Run zap in parallel by default for zap_regen_all.py (#23766)

* Run zap in parallel on regen all. This makes my local zap run in 88 seconds instead of about 222

* Restyle

* Fix arguments for generate.py

Author: andy31415

scripts/tools/zap/generate.py

@@ -32,6 +32,7 @@ class CmdLineArgs:
     templateFile: str
     outputDir: str
     runBootstrap: bool
+    parallel: bool = True
 
 
 CHIP_ROOT_DIR = os.path.realpath(
@@ -103,6 +104,9 @@ def runArgumentsParser() -> CmdLineArgs:
                         help='Output directory for the generated files (default: automatically selected)')
     parser.add_argument('--run-bootstrap', default=None, action='store_true',
                         help='Automatically run ZAP bootstrap. By default the bootstrap is not triggered')
+    parser.add_argument('--parallel', action='store_true')
+    parser.add_argument('--no-parallel', action='store_false', dest='parallel')
+    parser.set_defaults(parallel=True)
     args = parser.parse_args()
 
     # By default, this script assumes that the global CHIP template is used with
@@ -259,6 +263,11 @@ def main():
 
     # The maximum memory usage is over 4GB (#15620)
     os.environ["NODE_OPTIONS"] = "--max-old-space-size=8192"
+
+    if cmdLineArgs.parallel:
+        # Parallel-compatible runs will need separate state
+        os.environ["ZAP_TEMPSTATE"] = "1"
+
     runGeneration(cmdLineArgs.zapFile, cmdLineArgs.zclFile, cmdLineArgs.templateFile, cmdLineArgs.outputDir)
 
     prettifiers = [

scripts/tools/zap_regen_all.py

@@ -21,6 +21,8 @@
 import sys
 import subprocess
 import logging
+import multiprocessing
+
 from dataclasses import dataclass
 
 CHIP_ROOT_DIR = os.path.realpath(
@@ -113,6 +115,11 @@ def setupArgumentsParser():
                         help="Don't do any generation, just log what targets would be generated (default: False)")
     parser.add_argument('--run-bootstrap', default=None, action='store_true',
                         help='Automatically run ZAP bootstrap. By default the bootstrap is not triggered')
+
+    parser.add_argument('--parallel', action='store_true')
+    parser.add_argument('--no-parallel', action='store_false', dest='parallel')
+    parser.set_defaults(parallel=True)
+
     return parser.parse_args()
 
 
@@ -253,6 +260,14 @@ def getTargets(type, test_target):
     return targets
 
 
+def _ParallelGenerateOne(target):
+    """
+    Helper method to be passed to multiprocessing parallel generation of
+    items.
+    """
+    target.generate()
+
+
 def main():
     logging.basicConfig(
         level=logging.INFO,
@@ -269,8 +284,15 @@ def main():
         if args.run_bootstrap:
             subprocess.check_call(os.path.join(CHIP_ROOT_DIR, "scripts/tools/zap/zap_bootstrap.sh"), shell=True)
 
-        for target in targets:
-            target.generate()
+        if args.parallel:
+            # Ensure each zap run is independent
+            os.environ['ZAP_TEMPSTATE'] = '1'
+            with multiprocessing.Pool() as pool:
+                for _ in pool.imap_unordered(_ParallelGenerateOne, targets):
+                    pass
+        else:
+            for target in targets:
+                target.generate()
 
 
 if __name__ == '__main__':