🔨 Build script improvements

Author: thinkyhead

buildroot/share/PlatformIO/scripts/configuration.py

@@ -110,7 +110,7 @@ def disable_all_options():
             match = regex.match(line)
             if match:
                 name = match[3].upper()
-                if name in ('CONFIGURATION_H_VERSION', 'CONFIGURATION_ADV_H_VERSION'): continue
+                if name in ('CONFIGURATION_H_VERSION', 'CONFIGURATION_ADV_H_VERSION', 'CONFIG_EXAMPLES_DIR'): continue
                 if name.startswith('_'): continue
                 found = True
                 # Comment out the define

buildroot/share/PlatformIO/scripts/schema.py

@@ -80,7 +80,7 @@ def load_boards():
     return ''
 
 #
-# Extract the current configuration files in the form of a structured schema.
+# Extract the specified configuration files in the form of a structured schema.
 # Contains the full schema for the configuration files, not just the enabled options,
 # Contains the current values of the options, not just data structure, so "schema" is a slight misnomer.
 #
@@ -99,9 +99,9 @@ def load_boards():
 #    - requires = The conditions that must be met for the define to be enabled
 #    - comment  = The comment for the define, if it has one
 #    - units    = The units for the define, if it has one
-#    - options  = The options for the define, if it has one
+#    - options  = The options for the define, if it has any
 #
-def extract():
+def extract_files(filekey):
     # Load board names from boards.h
     boards = load_boards()
 
@@ -114,10 +114,8 @@ class Parse:
         GET_SENSORS     = 4 # Gathering temperature sensor options
         ERROR           = 9 # Syntax error
 
-    # List of files to process, with shorthand
-    filekey = { 'Configuration.h':'basic', 'Configuration_adv.h':'advanced' }
     # A JSON object to store the data
-    sch_out = { 'basic':{}, 'advanced':{} }
+    sch_out = { key:{} for key in filekey.values() }
     # Regex for #define NAME [VALUE] [COMMENT] with sanitized line
     defgrep = re.compile(r'^(//)?\s*(#define)\s+([A-Za-z0-9_]+)\s*(.*?)\s*(//.+)?$')
     # Pattern to match a float value
@@ -180,6 +178,12 @@ class Parse:
                             cfield = 'notes' if 'comment' in last_added_ref else 'comment'
                             last_added_ref[cfield] = cline
 
+                #
+                # Add the given comment line to the comment buffer, unless:
+                # - The line starts with ':' and JSON values to assign to 'opt'.
+                # - The line starts with '@section' so a new section needs to be returned.
+                # - The line starts with '======' so just skip it.
+                #
                 def use_comment(c, opt, sec, bufref):
                     if c.startswith(':'):               # If the comment starts with : then it has magic JSON
                         d = c[1:].strip()               # Strip the leading :
@@ -199,7 +203,7 @@ def use_comment(c, opt, sec, bufref):
                 # The comment will be applied to the next #define.
                 if state == Parse.SLASH_COMMENT:
                     if not defmatch and the_line.startswith('//'):
-                        use_comment(the_line[2:].strip(), options_json, section, comment_buff)
+                        options_json, section = use_comment(the_line[2:].strip(), options_json, section, comment_buff)
                         continue
                     else:
                         state = Parse.NORMAL
@@ -219,7 +223,7 @@ def use_comment(c, opt, sec, bufref):
 
                         state = Parse.NORMAL
 
-                    # Strip the leading '*' from block comments
+                    # Strip the leading '* ' from block comments
                     cline = re.sub(r'^\* ?', '', cline)
 
                     # Collect temperature sensors
@@ -412,6 +416,13 @@ def atomize(s):
 
     return sch_out
 
+#
+# Extract the current configuration files in the form of a structured schema.
+#
+def extract():
+    # List of files to process, with shorthand
+    return extract_files({ 'Configuration.h':'basic', 'Configuration_adv.h':'advanced' })
+
 def dump_json(schema:dict, jpath:Path):
     with jpath.open('w') as jfile:
         json.dump(schema, jfile, ensure_ascii=False, indent=2)

buildroot/share/PlatformIO/scripts/signature.py

@@ -35,18 +35,27 @@ def enabled_defines(filepath):
     '''
     outdict = {}
     section = "user"
-    spatt = re.compile(r".*@section +([-a-zA-Z0-9_\s]+)$") # must match @section ...
+    spatt = re.compile(r".*@section +([-a-zA-Z0-9_\s]+)$") # @section ...
 
     f = open(filepath, encoding="utf8").read().split("\n")
 
-    # Get the full contents of the file and remove all block comments.
-    # This will avoid false positives from #defines in comments
-    f = re.sub(r'/\*.*?\*/', '', '\n'.join(f), flags=re.DOTALL).split("\n")
-
+    incomment = False
     for line in f:
         sline = line.strip()
+
         m = re.match(spatt, sline) # @section ...
         if m: section = m.group(1).strip() ; continue
+
+        if incomment:
+            if '*/' in sline:
+                incomment = False
+            continue
+        else:
+            mpos, spos = sline.find('/*'), sline.find('//')
+            if mpos >= 0 and (spos < 0 or spos > mpos):
+                incomment = True
+                continue
+
         if sline[:7] == "#define":
             # Extract the key here (we don't care about the value)
             kv = sline[8:].strip().split()
@@ -70,6 +79,11 @@ def compress_file(filepath, storedname, outpath):
     with zipfile.ZipFile(outpath, 'w', compression=zipfile.ZIP_BZIP2, compresslevel=9) as zipf:
         zipf.write(filepath, arcname=storedname, compress_type=zipfile.ZIP_BZIP2, compresslevel=9)
 
+ignore = ('CONFIGURATION_H_VERSION', 'CONFIGURATION_ADV_H_VERSION', 'CONFIG_EXAMPLES_DIR', 'CONFIG_EXPORT')
+
+#
+# Compute a build signature and/or export the configuration
+#
 def compute_build_signature(env):
     '''
     Compute the build signature by extracting all configuration settings and
@@ -81,11 +95,17 @@ def compute_build_signature(env):
     env.Append(BUILD_SIGNATURE=1)
 
     build_path = Path(env['PROJECT_BUILD_DIR'], env['PIOENV'])
-    marlin_json = build_path / 'marlin_config.json'
+    json_name = 'marlin_config.json'
+    marlin_json = build_path / json_name
     marlin_zip = build_path / 'mc.zip'
 
+    # ANSI colors
+    green = "\u001b[32m"
+    yellow = "\u001b[33m"
+    red = "\u001b[31m"
+
     # Definitions from these files will be kept
-    header_paths = [ 'Marlin/Configuration.h', 'Marlin/Configuration_adv.h' ]
+    header_paths = ('Marlin/Configuration.h', 'Marlin/Configuration_adv.h')
 
     # Check if we can skip processing
     hashes = ''
@@ -100,7 +120,7 @@ def compute_build_signature(env):
             conf = json.load(infile)
             same_hash = conf['__INITIAL_HASH'] == hashes
             if same_hash:
-                compress_file(marlin_json, 'marlin_config.json', marlin_zip)
+                compress_file(marlin_json, json_name, marlin_zip)
     except:
         pass
 
@@ -179,25 +199,28 @@ def tryint(key):
     extended_dump = config_dump > 100
     if extended_dump: config_dump -= 100
 
+    # Get the schema class for exports that require it
+    if config_dump in (3, 4) or (extended_dump and config_dump in (2, 5)):
+        try:
+            conf_schema = schema.extract()
+        except Exception as exc:
+            print(red + "Error: " + str(exc))
+            conf_schema = None
+
     #
     # Produce an INI file if CONFIG_EXPORT == 2
     #
     if config_dump == 2:
-        print("Generating config.ini ...")
+        print(yellow + "Generating config.ini ...")
 
         ini_fmt = '{0:40} = {1}'
         ext_fmt = '{0:40}   {1}'
-        ignore = ('CONFIGURATION_H_VERSION', 'CONFIGURATION_ADV_H_VERSION', 'CONFIG_EXAMPLES_DIR', 'CONFIG_EXPORT')
 
         if extended_dump:
             # Extended export will dump config options by section
 
             # We'll use Schema class to get the sections
-            try:
-                conf_schema = schema.extract()
-            except Exception as exc:
-                print("Error: " + str(exc))
-                exit(1)
+            if not conf_schema: exit(1)
 
             # Then group options by schema @section
             sections = {}
@@ -305,27 +328,22 @@ def tryint(key):
                 for header in real_config:
                     outfile.write(f'\n[{filegrp[header]}]\n')
                     for name in sorted(real_config[header]):
-                        if name not in ignore:
-                            val = real_config[header][name]['value']
-                            if val == '': val = 'on'
-                            outfile.write(ini_fmt.format(name.lower(), val) + '\n')
+                        if name in ignore: continue
+                        val = real_config[header][name]['value']
+                        if val == '': val = 'on'
+                        outfile.write(ini_fmt.format(name.lower(), val) + '\n')
 
     #
     # CONFIG_EXPORT 3 = schema.json, 4 = schema.yml
     #
-    if config_dump >= 3:
-        try:
-            conf_schema = schema.extract()
-        except Exception as exc:
-            print("Error: " + str(exc))
-            conf_schema = None
+    if config_dump in (3, 4):
 
         if conf_schema:
             #
             # 3 = schema.json
             #
             if config_dump in (3, 13):
-                print("Generating schema.json ...")
+                print(yellow + "Generating schema.json ...")
                 schema.dump_json(conf_schema, build_path / 'schema.json')
                 if config_dump == 13:
                     schema.group_options(conf_schema)
@@ -335,7 +353,7 @@ def tryint(key):
             # 4 = schema.yml
             #
             elif config_dump == 4:
-                print("Generating schema.yml ...")
+                print(yellow + "Generating schema.yml ...")
                 try:
                     import yaml
                 except ImportError:
@@ -355,7 +373,7 @@ def tryint(key):
 
             json_data = {}
             if extended_dump:
-                print("Extended dump ...")
+                print(yellow + "Extended dump ...")
                 for header in real_config:
                     confs = real_config[header]
                     json_data[header] = {}
@@ -395,7 +413,7 @@ def tryint(key):
 
     # Compress the JSON file as much as we can
     if not same_hash:
-        compress_file(marlin_json, 'marlin_config.json', marlin_zip)
+        compress_file(marlin_json, json_name, marlin_zip)
 
     # Generate a C source file containing the entire ZIP file as an array
     with open('Marlin/src/mczip.h','wb') as result_file:

buildroot/share/extras/file_header.h

@@ -1,6 +1,6 @@
 /**
  * Marlin 3D Printer Firmware
- * Copyright (c) 2023 MarlinFirmware [https://github.com/MarlinFirmware/Marlin]
+ * Copyright (c) 2024 MarlinFirmware [https://github.com/MarlinFirmware/Marlin]
  *
  * Based on Sprinter and grbl.
  * Copyright (c) 2011 Camiel Gubbels / Erik van der Zalm