Skip to content

Commit a715884

Browse files
andy31415andy31415
authored
Code pre-generation support (#23763)
* Start implementing a pregenerator * Start moving pregenerate logic into a separate directory * Start adding some ability to figure out pregeneration output locations * Pregeneration for bridge seems to work * Add missing files * Restyle * Better log level logic * Pregeneration of java also exists * Allow pregeneration of cpp app data. Full codegen usage is now enabled * Move sdk root around * Make things run * Restyle * Parallel code pregen - can do pregen in 1.08 seconds on my machine * Restyle * Make sure pregen folders are split * Pregeneration compile when using GN works * Restyle * Direct pregen usage works now * Restyle * Minor sort * Add support for both pregen and no pregen * Restyle * Fix pregen dir output logic * Support pregenerated directory in build_examples.py * Fix gn build logic * Somewhat simpler code for parallel vs serial codegen * Use imap_unordered to not care about actual parallel generation order * Also fix java jni codegen with pregenerated data * Fix java compilation deps: java codegen uses data model files * NRF now can use pregen folder * Allow telink to also use a pregen dir * Better shell escape, make mbedos cmake flags work with pregen dir * Restyle * Add pregen support for esp32 as well * Add a test for esp32 pregeneration support * Also test pregen support in linux builds (to check gn builds) * Remove unused file * Fix spelling * Fix esp32 compilation - gn arguments need to be passed from cmake * Fix some define forwarding logic in codegen * Make sure java build config (which includes header paths) is set as a config and applies to generated sources * java build config should apply to all sources, not just transitively * Restyle * Replace codege with codegen. * Fix naming typo * Update text for build steps * Update spacing logic to make the code cleaner. The nospace args is odd
1 parent 4a8f8eb commit a715884

File tree

19 files changed

+661
-95
lines changed

19 files changed

+661
-95
lines changed

‎.github/workflows/build.yaml

+18-2
Original file line numberDiff line numberDiff line change
@@ -229,12 +229,28 @@ jobs:
229229
run: |
230230
./scripts/run_in_build_env.sh \
231231
"./scripts/build/build_examples.py --no-log-timestamps \
232-
--target linux-x64-all-clusters-ipv6only-clang \
233-
--target linux-x64-chip-tool-ipv6only-clang \
234232
--target linux-x64-minmdns-ipv6only-clang \
235233
--target linux-x64-rpc-console \
236234
build \
237235
"
236+
- name: Create a pre-generate directory and ensure compile-time codegen would fail
237+
run: |
238+
./scripts/run_in_build_env.sh "./scripts/codepregen.py ./zzz_pregenerated"
239+
mv scripts/codegen.py scripts/codegen.py.renamed
240+
- name: Build using build_examples.py (pregen)
241+
timeout-minutes: 60
242+
run: |
243+
./scripts/run_in_build_env.sh \
244+
"./scripts/build/build_examples.py --no-log-timestamps \
245+
--target linux-x64-all-clusters-ipv6only-clang \
246+
--target linux-x64-chip-tool-ipv6only-clang \
247+
--pregen-dir ./zzz_pregenerated \
248+
build \
249+
"
250+
- name: Undo code pre-generation changes (make compile time codegen work again)
251+
run: |
252+
rm -rf ./zzz_pregenerated
253+
mv scripts/codegen.py.renamed scripts/codegen.py
238254
- name: Run fake linux tests with build_examples
239255
timeout-minutes: 15
240256
run: |

‎.github/workflows/examples-esp32.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -71,11 +71,29 @@ jobs:
7171
"./scripts/build/build_examples.py \
7272
--enable-flashbundle \
7373
--target esp32-m5stack-all-clusters \
74+
build \
75+
--copy-artifacts-to out/artifacts \
76+
"
77+
- name: Prepare code pregen and ensure compile time pregen not possible
78+
run: |
79+
./scripts/run_in_build_env.sh "./scripts/codepregen.py ./zzz_pregenerated"
80+
mv scripts/codegen.py scripts/codegen.py.renamed
81+
- name: Build some M5Stack variations with pregen
82+
timeout-minutes: 60
83+
run: |
84+
./scripts/run_in_build_env.sh \
85+
"./scripts/build/build_examples.py \
86+
--enable-flashbundle \
7487
--target esp32-m5stack-all-clusters-minimal \
7588
--target esp32-m5stack-all-clusters-rpc-ipv6only \
89+
--pregen-dir ./zzz_pregenerated \
7690
build \
7791
--copy-artifacts-to out/artifacts \
7892
"
93+
- name: Undo code pregeneration changes
94+
run: |
95+
rm -rf ./zzz_pregenerated
96+
mv scripts/codegen.py.renamed scripts/codegen.py
7997
- name: Prepare bloat report
8098
run: |
8199
.environment/pigweed-venv/bin/python3 scripts/tools/memory/gh_sizes.py \

‎build/chip/chip_codegen.cmake

+61-30
Original file line numberDiff line numberDiff line change
@@ -31,40 +31,71 @@ function(chip_codegen TARGET_NAME)
3131
${ARGN}
3232
)
3333

34-
set(GEN_FOLDER "${CMAKE_BINARY_DIR}/gen/${TARGET_NAME}/${ARG_GENERATOR}")
34+
set(CHIP_CODEGEN_PREGEN_DIR "" CACHE PATH "Pre-generated directory to use instead of compile-time code generation.")
3535

36-
string(REPLACE ";" "\n" OUTPUT_AS_NEWLINES "${ARG_OUTPUTS}")
36+
if ("${CHIP_CODEGEN_PREGEN_DIR}" STREQUAL "")
37+
set(GEN_FOLDER "${CMAKE_BINARY_DIR}/gen/${TARGET_NAME}/${ARG_GENERATOR}")
3738

38-
file(MAKE_DIRECTORY "${GEN_FOLDER}")
39-
file(GENERATE
40-
OUTPUT "${GEN_FOLDER}/expected.outputs"
41-
CONTENT "${OUTPUT_AS_NEWLINES}"
42-
)
39+
string(REPLACE ";" "\n" OUTPUT_AS_NEWLINES "${ARG_OUTPUTS}")
4340

41+
file(MAKE_DIRECTORY "${GEN_FOLDER}")
42+
file(GENERATE
43+
OUTPUT "${GEN_FOLDER}/expected.outputs"
44+
CONTENT "${OUTPUT_AS_NEWLINES}"
45+
)
46+
47+
48+
set(OUT_NAMES)
49+
foreach(NAME IN LISTS ARG_OUTPUTS)
50+
list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}")
51+
endforeach()
52+
53+
# Python is expected to be in the path
54+
#
55+
# find_package(Python3 REQUIRED)
56+
add_custom_command(
57+
OUTPUT ${OUT_NAMES}
58+
COMMAND "${CHIP_ROOT}/scripts/codegen.py"
59+
ARGS "--generator" "${ARG_GENERATOR}"
60+
"--output-dir" "${GEN_FOLDER}"
61+
"--expected-outputs" "${GEN_FOLDER}/expected.outputs"
62+
"${ARG_INPUT}"
63+
DEPENDS
64+
"${ARG_INPUT}"
65+
VERBATIM
66+
)
67+
68+
add_custom_target(${TARGET_NAME} DEPENDS "${OUT_NAMES}")
69+
70+
# Forward outputs to the parent
71+
set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE)
72+
set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE)
73+
else()
74+
# Gets a path such as:
75+
# examples/lock-app/lock-common/lock-app.matter
76+
file(RELATIVE_PATH MATTER_FILE_PATH "${CHIP_ROOT}" ${ARG_INPUT})
77+
78+
# Removes the trailing file extension to get something like:
79+
# examples/lock-app/lock-common/lock-app
80+
string(REGEX REPLACE "\.matter$" "" CODEGEN_DIR_PATH "${MATTER_FILE_PATH}")
81+
82+
83+
# Build the final location within the pregen directory
84+
set(GEN_FOLDER "${CHIP_CODEGEN_PREGEN_DIR}/${CODEGEN_DIR_PATH}/codegen/${ARG_GENERATOR}")
85+
86+
# TODO: build a fake target of ${TARGET_NAME}
87+
88+
# Here we have ${CHIP_CODEGEN_PREGEN_DIR}
89+
set(OUT_NAMES)
90+
foreach(NAME IN LISTS ARG_OUTPUTS)
91+
list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}")
92+
endforeach()
4493

45-
set(OUT_NAMES)
46-
foreach(NAME IN LISTS ARG_OUTPUTS)
47-
list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}")
48-
endforeach()
49-
50-
# Python is expected to be in the path
51-
#
52-
# find_package(Python3 REQUIRED)
53-
add_custom_command(
54-
OUTPUT ${OUT_NAMES}
55-
COMMAND "${CHIP_ROOT}/scripts/codegen.py"
56-
ARGS "--generator" "${ARG_GENERATOR}"
57-
"--output-dir" "${GEN_FOLDER}"
58-
"--expected-outputs" "${GEN_FOLDER}/expected.outputs"
59-
"${ARG_INPUT}"
60-
DEPENDS
61-
"${ARG_INPUT}"
62-
VERBATIM
63-
)
6494

65-
add_custom_target(${TARGET_NAME} DEPENDS "${OUT_NAMES}")
95+
set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE)
96+
set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE)
6697

67-
# Forward outputs to the parent
68-
set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE)
69-
set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE)
98+
# allow adding dependencies to a phony target since no codegen is done
99+
add_custom_target(${TARGET_NAME})
100+
endif()
70101
endfunction()

‎build/chip/chip_codegen.gni

+120-32
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,75 @@ import("//build_overrides/pigweed.gni")
1818

1919
import("$dir_pw_build/python.gni")
2020

21+
declare_args() {
22+
# Location where code has been pre-generated
23+
chip_code_pre_generated_directory = ""
24+
}
25+
26+
# Code generation that will happen at build time.
27+
#
28+
#
29+
template("_chip_build_time_codegen") {
30+
_name = target_name
31+
_generator = invoker.generator
32+
33+
config("${_name}_config") {
34+
include_dirs = [ target_gen_dir ]
35+
}
36+
37+
pw_python_action("${_name}_codegen") {
38+
script = "${chip_root}/scripts/codegen.py"
39+
40+
_idl_file = invoker.input
41+
_expected_outputs = "${target_gen_dir}/${_name}.expected.outputs"
42+
43+
write_file(_expected_outputs, invoker.outputs, "list lines")
44+
45+
args = [
46+
"--generator",
47+
_generator,
48+
"--output-dir",
49+
rebase_path(target_gen_dir, root_build_dir),
50+
"--expected-outputs",
51+
rebase_path(_expected_outputs, root_build_dir),
52+
rebase_path(_idl_file, root_build_dir),
53+
]
54+
55+
deps = [ "${chip_root}/scripts/idl" ]
56+
57+
inputs = [
58+
_idl_file,
59+
_expected_outputs,
60+
]
61+
sources = [ _idl_file ]
62+
63+
outputs = []
64+
foreach(name, invoker.outputs) {
65+
outputs += [ "${target_gen_dir}/${name}" ]
66+
}
67+
}
68+
69+
source_set(_name) {
70+
sources = []
71+
foreach(name, invoker.outputs) {
72+
sources += [ "${target_gen_dir}/${name}" ]
73+
}
74+
75+
public_configs = [ ":${_name}_config" ]
76+
77+
if (defined(invoker.public_configs)) {
78+
public_configs += invoker.public_configs
79+
}
80+
81+
forward_variables_from(invoker, [ "deps" ])
82+
83+
if (!defined(deps)) {
84+
deps = []
85+
}
86+
deps += [ ":${_name}_codegen" ]
87+
}
88+
}
89+
2190
# Defines a target that runs code generation based on
2291
# scripts/codegen.py
2392
#
@@ -32,13 +101,29 @@ import("$dir_pw_build/python.gni")
32101
# Explicit names of the expected outputs. Enforced to validate that
33102
# expected outputs are generated when processing input files.
34103
#
104+
# deps, public_configs
105+
# Forwarded to the resulting source set
106+
#
107+
# Command line parameters:
108+
#
109+
# chip_code_pre_generated_directory:
110+
# - If this is set, generation will NOT happen at compile time but rather
111+
# the code generation is assumed to have already happened and reside in
112+
# the given location.
113+
# - The TOP LEVEL directory is assumed to be given. Actual location for
114+
# individual generators is expected to be of the form
115+
# <top_dir>/<matter_path>/<generator>
116+
#
35117
# NOTE: content of "outputs" is verified to match the output of codegen.py
36118
# exactly. It is not inferred on purpose, to make build-rules explicit
37-
# and verifiable (even though codege.py can at runtime report its outputs)
119+
# and verifiable (even though codegen.py can at runtime report its outputs)
38120
#
39121
# To find the list of generated files, you can run codegen.py with the
40122
# "--name-only" argument
41123
#
124+
# NOTE:
125+
# the result of the target_name WILL BE a `source_set`. Treat it as such.
126+
#
42127
# Example usage:
43128
#
44129
# chip_codegen("java-jni-generate") {
@@ -53,43 +138,46 @@ import("$dir_pw_build/python.gni")
53138
# }
54139
#
55140
template("chip_codegen") {
56-
_name = target_name
57-
_generator = invoker.generator
58-
59-
config("${_name}_config") {
60-
include_dirs = [ target_gen_dir ]
61-
}
141+
if (chip_code_pre_generated_directory == "") {
142+
_chip_build_time_codegen(target_name) {
143+
forward_variables_from(invoker,
144+
[
145+
"deps",
146+
"generator",
147+
"input",
148+
"outputs",
149+
"public_configs",
150+
])
151+
}
152+
} else {
153+
_name = target_name
62154

63-
pw_python_action(_name) {
64-
script = "${chip_root}/scripts/codegen.py"
155+
# This contstructs a path like:
156+
# FROM all-clusters-app.matter (inside examples/all-clusters-app/all-clusters-common/)
157+
# USING "cpp-app" for generator:
158+
# => ${pregen_dir}/examples/all-clusters-app/all-clusters-common/all-clusters-app/codegen/cpp-app
159+
_generation_dir =
160+
chip_code_pre_generated_directory + "/" +
161+
string_replace(rebase_path(invoker.input, chip_root), ".matter", "") +
162+
"/codegen/" + invoker.generator
65163

66-
_idl_file = invoker.input
67-
_expected_outputs = "${target_gen_dir}/${_name}.expected.outputs"
68-
69-
write_file(_expected_outputs, invoker.outputs, "list lines")
164+
config("${_name}_config") {
165+
include_dirs = [ "${_generation_dir}" ]
166+
}
70167

71-
args = [
72-
"--generator",
73-
_generator,
74-
"--output-dir",
75-
rebase_path(target_gen_dir, root_build_dir),
76-
"--expected-outputs",
77-
rebase_path(_expected_outputs, root_build_dir),
78-
rebase_path(_idl_file, root_build_dir),
79-
]
168+
source_set(_name) {
169+
public_configs = [ ":${_name}_config" ]
80170

81-
deps = [ "${chip_root}/scripts/idl" ]
82-
public_configs = [ ":${_name}_config" ]
171+
if (defined(invoker.public_configs)) {
172+
public_configs += invoker.public_configs
173+
}
83174

84-
inputs = [
85-
_idl_file,
86-
_expected_outputs,
87-
]
88-
sources = [ _idl_file ]
175+
forward_variables_from(invoker, [ "deps" ])
89176

90-
outputs = []
91-
foreach(name, invoker.outputs) {
92-
outputs += [ "${target_gen_dir}/${name}" ]
177+
sources = []
178+
foreach(name, invoker.outputs) {
179+
sources += [ "${_generation_dir}/${name}" ]
180+
}
93181
}
94182
}
95183
}

‎config/esp32/components/chip/CMakeLists.txt

+4
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,10 @@ if(CONFIG_DISABLE_IPV4)
101101
chip_gn_arg_append("chip_inet_config_enable_ipv4" "false")
102102
endif()
103103

104+
if(CHIP_CODEGEN_PREGEN_DIR)
105+
chip_gn_arg_append("chip_code_pre_generated_directory" "\"${CHIP_CODEGEN_PREGEN_DIR}\"")
106+
endif()
107+
104108
if(CONFIG_ENABLE_PW_RPC)
105109
string(APPEND chip_gn_args "import(\"//build_overrides/pigweed.gni\")\n")
106110
chip_gn_arg_append("remove_default_configs" "[\"//third_party/connectedhomeip/third_party/pigweed/repo/pw_build:toolchain_cpp_standard\"]")

‎docs/code_generation.md

+23-3
Original file line numberDiff line numberDiff line change
@@ -176,12 +176,32 @@ enerated
176176

177177
### `*.matter` code generation
178178

179-
Currently `*.matter` code generation is done at compile time.
179+
`*.matter` code generation can be done either at compile time or it can use
180+
pre-generated output.
180181

181182
Rules for how `codegen.py` is invoked and how includes/sources are set are
182183
defined at:
183184

184185
- `src/app/chip_data_model.cmake`
185-
- `build/chip/esp32/esp32_codegen.cmake` (support for 2-pass cmake builds used
186-
by the Espressif `idf.py` build system)
187186
- `src/app/chip_data_model.gni`
187+
188+
Additionally, `build/chip/esp32/esp32_codegen.cmake` adds processing support for
189+
the 2-pass cmake builds used by the Espressif `idf.py` build system.
190+
191+
## Pre-generation
192+
193+
Code pre-generation can be used:
194+
195+
- when compile-time code generation is not desirable. This may be for
196+
importing into build systems that do not have the pre-requisites to run code
197+
generation at build time or to save the code generation time at the expense
198+
of running code generation for every possible zap/generation type
199+
- To check changes in generated code across versions, beyond the comparisons
200+
of golden image tests in `scripts/idl/tests`
201+
202+
The script to trigger code pre-generation is `scripts/code_pregenerate.py` and
203+
requires the pre-generation output directory as an argument
204+
205+
```bash
206+
scripts/code_pregenerate.py ${OUTPUT_DIRECTORY:-./zzz_pregenerated/}
207+
```

0 commit comments

Comments
 (0)