Skip to content

Commit f66077a

Browse files
charlielyecharlielye
authored
feat: Build system handles dynamic deps first class. (#2283)
If no deps provided, query-manifest exectutes script in project, uses 1 line of jq to get deps for ts.
1 parent 262dbb9 commit f66077a

File tree

3 files changed

+72
-174
lines changed

3 files changed

+72
-174
lines changed

‎build-system/scripts/query_manifest

+42-13
Original file line numberDiff line numberDiff line change
@@ -11,18 +11,46 @@ if [ $(jq "has(\"$REPO\")" $MANIFEST) == "false" ]; then
1111
exit 1
1212
fi
1313

14-
function addRebuildPatterns {
15-
local TYPE=$(jq -r ".\"$1\".rebuildPatterns | type" $MANIFEST)
16-
if [ "$TYPE" == "string" ]; then
17-
local FILE=$(jq -r ".\"$1\".rebuildPatterns" $MANIFEST)
18-
local BUILD_DIR=$($0 buildDir $1)
19-
PATTERNS=(${PATTERNS[@]} $(cat $BUILD_DIR/$FILE))
20-
elif [ "$TYPE" == "array" ]; then
21-
PATTERNS=(${PATTERNS[@]} $(jq -r ".\"$1\".rebuildPatterns | .[]" $MANIFEST))
22-
else
23-
>&2 echo "Missing rebuildPatterns property. Either filename as string, or patterns as array."
14+
function get_deps {
15+
local TYPE=$(jq -r ".\"$1\".dependencies | type" $MANIFEST)
16+
if [ "$TYPE" == "string" ]; then
17+
# Execute string as command relative to buildDir to retrieve dependencies.
18+
local CMD=$BUILD_DIR/$(jq -r ".\"$1\".dependencies")
19+
if [ ! -f "$CMD" ]; then
20+
>&2 echo "Dependency script not found: $CMD"
2421
exit 1
2522
fi
23+
local PROJECT_DIR=$($0 projectDir $1)
24+
DEPS=($($CMD $PROJECT_DIR))
25+
elif [ "$TYPE" == "null" ]; then
26+
# Execute default script relative to buildDir to retrieve dependencies.
27+
local CMD=$BUILD_DIR/scripts/get_dependencies.sh
28+
if [ ! -f "$CMD" ]; then
29+
>&2 echo "Dependency script not found: $CMD"
30+
exit 1
31+
fi
32+
local PROJECT_DIR=$($0 projectDir $1)
33+
DEPS=($($CMD $PROJECT_DIR))
34+
elif [ "$TYPE" == "array" ]; then
35+
DEPS=($(jq -r ".\"$1\".dependencies // [] | .[]" $MANIFEST))
36+
else
37+
>&2 echo "dependencies must be a array, string or null."
38+
exit 1
39+
fi
40+
}
41+
42+
function add_rebuild_patterns {
43+
local TYPE=$(jq -r ".\"$1\".rebuildPatterns | type" $MANIFEST)
44+
if [ "$TYPE" == "string" ]; then
45+
local FILE=$(jq -r ".\"$1\".rebuildPatterns" $MANIFEST)
46+
local BUILD_DIR=$($0 buildDir $1)
47+
PATTERNS=(${PATTERNS[@]} $(cat $BUILD_DIR/$FILE))
48+
elif [ "$TYPE" == "array" ]; then
49+
PATTERNS=(${PATTERNS[@]} $(jq -r ".\"$1\".rebuildPatterns | .[]" $MANIFEST))
50+
else
51+
>&2 echo "Missing rebuildPatterns property. Either filename as string, or patterns as array."
52+
exit 1
53+
fi
2654
}
2755

2856
case "$CMD" in
@@ -46,13 +74,14 @@ case "$CMD" in
4674
jq -r ".\"$REPO\".projectDir // .\"$REPO\".buildDir" $MANIFEST
4775
;;
4876
dependencies)
77+
BUILD_DIR=$($0 buildDir $REPO)
4978
declare -A ALL_DEPS
5079
add_deps() {
5180
if [[ -v ALL_DEPS[$1] ]]; then
5281
return
5382
fi
5483
ALL_DEPS["$1"]=1
55-
DEPS=($(jq -r ".\"$1\".dependencies // [] | .[]" $MANIFEST))
84+
get_deps $1
5685
for DEP in "${DEPS[@]}"; do
5786
add_deps $DEP
5887
done
@@ -65,9 +94,9 @@ case "$CMD" in
6594
rebuildPatterns)
6695
DEPS=($($0 dependencies $REPO))
6796
PATTERNS=()
68-
addRebuildPatterns $REPO
97+
add_rebuild_patterns $REPO
6998
for DEP in "${DEPS[@]}"; do
70-
addRebuildPatterns $DEP
99+
add_rebuild_patterns $DEP
71100
done
72101
printf "%s\n" "${PATTERNS[@]}" | sort | uniq
73102
;;

‎build_manifest.json

+23-161
Original file line numberDiff line numberDiff line change
@@ -113,168 +113,85 @@
113113
"buildDir": "yarn-project",
114114
"projectDir": "yarn-project/acir-simulator",
115115
"dockerfile": "acir-simulator/Dockerfile",
116-
"rebuildPatterns": ["^yarn-project/acir-simulator/"],
117-
"dependencies": [
118-
"yarn-project-base",
119-
"circuits.js",
120-
"foundation",
121-
"types",
122-
"merkle-tree",
123-
"noir-contracts"
124-
]
116+
"rebuildPatterns": ["^yarn-project/acir-simulator/"]
125117
},
126118
"archiver": {
127119
"buildDir": "yarn-project",
128120
"projectDir": "yarn-project/archiver",
129121
"dockerfile": "archiver/Dockerfile",
130-
"rebuildPatterns": ["^yarn-project/archiver/"],
131-
"dependencies": [
132-
"yarn-project-base",
133-
"circuits.js",
134-
"ethereum",
135-
"foundation",
136-
"l1-artifacts",
137-
"types"
138-
]
122+
"rebuildPatterns": ["^yarn-project/archiver/"]
139123
},
140124
"cli": {
141125
"buildDir": "yarn-project",
142126
"projectDir": "yarn-project/cli",
143127
"dockerfile": "cli/Dockerfile",
144-
"rebuildPatterns": ["^yarn-project/cli/"],
145-
"dependencies": [
146-
"yarn-project-base",
147-
"aztec.js",
148-
"ethereum",
149-
"foundation",
150-
"noir-compiler",
151-
"noir-contracts",
152-
"types"
153-
]
128+
"rebuildPatterns": ["^yarn-project/cli/"]
154129
},
155130
"aztec-rpc": {
156131
"buildDir": "yarn-project",
157132
"projectDir": "yarn-project/aztec-rpc",
158133
"dockerfile": "aztec-rpc/Dockerfile",
159-
"rebuildPatterns": ["^yarn-project/aztec-rpc/"],
160-
"dependencies": [
161-
"yarn-project-base",
162-
"acir-simulator",
163-
"circuits.js",
164-
"foundation",
165-
"key-store",
166-
"types"
167-
]
134+
"rebuildPatterns": ["^yarn-project/aztec-rpc/"]
168135
},
169136
"aztec-sandbox": {
170137
"buildDir": "yarn-project",
171138
"projectDir": "yarn-project/aztec-sandbox",
172139
"dockerfile": "aztec-sandbox/Dockerfile",
173-
"rebuildPatterns": ["^yarn-project/aztec-sandbox/"],
174-
"dependencies": [
175-
"yarn-project-base",
176-
"aztec-node",
177-
"aztec-rpc",
178-
"aztec.js",
179-
"circuits.js",
180-
"ethereum",
181-
"foundation",
182-
"l1-artifacts",
183-
"noir-contracts",
184-
"types"
185-
]
140+
"rebuildPatterns": ["^yarn-project/aztec-sandbox/"]
186141
},
187142
"aztec.js": {
188143
"buildDir": "yarn-project",
189144
"projectDir": "yarn-project/aztec.js",
190145
"dockerfile": "aztec.js/Dockerfile",
191-
"rebuildPatterns": ["^yarn-project/aztec.js/"],
192-
"dependencies": ["yarn-project-base", "circuits.js", "foundation", "types"]
146+
"rebuildPatterns": ["^yarn-project/aztec.js/"]
193147
},
194148
"canary-build": {
195149
"buildDir": "yarn-project",
196150
"projectDir": "yarn-project/canary",
197151
"dockerfile": "canary/Dockerfile.build",
198-
"rebuildPatterns": ["^yarn-project/canary/"],
199-
"dependencies": [
200-
"yarn-project-base",
201-
"aztec.js",
202-
"l1-artifacts",
203-
"noir-contracts",
204-
"aztec-sandbox"
205-
]
152+
"rebuildPatterns": ["^yarn-project/canary/"]
206153
},
207154
"canary": {
208155
"buildDir": "yarn-project",
209156
"projectDir": "yarn-project/canary",
210157
"dockerfile": "canary/Dockerfile",
211-
"rebuildPatterns": ["^yarn-project/canary/"],
212-
"dependencies": [
213-
"yarn-project-base",
214-
"aztec.js",
215-
"foundation",
216-
"l1-artifacts",
217-
"noir-contracts"
218-
]
158+
"rebuildPatterns": ["^yarn-project/canary/"]
219159
},
220160
"circuits.js": {
221161
"buildDir": "yarn-project",
222162
"projectDir": "yarn-project/circuits.js",
223163
"dockerfile": "circuits.js/Dockerfile",
224-
"rebuildPatterns": ["^yarn-project/circuits.js/"],
225-
"dependencies": ["yarn-project-base", "foundation"]
164+
"rebuildPatterns": ["^yarn-project/circuits.js/"]
226165
},
227166
"end-to-end": {
228167
"buildDir": "yarn-project",
229168
"projectDir": "yarn-project/end-to-end",
230169
"dockerfile": "end-to-end/Dockerfile",
231-
"rebuildPatterns": ["^yarn-project/end-to-end/"],
232-
"dependencies": [
233-
"yarn-project-base",
234-
"archiver",
235-
"aztec-node",
236-
"aztec-rpc",
237-
"aztec-sandbox",
238-
"aztec.js",
239-
"circuits.js",
240-
"cli",
241-
"ethereum",
242-
"foundation",
243-
"l1-artifacts",
244-
"noir-contracts",
245-
"p2p",
246-
"sequencer-client",
247-
"types",
248-
"world-state"
249-
]
170+
"rebuildPatterns": ["^yarn-project/end-to-end/"]
250171
},
251172
"ethereum": {
252173
"buildDir": "yarn-project",
253174
"projectDir": "yarn-project/ethereum",
254175
"dockerfile": "ethereum/Dockerfile",
255-
"rebuildPatterns": ["^yarn-project/ethereum/"],
256-
"dependencies": ["yarn-project-base", "foundation", "l1-artifacts"]
176+
"rebuildPatterns": ["^yarn-project/ethereum/"]
257177
},
258178
"foundation": {
259179
"buildDir": "yarn-project",
260180
"projectDir": "yarn-project/foundation",
261181
"dockerfile": "foundation/Dockerfile",
262-
"rebuildPatterns": ["^yarn-project/foundation/"],
263-
"dependencies": ["yarn-project-base"]
182+
"rebuildPatterns": ["^yarn-project/foundation/"]
264183
},
265184
"key-store": {
266185
"buildDir": "yarn-project",
267186
"projectDir": "yarn-project/key-store",
268187
"dockerfile": "key-store/Dockerfile",
269-
"rebuildPatterns": ["^yarn-project/key-store/"],
270-
"dependencies": ["yarn-project-base", "circuits.js", "foundation", "types"]
188+
"rebuildPatterns": ["^yarn-project/key-store/"]
271189
},
272190
"merkle-tree": {
273191
"buildDir": "yarn-project",
274192
"projectDir": "yarn-project/merkle-tree",
275193
"dockerfile": "merkle-tree/Dockerfile",
276-
"rebuildPatterns": ["^yarn-project/merkle-tree/"],
277-
"dependencies": ["yarn-project-base", "circuits.js", "foundation", "types"]
194+
"rebuildPatterns": ["^yarn-project/merkle-tree/"]
278195
},
279196
"noir-contracts-build": {
280197
"buildDir": "yarn-project",
@@ -283,12 +200,6 @@
283200
"rebuildPatterns": [
284201
"^yarn-project/noir-contracts/",
285202
"^yarn-project/noir-libs/"
286-
],
287-
"dependencies": [
288-
"yarn-project-base",
289-
"aztec.js",
290-
"foundation",
291-
"noir-compiler"
292203
]
293204
},
294205
"noir-contracts": {
@@ -298,109 +209,60 @@
298209
"rebuildPatterns": [
299210
"^yarn-project/noir-contracts/",
300211
"^yarn-project/noir-libs/"
301-
],
302-
"dependencies": [
303-
"yarn-project-base",
304-
"aztec.js",
305-
"foundation",
306-
"noir-compiler"
307212
]
308213
},
309214
"noir-compiler": {
310215
"buildDir": "yarn-project",
311216
"projectDir": "yarn-project/noir-compiler",
312217
"dockerfile": "noir-compiler/Dockerfile",
313-
"rebuildPatterns": ["^yarn-project/noir-compiler/"],
314-
"dependencies": ["yarn-project-base", "foundation"]
218+
"rebuildPatterns": ["^yarn-project/noir-compiler/"]
315219
},
316220
"p2p": {
317221
"buildDir": "yarn-project",
318222
"projectDir": "yarn-project/p2p",
319223
"dockerfile": "p2p/Dockerfile",
320-
"rebuildPatterns": ["^yarn-project/p2p/"],
321-
"dependencies": ["yarn-project-base", "circuits.js", "foundation", "types"]
224+
"rebuildPatterns": ["^yarn-project/p2p/"]
322225
},
323226
"p2p-bootstrap": {
324227
"buildDir": "yarn-project",
325228
"projectDir": "yarn-project/p2p-bootstrap",
326229
"dockerfile": "p2p/Dockerfile",
327-
"rebuildPatterns": ["^yarn-project/p2p-bootstrap/"],
328-
"dependencies": ["yarn-project-base", "foundation", "p2p"]
230+
"rebuildPatterns": ["^yarn-project/p2p-bootstrap/"]
329231
},
330232
"prover-client": {
331233
"buildDir": "yarn-project",
332234
"projectDir": "yarn-project/prover-client",
333235
"dockerfile": "prover-client/Dockerfile",
334-
"rebuildPatterns": ["^yarn-project/prover-client/"],
335-
"dependencies": ["yarn-project-base", "foundation"]
236+
"rebuildPatterns": ["^yarn-project/prover-client/"]
336237
},
337238
"rollup-provider": {
338239
"buildDir": "yarn-project",
339240
"projectDir": "yarn-project/rollup-provider",
340241
"dockerfile": "rollup-provider/Dockerfile",
341-
"rebuildPatterns": ["^yarn-project/rollup-provider/"],
342-
"dependencies": [
343-
"yarn-project-base",
344-
"aztec-node",
345-
"circuits.js",
346-
"foundation",
347-
"types"
348-
]
242+
"rebuildPatterns": ["^yarn-project/rollup-provider/"]
349243
},
350244
"aztec-node": {
351245
"buildDir": "yarn-project",
352246
"projectDir": "yarn-project/aztec-node",
353247
"dockerfile": "aztec-node/Dockerfile",
354-
"rebuildPatterns": ["^yarn-project/aztec-node/"],
355-
"dependencies": [
356-
"yarn-project-base",
357-
"archiver",
358-
"circuits.js",
359-
"foundation",
360-
"l1-artifacts",
361-
"merkle-tree",
362-
"p2p",
363-
"sequencer-client",
364-
"types",
365-
"world-state"
366-
]
248+
"rebuildPatterns": ["^yarn-project/aztec-node/"]
367249
},
368250
"sequencer-client": {
369251
"buildDir": "yarn-project",
370252
"projectDir": "yarn-project/sequencer-client",
371253
"dockerfile": "sequencer-client/Dockerfile",
372-
"rebuildPatterns": ["^yarn-project/sequencer-client/"],
373-
"dependencies": [
374-
"yarn-project-base",
375-
"acir-simulator",
376-
"circuits.js",
377-
"ethereum",
378-
"foundation",
379-
"l1-artifacts",
380-
"merkle-tree",
381-
"p2p",
382-
"types",
383-
"world-state"
384-
]
254+
"rebuildPatterns": ["^yarn-project/sequencer-client/"]
385255
},
386256
"types": {
387257
"buildDir": "yarn-project",
388258
"projectDir": "yarn-project/types",
389259
"dockerfile": "types/Dockerfile",
390-
"rebuildPatterns": ["^yarn-project/types/"],
391-
"dependencies": ["yarn-project-base", "circuits.js", "foundation"]
260+
"rebuildPatterns": ["^yarn-project/types/"]
392261
},
393262
"world-state": {
394263
"buildDir": "yarn-project",
395264
"projectDir": "yarn-project/world-state",
396265
"dockerfile": "world-state/Dockerfile",
397-
"rebuildPatterns": ["^yarn-project/world-state/"],
398-
"dependencies": [
399-
"yarn-project-base",
400-
"circuits.js",
401-
"foundation",
402-
"merkle-tree",
403-
"types"
404-
]
266+
"rebuildPatterns": ["^yarn-project/world-state/"]
405267
}
406268
}

0 commit comments

Comments
 (0)