Skip to content

Commit c8a5cfb

Browse files
charlielyecharlielye
authored
feat: build_manifest default tweaks. (#2287)
If rebuildPatterns are not provided in build_manifest.json, default to the project dir. Can also exclude empty dependencies.
1 parent c46b3c8 commit c8a5cfb

File tree

27 files changed

+57
-252
lines changed

27 files changed

+57
-252
lines changed

‎build-system/scripts/query_manifest

+26-9
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ function get_deps {
1515
local TYPE=$(jq -r ".\"$1\".dependencies | type" $MANIFEST)
1616
if [ "$TYPE" == "string" ]; then
1717
# Execute string as command relative to buildDir to retrieve dependencies.
18+
local BUILD_DIR=$($0 buildDir $1)
1819
local CMD=$BUILD_DIR/$(jq -r ".\"$1\".dependencies")
1920
if [ ! -f "$CMD" ]; then
2021
>&2 echo "Dependency script not found: $CMD"
@@ -24,10 +25,11 @@ function get_deps {
2425
DEPS=($($CMD $PROJECT_DIR))
2526
elif [ "$TYPE" == "null" ]; then
2627
# Execute default script relative to buildDir to retrieve dependencies.
28+
local BUILD_DIR=$($0 buildDir $1)
2729
local CMD=$BUILD_DIR/scripts/get_dependencies.sh
2830
if [ ! -f "$CMD" ]; then
29-
>&2 echo "Dependency script not found: $CMD"
30-
exit 1
31+
DEPS=()
32+
return
3133
fi
3234
local PROJECT_DIR=$($0 projectDir $1)
3335
DEPS=($($CMD $PROJECT_DIR))
@@ -43,22 +45,25 @@ function add_rebuild_patterns {
4345
local TYPE=$(jq -r ".\"$1\".rebuildPatterns | type" $MANIFEST)
4446
if [ "$TYPE" == "string" ]; then
4547
local FILE=$(jq -r ".\"$1\".rebuildPatterns" $MANIFEST)
46-
local BUILD_DIR=$($0 buildDir $1)
47-
PATTERNS=(${PATTERNS[@]} $(cat $BUILD_DIR/$FILE))
48+
local PROJECT_DIR=$($0 projectDir $1)
49+
PATTERNS=(${PATTERNS[@]} $(cat $PROJECT_DIR/$FILE))
4850
elif [ "$TYPE" == "array" ]; then
4951
PATTERNS=(${PATTERNS[@]} $(jq -r ".\"$1\".rebuildPatterns | .[]" $MANIFEST))
52+
elif [ "$TYPE" == "null" ]; then
53+
local PROJECT_DIR=$($0 relativeProjectDir $1)
54+
PATTERNS=(${PATTERNS[@]} "^$PROJECT_DIR/")
5055
else
51-
>&2 echo "Missing rebuildPatterns property. Either filename as string, or patterns as array."
56+
>&2 echo "rebuildPatterns must be array, string, or null."
5257
exit 1
5358
fi
5459
}
5560

5661
case "$CMD" in
5762
dockerfile)
58-
# In the manifest, the path is relative to buildDir. Return absolute path.
59-
BUILD_DIR=$($0 buildDir $REPO)
63+
# In the manifest, the path is relative to projectDir. Return absolute path.
64+
PROJECT_DIR=$($0 projectDir $REPO)
6065
DOCKERFILE=$(jq -r ".\"$REPO\".dockerfile // \"Dockerfile\"" $MANIFEST)
61-
echo $BUILD_DIR/$DOCKERFILE
66+
echo $PROJECT_DIR/$DOCKERFILE
6267
;;
6368
buildDir)
6469
# In the manifest, the path is relative to the repo root. Return absolute path.
@@ -71,10 +76,15 @@ case "$CMD" in
7176
echo $ROOT_PATH/$PROJECT_DIR
7277
;;
7378
relativeProjectDir)
79+
# Return the relative path as it is in the manifest.
7480
jq -r ".\"$REPO\".projectDir // .\"$REPO\".buildDir" $MANIFEST
7581
;;
7682
dependencies)
77-
BUILD_DIR=$($0 buildDir $REPO)
83+
# Get dependencies for a given repo.
84+
# If no entry in the manifest file, attempt to call <projectDir>/scripts/get_dependencies.sh if exists, else empty.
85+
# If a string, attempt to call <projectDir>/<string> if exists, else error.
86+
# If an array, the array lists the dependencies.
87+
# Recursively descend "unvisited" dependencies to collect all dependencies.
7888
declare -A ALL_DEPS
7989
add_deps() {
8090
if [[ -v ALL_DEPS[$1] ]]; then
@@ -87,11 +97,18 @@ case "$CMD" in
8797
done
8898
}
8999
add_deps $REPO
100+
# Remove ourself as a dependency.
101+
unset ALL_DEPS["$REPO"]
90102
for KEY in "${!ALL_DEPS[@]}"; do
91103
echo $KEY
92104
done | sort
93105
;;
94106
rebuildPatterns)
107+
# Get rebuild patterns for a given repo (the file patterns that if changed result in rebuilds).
108+
# First add rebuild patterns for requested repo, then add rebuild patterns for each dependency.
109+
# If no rebuild patterns are given, the result is ["^<projectDir>/"].
110+
# If a projects rebuildPattern is a string, the rebuild patterns are in <projectDir>/<string>.
111+
# If an array, the array lists the rebuild patterns.
95112
DEPS=($($0 dependencies $REPO))
96113
PATTERNS=()
97114
add_rebuild_patterns $REPO

‎build_manifest.json

+31-89
Original file line numberDiff line numberDiff line change
@@ -2,48 +2,40 @@
22
"barretenberg-x86_64-linux-clang": {
33
"buildDir": "barretenberg/cpp",
44
"dockerfile": "dockerfiles/Dockerfile.x86_64-linux-clang",
5-
"rebuildPatterns": ".rebuild_patterns",
6-
"dependencies": []
5+
"rebuildPatterns": ".rebuild_patterns"
76
},
87
"barretenberg-x86_64-linux-clang-assert": {
98
"buildDir": "barretenberg/cpp",
109
"dockerfile": "dockerfiles/Dockerfile.x86_64-linux-clang-assert",
11-
"rebuildPatterns": ".rebuild_patterns",
12-
"dependencies": []
10+
"rebuildPatterns": ".rebuild_patterns"
1311
},
1412
"barretenberg-x86_64-linux-clang-fuzzing": {
1513
"buildDir": "barretenberg/cpp",
1614
"dockerfile": "dockerfiles/Dockerfile.x86_64-linux-clang-fuzzing",
17-
"rebuildPatterns": ".rebuild_patterns",
18-
"dependencies": []
15+
"rebuildPatterns": ".rebuild_patterns"
1916
},
2017
"barretenberg-x86_64-linux-gcc": {
2118
"buildDir": "barretenberg/cpp",
2219
"dockerfile": "dockerfiles/Dockerfile.x86_64-linux-gcc",
23-
"rebuildPatterns": ".rebuild_patterns",
24-
"dependencies": []
20+
"rebuildPatterns": ".rebuild_patterns"
2521
},
2622
"barretenberg-wasm-linux-clang": {
2723
"buildDir": "barretenberg/cpp",
2824
"dockerfile": "dockerfiles/Dockerfile.wasm-linux-clang",
29-
"rebuildPatterns": ".rebuild_patterns",
30-
"dependencies": []
25+
"rebuildPatterns": ".rebuild_patterns"
3126
},
3227
"bb.js": {
3328
"buildDir": "barretenberg/ts",
34-
"rebuildPatterns": ["^barretenberg/ts/"],
3529
"dependencies": ["barretenberg-wasm-linux-clang"]
3630
},
3731
"barretenberg-acir-tests-bb": {
3832
"buildDir": "barretenberg/acir_tests",
3933
"dockerfile": "Dockerfile.bb",
40-
"rebuildPatterns": ["^barretenberg/acir_tests/"],
4134
"dependencies": ["barretenberg-x86_64-linux-clang-assert"]
4235
},
4336
"barretenberg-acir-tests-bb.js": {
4437
"buildDir": "barretenberg/acir_tests",
4538
"dockerfile": "Dockerfile.bb.js",
46-
"rebuildPatterns": ["^barretenberg/acir_tests/"],
4739
"dependencies": ["bb.js"]
4840
},
4941
"circuits-wasm-linux-clang": {
@@ -79,19 +71,14 @@
7971
"docs": {
8072
"buildDir": ".",
8173
"dockerfile": "docs/Dockerfile",
82-
"rebuildPatterns": ["^docs/", "^.*.cpp$", "^.*.ts$"],
83-
"dependencies": []
74+
"rebuildPatterns": ["^docs/", "^.*.cpp$", "^.*.ts$"]
8475
},
8576
"l1-contracts": {
86-
"buildDir": "l1-contracts",
87-
"dockerfile": "Dockerfile",
88-
"rebuildPatterns": ["^l1-contracts/"],
89-
"dependencies": []
77+
"buildDir": "l1-contracts"
9078
},
9179
"l1-artifacts": {
9280
"buildDir": "yarn-project",
9381
"projectDir": "yarn-project/l1-artifacts",
94-
"dockerfile": "l1-artifacts/Dockerfile",
9582
"rebuildPatterns": ["^l1-contracts/", "^yarn-project/l1-artifacts/"],
9683
"dependencies": []
9784
},
@@ -111,92 +98,65 @@
11198
},
11299
"acir-simulator": {
113100
"buildDir": "yarn-project",
114-
"projectDir": "yarn-project/acir-simulator",
115-
"dockerfile": "acir-simulator/Dockerfile",
116-
"rebuildPatterns": ["^yarn-project/acir-simulator/"]
101+
"projectDir": "yarn-project/acir-simulator"
117102
},
118103
"archiver": {
119104
"buildDir": "yarn-project",
120-
"projectDir": "yarn-project/archiver",
121-
"dockerfile": "archiver/Dockerfile",
122-
"rebuildPatterns": ["^yarn-project/archiver/"]
105+
"projectDir": "yarn-project/archiver"
123106
},
124107
"cli": {
125108
"buildDir": "yarn-project",
126-
"projectDir": "yarn-project/cli",
127-
"dockerfile": "cli/Dockerfile",
128-
"rebuildPatterns": ["^yarn-project/cli/"]
109+
"projectDir": "yarn-project/cli"
129110
},
130111
"aztec-rpc": {
131112
"buildDir": "yarn-project",
132-
"projectDir": "yarn-project/aztec-rpc",
133-
"dockerfile": "aztec-rpc/Dockerfile",
134-
"rebuildPatterns": ["^yarn-project/aztec-rpc/"]
113+
"projectDir": "yarn-project/aztec-rpc"
135114
},
136115
"aztec-sandbox": {
137116
"buildDir": "yarn-project",
138-
"projectDir": "yarn-project/aztec-sandbox",
139-
"dockerfile": "aztec-sandbox/Dockerfile",
140-
"rebuildPatterns": ["^yarn-project/aztec-sandbox/"]
117+
"projectDir": "yarn-project/aztec-sandbox"
141118
},
142119
"aztec.js": {
143120
"buildDir": "yarn-project",
144-
"projectDir": "yarn-project/aztec.js",
145-
"dockerfile": "aztec.js/Dockerfile",
146-
"rebuildPatterns": ["^yarn-project/aztec.js/"]
121+
"projectDir": "yarn-project/aztec.js"
147122
},
148123
"canary-build": {
149124
"buildDir": "yarn-project",
150125
"projectDir": "yarn-project/canary",
151-
"dockerfile": "canary/Dockerfile.build",
152-
"rebuildPatterns": ["^yarn-project/canary/"]
126+
"dockerfile": "Dockerfile.build"
153127
},
154128
"canary": {
155129
"buildDir": "yarn-project",
156-
"projectDir": "yarn-project/canary",
157-
"dockerfile": "canary/Dockerfile",
158-
"rebuildPatterns": ["^yarn-project/canary/"]
130+
"projectDir": "yarn-project/canary"
159131
},
160132
"circuits.js": {
161133
"buildDir": "yarn-project",
162-
"projectDir": "yarn-project/circuits.js",
163-
"dockerfile": "circuits.js/Dockerfile",
164-
"rebuildPatterns": ["^yarn-project/circuits.js/"]
134+
"projectDir": "yarn-project/circuits.js"
165135
},
166136
"end-to-end": {
167137
"buildDir": "yarn-project",
168-
"projectDir": "yarn-project/end-to-end",
169-
"dockerfile": "end-to-end/Dockerfile",
170-
"rebuildPatterns": ["^yarn-project/end-to-end/"]
138+
"projectDir": "yarn-project/end-to-end"
171139
},
172140
"ethereum": {
173141
"buildDir": "yarn-project",
174-
"projectDir": "yarn-project/ethereum",
175-
"dockerfile": "ethereum/Dockerfile",
176-
"rebuildPatterns": ["^yarn-project/ethereum/"]
142+
"projectDir": "yarn-project/ethereum"
177143
},
178144
"foundation": {
179145
"buildDir": "yarn-project",
180-
"projectDir": "yarn-project/foundation",
181-
"dockerfile": "foundation/Dockerfile",
182-
"rebuildPatterns": ["^yarn-project/foundation/"]
146+
"projectDir": "yarn-project/foundation"
183147
},
184148
"key-store": {
185149
"buildDir": "yarn-project",
186-
"projectDir": "yarn-project/key-store",
187-
"dockerfile": "key-store/Dockerfile",
188-
"rebuildPatterns": ["^yarn-project/key-store/"]
150+
"projectDir": "yarn-project/key-store"
189151
},
190152
"merkle-tree": {
191153
"buildDir": "yarn-project",
192-
"projectDir": "yarn-project/merkle-tree",
193-
"dockerfile": "merkle-tree/Dockerfile",
194-
"rebuildPatterns": ["^yarn-project/merkle-tree/"]
154+
"projectDir": "yarn-project/merkle-tree"
195155
},
196156
"noir-contracts-build": {
197157
"buildDir": "yarn-project",
198158
"projectDir": "yarn-project/noir-contracts",
199-
"dockerfile": "noir-contracts/Dockerfile.build",
159+
"dockerfile": "Dockerfile.build",
200160
"rebuildPatterns": [
201161
"^yarn-project/noir-contracts/",
202162
"^yarn-project/aztec-nr/"
@@ -205,64 +165,46 @@
205165
"noir-contracts": {
206166
"buildDir": "yarn-project",
207167
"projectDir": "yarn-project/noir-contracts",
208-
"dockerfile": "noir-contracts/Dockerfile",
209168
"rebuildPatterns": [
210169
"^yarn-project/noir-contracts/",
211170
"^yarn-project/aztec-nr/"
212171
]
213172
},
214173
"noir-compiler": {
215174
"buildDir": "yarn-project",
216-
"projectDir": "yarn-project/noir-compiler",
217-
"dockerfile": "noir-compiler/Dockerfile",
218-
"rebuildPatterns": ["^yarn-project/noir-compiler/"]
175+
"projectDir": "yarn-project/noir-compiler"
219176
},
220177
"p2p": {
221178
"buildDir": "yarn-project",
222-
"projectDir": "yarn-project/p2p",
223-
"dockerfile": "p2p/Dockerfile",
224-
"rebuildPatterns": ["^yarn-project/p2p/"]
179+
"projectDir": "yarn-project/p2p"
225180
},
226181
"p2p-bootstrap": {
227182
"buildDir": "yarn-project",
228183
"projectDir": "yarn-project/p2p-bootstrap",
229-
"dockerfile": "p2p/Dockerfile",
230-
"rebuildPatterns": ["^yarn-project/p2p-bootstrap/"]
184+
"dockerfile": "../p2p/Dockerfile"
231185
},
232186
"prover-client": {
233187
"buildDir": "yarn-project",
234-
"projectDir": "yarn-project/prover-client",
235-
"dockerfile": "prover-client/Dockerfile",
236-
"rebuildPatterns": ["^yarn-project/prover-client/"]
188+
"projectDir": "yarn-project/prover-client"
237189
},
238190
"rollup-provider": {
239191
"buildDir": "yarn-project",
240-
"projectDir": "yarn-project/rollup-provider",
241-
"dockerfile": "rollup-provider/Dockerfile",
242-
"rebuildPatterns": ["^yarn-project/rollup-provider/"]
192+
"projectDir": "yarn-project/rollup-provider"
243193
},
244194
"aztec-node": {
245195
"buildDir": "yarn-project",
246-
"projectDir": "yarn-project/aztec-node",
247-
"dockerfile": "aztec-node/Dockerfile",
248-
"rebuildPatterns": ["^yarn-project/aztec-node/"]
196+
"projectDir": "yarn-project/aztec-node"
249197
},
250198
"sequencer-client": {
251199
"buildDir": "yarn-project",
252-
"projectDir": "yarn-project/sequencer-client",
253-
"dockerfile": "sequencer-client/Dockerfile",
254-
"rebuildPatterns": ["^yarn-project/sequencer-client/"]
200+
"projectDir": "yarn-project/sequencer-client"
255201
},
256202
"types": {
257203
"buildDir": "yarn-project",
258-
"projectDir": "yarn-project/types",
259-
"dockerfile": "types/Dockerfile",
260-
"rebuildPatterns": ["^yarn-project/types/"]
204+
"projectDir": "yarn-project/types"
261205
},
262206
"world-state": {
263207
"buildDir": "yarn-project",
264-
"projectDir": "yarn-project/world-state",
265-
"dockerfile": "world-state/Dockerfile",
266-
"rebuildPatterns": ["^yarn-project/world-state/"]
208+
"projectDir": "yarn-project/world-state"
267209
}
268210
}

‎yarn-project/acir-simulator/package.json

-2
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@
1111
"tsconfig": "./tsconfig.json"
1212
},
1313
"scripts": {
14-
"prepare": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json",
15-
"prepare:check": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json --check",
1614
"build": "yarn clean && tsc -b",
1715
"build:dev": "tsc -b --watch",
1816
"clean": "rm -rf ./dest .tsbuildinfo",

‎yarn-project/archiver/package.json

-2
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@
1111
"tsconfig": "./tsconfig.json"
1212
},
1313
"scripts": {
14-
"prepare": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json",
15-
"prepare:check": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json --check",
1614
"build": "yarn clean && tsc -b",
1715
"build:dev": "tsc -b --watch",
1816
"clean": "rm -rf ./dest .tsbuildinfo",

‎yarn-project/aztec-node/package.json

-2
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@
1212
"tsconfig": "./tsconfig.json"
1313
},
1414
"scripts": {
15-
"prepare": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json",
16-
"prepare:check": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json --check",
1715
"build": "yarn clean && tsc -b",
1816
"build:dev": "tsc -b --watch",
1917
"clean": "rm -rf ./dest .tsbuildinfo",

‎yarn-project/aztec-rpc/package.json

-2
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@
1111
"tsconfig": "./tsconfig.json"
1212
},
1313
"scripts": {
14-
"prepare": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json",
15-
"prepare:check": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json --check",
1614
"build": "yarn clean && tsc -b",
1715
"build:dev": "tsc -b --watch",
1816
"clean": "rm -rf ./dest .tsbuildinfo",

‎yarn-project/aztec-sandbox/package.json

-2
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,11 @@
1414
"tsconfig": "./tsconfig.json"
1515
},
1616
"scripts": {
17-
"prepare": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json",
1817
"build": "yarn clean && tsc -b",
1918
"start": "node --no-warnings ./dest/bin",
2019
"clean": "rm -rf ./dest .tsbuildinfo",
2120
"formatting": "run -T prettier --check ./src && run -T eslint ./src",
2221
"formatting:fix": "run -T prettier -w ./src",
23-
"prepare:check": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json --check",
2422
"build:dev": "tsc -b --watch",
2523
"test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --passWithNoTests",
2624
"run:example:token": "DEBUG='aztec:*' node ./dest/examples/private_token_contract.js",

0 commit comments

Comments
 (0)