chore: add support for caching node_modules within a nested repository (#12862)

This PR removes a hack added in #12760

This fixes an issue where we were querying the root git repository for
hashes for files which are only tracked in the `noir/noir-repo`
repository. We now set the `REPO_PATH` env variable so that we run `git
ls-tree` on the correct repository.

Author: TomAFrench

bootstrap.sh

@@ -177,6 +177,8 @@ function test {
 function build {
   echo_header "pull submodules"
   denoise "git submodule update --init --recursive"
+  echo_header "sync noir repo"
+  export NOIR_HASH=$(./noir/bootstrap.sh hash)
 
   check_toolchains
 

ci3/npm_install_deps

@@ -3,14 +3,19 @@
 # Note this cache pull won't restore .yarn, which means a subsequent "yarn install" may still be slow.
 # This doesn't matter in CI however.
 NO_CD=1 source ${root:-$(git rev-parse --show-toplevel)}/ci3/source
+# We assume that `yarn.lock` sits within the root directory of the git repository we're currently in.
+#
+# We cannot assume that this is equal to `$root` as we may be within a nested repository
+# whereas `$root` points to the root of the aztec-packages monorepo.
+REPO_PATH=$(git rev-parse --show-toplevel)
 
 if [ ! -f yarn.lock ]; then
   echo_stderr "yarn.lock not found in $PWD."
   exit 1
 fi
 
-yarn_lock_path=$(realpath yarn.lock --relative-to=$root)
-package_json_path=$(realpath package.json --relative-to=$root)
+yarn_lock_path=$(realpath yarn.lock --relative-to=$REPO_PATH)
+package_json_path=$(realpath package.json --relative-to=$REPO_PATH)
 
 if [ "$CI" -eq 1 ]; then
   nm_hash=$(cache_content_hash "^$yarn_lock_path" "^$package_json_path")

noir-projects/aztec-nr/bootstrap.sh

@@ -6,7 +6,8 @@ cmd=${1:-}
 export RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-16}
 export HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16}
 export NARGO=${NARGO:-../../noir/noir-repo/target/release/nargo}
-hash=$(hash_str $(../../noir/bootstrap.sh hash) $(cache_content_hash  "^noir-projects/aztec-nr"))
+export NOIR_HASH=${NOIR_HASH:- $(../../noir/bootstrap.sh hash)}
+hash=$(hash_str "$NOIR_HASH" $(cache_content_hash  "^noir-projects/aztec-nr"))
 
 function build {
   # Being a library, aztec-nr does not technically need to be built. But we can still run nargo check to find any type

noir-projects/bootstrap.sh

@@ -3,6 +3,8 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap
 
 cmd=${1:-}
 
+export NOIR_HASH=${NOIR_HASH-$(../noir/bootstrap.sh hash)}
+
 function build {
   echo_header "noir-projects build"
 

noir-projects/noir-contracts/bootstrap.sh

@@ -27,6 +27,7 @@ export BB=${BB:-../../barretenberg/cpp/build/bin/bb}
 export NARGO=${NARGO:-../../noir/noir-repo/target/release/nargo}
 export TRANSPILER=${TRANSPILER:-../../avm-transpiler/target/release/avm-transpiler}
 export BB_HASH=$(cache_content_hash ../../barretenberg/cpp/.rebuild_patterns)
+export NOIR_HASH=${NOIR_HASH:-$(../../noir/bootstrap.sh hash)}
 
 export tmp_dir=./target/tmp
 
@@ -98,7 +99,7 @@ export -f process_function
 # Compute hash for a given contract.
 function get_contract_hash {
   hash_str \
-    $(../../noir/bootstrap.sh hash) \
+    $NOIR_HASH \
     $(cache_content_hash \
       ../../avm-transpiler/.rebuild_patterns \
       "^noir-projects/noir-contracts/contracts/$1/" \

noir-projects/noir-protocol-circuits/bootstrap.sh

@@ -12,16 +12,16 @@ export PLATFORM_TAG=any
 export BB=${BB:-../../barretenberg/cpp/build/bin/bb}
 export NARGO=${NARGO:-../../noir/noir-repo/target/release/nargo}
 export BB_HASH=$(cache_content_hash ../../barretenberg/cpp/.rebuild_patterns)
-export NARGO_HASH=$(../../noir/bootstrap.sh hash)
+export NOIR_HASH=${NOIR_HASH:-$(../../noir/bootstrap.sh hash)}
 
-test_flag=$project_name-tests-$(hash_str "$NARGO_HASH" $(cache_content_hash "^noir-projects/$project_name"))
+test_flag=$project_name-tests-$(hash_str "$NOIR_HASH" $(cache_content_hash "^noir-projects/$project_name"))
 key_dir=./target/keys
 mkdir -p $key_dir
 
 # Hash of the entire protocol circuits.
 # Needed for test hash, as we presently don't have a program hash for each individual test.
 # Means if anything within the dir changes, the tests will rerun.
-circuits_hash=$(hash_str "$NARGO_HASH" $(cache_content_hash "^noir-projects/$project_name/crates/"))
+circuits_hash=$(hash_str "$NOIR_HASH" $(cache_content_hash "^noir-projects/$project_name/crates/"))
 
 # Circuits matching these patterns we have client-ivc keys computed, rather than ultra-honk.
 ivc_patterns=(
@@ -68,8 +68,8 @@ function compile {
   local program_hash_cmd="$NARGO check --package $name --silence-warnings --show-program-hash | cut -d' ' -f2"
   # echo_stderr $program_hash_cmd
   program_hash=$(dump_fail "$program_hash_cmd")
-  echo_stderr "Hash preimage: $NARGO_HASH-$program_hash"
-  hash=$(hash_str "$NARGO_HASH-$program_hash")
+  echo_stderr "Hash preimage: $NOIR_HASH-$program_hash"
+  hash=$(hash_str "$NOIR_HASH-$program_hash")
 
   if ! cache_download circuit-$hash.tar.gz 1>&2; then
     SECONDS=0

noir/bootstrap.sh

@@ -64,10 +64,16 @@ function noir_content_hash {
   fi
 }
 
+if [ ! -v NOIR_HASH ]; then
+  noir_sync
+  export NOIR_HASH=$(noir_content_hash)
+fi
+
+
 # Builds nargo, acvm and profiler binaries.
 function build_native {
   set -euo pipefail
-  local hash=$(noir_content_hash)
+  local hash=$NOIR_HASH
   if cache_download noir-$hash.tar.gz; then
     return
   fi
@@ -83,23 +89,17 @@ function build_native {
 # Builds js packages.
 function build_packages {
   set -euo pipefail
-  local hash=$(noir_content_hash)
+  local hash=$NOIR_HASH
 
   if cache_download noir-packages-$hash.tar.gz; then
     cd noir-repo
     npm_install_deps
-    # Hack to get around failure introduced by https://github.com/AztecProtocol/aztec-packages/pull/12371
-    # Tests fail with message "env: ‘mocha’: No such file or directory"
-    yarn install
     return
   fi
 
   cd noir-repo
   npm_install_deps
 
-  # Hack to get around failure introduced by https://github.com/AztecProtocol/aztec-packages/pull/12371
-  # Tests fail with message "env: ‘mocha’: No such file or directory"
-  yarn install
   yarn workspaces foreach  -A --parallel --topological-dev --verbose $js_include run build
 
   # We create a folder called packages, that contains each package as it would be published to npm, named correctly.
@@ -156,7 +156,7 @@ function test {
 
 # Prints the commands to run tests, one line per test, prefixed with the appropriate content hash.
 function test_cmds {
-  local test_hash=$(noir_content_hash 1)
+  local test_hash=$NOIR_HASH
   cd noir-repo
   cargo nextest list --workspace --locked --release -Tjson-pretty 2>/dev/null | \
       jq -r '
@@ -231,25 +231,20 @@ case "$cmd" in
     git clean -ffdx
     ;;
   "ci")
-    noir_sync
     build
     test
     ;;
   ""|"fast"|"full")
-    noir_sync
     build
     ;;
   test_cmds|build_native|build_packages|format|test|release)
-    noir_sync
     $cmd "$@"
     ;;
   "hash")
-    noir_sync
-    echo $(noir_content_hash)
+    echo $NOIR_HASH
     ;;
   "hash-tests")
-    noir_sync
-    echo $(noir_content_hash 1)
+    echo $NOIR_HASH
     ;;
   "make-patch")
     scripts/sync.sh make-patch