chore: replace benchmarks on fast test suites with a cut-off (#7276)

Author: TomAFrench

.github/workflows/test-js-packages.yml

@@ -534,7 +534,7 @@ jobs:
       - name: Build list of libraries
         id: get_critical_libraries
         run: |
-          LIBRARIES=$(grep -Po "^https://github.com/\K.+"  ./CRITICAL_NOIR_LIBRARIES | jq -R -s -c 'split("\n") | map(select(. != "")) | map({ repo: ., path: ""})')
+          LIBRARIES=$(yq ./EXTERNAL_NOIR_LIBRARIES.yml -o json | jq -c '.libraries | map({ repo, path: (.path // ""), timeout:(.timeout // 2), nargo_args })')
           echo "libraries=$LIBRARIES"
           echo "libraries=$LIBRARIES" >> $GITHUB_OUTPUT
         env:
@@ -548,17 +548,7 @@ jobs:
       fail-fast: false
       matrix:
         project: ${{ fromJson( needs.critical-library-list.outputs.libraries )}}
-        include:
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/aztec-nr }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/blob }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-lib }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-lib }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/reset-kernel-lib }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/types }
-          # Use 1 test threads for rollup-lib because each test requires a lot of memory, and multiple ones in parallel exceed the maximum memory limit.
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-lib, nargo_args: "--test-threads 1" }
-
+         
     name: Check external repo - ${{ matrix.project.repo }}/${{ matrix.project.path }}
     steps:
       - name: Checkout
@@ -604,6 +594,12 @@ jobs:
           nargo test --silence-warnings --skip-brillig-constraints-check --format json ${{ matrix.project.nargo_args }} | tee $output_file
           TIME=$(($SECONDS-$BEFORE))
 
+          if [ "TIME" -gt "${{ matrix.project.timeout }}" ]; then
+            # Library testing time has regressed past the set timeout.
+            # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause.
+            exit 1
+          fi
+
           NAME=${{ matrix.project.repo }}/${{ matrix.project.path }}
           # Replace any slashes with underscores
           NAME=${NAME//\//_}
@@ -618,12 +614,13 @@ jobs:
           fi
         env:
           NARGO_IGNORE_TEST_FAILURES_FROM_FOREIGN_CALLS: true
-      
+
       - name: Compare test results
         working-directory: ./noir-repo
         run: .github/scripts/check_test_results.sh .github/critical_libraries_status/${{ matrix.project.repo }}/${{ matrix.project.path }}.failures.jsonl .github/critical_libraries_status/${{ matrix.project.repo }}/${{ matrix.project.path }}.actual.jsonl
 
       - name: Upload test report
+        if: ${{ matrix.project.timeout > 10 }} # We want to avoid recording benchmarking for a ton of tiny libraries, these should be covered with aggressive timeouts
         uses: actions/upload-artifact@v4
         with:
           name: ${{ steps.test_report.outputs.test_report_name }}

EXTERNAL_NOIR_LIBRARIES.yml

@@ -0,0 +1,76 @@
+
+libraries:
+  noir_check_shuffle:
+    repo: noir-lang/noir_check_shuffle
+    timeout: 2
+  ec:
+    repo: noir-lang/ec
+    timeout: 2
+  eddsa:
+    repo: noir-lang/eddsa
+    timeout: 2
+  mimc:
+    repo: noir-lang/mimc
+    timeout: 2
+  schnorr:
+    repo: noir-lang/schnorr
+    timeout: 2
+  noir_sort:
+    repo: noir-lang/noir_sort
+    timeout: 2
+  noir-edwards:
+    repo: noir-lang/noir-edwards
+    timeout: 2
+  noir-bignum:
+    repo: noir-lang/noir-bignum
+  noir_bigcurve:
+    repo: noir-lang/noir_bigcurve
+  noir_base64:
+    repo: noir-lang/noir_base64
+    timeout: 2
+  noir_string_search:
+    repo: noir-lang/noir_string_search
+    timeout: 2
+  sparse_array:
+    repo: noir-lang/sparse_array
+    timeout: 2
+  noir_rsa:
+    repo: noir-lang/noir_rsa
+    timeout: 2
+  noir_json_parser:
+    repo: noir-lang/noir_json_parser
+    timeout: 10
+  aztec_nr:
+    repo: AztecProtocol/aztec-packages
+    path: noir-projects/aztec-nr
+    timeout: 60
+  noir_contracts:
+    repo: AztecProtocol/aztec-packages
+    path: noir-projects/noir-contracts
+    timeout: 60
+  blob:
+    repo: AztecProtocol/aztec-packages
+    path: noir-projects/noir-protocol-circuits/crates/blob
+    timeout: 70
+  protocol_circuits_parity_lib:
+    repo: AztecProtocol/aztec-packages
+    path: noir-projects/noir-protocol-circuits/crates/parity-lib
+    timeout: 4
+  protocol_circuits_private_kernel_lib:
+    repo: AztecProtocol/aztec-packages
+    path: noir-projects/noir-protocol-circuits/crates/private-kernel-lib
+    timeout: 250
+  protocol_circuits_reset_kernel_lib:
+    repo: AztecProtocol/aztec-packages
+    path: noir-projects/noir-protocol-circuits/crates/reset-kernel-lib
+    timeout: 15
+  protocol_circuits_types:
+    repo: AztecProtocol/aztec-packages
+    path: noir-projects/noir-protocol-circuits/crates/types
+    timeout: 60
+  protocol_circuits_rollup_lib:
+    repo: AztecProtocol/aztec-packages
+    path: noir-projects/noir-protocol-circuits/crates/rollup-lib
+    timeout: 300
+    # Use 1 test threads for rollup-lib because each test requires a lot of memory, and multiple ones in parallel exceed the maximum memory limit.
+    nargo_args: "--test-threads 1"