Merge branch 'master' into sync-noir

* master:
  feat: verify public validation requests (#8150)
  feat: ultra keccak honk verifier (#8261)
  git subrepo push --branch=master noir-projects/aztec-nr
  git_subrepo.sh: Fix parent in .gitrepo file. [skip ci]
  chore: replace relative paths to noir-protocol-circuits
  git subrepo push --branch=master barretenberg
  chore(bb): use std::span for srs (#8371)
  feat: router contract (#8352)
  chore: Merge provernet to master (#8373)
  feat: Sync from noir (#8363)
  chore: Change efs volumes to use bursting throughput (#8370)
  chore: move spartan network tests to nightly (#8369)
  chore(ci): Test lowering of non-persistent ebs provisions (#8360)
  chore(bb): reinstate "chore: uncomment asserts in oink rec verifier"" (#8356)
  fix: Split stores per component and split merkle tree operations (#8299)
  fix: TXE logs in docker (#8365)
  feat: Liveness analysis for constants (#8294)
  chore(avm): remove some unused deps (#8366)

Author: TomAFrench

.github/ensure-tester/action.yml

@@ -70,7 +70,10 @@ runs:
 
     - name: Print Startup Log
       shell: bash
-      run: cat /run/log.out || true # don't fail
+      run:
+        # Try 6 times to see if .user-data-finished has been written to
+        for i in {0..6} ; do scripts/run_on_tester "cat /home/ubuntu/.user-data-finished" && break ; sleep 5 ; done ;
+        scripts/run_on_tester "cat /run/log.out"
 
     # Set up a context for this run
     - name: Copy Repo to Tester

.github/spot-runner-action/dist/index.js

@@ -281,8 +281,8 @@ class Ec2Instance {
                             Ebs: {
                                 VolumeSize: 64,
                                 VolumeType: 'gp3',
-                                Throughput: 1000,
-                                Iops: 5000
+                                Throughput: 250,
+                                Iops: 3000
                             },
                         },
                     ],
@@ -1040,7 +1040,14 @@ class UserData {
                 `sudo service docker restart`,
                 "sudo wget -q https://github.com/earthly/earthly/releases/download/v0.8.10/earthly-linux-$(dpkg --print-architecture) -O /usr/local/bin/earthly",
                 "sudo chmod +x /usr/local/bin/earthly",
-                `sudo bash -c 'echo \'Acquire::Retries "3"; Acquire::https::Timeout "240"; Acquire::http::Timeout "240"; APT::Get::Assume-Yes "true"; APT::Install-Recommends "false"; APT::Install-Suggests "false";\' > /etc/apt/apt.conf.d/99-aztec-build'`,
+                `sudo bash -c 'cat <<EOF > /etc/apt/apt.conf.d/99-aztec-build
+Acquire::Retries "3";
+Acquire::https::Timeout "240";
+Acquire::http::Timeout "240";
+APT::Get::Assume-Yes "true";
+APT::Install-Recommends "false";
+APT::Install-Suggests "false";
+EOF'`,
                 "sudo apt install -y brotli",
                 'echo "MaxStartups 1000" >> /etc/ssh/sshd_config',
                 'echo "ClientAliveInterval=30" >> /etc/ssh/sshd_config',

.github/spot-runner-action/src/ec2.ts

@@ -200,8 +200,8 @@ export class Ec2Instance {
             Ebs: {
               VolumeSize: 64,
               VolumeType: 'gp3',
-              Throughput: 1000,
-              Iops: 5000
+              Throughput: 250,
+              Iops: 3000
             },
           },
         ],

.github/spot-runner-action/src/userdata.ts

@@ -22,7 +22,14 @@ export class UserData {
       `sudo service docker restart`,
       "sudo wget -q https://github.com/earthly/earthly/releases/download/v0.8.10/earthly-linux-$(dpkg --print-architecture) -O /usr/local/bin/earthly",
       "sudo chmod +x /usr/local/bin/earthly",
-      `sudo bash -c 'echo \'Acquire::Retries "3"; Acquire::https::Timeout "240"; Acquire::http::Timeout "240"; APT::Get::Assume-Yes "true"; APT::Install-Recommends "false"; APT::Install-Suggests "false";\' > /etc/apt/apt.conf.d/99-aztec-build'`,
+      `sudo bash -c 'cat <<EOF > /etc/apt/apt.conf.d/99-aztec-build
+Acquire::Retries "3";
+Acquire::https::Timeout "240";
+Acquire::http::Timeout "240";
+APT::Get::Assume-Yes "true";
+APT::Install-Recommends "false";
+APT::Install-Suggests "false";
+EOF'`,
       "sudo apt install -y brotli",
       'echo "MaxStartups 1000" >> /etc/ssh/sshd_config',
       'echo "ClientAliveInterval=30" >> /etc/ssh/sshd_config',

.github/workflows/ci.yml

@@ -161,37 +161,6 @@ jobs:
             export FORCE_COLOR=1
             ../../scripts/earthly-ci -P --no-output +${{ matrix.test }}
 
-  network-e2e:
-    needs: [build, changes]
-    if: ${{ needs.changes.outputs.non-barretenberg-cpp == 'true' }}
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        values_file: ["default.yaml", "3-validators.yaml"]
-    steps:
-      - uses: actions/checkout@v4
-        with: { ref: "${{ env.GIT_COMMIT }}" }
-      - uses: ./.github/ci-setup-action
-      - name: Setup and Test
-        uses: ./.github/ensure-tester-with-images
-        timeout-minutes: 45
-        with:
-          runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '16core-tester-x86' }}
-          builder_type: builder-x86
-          # these are copied to the tester and expected by the earthly command below
-          # if they fail to copy, it will try to build them on the tester and fail
-          builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
-          # command to produce the images in case they don't exist
-          builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
-          tester_ttl: 40
-          run: |
-            set -eux
-            cd ./yarn-project/end-to-end/
-            ./scripts/setup_local_k8s.sh
-            export FORCE_COLOR=1
-            export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}"
-            ../../scripts/earthly-ci --exec-stats -P --no-output ./+network-transfer --values-file=${{ matrix.values_file }}
-
   # all the benchmarking end-to-end integration tests for aztec (not required to merge)
   bench-e2e:
     needs: [build, changes]

.github/workflows/devnet-deploys.yml

@@ -66,6 +66,8 @@ env:
   TF_VAR_FORK_MNEMONIC: ${{ secrets.FORK_MNEMONIC }}
   TF_VAR_INFURA_API_KEY: ${{ secrets.INFURA_API_KEY }}
   TF_VAR_FORK_ADMIN_API_KEY: ${{ secrets.DEVNET_API_KEY }}
+  TF_VAR_MAINNET_FORK_CPU_UNITS: 2048
+  TF_VAR_MAINNET_FORK_MEMORY_UNITS: 4096
 
   # Faucet
   TF_VAR_FAUCET_ACCOUNT_INDEX: 9
@@ -123,6 +125,12 @@ jobs:
       min_txs_per_block: ${{ steps.set_network_vars.outputs.min_txs_per_block }}
       bot_flush_setup_txs: ${{ steps.set_network_vars.outputs.bot_flush_setup_txs }}
       bot_max_pending_txs: ${{ steps.set_network_vars.outputs.bot_max_pending_txs }}
+      mainnet_fork_cpu_units: ${{ steps.set_network_vars.outputs.mainnet_fork_cpu_units }}
+      mainnet_fork_memory_units: ${{ steps.set_network_vars.outputs.mainnet_fork_memory_units }}
+      bot_skip_simulation: ${{ steps.set_network_vars.outputs.bot_skip_simulation }}
+      bot_l2_gas_limit: ${{ steps.set_network_vars.outputs.bot_l2_gas_limit }}
+      bot_da_gas_limit: ${{ steps.set_network_vars.outputs.bot_da_gas_limit }}
+      bot_count: ${{ steps.set_network_vars.outputs.bot_count }}
     steps:
       - name: Set network vars
         shell: bash
@@ -135,7 +143,7 @@ jobs:
               echo "branch_name=devnet" >> $GITHUB_OUTPUT
               echo "network_api_key=DEVNET_API_KEY" >> $GITHUB_OUTPUT
               echo "network_fork_admin_api_key=DEVNET_API_KEY" >> $GITHUB_OUTPUT
-              echo "agents_per_prover=4" >> $GITHUB_OUTPUT
+              echo "agents_per_prover=2" >> $GITHUB_OUTPUT
               echo "bot_interval=180" >> $GITHUB_OUTPUT
               echo "node_tcp_range_start=40100" >> $GITHUB_OUTPUT
               echo "node_udp_range_start=45100" >> $GITHUB_OUTPUT
@@ -147,9 +155,15 @@ jobs:
               echo "faucet_lb_priority=601" >> $GITHUB_OUTPUT
               echo "min_txs_per_block=1" >> $GITHUB_OUTPUT
               echo "max_txs_per_block=64" >> $GITHUB_OUTPUT
-              echo "bot_follow_chain=NONE" >> $GITHUB_OUTPUT
+              echo "bot_follow_chain=PROVEN" >> $GITHUB_OUTPUT
               echo "bot_flush_setup_txs=false" >> $GITHUB_OUTPUT
               echo "bot_max_pending_txs=1" >> $GITHUB_OUTPUT
+              echo "mainnet_fork_cpu_units=2048" >> $GITHUB_OUTPUT
+              echo "mainnet_fork_memory_units=4096" >> $GITHUB_OUTPUT
+              echo "bot_skip_simulation=false" >> $GITHUB_OUTPUT
+              echo "bot_l2_gas_limit=" >> $GITHUB_OUTPUT
+              echo "bot_da_gas_limit=" >> $GITHUB_OUTPUT
+              echo "bot_count=1" >> $GITHUB_OUTPUT
           elif [ "$BRANCH_NAME" = "provernet" ]
           then
               echo "deploy_tag=provernet" >> $GITHUB_OUTPUT
@@ -171,14 +185,20 @@ jobs:
               echo "bot_follow_chain=NONE" >> $GITHUB_OUTPUT
               echo "bot_flush_setup_txs=true" >> $GITHUB_OUTPUT
               echo "bot_max_pending_txs=32" >> $GITHUB_OUTPUT
+              echo "mainnet_fork_cpu_units=8192" >> $GITHUB_OUTPUT
+              echo "mainnet_fork_memory_units=32768" >> $GITHUB_OUTPUT
+              echo "bot_skip_simulation=true" >> $GITHUB_OUTPUT
+              echo "bot_l2_gas_limit=1000000000" >> $GITHUB_OUTPUT
+              echo "bot_da_gas_limit=1000000000" >> $GITHUB_OUTPUT
+              echo "bot_count=1" >> $GITHUB_OUTPUT
           elif [ "$BRANCH_NAME" = "alphanet" ]
           then
               echo "deploy_tag=alphanet" >> $GITHUB_OUTPUT
               echo "branch_name=alphanet" >> $GITHUB_OUTPUT
               echo "network_api_key=ALPHANET_API_KEY" >> $GITHUB_OUTPUT
               echo "network_fork_admin_api_key=ALPHANET_API_KEY" >> $GITHUB_OUTPUT
               echo "agents_per_prover=1" >> $GITHUB_OUTPUT
-              echo "bot_interval=30" >> $GITHUB_OUTPUT
+              echo "bot_interval=10" >> $GITHUB_OUTPUT
               echo "node_tcp_range_start=40000" >> $GITHUB_OUTPUT
               echo "node_udp_range_start=45000" >> $GITHUB_OUTPUT
               echo "prover_node_tcp_range_start=41000" >> $GITHUB_OUTPUT
@@ -192,6 +212,12 @@ jobs:
               echo "bot_follow_chain=PROVEN" >> $GITHUB_OUTPUT
               echo "bot_flush_setup_txs=false" >> $GITHUB_OUTPUT
               echo "bot_max_pending_txs=1" >> $GITHUB_OUTPUT
+              echo "mainnet_fork_cpu_units=2048" >> $GITHUB_OUTPUT
+              echo "mainnet_fork_memory_units=4096" >> $GITHUB_OUTPUT
+              echo "bot_skip_simulation=false" >> $GITHUB_OUTPUT
+              echo "bot_l2_gas_limit=" >> $GITHUB_OUTPUT
+              echo "bot_da_gas_limit=" >> $GITHUB_OUTPUT
+              echo "bot_count=1" >> $GITHUB_OUTPUT
           else
               echo "Unrecognized Branch!!"
               exit 1
@@ -462,6 +488,12 @@ jobs:
       TF_VAR_PROVER_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.prover_node_lb_priority_range_start }}
       TF_VAR_SEQ_MIN_TX_PER_BLOCK: 1
       TF_VAR_SEQ_MAX_TX_PER_BLOCK: ${{ needs.set-network.outputs.max_txs_per_block }}
+      TF_VAR_MAINNET_FORK_CPU_UNITS: ${{ needs.set-network.outputs.mainnet_fork_cpu_units }}
+      TF_VAR_MAINNET_FORK_MEMORY_UNITS: ${{ needs.set-network.outputs.mainnet_fork_memory_units }}
+      TF_VAR_BOT_SKIP_PUBLIC_SIMULATION: ${{ needs.set-network.outputs.bot_skip_simulation }}
+      TF_VAR_BOT_L2_GAS_LIMIT: ${{ needs.set-network.outputs.bot_l2_gas_limit }}
+      TF_VAR_BOT_DA_GAS_LIMIT: ${{ needs.set-network.outputs.bot_da_gas_limit }}
+      TF_VAR_BOT_COUNT: ${{ needs.set-network.outputs.bot_count }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -679,6 +711,10 @@ jobs:
       TF_VAR_BOT_FOLLOW_CHAIN: ${{ needs.set-network.outputs.bot_follow_chain }}
       TF_VAR_PROVING_ENABLED: true
       TF_VAR_BOT_NO_START: false
+      TF_VAR_BOT_SKIP_PUBLIC_SIMULATION: ${{ needs.set-network.outputs.bot_skip_simulation }}
+      TF_VAR_BOT_L2_GAS_LIMIT: ${{ needs.set-network.outputs.bot_l2_gas_limit }}
+      TF_VAR_BOT_DA_GAS_LIMIT: ${{ needs.set-network.outputs.bot_da_gas_limit }}
+      TF_VAR_BOT_COUNT: ${{ needs.set-network.outputs.bot_count }}
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/spartan-test.yml

@@ -0,0 +1,175 @@
+name: Run spartan network tests
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "00 08 * * 1-5"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+  GIT_COMMIT: ${{ github.sha }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+jobs:
+  setup:
+    uses: ./.github/workflows/setup-runner.yml
+    with:
+      username: ${{ github.event.pull_request.user.login || github.actor }}
+      runner_type: builder-x86
+    secrets: inherit
+
+  changes:
+    runs-on: ubuntu-20.04
+    # Required permissions.
+    permissions:
+      pull-requests: read
+    # Set job outputs to values from filter step
+    outputs:
+      avm-transpiler: ${{ steps.filter.outputs.avm-transpiler }}
+      build-images: ${{ steps.filter.outputs.build-images }}
+      barretenberg: ${{ steps.filter.outputs.barretenberg }}
+      barretenberg-cpp: ${{ steps.filter.outputs.barretenberg-cpp }}
+      noir: ${{ steps.filter.outputs.noir }}
+      noir-projects: ${{ steps.filter.outputs.noir-projects }}
+      l1-contracts: ${{ steps.filter.outputs.l1-contracts }}
+      non-docs: ${{ steps.filter.outputs.non-docs }}
+      non-misc-ci: ${{ steps.filter.outputs.non-misc-ci }}
+      non-barretenberg-cpp: ${{ steps.filter.outputs.non-barretenberg-cpp }}
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
+        id: filter
+        with:
+          filters: |
+            barretenberg:
+              - 'barretenberg/**'
+            barretenberg-cpp:
+              - 'barretenberg/cpp/**'
+            build-images:
+              - 'build-images/**'
+            noir:
+              - 'noir/**'
+            avm-transpiler:
+              - 'avm-transpiler/**'
+            l1-contracts:
+              - 'l1-contracts/**'
+            noir-projects:
+              - 'noir-projects/**'
+            non-barretenberg-cpp:
+              - '!(barretenberg/cpp/**)'
+              # don't consider AVM stuff 'core bb'
+              - barretenberg/cpp/pil/**
+              - barretenberg/cpp/src/barretenberg/vm/**
+              - barretenberg/cpp/src/barretenberg/**/generated/*
+              - barretenberg/cpp/src/barretenberg/client_ivc.{hpp,cpp}
+            non-docs:
+              - '!(docs/**)'
+            non-misc-ci:
+              - '!(.github/**)'
+              # Always rebuild when this file changes
+              - .github/workflows/ci.yml
+
+  build-images:
+    needs: [setup, changes]
+    # Note we don't but the 'if' here as that would also apply to dependent jobs, instead we just copy it into each step
+    runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
+    steps:
+      - uses: actions/checkout@v4
+        if: ${{ needs.changes.outputs.build-images == 'true' }}
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+        if: ${{ needs.changes.outputs.build-images == 'true' }}
+        with:
+          concurrency_key: build-images-x86
+      - name: "Push Build Images If Changed"
+        if: ${{ needs.changes.outputs.build-images == 'true' }}
+        timeout-minutes: 40
+        run: |
+          earthly-ci --push ./build-images/+build
+
+  build:
+    needs: [build-images, changes]
+    if: ${{ needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true' }}
+    runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
+    outputs:
+      e2e_list: ${{ steps.e2e_list.outputs.list }}
+      bench_list: ${{ steps.bench_list.outputs.list }}
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+        with:
+          concurrency_key: build-x86
+      # prepare images locally, tagged by commit hash
+      - name: "Build E2E Image"
+        timeout-minutes: 40
+        run: |
+          earthly-ci ./yarn-project+export-e2e-test-images
+
+  network-e2e:
+    needs: [build, changes]
+    if: ${{ needs.changes.outputs.non-barretenberg-cpp == 'true' }}
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        values_file: ["default.yaml", "3-validators.yaml"]
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+      - name: Setup and Test
+        uses: ./.github/ensure-tester-with-images
+        timeout-minutes: 45
+        with:
+          runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '16core-tester-x86' }}
+          builder_type: builder-x86
+          # these are copied to the tester and expected by the earthly command below
+          # if they fail to copy, it will try to build them on the tester and fail
+          builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
+          # command to produce the images in case they don't exist
+          builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
+          tester_ttl: 40
+          run: |
+            set -eux
+            cd ./yarn-project/end-to-end/
+            ./scripts/setup_local_k8s.sh
+            export FORCE_COLOR=1
+            export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}"
+            ../../scripts/earthly-ci --exec-stats -P --no-output ./+network-transfer --values-file=${{ matrix.values_file }}
+
+  success-check:
+    runs-on: ubuntu-20.04
+    needs:
+      - build-and-test
+    if: always()
+    steps:
+      - name: Report overall success
+        env:
+          # We treat any skipped or failing jobs as a failure for the workflow as a whole.
+          FAIL: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
+        run: |
+          if [[ $FAIL == true ]]; then
+              echo "Test failed."
+              exit 1
+          fi
+
+  notify:
+    needs:
+      - success-check
+    runs-on: ubuntu-20.04
+    if: ${{ github.ref == 'refs/heads/master' && failure() }}
+    steps:
+      - name: Send notification to aztec3-ci channel if workflow failed on master
+        uses: slackapi/slack-github-action@v1.25.0
+        with:
+          payload: |
+            {
+              "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WORKFLOW_TRIGGER_URL }}