chore(ci): use labels and if branch=master to control jobs (#8508)

Implements a default CI list of e2e tests:
```
  "e2e-2-pxes"
  "e2e-authwit"
  "e2e-avm-simulator"
  "e2e-block-building"
  "e2e-cross-chain-messaging"
  "e2e-deploy-contract"
  "e2e-fees"
  "e2e-fees-gas-estimation"
  "e2e-fees-private-payments"
  "e2e-max-block-number"
  "e2e-nested-contract"
  "e2e-ordering"
  "e2e-prover-full"
  "e2e-static-calls"
```

More tests can be added on with labels.
All tests run on master.

Author: ludamad

.github/workflows/ci.yml

@@ -8,6 +8,7 @@ on:
       - reopened
       - synchronize
       - ready_for_review
+      - labeled
     branches-ignore: [devnet]
   workflow_dispatch:
     inputs: {}
@@ -113,7 +114,7 @@ jobs:
 
   build:
     needs: [build-images, changes]
-    if: needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true'
+    if: (needs.changes.outputs.non-docs == 'true' && needs.changes.outputs.non-misc-ci == 'true' && needs.changes.outputs.non-barretenberg-cpp == 'true') || github.ref_name == 'master'
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
     outputs:
       e2e_list: ${{ steps.e2e_list.outputs.list }}
@@ -133,15 +134,21 @@ jobs:
       # (Note ARM uses just 2 tests as a smoketest)
       - name: Create list of non-bench end-to-end jobs
         id: e2e_list
-        run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | grep -v '+bench' | grep -v "+network" | grep -v 'devnet' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT
+        run: |
+          LABELS=$(jq -r '.pull_request.labels | map(.name) | join(",")' "$GITHUB_EVENT_PATH")
+          echo Labels: $LABELS
+          echo "list=$(./scripts/ci/get_e2e_jobs.sh ${{github.ref_name}} "$LABELS")" >> $GITHUB_OUTPUT
       - name: Create list of bench end-to-end jobs
         id: bench_list
-        run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep '+bench' |  sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT
+        run: |
+          LABELS=$(jq -r '.pull_request.labels | map(.name) | join(",")' "$GITHUB_EVENT_PATH")
+          echo Labels: $LABELS
+          echo "list=$(./scripts/ci/get_bench_jobs.sh ${{github.ref_name}} "$LABELS")" >> $GITHUB_OUTPUT
 
   # all the non-bench end-to-end integration tests for aztec
   e2e:
     needs: [build, changes]
-    if: needs.changes.outputs.non-barretenberg-cpp == 'true'
+    if: needs.changes.outputs.non-barretenberg-cpp == 'true' || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'e2e')
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
@@ -171,7 +178,7 @@ jobs:
   # all the benchmarking end-to-end integration tests for aztec (not required to merge)
   bench-e2e:
     needs: [build, changes]
-    if: needs.changes.outputs.non-barretenberg-cpp == 'true'
+    if: needs.changes.outputs.non-barretenberg-cpp == 'true' || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'bench')
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false

scripts/ci/get_bench_jobs.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+set -eu
+
+# Enter repo root.
+cd "$(dirname "$0")"/../..
+
+BRANCH=$1
+LABELS=$2
+
+# Define the allow_list
+allow_list=()
+
+# Add labels from input to the allow_list
+IFS=',' read -r -a input_labels <<< "$LABELS"
+allow_list+=("${input_labels[@]}")
+
+# Generate full list of targets on one line
+full_list=$(earthly ls ./yarn-project/end-to-end | grep '+bench' | sed 's/+//' | xargs echo)
+
+# If branch is master or allow_list contains 'bench-all', return full list
+if [[ "$BRANCH" == "master" ]] || [[ " ${allow_list[@]} " =~ "bench-all" ]]; then
+  # print as JSON list
+  echo "$full_list" | jq -Rc 'split(" ")'
+  exit 0
+fi
+
+# Filter the full_list to include only items in the allow_list
+filtered_list=()
+for item in $full_list; do
+  for allowed in "${allow_list[@]}"; do
+    if [[ "$item" == "$allowed" ]]; then
+      filtered_list+=("$item")
+    fi
+  done
+done
+
+# Print the filtered list in JSON format
+echo ${filtered_list[@]} | jq -Rc 'split(" ")'

scripts/ci/get_e2e_jobs.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+set -eu
+
+# Enter repo root.
+cd "$(dirname "$0")"/../..
+
+BRANCH=$1
+LABELS=$2
+
+# Define the allow_list
+allow_list=(
+  "e2e-2-pxes"
+  "e2e-authwit"
+  "e2e-avm-simulator"
+  "e2e-block-building"
+  "e2e-cross-chain-messaging"
+  "e2e-deploy-contract"
+  "e2e-fees"
+  "e2e-fees-gas-estimation"
+  "e2e-fees-private-payments"
+  "e2e-max-block-number"
+  "e2e-nested-contract"
+  "e2e-ordering"
+  "e2e-prover-full"
+  "e2e-static-calls"
+)
+
+# Add labels from input to the allow_list
+IFS=',' read -r -a input_labels <<< "$LABELS"
+allow_list+=("${input_labels[@]}")
+
+# Generate full list of targets, excluding specific entries, on one line
+full_list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | grep -v '+bench' | grep -v "+network" | grep -v 'devnet' | sed 's/+//' | xargs echo)
+
+# If branch is master or allow_list contains 'e2e-all', return full list
+if [[ "$BRANCH" == "master" ]] || [[ " ${allow_list[@]} " =~ "e2e-all" ]]; then
+  # print as JSON list
+  echo "$full_list" | jq -Rc 'split(" ")'
+  exit 0
+fi
+
+# Filter the full_list to include only items in the allow_list
+filtered_list=()
+for item in $full_list; do
+  for allowed in "${allow_list[@]}"; do
+    if [[ "$item" == "$allowed" ]]; then
+      filtered_list+=("$item")
+    fi
+  done
+done
+
+# Print the filtered list in JSON format
+echo ${filtered_list[@]} | jq -Rc 'split(" ")'