feat(spartan): blob sink in spartan

.github/workflows/ci.yml

@@ -415,6 +415,11 @@ jobs:
             values: ci
             runner_type: 16core-tester-x86
             timeout: 40
+          - test: 4epochs.test.ts
+            values: ci
+            overrides: "blobSink.enabled=true"
+            runner_type: 16core-tester-x86
+            timeout: 40
           # - test: gating-passive.test.ts
           #   values: ci
           #   runner_type: 16core-tester-x86
@@ -437,7 +442,7 @@ jobs:
             if ci3/test_should_run "$artifact"; then
               docker pull aztecprotocol/aztec:${{ env.GIT_COMMIT }}
               docker pull aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
-              ./ci.sh test-kind-network ${{ matrix.config.test }} ${{ matrix.config.values }}
+              ./ci.sh test-kind-network ${{ matrix.config.test }} ${{ matrix.config.values }} ${{ matrix.config.overrides }}
               ci3/cache_upload_flag "$artifact"
             fi
       - name: Copy Network Logs
@@ -448,7 +453,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: kind-network-test-${{ matrix.config.values }}-${{ matrix.config.test }}.log
+          name: kind-network-test-${{ matrix.config.values }}-${{ matrix.config.test }}-${{ matrix.config.overrides }}.log
           path: network-test.log
 
   bb-bench:

ci.sh

@@ -192,9 +192,11 @@ case "$cmd" in
   "test-kind-network")
     test=${1:-transfer.test.ts}
     values=${2:-3-validators}
+    overrides=${3:-}
+
     ./bootstrap.sh image-e2e
     cd yarn-project/end-to-end
-    NAMESPACE="kind-network-test" FRESH_INSTALL=true VALUES_FILE=$values.yaml ./scripts/network_test.sh ./src/spartan/$test
+    NAMESPACE="kind-network-test" FRESH_INSTALL=true VALUES_FILE=$values.yaml OVERRIDES="$overrides" ./scripts/network_test.sh ./src/spartan/$test
     exit 0
     ;;
   "test-network")

spartan/aztec-network/templates/_helpers.tpl

@@ -72,6 +72,10 @@ http://{{ include "aztec-network.fullname" . }}-boot-node-0.{{ include "aztec-ne
 http://{{ include "aztec-network.fullname" . }}-validator.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.validator.service.nodePort }}
 {{- end -}}
 
+{{- define "aztec-network.blobSinkUrl" -}}
+http://{{ include "aztec-network.fullname" . }}-blob-sink.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.blobSink.service.nodePort }}
+{{- end -}}
+
 {{- define "aztec-network.metricsHost" -}}
 http://{{ include "aztec-network.fullname" . }}-metrics.{{ .Release.Namespace }}
 {{- end -}}

spartan/aztec-network/templates/blob-sink.yaml

@@ -0,0 +1,134 @@
+{{- if .Values.blobSink.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "aztec-network.fullname" . }}-blob-sink
+  labels:
+    {{- include "aztec-network.labels" . | nindent 4 }}
+spec:
+  serviceName: {{ include "aztec-network.fullname" . }}-blob-sink
+  replicas: {{ .Values.blobSink.replicas }}
+  selector:
+    matchLabels:
+      {{- include "aztec-network.selectorLabels" . | nindent 6 }}
+      app: blob-sink
+  {{- if not .Values.storage.localSsd }}
+  volumeClaimTemplates:
+    - metadata:
+        name: blob-sink-data
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: {{ .Values.blobSink.dataStoreConfig.storageSize }}
+  {{- end }}
+  template:
+    metadata:
+      labels:
+        {{- include "aztec-network.selectorLabels" . | nindent 8 }}
+        app: blob-sink
+    spec:
+      {{- if .Values.storage.localSsd }}
+      {{- include "aztec-network.gcpLocalSsd" . | nindent 6 }}
+      {{- end }}
+      dnsPolicy: ClusterFirstWithHostNet
+      {{- if .Values.network.public }}
+      hostNetwork: true
+      {{- include "aztec-network.publicAntiAffinity" . | nindent 6 }}
+      {{- end }}
+      serviceAccountName: {{ include "aztec-network.fullname" . }}-node
+      initContainers:
+      containers:
+        - name: blob-sink
+          {{- include "aztec-network.image" . | nindent 10 }}
+          command:
+            - /bin/bash
+            - -c
+            - |
+              env && \
+              node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --blob-sink
+          startupProbe:
+            httpGet:
+              path: /status
+              port: {{ .Values.blobSink.service.nodePort }}
+            periodSeconds: {{ .Values.blobSink.startupProbe.periodSeconds }}
+            failureThreshold: {{ .Values.blobSink.startupProbe.failureThreshold }}
+          livenessProbe:
+            httpGet:
+              path: /status
+              port: {{ .Values.blobSink.service.nodePort }}
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            timeoutSeconds: 30
+            failureThreshold: 3
+          volumeMounts:
+            - name: blob-sink-data
+              mountPath: {{ .Values.blobSink.dataStoreConfig.dataDir }}
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: K8S_POD_UID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.uid
+            - name: K8S_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: OTEL_SERVICE_NAME
+              value: blob-sink
+            - name: K8S_NAMESPACE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: BLOB_SINK_PORT
+              value: "{{ .Values.blobSink.service.nodePort }}"
+            - name: LOG_LEVEL
+              value: "{{ .Values.blobSink.logLevel }}"
+            - name: LOG_JSON
+              value: "1"
+            - name: DATA_DIRECTORY
+              value: "{{ .Values.blobSink.dataStoreConfig.dataDir }}"
+            - name: DATA_STORE_MAP_SIZE_KB
+              value: "{{ .Values.blobSink.dataStoreConfig.dataStoreMapSize }}"
+            - name: USE_GCLOUD_OBSERVABILITY
+              value: "{{ .Values.telemetry.useGcloudObservability }}"
+          ports:
+            - containerPort: {{ .Values.blobSink.service.nodePort }}
+          resources:
+            {{- toYaml .Values.blobSink.resources | nindent 12 }}
+      volumes:
+        - name: blob-sink-data
+          emptyDir: {}
+      {{- if .Values.storage.localSsd }}
+        - name: blob-sink-data
+          emptyDir: {}
+      {{ else }}
+        - name: blob-sink-data
+          persistentVolumeClaim:
+            claimName: blob-sink-data
+      {{- end }}
+---
+# Headless service for StatefulSet DNS entries
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "aztec-network.fullname" . }}-blob-sink
+  labels:
+    {{- include "aztec-network.labels" . | nindent 4 }}
+spec:
+  {{- if .Values.network.public }}
+  type: LoadBalancer
+  {{- else }}
+  type: ClusterIP
+  clusterIP: None
+  {{- end }}
+  selector:
+    {{- include "aztec-network.selectorLabels" . | nindent 4 }}
+    app: blob-sink
+  ports:
+    - port: {{ .Values.blobSink.service.nodePort }}
+      name: node
+{{- end }}

spartan/aztec-network/templates/boot-node.yaml

@@ -222,6 +222,10 @@ spec:
               value: "{{ .Values.telemetry.useGcloudObservability }}"
             - name: OTEL_EXCLUDE_METRICS
               value: "{{ .Values.telemetry.excludeMetrics }}"
+            {{- if .Values.blobSink.enabled }}
+            - name: BLOB_SINK_URL
+              value: {{ include "aztec-network.blobSinkUrl" . }}
+            {{- end }}
           ports:
             - containerPort: {{ .Values.bootNode.service.nodePort }}
             - containerPort: {{ .Values.bootNode.service.p2pTcpPort }}

spartan/aztec-network/templates/prover-node.yaml

@@ -192,6 +192,10 @@ spec:
               value: "{{ .Values.telemetry.useGcloudObservability }}"
             - name: OTEL_EXCLUDE_METRICS
               value: "{{ .Values.telemetry.excludeMetrics }}"
+            {{- if .Values.blobSink.enabled }}
+            - name: BLOB_SINK_URL
+              value: {{ include "aztec-network.blobSinkUrl" . }}
+            {{- end }}
           ports:
             - containerPort: {{ .Values.proverNode.service.nodePort }}
             - containerPort: {{ .Values.proverNode.service.p2pTcpPort }}

spartan/aztec-network/templates/validator.yaml

@@ -215,6 +215,10 @@ spec:
               value: "{{ .Values.telemetry.useGcloudObservability }}"
             - name: OTEL_EXCLUDE_METRICS
               value: "{{ .Values.telemetry.excludeMetrics }}"
+            {{- if .Values.blobSink.enabled }}
+            - name: BLOB_SINK_URL
+              value: {{ include "aztec-network.blobSinkUrl" . }}
+            {{- end }}
           ports:
             - containerPort: {{ .Values.validator.service.nodePort }}
             - containerPort: {{ .Values.validator.service.p2pTcpPort }}

spartan/aztec-network/values.yaml

@@ -307,6 +307,19 @@ jobs:
   deployL1Verifier:
     enable: false
 
+blobSink:
+  enabled: false
+  replicas: 1
+  service:
+    nodePort: 5052
+  startupProbe:
+    periodSeconds: 10
+    failureThreshold: 120
+  dataStoreConfig:
+    dataDir: "/data"
+    storageSize: "8Gi"
+    dataStoreMapSize: "134217728" # 128 GB
+
 faucet:
   replicas: 1
   service:

yarn-project/archiver/src/archiver/archiver.test.ts

@@ -198,6 +198,8 @@ describe('Archiver', () => {
 
     blocks.forEach(b => blobSinkClient.getBlobSidecar.mockResolvedValueOnce([makeBlobFromBlock(b)]));
 
+    blocks.forEach(b => blobSinkClient.getBlobSidecar.mockResolvedValueOnce([makeBlobFromBlock(b)]));
+
     makeMessageSentEvent(98n, 1n, 0n);
     makeMessageSentEvent(99n, 1n, 1n);
     makeL2BlockProposedEvent(101n, 1n, blocks[0].archive.root.toString());

yarn-project/aztec/src/cli/aztec_start_action.ts

@@ -71,6 +71,9 @@ export async function aztecStart(options: any, userLog: LogFn, debugLogger: Logg
     } else if (options.proverNode) {
       const { startProverNode } = await import('./cmds/start_prover_node.js');
       await startProverNode(options, signalHandlers, services, userLog);
+    } else if (options.blobSink) {
+      const { startBlobSink } = await import('./cmds/start_blob_sink.js');
+      await startBlobSink(options, signalHandlers, userLog);
     } else if (options.pxe) {
       const { startPXE } = await import('./cmds/start_pxe.js');
       await startPXE(options, signalHandlers, services, userLog);

yarn-project/aztec/src/cli/aztec_start_options.ts

@@ -1,7 +1,7 @@
 import { type ArchiverConfig, archiverConfigMappings } from '@aztec/archiver/config';
 import { faucetConfigMapping } from '@aztec/aztec-faucet/config';
 import { sequencerClientConfigMappings } from '@aztec/aztec-node/config';
-import { blobSinkConfigMapping } from '@aztec/blob-sink/client';
+import { blobSinkConfigMappings } from '@aztec/blob-sink/server';
 import { botConfigMappings } from '@aztec/bot/config';
 import {
   type ConfigMapping,
@@ -259,7 +259,7 @@ export const aztecStartOptions: { [key: string]: AztecStartOption[] } = {
       defaultValue: undefined,
       envVar: undefined,
     },
-    ...getOptions('blobSink', blobSinkConfigMapping),
+    ...getOptions('blobSink', blobSinkConfigMappings),
   ],
   'PROVER NODE': [
     {

yarn-project/aztec/src/cli/cmds/start_blob_sink.ts

@@ -0,0 +1,31 @@
+import {
+  type BlobSinkConfig,
+  blobSinkConfigMappings,
+  createBlobSinkServer,
+  getBlobSinkConfigFromEnv,
+} from '@aztec/blob-sink/server';
+import { type LogFn } from '@aztec/foundation/log';
+import { getConfigEnvVars as getTelemetryClientConfig, initTelemetryClient } from '@aztec/telemetry-client';
+
+import { extractRelevantOptions } from '../util.js';
+
+export async function startBlobSink(options: any, signalHandlers: (() => Promise<void>)[], userLog: LogFn) {
+  if (options.prover || options.node || options.sequencer || options.pxe || options.p2pBootstrap || options.txe) {
+    userLog(
+      `Starting a blob sink with --node, --sequencer, --pxe, --p2p-bootstrap, --prover or --txe is not supported.`,
+    );
+    process.exit(1);
+  }
+
+  const blobSinkConfig = {
+    ...getBlobSinkConfigFromEnv(), // get default config from env
+    ...extractRelevantOptions<BlobSinkConfig>(options, blobSinkConfigMappings, 'blobSink'), // override with command line options
+  };
+
+  const telemetry = initTelemetryClient(getTelemetryClientConfig());
+
+  const blobSink = await createBlobSinkServer(blobSinkConfig, telemetry);
+  signalHandlers.push(blobSink.stop.bind(blobSink));
+
+  await blobSink.start();
+}

yarn-project/blob-sink/src/client/http.ts

@@ -57,8 +57,7 @@ export class HttpBlobSinkClient implements BlobSinkClientInterface {
    *
    * 1. First atttempts to get blobs from a configured blob sink
    * 2. If no blob sink is configured, attempts to get blobs from a configured consensus host
-
-   * // TODO(md): blow up?
+   *
    * 3. If none configured, fails
    *
    * @param blockHash - The block hash

yarn-project/blob-sink/src/server/server.test.ts

@@ -19,6 +19,13 @@ describe('BlobSinkService', () => {
     await service.stop();
   });
 
+  describe('status', () => {
+    it('should return 200', async () => {
+      const response = await request(service.getApp()).get('/status');
+      expect(response.status).toBe(200);
+    });
+  });
+
   describe('should store and retrieve a blob sidecar', () => {
     const blob = makeEncodedBlob(3);
     const blob2 = makeEncodedBlob(3);

yarn-project/blob-sink/src/server/server.ts

@@ -48,7 +48,7 @@ export class BlobSinkServer {
   }
 
   private setupRoutes() {
-    // TODO(md): needed?
+    this.app.get('/status', this.status.bind(this));
     this.app.get('/eth/v1/beacon/headers/:block_id', this.handleGetBlockHeader.bind(this));
     this.app.get('/eth/v1/beacon/blob_sidecars/:block_id', this.handleGetBlobSidecar.bind(this));
     this.app.post('/blob_sidecar', this.handlePostBlobSidecar.bind(this));
@@ -72,6 +72,13 @@ export class BlobSinkServer {
     return;
   }
 
+  private status(_req: Request, res: Response) {
+    res.status(200).json({
+      message: 'Ok',
+    });
+    return;
+  }
+
   private async handleGetBlobSidecar(req: Request, res: Response) {
     // eslint-disable-next-line camelcase
     const { block_id } = req.params;

yarn-project/end-to-end/scripts/network_test.sh

@@ -145,12 +145,34 @@ source "$REPO/yarn-project/end-to-end/scripts/bash/read_values_file.sh"
 ## so that it can generate the genesis.json and config.yaml file with the correct values.
 $REPO/yarn-project/end-to-end/scripts/bash/generate_devnet_config.sh
 
+function read_values_file() {
+  local key="$1"
+
+  value=$(yq -r ".$key" "$VALUES_PATH")
+  if [ -z "$value" ] || [ "$value" = "null" ]; then
+    value=$(yq -r ".$key" "$DEFAULT_VALUES_PATH")
+  fi
+  echo "$value"
+}
+
+function generate_overrides() {
+  local overrides="$1"
+  if [ -n "$overrides" ]; then
+    # Split the comma-separated string into an array and generate --set arguments
+    IFS=',' read -ra OVERRIDE_ARRAY <<< "$overrides"
+    for override in "${OVERRIDE_ARRAY[@]}"; do
+      echo "--set $override"
+    done
+  fi
+}
+
 # Install the Helm chart
 helm upgrade --install spartan "$REPO/spartan/aztec-network/" \
   --namespace "$NAMESPACE" \
   --create-namespace \
   --values "$VALUES_PATH" \
   --set images.aztec.image="aztecprotocol/aztec:$AZTEC_DOCKER_TAG" \
+  $(generate_overrides "$OVERRIDES") \
   --wait \
   --wait-for-jobs=true \
   --timeout="$INSTALL_TIMEOUT"