From 2ea17670b5e7f7cc047a49313bd99032e39323de Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Fri, 7 Mar 2025 16:24:19 +0000 Subject: [PATCH] feat: provision alerts (#12561) This PR updates the in-repo dashboards to the latest version and adds alerting rules. A new secret is added to Gcloud to hold the webhook URL for a Slack channel where we want alerts to fire to. One thing that needs to kept in mind if editing the rules is that Grafana templates need to be escaped otherwise Helm will try to execute them and fail (use `` {{ ` escaped content {{ $some_grafan_var }} ` }}``) --- .github/workflows/metrics-deploy.yml | 18 + spartan/metrics/copy-dashboard.sh | 26 +- .../metrics/grafana/alerts/contactpoints.yaml | 11 + spartan/metrics/grafana/alerts/policies.yaml | 11 + spartan/metrics/grafana/alerts/rules.yaml | 742 +++++ .../dashboards/aztec_network.json} | 1989 +++++++------ .../dashboards/aztec_provers.json} | 117 +- .../dashboards/aztec_validators.json} | 69 +- .../dashboards/metrics_prometheus.json} | 28 +- .../aztec-dashboard-all-in-one.json | 2537 ----------------- spartan/metrics/values.tmp.yaml | 30 +- spartan/terraform/deploy-metrics/main.tf | 5 + spartan/terraform/deploy-metrics/variables.tf | 6 + 13 files changed, 2075 insertions(+), 3514 deletions(-) create mode 100644 spartan/metrics/grafana/alerts/contactpoints.yaml create mode 100644 spartan/metrics/grafana/alerts/policies.yaml create mode 100644 spartan/metrics/grafana/alerts/rules.yaml rename spartan/metrics/{grafana_dashboards/network.json => grafana/dashboards/aztec_network.json} (69%) rename spartan/metrics/{grafana_dashboards/provers.json => grafana/dashboards/aztec_provers.json} (95%) rename spartan/metrics/{grafana_dashboards/validators.json => grafana/dashboards/aztec_validators.json} (98%) rename spartan/metrics/{grafana_dashboards/metrics_stack.json => grafana/dashboards/metrics_prometheus.json} (98%) delete mode 100644 spartan/metrics/grafana_dashboards/aztec-dashboard-all-in-one.json diff --git a/.github/workflows/metrics-deploy.yml b/.github/workflows/metrics-deploy.yml index 158e7fa0044..81d5876f0fc 100644 --- a/.github/workflows/metrics-deploy.yml +++ b/.github/workflows/metrics-deploy.yml @@ -37,6 +37,11 @@ on: required: true type: string default: "grafana-dashboard-password" + slack_webhook_url_secret_name: + description: The name of the secret which holds the Slack webhook URL + required: true + type: string + default: "slack-webhook-url" secrets: GCP_SA_KEY: required: true @@ -70,6 +75,10 @@ on: description: The name of the secret which holds the Grafana dashboard password required: true default: "grafana-dashboard-password" + slack_webhook_url_secret_name: + description: The name of the secret which holds the Slack webhook URL + required: true + default: "slack-webhook-url" jobs: metrics_deployment: @@ -89,6 +98,7 @@ jobs: TF_STATE_BUCKET: aztec-terraform GKE_CLUSTER_CONTEXT: "gke_testnet-440309_us-west1-a_${{ inputs.cluster }}" GRAFANA_DASHBOARD_PASSWORD_SECRET_NAME: ${{ inputs.grafana_dashboard_password_secret_name }} + SLACK_WEBHOOK_URL_SECRET_NAME: ${{ inputs.slack_webhook_url_name }} steps: - name: Checkout code @@ -118,6 +128,12 @@ jobs: echo "::add-mask::$(gcloud secrets versions access latest --secret=${{ env.GRAFANA_DASHBOARD_PASSWORD_SECRET_NAME }})" echo "grafana_dashboard_password=$(gcloud secrets versions access latest --secret=${{ env.GRAFANA_DASHBOARD_PASSWORD_SECRET_NAME }})" >> "$GITHUB_OUTPUT" + - name: Grab the Slack webhook URL + id: get-slack-webhook-url + run: | + echo "::add-mask::$(gcloud secrets versions access latest --secret=${{ env.SLACK_WEBHOOK_URL_SECRET_NAME }})" + echo "slack_webhook_url=$(gcloud secrets versions access latest --secret=${{ env.SLACK_WEBHOOK_URL_SECRET_NAME }})" >> "$GITHUB_OUTPUT" + - name: Ensure Terraform state bucket exists run: | if ! gsutil ls gs://${{ env.TF_STATE_BUCKET }} >/dev/null 2>&1; then @@ -155,6 +171,7 @@ jobs: -var="VALUES_FILE=${{ env.VALUES_FILE }}" \ -var="GKE_CLUSTER_CONTEXT=${{ env.GKE_CLUSTER_CONTEXT }}" \ -var="GRAFANA_DASHBOARD_PASSWORD=${{ steps.get-grafana-dashboard-password.outputs.grafana_dashboard_password }}" \ + -var="SLACK_WEBHOOK_URL=${{ steps.get-slack-webhook-url.outputs.slack_webhook_url }}" \ -lock=${{ inputs.respect_tf_lock }} - name: Terraform Plan @@ -165,6 +182,7 @@ jobs: -var="VALUES_FILE=${{ env.VALUES_FILE }}" \ -var="GKE_CLUSTER_CONTEXT=${{ env.GKE_CLUSTER_CONTEXT }}" \ -var="GRAFANA_DASHBOARD_PASSWORD=${{ steps.get-grafana-dashboard-password.outputs.grafana_dashboard_password }}" \ + -var="SLACK_WEBHOOK_URL=${{ steps.get-slack-webhook-url.outputs.slack_webhook_url }}" \ -out=tfplan \ -lock=${{ inputs.respect_tf_lock }} diff --git a/spartan/metrics/copy-dashboard.sh b/spartan/metrics/copy-dashboard.sh index 50a1b89d175..30d8f40d49d 100755 --- a/spartan/metrics/copy-dashboard.sh +++ b/spartan/metrics/copy-dashboard.sh @@ -5,8 +5,26 @@ cd "$(dirname "${BASH_SOURCE[0]}")" cp values.tmp.yaml values.yaml -for dashboard in ./grafana_dashboards/*.json; do - dashboard_name=$(basename "$dashboard" .json) - export DASHBOARD_JSON=$(jq -c '.' "$dashboard") - yq -i ".grafana.dashboards.default.\"$dashboard_name\".json = strenv(DASHBOARD_JSON)" values.yaml +for dashboard_file in ./grafana/dashboards/*.json; do + full_filename=$(basename "$dashboard_file" .json) + + # Extract folder name and dashboard name using underscore as separator + # Format: foldername_dashboardname.json + if [[ "$full_filename" == *"_"* ]]; then + folder_name=${full_filename%%_*} + dashboard_name=${full_filename#*_} + else + # If no underscore, use "default" as the folder + folder_name="default" + dashboard_name=$full_filename + fi + + export dashboard_content=$(jq -c '.' "$dashboard_file") + yq -i ".grafana.dashboards.${folder_name}.${dashboard_name}.json = strenv(dashboard_content)" values.yaml +done + +for file in ./grafana/alerts/*.yaml; do + file_name=$(basename "$file" .yaml) + export file_content=$(cat "$file" ) + yq -i ".grafana.alerting.\"${file_name}.yaml\" = env(file_content)" values.yaml done diff --git a/spartan/metrics/grafana/alerts/contactpoints.yaml b/spartan/metrics/grafana/alerts/contactpoints.yaml new file mode 100644 index 00000000000..86b094cf174 --- /dev/null +++ b/spartan/metrics/grafana/alerts/contactpoints.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 +contactPoints: + - orgId: 1 + name: 'Slack #network-alerts channel' + receivers: + - uid: deexubp9hzpc1b + type: slack + settings: + url: $SLACK_WEBHOOK_URL + disableResolveMessage: false + diff --git a/spartan/metrics/grafana/alerts/policies.yaml b/spartan/metrics/grafana/alerts/policies.yaml new file mode 100644 index 00000000000..87217cb46cc --- /dev/null +++ b/spartan/metrics/grafana/alerts/policies.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 +policies: + - orgId: 1 + receiver: 'Slack #network-alerts channel' + object_matchers: + - - k8s_namespace_name + - =~ + - $PRODUCTION_NAMESPACES_REGEX + group_by: + - grafana_folder + - alertname diff --git a/spartan/metrics/grafana/alerts/rules.yaml b/spartan/metrics/grafana/alerts/rules.yaml new file mode 100644 index 00000000000..92084643688 --- /dev/null +++ b/spartan/metrics/grafana/alerts/rules.yaml @@ -0,0 +1,742 @@ +apiVersion: 1 +groups: + - orgId: 1 + name: Every minute + folder: Aztec + interval: 1m + rules: + - uid: cef0c3pw50bnkf + title: L1 - ETH spend + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: spartan-metrics-prometheus + model: + editorMode: code + expr: sum by (k8s_namespace_name) (-delta(aztec_l1_publisher_balance_eth[$__rate_interval])) + instant: true + intervalMs: 60000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.001 + - 0.01 + type: outside_range + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 1m + annotations: {} + labels: {} + isPaused: true + - uid: bef0du59q8sg0b + title: Chain - no new blocks + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: spartan-metrics-prometheus + model: + editorMode: code + expr: max by (k8s_namespace_name) (delta(aztec_archiver_block_height{k8s_namespace_name!="",aztec_status=""}[$__rate_interval])) + instant: true + intervalMs: 60000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 5m + annotations: + summary: Pending chain in {{ ` {{ $labels.k8s_namespace_name }} ` }} hasn't advanced in 5 minutes + labels: {} + isPaused: false + - uid: def0g11bn64n4a + title: Chain - reorg + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: spartan-metrics-prometheus + model: + editorMode: code + expr: max by (k8s_namespace_name) (increase(aztec_archiver_prune_count{k8s_namespace_name!=""}[$__rate_interval])) + instant: true + intervalMs: 300000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 0s + annotations: + summary: Pending chain has re-orged in {{ ` {{ $labels.k8s_namespace_name }} ` }} + labels: {} + isPaused: false + - uid: fef0j7owjyebkf + title: Sequencer - slow attestation gathering + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: spartan-metrics-prometheus + model: + editorMode: code + expr: avg by (k8s_namespace_name) (aztec_sequencer_time_to_collect_attestations) + instant: true + intervalMs: 60000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 10000 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 1m + annotations: + summary: Attestations collection is taking over 10 seconds in {{ ` {{ $labels.k8s_namespace_name }} ` }} + labels: {} + isPaused: false + - uid: fef0jfew2nls0e + title: Process - high CPU + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: spartan-metrics-prometheus + model: + editorMode: code + expr: avg by (k8s_namespace_name,service_name) (process_cpu_utilization{process_cpu_state="user"}) + instant: true + intervalMs: 60000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.5 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 5m + annotations: + summary: High CPU usage in {{ ` {{ $labels.k8s_namespace_name }} by {{ $labels.service_name }} ` }} + labels: {} + isPaused: false + - uid: cef0js2plm8zka + title: Process - high memory + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: spartan-metrics-prometheus + model: + editorMode: code + expr: avg by (k8s_namespace_name,service_name) (process_memory_usage) / 1024^3 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 5 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 5m + annotations: + summary: High memory usage in {{ ` {{ $labels.k8s_namespace_name }} by {{ $labels.service_name }} ` }} + labels: {} + isPaused: false + - uid: cef0k7rx404qof + title: Process - high event loop lag + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: spartan-metrics-prometheus + model: + editorMode: code + expr: avg by (k8s_namespace_name,service_name) (nodejs_eventloop_delay_p90_nanoseconds) / 1e6 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 25 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 1m + annotations: + summary: Something is blocking the main thread in {{ ` {{ $labels.service_name }} ({{ $labels.k8s_namespace_name }} ` }}) + labels: {} + isPaused: false + - uid: eef0km6vckxs0a + title: World State - critical errors + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: spartan-metrics-prometheus + model: + editorMode: code + expr: sum by(k8s_namespace_name,service_name) (increase(aztec_world_state_critical_error_count[$__rate_interval])) or vector(0) + instant: true + intervalMs: 60000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 1m + annotations: + summary: Critical errors in the WorldState of {{ ` {{ $labels.service_name }} ({{ $labels.k8s_namespace_name }} ` }}) + labels: {} + isPaused: false + - orgId: 1 + name: Every minute + folder: Metrics + interval: 1m + rules: + - uid: fef1519rsbw8wa + title: 'Prometheus - slow scraping ' + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: spartan-metrics-prometheus + model: + editorMode: code + expr: scrape_duration_seconds + instant: true + intervalMs: 60000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [] + type: gt + operator: + type: and + query: + params: + - B + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 7.5 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 1m + annotations: + summary: Prometheus scraping for job {{ ` {{ $labels.job }} ` }} takes more than 7.5s + labels: {} + isPaused: false + diff --git a/spartan/metrics/grafana_dashboards/network.json b/spartan/metrics/grafana/dashboards/aztec_network.json similarity index 69% rename from spartan/metrics/grafana_dashboards/network.json rename to spartan/metrics/grafana/dashboards/aztec_network.json index 0590cca0dec..ba7fd0967f4 100644 --- a/spartan/metrics/grafana_dashboards/network.json +++ b/spartan/metrics/grafana/dashboards/aztec_network.json @@ -12,6 +12,18 @@ "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "enable": false, + "expr": "max(increase(aztec_archiver_prune_count{k8s_namespace_name=\"$namespace\"}[5m])) > 1", + "iconColor": "yellow", + "name": "Reorg", + "step": "1m", + "titleFormat": "Reorg" } ] }, @@ -168,7 +180,7 @@ "uid": "spartan-metrics-prometheus" }, "editorMode": "code", - "expr": "count by (exported_job) (target_info{k8s_namespace_name=~\"$namespace\"})", + "expr": "count by (exported_job) (target_info{k8s_namespace_name=~\"$namespace\", service_name=\"\"})", "instant": false, "interval": "", "legendFormat": "{{service_name}}", @@ -191,6 +203,7 @@ }, "decimals": 0, "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ @@ -216,7 +229,7 @@ "graphMode": "none", "justifyMode": "center", "orientation": "auto", - "percentChangeColorMode": "standard", + "percentChangeColorMode": "same_as_value", "reduceOptions": { "calcs": [ "lastNotNull" @@ -225,7 +238,8 @@ "values": false }, "showPercentChange": false, - "textMode": "auto", + "text": {}, + "textMode": "value_and_name", "wideLayout": true }, "pluginVersion": "11.2.1", @@ -237,12 +251,26 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg by (aztec_status) (label_replace(aztec_archiver_block_height{k8s_namespace_name=\"$namespace\"}, \"aztec_status\", \"pending\", \"aztec_status\", \"^$\"))", + "expr": "max by (aztec_status) (label_replace(aztec_archiver_block_height{k8s_namespace_name=\"$namespace\",aztec_status=\"\"}, \"aztec_status\", \"Pending chain\", \"aztec_status\", \"^$\"))", "instant": true, - "legendFormat": "__auto", + "legendFormat": "{{aztec_status}}", "range": false, "refId": "A" }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by (aztec_status) (label_replace(aztec_archiver_block_height{k8s_namespace_name=\"$namespace\",aztec_status=\"proven\"}, \"aztec_status\", \"Proven chain\", \"aztec_status\", \"^proven$\"))", + "hide": false, + "instant": true, + "legendFormat": "{{aztec_status}}", + "range": false, + "refId": "E" + }, { "datasource": { "type": "prometheus", @@ -250,10 +278,10 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg by (aztec_status) (label_replace(aztec_archiver_l1_block_height{k8s_namespace_name=\"$namespace\"}, \"aztec_status\", \"l1\", \"aztec_status\", \"^$\"))", + "expr": "max by (aztec_status) (label_replace(aztec_archiver_l1_block_height{k8s_namespace_name=\"$namespace\"}, \"aztec_status\", \"l1\", \"aztec_status\", \"^$\"))", "hide": false, "instant": true, - "legendFormat": "__auto", + "legendFormat": "L1", "range": false, "refId": "B" }, @@ -263,12 +291,25 @@ "uid": "spartan-metrics-prometheus" }, "editorMode": "code", - "expr": "avg(aztec_archiver_block_height{k8s_namespace_name=\"$namespace\", aztec_status=\"\"}) - avg(aztec_archiver_block_height{k8s_namespace_name=\"$namespace\", aztec_status=\"proven\"})", + "expr": "max(aztec_archiver_block_height{k8s_namespace_name=\"$namespace\", aztec_status=\"\"}) - max(aztec_archiver_block_height{k8s_namespace_name=\"$namespace\", aztec_status=\"proven\"})", "hide": false, "instant": false, "legendFormat": "Pending chain depth", "range": true, "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "editorMode": "code", + "expr": "max(aztec_archiver_prune_count{k8s_namespace_name=\"$namespace\"}) or vector(0)", + "hide": false, + "instant": false, + "legendFormat": "Total re-orgs", + "range": true, + "refId": "D" } ], "title": "Current Block Heights", @@ -336,7 +377,7 @@ }, "gridPos": { "h": 9, - "w": 8, + "w": 17, "x": 7, "y": 7 }, @@ -373,6 +414,211 @@ "title": "AVG Mana Consumption Rate", "type": "timeseries" }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "description": "This panel calculates how much ETH has been consumed in the given time range", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 4, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "super-light-blue", + "value": null + } + ] + }, + "unit": "ETH" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 4, + "x": 0, + "y": 16 + }, + "id": 32, + "interval": "1m", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "inverted", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "-sum(delta(aztec_l1_publisher_balance_eth{k8s_namespace_name=\"$namespace\"}[$__range]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total ETH spent in time range", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "description": "How much eth is used. Accumulated by service type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Ξ" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "prover-node" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 10, + "x": 4, + "y": 16 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(service_name) (delta(aztec_l1_publisher_balance_eth{k8s_namespace_name=\"$namespace\"}[1m]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "ETH spend per minute by service type", + "type": "timeseries" + }, { "datasource": { "default": true, @@ -433,10 +679,10 @@ "overrides": [] }, "gridPos": { - "h": 9, - "w": 9, - "x": 15, - "y": 7 + "h": 11, + "w": 10, + "x": 14, + "y": 16 }, "id": 25, "options": { @@ -445,7 +691,7 @@ "lastNotNull" ], "displayMode": "table", - "placement": "right", + "placement": "bottom", "showLegend": true }, "tooltip": { @@ -472,116 +718,341 @@ "type": "timeseries" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 16 + "y": 27 }, "id": 12, - "panels": [ - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "spartan-metrics-prometheus" + "panels": [], + "title": "Resource use", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "description": "Tracks how much CPU the main process is using (this does not account for bb/acvm)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg by(service_name) (sum without(process_cpu_state) (process_cpu_utilization{service_namespace=\"$namespace\"}))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "CPU use", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "description": "Tracks how much memory the main process is using (this does not account for bb/acvm)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "avg by(service_name) (sum without(process_cpu_state) (process_cpu_utilization{service_namespace=\"$namespace\"}))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg by(service_name) (process_memory_usage{service_namespace=\"$namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Memory use", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - ], - "title": "CPU use", - "type": "timeseries" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "editorMode": "code", + "expr": "sum by (aztec_db_type) (aztec_db_used_size_bytes{k8s_namespace_name=~\"$namespace\"})", + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" }, + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "editorMode": "code", + "expr": "sum (avg by (aztec_merkle_tree_name, aztec_world_state_db_type) (aztec_world_state_db_used_size{service_namespace=\"$namespace\"}))", + "hide": false, + "instant": false, + "legendFormat": "world-state", + "range": true, + "refId": "B" + } + ], + "title": "Database Used Size", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 13, + "panels": [ { "datasource": { "default": true, @@ -591,40 +1062,7 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], "thresholds": { @@ -637,31 +1075,36 @@ "color": "red", "value": 80 } - ] - }, - "unit": "bytes" + ] + } }, "overrides": [] }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 18 + "w": 4, + "x": 0, + "y": 34 }, - "id": 9, + "id": 5, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.2.1", "targets": [ { "datasource": { @@ -669,8 +1112,8 @@ "uid": "spartan-metrics-prometheus" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "avg by(service_name) (process_memory_usage{service_namespace=\"$namespace\"})", + "editorMode": "code", + "expr": "sum(rate(aztec_node_receive_tx_count{service_namespace=\"$namespace\"}[$__rate_interval]))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -680,8 +1123,8 @@ "useBackend": false } ], - "title": "Memory use", - "type": "timeseries" + "title": "TPS", + "type": "stat" }, { "datasource": { @@ -692,40 +1135,7 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], "thresholds": { @@ -735,755 +1145,590 @@ "color": "green" } ] - }, - "unit": "bytes" + } }, "overrides": [] }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 26 + "w": 4, + "x": 4, + "y": 34 }, - "id": 21, + "id": 6, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "8.0.0", + "pluginVersion": "11.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "spartan-metrics-prometheus" }, - "editorMode": "code", - "expr": "sum by (aztec_db_type) (aztec_db_used_size_bytes{k8s_namespace_name=~\"$namespace\"})", - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "editorMode": "code", - "expr": "sum (avg by (aztec_merkle_tree_name, aztec_world_state_db_type) (aztec_world_state_db_used_size{service_namespace=\"$namespace\"}))", - "hide": false, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(aztec_node_receive_tx_count{aztec_ok=\"true\", service_namespace=\"$namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, "instant": false, - "legendFormat": "world-state", + "legendFormat": "__auto", "range": true, - "refId": "B" + "refId": "A", + "useBackend": false } ], - "title": "Database Used Size", - "type": "timeseries" - } - ], - "title": "Resource use", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 13, - "panels": [], - "title": "Mempool status", - "type": "row" - }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 0, - "y": 18 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "title": "Total received txs", + "type": "stat" }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.1", - "targets": [ { "datasource": { + "default": true, "type": "prometheus", "uid": "spartan-metrics-prometheus" }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "sum(rate(aztec_node_receive_tx_count{service_namespace=\"$namespace\"}[$__rate_interval]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "TPS", - "type": "stat" - }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow" + } + ] } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 4, - "y": 18 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" + }, + "overrides": [] }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum(aztec_node_receive_tx_count{aztec_ok=\"true\", service_namespace=\"$namespace\"})", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Total received txs", - "type": "stat" - }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "gridPos": { + "h": 8, + "w": 4, + "x": 8, + "y": 34 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 8, - "y": 18 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(aztec_node_receive_tx_count{aztec_ok=\"false\", service_namespace=\"$namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } ], - "fields": "", - "values": false + "title": "Total rejected txs", + "type": "stat" }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.1", - "targets": [ { "datasource": { + "default": true, "type": "prometheus", "uid": "spartan-metrics-prometheus" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum(aztec_node_receive_tx_count{aztec_ok=\"false\", service_namespace=\"$namespace\"})", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Total rejected txs", - "type": "stat" - }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "noValue": "0", + "unit": "none" + }, + "overrides": [] }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "gridPos": { + "h": 8, + "w": 4, + "x": 12, + "y": 34 + }, + "id": 4, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" } }, - "decimals": 0, - "mappings": [], - "min": 0, - "noValue": "0", - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 12, - "y": 18 - }, - "id": 4, - "options": { - "displayLabels": [ - "name", - "value" - ], - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg by(aztec_status) (aztec_mempool_tx_count{service_namespace=\"$namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } ], - "fields": "", - "values": false + "title": "Tx mempool status", + "type": "piechart" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { + "default": true, "type": "prometheus", "uid": "spartan-metrics-prometheus" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "avg by(aztec_status) (aztec_mempool_tx_count{service_namespace=\"$namespace\"})", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Tx mempool status", - "type": "piechart" - }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 34 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "min": 0, - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 18 - }, - "id": 29, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "editorMode": "code", - "expr": "avg(rate(aztec_archiver_tx_count{k8s_namespace_name=\"$namespace\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "editorMode": "code", + "expr": "avg(rate(aztec_archiver_tx_count{k8s_namespace_name=\"$namespace\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Mined TPS", + "type": "timeseries" } ], - "title": "Mined TPS", - "type": "timeseries" + "title": "Mempool status", + "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 26 + "y": 45 }, "id": 17, - "panels": [], - "title": "World state", - "type": "row" - }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 8, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 14, - "x": 0, - "y": 27 - }, - "id": 19, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ + "panels": [ { "datasource": { + "default": true, "type": "prometheus", "uid": "spartan-metrics-prometheus" }, - "editorMode": "code", - "expr": "sum by (aztec_merkle_tree_name) (avg by (aztec_merkle_tree_name, aztec_world_state_db_type) (aztec_world_state_db_used_size{service_namespace=\"$namespace\"}))", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Approx tree size on disk", - "type": "timeseries" - }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 35 }, - "decimals": 0, - "mappings": [], - "min": 0, - "unit": "Leaves" - }, - "overrides": [] - }, - "gridPos": { - "h": 16, - "w": 10, - "x": 14, - "y": 27 - }, - "id": 16, - "options": { - "displayLabels": [ - "name" - ], - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "editorMode": "code", + "expr": "sum by (aztec_merkle_tree_name) (avg by (aztec_merkle_tree_name, aztec_world_state_db_type) (aztec_world_state_db_used_size{service_namespace=\"$namespace\"}))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } ], - "fields": "", - "values": false + "title": "Approx tree size on disk", + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.2.1", - "targets": [ { "datasource": { + "default": true, "type": "prometheus", "uid": "spartan-metrics-prometheus" }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "max by(aztec_merkle_tree_name) (aztec_world_state_tree_size{service_namespace=\"$namespace\"})", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A", - "useBackend": false - } - ], - "title": "Relative tree sizes", - "type": "piechart" - }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "description": "Tracks tree nodes, indexes, metadata", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 8, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "description": "How big trees are relative to each other. \nNOTE: The L1 to L2 message tree has empty leaves added to it for every block!", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "unit": "Leaves" }, - "insertNulls": false, - "lineInterpolation": "stepBefore", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 10, + "x": 14, + "y": 35 + }, + "id": 16, + "options": { + "displayLabels": [ + "name" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "decimals": 0, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Entries" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 14, - "x": 0, - "y": 35 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "max by(aztec_merkle_tree_name) (aztec_world_state_tree_size{service_namespace=\"$namespace\"})", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Relative tree sizes", + "type": "piechart" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.2.1", - "targets": [ { "datasource": { + "default": true, "type": "prometheus", "uid": "spartan-metrics-prometheus" }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "sum by (aztec_merkle_tree_name) (avg by(aztec_merkle_tree_name, aztec_world_state_db_type) (aztec_world_state_db_num_items{service_namespace=\"$namespace\"}))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false + "description": "Tracks tree nodes, indexes, metadata. It's normal for tree sizes to fluctuate even if there are no reorgs (trees garbage collect old data)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 8, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepBefore", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "Entries" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 43 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by (aztec_merkle_tree_name) (avg by(aztec_merkle_tree_name, aztec_world_state_db_type) (aztec_world_state_db_num_items{service_namespace=\"$namespace\"}))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Tree size", + "type": "timeseries" } ], - "title": "Tree size", - "type": "timeseries" + "title": "World state", + "type": "row" }, { "collapsed": false, @@ -1491,7 +1736,7 @@ "h": 1, "w": 24, "x": 0, - "y": 43 + "y": 46 }, "id": 14, "panels": [], @@ -1548,8 +1793,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1564,7 +1808,7 @@ "h": 8, "w": 12, "x": 0, - "y": 44 + "y": 47 }, "id": 1, "options": { @@ -1606,6 +1850,7 @@ "type": "prometheus", "uid": "spartan-metrics-prometheus" }, + "description": "Components synch at different rates, can restart for upgrades (resetting their graph), see multiple reorgs, etc", "fieldConfig": { "defaults": { "color": { @@ -1649,8 +1894,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -1662,7 +1906,7 @@ "h": 8, "w": 12, "x": 12, - "y": 44 + "y": 47 }, "id": 11, "options": { @@ -1748,8 +1992,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1765,7 +2008,7 @@ "h": 8, "w": 12, "x": 0, - "y": 52 + "y": 55 }, "id": 24, "interval": "1m", @@ -1813,8 +2056,8 @@ { "current": { "selected": false, - "text": "master-rc-1", - "value": "master-rc-1" + "text": "ignition-testnet", + "value": "ignition-testnet" }, "datasource": { "type": "prometheus", @@ -1842,13 +2085,13 @@ ] }, "time": { - "from": "now-1h", + "from": "now-3h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Network overview", "uid": "aztec-network", - "version": 10, + "version": 29, "weekStart": "" } diff --git a/spartan/metrics/grafana_dashboards/provers.json b/spartan/metrics/grafana/dashboards/aztec_provers.json similarity index 95% rename from spartan/metrics/grafana_dashboards/provers.json rename to spartan/metrics/grafana/dashboards/aztec_provers.json index a92f4fa855e..5aafbf0a5c3 100644 --- a/spartan/metrics/grafana_dashboards/provers.json +++ b/spartan/metrics/grafana/dashboards/aztec_provers.json @@ -330,7 +330,7 @@ "values": false }, "showPercentChange": false, - "textMode": "auto", + "textMode": "value_and_name", "wideLayout": true }, "pluginVersion": "11.2.1", @@ -352,7 +352,7 @@ "uid": "spartan-metrics-prometheus" }, "editorMode": "code", - "expr": "sum(aztec_proving_queue_rejected_jobs_count{k8s_namespace_name=\"$namespace\"})", + "expr": "sum(aztec_proving_queue_rejected_jobs_count{k8s_namespace_name=\"$namespace\"}) or vector(0)", "hide": false, "instant": false, "legendFormat": "Failed jobs", @@ -1031,7 +1031,7 @@ "y": 34 }, "id": 12, - "interval": "10m", + "maxDataPoints": 32, "options": { "calculate": false, "cellGap": 1, @@ -1068,7 +1068,7 @@ "yAxis": { "axisPlacement": "left", "reverse": false, - "unit": "ms" + "unit": "s" } }, "pluginVersion": "11.2.1", @@ -1333,8 +1333,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1451,8 +1450,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1495,6 +1493,7 @@ }, "editorMode": "code", "expr": "sum(aztec_proving_queue_size{k8s_namespace_name=\"$namespace\"}) by (aztec_proving_job_type)", + "interval": "", "legendFormat": "{{aztec_proving_job_type}}", "range": true, "refId": "A" @@ -1553,8 +1552,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1611,6 +1609,7 @@ "type": "prometheus", "uid": "spartan-metrics-prometheus" }, + "description": "How long jobs spend in the queue before being taken by an agent", "fieldConfig": { "defaults": { "color": { @@ -1654,8 +1653,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1674,14 +1672,11 @@ "y": 68 }, "id": 19, + "interval": "20m", "options": { "legend": { - "calcs": [ - "mean", - "p95", - "max" - ], - "displayMode": "table", + "calcs": [], + "displayMode": "list", "placement": "bottom", "showLegend": true }, @@ -1698,15 +1693,54 @@ "uid": "spartan-metrics-prometheus" }, "editorMode": "code", - "expr": "sum by (aztec_proving_job_type) (rate(aztec_proving_queue_job_wait_milliseconds_sum{k8s_namespace_name=\"$namespace\"}[$__rate_interval])) / sum by (aztec_proving_job_type) (rate(aztec_proving_queue_job_wait_milliseconds_count{k8s_namespace_name=\"$namespace\"}[$__rate_interval]))", - "hide": false, + "expr": "avg (rate(aztec_proving_queue_job_wait_milliseconds_sum{k8s_namespace_name=\"$namespace\"}[$__rate_interval]) / rate(aztec_proving_queue_job_wait_milliseconds_count{k8s_namespace_name=\"$namespace\"}[$__rate_interval]))", + "hide": true, "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{label_name}}", "range": true, "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(aztec_proving_queue_job_wait_milliseconds_bucket{k8s_namespace_name=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": false, + "instant": false, + "legendFormat": "50th Perc", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum(rate(aztec_proving_queue_job_wait_milliseconds_bucket{k8s_namespace_name=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": false, + "instant": false, + "legendFormat": "90th Perc", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "spartan-metrics-prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(aztec_proving_queue_job_wait_milliseconds_bucket{k8s_namespace_name=\"$namespace\"}[$__rate_interval])) by (le))", + "hide": false, + "instant": false, + "legendFormat": "95th Perc", + "range": true, + "refId": "D" } ], - "title": "AVG Job Wait Time by Type", + "title": "AVG Job Wait Time", "type": "timeseries" }, { @@ -1727,8 +1761,7 @@ "mode": "absolute", "steps": [ { - "color": "orange", - "value": null + "color": "orange" } ] }, @@ -1827,8 +1860,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -1843,7 +1875,7 @@ "y": 77 }, "id": 22, - "interval": "5m", + "interval": "20m", "options": { "barRadius": 0, "barWidth": 0.97, @@ -1875,7 +1907,7 @@ "uid": "spartan-metrics-prometheus" }, "editorMode": "code", - "expr": "sum(rate(aztec_proving_queue_job_duration_milliseconds_sum{k8s_namespace_name=\"$namespace\"}[$__rate_interval])) by (aztec_proving_job_type) / sum(rate(aztec_proving_queue_job_duration_milliseconds_count{k8s_namespace_name=\"$namespace\"}[$__rate_interval])) by (aztec_proving_job_type)", + "expr": "avg by (aztec_proving_job_type) ((rate(aztec_proving_queue_job_duration_milliseconds_sum{k8s_namespace_name=\"$namespace\"}[$__rate_interval])) / rate(aztec_proving_queue_job_duration_milliseconds_count{k8s_namespace_name=\"$namespace\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "__auto", @@ -1915,7 +1947,7 @@ "y": 77 }, "id": 23, - "interval": "5m", + "maxDataPoints": 32, "options": { "calculate": false, "cellGap": 1, @@ -1952,7 +1984,7 @@ "yAxis": { "axisPlacement": "left", "reverse": false, - "unit": "ms" + "unit": "s" } }, "pluginVersion": "11.2.1", @@ -2025,8 +2057,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2108,8 +2139,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2183,8 +2213,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2286,8 +2315,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2358,8 +2386,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2420,9 +2447,9 @@ "list": [ { "current": { - "selected": true, - "text": "master-rc-1", - "value": "master-rc-1" + "selected": false, + "text": "ignition-testnet", + "value": "ignition-testnet" }, "datasource": { "type": "prometheus", @@ -2449,13 +2476,13 @@ ] }, "time": { - "from": "now-1h", + "from": "now-3h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Prover Node Overview", "uid": "aztec-prover-nodes", - "version": 22, + "version": 26, "weekStart": "" } diff --git a/spartan/metrics/grafana_dashboards/validators.json b/spartan/metrics/grafana/dashboards/aztec_validators.json similarity index 98% rename from spartan/metrics/grafana_dashboards/validators.json rename to spartan/metrics/grafana/dashboards/aztec_validators.json index 80d100fef7c..16e4f079c19 100644 --- a/spartan/metrics/grafana_dashboards/validators.json +++ b/spartan/metrics/grafana/dashboards/aztec_validators.json @@ -552,7 +552,7 @@ "y": 12 }, "id": 105, - "interval": "30s", + "interval": "1m", "options": { "legend": { "calcs": [ @@ -577,7 +577,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "delta(aztec_sequencer_block_count{k8s_namespace_name=\"$namespace\", aztec_status=\"built\"}[$__rate_interval])", + "expr": "delta(aztec_sequencer_block_count{k8s_namespace_name=\"$namespace\", aztec_status=\"built\"}[1m])", "hide": false, "legendFormat": "{{k8s_pod_name}}", "range": true, @@ -606,6 +606,7 @@ "type": "prometheus", "uid": "spartan-metrics-prometheus" }, + "description": "Count of blocks that fall within each bucket", "fieldConfig": { "defaults": { "custom": { @@ -628,14 +629,20 @@ "y": 21 }, "id": 201, + "interval": "36s", + "maxDataPoints": 64, "options": { "calculate": false, "cellGap": 1, + "cellValues": { + "decimals": 0, + "unit": "none" + }, "color": { "exponent": 0.5, "fill": "dark-orange", "mode": "scheme", - "reverse": false, + "reverse": true, "scale": "exponential", "scheme": "Purples", "steps": 64 @@ -650,7 +657,8 @@ "show": true }, "rowsFrame": { - "layout": "auto" + "layout": "auto", + "value": "Block count" }, "tooltip": { "mode": "single", @@ -659,7 +667,8 @@ }, "yAxis": { "axisPlacement": "left", - "reverse": false + "reverse": false, + "unit": "ms" } }, "pluginVersion": "11.2.1", @@ -670,8 +679,9 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(increase(aztec_sequencer_block_build_duration_milliseconds_bucket{k8s_namespace_name=\"$namespace\"}[5m])) by (le)", + "expr": "sum(increase(aztec_sequencer_block_build_duration_milliseconds_bucket{k8s_namespace_name=\"$namespace\"}[$__interval])) by (le)", "format": "heatmap", + "interval": "", "legendFormat": "{{le}}", "range": true, "refId": "A" @@ -1090,19 +1100,6 @@ "title": "Time to Collect Attestations", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 38 - }, - "id": 300, - "panels": [], - "title": "State Machine Performance", - "type": "row" - }, { "datasource": { "default": true, @@ -1164,7 +1161,7 @@ "h": 8, "w": 12, "x": 0, - "y": 39 + "y": 38 }, "id": 303, "interval": "1m", @@ -1271,7 +1268,7 @@ "h": 8, "w": 12, "x": 12, - "y": 39 + "y": 38 }, "id": 304, "options": { @@ -1312,7 +1309,7 @@ "h": 1, "w": 24, "x": 0, - "y": 47 + "y": 46 }, "id": 400, "panels": [], @@ -1396,7 +1393,7 @@ "h": 8, "w": 8, "x": 0, - "y": 48 + "y": 47 }, "id": 401, "options": { @@ -1518,7 +1515,7 @@ "h": 8, "w": 8, "x": 8, - "y": 48 + "y": 47 }, "id": 402, "options": { @@ -1659,7 +1656,7 @@ "h": 8, "w": 8, "x": 16, - "y": 48 + "y": 47 }, "id": 403, "options": { @@ -1754,7 +1751,7 @@ "h": 8, "w": 8, "x": 0, - "y": 56 + "y": 55 }, "id": 404, "interval": "1m", @@ -1863,7 +1860,7 @@ "h": 8, "w": 8, "x": 8, - "y": 56 + "y": 55 }, "id": 405, "options": { @@ -1958,7 +1955,7 @@ "h": 8, "w": 8, "x": 16, - "y": 56 + "y": 55 }, "id": 406, "interval": "5m", @@ -2066,7 +2063,7 @@ "h": 8, "w": 8, "x": 0, - "y": 64 + "y": 63 }, "id": 505, "interval": "5m", @@ -2177,7 +2174,7 @@ "h": 8, "w": 8, "x": 8, - "y": 64 + "y": 63 }, "id": 506, "interval": "5m", @@ -2218,7 +2215,7 @@ "h": 1, "w": 24, "x": 0, - "y": 72 + "y": 71 }, "id": 500, "panels": [], @@ -2286,7 +2283,7 @@ "h": 8, "w": 12, "x": 0, - "y": 73 + "y": 72 }, "id": 501, "options": { @@ -2381,7 +2378,7 @@ "h": 8, "w": 12, "x": 12, - "y": 73 + "y": 72 }, "id": 502, "options": { @@ -2476,7 +2473,7 @@ "h": 8, "w": 12, "x": 0, - "y": 81 + "y": 80 }, "id": 503, "options": { @@ -2570,7 +2567,7 @@ "h": 8, "w": 12, "x": 12, - "y": 81 + "y": 80 }, "id": 504, "options": { @@ -2693,6 +2690,6 @@ "timezone": "", "title": "Validator node overview", "uid": "aztec-validators", - "version": 5, + "version": 7, "weekStart": "" } diff --git a/spartan/metrics/grafana_dashboards/metrics_stack.json b/spartan/metrics/grafana/dashboards/metrics_prometheus.json similarity index 98% rename from spartan/metrics/grafana_dashboards/metrics_stack.json rename to spartan/metrics/grafana/dashboards/metrics_prometheus.json index 3d1f707af7d..79344fb5703 100644 --- a/spartan/metrics/grafana_dashboards/metrics_stack.json +++ b/spartan/metrics/grafana/dashboards/metrics_prometheus.json @@ -18,6 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, + "id": 2, "links": [], "panels": [ { @@ -185,7 +186,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] } @@ -363,7 +365,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -463,7 +466,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -967,8 +971,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1085,8 +1088,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1203,8 +1205,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1305,8 +1306,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1407,8 +1407,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1517,8 +1516,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", diff --git a/spartan/metrics/grafana_dashboards/aztec-dashboard-all-in-one.json b/spartan/metrics/grafana_dashboards/aztec-dashboard-all-in-one.json deleted file mode 100644 index c4a917108fa..00000000000 --- a/spartan/metrics/grafana_dashboards/aztec-dashboard-all-in-one.json +++ /dev/null @@ -1,2537 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 3, - "links": [], - "panels": [ - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 92, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "aztec_archiver_block_height{k8s_namespace_name=\"$namespace\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "L2 Block Height", - "type": "timeseries" - }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 137, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum(rate(aztec_node_receive_tx_count{k8s_namespace_name=\"$namespace\"}[5m]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "TX Received Rate", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 2, - "panels": [], - "title": "Node system resources", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 12 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "1 - avg by(exported_job, k8s_namespace_name) (system_cpu_utilization{system_cpu_state=\"idle\", k8s_namespace_name=~\"$namespace\"})", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{exported_job}}-{{k8s_namespace_name}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "CPU utilization", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 12 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "avg by(exported_job, k8s_namespace_name) (system_memory_utilization{k8s_namespace_name=~\"$namespace\", system_memory_state=\"used\"})", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{exported_job}}-{{k8s_namespace_name}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Memory utilization", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 4, - "panels": [], - "title": "L2 network status", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 5, - "x": 0, - "y": 23 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "max(aztec_archiver_block_height{k8s_namespace_name=~\"$namespace\"})", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Last block number", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "fieldMinMax": false, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 5, - "y": 23 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "last_over_time(aztec_archiver_block_size{k8s_namespace_name=~\"$namespace\"}[$__interval])", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "Last block size", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "avg (aztec_archiver_block_size)", - "fullMetaSearch": false, - "hide": true, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Block size", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 13, - "x": 11, - "y": 23 - }, - "id": 23, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "avg (rate(aztec_archiver_sync_duration_milliseconds_sum{k8s_namespace_name=~\"$namespace\"}[$interval]) / rate(aztec_archiver_sync_duration_milliseconds_count{k8s_namespace_name=~\"$namespace\"}[$interval])) by (exported_job, k8s_namespace_name)", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "AVG - {{exported_job}}-{{k8s_namespace_name}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "timeShift": "6h", - "title": "Block synch", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "true" - }, - "properties": [ - { - "id": "displayName", - "value": "Accepted" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 7, - "x": 0, - "y": 31 - }, - "id": 21, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "increase(aztec_node_receive_tx_count{k8s_namespace_name=~\"$namespace\"}[$__rate_interval])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{aztec_ok}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Incoming transactions", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 5, - "x": 7, - "y": 31 - }, - "id": 6, - "options": { - "displayLabels": ["name", "value"], - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "pieType": "pie", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "avg by(aztec_status) (aztec_mempool_tx_count{k8s_namespace_name=~\"$namespace\"})", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Mempool status", - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 7, - "x": 12, - "y": 31 - }, - "id": 22, - "interval": "5m", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "avg (rate(aztec_node_receive_tx_duration_milliseconds_sum{k8s_namespace_name=~\"$namespace\"}[$__rate_interval]) / rate(aztec_node_receive_tx_duration_milliseconds_count{k8s_namespace_name=~\"$namespace\"}[$__rate_interval]))", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "AVG", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Tx receive time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 5, - "x": 19, - "y": 31 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "avg(rate(aztec_mempool_tx_size_bytes_sum{k8s_namespace_name=~\"$namespace\"}[$interval]) / rate(aztec_mempool_tx_size_bytes_count{k8s_namespace_name=~\"$namespace\"}[$interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Tx size", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 39 - }, - "id": 9, - "panels": [], - "title": "Sequencer", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "0": { - "index": 0, - "text": "Idle" - }, - "1": { - "index": 1, - "text": "Waiting for txs" - }, - "2": { - "index": 2, - "text": "Creating block" - }, - "3": { - "index": 3, - "text": "Publishing contract data" - }, - "4": { - "index": 4, - "text": "Publishing block" - }, - "5": { - "index": 5, - "text": "Stopped" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 7, - "x": 0, - "y": 40 - }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [], - "fields": "", - "values": true - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "aztec_sequencer_current_state{k8s_namespace_name=~\"$namespace\"}", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A", - "useBackend": false - } - ], - "title": "Sequencer state", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 9, - "x": 7, - "y": 40 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "aztec_sequencer_current_block_number{k8s_namespace_name=~\"$namespace\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "Next block number", - "range": false, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "aztec_sequencer_current_block_size{k8s_namespace_name=~\"$namespace\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "Next block size", - "range": false, - "refId": "B", - "useBackend": false - } - ], - "title": "Next block", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 16, - "y": 40 - }, - "hideTimeOverride": true, - "id": 26, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "/^Time$/", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "changes(aztec_sequencer_current_block_number{k8s_namespace_name=~\"$namespace\"}[1h])", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "interval": "", - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "timeFrom": "6h", - "title": "Next block started at", - "transformations": [ - { - "id": "filterByValue", - "options": { - "filters": [ - { - "config": { - "id": "equal", - "options": { - "value": 0 - } - }, - "fieldName": "{k8s_namespace_name=\"22828b91-92ae-4b04-a5e0-56ebf866edef\", exported_job=\"alphanet-aztec-node-1\", instance=\"aztec-otel.local:8889\", job=\"aztec\"}" - } - ], - "match": "any", - "type": "exclude" - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 20, - "y": 40 - }, - "id": 25, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "avg(rate(aztec_public_processor_deploy_bytecode_size_bytes_sum{k8s_namespace_name=~\"$namespace\"}[$interval]) / rate(aztec_public_processor_deploy_bytecode_size_bytes_count{k8s_namespace_name=~\"$namespace\"}[$interval]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "AVG", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Deployed bytecode", - "type": "stat" - }, - { - "datasource": {}, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 46 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "increase(aztec_l1_publisher_tx_count{k8s_namespace_name=~\"$namespace\"}[$__rate_interval])", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{aztec_l1_tx_type}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "L1 transactions", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 13, - "x": 8, - "y": 46 - }, - "id": 11, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(le) (rate(aztec_sequencer_block_build_duration_milliseconds_bucket{k8s_namespace_name=~\"$namespace\"}[$interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "interval": "", - "legendFormat": "Median", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(0.95, sum by(le) (rate(aztec_sequencer_block_build_duration_milliseconds_bucket{k8s_namespace_name=~\"$namespace\"}[$interval])))", - "fullMetaSearch": false, - "hide": true, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "95th percentile", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "histogram_quantile(1, sum by(le) (rate(aztec_sequencer_block_build_duration_milliseconds_bucket{k8s_namespace_name=~\"$namespace\"}[$interval])))", - "fullMetaSearch": false, - "hide": true, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "MAX", - "range": true, - "refId": "C", - "useBackend": false - } - ], - "timeShift": "6h", - "title": "Block building duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 10, - "x": 0, - "y": 54 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "avg(rate(aztec_public_processor_tx_duration_milliseconds_sum{k8s_namespace_name=~\"$namespace\"}[$__rate_interval]) / rate(aztec_public_processor_tx_duration_milliseconds_count{k8s_namespace_name=~\"$namespace\"}[$__rate_interval]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "interval": "5m", - "legendFormat": "AVG - tx", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "avg by (aztec_tx_phase_name) (rate(aztec_public_processor_phase_duration_milliseconds_sum{k8s_namespace_name=~\"$namespace\"}[$__rate_interval]) / rate(aztec_public_processor_phase_duration_milliseconds_count{k8s_namespace_name=~\"$namespace\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "5m", - "legendFormat": "AVG - {{aztec_tx_phase_name}}", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "avg(rate(aztec_public_executor_simulation_duration_milliseconds_sum{k8s_namespace_name=~\"$namespace\"}[$__rate_interval]) / rate(aztec_public_executor_simulation_duration_milliseconds_count{k8s_namespace_name=~\"$namespace\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "5m", - "legendFormat": "AVG - fn", - "range": true, - "refId": "C", - "useBackend": false - } - ], - "title": "Tx simulation", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 62 - }, - "id": 16, - "panels": [], - "title": "Prover", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 9, - "x": 0, - "y": 63 - }, - "id": 19, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.1.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "aztec_proving_queue_size{k8s_namespace_name=~\"$namespace\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Queue depth", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 15, - "x": 9, - "y": 63 - }, - "id": 20, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "avg by (aztec_proving_job_type) (rate(aztec_proving_queue_job_size_by_sum{k8s_namespace_name=~\"$namespace\"}[$interval]) / rate(aztec_proving_queue_job_size_by_count{k8s_namespace_name=~\"$namespace\"}[$interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "{{aztec_proving_job_type}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Job size", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 72 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(aztec_proving_orchestrator_base_rollup_inputs_duration_milliseconds_sum{k8s_namespace_name=~\"$namespace\"}[$interval]) / rate(aztec_proving_orchestrator_base_rollup_inputs_duration_milliseconds_count{k8s_namespace_name=~\"$namespace\"}[$interval])", - "fullMetaSearch": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "AVG", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Tree updates", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 80 - }, - "id": 17, - "panels": [], - "repeat": "protocol_circuit", - "repeatDirection": "h", - "title": "Circuits stats - $protocol_circuit", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "aztec_circuit_public_inputs_count" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "displayName", - "value": "Public inputs counts" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "aztec_circuit_proving_proof_size_bytes" - }, - "properties": [ - { - "id": "displayName", - "value": "Proof size" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 81 - }, - "id": 47, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "aztec_circuit_proving_proof_size_bytes{k8s_namespace_name=~\"$namespace\", aztec_circuit_protocol_circuit_name=\"$protocol_circuit\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "aztec_circuit_public_inputs_count{k8s_namespace_name=~\"$namespace\", aztec_circuit_protocol_circuit_name=\"$protocol_circuit\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Stats - $protocol_circuit", - "type": "stat" - }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 6, - "y": 81 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "sum(rate(aztec_circuit_witness_generation_duration_milliseconds_sum{aztec_circuit_protocol_circuit_name=\"$protocol_circuit\", k8s_namespace_name=~\"$namespace\"}[$interval]))\n/(sum(rate(aztec_circuit_witness_generation_duration_milliseconds_count{aztec_circuit_protocol_circuit_name=\"$protocol_circuit\", k8s_namespace_name=~\"$namespace\"}[$interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "", - "legendFormat": "AVG", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Witness generation - $protocol_circuit", - "type": "timeseries" - }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 10, - "x": 14, - "y": 81 - }, - "id": 46, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "sum(rate(aztec_circuit_proving_duration_milliseconds_sum{aztec_circuit_protocol_circuit_name=\"$protocol_circuit\", k8s_namespace_name=~\"$namespace\"}[$interval]))\n/\nsum(rate(aztec_circuit_proving_duration_milliseconds_count{aztec_circuit_protocol_circuit_name=\"$protocol_circuit\", k8s_namespace_name=~\"$namespace\"}[$interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "", - "legendFormat": "AVG", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Proving - $protocol_circuit", - "type": "timeseries" - } - ], - "refresh": "10s", - "schemaVersion": 39, - "tags": [], - "templating": { - "list": [ - { - "current": { - "selected": true, - "text": "rc1", - "value": "rc1" - }, - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "definition": "label_values(k8s_namespace_name)", - "hide": 0, - "includeAll": true, - "multi": false, - "name": "namespace", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(k8s_namespace_name)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, - { - "current": { - "selected": false, - "text": "4h", - "value": "4h" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "interval", - "options": [ - { - "selected": true, - "text": "4h", - "value": "4h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "24h", - "value": "24h" - } - ], - "query": "4h, 12h, 24h", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" - }, - { - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "spartan-metrics-prometheus" - }, - "definition": "label_values(aztec_circuit_protocol_circuit_name)", - "hide": 0, - "includeAll": true, - "label": "Protocol circuit", - "multi": false, - "name": "protocol_circuit", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(aztec_circuit_protocol_circuit_name)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Aztec Network Dashboard", - "uid": "cdtxao66xa1ogc", - "version": 11, - "weekStart": "" -} diff --git a/spartan/metrics/values.tmp.yaml b/spartan/metrics/values.tmp.yaml index c03a1e275b4..05833bb76aa 100644 --- a/spartan/metrics/values.tmp.yaml +++ b/spartan/metrics/values.tmp.yaml @@ -112,6 +112,9 @@ prometheus: # Enable and configure Grafana # https://artifacthub.io/packages/helm/grafana/grafana grafana: + env: + PRODUCTION_NAMESPACES_REGEX: "devnet|master-rc-1|ignition-testnet" + SLACK_WEBHOOK_URL: "http://127.0.0.1" # dummy value datasources: datasources.yaml: apiVersion: 1 @@ -136,9 +139,28 @@ grafana: editable: true options: path: /var/lib/grafana/dashboards/default + - name: "aztec" + orgId: 1 + folder: "Aztec" + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/aztec + - name: "metrics" + orgId: 1 + folder: "Metrics" + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/metrics + # unfortunately, we can't use the `file` helper in values.yaml, so we have to inline the dashboard + # json. This is a limitation of Helm. + # See the install scripts: we inject the dashboard json into a copy of this file, which is the + # version that actually gets helm installed. + alerting: dashboards: default: - # unfortunately, we can't use the `file` helper here, so we have to inline the dashboard - # json. This is a limitation of Helm. - # See the install scripts: we inject the dashboard json into a copy of this file, which is the - # version that actually gets helm installed. + aztec: + metrics: diff --git a/spartan/terraform/deploy-metrics/main.tf b/spartan/terraform/deploy-metrics/main.tf index da948f22905..b3a24d3528b 100644 --- a/spartan/terraform/deploy-metrics/main.tf +++ b/spartan/terraform/deploy-metrics/main.tf @@ -83,6 +83,11 @@ resource "helm_release" "aztec-gke-cluster" { value = var.GRAFANA_DASHBOARD_PASSWORD } + set { + name = "grafana.env.SLACK_WEBHOOK_URL" + value = var.SLACK_WEBHOOK_URL + } + set { name = "opentelemetry-collector.service.loadBalancerIP" value = google_compute_address.otel_collector_ip.address diff --git a/spartan/terraform/deploy-metrics/variables.tf b/spartan/terraform/deploy-metrics/variables.tf index 7611d25b281..b1827052043 100644 --- a/spartan/terraform/deploy-metrics/variables.tf +++ b/spartan/terraform/deploy-metrics/variables.tf @@ -20,6 +20,12 @@ variable "GRAFANA_DASHBOARD_PASSWORD" { sensitive = true } +variable "SLACK_WEBHOOK_URL" { + description = "Webhook to use to send to notifications" + type = string + sensitive = true +} + variable "project" { default = "testnet-440309" type = string