feat: provision alerts

Author: alexghr

.github/workflows/metrics-deploy.yml

@@ -37,6 +37,11 @@ on:
         required: true
         type: string
         default: "grafana-dashboard-password"
+      slack_webhook_url_secret_name:
+        description: The name of the secret which holds the Slack webhook URL
+        required: true
+        type: string
+        default: "slack-webhook-url"
     secrets:
       GCP_SA_KEY:
         required: true
@@ -70,6 +75,10 @@ on:
         description: The name of the secret which holds the Grafana dashboard password
         required: true
         default: "grafana-dashboard-password"
+      slack_webhook_url_secret_name:
+        description: The name of the secret which holds the Slack webhook URL
+        required: true
+        default: "slack-webhook-url"
 
 jobs:
   metrics_deployment:
@@ -89,6 +98,7 @@ jobs:
       TF_STATE_BUCKET: aztec-terraform
       GKE_CLUSTER_CONTEXT: "gke_testnet-440309_us-west1-a_${{ inputs.cluster }}"
       GRAFANA_DASHBOARD_PASSWORD_SECRET_NAME: ${{ inputs.grafana_dashboard_password_secret_name }}
+      SLACK_WEBHOOK_URL_SECRET_NAME: ${{ inputs.slack_webhook_url_name }}
 
     steps:
       - name: Checkout code
@@ -118,6 +128,12 @@ jobs:
           echo "::add-mask::$(gcloud secrets versions access latest --secret=${{ env.GRAFANA_DASHBOARD_PASSWORD_SECRET_NAME }})"
           echo "grafana_dashboard_password=$(gcloud secrets versions access latest --secret=${{ env.GRAFANA_DASHBOARD_PASSWORD_SECRET_NAME }})" >> "$GITHUB_OUTPUT"
 
+      - name: Grab the Slack webhook URL
+        id: get-slack-webhook-url
+        run: |
+          echo "::add-mask::$(gcloud secrets versions access latest --secret=${{ env.GRAFANA_DASHBOARD_PASSWORD_SECRET_NAME }})"
+          echo "slack_webhook_url=$(gcloud secrets versions access latest --secret=${{ env.SLACK_WEBHOOK_URL_SECRET_NAME }})" >> "$GITHUB_OUTPUT"
+
       - name: Ensure Terraform state bucket exists
         run: |
           if ! gsutil ls gs://${{ env.TF_STATE_BUCKET }} >/dev/null 2>&1; then
@@ -155,6 +171,7 @@ jobs:
             -var="VALUES_FILE=${{ env.VALUES_FILE }}" \
             -var="GKE_CLUSTER_CONTEXT=${{ env.GKE_CLUSTER_CONTEXT }}" \
             -var="GRAFANA_DASHBOARD_PASSWORD=${{ steps.get-grafana-dashboard-password.outputs.grafana_dashboard_password }}" \
+            -var="SLACK_WEBHOOK_URL=${{ steps.get-slack-webhook-url.outputs.slack_webhook_url }}" \
             -lock=${{ inputs.respect_tf_lock }}
 
       - name: Terraform Plan
@@ -165,6 +182,7 @@ jobs:
             -var="VALUES_FILE=${{ env.VALUES_FILE }}" \
             -var="GKE_CLUSTER_CONTEXT=${{ env.GKE_CLUSTER_CONTEXT }}" \
             -var="GRAFANA_DASHBOARD_PASSWORD=${{ steps.get-grafana-dashboard-password.outputs.grafana_dashboard_password }}" \
+            -var="SLACK_WEBHOOK_URL=${{ steps.get-slack-webhook-url.outputs.slack_webhook_url }}" \
             -out=tfplan \
             -lock=${{ inputs.respect_tf_lock }}
 

spartan/metrics/copy-dashboard.sh

@@ -5,8 +5,34 @@ cd "$(dirname "${BASH_SOURCE[0]}")"
 
 cp values.tmp.yaml values.yaml
 
-for dashboard in ./grafana_dashboards/*.json; do
-  dashboard_name=$(basename "$dashboard" .json)
-  export DASHBOARD_JSON=$(jq -c '.' "$dashboard")
-  yq -i ".grafana.dashboards.default.\"$dashboard_name\".json = strenv(DASHBOARD_JSON)" values.yaml
+for dashboard_file in ./grafana/dashboards/*.json; do
+  if [ -f "$dashboard_file" ]; then
+    # Get the filename without path and extension
+    full_filename=$(basename "$dashboard_file" .json)
+
+    # Extract folder name and dashboard name using underscore as separator
+    # Format: foldername_dashboardname.json
+    if [[ "$full_filename" == *"_"* ]]; then
+      folder_name=${full_filename%%_*}
+      dashboard_name=${full_filename#*_}
+    else
+      # If no underscore, use "default" as the folder
+      folder_name="default"
+      dashboard_name=$full_filename
+    fi
+
+    # Read the dashboard JSON content
+    export dashboard_content=$(jq -c '.' "$dashboard_file")
+
+    # Use yq to add the dashboard to the values file
+    yq -i ".grafana.dashboards.${folder_name}.${dashboard_name}.json = strenv(dashboard_content)" values.yaml
+
+    echo "Injected dashboard: $dashboard_name into folder: $folder_name"
+  fi
+done
+
+for file in ./grafana/alerts/*.yaml; do
+  file_name=$(basename "$file" .yaml)
+  export file_content=$(cat "$file" )
+  yq -i ".grafana.alerting.\"${file_name}.yaml\" = env(file_content)" values.yaml
 done

spartan/metrics/grafana/alerts/contactpoints.yaml

@@ -0,0 +1,11 @@
+apiVersion: 1
+contactPoints:
+    - orgId: 1
+      name: 'Slack #network-alerts channel'
+      receivers:
+        - uid: deexubp9hzpc1b
+          type: slack
+          settings:
+            url: $SLACK_WEBHOOK_URL
+          disableResolveMessage: false
+

spartan/metrics/grafana/alerts/policies.yaml

@@ -0,0 +1,11 @@
+apiVersion: 1
+policies:
+    - orgId: 1
+      receiver: 'Slack #network-alerts channel'
+      object_matchers:
+        - - k8s_namespace_name
+          - =~
+          - $PRODUCTION_NAMESPACES_REGEX
+      group_by:
+        - grafana_folder
+        - alertname