feat(o11y): adding an alert for the RPC providers availability (#948)

Author: geekbrother

terraform/monitoring/panels/usage/provider.libsonnet

@@ -1,8 +1,10 @@
 local grafana   = import '../../grafonnet-lib/grafana.libsonnet';
 local defaults  = import '../../grafonnet-lib/defaults.libsonnet';
 
-local panels    = grafana.panels;
-local targets   = grafana.targets;
+local panels         = grafana.panels;
+local targets        = grafana.targets;
+local alert          = grafana.alert;
+local alertCondition = grafana.alertCondition;
 
 {
   new(ds, vars, provider)::
@@ -14,7 +16,41 @@ local targets   = grafana.targets;
 
     .addTarget(targets.prometheus(
       datasource  = ds.prometheus,
-      expr          = 'sum by(chain_id) (increase(provider_status_code_counter_total{provider="%s"}[5m]))' % provider,
+      expr          = 'sum by(chain_id) (increase(provider_status_code_counter_total{provider="%s"}[$__rate_interval]))' % provider,
       legendFormat  = '__auto',
     ))
+
+    // Hidden target for the provider availability alert
+
+    .addTarget(targets.prometheus(
+      datasource  = ds.prometheus,
+      expr          = '(sum(increase(provider_status_code_counter_total{provider="%s", status_code="200"}[$__rate_interval])) / sum(increase(provider_status_code_counter_total{provider="%s"}[$__rate_interval]))) * 100' % [provider, provider],
+      legendFormat  = '__auto',
+      exemplar    = false,
+      refId       = 'providerAvailabilityPercent',
+      hide        = true,
+    ))
+
+    .setAlert(vars.environment, alert.new(
+      namespace     = 'Blockchain API',
+      name          = "%s - Provider availability drop" % vars.environment,
+      message       = "%s - Provider availability drop" % vars.environment,
+      period        = '5m',
+      frequency     = '1m',
+      noDataState   = 'no_data',
+      notifications = vars.notifications,
+      alertRuleTags = {
+        'og_priority': 'P3',
+      },
+      conditions  = [
+        alertCondition.new(
+          evaluatorParams = [ 90 ],
+          evaluatorType   = 'lt',
+          operatorType    = 'or',
+          queryRefId      = 'providerAvailabilityPercent',
+          queryTimeStart  = '5m',
+          reducerType     = 'avg',
+        ),
+      ]
+    ))
 }