feat: operator liveness metric (#1621)

Author: avilagaston9

grafana/provisioning/dashboards/aligned/aggregator_batcher.json

@@ -2721,6 +2721,123 @@
       ],
       "type": "timeseries"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "continuous-GrYlRd"
+          },
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "inspect": false
+          },
+          "mappings": [],
+          "noValue": "none",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Max"
+            },
+            "properties": [
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "type": "gauge"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 61
+      },
+      "id": 49,
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": false
+      },
+      "pluginVersion": "10.1.10",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "disableTextWrap": false,
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "floor(increase(missing_operator_count{job=\"aligned-tracker\"}[$__range]))",
+          "format": "time_series",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "interval": "1",
+          "legendFormat": "{{operator}}",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "# Operator Missing Tasks",
+      "transformations": [
+        {
+          "id": "reduce",
+          "options": {
+            "labelsToFields": false,
+            "reducers": [
+              "max"
+            ]
+          }
+        },
+        {
+          "id": "sortBy",
+          "options": {
+            "fields": {},
+            "sort": [
+              {
+                "desc": true,
+                "field": "Max"
+              }
+            ]
+          }
+        }
+      ],
+      "type": "table"
+    },
     {
       "datasource": {
         "type": "prometheus",
@@ -3176,13 +3293,13 @@
     "list": []
   },
   "time": {
-    "from": "now-5m",
+    "from": "now-30m",
     "to": "now"
   },
   "timepicker": {},
   "timezone": "browser",
   "title": "System Data",
   "uid": "aggregator",
-  "version": 7,
+  "version": 1,
   "weekStart": ""
 }

telemetry_api/lib/telemetry_api/ethereum_metrics.ex

@@ -9,6 +9,7 @@ defmodule TelemetryApi.Operators do
   alias TelemetryApi.Operators.Operator
   alias TelemetryApi.ContractManagers.OperatorStateRetriever
   alias TelemetryApi.ContractManagers.DelegationManager
+  alias TelemetryApi.PrometheusMetrics
 
   @doc """
   Returns the list of operators.
@@ -95,6 +96,18 @@ defmodule TelemetryApi.Operators do
         |> Enum.filter(fn {status, _} -> status == :ok end)
         |> Enum.map(fn {_, data} -> data end)
 
+      # Initialize new_operators metrics
+      Enum.map(new_operators, fn {_, op_data} ->
+        op_name_address = op_data.name <> " - " <> String.slice(op_data.address, 0..7)
+        PrometheusMetrics.initialize_operator_metrics(op_name_address)
+      end)
+
+      # If the server was restarted, initialize old_operators metrics
+      Enum.map(old_operators, fn {op, _} ->
+        op_name_address = op.name <> " - " <> String.slice(op.address, 0..7)
+        PrometheusMetrics.initialize_operator_metrics(op_name_address)
+      end)
+
       # Merge both lists
       operators = (new_operators ++ old_operators)
 

telemetry_api/lib/telemetry_api/operators.ex

@@ -1,7 +1,7 @@
 defmodule TelemetryApi.Periodically do
   use GenServer
   alias TelemetryApi.Operators
-  alias TelemetryApi.EthereumMetrics
+  alias TelemetryApi.PrometheusMetrics
   alias TelemetryApi.ContractManagers.RegistryCoordinatorManager
   require Logger
 
@@ -45,7 +45,7 @@ defmodule TelemetryApi.Periodically do
   def handle_info(:gas_price, _state) do
     case Ethers.current_gas_price() do
       {:ok, gas_price} ->
-        EthereumMetrics.new_gas_price(gas_price)
+        PrometheusMetrics.new_gas_price(gas_price)
 
       {:error, error} ->
         IO.inspect("Error fetching gas price: #{error}")

telemetry_api/lib/telemetry_api/periodically.ex

@@ -0,0 +1,35 @@
+defmodule TelemetryApi.PrometheusMetrics do
+  use Prometheus.Metric
+
+  @gauge [name: :gas_price, help: "Ethereum Gas Price.", labels: []]
+  @counter [name: :missing_operator_count, help: "Missing Operators", labels: [:operator]]
+
+  def new_gas_price(gas_price) do
+    Gauge.set(
+      [name: :gas_price, labels: []],
+      gas_price
+    )
+  end
+
+  def missing_operator(operator) do
+    Counter.inc(
+      name: :missing_operator_count,
+      labels: [operator]
+    )
+  end
+
+  def initialize_operator_metrics(operator) do
+    value =
+      Counter.value(
+        name: :missing_operator_count,
+        labels: [operator]
+      )
+
+    if value == :undefined do
+      Counter.inc(
+        [name: :missing_operator_count, labels: [operator]],
+        0
+      )
+    end
+  end
+end

telemetry_api/lib/telemetry_api/prometheus_metrics.ex

@@ -5,6 +5,7 @@ defmodule TelemetryApi.Traces do
   alias TelemetryApi.Traces.Trace
   alias TelemetryApi.Operators
   alias TelemetryApi.ContractManagers.StakeRegistry
+  alias TelemetryApi.PrometheusMetrics
 
   require OpenTelemetry.Tracer
   require OpenTelemetry.Ctx
@@ -208,7 +209,6 @@ defmodule TelemetryApi.Traces do
       :ok
     end
   end
-  
 
   @doc """
   Registers the sending of a batcher task to Ethereum in the task trace.
@@ -298,7 +298,7 @@ defmodule TelemetryApi.Traces do
       :ok
     end
   end
-  
+
   @doc """
   Registers a set gas price when the aggregator tries to respond to a task in the task trace.
 
@@ -368,8 +368,17 @@ defmodule TelemetryApi.Traces do
   defp add_missing_operators([]), do: :ok
 
   defp add_missing_operators(missing_operators) do
+    # Concatenate name + address
+    missing_operators =
+      missing_operators
+      |> Enum.map(fn op -> op.name <> " - " <> String.slice(op.address, 0..7) end)
+
+    # Send to prometheus
+    missing_operators
+    |> Enum.map(fn o -> PrometheusMetrics.missing_operator(o) end)
+
     missing_operators =
-      missing_operators |> Enum.map(fn o -> o.name end) |> Enum.join(";")
+      missing_operators |> Enum.join(";")
 
     Tracer.add_event("Missing Operators", [{:operators, missing_operators}])
   end