feat: add validator activity metrics (#7634)

**Motivation**

Create metrics for monitoring validator activity.

**Description**

- validators in the queue to active
- validators in the queue to exit 
- pending consolidations 
- pending partial withdrawals

Closes #7579

**Steps to test or reproduce**

The metrics have been added to the `Lodestar` dashboard
(lodestar_summary.json)
<img width="1120" alt="Screenshot 2025-04-17 at 00 09 32"
src="https://github.com/user-attachments/assets/f35e5e38-bb37-4bbc-906a-b832c475641e"
/>

---------

Co-authored-by: Nico Flaig <nflaig@protonmail.com>

dashboards/lodestar_summary.json

@@ -1862,7 +1862,7 @@
       },
       "gridPos": {
         "h": 3,
-        "w": 4,
+        "w": 6,
         "x": 0,
         "y": 39
       },
@@ -1916,8 +1916,8 @@
       },
       "gridPos": {
         "h": 3,
-        "w": 4,
-        "x": 4,
+        "w": 6,
+        "x": 6,
         "y": 39
       },
       "id": 492,
@@ -1970,8 +1970,8 @@
       },
       "gridPos": {
         "h": 3,
-        "w": 4,
-        "x": 8,
+        "w": 6,
+        "x": 12,
         "y": 39
       },
       "id": 484,
@@ -2024,8 +2024,8 @@
       },
       "gridPos": {
         "h": 3,
-        "w": 4,
-        "x": 12,
+        "w": 6,
+        "x": 18,
         "y": 39
       },
       "id": 483,
@@ -2079,8 +2079,8 @@
       "gridPos": {
         "h": 3,
         "w": 4,
-        "x": 16,
-        "y": 39
+        "x": 0,
+        "y": 42
       },
       "id": 485,
       "options": {
@@ -2106,11 +2106,14 @@
             "type": "prometheus",
             "uid": "${DS_PROMETHEUS}"
           },
+          "editorMode": "code",
           "exemplar": false,
           "expr": "beacon_current_validators{status=\"active\"}",
           "hide": false,
+          "instant": true,
           "interval": "",
           "legendFormat": "",
+          "range": false,
           "refId": "B"
         },
         {
@@ -2118,17 +2121,136 @@
             "type": "prometheus",
             "uid": "${DS_PROMETHEUS}"
           },
+          "editorMode": "code",
           "exemplar": false,
           "expr": "beacon_current_active_validators",
           "hide": true,
+          "instant": true,
           "interval": "",
           "legendFormat": "",
+          "range": false,
           "refId": "A"
         }
       ],
       "title": "current active validators",
       "type": "stat"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": []
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 4,
+        "x": 4,
+        "y": 42
+      },
+      "id": 539,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "lodestar_stfn_validators_in_activation_queue",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "validators in the activation queue",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": []
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 4,
+        "x": 8,
+        "y": 42
+      },
+      "id": 540,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "lodestar_stfn_validators_in_exit_queue",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "validators in the exit queue",
+      "type": "stat"
+    },
     {
       "datasource": {
         "type": "prometheus",
@@ -2146,8 +2268,8 @@
       "gridPos": {
         "h": 3,
         "w": 4,
-        "x": 20,
-        "y": 39
+        "x": 12,
+        "y": 42
       },
       "id": 488,
       "options": {
@@ -2173,14 +2295,133 @@
             "type": "prometheus",
             "uid": "${DS_PROMETHEUS}"
           },
+          "editorMode": "code",
           "exemplar": false,
-          "expr": "beacon_processed_deposits_total",
+          "expr": "beacon_pending_deposits",
+          "instant": true,
           "interval": "",
           "legendFormat": "",
+          "range": false,
           "refId": "A"
         }
       ],
-      "title": "processed deposits",
+      "title": "pending deposits",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": []
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 4,
+        "x": 16,
+        "y": 42
+      },
+      "id": 542,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "beacon_pending_partial_withdrawals",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "pending partial withdrawals",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": []
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 4,
+        "x": 20,
+        "y": 42
+      },
+      "id": 541,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "beacon_pending_consolidations",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "pending consolidations",
       "type": "stat"
     },
     {
@@ -2233,7 +2474,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 42
+        "y": 45
       },
       "id": 490,
       "options": {
@@ -2297,7 +2538,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 42
+        "y": 45
       },
       "heatmap": {},
       "hideZeroBuckets": false,
@@ -2410,7 +2651,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 50
+        "y": 53
       },
       "heatmap": {},
       "hideZeroBuckets": false,
@@ -2447,7 +2688,8 @@
         },
         "showValue": "never",
         "tooltip": {
-          "show": true,
+          "mode": "single",
+          "showColorScale": false,
           "yHistogram": false
         },
         "yAxis": {
@@ -2456,7 +2698,7 @@
           "unit": "short"
         }
       },
-      "pluginVersion": "10.1.1",
+      "pluginVersion": "10.4.1",
       "reverseYBuckets": false,
       "targets": [
         {
@@ -2498,7 +2740,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 50
+        "y": 53
       },
       "id": 497,
       "options": {
@@ -2510,7 +2752,7 @@
         "content": "",
         "mode": "markdown"
       },
-      "pluginVersion": "10.1.1",
+      "pluginVersion": "10.4.1",
       "title": "-",
       "type": "text"
     },
@@ -2524,7 +2766,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 58
+        "y": 61
       },
       "id": 104,
       "panels": [],
@@ -2551,6 +2793,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "slots",
@@ -2591,7 +2834,7 @@
         "h": 6,
         "w": 12,
         "x": 0,
-        "y": 59
+        "y": 62
       },
       "id": 102,
       "options": {
@@ -2635,6 +2878,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -2701,7 +2945,7 @@
         "h": 6,
         "w": 12,
         "x": 12,
-        "y": 59
+        "y": 62
       },
       "id": 172,
       "options": {
@@ -2747,6 +2991,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "slots",
@@ -2787,7 +3032,7 @@
         "h": 6,
         "w": 12,
         "x": 0,
-        "y": 65
+        "y": 68
       },
       "id": 171,
       "options": {
@@ -2830,7 +3075,7 @@
         "h": 6,
         "w": 12,
         "x": 12,
-        "y": 65
+        "y": 68
       },
       "id": 99,
       "options": {
@@ -2842,7 +3087,7 @@
         "content": "",
         "mode": "markdown"
       },
-      "pluginVersion": "10.1.1",
+      "pluginVersion": "10.4.1",
       "targets": [
         {
           "datasource": {

packages/beacon-node/src/chain/archiveStore/historicalState/metrics.ts

@@ -66,6 +66,14 @@ export function createHistoricalStateTransitionMetrics(
       name: "lodestar_historical_state_stfn_num_effective_balance_updates_count",
       help: "Count of effective balance updates in epoch transition",
     }),
+    validatorsInActivationQueue: metricsRegister.gauge({
+      name: "lodestar_historical_state_stfn_validators_in_activation_queue",
+      help: "Current number of validators in the activation queue",
+    }),
+    validatorsInExitQueue: metricsRegister.gauge({
+      name: "lodestar_historical_state_stfn_validators_in_exit_queue",
+      help: "Current number of validators in the exit queue",
+    }),
     preStateBalancesNodesPopulatedMiss: metricsRegister.gauge<{source: StateCloneSource}>({
       name: "lodestar_historical_state_stfn_balances_nodes_populated_miss_total",
       help: "Total count state.balances nodesPopulated is false on stfn",

packages/beacon-node/src/chain/chain.ts

@@ -1114,6 +1114,14 @@ export class BeaconChain implements IBeaconChain {
     metrics.forkChoice.balancesLength.set(forkChoiceMetrics.balancesLength);
     metrics.forkChoice.nodes.set(forkChoiceMetrics.nodes);
     metrics.forkChoice.indices.set(forkChoiceMetrics.indices);
+
+    const fork = this.config.getForkName(this.clock.currentSlot);
+    if (isForkPostElectra(fork)) {
+      const headStateElectra = this.getHeadState() as BeaconStateElectra;
+      metrics.pendingDeposits.set(headStateElectra.pendingDeposits.length);
+      metrics.pendingPartialWithdrawals.set(headStateElectra.pendingPartialWithdrawals.length);
+      metrics.pendingConsolidations.set(headStateElectra.pendingConsolidations.length);
+    }
   }
 
   private onClockSlot(slot: Slot): void {

packages/beacon-node/src/metrics/metrics/beacon.ts

@@ -56,6 +56,21 @@ export function createBeaconMetrics(register: RegistryMetricCreator) {
       help: "number of validators in current epoch",
     }),
 
+    pendingDeposits: register.gauge({
+      name: "beacon_pending_deposits",
+      help: "Current number of pending deposits",
+    }),
+
+    pendingConsolidations: register.gauge({
+      name: "beacon_pending_consolidations",
+      help: "Current number of pending consolidations",
+    }),
+
+    pendingPartialWithdrawals: register.gauge({
+      name: "beacon_pending_partial_withdrawals",
+      help: "Current number of pending partial withdrawals",
+    }),
+
     // Non-spec'ed
 
     forkChoice: {

packages/state-transition/src/epoch/index.ts

@@ -110,6 +110,8 @@ export function processEpoch(
   // after processSlashings() to update balances only once
   // processRewardsAndPenalties(state, cache);
   {
+    metrics?.validatorsInActivationQueue.set(cache.indicesEligibleForActivationQueue.length);
+    metrics?.validatorsInExitQueue.set(cache.indicesToEject.length);
     const timer = metrics?.epochTransitionStepTime.startTimer({step: EpochTransitionStep.processRegistryUpdates});
     processRegistryUpdates(fork, state, cache);
     timer?.();

packages/state-transition/src/epoch/processPendingDeposits.ts

@@ -9,7 +9,7 @@ import {getActivationExitChurnLimit} from "../util/validator.js";
 
 /**
  * Starting from Electra:
- * Process pending balance deposits from state subject to churn limit and depsoitBalanceToConsume.
+ * Process pending balance deposits from state subject to churn limit and depositBalanceToConsume.
  * For each eligible `deposit`, call `increaseBalance()`.
  * Remove the processed deposits from `state.pendingDeposits`.
  * Update `state.depositBalanceToConsume` for the next epoch

packages/state-transition/src/metrics.ts

@@ -51,6 +51,14 @@ export function getMetrics(register: MetricsRegister) {
       name: "lodestar_stfn_effective_balance_updates_count",
       help: "Total count of effective balance updates",
     }),
+    validatorsInActivationQueue: register.gauge({
+      name: "lodestar_stfn_validators_in_activation_queue",
+      help: "Current number of validators in the activation queue",
+    }),
+    validatorsInExitQueue: register.gauge({
+      name: "lodestar_stfn_validators_in_exit_queue",
+      help: "Current number of validators in the exit queue",
+    }),
     preStateBalancesNodesPopulatedMiss: register.gauge<{source: StateCloneSource}>({
       name: "lodestar_stfn_balances_nodes_populated_miss_total",
       help: "Total count state.balances nodesPopulated is false on stfn",