Re-org dashboards into common folder (#3905)

.github/workflows/test.yml

@@ -42,8 +42,8 @@ jobs:
           key: spec-test-data-${{ hashFiles('packages/validator/test/spec/params.ts') }}
 
       # Misc sanity checks
-      - name: Lint Grafana Dashboard
-        run: yarn validate-gdash
+      - name: Lint Grafana dashboards
+        run: node scripts/validate-grafana-dashboard.mjs ./dashboards
       - name: Test root binary exists
         run: ./lodestar --version
       - name: Reject yarn.lock changes

docker-compose.yml

@@ -24,15 +24,15 @@ services:
       - "prometheus:/prometheus"
 
   grafana:
-    build:
-      context: docker/grafana
-      args:
-        GF_SECURITY_ADMIN_PASSWORD: ${GF_SECURITY_ADMIN_PASSWORD}
+    build: docker/grafana
     restart: always
     ports:
       - "3000:3000"
     volumes:
       - "grafana:/var/lib/grafana"
+      - "./dashboards:/dashboards"
+    environment:
+      GF_SECURITY_ADMIN_PASSWORD: ${GF_SECURITY_ADMIN_PASSWORD}
 
 volumes:
   beacon_node:

docker/docker-compose.local.mac.yml

@@ -1,36 +0,0 @@
-version: "3.4"
-
-# Configuration to work with a local non-dockerized Lodestar node on MacOS
-# For local testing and quick debugging
-#
-# HOW TO USE: Start a Lodestar node, then run
-#
-# docker-compose -f docker/docker-compose.local.mac.yml up -d
-
-services:
-  prometheus:
-    build:
-      context: prometheus
-      args:
-        config_file: prometheus.local.mac.yml
-    restart: always
-    ports:
-      - "9090:9090"
-    volumes:
-      - "prometheus:/prometheus"
-
-  grafana:
-    build:
-      context: grafana
-      args:
-        DATASOURCE_FILE: datasource.local.mac.yml
-    restart: always
-    ports:
-      - "3000:3000"
-    volumes:
-      - "grafana:/var/lib/grafana"
-    depends_on: [prometheus]
-
-volumes:
-  prometheus:
-  grafana:

docker/docker-compose.local.yml

@@ -12,22 +12,25 @@ services:
     build:
       context: prometheus
       args:
-        config_file: prometheus.local.yml
+        # Linux:  http://localhost:8008
+        # MacOSX: http://host.docker.internal:8008
+        BEACON_URL: localhost:8008
     restart: always
+    network_mode: host
     volumes:
       - "prometheus:/prometheus"
-    network_mode: host
 
   grafana:
-    build:
-      context: grafana
-      args:
-        DATASOURCE_FILE: datasource.local.yml
+    build: grafana
     restart: always
+    network_mode: host
     volumes:
       - "grafana:/var/lib/grafana"
-    depends_on: [prometheus]
-    network_mode: host
+      - "../dashboards:/dashboards"
+    environment:
+      # Linux:  http://localhost:9090
+      # MacOSX: http://host.docker.internal:9090
+      PROMETHEUS_URL: http://localhost:9090
 
 volumes:
   prometheus:

docker/grafana/Dockerfile

@@ -1,24 +1,22 @@
 # Same version as our ansible deployments, to minimize the diff in the dashboard on export
 FROM grafana/grafana:8.4.2
 
-COPY provisioning/ /etc/grafana/provisioning/
-COPY provisioning/dashboards/*.json /provisioning/dashboards/
-COPY grafana.ini /etc/grafana.ini
-
-# Modified datasource to work with a network_mode: host
-ARG DATASOURCE_FILE=./datasource.yml
-COPY ${DATASOURCE_FILE} /etc/grafana/provisioning/datasources/datasource.yml
+# Datasource URL is configured with ENV variables
+COPY datasource.yml /etc/grafana/provisioning/datasources/datasource.yml
+# Note: Dashboard as linked via a bind volume
+COPY dashboard.yml /etc/grafana/provisioning/dashboards/dashboard.yml
 
 # Set GF_SECURITY_ADMIN_PASSWORD=your-password to the root .env file
 ARG GF_SECURITY_ADMIN_PASSWORD
 ENV GF_SECURITY_ADMIN_USER=admin \
   GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD}
 
+# Modified datasource to work with a network_mode: host
+ENV PROMETHEUS_URL=http://prometheus:9090
+ENV DASHBOARDS_DIR=/dashboards
+
 CMD [ \
   "--homepath=/usr/share/grafana", \
-  "--config=/etc/grafana.ini", \
   "--packaging=docker", \
-  "cfg:default.paths.data=/var/lib/grafana", \
-  "cfg:default.log.mode=console", \
-  "cfg:default.log.level=error" \
+  "cfg:default.paths.data=/var/lib/grafana" \
   ]

docker/grafana/dashboard.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+providers:
+  - name: lodestar_github
+    type: file
+    updateIntervalSeconds: 60
+    allowUiUpdates: true
+    options:
+      path: $DASHBOARDS_DIR
+      foldersFromFilesStructure: true

docker/grafana/datasource.local.mac.yml

@@ -1,9 +0,0 @@
-apiVersion: 1
-
-datasources:
-  - name: Prometheus
-    type: prometheus
-    access: proxy
-    url: http://host.docker.internal:9090
-    uid: prometheus_local
-    isDefault: true

docker/grafana/datasource.local.yml

@@ -1,9 +0,0 @@
-apiVersion: 1
-
-datasources:
-  - name: Prometheus
-    type: prometheus
-    access: proxy
-    url: http://localhost:9090
-    uid: prometheus_local
-    isDefault: true

docker/grafana/datasource.yml

@@ -1,9 +1,9 @@
 apiVersion: 1
 
 datasources:
-  - name: Prometheus
+  - name: prometheus
     type: prometheus
     access: proxy
-    url: http://prometheus:9090
-    uid: prometheus_local
+    url: $PROMETHEUS_URL
+    uid: prometheus
     isDefault: true

docker/grafana/grafana.ini

@@ -1,18 +0,0 @@
-##################### Grafana Configuration #####################
-
-[paths]
-# folder that contains provisioning config files that grafana will apply on startup and while running.
-provisioning = /etc/grafana/provisioning
-data = /var/lib/grafana
-
-[log]
-# Either "console", "file", "syslog". Default is console and  file
-# Use space to separate multiple modes, e.g. "console file"
-mode = console file
-
-# Either "debug", "info", "warn", "error", "critical", default is "info"
-level = error
-
-[explore]
-# Enable the Explore section
-;enabled = true

docker/grafana/provisioning/dashboards/dashboard.yml

@@ -1,12 +0,0 @@
-apiVersion: 1
-
-providers:
-  - name: "Prometheus"
-    orgId: 1
-    folder: ""
-    type: file
-    disableDeletion: false
-    editable: true
-    allowUiUpdates: true
-    options:
-      path: /provisioning/dashboards

docker/prometheus/Dockerfile

@@ -1,8 +1,14 @@
 FROM prom/prometheus:latest
 
+COPY prometheus.yml /etc/prometheus/prometheus.yml
+
 # Modified datasource to work with a network_mode: host
-ARG config_file=./prometheus.yml
-COPY ${config_file} /etc/prometheus/prometheus.yml
+# Docker DNS: "beacon_node:8008"
+# net host: "localhost:8008"
+# MacOSX: "host.docker.internal:8008"
+ARG BEACON_URL=beacon_node:8008
+RUN sed -i 's/#BEACON_URL/'"$BEACON_URL"'/' /etc/prometheus/prometheus.yml
+RUN cat /etc/prometheus/prometheus.yml
 
 CMD [ \
   "--config.file=/etc/prometheus/prometheus.yml", \

docker/prometheus/prometheus.local.mac.yml

@@ -1,7 +0,0 @@
-scrape_configs:
-  - job_name: Lodestar
-    scrape_interval: 20s
-    scrape_timeout: 20s
-    metrics_path: /metrics
-    static_configs:
-      - targets: ["host.docker.internal:8008"]

docker/prometheus/prometheus.local.yml

@@ -1,7 +0,0 @@
-scrape_configs:
-  - job_name: Lodestar
-    scrape_interval: 20s
-    scrape_timeout: 20s
-    metrics_path: /metrics
-    static_configs:
-      - targets: ["localhost:8008"]

docker/prometheus/prometheus.yml

@@ -4,5 +4,6 @@ scrape_configs:
     scrape_timeout: 20s
     metrics_path: /metrics
     static_configs:
-      # Run in a docker-compose context with a "beacon_node" service. Uses internal docker DNS
-      - targets: ["beacon_node:8008"]
+      # This tag is to be replaced in the Dockerfile with sed
+      # Modified datasource to work with a network_mode: host
+      - targets: ["#BEACON_URL"]

package.json

@@ -29,8 +29,7 @@
     "publish:release": "lerna publish from-package --yes --no-verify-access",
     "release": "lerna version --no-push --sign-git-commit",
     "postrelease": "git tag -d $(git describe --abbrev=0)",
-    "check-readme": "lerna run check-readme",
-    "validate-gdash": "node scripts/validate-grafana-dashboard.js"
+    "check-readme": "lerna run check-readme"
   },
   "devDependencies": {
     "@dapplion/benchmark": "^0.2.2",

scripts/grafana_push_dashboards.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# Utility script to push dashboards to Grafana via its HTTP API. Useful to quickly sync local dashboards
+#
+# USAGE:
+#
+# API_URL=http://localhost:3000 API_USERPASS=admin:admin ./grafana_push_dashboards.sh dashboards/*.json
+#
+# - Accepts a single file, a file glob, or multiple combinations of both
+# - Set API_URL to the root Grafana API url: API_URL=https://yourgrafana.server
+# - Set API_USERPASS to `$user:$password`: API_USERPASS=admin:admin
+
+if [ $# -eq 0 ]; then
+  echo "No arguments supplied"
+  exit 1
+fi
+
+# Accepts a single file, a file glob, or multiple combinations of both
+for fileglob in "${@:1}"; do
+  for filename in $fileglob; do
+    echo "Uploading dashboard $filename"
+
+    # https://grafana.com/docs/grafana/latest/http_api/dashboard/#create--update-dashboard
+    # 
+    # POST /api/dashboards/db HTTP/1.1
+    # Accept: application/json
+    # Content-Type: application/json
+
+    # To authenticate with token. However, token cannot be provisioned so it's harder to use with ansible
+    # Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk
+    #
+    # To authenticate with user:password
+    # curl http://admin:admin@localhost:3000/api/search
+
+    curl -X POST \
+      -H "Accept: application/json" \
+      -H "Content-Type: application/json" \
+      -u $API_USERPASS \
+      -d @$filename \
+      ${API_URL}/api/dashboards/import
+  done
+done
+
+# TODO: Only overwrite if changes are detected. But it's not easy to do.
+#
+# 1. Check version of existing dashboard. However, we may not bump the version
+#
+# GET /api/dashboards/uid/:uid
+# {
+#   "uid": "lodestar",
+#   "version": 2,
+# }
+#
+# 2. Hash existing dashboard and compare with new dashboard. However, Grafana may render them differently
+

scripts/validate-grafana-dashboard.js

@@ -1,17 +0,0 @@
-const fs = require("node:fs");
-const path = require("node:path");
-
-const dashboardsDir = path.join(__dirname, "../docker/grafana/provisioning/dashboards");
-
-for (const dashboardName of fs.readdirSync(dashboardsDir)) {
-  if (dashboardName.endsWith(".json")) {
-    let lodestarDash = require(path.join(dashboardsDir, dashboardName));
-    lodestarDash = JSON.stringify(lodestarDash); // get everything in the same line
-
-    // match something like exemplar : true
-    const matches = lodestarDash.match(/(exemplar)(\s)*(["])?(\s)*:(\s)*(["])?(\s)*true/gi);
-    if (matches && matches.length > 0) {
-      throw new Error(`ExemplarQueryNotSupported: ${dashboardName}`);
-    }
-  }
-}

scripts/validate-grafana-dashboard.mjs

@@ -0,0 +1,25 @@
+import fs from "node:fs";
+import path from "node:path";
+
+// USAGE:
+// node validate-grafana-dashboard.mjs ./dashboards
+
+const dirpath = process.argv[2];
+if (!dirpath) throw Error("Must provide dirpath argument");
+
+const filenames = fs.readdirSync(dirpath);
+if (filenames.length === 0) throw Error(`Empty dir ${dirpath}`);
+
+for (const filename of filenames) {
+  if (filename.endsWith(".json")) {
+    const jsonStr = fs.readFileSync(path.join(dirpath, filename), "utf8");
+    // get everything in the same line
+    const jsonStrSameLine = JSON.stringify(JSON.parse(jsonStr));
+
+    // match something like exemplar : true
+    const matches = jsonStrSameLine.match(/(exemplar)(\s)*(["])?(\s)*:(\s)*(["])?(\s)*true/gi);
+    if (matches && matches.length > 0) {
+      throw new Error(`ExemplarQueryNotSupported: ${filename}`);
+    }
+  }
+}