cli to push metrics to aggregator service (#8835)

* new prometheus metrics for client-stats metrics

* adds client-stats types beacause they are
  used by some of the prometheus collection code.
* new prometheus collector for db disk size
* new prometheus collector for web3 client
  connection status

* adds client-stats api push cli in cmd/client-stats

* adds api metadata to client-stats collector posts

* appease deepsource

* mop up copypasta

* use prysm assert package for testing

cmd/client-stats/BUILD.bazel

@@ -0,0 +1,31 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary")
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "log.go",
+        "main.go",
+        "usage.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/cmd/client-stats",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//cmd/client-stats/flags:go_default_library",
+        "//shared/clientstats:go_default_library",
+        "//shared/cmd:go_default_library",
+        "//shared/journald:go_default_library",
+        "//shared/logutil:go_default_library",
+        "//shared/version:go_default_library",
+        "@com_github_joonix_log//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
+        "@com_github_x_cray_logrus_prefixed_formatter//:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "client-stats",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)

cmd/client-stats/flags/BUILD.bazel

@@ -0,0 +1,9 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["flags.go"],
+    importpath = "github.com/prysmaticlabs/prysm/cmd/client-stats/flags",
+    visibility = ["//visibility:public"],
+    deps = ["@com_github_urfave_cli_v2//:go_default_library"],
+)

cmd/client-stats/flags/flags.go

@@ -0,0 +1,30 @@
+// Package flags contains all configuration runtime flags for
+// the client-stats daemon.
+package flags
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	// BeaconCertFlag defines a flag for the beacon api certificate.
+	ValidatorMetricsURLFlag = &cli.StringFlag{
+		Name:  "validator-metrics-url",
+		Usage: "Full URL to the validator /metrics prometheus endpoint to scrape. eg http://localhost:8081/metrics",
+	}
+	// BeaconRPCProviderFlag defines a flag for the beacon host ip or address.
+	BeaconnodeMetricsURLFlag = &cli.StringFlag{
+		Name:  "beacon-node-metrics-url",
+		Usage: "Full URL to the beacon-node /metrics prometheus endpoint to scrape. eg http://localhost:8080/metrics",
+	}
+	// CertFlag defines a flag for the node's TLS certificate.
+	ClientStatsAPIURLFlag = &cli.StringFlag{
+		Name:  "clientstats-api-url",
+		Usage: "Full URL to the client stats endpoint where collected metrics should be sent.",
+	}
+	// CertFlag defines a flag for the node's TLS certificate.
+	ScrapeIntervalFlag = &cli.DurationFlag{
+		Name:  "scrape-interval",
+		Usage: "Frequency of scraping expressed as a duration, eg 2m or 1m5s. Default is 60s.",
+	}
+)

cmd/client-stats/log.go

@@ -0,0 +1,5 @@
+package main
+
+import "github.com/sirupsen/logrus"
+
+var log = logrus.WithField("prefix", "main")

cmd/client-stats/main.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	runtimeDebug "runtime/debug"
+	"time"
+
+	joonix "github.com/joonix/log"
+	"github.com/prysmaticlabs/prysm/cmd/client-stats/flags"
+	"github.com/prysmaticlabs/prysm/shared/clientstats"
+	"github.com/prysmaticlabs/prysm/shared/cmd"
+	"github.com/prysmaticlabs/prysm/shared/journald"
+	"github.com/prysmaticlabs/prysm/shared/logutil"
+	"github.com/prysmaticlabs/prysm/shared/version"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+	prefixed "github.com/x-cray/logrus-prefixed-formatter"
+)
+
+var appFlags = []cli.Flag{
+	cmd.VerbosityFlag,
+	cmd.LogFormat,
+	cmd.LogFileName,
+	cmd.ConfigFileFlag,
+	flags.BeaconnodeMetricsURLFlag,
+	flags.ValidatorMetricsURLFlag,
+	flags.ClientStatsAPIURLFlag,
+	flags.ScrapeIntervalFlag,
+}
+var scrapeInterval = 60 * time.Second
+
+func main() {
+	app := cli.App{}
+	app.Name = "client-stats"
+	app.Usage = "daemon to scrape client-stats from prometheus and ship to a remote endpoint"
+	app.Action = run
+	app.Version = version.Version()
+
+	app.Flags = appFlags
+
+	// logging/config setup cargo-culted from beaconchain
+	app.Before = func(ctx *cli.Context) error {
+		// Load flags from config file, if specified.
+		if err := cmd.LoadFlagsFromConfig(ctx, app.Flags); err != nil {
+			return err
+		}
+
+		verbosity := ctx.String(cmd.VerbosityFlag.Name)
+		level, err := logrus.ParseLevel(verbosity)
+		if err != nil {
+			return err
+		}
+		logrus.SetLevel(level)
+
+		format := ctx.String(cmd.LogFormat.Name)
+		switch format {
+		case "text":
+			formatter := new(prefixed.TextFormatter)
+			formatter.TimestampFormat = "2006-01-02 15:04:05"
+			formatter.FullTimestamp = true
+			// If persistent log files are written - we disable the log messages coloring because
+			// the colors are ANSI codes and seen as gibberish in the log files.
+			formatter.DisableColors = ctx.String(cmd.LogFileName.Name) != ""
+			logrus.SetFormatter(formatter)
+		case "fluentd":
+			f := joonix.NewFormatter()
+			if err := joonix.DisableTimestampFormat(f); err != nil {
+				panic(err)
+			}
+			logrus.SetFormatter(f)
+		case "json":
+			logrus.SetFormatter(&logrus.JSONFormatter{})
+		case "journald":
+			if err := journald.Enable(); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("unknown log format %s", format)
+		}
+
+		logFileName := ctx.String(cmd.LogFileName.Name)
+		if logFileName != "" {
+			if err := logutil.ConfigurePersistentLogging(logFileName); err != nil {
+				log.WithError(err).Error("Failed to configuring logging to disk.")
+			}
+		}
+		return nil
+	}
+
+	defer func() {
+		if x := recover(); x != nil {
+			log.Errorf("Runtime panic: %v\n%v", x, string(runtimeDebug.Stack()))
+			panic(x)
+		}
+	}()
+
+	if err := app.Run(os.Args); err != nil {
+		log.Error(err.Error())
+	}
+}
+
+func run(ctx *cli.Context) error {
+	if ctx.IsSet(flags.ScrapeIntervalFlag.Name) {
+		scrapeInterval = ctx.Duration(flags.ScrapeIntervalFlag.Name)
+	}
+
+	var upd clientstats.Updater
+	if ctx.IsSet(flags.ClientStatsAPIURLFlag.Name) {
+		u := ctx.String(flags.ClientStatsAPIURLFlag.Name)
+		upd = clientstats.NewClientStatsHTTPPostUpdater(u)
+	} else {
+		log.Warn("No --clientstats-api-url flag set, writing to stdout as default metrics sink.")
+		upd = clientstats.NewGenericClientStatsUpdater(os.Stdout)
+	}
+
+	scrapers := make([]clientstats.Scraper, 0)
+	if ctx.IsSet(flags.BeaconnodeMetricsURLFlag.Name) {
+		u := ctx.String(flags.BeaconnodeMetricsURLFlag.Name)
+		scrapers = append(scrapers, clientstats.NewBeaconNodeScraper(u))
+	}
+	if ctx.IsSet(flags.ValidatorMetricsURLFlag.Name) {
+		u := ctx.String(flags.ValidatorMetricsURLFlag.Name)
+		scrapers = append(scrapers, clientstats.NewValidatorScraper(u))
+	}
+
+	ticker := time.NewTicker(scrapeInterval)
+	for {
+		select {
+		case <-ticker.C:
+			for _, s := range scrapers {
+				r, err := s.Scrape()
+				if err != nil {
+					log.Errorf("Scraper error: %s", err)
+					continue
+				}
+				err = upd.Update(r)
+				if err != nil {
+					log.Errorf("client-stats collector error: %s", err)
+					continue
+				}
+			}
+		case <-ctx.Done():
+			ticker.Stop()
+			return nil
+		}
+	}
+}

cmd/client-stats/usage.go

@@ -0,0 +1,81 @@
+// This code was adapted from https://github.com/ethereum/go-ethereum/blob/master/cmd/geth/usage.go
+package main
+
+import (
+	"io"
+	"sort"
+
+	"github.com/prysmaticlabs/prysm/cmd/client-stats/flags"
+	"github.com/prysmaticlabs/prysm/shared/cmd"
+	"github.com/urfave/cli/v2"
+)
+
+var appHelpTemplate = `NAME:
+   {{.App.Name}} - {{.App.Usage}}
+USAGE:
+   {{.App.HelpName}} [options]{{if .App.Commands}} command [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}}
+   {{if .App.Version}}
+AUTHOR:
+   {{range .App.Authors}}{{ . }}{{end}}
+   {{end}}{{if .App.Commands}}
+GLOBAL OPTIONS:
+   {{range .App.Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
+   {{end}}{{end}}{{if .FlagGroups}}
+{{range .FlagGroups}}{{.Name}} OPTIONS:
+  {{range .Flags}}{{.}}
+  {{end}}
+{{end}}{{end}}{{if .App.Copyright }}
+COPYRIGHT:
+   {{.App.Copyright}}
+VERSION:
+   {{.App.Version}}
+   {{end}}{{if len .App.Authors}}
+   {{end}}
+`
+
+type flagGroup struct {
+	Name  string
+	Flags []cli.Flag
+}
+
+var appHelpFlagGroups = []flagGroup{
+	{
+		Name: "cmd",
+		Flags: []cli.Flag{
+			cmd.VerbosityFlag,
+			cmd.LogFormat,
+			cmd.LogFileName,
+			cmd.ConfigFileFlag,
+		},
+	},
+	{
+		Name: "client-stats",
+		Flags: []cli.Flag{
+			flags.BeaconnodeMetricsURLFlag,
+			flags.ValidatorMetricsURLFlag,
+			flags.ClientStatsAPIURLFlag,
+			flags.ScrapeIntervalFlag,
+		},
+	},
+}
+
+func init() {
+	cli.AppHelpTemplate = appHelpTemplate
+
+	type helpData struct {
+		App        interface{}
+		FlagGroups []flagGroup
+	}
+
+	originalHelpPrinter := cli.HelpPrinter
+	cli.HelpPrinter = func(w io.Writer, tmpl string, data interface{}) {
+		if tmpl == appHelpTemplate {
+			for _, group := range appHelpFlagGroups {
+				sort.Sort(cli.FlagsByName(group.Flags))
+			}
+			originalHelpPrinter(w, tmpl, helpData{data, appHelpFlagGroups})
+		} else {
+			originalHelpPrinter(w, tmpl, data)
+		}
+	}
+}