zpage server to help e2e debugging

deps.bzl

@@ -1274,6 +1274,12 @@ def prysm_deps():
         sum = "h1:utua3L2IbQJmauC5IXdEA547bcoU5dozgQAfc8Onsg4=",
         version = "v0.0.0-20181222135242-d2cdd8c08219",
     )
+    go_repository(
+        name = "com_github_gomarkdown_markdown",
+        importpath = "github.com/gomarkdown/markdown",
+        sum = "h1:YVvt637ygnOO9qjLBVmPOvrUmCz/i8YECSu/8UlOQW0=",
+        version = "v0.0.0-20220310201231-552c6011c0b8",
+    )
 
     go_repository(
         name = "com_github_google_btree",

go.mod

@@ -255,6 +255,7 @@ require (
 	github.com/go-logr/logr v0.2.1 // indirect
 	github.com/go-ole/go-ole v1.2.5 // indirect
 	github.com/go-playground/validator/v10 v10.10.0
+	github.com/gomarkdown/markdown v0.0.0-20220310201231-552c6011c0b8
 	github.com/peterh/liner v1.2.0 // indirect
 	github.com/prometheus/tsdb v0.10.0 // indirect
 	github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220303211031-f753e083138c

go.sum

@@ -447,6 +447,8 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
+github.com/gomarkdown/markdown v0.0.0-20220310201231-552c6011c0b8 h1:YVvt637ygnOO9qjLBVmPOvrUmCz/i8YECSu/8UlOQW0=
+github.com/gomarkdown/markdown v0.0.0-20220310201231-552c6011c0b8/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=

testing/endtoend/BUILD.bazel

@@ -41,6 +41,7 @@ go_test(
         "//testing/assert:go_default_library",
         "//testing/endtoend/components:go_default_library",
         "//testing/endtoend/components/eth1:go_default_library",
+        "//testing/endtoend/e2ez:go_default_library",
         "//testing/endtoend/evaluators:go_default_library",
         "//testing/endtoend/helpers:go_default_library",
         "//testing/endtoend/params:go_default_library",

testing/endtoend/components/BUILD.bazel

@@ -31,6 +31,7 @@ go_library(
         "//io/file:go_default_library",
         "//runtime/interop:go_default_library",
         "//testing/endtoend/components/eth1:go_default_library",
+        "//testing/endtoend/e2ez:go_default_library",
         "//testing/endtoend/helpers:go_default_library",
         "//testing/endtoend/params:go_default_library",
         "//testing/endtoend/types:go_default_library",

testing/endtoend/components/beacon_node.go

@@ -3,14 +3,18 @@
 package components
 
 import (
+	"bytes"
 	"context"
-	"errors"
+	"github.com/pkg/errors"
 	"fmt"
+	"github.com/prysmaticlabs/prysm/testing/endtoend/e2ez"
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
+	"text/template"
 
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
 	cmdshared "github.com/prysmaticlabs/prysm/cmd"
@@ -24,7 +28,6 @@ import (
 
 var _ e2etypes.ComponentRunner = (*BeaconNode)(nil)
 var _ e2etypes.ComponentRunner = (*BeaconNodeSet)(nil)
-var _ e2etypes.BeaconNodeSet = (*BeaconNodeSet)(nil)
 
 // BeaconNodeSet represents set of beacon nodes.
 type BeaconNodeSet struct {
@@ -33,18 +36,24 @@ type BeaconNodeSet struct {
 	enr     string
 	ids     []string
 	started chan struct{}
-}
-
-// SetENR assigns ENR to the set of beacon nodes.
-func (s *BeaconNodeSet) SetENR(enr string) {
-	s.enr = enr
+	nodes []*BeaconNode
 }
 
 // NewBeaconNodes creates and returns a set of beacon nodes.
-func NewBeaconNodes(config *e2etypes.E2EConfig) *BeaconNodeSet {
+func NewBeaconNodes(config *e2etypes.E2EConfig, enr string) *BeaconNodeSet {
+	// Create beacon nodes.
+	//nodes := make([]e2etypes.ComponentRunner, e2e.TestParams.BeaconNodeCount)
+	nodes := make([]*BeaconNode, e2e.TestParams.BeaconNodeCount)
+	for i := 0; i < e2e.TestParams.BeaconNodeCount; i++ {
+		nodes[i] = NewBeaconNode(config, i, enr)
+		//nodes[i] = s.nodes[i]
+	}
+
 	return &BeaconNodeSet{
 		config:  config,
+		nodes: nodes,
 		started: make(chan struct{}, 1),
+		enr: enr,
 	}
 }
 
@@ -54,10 +63,9 @@ func (s *BeaconNodeSet) Start(ctx context.Context) error {
 		return errors.New("empty ENR")
 	}
 
-	// Create beacon nodes.
-	nodes := make([]e2etypes.ComponentRunner, e2e.TestParams.BeaconNodeCount)
-	for i := 0; i < e2e.TestParams.BeaconNodeCount; i++ {
-		nodes[i] = NewBeaconNode(s.config, i, s.enr)
+	nodes := make([]e2etypes.ComponentRunner, len(s.nodes))
+	for i, n := range s.nodes {
+		nodes[i] = n
 	}
 
 	// Wait for all nodes to finish their job (blocking).
@@ -74,6 +82,31 @@ func (s *BeaconNodeSet) Start(ctx context.Context) error {
 	})
 }
 
+func (s *BeaconNodeSet) ZPath() string {
+	return "/beacon-nodes"
+}
+
+func (s *BeaconNodeSet) ZMarkdown() (string, error) {
+	tmpl := `
+%d beacon nodes
+---------------
+
+%s`
+	nodeList := ""
+	for _, node := range s.nodes {
+		nodeList = nodeList + fmt.Sprintf("\n - [beacon node #%d](%s)", node.index, node.ZPath())
+	}
+	return fmt.Sprintf(tmpl, len(s.nodes), nodeList), nil
+}
+
+func (s *BeaconNodeSet) ZChildren() []e2ez.ZPage {
+	zps := make([]e2ez.ZPage, len(s.nodes))
+	for i := 0; i < len(s.nodes); i++ {
+		zps[i] = s.nodes[i]
+	}
+	return zps
+}
+
 // Started checks whether beacon node set is started and all nodes are ready to be queried.
 func (s *BeaconNodeSet) Started() <-chan struct{} {
 	return s.started
@@ -89,6 +122,62 @@ type BeaconNode struct {
 	peerID  string
 }
 
+func (node *BeaconNode) ZPath() string {
+	return fmt.Sprintf("/beacon-node/%d", node.index)
+}
+
+var bnzm = template.Must(template.New("BeaconNode.ZMarkdown").Parse("" +
+	"beacon node {{.Index}}\n" +
+	"--------------\n\n" +
+	"```\n" +
+	"{{.StartCmd}}" +
+	"```\n\n" +
+	"http addr={{.HTTPAddr}}\n\n" +
+	"grpc addr={{.GRPCAddr}}\n\n" +
+	"db path={{.DBPath}}\n\n" +
+	"log path={{.LogPath}}\n\n" +
+	"stdout path={{.StdoutPath}}\n\n" +
+	"stderr path={{.StderrPath}}\n\n"))
+
+func (node *BeaconNode) ZMarkdown() (string, error) {
+	bin, args, err := node.startCommand()
+	if err != nil {
+		return "", err
+	}
+	cmd := path.Join(bin, args[0])
+	for _, a := range args {
+		cmd += fmt.Sprintf("\n%s \\", a)
+	}
+
+	buf := bytes.NewBuffer(nil)
+	err = bnzm.Execute(buf, struct{
+		Index int
+		StartCmd string
+		DBPath string
+		LogPath string
+		StdoutPath string
+		StderrPath string
+		HTTPAddr string
+		GRPCAddr string
+	}{
+		Index: node.index,
+		StartCmd: cmd,
+		DBPath: node.dbPath(),
+		LogPath: node.logPath(),
+		StdoutPath: node.stdoutPath(),
+		StderrPath: node.stderrPath(),
+		HTTPAddr: node.httpAddr(),
+		GRPCAddr: node.grpcAddr(),
+	})
+	return buf.String(), err
+}
+
+func (node *BeaconNode) ZChildren() []e2ez.ZPage {
+	return []e2ez.ZPage{}
+}
+
+var _ e2ez.ZPage = &BeaconNode{}
+
 // NewBeaconNode creates and returns a beacon node.
 func NewBeaconNode(config *e2etypes.E2EConfig, index int, enr string) *BeaconNode {
 	return &BeaconNode{
@@ -99,19 +188,13 @@ func NewBeaconNode(config *e2etypes.E2EConfig, index int, enr string) *BeaconNod
 	}
 }
 
-// Start starts a fresh beacon node, connecting to all passed in beacon nodes.
-func (node *BeaconNode) Start(ctx context.Context) error {
+func (node *BeaconNode) startCommand() (string, []string, error) {
 	binaryPath, found := bazel.FindBinary("cmd/beacon-chain", "beacon-chain")
 	if !found {
 		log.Info(binaryPath)
-		return errors.New("beacon chain binary not found")
+		return "", []string{}, errors.New("beacon chain binary not found")
 	}
-
 	config, index, enr := node.config, node.index, node.enr
-	stdOutFile, err := helpers.DeleteAndCreateFile(e2e.TestParams.LogPath, fmt.Sprintf(e2e.BeaconNodeLogFileName, index))
-	if err != nil {
-		return err
-	}
 	expectedNumOfPeers := e2e.TestParams.BeaconNodeCount + e2e.TestParams.LighthouseBeaconNodeCount - 1
 	if node.config.TestSync {
 		expectedNumOfPeers += 1
@@ -122,8 +205,8 @@ func (node *BeaconNode) Start(ctx context.Context) error {
 	}
 	jwtPath = path.Join(jwtPath, "geth/jwtsecret")
 	args := []string{
-		fmt.Sprintf("--%s=%s/eth2-beacon-node-%d", cmdshared.DataDirFlag.Name, e2e.TestParams.TestPath, index),
-		fmt.Sprintf("--%s=%s", cmdshared.LogFileName.Name, stdOutFile.Name()),
+		fmt.Sprintf("--%s=%s", cmdshared.DataDirFlag.Name, node.dbPath()),
+		fmt.Sprintf("--%s=%s", cmdshared.LogFileName.Name, node.logPath()),
 		fmt.Sprintf("--%s=%s", flags.DepositContractFlag.Name, e2e.TestParams.ContractAddress.Hex()),
 		fmt.Sprintf("--%s=%d", flags.RPCPort.Name, e2e.TestParams.Ports.PrysmBeaconNodeRPCPort+index),
 		fmt.Sprintf("--%s=http://127.0.0.1:%d", flags.HTTPWeb3ProviderFlag.Name, e2e.TestParams.Ports.Eth1RPCPort+index),
@@ -154,13 +237,53 @@ func (node *BeaconNode) Start(ctx context.Context) error {
 	}
 	args = append(args, config.BeaconFlags...)
 
-	cmd := exec.CommandContext(ctx, binaryPath, args...) // #nosec G204 -- Safe
-	// Write stdout and stderr to log files.
-	stdout, err := os.Create(path.Join(e2e.TestParams.LogPath, fmt.Sprintf("beacon_node_%d_stdout.log", index)))
+	return binaryPath, args, nil
+}
+
+func (node *BeaconNode) dbPath() string {
+	return fmt.Sprintf("%s/eth2-beacon-node-%d", e2e.TestParams.TestPath, node.index)
+}
+
+func (node *BeaconNode) logPath() string {
+	return filepath.Clean(path.Join(e2e.TestParams.LogPath, fmt.Sprintf(e2e.BeaconNodeLogFileName, node.index)))
+}
+
+func (node *BeaconNode) stdoutPath() string {
+	return path.Join(e2e.TestParams.LogPath, fmt.Sprintf("beacon_node_%d_stdout.log", node.index))
+}
+
+func (node *BeaconNode) stderrPath() string {
+	return path.Join(e2e.TestParams.LogPath, fmt.Sprintf("beacon_node_%d_stderr.log", node.index))
+}
+
+func (node *BeaconNode) httpAddr() string {
+	port := e2e.TestParams.Ports.PrysmBeaconNodeGatewayPort+node.index
+	return fmt.Sprintf("http://localhost:%d", port)
+}
+
+func (node *BeaconNode) grpcAddr() string {
+	port := e2e.TestParams.Ports.PrysmBeaconNodeRPCPort+node.index
+	return fmt.Sprintf("localhost:%d", port)
+}
+
+// Start starts a fresh beacon node, connecting to all passed in beacon nodes.
+func (node *BeaconNode) Start(ctx context.Context) error {
+	stdOutFile, err := helpers.DeleteAndCreateFile(node.logPath(), "")
 	if err != nil {
 		return err
 	}
-	stderr, err := os.Create(path.Join(e2e.TestParams.LogPath, fmt.Sprintf("beacon_node_%d_stderr.log", index)))
+
+	bin, args, err := node.startCommand()
+	if err != nil {
+		return errors.Wrap(err, "filed to generate start command")
+	}
+	cmd := exec.CommandContext(ctx, bin, args...) // #nosec G204 -- Safe
+	// Write stdout and stderr to log files.
+	stdout, err := os.Create(node.stdoutPath())
+	if err != nil {
+		return err
+	}
+	stderr, err := os.Create(node.stderrPath())
 	if err != nil {
 		return err
 	}
@@ -174,16 +297,16 @@ func (node *BeaconNode) Start(ctx context.Context) error {
 	}()
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
-	log.Infof("Starting beacon chain %d with flags: %s", index, strings.Join(args[2:], " "))
+	log.Infof("Starting beacon chain %d with flags: %s", node.index, strings.Join(args[2:], " "))
 	if err = cmd.Start(); err != nil {
 		return fmt.Errorf("failed to start beacon node: %w", err)
 	}
 
 	if err = helpers.WaitForTextInFile(stdOutFile, "gRPC server listening on port"); err != nil {
-		return fmt.Errorf("could not find multiaddr for node %d, this means the node had issues starting: %w", index, err)
+		return fmt.Errorf("could not find multiaddr for node %d, this means the node had issues starting: %w", node.index, err)
 	}
 
-	if config.UseFixedPeerIDs {
+	if node.config.UseFixedPeerIDs {
 		peerId, err := helpers.FindFollowingTextInFile(stdOutFile, "Running node with peer id of ")
 		if err != nil {
 			return fmt.Errorf("could not find peer id: %w", err)

testing/endtoend/components/lighthouse_beacon.go

@@ -1,15 +1,18 @@
 package components
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
+	"github.com/prysmaticlabs/prysm/testing/endtoend/e2ez"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
+	"text/template"
 
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
 	"github.com/prysmaticlabs/prysm/config/params"
@@ -21,7 +24,6 @@ import (
 
 var _ e2etypes.ComponentRunner = (*LighthouseBeaconNode)(nil)
 var _ e2etypes.ComponentRunner = (*LighthouseBeaconNodeSet)(nil)
-var _ e2etypes.BeaconNodeSet = (*LighthouseBeaconNodeSet)(nil)
 
 // LighthouseBeaconNodeSet represents set of lighthouse beacon nodes.
 type LighthouseBeaconNodeSet struct {
@@ -29,18 +31,20 @@ type LighthouseBeaconNodeSet struct {
 	config  *e2etypes.E2EConfig
 	enr     string
 	started chan struct{}
-}
-
-// SetENR assigns ENR to the set of beacon nodes.
-func (s *LighthouseBeaconNodeSet) SetENR(enr string) {
-	s.enr = enr
+	nodes []*LighthouseBeaconNode
 }
 
 // NewLighthouseBeaconNodes creates and returns a set of lighthouse beacon nodes.
-func NewLighthouseBeaconNodes(config *e2etypes.E2EConfig) *LighthouseBeaconNodeSet {
+func NewLighthouseBeaconNodes(config *e2etypes.E2EConfig, enr string) *LighthouseBeaconNodeSet {
+	nodes := make([]*LighthouseBeaconNode, e2e.TestParams.LighthouseBeaconNodeCount)
+	for i := 0; i < e2e.TestParams.LighthouseBeaconNodeCount; i++ {
+		nodes[i] = NewLighthouseBeaconNode(config, i, enr)
+	}
 	return &LighthouseBeaconNodeSet{
 		config:  config,
 		started: make(chan struct{}, 1),
+		enr: enr,
+		nodes: nodes,
 	}
 }
 
@@ -51,9 +55,9 @@ func (s *LighthouseBeaconNodeSet) Start(ctx context.Context) error {
 	}
 
 	// Create beacon nodes.
-	nodes := make([]e2etypes.ComponentRunner, e2e.TestParams.LighthouseBeaconNodeCount)
+	nodes := make([]e2etypes.ComponentRunner, len(s.nodes))
 	for i := 0; i < e2e.TestParams.LighthouseBeaconNodeCount; i++ {
-		nodes[i] = NewLighthouseBeaconNode(s.config, i, s.enr)
+		nodes[i] = s.nodes[i]
 	}
 
 	// Wait for all nodes to finish their job (blocking).
@@ -69,6 +73,32 @@ func (s *LighthouseBeaconNodeSet) Started() <-chan struct{} {
 	return s.started
 }
 
+func (s *LighthouseBeaconNodeSet) ZPath() string {
+	return "/lh-beacon-nodes"
+}
+
+func (s *LighthouseBeaconNodeSet) ZMarkdown() (string, error) {
+	tmpl := `
+%d beacon nodes
+---------------
+
+%s`
+	nodeList := ""
+	for _, node := range s.nodes {
+		nodeList = nodeList + fmt.Sprintf("\n - [beacon node #%d](%s)", node.index, node.ZPath())
+	}
+	return fmt.Sprintf(tmpl, len(s.nodes), nodeList), nil
+}
+
+func (s *LighthouseBeaconNodeSet) ZChildren() []e2ez.ZPage {
+	zps := make([]e2ez.ZPage, len(s.nodes))
+	for i := 0; i < len(s.nodes); i++ {
+		zps[i] = s.nodes[i]
+	}
+	return zps
+}
+
+
 // LighthouseBeaconNode represents a lighthouse beacon node.
 type LighthouseBeaconNode struct {
 	e2etypes.ComponentRunner
@@ -88,39 +118,51 @@ func NewLighthouseBeaconNode(config *e2etypes.E2EConfig, index int, enr string)
 	}
 }
 
-// Start starts a fresh beacon node, connecting to all passed in beacon nodes.
-func (node *LighthouseBeaconNode) Start(ctx context.Context) error {
+func (node *LighthouseBeaconNode) dbPath() string {
+	return fmt.Sprintf("%s/lighthouse-beacon-node-%d", e2e.TestParams.TestPath, node.index)
+}
+
+func (node *LighthouseBeaconNode) stdoutPath() string {
+	return path.Join(e2e.TestParams.LogPath, fmt.Sprintf("lighthouse_beacon_node_%d_stdout.log", node.index))
+}
+
+func (node *LighthouseBeaconNode) stderrPath() string {
+	return path.Join(e2e.TestParams.LogPath, fmt.Sprintf("lighthouse_beacon_node_%d_stderr.log", node.index))
+}
+
+func (node *LighthouseBeaconNode) httpPort() int {
+	return e2e.TestParams.Ports.LighthouseBeaconNodeHTTPPort+node.index
+}
+
+func (node *LighthouseBeaconNode) startCommand() (string, []string, error) {
 	binaryPath, found := bazel.FindBinary("external/lighthouse", "lighthouse")
 	if !found {
 		log.Info(binaryPath)
 		log.Error("beacon chain binary not found")
 	}
-
-	_, index, _ := node.config, node.index, node.enr
-	testDir, err := node.createTestnetDir(index)
+	testDir, err := node.createTestnetDir(node.index)
 	if err != nil {
-		return err
+		return "", []string{}, err
 	}
-
 	prysmNodeCount := e2e.TestParams.BeaconNodeCount
 	jwtPath := path.Join(e2e.TestParams.TestPath, "eth1data/"+strconv.Itoa(node.index+prysmNodeCount)+"/")
 	jwtPath = path.Join(jwtPath, "geth/jwtsecret")
 	args := []string{
 		"beacon_node",
-		fmt.Sprintf("--datadir=%s/lighthouse-beacon-node-%d", e2e.TestParams.TestPath, index),
+		fmt.Sprintf("--datadir=%s", node.dbPath()),
 		fmt.Sprintf("--testnet-dir=%s", testDir),
 		"--staking",
 		"--enr-address=127.0.0.1",
-		fmt.Sprintf("--enr-udp-port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeP2PPort+index),
-		fmt.Sprintf("--enr-tcp-port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeP2PPort+index),
-		fmt.Sprintf("--port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeP2PPort+index),
-		fmt.Sprintf("--http-port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeHTTPPort+index),
+		fmt.Sprintf("--enr-udp-port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeP2PPort+node.index),
+		fmt.Sprintf("--enr-tcp-port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeP2PPort+node.index),
+		fmt.Sprintf("--port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeP2PPort+node.index),
+		fmt.Sprintf("--http-port=%d", node.httpPort()),
 		fmt.Sprintf("--target-peers=%d", 10),
-		fmt.Sprintf("--eth1-endpoints=http://127.0.0.1:%d", e2e.TestParams.Ports.Eth1RPCPort+prysmNodeCount+index),
-		fmt.Sprintf("--execution-endpoints=http://127.0.0.1:%d", e2e.TestParams.Ports.Eth1AuthRPCPort+prysmNodeCount+index),
+		fmt.Sprintf("--eth1-endpoints=http://127.0.0.1:%d", e2e.TestParams.Ports.Eth1RPCPort+prysmNodeCount+node.index),
+		fmt.Sprintf("--execution-endpoints=http://127.0.0.1:%d", e2e.TestParams.Ports.Eth1AuthRPCPort+prysmNodeCount+node.index),
 		fmt.Sprintf("--jwt-secrets=%s", jwtPath),
 		fmt.Sprintf("--boot-nodes=%s", node.enr),
-		fmt.Sprintf("--metrics-port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeMetricsPort+index),
+		fmt.Sprintf("--metrics-port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeMetricsPort+node.index),
 		"--metrics",
 		"--http",
 		"--http-allow-sync-stalled",
@@ -133,13 +175,22 @@ func (node *LighthouseBeaconNode) Start(ctx context.Context) error {
 		args = append(args,
 			fmt.Sprintf("--trusted-peers=%s", flagVal))
 	}
-	cmd := exec.CommandContext(ctx, binaryPath, args...) /* #nosec G204 */
-	// Write stdout and stderr to log files.
-	stdout, err := os.Create(path.Join(e2e.TestParams.LogPath, fmt.Sprintf("lighthouse_beacon_node_%d_stdout.log", index)))
+	return binaryPath, args, nil
+}
+
+// Start starts a fresh beacon node, connecting to all passed in beacon nodes.
+func (node *LighthouseBeaconNode) Start(ctx context.Context) error {
+	binaryPath, args, err := node.startCommand()
 	if err != nil {
 		return err
 	}
-	stderr, err := os.Create(path.Join(e2e.TestParams.LogPath, fmt.Sprintf("lighthouse_beacon_node_%d_stderr.log", index)))
+	cmd := exec.CommandContext(ctx, binaryPath, args...) /* #nosec G204 */
+	// Write stdout and stderr to log files.
+	stdout, err := os.Create(node.stdoutPath())
+	if err != nil {
+		return err
+	}
+	stderr, err := os.Create(node.stderrPath())
 	if err != nil {
 		return err
 	}
@@ -153,13 +204,13 @@ func (node *LighthouseBeaconNode) Start(ctx context.Context) error {
 	}()
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
-	log.Infof("Starting lighthouse beacon chain %d with flags: %s", index, strings.Join(args[2:], " "))
+	log.Infof("Starting lighthouse beacon chain %d with flags: %s", node.index, strings.Join(args[2:], " "))
 	if err = cmd.Start(); err != nil {
 		return fmt.Errorf("failed to start beacon node: %w", err)
 	}
 
 	if err = helpers.WaitForTextInFile(stderr, "Configured for network"); err != nil {
-		return fmt.Errorf("could not find initialization for node %d, this means the node had issues starting: %w", index, err)
+		return fmt.Errorf("could not find initialization for node %d, this means the node had issues starting: %w", node.index, err)
 	}
 
 	// Mark node as ready.
@@ -196,3 +247,53 @@ func (node *LighthouseBeaconNode) createTestnetDir(index int) (string, error) {
 	deployYaml := []byte("0")
 	return testNetDir, file.WriteFile(deployPath, deployYaml)
 }
+
+func (node *LighthouseBeaconNode) ZPath() string {
+	return fmt.Sprintf("/lh-beacon-node/%d", node.index)
+}
+
+var lbnzm = template.Must(template.New("BeaconNode.ZMarkdown").Parse("" +
+	"beacon node {{.Index}}\n" +
+	"--------------\n\n" +
+	"http addr={{.HTTPAddr}}\n\n" +
+	"db path={{.DBPath}}\n\n" +
+	"stdout path={{.StdoutPath}}\n\n" +
+	"stderr path={{.StderrPath}}\n\n" +
+	"```\n" +
+	"{{.StartCmd}}" +
+	"```\n\n"))
+
+func (node *LighthouseBeaconNode) ZMarkdown() (string, error) {
+	bin, args, err := node.startCommand()
+	if err != nil {
+		return "", err
+	}
+	cmd := path.Join(bin, args[0])
+	for _, a := range args {
+		cmd += fmt.Sprintf("\n%s \\", a)
+	}
+
+	buf := bytes.NewBuffer(nil)
+	err = lbnzm.Execute(buf, struct{
+		Index int
+		StartCmd string
+		DBPath string
+		StdoutPath string
+		StderrPath string
+		HTTPAddr string
+	}{
+		Index: node.index,
+		StartCmd: cmd,
+		DBPath: node.dbPath(),
+		StdoutPath: node.stdoutPath(),
+		StderrPath: node.stderrPath(),
+		HTTPAddr: fmt.Sprintf("http://localhost:%d", node.httpPort()),
+	})
+	return buf.String(), err
+}
+
+func (node *LighthouseBeaconNode) ZChildren() []e2ez.ZPage {
+	return []e2ez.ZPage{}
+}
+
+var _ e2ez.ZPage = &LighthouseBeaconNode{}

testing/endtoend/e2ez/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["server.go"],
+    importpath = "github.com/prysmaticlabs/prysm/testing/endtoend/e2ez",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_gomarkdown_markdown//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+    ],
+)

testing/endtoend/e2ez/server.go

@@ -0,0 +1,115 @@
+package e2ez
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+
+	"github.com/gomarkdown/markdown"
+	log "github.com/sirupsen/logrus"
+)
+
+// Server is an http server that serves all zpages.
+// if zpages register using the HandleMarkdown method, their responses will be transformed from markdown
+// to html before being streamed back to the client.
+type Server struct {
+	handler *http.ServeMux
+	ec      chan error
+}
+
+// NewServer should be used to instantiate a Server, so that it can set up the internal http.Handler
+// and http.Server values.
+func NewServer() *Server {
+	return &Server{
+		handler: http.NewServeMux(),
+		ec:      make(chan error),
+	}
+}
+
+// ListenAndServe just starts the underlying http.Server using the provided addr.
+// This method does not use a goroutine and will block, call it in a goroutine
+// if you do not want the caller to block.
+func (s *Server) ListenAndServe(ctx context.Context, addr string) {
+	srv := &http.Server{
+		Addr:    addr,
+		Handler: s.handler,
+	}
+	go func() {
+		if err := srv.ListenAndServe(); err != nil {
+			s.ec <- err
+		}
+	}()
+	for {
+		select {
+		case err := <-s.ec:
+			log.Error(err)
+		case <-ctx.Done():
+			err := srv.Shutdown(ctx)
+			if err != nil {
+				log.Error(err)
+			}
+		}
+	}
+}
+
+// HandleMarkdown mirrors http.HandleFunc. It wraps the given handler in a "middleware" enclosure that assumes
+// a successful response body is a markdown document, translating the markdown to an html page.
+func (s *Server) HandleMarkdown(pattern string, handler func(http.ResponseWriter, *http.Request)) {
+	s.handler.HandleFunc(pattern, handleMarkdown(handler, s.ec))
+}
+
+// HandleZPage allows any type that implements the minimal ZPage interface to
+// handle requests for information about itself.
+func (s *Server) HandleZPages(zps ...ZPage)  {
+	for i := 0; i < len(zps); i++ {
+		z := zps[i]
+		f := func(rw http.ResponseWriter, req *http.Request) {
+			md, err := z.ZMarkdown()
+			if err != nil {
+				rw.WriteHeader(http.StatusInternalServerError)
+				s.ec <- err
+				return
+			}
+			rw.Header().Add("Content-Type", "text/html")
+			rw.WriteHeader(http.StatusOK)
+			_, err = rw.Write(markdown.ToHTML([]byte(md), nil, nil))
+			if err != nil {
+				s.ec <- err
+				return
+			}
+		}
+		s.handler.HandleFunc(z.ZPath(), f)
+		s.HandleZPages(z.ZChildren()...)
+	}
+}
+
+type markdownResponseWriter struct {
+	http.ResponseWriter
+	buf *bytes.Buffer
+}
+
+func (w *markdownResponseWriter) Write(i []byte) (int, error) {
+	return w.buf.Write(i)
+}
+
+func handleMarkdown(wrapped http.HandlerFunc, ec chan error) http.HandlerFunc {
+	return func(rw http.ResponseWriter, req *http.Request) {
+		w := &markdownResponseWriter{ResponseWriter: rw, buf: bytes.NewBuffer(nil)}
+		wrapped(w, req)
+		hb := markdown.ToHTML(w.buf.Bytes(), nil, nil)
+		_, err := rw.Write(hb)
+		if err != nil {
+			ec <- err
+			return
+		}
+	}
+}
+
+// ZPage allows a type to generate a markdown zpage without consideration for http server semantics.
+// ZPath() is used to claim a path for itself in the zpage namespace, ZMarkdown returns the markdown
+// value to translate to HTML.
+type ZPage interface {
+	ZPath() string
+	ZMarkdown() (string, error)
+	ZChildren() []ZPage
+}

testing/endtoend/endtoend_test.go

@@ -7,6 +7,7 @@ package endtoend
 import (
 	"context"
 	"fmt"
+	"github.com/prysmaticlabs/prysm/testing/endtoend/e2ez"
 	"os"
 	"path"
 	"strings"
@@ -50,6 +51,7 @@ func init() {
 type testRunner struct {
 	t      *testing.T
 	config *e2etypes.E2EConfig
+	z      *e2ez.Server
 }
 
 // newTestRunner creates E2E test runner.
@@ -57,9 +59,30 @@ func newTestRunner(t *testing.T, config *e2etypes.E2EConfig) *testRunner {
 	return &testRunner{
 		t:      t,
 		config: config,
+		z:      e2ez.NewServer(),
 	}
 }
 
+type zPageMenu struct {}
+
+func (z *zPageMenu) ZPath() string {
+	return "/"
+}
+
+func (z *zPageMenu) ZMarkdown() (string, error) {
+	return `
+e2e admin
+===========
+
+- [prysm beacon nodes](/beacon-nodes)
+- [lh beacon nodes](/lh-beacon-nodes)
+`, nil
+}
+
+func (z *zPageMenu) ZChildren() []e2ez.ZPage {
+	return []e2ez.ZPage{}
+}
+
 // run executes configured E2E test.
 func (r *testRunner) run() {
 	t, config := r.t, r.config
@@ -67,18 +90,26 @@ func (r *testRunner) run() {
 	t.Logf("Starting time: %s\n", time.Now().String())
 	t.Logf("Log Path: %s\n", e2e.TestParams.LogPath)
 
+	if e2e.TestParams.ZPageAddr == "" {
+		e2e.TestParams.ZPageAddr = ":8080"
+	}
+
 	minGenesisActiveCount := int(params.BeaconConfig().MinGenesisActiveValidatorCount)
 	multiClientActive := e2e.TestParams.LighthouseBeaconNodeCount > 0
 	var keyGen, lighthouseValidatorNodes e2etypes.ComponentRunner
 	var lighthouseNodes *components.LighthouseBeaconNodeSet
 
 	ctx, done := context.WithCancel(context.Background())
+	defer done()
 	g, ctx := errgroup.WithContext(ctx)
 
 	tracingSink := components.NewTracingSink(config.TracingSinkEndpoint)
 	g.Go(func() error {
 		return tracingSink.Start(ctx)
 	})
+	zp := e2ez.NewServer()
+	zp.HandleZPages(&zPageMenu{})
+	go zp.ListenAndServe(ctx, e2e.TestParams.ZPageAddr)
 
 	if multiClientActive {
 		keyGen = components.NewKeystoreGenerator()
@@ -133,19 +164,6 @@ func (r *testRunner) run() {
 		return nil
 	})
 
-	// Beacon nodes.
-	beaconNodes := components.NewBeaconNodes(config)
-	g.Go(func() error {
-		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Nodes, bootNode}); err != nil {
-			return errors.Wrap(err, "beacon nodes require ETH1 and boot node to run")
-		}
-		beaconNodes.SetENR(bootNode.ENR())
-		if err := beaconNodes.Start(ctx); err != nil {
-			return errors.Wrap(err, "failed to start beacon nodes")
-		}
-		return nil
-	})
-
 	// Web3 remote signer.
 	var web3RemoteSigner *components.Web3RemoteSigner
 	if config.UseWeb3RemoteSigner {
@@ -158,13 +176,29 @@ func (r *testRunner) run() {
 		})
 	}
 
+	// Beacon nodes.
+	if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{bootNode}); err != nil {
+		t.Fatal(err, errors.Wrap(err, "beacon nodes require ETH1 and boot node to run"))
+	}
+	beaconNodes := components.NewBeaconNodes(config, bootNode.ENR())
+	zp.HandleZPages(beaconNodes)
+	g.Go(func() error {
+		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Nodes, bootNode}); err != nil {
+			t.Fatal(err, errors.Wrap(err, "beacon nodes require ETH1 and boot node to run"))
+		}
+		if err := beaconNodes.Start(ctx); err != nil {
+			return errors.Wrap(err, "failed to start beacon nodes")
+		}
+		return nil
+	})
+
 	if multiClientActive {
-		lighthouseNodes = components.NewLighthouseBeaconNodes(config)
+		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Nodes, bootNode, beaconNodes}); err != nil {
+			t.Fatal(errors.Wrap(err, "lighthouse beacon nodes require ETH1 and boot node to run"))
+		}
+		lighthouseNodes = components.NewLighthouseBeaconNodes(config, bootNode.ENR())
+		zp.HandleZPages(lighthouseNodes)
 		g.Go(func() error {
-			if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Nodes, bootNode, beaconNodes}); err != nil {
-				return errors.Wrap(err, "lighthouse beacon nodes require ETH1 and boot node to run")
-			}
-			lighthouseNodes.SetENR(bootNode.ENR())
 			if err := lighthouseNodes.Start(ctx); err != nil {
 				return errors.Wrap(err, "failed to start lighthouse beacon nodes")
 			}

testing/endtoend/params/params.go

@@ -22,6 +22,7 @@ type params struct {
 	LighthouseBeaconNodeCount int
 	ContractAddress           common.Address
 	Ports                     *ports
+	ZPageAddr                 string
 }
 
 type ports struct {

testing/endtoend/types/types.go

@@ -43,11 +43,3 @@ type ComponentRunner interface {
 	// Started checks whether an underlying component is started and ready to be queried.
 	Started() <-chan struct{}
 }
-
-// BeaconNodeSet defines an interface for an object that fulfills the duties
-// of a group of beacon nodes.
-type BeaconNodeSet interface {
-	ComponentRunner
-	// SetENR provides the relevant bootnode's enr to the beacon nodes.
-	SetENR(enr string)
-}