Run multiple go-ethereum nodes in e2e (#10277)

Co-authored-by: Nishant Das <nishdas93@gmail.com>

testing/endtoend/BUILD.bazel

@@ -40,6 +40,7 @@ go_test(
         "//proto/prysm/v1alpha1:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/endtoend/components:go_default_library",
+        "//testing/endtoend/components/eth1:go_default_library",
         "//testing/endtoend/evaluators:go_default_library",
         "//testing/endtoend/helpers:go_default_library",
         "//testing/endtoend/params:go_default_library",
@@ -93,6 +94,7 @@ go_test(
         "//proto/prysm/v1alpha1:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/endtoend/components:go_default_library",
+        "//testing/endtoend/components/eth1:go_default_library",
         "//testing/endtoend/evaluators:go_default_library",
         "//testing/endtoend/helpers:go_default_library",
         "//testing/endtoend/params:go_default_library",

testing/endtoend/components/BUILD.bazel

@@ -14,7 +14,10 @@ go_library(
         "validator.go",
         "web3remotesigner.go",
     ],
-    data = ["@lighthouse//:lighthouse_bin"],
+    data = [
+        "//testing/endtoend/static-files/eth1:eth1data",
+        "@lighthouse//:lighthouse_bin",
+    ],
     importpath = "github.com/prysmaticlabs/prysm/testing/endtoend/components",
     visibility = ["//testing/endtoend:__subpackages__"],
     deps = [
@@ -29,6 +32,7 @@ go_library(
         "//encoding/bytesutil:go_default_library",
         "//io/file:go_default_library",
         "//runtime/interop:go_default_library",
+        "//testing/endtoend/components/eth1:go_default_library",
         "//testing/endtoend/helpers:go_default_library",
         "//testing/endtoend/params:go_default_library",
         "//testing/endtoend/types:go_default_library",

testing/endtoend/components/beacon_node.go

@@ -112,14 +112,16 @@ func (node *BeaconNode) Start(ctx context.Context) error {
 		return err
 	}
 	expectedNumOfPeers := e2e.TestParams.BeaconNodeCount + e2e.TestParams.LighthouseBeaconNodeCount - 1
-
+	if node.config.TestSync {
+		expectedNumOfPeers += 1
+	}
 	args := []string{
 		fmt.Sprintf("--%s=%s/eth2-beacon-node-%d", cmdshared.DataDirFlag.Name, e2e.TestParams.TestPath, index),
 		fmt.Sprintf("--%s=%s", cmdshared.LogFileName.Name, stdOutFile.Name()),
 		fmt.Sprintf("--%s=%s", flags.DepositContractFlag.Name, e2e.TestParams.ContractAddress.Hex()),
 		fmt.Sprintf("--%s=%d", flags.RPCPort.Name, e2e.TestParams.Ports.PrysmBeaconNodeRPCPort+index),
 		fmt.Sprintf("--%s=http://127.0.0.1:%d", flags.HTTPWeb3ProviderFlag.Name, e2e.TestParams.Ports.Eth1RPCPort),
-		fmt.Sprintf("--%s=%d", flags.MinSyncPeers.Name, e2e.TestParams.BeaconNodeCount-1),
+		fmt.Sprintf("--%s=%d", flags.MinSyncPeers.Name, 1),
 		fmt.Sprintf("--%s=%d", cmdshared.P2PUDPPort.Name, e2e.TestParams.Ports.PrysmBeaconNodeUDPPort+index),
 		fmt.Sprintf("--%s=%d", cmdshared.P2PTCPPort.Name, e2e.TestParams.Ports.PrysmBeaconNodeTCPPort+index),
 		fmt.Sprintf("--%s=%d", cmdshared.P2PMaxPeers.Name, expectedNumOfPeers),

testing/endtoend/components/eth1/BUILD.bazel

@@ -0,0 +1,30 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    testonly = True,
+    srcs = [
+        "helpers.go",
+        "miner.go",
+        "node.go",
+        "node_set.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/testing/endtoend/components/eth1",
+    visibility = ["//testing/endtoend:__subpackages__"],
+    deps = [
+        "//config/params:go_default_library",
+        "//contracts/deposit/mock:go_default_library",
+        "//io/file:go_default_library",
+        "//testing/endtoend/helpers:go_default_library",
+        "//testing/endtoend/params:go_default_library",
+        "//testing/endtoend/types:go_default_library",
+        "@com_github_ethereum_go_ethereum//accounts/abi/bind:go_default_library",
+        "@com_github_ethereum_go_ethereum//accounts/keystore:go_default_library",
+        "@com_github_ethereum_go_ethereum//core/types:go_default_library",
+        "@com_github_ethereum_go_ethereum//ethclient:go_default_library",
+        "@com_github_ethereum_go_ethereum//rpc:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+        "@io_bazel_rules_go//go/tools/bazel:go_default_library",
+    ],
+)

testing/endtoend/components/eth1/helpers.go

@@ -0,0 +1,64 @@
+package eth1
+
+import (
+	"context"
+	"math/big"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethclient"
+	e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types"
+)
+
+// NetworkId is the ID of the ETH1 chain.
+const NetworkId = 1337
+
+// KeystorePassword is the password used to decrypt ETH1 keystores.
+const KeystorePassword = "password"
+
+const MinerPort = 30303
+const minerPasswordFile = "password.txt"
+const minerFile = "UTC--2021-12-22T19-14-08.590377700Z--878705ba3f8bc32fcf7f4caa1a35e72af65cf766"
+const timeGapPerTX = 100 * time.Millisecond
+const staticFilesPath = "/testing/endtoend/static-files/eth1"
+const timeGapPerMiningTX = 250 * time.Millisecond
+
+var _ e2etypes.ComponentRunner = (*NodeSet)(nil)
+var _ e2etypes.ComponentRunner = (*Miner)(nil)
+var _ e2etypes.ComponentRunner = (*Node)(nil)
+
+// WaitForBlocks waits for a certain amount of blocks to be mined by the ETH1 chain before returning.
+func WaitForBlocks(web3 *ethclient.Client, keystore *keystore.Key, blocksToWait uint64) error {
+	nonce, err := web3.PendingNonceAt(context.Background(), keystore.Address)
+	if err != nil {
+		return err
+	}
+	chainID, err := web3.NetworkID(context.Background())
+	if err != nil {
+		return err
+	}
+	block, err := web3.BlockByNumber(context.Background(), nil)
+	if err != nil {
+		return err
+	}
+	finishBlock := block.NumberU64() + blocksToWait
+
+	for block.NumberU64() <= finishBlock {
+		spamTX := types.NewTransaction(nonce, keystore.Address, big.NewInt(0), 21000, big.NewInt(1e6), []byte{})
+		signed, err := types.SignTx(spamTX, types.NewEIP155Signer(chainID), keystore.PrivateKey)
+		if err != nil {
+			return err
+		}
+		if err = web3.SendTransaction(context.Background(), signed); err != nil {
+			return err
+		}
+		nonce++
+		time.Sleep(timeGapPerMiningTX)
+		block, err = web3.BlockByNumber(context.Background(), nil)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

testing/endtoend/components/eth1/miner.go

@@ -0,0 +1,254 @@
+package eth1
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"math/big"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/bazelbuild/rules_go/go/tools/bazel"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/config/params"
+	contracts "github.com/prysmaticlabs/prysm/contracts/deposit/mock"
+	io "github.com/prysmaticlabs/prysm/io/file"
+	"github.com/prysmaticlabs/prysm/testing/endtoend/helpers"
+	e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params"
+	e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types"
+	log "github.com/sirupsen/logrus"
+)
+
+// Miner represents an ETH1 node which mines blocks.
+type Miner struct {
+	e2etypes.ComponentRunner
+	started      chan struct{}
+	bootstrapEnr string
+	enr          string
+	keystorePath string
+}
+
+// NewMiner creates and returns an ETH1 node miner.
+func NewMiner() *Miner {
+	return &Miner{
+		started: make(chan struct{}, 1),
+	}
+}
+
+// KeystorePath returns the path of the keystore file.
+func (m *Miner) KeystorePath() string {
+	return m.keystorePath
+}
+
+// ENR returns the miner's enode.
+func (m *Miner) ENR() string {
+	return m.enr
+}
+
+// SetBootstrapENR sets the bootstrap record.
+func (m *Miner) SetBootstrapENR(bootstrapEnr string) {
+	m.bootstrapEnr = bootstrapEnr
+}
+
+// Start runs a mining ETH1 node.
+// The miner is responsible for moving the ETH1 chain forward and for deploying the deposit contract.
+func (m *Miner) Start(ctx context.Context) error {
+	binaryPath, found := bazel.FindBinary("cmd/geth", "geth")
+	if !found {
+		return errors.New("go-ethereum binary not found")
+	}
+
+	eth1Path := path.Join(e2e.TestParams.TestPath, "eth1data/miner/")
+	// Clear out potentially existing dir to prevent issues.
+	if _, err := os.Stat(eth1Path); !os.IsNotExist(err) {
+		if err = os.RemoveAll(eth1Path); err != nil {
+			return err
+		}
+	}
+
+	genesisSrcPath, err := bazel.Runfile(path.Join(staticFilesPath, "genesis.json"))
+	if err != nil {
+		return err
+	}
+	genesisDstPath := binaryPath[:strings.LastIndex(binaryPath, "/")]
+	cpCmd := exec.CommandContext(ctx, "cp", genesisSrcPath, genesisDstPath) // #nosec G204 -- Safe
+	if err = cpCmd.Start(); err != nil {
+		return err
+	}
+	if err = cpCmd.Wait(); err != nil {
+		return err
+	}
+
+	initCmd := exec.CommandContext(
+		ctx,
+		binaryPath,
+		"init",
+		genesisDstPath+"/genesis.json",
+		fmt.Sprintf("--datadir=%s", eth1Path)) // #nosec G204 -- Safe
+	initFile, err := helpers.DeleteAndCreateFile(e2e.TestParams.LogPath, "eth1-init_miner.log")
+	if err != nil {
+		return err
+	}
+	initCmd.Stderr = initFile
+	if err = initCmd.Start(); err != nil {
+		return err
+	}
+	if err = initCmd.Wait(); err != nil {
+		return err
+	}
+
+	args := []string{
+		fmt.Sprintf("--datadir=%s", eth1Path),
+		fmt.Sprintf("--http.port=%d", e2e.TestParams.Ports.Eth1RPCPort),
+		fmt.Sprintf("--ws.port=%d", e2e.TestParams.Ports.Eth1WSPort),
+		fmt.Sprintf("--bootnodes=%s", m.bootstrapEnr),
+		fmt.Sprintf("--port=%d", e2e.TestParams.Ports.Eth1Port),
+		fmt.Sprintf("--networkid=%d", NetworkId),
+		"--http",
+		"--http.addr=127.0.0.1",
+		"--http.corsdomain=\"*\"",
+		"--http.vhosts=\"*\"",
+		"--rpc.allow-unprotected-txs",
+		"--ws",
+		"--ws.addr=127.0.0.1",
+		"--ws.origins=\"*\"",
+		"--ipcdisable",
+		"--verbosity=4",
+		"--mine",
+		"--unlock=0x878705ba3f8bc32fcf7f4caa1a35e72af65cf766",
+		"--allow-insecure-unlock",
+		fmt.Sprintf("--password=%s", eth1Path+"/keystore/"+minerPasswordFile),
+	}
+
+	keystorePath, err := bazel.Runfile(path.Join(staticFilesPath, minerFile))
+	if err != nil {
+		return err
+	}
+	jsonBytes, err := ioutil.ReadFile(keystorePath) // #nosec G304 -- ReadFile is safe
+	if err != nil {
+		return err
+	}
+	err = io.WriteFile(eth1Path+"/keystore/"+minerFile, jsonBytes)
+	if err != nil {
+		return err
+	}
+	err = io.WriteFile(eth1Path+"/keystore/"+minerPasswordFile, []byte(KeystorePassword))
+	if err != nil {
+		return err
+	}
+
+	runCmd := exec.CommandContext(ctx, binaryPath, args...) // #nosec G204 -- Safe
+	file, err := helpers.DeleteAndCreateFile(e2e.TestParams.LogPath, "eth1_miner.log")
+	if err != nil {
+		return err
+	}
+	runCmd.Stdout = file
+	runCmd.Stderr = file
+	log.Infof("Starting eth1 miner with flags: %s", strings.Join(args[2:], " "))
+
+	if err = runCmd.Start(); err != nil {
+		return fmt.Errorf("failed to start eth1 chain: %w", err)
+	}
+
+	if err = helpers.WaitForTextInFile(file, "Commit new mining work"); err != nil {
+		return fmt.Errorf("mining log not found, this means the eth1 chain had issues starting: %w", err)
+	}
+	if err = helpers.WaitForTextInFile(file, "Started P2P networking"); err != nil {
+		return fmt.Errorf("P2P log not found, this means the eth1 chain had issues starting: %w", err)
+	}
+
+	enode, err := enodeFromLogFile(file.Name())
+	if err != nil {
+		return err
+	}
+	enode = "enode://" + enode + "@127.0.0.1:" + fmt.Sprintf("%d", e2e.TestParams.Ports.Eth1Port)
+	m.enr = enode
+	log.Infof("Communicated enode. Enode is %s", enode)
+
+	// Connect to the started geth dev chain.
+	client, err := rpc.DialHTTP(fmt.Sprintf("http://127.0.0.1:%d", e2e.TestParams.Ports.Eth1RPCPort))
+	if err != nil {
+		return fmt.Errorf("failed to connect to ipc: %w", err)
+	}
+	web3 := ethclient.NewClient(client)
+
+	// Deploy the contract.
+	store, err := keystore.DecryptKey(jsonBytes, KeystorePassword)
+	if err != nil {
+		return err
+	}
+	// Advancing the blocks eth1follow distance to prevent issues reading the chain.
+	if err = WaitForBlocks(web3, store, params.BeaconConfig().Eth1FollowDistance); err != nil {
+		return fmt.Errorf("unable to advance chain: %w", err)
+	}
+	txOpts, err := bind.NewTransactorWithChainID(bytes.NewReader(jsonBytes), KeystorePassword, big.NewInt(NetworkId))
+	if err != nil {
+		return err
+	}
+	nonce, err := web3.PendingNonceAt(ctx, store.Address)
+	if err != nil {
+		return err
+	}
+	txOpts.Nonce = big.NewInt(int64(nonce))
+	txOpts.Context = ctx
+	contractAddr, tx, _, err := contracts.DeployDepositContract(txOpts, web3)
+	if err != nil {
+		return fmt.Errorf("failed to deploy deposit contract: %w", err)
+	}
+	e2e.TestParams.ContractAddress = contractAddr
+
+	// Wait for contract to mine.
+	for pending := true; pending; _, pending, err = web3.TransactionByHash(ctx, tx.Hash()) {
+		if err != nil {
+			return err
+		}
+		time.Sleep(timeGapPerTX)
+	}
+
+	// Advancing the blocks another eth1follow distance to prevent issues reading the chain.
+	if err = WaitForBlocks(web3, store, params.BeaconConfig().Eth1FollowDistance); err != nil {
+		return fmt.Errorf("unable to advance chain: %w", err)
+	}
+
+	// Save keystore path (used for saving and mining deposits).
+	m.keystorePath = keystorePath
+
+	// Mark node as ready.
+	close(m.started)
+
+	return runCmd.Wait()
+}
+
+// Started checks whether ETH1 node is started and ready to be queried.
+func (m *Miner) Started() <-chan struct{} {
+	return m.started
+}
+
+func enodeFromLogFile(name string) (string, error) {
+	byteContent, err := ioutil.ReadFile(name) // #nosec G304
+	if err != nil {
+		return "", err
+	}
+	contents := string(byteContent)
+
+	searchText := "self=enode://"
+	startIdx := strings.Index(contents, searchText)
+	if startIdx == -1 {
+		return "", fmt.Errorf("did not find ENR text in %s", contents)
+	}
+	startIdx += len(searchText)
+	endIdx := strings.Index(contents[startIdx:], "@")
+	if endIdx == -1 {
+		return "", fmt.Errorf("did not find ENR text in %s", contents)
+	}
+	enode := contents[startIdx : startIdx+endIdx]
+	return strings.TrimPrefix(enode, "-"), nil
+}

testing/endtoend/components/eth1/node.go

@@ -0,0 +1,112 @@
+package eth1
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"strconv"
+	"strings"
+
+	"github.com/bazelbuild/rules_go/go/tools/bazel"
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/testing/endtoend/helpers"
+	e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params"
+	e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types"
+	log "github.com/sirupsen/logrus"
+)
+
+// Node represents an ETH1 node.
+type Node struct {
+	e2etypes.ComponentRunner
+	started chan struct{}
+	index   int
+	enr     string
+}
+
+// NewNode creates and returns ETH1 node.
+func NewNode(index int, enr string) *Node {
+	return &Node{
+		started: make(chan struct{}, 1),
+		index:   index,
+		enr:     enr,
+	}
+}
+
+// Start runs a non-mining ETH1 node.
+// To connect to a miner and start working properly, this node should be a part of a NodeSet.
+func (node *Node) Start(ctx context.Context) error {
+	binaryPath, found := bazel.FindBinary("cmd/geth", "geth")
+	if !found {
+		return errors.New("go-ethereum binary not found")
+	}
+
+	eth1Path := path.Join(e2e.TestParams.TestPath, "eth1data/"+strconv.Itoa(node.index)+"/")
+	// Clear out potentially existing dir to prevent issues.
+	if _, err := os.Stat(eth1Path); !os.IsNotExist(err) {
+		if err = os.RemoveAll(eth1Path); err != nil {
+			return err
+		}
+	}
+
+	initCmd := exec.CommandContext(
+		ctx,
+		binaryPath,
+		"init",
+		binaryPath[:strings.LastIndex(binaryPath, "/")]+"/genesis.json",
+		fmt.Sprintf("--datadir=%s", eth1Path)) // #nosec G204 -- Safe
+	initFile, err := helpers.DeleteAndCreateFile(e2e.TestParams.LogPath, "eth1-init_"+strconv.Itoa(node.index)+".log")
+	if err != nil {
+		return err
+	}
+	initCmd.Stderr = initFile
+	if err = initCmd.Start(); err != nil {
+		return err
+	}
+	if err = initCmd.Wait(); err != nil {
+		return err
+	}
+
+	args := []string{
+		fmt.Sprintf("--datadir=%s", eth1Path),
+		fmt.Sprintf("--http.port=%d", e2e.TestParams.Ports.Eth1RPCPort+node.index),
+		fmt.Sprintf("--ws.port=%d", e2e.TestParams.Ports.Eth1WSPort+node.index),
+		fmt.Sprintf("--bootnodes=%s", node.enr),
+		fmt.Sprintf("--port=%d", e2e.TestParams.Ports.Eth1Port+node.index),
+		fmt.Sprintf("--networkid=%d", NetworkId),
+		"--http",
+		"--http.addr=127.0.0.1",
+		"--http.corsdomain=\"*\"",
+		"--http.vhosts=\"*\"",
+		"--rpc.allow-unprotected-txs",
+		"--ws",
+		"--ws.addr=127.0.0.1",
+		"--ws.origins=\"*\"",
+		"--ipcdisable",
+		"--verbosity=4",
+	}
+
+	runCmd := exec.CommandContext(ctx, binaryPath, args...) // #nosec G204 -- Safe
+	file, err := helpers.DeleteAndCreateFile(e2e.TestParams.LogPath, "eth1_"+strconv.Itoa(node.index)+".log")
+	if err != nil {
+		return err
+	}
+	runCmd.Stdout = file
+	runCmd.Stderr = file
+	log.Infof("Starting eth1 node %d with flags: %s", node.index, strings.Join(args[2:], " "))
+
+	if err = runCmd.Start(); err != nil {
+		return fmt.Errorf("failed to start eth1 chain: %w", err)
+	}
+
+	// Mark node as ready.
+	close(node.started)
+
+	return runCmd.Wait()
+}
+
+// Started checks whether ETH1 node is started and ready to be queried.
+func (node *Node) Started() <-chan struct{} {
+	return node.started
+}

testing/endtoend/components/eth1/node_set.go

@@ -0,0 +1,54 @@
+package eth1
+
+import (
+	"context"
+
+	"github.com/prysmaticlabs/prysm/testing/endtoend/helpers"
+	e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params"
+	e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types"
+)
+
+// NodeSet represents a set of Eth1 nodes, none of which is a mining node.
+type NodeSet struct {
+	e2etypes.ComponentRunner
+	started chan struct{}
+	enr     string
+}
+
+// NewNodeSet creates and returns a set of Eth1 nodes.
+func NewNodeSet() *NodeSet {
+	return &NodeSet{
+		started: make(chan struct{}, 1),
+	}
+}
+
+// SetMinerENR sets the miner's enode, used to connect to the miner through P2P.
+func (s *NodeSet) SetMinerENR(enr string) {
+	s.enr = enr
+}
+
+// Start starts all the beacon nodes in set.
+func (s *NodeSet) Start(ctx context.Context) error {
+	// Create Eth1 nodes. The number of nodes is the same as the number of beacon nodes.
+	// We want each beacon node to connect to its own Eth1 node.
+	// We start up one Eth1 node less than the beacon node count because the first
+	// beacon node will connect to the already existing Eth1 miner.
+	nodes := make([]e2etypes.ComponentRunner, e2e.TestParams.BeaconNodeCount-1)
+	for i := 0; i < e2e.TestParams.BeaconNodeCount-1; i++ {
+		// We start indexing nodes from 1 because the miner has an implicit 0 index.
+		node := NewNode(i+1, s.enr)
+		nodes[i] = node
+	}
+
+	// Wait for all nodes to finish their job (blocking).
+	// Once nodes are ready passed in handler function will be called.
+	return helpers.WaitOnNodes(ctx, nodes, func() {
+		// All nodes started, close channel, so that all services waiting on a set, can proceed.
+		close(s.started)
+	})
+}
+
+// Started checks whether beacon node set is started and all nodes are ready to be queried.
+func (s *NodeSet) Started() <-chan struct{} {
+	return s.started
+}

testing/endtoend/components/tracing_sink.go

@@ -10,6 +10,7 @@ import (
 	"os/signal"
 	"syscall"
 
+	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/testing/endtoend/helpers"
 	e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params"
 )
@@ -40,6 +41,9 @@ func NewTracingSink(endpoint string) *TracingSink {
 
 // Start the tracing sink.
 func (ts *TracingSink) Start(_ context.Context) error {
+	if ts.endpoint == "" {
+		return errors.New("empty endpoint provided")
+	}
 	go ts.initializeSink()
 	close(ts.started)
 	return nil

testing/endtoend/components/validator.go

@@ -25,6 +25,7 @@ import (
 	contracts "github.com/prysmaticlabs/prysm/contracts/deposit"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 	"github.com/prysmaticlabs/prysm/runtime/interop"
+	"github.com/prysmaticlabs/prysm/testing/endtoend/components/eth1"
 	"github.com/prysmaticlabs/prysm/testing/endtoend/helpers"
 	e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params"
 	e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types"
@@ -229,11 +230,11 @@ func SendAndMineDeposits(keystorePath string, validatorNum, offset int, partial
 	if err = sendDeposits(web3, keystoreBytes, validatorNum, offset, partial); err != nil {
 		return err
 	}
-	mineKey, err := keystore.DecryptKey(keystoreBytes, "" /*password*/)
+	mineKey, err := keystore.DecryptKey(keystoreBytes, eth1.KeystorePassword)
 	if err != nil {
 		return err
 	}
-	if err = mineBlocks(web3, mineKey, params.BeaconConfig().Eth1FollowDistance); err != nil {
+	if err = eth1.WaitForBlocks(web3, mineKey, params.BeaconConfig().Eth1FollowDistance); err != nil {
 		return fmt.Errorf("failed to mine blocks %w", err)
 	}
 	return nil
@@ -241,7 +242,7 @@ func SendAndMineDeposits(keystorePath string, validatorNum, offset int, partial
 
 // sendDeposits uses the passed in web3 and keystore bytes to send the requested deposits.
 func sendDeposits(web3 *ethclient.Client, keystoreBytes []byte, num, offset int, partial bool) error {
-	txOps, err := bind.NewTransactorWithChainID(bytes.NewReader(keystoreBytes), "" /*password*/, big.NewInt(1337))
+	txOps, err := bind.NewTransactorWithChainID(bytes.NewReader(keystoreBytes), eth1.KeystorePassword, big.NewInt(eth1.NetworkId))
 	if err != nil {
 		return err
 	}

testing/endtoend/endtoend_test.go

@@ -21,6 +21,7 @@ import (
 	eth "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/testing/assert"
 	"github.com/prysmaticlabs/prysm/testing/endtoend/components"
+	"github.com/prysmaticlabs/prysm/testing/endtoend/components/eth1"
 	ev "github.com/prysmaticlabs/prysm/testing/endtoend/evaluators"
 	"github.com/prysmaticlabs/prysm/testing/endtoend/helpers"
 	e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params"
@@ -88,24 +89,6 @@ func (r *testRunner) run() {
 		})
 	}
 
-	// ETH1 node.
-	eth1Node := components.NewEth1Node()
-	g.Go(func() error {
-		if err := eth1Node.Start(ctx); err != nil {
-			return errors.Wrap(err, "failed to start eth1node")
-		}
-		return nil
-	})
-	g.Go(func() error {
-		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Node}); err != nil {
-			return errors.Wrap(err, "sending and mining deposits require ETH1 node to run")
-		}
-		if err := components.SendAndMineDeposits(eth1Node.KeystorePath(), minGenesisActiveCount, 0, true /* partial */); err != nil {
-			return errors.Wrap(err, "failed to send and mine deposits")
-		}
-		return nil
-	})
-
 	// Boot node.
 	bootNode := components.NewBootNode()
 	g.Go(func() error {
@@ -114,10 +97,46 @@ func (r *testRunner) run() {
 		}
 		return nil
 	})
+
+	// ETH1 miner.
+	eth1Miner := eth1.NewMiner()
+	g.Go(func() error {
+		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{bootNode}); err != nil {
+			return errors.Wrap(err, "sending and mining deposits require ETH1 nodes to run")
+		}
+		eth1Miner.SetBootstrapENR(bootNode.ENR())
+		if err := eth1Miner.Start(ctx); err != nil {
+			return errors.Wrap(err, "failed to start the ETH1 miner")
+		}
+		return nil
+	})
+
+	// ETH1 non-mining nodes.
+	eth1Nodes := eth1.NewNodeSet()
+	g.Go(func() error {
+		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Miner}); err != nil {
+			return errors.Wrap(err, "sending and mining deposits require ETH1 nodes to run")
+		}
+		eth1Nodes.SetMinerENR(eth1Miner.ENR())
+		if err := eth1Nodes.Start(ctx); err != nil {
+			return errors.Wrap(err, "failed to start ETH1 nodes")
+		}
+		return nil
+	})
+	g.Go(func() error {
+		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Nodes}); err != nil {
+			return errors.Wrap(err, "sending and mining deposits require ETH1 nodes to run")
+		}
+		if err := components.SendAndMineDeposits(eth1Miner.KeystorePath(), minGenesisActiveCount, 0, true /* partial */); err != nil {
+			return errors.Wrap(err, "failed to send and mine deposits")
+		}
+		return nil
+	})
+
 	// Beacon nodes.
 	beaconNodes := components.NewBeaconNodes(config)
 	g.Go(func() error {
-		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Node, bootNode}); err != nil {
+		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Nodes, bootNode}); err != nil {
 			return errors.Wrap(err, "beacon nodes require ETH1 and boot node to run")
 		}
 		beaconNodes.SetENR(bootNode.ENR())
@@ -142,7 +161,7 @@ func (r *testRunner) run() {
 	if multiClientActive {
 		lighthouseNodes = components.NewLighthouseBeaconNodes(config)
 		g.Go(func() error {
-			if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Node, bootNode, beaconNodes}); err != nil {
+			if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Nodes, bootNode, beaconNodes}); err != nil {
 				return errors.Wrap(err, "lighthouse beacon nodes require ETH1 and boot node to run")
 			}
 			lighthouseNodes.SetENR(bootNode.ENR())
@@ -192,7 +211,7 @@ func (r *testRunner) run() {
 
 		// Wait for all required nodes to start.
 		requiredComponents := []e2etypes.ComponentRunner{
-			tracingSink, eth1Node, bootNode, beaconNodes, validatorNodes,
+			tracingSink, eth1Nodes, bootNode, beaconNodes, validatorNodes,
 		}
 		if multiClientActive {
 			requiredComponents = append(requiredComponents, []e2etypes.ComponentRunner{keyGen, lighthouseNodes, lighthouseValidatorNodes}...)
@@ -224,7 +243,7 @@ func (r *testRunner) run() {
 
 		if config.TestDeposits {
 			log.Info("Running deposit tests")
-			r.testDeposits(ctx, g, eth1Node, []e2etypes.ComponentRunner{beaconNodes})
+			r.testDeposits(ctx, g, eth1Miner.KeystorePath(), []e2etypes.ComponentRunner{beaconNodes})
 		}
 
 		// Create GRPC connection to beacon nodes.
@@ -247,7 +266,7 @@ func (r *testRunner) run() {
 		if !config.TestSync {
 			return nil
 		}
-		if err := r.testBeaconChainSync(ctx, g, conns, tickingStartTime, bootNode.ENR()); err != nil {
+		if err := r.testBeaconChainSync(ctx, g, conns, tickingStartTime, bootNode.ENR(), eth1Miner.ENR()); err != nil {
 			return errors.Wrap(err, "beacon chain sync test failed")
 		}
 		if err := r.testDoppelGangerProtection(ctx); err != nil {
@@ -314,7 +333,7 @@ func (r *testRunner) runEvaluators(conns []*grpc.ClientConn, tickingStartTime ti
 
 // testDeposits runs tests when config.TestDeposits is enabled.
 func (r *testRunner) testDeposits(ctx context.Context, g *errgroup.Group,
-	eth1Node *components.Eth1Node, requiredNodes []e2etypes.ComponentRunner) {
+	keystorePath string, requiredNodes []e2etypes.ComponentRunner) {
 	minGenesisActiveCount := int(params.BeaconConfig().MinGenesisActiveValidatorCount)
 
 	depositCheckValidator := components.NewValidatorNode(r.config, int(e2e.DepositCount), e2e.TestParams.BeaconNodeCount, minGenesisActiveCount)
@@ -323,7 +342,7 @@ func (r *testRunner) testDeposits(ctx context.Context, g *errgroup.Group,
 			return fmt.Errorf("deposit check validator node requires beacon nodes to run: %w", err)
 		}
 		go func() {
-			err := components.SendAndMineDeposits(eth1Node.KeystorePath(), int(e2e.DepositCount), minGenesisActiveCount, false /* partial */)
+			err := components.SendAndMineDeposits(keystorePath, int(e2e.DepositCount), minGenesisActiveCount, false /* partial */)
 			if err != nil {
 				r.t.Fatal(err)
 			}
@@ -334,14 +353,18 @@ func (r *testRunner) testDeposits(ctx context.Context, g *errgroup.Group,
 
 // testBeaconChainSync creates another beacon node, and tests whether it can sync to head using previous nodes.
 func (r *testRunner) testBeaconChainSync(ctx context.Context, g *errgroup.Group,
-	conns []*grpc.ClientConn, tickingStartTime time.Time, enr string) error {
+	conns []*grpc.ClientConn, tickingStartTime time.Time, bootnodeEnr, minerEnr string) error {
 	t, config := r.t, r.config
 	index := e2e.TestParams.BeaconNodeCount
-	syncBeaconNode := components.NewBeaconNode(config, index, enr)
+	ethNode := eth1.NewNode(index, minerEnr)
+	g.Go(func() error {
+		return ethNode.Start(ctx)
+	})
+	syncBeaconNode := components.NewBeaconNode(config, index, bootnodeEnr)
 	g.Go(func() error {
 		return syncBeaconNode.Start(ctx)
 	})
-	if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{syncBeaconNode}); err != nil {
+	if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{ethNode, syncBeaconNode}); err != nil {
 		return fmt.Errorf("sync beacon node not ready: %w", err)
 	}
 	syncConn, err := grpc.Dial(fmt.Sprintf("127.0.0.1:%d", e2e.TestParams.Ports.PrysmBeaconNodeRPCPort+index), grpc.WithInsecure())

testing/endtoend/mainnet_e2e_test.go

@@ -36,7 +36,7 @@ func e2eMainnet(t *testing.T, usePrysmSh bool) {
 		// TODO(#9166): remove this block once v2 changes are live.
 		epochsToRun = helpers.AltairE2EForkEpoch - 1
 	}
-	tracingPort := 9411 + e2eParams.TestParams.TestShardIndex
+	tracingPort := e2eParams.TestParams.Ports.JaegerTracingPort
 	tracingEndpoint := fmt.Sprintf("127.0.0.1:%d", tracingPort)
 	evals := []types.Evaluator{
 		ev.PeersConnect,

testing/endtoend/minimal_e2e_test.go

@@ -59,7 +59,7 @@ func e2eMinimal(t *testing.T, args *testArgs) {
 		// TODO(#9166): remove this block once v2 changes are live.
 		epochsToRun = helpers.AltairE2EForkEpoch - 1
 	}
-	tracingPort := 9411 + e2eParams.TestParams.TestShardIndex
+	tracingPort := e2eParams.TestParams.Ports.JaegerTracingPort
 	tracingEndpoint := fmt.Sprintf("127.0.0.1:%d", tracingPort)
 	evals := []types.Evaluator{
 		ev.PeersConnect,

testing/endtoend/minimal_slashing_e2e_test.go

@@ -1,6 +1,7 @@
 package endtoend
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/config/params"
@@ -14,6 +15,9 @@ func TestEndToEnd_Slasher_MinimalConfig(t *testing.T) {
 	params.UseE2EConfig()
 	require.NoError(t, e2eParams.Init(e2eParams.StandardBeaconCount))
 
+	tracingPort := e2eParams.TestParams.Ports.JaegerTracingPort
+	tracingEndpoint := fmt.Sprintf("127.0.0.1:%d", tracingPort)
+
 	testConfig := &types.E2EConfig{
 		BeaconFlags: []string{
 			"--slasher",
@@ -30,6 +34,7 @@ func TestEndToEnd_Slasher_MinimalConfig(t *testing.T) {
 			ev.InjectDoubleVoteOnEpoch(2),
 			ev.InjectDoubleBlockOnEpoch(2),
 		},
+		TracingSinkEndpoint: tracingEndpoint,
 	}
 
 	newTestRunner(t, testConfig).run()

testing/endtoend/params/params.go

@@ -27,6 +27,7 @@ type params struct {
 type ports struct {
 	BootNodePort                    int
 	BootNodeMetricsPort             int
+	Eth1Port                        int
 	Eth1RPCPort                     int
 	Eth1WSPort                      int
 	PrysmBeaconNodeRPCPort          int
@@ -40,6 +41,7 @@ type ports struct {
 	LighthouseBeaconNodeMetricsPort int
 	ValidatorMetricsPort            int
 	ValidatorGatewayPort            int
+	JaegerTracingPort               int
 }
 
 // TestParams is the globally accessible var for getting config elements.
@@ -73,8 +75,9 @@ const (
 	BootNodePort        = 2150
 	BootNodeMetricsPort = BootNodePort + portSpan
 
-	Eth1RPCPort = 3150
-	Eth1WSPort  = Eth1RPCPort + portSpan
+	Eth1Port    = 3150
+	Eth1RPCPort = Eth1Port + portSpan
+	Eth1WSPort  = Eth1Port + 2*portSpan
 
 	PrysmBeaconNodeRPCPort     = 4150
 	PrysmBeaconNodeUDPPort     = PrysmBeaconNodeRPCPort + portSpan
@@ -89,6 +92,8 @@ const (
 
 	ValidatorGatewayPort = 6150
 	ValidatorMetricsPort = ValidatorGatewayPort + portSpan
+
+	JaegerTracingPort = 9150
 )
 
 // Init initializes the E2E config, properly handling test sharding.
@@ -124,6 +129,10 @@ func Init(beaconNodeCount int) error {
 	if err != nil {
 		return err
 	}
+	eth1Port, err := port(Eth1Port, testTotalShards, testShardIndex, &existingRegistrations)
+	if err != nil {
+		return err
+	}
 	eth1RPCPort, err := port(Eth1RPCPort, testTotalShards, testShardIndex, &existingRegistrations)
 	if err != nil {
 		return err
@@ -164,9 +173,14 @@ func Init(beaconNodeCount int) error {
 	if err != nil {
 		return err
 	}
+	jaegerTracingPort, err := port(JaegerTracingPort, testTotalShards, testShardIndex, &existingRegistrations)
+	if err != nil {
+		return err
+	}
 	testPorts := &ports{
 		BootNodePort:               bootnodePort,
 		BootNodeMetricsPort:        bootnodeMetricsPort,
+		Eth1Port:                   eth1Port,
 		Eth1RPCPort:                eth1RPCPort,
 		Eth1WSPort:                 eth1WSPort,
 		PrysmBeaconNodeRPCPort:     beaconNodeRPCPort,
@@ -177,6 +191,7 @@ func Init(beaconNodeCount int) error {
 		PrysmBeaconNodePprofPort:   beaconNodePprofPort,
 		ValidatorMetricsPort:       validatorMetricsPort,
 		ValidatorGatewayPort:       validatorGatewayPort,
+		JaegerTracingPort:          jaegerTracingPort,
 	}
 
 	TestParams = &params{
@@ -222,6 +237,10 @@ func InitMultiClient(beaconNodeCount int, lighthouseNodeCount int) error {
 	if err != nil {
 		return err
 	}
+	eth1Port, err := port(Eth1Port, testTotalShards, testShardIndex, &existingRegistrations)
+	if err != nil {
+		return err
+	}
 	eth1RPCPort, err := port(Eth1RPCPort, testTotalShards, testShardIndex, &existingRegistrations)
 	if err != nil {
 		return err
@@ -277,6 +296,7 @@ func InitMultiClient(beaconNodeCount int, lighthouseNodeCount int) error {
 	testPorts := &ports{
 		BootNodePort:                    bootnodePort,
 		BootNodeMetricsPort:             bootnodeMetricsPort,
+		Eth1Port:                        eth1Port,
 		Eth1RPCPort:                     eth1RPCPort,
 		Eth1WSPort:                      eth1WSPort,
 		PrysmBeaconNodeRPCPort:          prysmBeaconNodeRPCPort,

testing/endtoend/static-files/eth1/BUILD.bazel

@@ -0,0 +1,8 @@
+filegroup(
+    name = "eth1data",
+    srcs = [
+        "UTC--2021-12-22T19-14-08.590377700Z--878705ba3f8bc32fcf7f4caa1a35e72af65cf766",
+        "genesis.json",
+    ],
+    visibility = ["//testing/endtoend:__subpackages__"],
+)

testing/endtoend/static-files/eth1/UTC--2021-12-22T19-14-08.590377700Z--878705ba3f8bc32fcf7f4caa1a35e72af65cf766

@@ -0,0 +1 @@
+{"address":"878705ba3f8bc32fcf7f4caa1a35e72af65cf766","crypto":{"cipher":"aes-128-ctr","ciphertext":"f02daebbf456faf787c5cd61a33ce780857c1ca10b00972aa451f0e9688e4ead","cipherparams":{"iv":"ef1668814155862f0653f15dae845e58"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"55e5ee70d3e882d2f00a073eda252ff01437abf51d7bfa76c06dcc73f7e8f1a3"},"mac":"d8d04625d0769fe286756734f946c78663961b74f0caaff1d768f0d255632f04"},"id":"5fb9083a-a221-412b-b0e0-921e22cc9645","version":3}

testing/endtoend/static-files/eth1/genesis.json

@@ -0,0 +1,30 @@
+{
+  "config": {
+    "chainId": 1337,
+    "homesteadBlock": 0,
+    "eip150Block": 0,
+    "eip155Block": 0,
+    "eip158Block": 0,
+    "byzantiumBlock": 0,
+    "constantinopleBlock": 0,
+    "petersburgBlock": 0,
+    "istanbulBlock": 0,
+    "berlinBlock": 0,
+    "londonBlock": 0,
+    "clique": {
+      "period": 2,
+      "epoch": 30000
+    }
+  },
+  "alloc": {
+    "0x878705ba3f8bc32fcf7f4caa1a35e72af65cf766": {"balance": "100000000000000000000000000000"}
+  },
+  "coinbase" : "0x0000000000000000000000000000000000000000",
+  "difficulty": "1",
+  "extradata": "0x0000000000000000000000000000000000000000000000000000000000000000878705ba3f8bc32fcf7f4caa1a35e72af65cf7660000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+  "gasLimit" : "0xffffff",
+  "nonce" : "0x0000000000000042",
+  "mixhash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "timestamp" : "0x00"
+}