E2E deposit testing overhaul (#11667)

* rewrite/refactor deposit testing code

keep track of sent deposits so that they can be compared in detail with
the validator set retreived from the API.

* fix bugs in evaluator and retry

* lint + deepsource appeasement

* typo s/Sprintf/Printf/

* gosec, more like nosec

* fix gosec number - 204->304

* type switch to get signed block from container

* improve comments

* centralizing constants and adding comments

* lock around Depositor to avoid future races

Co-authored-by: Kasey Kirkham <kasey@users.noreply.github.com>

consensus-types/blocks/factory.go

@@ -288,3 +288,20 @@ func BuildSignedBeaconBlockFromExecutionPayload(
 
 	return NewSignedBeaconBlock(fullBlock)
 }
+
+// BeaconBlockContainerToSignedBeaconBlock converts BeaconBlockContainer (API response) to a SignedBeaconBlock.
+// This is particularly useful for using the values from API calls.
+func BeaconBlockContainerToSignedBeaconBlock(obj *eth.BeaconBlockContainer) (interfaces.SignedBeaconBlock, error) {
+	switch obj.Block.(type) {
+	case *eth.BeaconBlockContainer_BlindedBellatrixBlock:
+		return NewSignedBeaconBlock(obj.GetBlindedBellatrixBlock())
+	case *eth.BeaconBlockContainer_BellatrixBlock:
+		return NewSignedBeaconBlock(obj.GetBellatrixBlock())
+	case *eth.BeaconBlockContainer_AltairBlock:
+		return NewSignedBeaconBlock(obj.GetAltairBlock())
+	case *eth.BeaconBlockContainer_Phase0Block:
+		return NewSignedBeaconBlock(obj.GetPhase0Block())
+	default:
+		return nil, errors.New("container block type not recognized")
+	}
+}

runtime/interop/generate_keys_test.go

@@ -39,6 +39,6 @@ func TestKeyGenerator(t *testing.T) {
 			continue
 		}
 		assert.DeepEqual(t, key.Marshal(), nKey)
-		fmt.Println(fmt.Sprintf("pubkey: %s privkey: %s ", hexutil.Encode(pubkeys[i].Marshal()), hexutil.Encode(key.Marshal())))
+		fmt.Printf("pubkey: %s privkey: %s \n", hexutil.Encode(pubkeys[i].Marshal()), hexutil.Encode(key.Marshal()))
 	}
 }

testing/endtoend/component_handler_test.go

@@ -6,7 +6,6 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/v3/config/params"
 	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/components"
 	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/components/eth1"
 	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/helpers"
@@ -35,24 +34,31 @@ type componentHandler struct {
 }
 
 func NewComponentHandler(cfg *e2etypes.E2EConfig, t *testing.T) *componentHandler {
-	return &componentHandler{cfg: cfg, t: t}
+	ctx, done := context.WithCancel(context.Background())
+	g, ctx := errgroup.WithContext(ctx)
+
+	return &componentHandler{
+		ctx:       ctx,
+		done:      done,
+		group:     g,
+		cfg:       cfg,
+		t:         t,
+		eth1Miner: eth1.NewMiner(),
+	}
 }
 
 func (c *componentHandler) setup() {
 	t, config := c.t, c.cfg
+	ctx, g := c.ctx, c.group
 	t.Logf("Shard index: %d\n", e2e.TestParams.TestShardIndex)
 	t.Logf("Starting time: %s\n", time.Now().String())
 	t.Logf("Log Path: %s\n", e2e.TestParams.LogPath)
 
-	minGenesisActiveCount := int(params.BeaconConfig().MinGenesisActiveValidatorCount)
 	multiClientActive := e2e.TestParams.LighthouseBeaconNodeCount > 0
 	var keyGen e2etypes.ComponentRunner
 	var lighthouseValidatorNodes e2etypes.MultipleComponentRunners
 	var lighthouseNodes *components.LighthouseBeaconNodeSet
 
-	c.ctx, c.done = context.WithCancel(context.Background())
-	g, ctx := errgroup.WithContext(c.ctx)
-
 	tracingSink := components.NewTracingSink(config.TracingSinkEndpoint)
 	g.Go(func() error {
 		return tracingSink.Start(ctx)
@@ -92,27 +98,32 @@ func (c *componentHandler) setup() {
 	})
 	c.bootnode = bootNode
 
+	miner, ok := c.eth1Miner.(*eth1.Miner)
+	if !ok {
+		g.Go(func() error {
+			return errors.New("c.eth1Miner fails type assertion to *eth1.Miner")
+		})
+		return
+	}
 	// ETH1 miner.
-	eth1Miner := eth1.NewMiner()
 	g.Go(func() error {
 		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{bootNode}); err != nil {
 			return errors.Wrap(err, "miner require boot node to run")
 		}
-		eth1Miner.SetBootstrapENR(bootNode.ENR())
-		if err := eth1Miner.Start(ctx); err != nil {
+		miner.SetBootstrapENR(bootNode.ENR())
+		if err := miner.Start(ctx); err != nil {
 			return errors.Wrap(err, "failed to start the ETH1 miner")
 		}
 		return nil
 	})
-	c.eth1Miner = eth1Miner
 
 	// ETH1 non-mining nodes.
 	eth1Nodes := eth1.NewNodeSet()
 	g.Go(func() error {
-		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Miner}); err != nil {
+		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{miner}); err != nil {
 			return errors.Wrap(err, "execution nodes require miner to run")
 		}
-		eth1Nodes.SetMinerENR(eth1Miner.ENR())
+		eth1Nodes.SetMinerENR(miner.ENR())
 		if err := eth1Nodes.Start(ctx); err != nil {
 			return errors.Wrap(err, "failed to start ETH1 nodes")
 		}
@@ -120,16 +131,6 @@ func (c *componentHandler) setup() {
 	})
 	c.eth1Nodes = eth1Nodes
 
-	g.Go(func() error {
-		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Nodes}); err != nil {
-			return errors.Wrap(err, "sending and mining deposits require ETH1 nodes to run")
-		}
-		if err := components.SendAndMineDeposits(eth1Miner.KeystorePath(), minGenesisActiveCount, 0, true /* partial */); err != nil {
-			return errors.Wrap(err, "failed to send and mine deposits")
-		}
-		return nil
-	})
-
 	if config.TestCheckpointSync {
 		appendDebugEndpoints(config)
 	}

testing/endtoend/components/BUILD.bazel

@@ -27,23 +27,15 @@ go_library(
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
         "//config/validator/service:go_default_library",
-        "//contracts/deposit:go_default_library",
         "//crypto/bls:go_default_library",
-        "//encoding/bytesutil:go_default_library",
         "//io/file:go_default_library",
         "//runtime/interop:go_default_library",
-        "//testing/endtoend/components/eth1:go_default_library",
         "//testing/endtoend/helpers:go_default_library",
         "//testing/endtoend/params:go_default_library",
         "//testing/endtoend/types:go_default_library",
-        "//testing/util:go_default_library",
         "//validator/keymanager:go_default_library",
-        "@com_github_ethereum_go_ethereum//accounts/abi/bind:go_default_library",
-        "@com_github_ethereum_go_ethereum//accounts/keystore:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
-        "@com_github_ethereum_go_ethereum//ethclient:go_default_library",
-        "@com_github_ethereum_go_ethereum//rpc:go_default_library",
         "@com_github_google_uuid//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",

testing/endtoend/components/eth1/BUILD.bazel

@@ -1,9 +1,10 @@
-load("@prysm//tools/go:def.bzl", "go_library")
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
     testonly = True,
     srcs = [
+        "depositor.go",
         "helpers.go",
         "miner.go",
         "node.go",
@@ -15,13 +16,17 @@ go_library(
     visibility = ["//testing/endtoend:__subpackages__"],
     deps = [
         "//config/params:go_default_library",
+        "//contracts/deposit:go_default_library",
         "//contracts/deposit/mock:go_default_library",
         "//crypto/rand:go_default_library",
+        "//encoding/bytesutil:go_default_library",
         "//io/file:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
         "//testing/endtoend/helpers:go_default_library",
         "//testing/endtoend/params:go_default_library",
         "//testing/endtoend/types:go_default_library",
         "//testing/middleware/engine-api-proxy:go_default_library",
+        "//testing/util:go_default_library",
         "@com_github_ethereum_go_ethereum//accounts/abi/bind:go_default_library",
         "@com_github_ethereum_go_ethereum//accounts/keystore:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
@@ -36,3 +41,13 @@ go_library(
         "@org_golang_x_sync//errgroup:go_default_library",
     ],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = ["depositor_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//config/params:go_default_library",
+        "//testing/require:go_default_library",
+    ],
+)

testing/endtoend/components/eth1/depositor.go

@@ -0,0 +1,220 @@
+package eth1
+
+import (
+	"context"
+	"fmt"
+	"math/big"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+	gethtypes "github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/v3/config/params"
+	contracts "github.com/prysmaticlabs/prysm/v3/contracts/deposit"
+	"github.com/prysmaticlabs/prysm/v3/encoding/bytesutil"
+	eth "github.com/prysmaticlabs/prysm/v3/proto/prysm/v1alpha1"
+	e2e "github.com/prysmaticlabs/prysm/v3/testing/endtoend/params"
+	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/types"
+	"github.com/prysmaticlabs/prysm/v3/testing/util"
+)
+
+var gweiPerEth = big.NewInt(int64(params.BeaconConfig().GweiPerEth))
+
+func amtInGwei(deposit *eth.Deposit) *big.Int {
+	amt := big.NewInt(0).SetUint64(deposit.Data.Amount)
+	return amt.Mul(amt, gweiPerEth)
+}
+
+// computeDeposits uses the deterministic validator generator to generate deposits for `nvals` (number of validators).
+// To control which validators should receive deposits, so that we can generate deposits at different stages of e2e,
+// the `offset` parameter skips the first N validators in the deterministic list.
+// In order to test the requirement that our deposit follower is able to handle multiple partial deposits,
+// the `partial` flag specifies that half of the deposits should be broken up into 2 transactions.
+func computeDeposits(offset, nvals int, partial bool) ([]*eth.Deposit, error) {
+	balances := make([]uint64, offset+nvals)
+	partialIndex := len(balances) // set beyond loop invariant so by default nothing gets partial
+	if partial {
+		// Validators in this range will get 2 half (MAX_EFFECTIVE_BALANCE/2) deposits.
+		// Upper half of the range of desired validator indices is used because these can be easily
+		// duplicated with a slice from this offset to the end.
+		partialIndex = offset + nvals/2
+	}
+	// Start setting values at `offset`. Lower elements will be discarded - setting them to zero
+	// ensures they don't slip through and add extra deposits.
+	for i := offset; i < len(balances); i++ {
+		if i >= partialIndex {
+			// these deposits will be duplicated, resulting in 2 deposits summing to MAX_EFFECTIVE_BALANCE
+			balances[i] = params.BeaconConfig().MaxEffectiveBalance / 2
+		} else {
+			balances[i] = params.BeaconConfig().MaxEffectiveBalance
+		}
+	}
+	deposits, _, err := util.DepositsWithBalance(balances)
+	if err != nil {
+		return []*eth.Deposit{}, err
+	}
+
+	// if partial = false, these will be a no-op (partialIndex == len(deposits)),
+	// otherwise it will duplicate the partial deposits.
+	deposits = append(deposits[offset:], deposits[partialIndex:]...)
+
+	return deposits, nil
+}
+
+type Depositor struct {
+	// The EmptyComponent type is embedded in the Depositor so that it satisfies the ComponentRunner interface.
+	// This allows other components or e2e set up code to block until its Start method has been called.
+	types.EmptyComponent
+	Key       *keystore.Key
+	Client    *ethclient.Client
+	ChainID   *big.Int
+	NetworkId *big.Int
+	cd        *contracts.DepositContract
+	sent      *DepositHistory
+}
+
+var _ types.ComponentRunner = &Depositor{}
+
+// History exposes the DepositHistory value for a Depositor.
+func (d *Depositor) History() *DepositHistory {
+	if d.sent == nil {
+		d.sent = &DepositHistory{}
+	}
+	return d.sent
+}
+
+// DepositHistory is a type used by Depositor to keep track of batches of deposit for test assertion purposes.
+type DepositHistory struct {
+	sync.RWMutex
+	byBatch  map[types.DepositBatch][]*SentDeposit
+	deposits []*SentDeposit
+}
+
+func (h *DepositHistory) record(sd *SentDeposit) {
+	h.Lock()
+	defer h.Unlock()
+	h.deposits = append(h.deposits, sd)
+	h.updateByBatch(sd)
+}
+
+func (h *DepositHistory) updateByBatch(sd *SentDeposit) {
+	if h.byBatch == nil {
+		h.byBatch = make(map[types.DepositBatch][]*SentDeposit)
+	}
+	h.byBatch[sd.batch] = append(h.byBatch[sd.batch], sd)
+}
+
+// Balances sums, by validator, all deposit amounts that were sent as part of the given batch.
+// This can be used in e2e evaluators to check that the results of deposit transactions are visible on chain.
+func (h *DepositHistory) Balances(batch types.DepositBatch) map[[48]byte]uint64 {
+	balances := make(map[[48]byte]uint64)
+	h.RLock()
+	defer h.RUnlock()
+	for _, d := range h.byBatch[batch] {
+		k := bytesutil.ToBytes48(d.deposit.Data.PublicKey)
+		if _, ok := balances[k]; !ok {
+			balances[k] = d.deposit.Data.Amount
+		} else {
+			balances[k] += d.deposit.Data.Amount
+		}
+	}
+	return balances
+}
+
+// SentDeposit is the record of an individual deposit which has been successfully submitted as a transaction.
+type SentDeposit struct {
+	root    [32]byte
+	deposit *eth.Deposit
+	tx      *gethtypes.Transaction
+	time    time.Time
+	batch   types.DepositBatch
+}
+
+// SendAndMine uses the deterministic validator generator to generate deposits for `nvals` (number of validators).
+// To control which validators should receive deposits, so that we can generate deposits at different stages of e2e,
+// the `offset` parameter skips the first N validators in the deterministic list.
+// In order to test the requirement that our deposit follower is able to handle multiple partial deposits,
+// the `partial` flag specifies that half of the deposits should be broken up into 2 transactions.
+// Once the set of deposits has been generated, it submits a transaction for each deposit
+// (using 2 transactions for partial deposits) and then uses WaitForBlocks (which spams the miner node with transactions
+// to and from its own address) to advance the chain until it has moved forward ETH1_FOLLOW_DISTANCE blocks.
+func (d *Depositor) SendAndMine(ctx context.Context, offset, nvals int, batch types.DepositBatch, partial bool) error {
+	// This is the "Send" part of the function. Compute deposits for `nvals` validators,
+	// with half of those deposits being split over 2 transactions if the `partial` flag is true,
+	// and throwing away any validators before `offset`.
+	deposits, err := computeDeposits(offset, nvals, partial)
+	if err != nil {
+		return err
+	}
+
+	txo, err := d.txops(ctx)
+	if err != nil {
+		return err
+	}
+	for _, dd := range deposits {
+		if err := d.SendDeposit(dd, txo, batch); err != nil {
+			return err
+		}
+	}
+
+	// This is the "AndMine" part of the function. WaitForBlocks will spam transactions to/from the given key
+	// to advance the EL chain and until the chain has advanced the requested amount.
+	if err = WaitForBlocks(d.Client, d.Key, params.BeaconConfig().Eth1FollowDistance); err != nil {
+		return fmt.Errorf("failed to mine blocks %w", err)
+	}
+	return nil
+}
+
+// SendDeposit sends a single deposit. A record of this deposit will be tracked for the life of the Depositor,
+// allowing evaluators to use the deposit history to make assertions about those deposits.
+func (d *Depositor) SendDeposit(dep *eth.Deposit, txo *bind.TransactOpts, batch types.DepositBatch) error {
+	contract, err := d.contractDepositor()
+	if err != nil {
+		return err
+	}
+	txo.Value = amtInGwei(dep)
+	root, err := dep.Data.HashTreeRoot()
+	if err != nil {
+		return err
+	}
+	sent := time.Now()
+	tx, err := contract.Deposit(txo, dep.Data.PublicKey, dep.Data.WithdrawalCredentials, dep.Data.Signature, root)
+	if err != nil {
+		return errors.Wrap(err, "unable to send transaction to contract")
+	}
+	d.History().record(&SentDeposit{deposit: dep, time: sent, tx: tx, root: root, batch: batch})
+	txo.Nonce = txo.Nonce.Add(txo.Nonce, big.NewInt(1))
+	return nil
+}
+
+// txops is a little helper method that creates a TransactOpts (type encapsulating auth/meta data needed to make a tx).
+func (d *Depositor) txops(ctx context.Context) (*bind.TransactOpts, error) {
+	txo, err := bind.NewKeyedTransactorWithChainID(d.Key.PrivateKey, d.NetworkId)
+	if err != nil {
+		return nil, err
+	}
+	txo.Context = ctx
+	txo.GasLimit = e2e.DepositGasLimit
+	nonce, err := d.Client.PendingNonceAt(ctx, txo.From)
+	if err != nil {
+		return nil, err
+	}
+	txo.Nonce = big.NewInt(0).SetUint64(nonce)
+	return txo, nil
+}
+
+// contractDepositor is a little helper method that inits and caches a DepositContract value.
+// DepositContract is a special-purpose client for calling the deposit contract.
+func (d *Depositor) contractDepositor() (*contracts.DepositContract, error) {
+	if d.cd == nil {
+		contract, err := contracts.NewDepositContract(e2e.TestParams.ContractAddress, d.Client)
+		if err != nil {
+			return nil, err
+		}
+		d.cd = contract
+	}
+	return d.cd, nil
+}

testing/endtoend/components/eth1/depositor_test.go

@@ -0,0 +1,95 @@
+package eth1
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/v3/config/params"
+
+	"github.com/prysmaticlabs/prysm/v3/testing/require"
+)
+
+func TestComputeDeposits(t *testing.T) {
+	cases := []struct {
+		nvals   int
+		offset  int
+		partial bool
+		err     error
+		len     int
+		name    string
+	}{
+		{
+			nvals:   100,
+			offset:  0,
+			partial: false,
+			len:     100,
+			name:    "offset 0",
+		},
+		{
+			nvals:   100,
+			offset:  50,
+			partial: false,
+			len:     100,
+			name:    "offset 50",
+		},
+		{
+			nvals:   100,
+			offset:  0,
+			partial: true,
+			len:     150,
+			name:    "offset 50, partial",
+		},
+		{
+			nvals:   100,
+			offset:  50,
+			partial: true,
+			len:     150,
+			name:    "offset 50, partial",
+		},
+		{
+			nvals:   100,
+			offset:  23,
+			partial: true,
+			len:     150,
+			name:    "offset 23, partial",
+		},
+		{
+			nvals:   23,
+			offset:  0,
+			partial: true,
+			len:     35, // half of 23 (rounding down) is 11
+			name:    "offset 23, partial",
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			totals := make(map[string]uint64, c.nvals)
+			d, err := computeDeposits(c.offset, c.nvals, c.partial)
+			for _, dd := range d {
+				require.Equal(t, 48, len(dd.Data.PublicKey))
+				k := fmt.Sprintf("%#x", dd.Data.PublicKey)
+				if _, ok := totals[k]; !ok {
+					totals[k] = dd.Data.Amount
+				} else {
+					totals[k] += dd.Data.Amount
+				}
+			}
+			complete := make([]string, 0)
+			incomplete := make([]string, 0)
+			for k, v := range totals {
+				if params.BeaconConfig().MaxEffectiveBalance != v {
+					incomplete = append(incomplete, fmt.Sprintf("%s=%d", k, v))
+				} else {
+					complete = append(complete, k)
+				}
+			}
+			require.Equal(t, 0, len(incomplete), strings.Join(incomplete, ", "))
+			require.Equal(t, c.nvals, len(complete), strings.Join(complete, ", "))
+			if err != nil {
+				require.ErrorIs(t, err, c.err)
+			}
+			require.Equal(t, c.len, len(d))
+		})
+	}
+}

testing/endtoend/components/eth1/helpers.go

@@ -8,6 +8,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/params"
 	e2etypes "github.com/prysmaticlabs/prysm/v3/testing/endtoend/types"
 )
 
@@ -20,7 +21,6 @@ const KeystorePassword = "password"
 const minerPasswordFile = "password.txt"
 const minerFile = "UTC--2021-12-22T19-14-08.590377700Z--878705ba3f8bc32fcf7f4caa1a35e72af65cf766"
 const timeGapPerTX = 100 * time.Millisecond
-const staticFilesPath = "/testing/endtoend/static-files/eth1"
 const timeGapPerMiningTX = 250 * time.Millisecond
 
 var _ e2etypes.ComponentRunner = (*NodeSet)(nil)
@@ -31,8 +31,8 @@ var _ e2etypes.ComponentRunner = (*Node)(nil)
 var _ e2etypes.EngineProxy = (*Proxy)(nil)
 
 // WaitForBlocks waits for a certain amount of blocks to be mined by the ETH1 chain before returning.
-func WaitForBlocks(web3 *ethclient.Client, keystore *keystore.Key, blocksToWait uint64) error {
-	nonce, err := web3.PendingNonceAt(context.Background(), keystore.Address)
+func WaitForBlocks(web3 *ethclient.Client, key *keystore.Key, blocksToWait uint64) error {
+	nonce, err := web3.PendingNonceAt(context.Background(), key.Address)
 	if err != nil {
 		return err
 	}
@@ -47,8 +47,8 @@ func WaitForBlocks(web3 *ethclient.Client, keystore *keystore.Key, blocksToWait
 	finishBlock := block.NumberU64() + blocksToWait
 
 	for block.NumberU64() <= finishBlock {
-		spamTX := types.NewTransaction(nonce, keystore.Address, big.NewInt(0), 21000, big.NewInt(1e6), []byte{})
-		signed, err := types.SignTx(spamTX, types.NewEIP155Signer(chainID), keystore.PrivateKey)
+		spamTX := types.NewTransaction(nonce, key.Address, big.NewInt(0), params.SpamTxGasLimit, big.NewInt(1e6), []byte{})
+		signed, err := types.SignTx(spamTX, types.NewEIP155Signer(chainID), key.PrivateKey)
 		if err != nil {
 			return err
 		}

testing/endtoend/components/eth1/miner.go

@@ -1,7 +1,6 @@
 package eth1
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"math/big"
@@ -14,7 +13,6 @@ import (
 
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
-	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/pkg/errors"
@@ -37,7 +35,6 @@ type Miner struct {
 	started      chan struct{}
 	bootstrapEnr string
 	enr          string
-	keystorePath string
 	cmd          *exec.Cmd
 }
 
@@ -48,11 +45,6 @@ func NewMiner() *Miner {
 	}
 }
 
-// KeystorePath returns the path of the keystore file.
-func (m *Miner) KeystorePath() string {
-	return m.keystorePath
-}
-
 // ENR returns the miner's enode.
 func (m *Miner) ENR() string {
 	return m.enr
@@ -63,56 +55,72 @@ func (m *Miner) SetBootstrapENR(bootstrapEnr string) {
 	m.bootstrapEnr = bootstrapEnr
 }
 
-// Start runs a mining ETH1 node.
-// The miner is responsible for moving the ETH1 chain forward and for deploying the deposit contract.
-func (m *Miner) Start(ctx context.Context) error {
-	binaryPath, found := bazel.FindBinary("cmd/geth", "geth")
-	if !found {
-		return errors.New("go-ethereum binary not found")
-	}
+func (*Miner) DataDir(sub ...string) string {
+	parts := append([]string{e2e.TestParams.TestPath, "eth1data/miner"}, sub...)
+	return path.Join(parts...)
+}
 
-	eth1Path := path.Join(e2e.TestParams.TestPath, "eth1data/miner/")
+func (*Miner) Password() string {
+	return KeystorePassword
+}
+
+func (m *Miner) initDataDir() error {
+	eth1Path := m.DataDir()
 	// Clear out potentially existing dir to prevent issues.
 	if _, err := os.Stat(eth1Path); !os.IsNotExist(err) {
 		if err = os.RemoveAll(eth1Path); err != nil {
 			return err
 		}
 	}
+	return nil
+}
 
-	genesisSrcPath, err := bazel.Runfile(path.Join(staticFilesPath, "genesis.json"))
+func (m *Miner) initAttempt(ctx context.Context, attempt int) (*os.File, error) {
+	if err := m.initDataDir(); err != nil {
+		return nil, err
+	}
+
+	// find geth so we can run it.
+	binaryPath, found := bazel.FindBinary("cmd/geth", "geth")
+	if !found {
+		return nil, errors.New("go-ethereum binary not found")
+	}
+
+	staticGenesis, err := e2e.TestParams.Paths.Eth1Runfile("genesis.json")
 	if err != nil {
-		return err
+		return nil, err
 	}
-	genesisDstPath := binaryPath[:strings.LastIndex(binaryPath, "/")]
-	cpCmd := exec.CommandContext(ctx, "cp", genesisSrcPath, genesisDstPath) // #nosec G204 -- Safe
-	if err = cpCmd.Start(); err != nil {
-		return err
-	}
-	if err = cpCmd.Wait(); err != nil {
-		return err
+	genesisPath := path.Join(path.Dir(binaryPath), "genesis.json")
+	if err := io.CopyFile(staticGenesis, genesisPath); err != nil {
+		return nil, errors.Wrapf(err, "error copying %s to %s", staticGenesis, genesisPath)
 	}
 
 	initCmd := exec.CommandContext(
 		ctx,
 		binaryPath,
 		"init",
-		fmt.Sprintf("--datadir=%s", eth1Path),
-		genesisDstPath+"/genesis.json") // #nosec G204 -- Safe
-	initFile, err := helpers.DeleteAndCreateFile(e2e.TestParams.LogPath, "eth1-init_miner.log")
+		fmt.Sprintf("--datadir=%s", m.DataDir()),
+		genesisPath) // #nosec G204 -- Safe
+
+	// redirect stderr to a log file
+	initFile, err := helpers.DeleteAndCreatePath(e2e.TestParams.Logfile("eth1-init_miner.log"))
 	if err != nil {
-		return err
+		return nil, err
 	}
 	initCmd.Stderr = initFile
+
+	// run init command and wait until it exits. this will initialize the geth node (required before starting).
 	if err = initCmd.Start(); err != nil {
-		return err
+		return nil, err
 	}
 	if err = initCmd.Wait(); err != nil {
-		return err
+		return nil, err
 	}
 
+	pwFile := m.DataDir("keystore", minerPasswordFile)
 	args := []string{
-		"--nat=none",
-		fmt.Sprintf("--datadir=%s", eth1Path),
+		"--nat=none", // disable nat traversal in e2e, it is failure prone and not needed
+		fmt.Sprintf("--datadir=%s", m.DataDir()),
 		fmt.Sprintf("--http.port=%d", e2e.TestParams.Ports.Eth1RPCPort),
 		fmt.Sprintf("--ws.port=%d", e2e.TestParams.Ports.Eth1WSPort),
 		fmt.Sprintf("--authrpc.port=%d", e2e.TestParams.Ports.Eth1AuthRPCPort),
@@ -136,46 +144,69 @@ func (m *Miner) Start(ctx context.Context) error {
 		"--allow-insecure-unlock",
 		"--syncmode=full",
 		fmt.Sprintf("--txpool.locals=%s", EthAddress),
-		fmt.Sprintf("--password=%s", eth1Path+"/keystore/"+minerPasswordFile),
+		fmt.Sprintf("--password=%s", pwFile),
 	}
 
-	keystorePath, err := bazel.Runfile(path.Join(staticFilesPath, minerFile))
+	keystorePath, err := e2e.TestParams.Paths.MinerKeyPath()
 	if err != nil {
-		return err
+		return nil, err
 	}
-	jsonBytes, err := os.ReadFile(keystorePath) // #nosec G304 -- ReadFile is safe
-	if err != nil {
-		return err
+	if err = io.CopyFile(keystorePath, m.DataDir("keystore", minerFile)); err != nil {
+		return nil, errors.Wrapf(err, "error copying %s to %s", keystorePath, m.DataDir("keystore", minerFile))
 	}
-	err = io.WriteFile(eth1Path+"/keystore/"+minerFile, jsonBytes)
+	err = io.WriteFile(pwFile, []byte(KeystorePassword))
 	if err != nil {
-		return err
-	}
-	err = io.WriteFile(eth1Path+"/keystore/"+minerPasswordFile, []byte(KeystorePassword))
-	if err != nil {
-		return err
+		return nil, err
 	}
 
 	runCmd := exec.CommandContext(ctx, binaryPath, args...) // #nosec G204 -- Safe
-	file, err := os.Create(path.Join(e2e.TestParams.LogPath, "eth1_miner.log"))
+	// redirect miner stderr to a log file
+	minerLog, err := helpers.DeleteAndCreatePath(e2e.TestParams.Logfile("eth1_miner.log"))
 	if err != nil {
-		return err
+		return nil, err
 	}
-	runCmd.Stderr = file
-	log.Infof("Starting eth1 miner with flags: %s", strings.Join(args[2:], " "))
-
+	runCmd.Stderr = minerLog
+	log.Infof("Starting eth1 miner, attempt %d, with flags: %s", attempt, strings.Join(args[2:], " "))
 	if err = runCmd.Start(); err != nil {
-		return fmt.Errorf("failed to start eth1 chain: %w", err)
+		return nil, fmt.Errorf("failed to start eth1 chain: %w", err)
+	}
+	// check logs for common issues that prevent the EL miner from starting up.
+	if err = helpers.WaitForTextInFile(minerLog, "Commit new sealing work"); err != nil {
+		kerr := runCmd.Process.Kill()
+		if kerr != nil {
+			log.WithError(kerr).Error("error sending kill to failed miner command process")
+		}
+		return nil, fmt.Errorf("mining log not found, this means the eth1 chain had issues starting: %w", err)
+	}
+	if err = helpers.WaitForTextInFile(minerLog, "Started P2P networking"); err != nil {
+		kerr := runCmd.Process.Kill()
+		if kerr != nil {
+			log.WithError(kerr).Error("error sending kill to failed miner command process")
+		}
+		return nil, fmt.Errorf("P2P log not found, this means the eth1 chain had issues starting: %w", err)
+	}
+	m.cmd = runCmd
+	return minerLog, nil
+}
+
+// Start runs a mining ETH1 node.
+// The miner is responsible for moving the ETH1 chain forward and for deploying the deposit contract.
+func (m *Miner) Start(ctx context.Context) error {
+	// give the miner start a couple of tries, since the p2p networking check is flaky
+	var retryErr error
+	var minerLog *os.File
+	for attempt := 0; attempt < 3; attempt++ {
+		minerLog, retryErr = m.initAttempt(ctx, attempt)
+		if retryErr == nil {
+			log.Infof("miner started after %d retries", attempt)
+			break
+		}
+	}
+	if retryErr != nil {
+		return retryErr
 	}
 
-	if err = helpers.WaitForTextInFile(file, "Commit new sealing work"); err != nil {
-		return fmt.Errorf("mining log not found, this means the eth1 chain had issues starting: %w", err)
-	}
-	if err = helpers.WaitForTextInFile(file, "Started P2P networking"); err != nil {
-		return fmt.Errorf("P2P log not found, this means the eth1 chain had issues starting: %w", err)
-	}
-
-	enode, err := enodeFromLogFile(file.Name())
+	enode, err := enodeFromLogFile(minerLog.Name())
 	if err != nil {
 		return err
 	}
@@ -184,26 +215,32 @@ func (m *Miner) Start(ctx context.Context) error {
 	log.Infof("Communicated enode. Enode is %s", enode)
 
 	// Connect to the started geth dev chain.
-	client, err := rpc.DialHTTP(fmt.Sprintf("http://127.0.0.1:%d", e2e.TestParams.Ports.Eth1RPCPort))
+	client, err := rpc.DialHTTP(e2e.TestParams.Eth1RPCURL(e2e.MinerComponentOffset).String())
 	if err != nil {
 		return fmt.Errorf("failed to connect to ipc: %w", err)
 	}
 	web3 := ethclient.NewClient(client)
-
-	// Deploy the contract.
-	store, err := keystore.DecryptKey(jsonBytes, KeystorePassword)
+	keystorePath, err := e2e.TestParams.Paths.MinerKeyPath()
 	if err != nil {
 		return err
 	}
-	// Advancing the blocks eth1follow distance to prevent issues reading the chain.
-	if err = WaitForBlocks(web3, store, params.BeaconConfig().Eth1FollowDistance); err != nil {
+	// this is the key for the miner account. miner account balance is pre-mined in genesis.json.
+	key, err := helpers.KeyFromPath(keystorePath, KeystorePassword)
+	if err != nil {
+		return err
+	}
+	// Waiting for the blocks to advance by eth1follow to prevent issues reading the chain.
+	// Note that WaitForBlocks spams transfer transactions (to and from the miner's address) in order to advance.
+	if err = WaitForBlocks(web3, key, params.BeaconConfig().Eth1FollowDistance); err != nil {
 		return fmt.Errorf("unable to advance chain: %w", err)
 	}
-	txOpts, err := bind.NewTransactorWithChainID(bytes.NewReader(jsonBytes), KeystorePassword, big.NewInt(NetworkId))
+
+	// Time to deploy the contract using the miner's key.
+	txOpts, err := bind.NewKeyedTransactorWithChainID(key.PrivateKey, big.NewInt(NetworkId))
 	if err != nil {
 		return err
 	}
-	nonce, err := web3.PendingNonceAt(ctx, store.Address)
+	nonce, err := web3.PendingNonceAt(ctx, key.Address)
 	if err != nil {
 		return err
 	}
@@ -224,18 +261,14 @@ func (m *Miner) Start(ctx context.Context) error {
 	}
 
 	// Advancing the blocks another eth1follow distance to prevent issues reading the chain.
-	if err = WaitForBlocks(web3, store, params.BeaconConfig().Eth1FollowDistance); err != nil {
+	if err = WaitForBlocks(web3, key, params.BeaconConfig().Eth1FollowDistance); err != nil {
 		return fmt.Errorf("unable to advance chain: %w", err)
 	}
 
-	// Save keystore path (used for saving and mining deposits).
-	m.keystorePath = keystorePath
-
 	// Mark node as ready.
 	close(m.started)
 
-	m.cmd = runCmd
-	return runCmd.Wait()
+	return m.cmd.Wait()
 }
 
 // Started checks whether ETH1 node is started and ready to be queried.

testing/endtoend/components/eth1/node.go

@@ -10,12 +10,13 @@ import (
 	"strings"
 	"syscall"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/helpers"
 	e2e "github.com/prysmaticlabs/prysm/v3/testing/endtoend/params"
 	e2etypes "github.com/prysmaticlabs/prysm/v3/testing/endtoend/types"
-	log "github.com/sirupsen/logrus"
 )
 
 // Node represents an ETH1 node.
@@ -71,7 +72,7 @@ func (node *Node) Start(ctx context.Context) error {
 	}
 
 	args := []string{
-		"--nat=none",
+		"--nat=none", // disable nat traversal in e2e, it is failure prone and not needed
 		fmt.Sprintf("--datadir=%s", eth1Path),
 		fmt.Sprintf("--http.port=%d", e2e.TestParams.Ports.Eth1RPCPort+node.index),
 		fmt.Sprintf("--ws.port=%d", e2e.TestParams.Ports.Eth1WSPort+node.index),
@@ -94,26 +95,41 @@ func (node *Node) Start(ctx context.Context) error {
 		"--syncmode=full",
 		fmt.Sprintf("--txpool.locals=%s", EthAddress),
 	}
-	runCmd := exec.CommandContext(ctx, binaryPath, args...) // #nosec G204 -- Safe
-	file, err := os.Create(path.Join(e2e.TestParams.LogPath, "eth1_"+strconv.Itoa(node.index)+".log"))
-	if err != nil {
-		return err
-	}
-	runCmd.Stderr = file
-	log.Infof("Starting eth1 node %d with flags: %s", node.index, strings.Join(args[2:], " "))
 
-	if err = runCmd.Start(); err != nil {
-		return fmt.Errorf("failed to start eth1 chain: %w", err)
+	// give the miner start a couple of tries, since the p2p networking check is flaky
+	var retryErr error
+	for retries := 0; retries < 3; retries++ {
+		retryErr = nil
+		log.Infof("Starting eth1 node %d, attempt %d with flags: %s", node.index, retries, strings.Join(args[2:], " "))
+		runCmd := exec.CommandContext(ctx, binaryPath, args...) // #nosec G204 -- Safe
+		errLog, err := os.Create(path.Join(e2e.TestParams.LogPath, "eth1_"+strconv.Itoa(node.index)+".log"))
+		if err != nil {
+			return err
+		}
+		runCmd.Stderr = errLog
+		if err = runCmd.Start(); err != nil {
+			return fmt.Errorf("failed to start eth1 chain: %w", err)
+		}
+		if err = helpers.WaitForTextInFile(errLog, "Started P2P networking"); err != nil {
+			kerr := runCmd.Process.Kill()
+			if kerr != nil {
+				log.WithError(kerr).Error("error sending kill to failed node command process")
+			}
+			retryErr = fmt.Errorf("P2P log not found, this means the eth1 chain had issues starting: %w", err)
+			continue
+		}
+		node.cmd = runCmd
+		log.Infof("eth1 node started after %d retries", retries)
+		break
 	}
-	if err = helpers.WaitForTextInFile(file, "Started P2P networking"); err != nil {
-		return fmt.Errorf("P2P log not found, this means the eth1 chain had issues starting: %w", err)
+	if retryErr != nil {
+		return retryErr
 	}
 
 	// Mark node as ready.
 	close(node.started)
-	node.cmd = runCmd
 
-	return runCmd.Wait()
+	return node.cmd.Wait()
 }
 
 // Started checks whether ETH1 node is started and ready to be queried.

testing/endtoend/components/validator.go

@@ -1,11 +1,9 @@
 package components
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
-	"math/big"
 	"os"
 	"os/exec"
 	"path"
@@ -14,12 +12,8 @@ import (
 	"syscall"
 
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
-	"github.com/ethereum/go-ethereum/accounts/abi/bind"
-	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/ethclient"
-	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/pkg/errors"
 	cmdshared "github.com/prysmaticlabs/prysm/v3/cmd"
 	"github.com/prysmaticlabs/prysm/v3/cmd/validator/flags"
@@ -27,18 +21,13 @@ import (
 	fieldparams "github.com/prysmaticlabs/prysm/v3/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v3/config/params"
 	validator_service_config "github.com/prysmaticlabs/prysm/v3/config/validator/service"
-	contracts "github.com/prysmaticlabs/prysm/v3/contracts/deposit"
-	"github.com/prysmaticlabs/prysm/v3/encoding/bytesutil"
 	"github.com/prysmaticlabs/prysm/v3/io/file"
 	"github.com/prysmaticlabs/prysm/v3/runtime/interop"
-	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/components/eth1"
 	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/helpers"
 	e2e "github.com/prysmaticlabs/prysm/v3/testing/endtoend/params"
 	e2etypes "github.com/prysmaticlabs/prysm/v3/testing/endtoend/types"
-	"github.com/prysmaticlabs/prysm/v3/testing/util"
 )
 
-const depositGasLimit = 4000000
 const DefaultFeeRecipientAddress = "0x099FB65722e7b2455043bfebF6177f1D2E9738d9"
 
 var _ e2etypes.ComponentRunner = (*ValidatorNode)(nil)
@@ -309,90 +298,6 @@ func (v *ValidatorNode) Stop() error {
 	return v.cmd.Process.Kill()
 }
 
-// SendAndMineDeposits sends the requested amount of deposits and mines the chain after to ensure the deposits are seen.
-func SendAndMineDeposits(keystorePath string, validatorNum, offset int, partial bool) error {
-	client, err := rpc.DialHTTP(fmt.Sprintf("http://127.0.0.1:%d", e2e.TestParams.Ports.Eth1RPCPort))
-	if err != nil {
-		return err
-	}
-	defer client.Close()
-	web3 := ethclient.NewClient(client)
-
-	keystoreBytes, err := os.ReadFile(keystorePath) // #nosec G304
-	if err != nil {
-		return err
-	}
-	if err = sendDeposits(web3, keystoreBytes, validatorNum, offset, partial); err != nil {
-		return err
-	}
-	mineKey, err := keystore.DecryptKey(keystoreBytes, eth1.KeystorePassword)
-	if err != nil {
-		return err
-	}
-	if err = eth1.WaitForBlocks(web3, mineKey, params.BeaconConfig().Eth1FollowDistance); err != nil {
-		return fmt.Errorf("failed to mine blocks %w", err)
-	}
-	return nil
-}
-
-// sendDeposits uses the passed in web3 and keystore bytes to send the requested deposits.
-func sendDeposits(web3 *ethclient.Client, keystoreBytes []byte, num, offset int, partial bool) error {
-	txOps, err := bind.NewTransactorWithChainID(bytes.NewReader(keystoreBytes), eth1.KeystorePassword, big.NewInt(eth1.NetworkId))
-	if err != nil {
-		return err
-	}
-	txOps.GasLimit = depositGasLimit
-	txOps.Context = context.Background()
-	nonce, err := web3.PendingNonceAt(context.Background(), txOps.From)
-	if err != nil {
-		return err
-	}
-	txOps.Nonce = big.NewInt(0).SetUint64(nonce)
-
-	contract, err := contracts.NewDepositContract(e2e.TestParams.ContractAddress, web3)
-	if err != nil {
-		return err
-	}
-
-	balances := make([]uint64, num+offset)
-	for i := 0; i < len(balances); i++ {
-		if i < len(balances)/2 && partial {
-			balances[i] = params.BeaconConfig().MaxEffectiveBalance / 2
-		} else {
-			balances[i] = params.BeaconConfig().MaxEffectiveBalance
-		}
-	}
-	deposits, trie, err := util.DepositsWithBalance(balances)
-	if err != nil {
-		return err
-	}
-	allDeposits := deposits
-	allRoots := trie.Items()
-	allBalances := balances
-	if partial {
-		deposits2, trie2, err := util.DepositsWithBalance(balances)
-		if err != nil {
-			return err
-		}
-		allDeposits = append(deposits, deposits2[:len(balances)/2]...)
-		allRoots = append(trie.Items(), trie2.Items()[:len(balances)/2]...)
-		allBalances = append(balances, balances[:len(balances)/2]...)
-	}
-	for index, dd := range allDeposits {
-		if index < offset {
-			continue
-		}
-		depositInGwei := big.NewInt(int64(allBalances[index]))
-		txOps.Value = depositInGwei.Mul(depositInGwei, big.NewInt(int64(params.BeaconConfig().GweiPerEth)))
-		_, err = contract.Deposit(txOps, dd.Data.PublicKey, dd.Data.WithdrawalCredentials, dd.Data.Signature, bytesutil.ToBytes32(allRoots[index]))
-		if err != nil {
-			return errors.Wrap(err, "unable to send transaction to contract")
-		}
-		txOps.Nonce = txOps.Nonce.Add(txOps.Nonce, big.NewInt(1))
-	}
-	return nil
-}
-
 func createProposerSettingsPath(pubkeys []string, validatorIndex int) (string, error) {
 	testNetDir := e2e.TestParams.TestPath + fmt.Sprintf("/proposer-settings/validator_%d", validatorIndex)
 	configPath := filepath.Join(testNetDir, "config.json")

testing/endtoend/endtoend_setup_test.go

@@ -44,7 +44,7 @@ func e2eMinimal(t *testing.T, cfgo ...types.E2EConfigOpt) *testRunner {
 		ev.VerifyBlockGraffiti,
 		ev.PeersCheck,
 		ev.ProposeVoluntaryExit,
-		ev.ValidatorHasExited,
+		ev.ValidatorsHaveExited,
 		ev.ProcessesDepositsInBlocks,
 		ev.ActivatesDepositedValidators,
 		ev.DepositedValidatorsAreActive,
@@ -119,7 +119,7 @@ func e2eMainnet(t *testing.T, usePrysmSh, useMultiClient bool, cfgo ...types.E2E
 		ev.ValidatorsParticipatingAtEpoch(2),
 		ev.FinalizationOccurs(3),
 		ev.ProposeVoluntaryExit,
-		ev.ValidatorHasExited,
+		ev.ValidatorsHaveExited,
 		ev.ColdStateCheckpoint,
 		ev.AltairForkTransition,
 		ev.BellatrixForkTransition,
@@ -166,7 +166,7 @@ func scenarioEvals() []types.Evaluator {
 		ev.FinalizationOccurs(3),
 		ev.VerifyBlockGraffiti,
 		ev.ProposeVoluntaryExit,
-		ev.ValidatorHasExited,
+		ev.ValidatorsHaveExited,
 		ev.ColdStateCheckpoint,
 		ev.AltairForkTransition,
 		ev.BellatrixForkTransition,
@@ -186,7 +186,7 @@ func scenarioEvalsMulti() []types.Evaluator {
 		ev.ValidatorsParticipatingAtEpoch(2),
 		ev.FinalizationOccurs(3),
 		ev.ProposeVoluntaryExit,
-		ev.ValidatorHasExited,
+		ev.ValidatorsHaveExited,
 		ev.ColdStateCheckpoint,
 		ev.AltairForkTransition,
 		ev.BellatrixForkTransition,

testing/endtoend/endtoend_test.go

@@ -7,6 +7,7 @@ package endtoend
 import (
 	"context"
 	"fmt"
+	"math/big"
 	"os"
 	"path"
 	"strings"
@@ -60,6 +61,7 @@ type testRunner struct {
 	t          *testing.T
 	config     *e2etypes.E2EConfig
 	comHandler *componentHandler
+	depositor  *eth1.Depositor
 }
 
 // newTestRunner creates E2E test runner.
@@ -70,13 +72,46 @@ func newTestRunner(t *testing.T, config *e2etypes.E2EConfig) *testRunner {
 	}
 }
 
-// run executes configured E2E test.
-func (r *testRunner) run() {
+type runEvent func() error
+
+func (r *testRunner) runBase(runEvents []runEvent) {
 	r.comHandler = NewComponentHandler(r.config, r.t)
+	r.comHandler.group.Go(func() error {
+		miner, ok := r.comHandler.eth1Miner.(*eth1.Miner)
+		if !ok {
+			return errors.New("in runBase, comHandler.eth1Miner fails type assertion to *eth1.Miner")
+		}
+		if err := helpers.ComponentsStarted(r.comHandler.ctx, []e2etypes.ComponentRunner{miner}); err != nil {
+			return errors.Wrap(err, "eth1Miner component never started - cannot send deposits")
+		}
+		// refactored send and mine goes here
+		minGenesisActiveCount := int(params.BeaconConfig().MinGenesisActiveValidatorCount)
+		keyPath, err := e2e.TestParams.Paths.MinerKeyPath()
+		if err != nil {
+			return errors.Wrap(err, "error getting miner key file from bazel static files")
+		}
+		key, err := helpers.KeyFromPath(keyPath, miner.Password())
+		if err != nil {
+			return errors.Wrap(err, "failed to read key from miner wallet")
+		}
+		client, err := helpers.MinerRPCClient()
+		if err != nil {
+			return errors.Wrap(err, "failed to initialize a client to connect to the miner EL node")
+		}
+		r.depositor = &eth1.Depositor{Key: key, Client: client, NetworkId: big.NewInt(eth1.NetworkId)}
+		if err := r.depositor.SendAndMine(r.comHandler.ctx, 0, minGenesisActiveCount, e2etypes.GenesisDepositBatch, true); err != nil {
+			return errors.Wrap(err, "failed to send and mine deposits")
+		}
+		if err := r.depositor.Start(r.comHandler.ctx); err != nil {
+			return errors.Wrap(err, "depositor.Start failed")
+		}
+		return nil
+	})
 	r.comHandler.setup()
 
-	// Run E2E evaluators and tests.
-	r.addEvent(r.defaultEndToEndRun)
+	for _, re := range runEvents {
+		r.addEvent(re)
+	}
 
 	if err := r.comHandler.group.Wait(); err != nil && !errors.Is(err, context.Canceled) {
 		// At the end of the main evaluator goroutine all nodes are killed, no need to fail the test.
@@ -87,20 +122,14 @@ func (r *testRunner) run() {
 	}
 }
 
+// run is the stock test runner
+func (r *testRunner) run() {
+	r.runBase([]runEvent{r.defaultEndToEndRun})
+}
+
+// scenarioRunner runs more complex scenarios to exercise error handling for unhappy paths
 func (r *testRunner) scenarioRunner() {
-	r.comHandler = NewComponentHandler(r.config, r.t)
-	r.comHandler.setup()
-
-	// Run E2E evaluators and tests.
-	r.addEvent(r.scenarioRun)
-
-	if err := r.comHandler.group.Wait(); err != nil && !errors.Is(err, context.Canceled) {
-		// At the end of the main evaluator goroutine all nodes are killed, no need to fail the test.
-		if strings.Contains(err.Error(), "signal: killed") {
-			return
-		}
-		r.t.Fatalf("E2E test ended in error: %v", err)
-	}
+	r.runBase([]runEvent{r.scenarioRun})
 }
 
 func (r *testRunner) waitExtra(ctx context.Context, e types.Epoch, conn *grpc.ClientConn, extra types.Epoch) error {
@@ -177,10 +206,16 @@ func (r *testRunner) testDepositsAndTx(ctx context.Context, g *errgroup.Group,
 		if err := helpers.ComponentsStarted(ctx, requiredNodes); err != nil {
 			return fmt.Errorf("deposit check validator node requires beacon nodes to run: %w", err)
 		}
+		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{r.depositor}); err != nil {
+			return errors.Wrap(err, "testDepositsAndTx unable to run, depositor did not Start")
+		}
 		go func() {
 			if r.config.TestDeposits {
 				log.Info("Running deposit tests")
-				err := components.SendAndMineDeposits(keystorePath, int(e2e.DepositCount), minGenesisActiveCount, false /* partial */)
+				// The validators with an index < minGenesisActiveCount all have deposits already from the chain start.
+				// Skip all of those chain start validators by seeking to minGenesisActiveCount in the validator list
+				// for further deposit testing.
+				err := r.depositor.SendAndMine(ctx, minGenesisActiveCount, int(e2e.DepositCount), e2etypes.PostGenesisDepositBatch, false)
 				if err != nil {
 					r.t.Fatal(err)
 				}
@@ -295,7 +330,7 @@ func (r *testRunner) testCheckpointSync(ctx context.Context, g *errgroup.Group,
 	syncEvaluators := []e2etypes.Evaluator{ev.FinishedSyncing, ev.AllNodesHaveSameHead}
 	for _, evaluator := range syncEvaluators {
 		r.t.Run(evaluator.Name, func(t *testing.T) {
-			assert.NoError(t, evaluator.Evaluation(conns...), "Evaluation failed for sync node")
+			assert.NoError(t, evaluator.Evaluation(nil, conns...), "Evaluation failed for sync node")
 		})
 	}
 	return nil
@@ -352,7 +387,7 @@ func (r *testRunner) testBeaconChainSync(ctx context.Context, g *errgroup.Group,
 	syncEvaluators := []e2etypes.Evaluator{ev.FinishedSyncing, ev.AllNodesHaveSameHead}
 	for _, evaluator := range syncEvaluators {
 		t.Run(evaluator.Name, func(t *testing.T) {
-			assert.NoError(t, evaluator.Evaluation(conns...), "Evaluation failed for sync node")
+			assert.NoError(t, evaluator.Evaluation(nil, conns...), "Evaluation failed for sync node")
 		})
 	}
 	return nil
@@ -444,7 +479,11 @@ func (r *testRunner) defaultEndToEndRun() error {
 		return errors.New("incorrect component type")
 	}
 
-	r.testDepositsAndTx(ctx, g, eth1Miner.KeystorePath(), []e2etypes.ComponentRunner{beaconNodes})
+	keypath, err := e2e.TestParams.Paths.MinerKeyPath()
+	if err != nil {
+		return errors.Wrap(err, "error getting miner key path from bazel static files in defaultEndToEndRun")
+	}
+	r.testDepositsAndTx(ctx, g, keypath, []e2etypes.ComponentRunner{beaconNodes})
 
 	// Create GRPC connection to beacon nodes.
 	conns, closeConns, err := helpers.NewLocalConnections(ctx, e2e.TestParams.BeaconNodeCount)
@@ -489,7 +528,7 @@ func (r *testRunner) defaultEndToEndRun() error {
 		syncEvaluators := []e2etypes.Evaluator{ev.FinishedSyncing, ev.AllNodesHaveSameHead}
 		for _, evaluator := range syncEvaluators {
 			t.Run(evaluator.Name, func(t *testing.T) {
-				assert.NoError(t, evaluator.Evaluation(conns...), "Evaluation failed for sync node")
+				assert.NoError(t, evaluator.Evaluation(nil, conns...), "Evaluation failed for sync node")
 			})
 		}
 	}
@@ -554,8 +593,9 @@ func (r *testRunner) executeProvidedEvaluators(currentEpoch uint64, conns []*grp
 			continue
 		}
 		wg.Add(1)
+		var ec e2etypes.EvaluationContext = r.depositor.History()
 		go r.t.Run(fmt.Sprintf(evaluator.Name, currentEpoch), func(t *testing.T) {
-			err := evaluator.Evaluation(conns...)
+			err := evaluator.Evaluation(ec, conns...)
 			assert.NoError(t, err, "Evaluation failed for epoch %d: %v", currentEpoch, err)
 			wg.Done()
 		})

testing/endtoend/evaluators/BUILD.bazel

@@ -53,6 +53,7 @@ go_library(
         "@com_github_ethereum_go_ethereum//rpc:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prysmaticlabs_go_bitfield//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
         "@io_bazel_rules_go//proto/wkt:empty_go_proto",
         "@org_golang_google_grpc//:go_default_library",
         "@org_golang_google_protobuf//types/known/emptypb:go_default_library",

testing/endtoend/evaluators/api_gateway_v1alpha1.go

@@ -33,7 +33,7 @@ const (
 
 type apiComparisonFunc func(beaconNodeIdx int, conn *grpc.ClientConn) error
 
-func apiGatewayV1Alpha1Verify(conns ...*grpc.ClientConn) error {
+func apiGatewayV1Alpha1Verify(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	for beaconNodeIdx, conn := range conns {
 		if err := runAPIComparisonFunctions(
 			beaconNodeIdx,

testing/endtoend/evaluators/api_middleware.go

@@ -51,7 +51,7 @@ const (
 	v1MiddlewarePathTemplate = "http://localhost:%d/eth/v1"
 )
 
-func apiMiddlewareVerify(conns ...*grpc.ClientConn) error {
+func apiMiddlewareVerify(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	for beaconNodeIdx, conn := range conns {
 		if err := runAPIComparisonFunctions(
 			beaconNodeIdx,

testing/endtoend/evaluators/data.go

@@ -22,7 +22,7 @@ var ColdStateCheckpoint = e2etypes.Evaluator{
 }
 
 // Checks the first node for an old checkpoint using cold state storage.
-func checkColdStateCheckpoint(conns ...*grpc.ClientConn) error {
+func checkColdStateCheckpoint(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	ctx := context.Background()
 	client := eth.NewBeaconChainClient(conns[0])
 

testing/endtoend/evaluators/execution_engine.go

@@ -22,7 +22,7 @@ var OptimisticSyncEnabled = types.Evaluator{
 	Evaluation: optimisticSyncEnabled,
 }
 
-func optimisticSyncEnabled(conns ...*grpc.ClientConn) error {
+func optimisticSyncEnabled(_ types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	for _, conn := range conns {
 		client := service.NewBeaconChainClient(conn)
 		head, err := client.GetBlockV2(context.Background(), &v2.BlockRequestV2{BlockId: []byte("head")})

testing/endtoend/evaluators/fee_recipient.go

@@ -29,7 +29,7 @@ var FeeRecipientIsPresent = types.Evaluator{
 	Evaluation: feeRecipientIsPresent,
 }
 
-func feeRecipientIsPresent(conns ...*grpc.ClientConn) error {
+func feeRecipientIsPresent(_ types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconChainClient(conn)
 	chainHead, err := client.GetChainHead(context.Background(), &emptypb.Empty{})
@@ -68,8 +68,7 @@ func feeRecipientIsPresent(conns ...*grpc.ClientConn) error {
 	}
 
 	for _, ctr := range blks.BlockContainers {
-		switch ctr.Block.(type) {
-		case *ethpb.BeaconBlockContainer_BellatrixBlock:
+		if fr := ctr.GetBellatrixBlock(); fr != nil {
 			var account common.Address
 
 			fr := ctr.GetBellatrixBlock().Block.Body.ExecutionPayload.FeeRecipient

testing/endtoend/evaluators/finality.go

@@ -23,7 +23,7 @@ var FinalizationOccurs = func(epoch ethtypes.Epoch) types.Evaluator {
 	}
 }
 
-func finalizationOccurs(conns ...*grpc.ClientConn) error {
+func finalizationOccurs(_ types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := eth.NewBeaconChainClient(conn)
 	chainHead, err := client.GetChainHead(context.Background(), &emptypb.Empty{})

testing/endtoend/evaluators/fork.go

@@ -30,7 +30,7 @@ var BellatrixForkTransition = types.Evaluator{
 	Evaluation: bellatrixForkOccurs,
 }
 
-func altairForkOccurs(conns ...*grpc.ClientConn) error {
+func altairForkOccurs(_ types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconNodeValidatorClient(conn)
 	ctx, cancel := context.WithTimeout(context.Background(), streamDeadline)
@@ -72,7 +72,7 @@ func altairForkOccurs(conns ...*grpc.ClientConn) error {
 	return nil
 }
 
-func bellatrixForkOccurs(conns ...*grpc.ClientConn) error {
+func bellatrixForkOccurs(_ types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconNodeValidatorClient(conn)
 	ctx, cancel := context.WithTimeout(context.Background(), streamDeadline)

testing/endtoend/evaluators/metrics.go

@@ -85,7 +85,7 @@ var metricComparisonTests = []comparisonTest{
 	},
 }
 
-func metricsTest(conns ...*grpc.ClientConn) error {
+func metricsTest(_ types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	genesis, err := eth.NewNodeClient(conns[0]).GetGenesis(context.Background(), &emptypb.Empty{})
 	if err != nil {
 		return err

testing/endtoend/evaluators/node.go

@@ -52,7 +52,7 @@ var AllNodesHaveSameHead = e2etypes.Evaluator{
 	Evaluation: allNodesHaveSameHead,
 }
 
-func healthzCheck(conns ...*grpc.ClientConn) error {
+func healthzCheck(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	count := len(conns)
 	for i := 0; i < count; i++ {
 		resp, err := http.Get(fmt.Sprintf("http://localhost:%d/healthz", e2e.TestParams.Ports.PrysmBeaconNodeMetricsPort+i))
@@ -94,7 +94,7 @@ func healthzCheck(conns ...*grpc.ClientConn) error {
 	return nil
 }
 
-func peersConnect(conns ...*grpc.ClientConn) error {
+func peersConnect(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	if len(conns) == 1 {
 		return nil
 	}
@@ -114,7 +114,7 @@ func peersConnect(conns ...*grpc.ClientConn) error {
 	return nil
 }
 
-func finishedSyncing(conns ...*grpc.ClientConn) error {
+func finishedSyncing(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	syncNodeClient := eth.NewNodeClient(conn)
 	syncStatus, err := syncNodeClient.GetSyncStatus(context.Background(), &emptypb.Empty{})
@@ -127,7 +127,7 @@ func finishedSyncing(conns ...*grpc.ClientConn) error {
 	return nil
 }
 
-func allNodesHaveSameHead(conns ...*grpc.ClientConn) error {
+func allNodesHaveSameHead(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	headEpochs := make([]types.Epoch, len(conns))
 	justifiedRoots := make([][]byte, len(conns))
 	prevJustifiedRoots := make([][]byte, len(conns))

testing/endtoend/evaluators/operations.go

@@ -5,13 +5,15 @@ import (
 	"context"
 	"fmt"
 	"math"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
 
 	"github.com/pkg/errors"
 	corehelpers "github.com/prysmaticlabs/prysm/v3/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/v3/beacon-chain/core/signing"
 	"github.com/prysmaticlabs/prysm/v3/config/params"
 	"github.com/prysmaticlabs/prysm/v3/consensus-types/blocks"
-	"github.com/prysmaticlabs/prysm/v3/consensus-types/interfaces"
 	types "github.com/prysmaticlabs/prysm/v3/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v3/encoding/bytesutil"
 	ethpb "github.com/prysmaticlabs/prysm/v3/proto/prysm/v1alpha1"
@@ -25,15 +27,10 @@ import (
 	"google.golang.org/protobuf/types/known/emptypb"
 )
 
-// exitedIndex holds the exited index from ProposeVoluntaryExit in memory so other functions don't confuse it
-// for a normal validator.
-var exitedIndex types.ValidatorIndex
-
-// valExited is used to know if exitedIndex is set, since default value is 0.
-var valExited bool
+var exitedVals = make(map[[48]byte]bool)
 
 // churnLimit is normally 4 unless the validator set is extremely large.
-var churnLimit = uint64(4)
+var churnLimit = 4
 var depositValCount = e2e.DepositCount
 
 // Deposits should be processed in twice the length of the epochs per eth1 voting period.
@@ -78,11 +75,11 @@ var ProposeVoluntaryExit = e2etypes.Evaluator{
 	Evaluation: proposeVoluntaryExit,
 }
 
-// ValidatorHasExited checks the beacon state for the exited validator and ensures its marked as exited.
-var ValidatorHasExited = e2etypes.Evaluator{
+// ValidatorsHaveExited checks the beacon state for the exited validator and ensures its marked as exited.
+var ValidatorsHaveExited = e2etypes.Evaluator{
 	Name:       "voluntary_has_exited_%d",
 	Policy:     policies.OnEpoch(8),
-	Evaluation: validatorIsExited,
+	Evaluation: validatorsHaveExited,
 }
 
 // ValidatorsVoteWithTheMajority verifies whether validator vote for eth1data using the majority algorithm.
@@ -92,7 +89,18 @@ var ValidatorsVoteWithTheMajority = e2etypes.Evaluator{
 	Evaluation: validatorsVoteWithTheMajority,
 }
 
-func processesDepositsInBlocks(conns ...*grpc.ClientConn) error {
+type mismatch struct {
+	k [48]byte
+	e uint64
+	o uint64
+}
+
+func (m mismatch) String() string {
+	return fmt.Sprintf("(%#x:%d:%d)", m.k, m.e, m.o)
+}
+
+func processesDepositsInBlocks(ec e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+	expected := ec.Balances(e2etypes.PostGenesisDepositBatch)
 	conn := conns[0]
 	client := ethpb.NewBeaconChainClient(conn)
 	chainHead, err := client.GetChainHead(context.Background(), &emptypb.Empty{})
@@ -105,40 +113,42 @@ func processesDepositsInBlocks(conns ...*grpc.ClientConn) error {
 	if err != nil {
 		return errors.Wrap(err, "failed to get blocks from beacon-chain")
 	}
-	var numDeposits uint64
+	observed := make(map[[48]byte]uint64)
 	for _, blk := range blks.BlockContainers {
-		var slot types.Slot
-		var eth1Data *ethpb.Eth1Data
-		var deposits []*ethpb.Deposit
-		switch blk.Block.(type) {
-		case *ethpb.BeaconBlockContainer_Phase0Block:
-			b := blk.GetPhase0Block().Block
-			slot = b.Slot
-			eth1Data = b.Body.Eth1Data
-			deposits = b.Body.Deposits
-		case *ethpb.BeaconBlockContainer_AltairBlock:
-			b := blk.GetAltairBlock().Block
-			slot = b.Slot
-			eth1Data = b.Body.Eth1Data
-			deposits = b.Body.Deposits
-		default:
-			return errors.New("block neither phase0 nor altair")
+		sb, err := blocks.BeaconBlockContainerToSignedBeaconBlock(blk)
+		if err != nil {
+			return errors.Wrap(err, "failed to convert api response type to SignedBeaconBlock interface")
 		}
+		b := sb.Block()
+		slot := b.Slot()
+		eth1Data := b.Body().Eth1Data()
+		deposits := b.Body().Deposits()
 		fmt.Printf(
 			"Slot: %d with %d deposits, Eth1 block %#x with %d deposits\n",
 			slot,
 			len(deposits),
 			eth1Data.BlockHash, eth1Data.DepositCount,
 		)
-		numDeposits += uint64(len(deposits))
+		for _, d := range deposits {
+			k := bytesutil.ToBytes48(d.Data.PublicKey)
+			v := observed[k]
+			observed[k] = v + d.Data.Amount
+		}
 	}
-	if numDeposits != depositValCount {
-		return fmt.Errorf("expected %d deposits to be processed, received %d", depositValCount, numDeposits)
+	mismatches := []string{}
+	for k, ev := range expected {
+		ov := observed[k]
+		if ev != ov {
+			mismatches = append(mismatches, mismatch{k: k, e: ev, o: ov}.String())
+		}
+	}
+	if len(mismatches) != 0 {
+		return fmt.Errorf("not all expected deposits observed on chain, len(expected)=%d, len(observed)=%d, mismatches=%d; details(key:expected:observed): %s", len(expected), len(observed), len(mismatches), strings.Join(mismatches, ","))
 	}
 	return nil
 }
 
-func verifyGraffitiInBlocks(conns ...*grpc.ClientConn) error {
+func verifyGraffitiInBlocks(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconChainClient(conn)
 	chainHead, err := client.GetChainHead(context.Background(), &emptypb.Empty{})
@@ -151,7 +161,7 @@ func verifyGraffitiInBlocks(conns ...*grpc.ClientConn) error {
 		return errors.Wrap(err, "failed to get blocks from beacon-chain")
 	}
 	for _, ctr := range blks.BlockContainers {
-		blk, err := convertToBlockInterface(ctr)
+		blk, err := blocks.BeaconBlockContainerToSignedBeaconBlock(ctr)
 		if err != nil {
 			return err
 		}
@@ -172,7 +182,7 @@ func verifyGraffitiInBlocks(conns ...*grpc.ClientConn) error {
 	return nil
 }
 
-func activatesDepositedValidators(conns ...*grpc.ClientConn) error {
+func activatesDepositedValidators(ec e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconChainClient(conn)
 
@@ -180,108 +190,133 @@ func activatesDepositedValidators(conns ...*grpc.ClientConn) error {
 	if err != nil {
 		return errors.Wrap(err, "failed to get chain head")
 	}
+	epoch := chainHead.HeadEpoch
 
-	validatorRequest := &ethpb.ListValidatorsRequest{
-		PageSize:  int32(params.BeaconConfig().MinGenesisActiveValidatorCount),
-		PageToken: "1",
-	}
-	validators, err := client.ListValidators(context.Background(), validatorRequest)
+	validators, err := getAllValidators(client)
 	if err != nil {
 		return errors.Wrap(err, "failed to get validators")
 	}
+	expected := ec.Balances(e2etypes.PostGenesisDepositBatch)
 
-	expectedCount := depositValCount
-	receivedCount := uint64(len(validators.ValidatorList))
-	if expectedCount != receivedCount {
-		return fmt.Errorf("expected validator count to be %d, recevied %d", expectedCount, receivedCount)
-	}
-
-	epoch := chainHead.HeadEpoch
-	depositsInEpoch := uint64(0)
-	var effBalanceLowCount, exitEpochWrongCount, withdrawEpochWrongCount uint64
-	for _, item := range validators.ValidatorList {
-		if item.Validator.ActivationEpoch == epoch {
-			depositsInEpoch++
-			if item.Validator.EffectiveBalance < params.BeaconConfig().MaxEffectiveBalance {
-				effBalanceLowCount++
-			}
-			if item.Validator.ExitEpoch != params.BeaconConfig().FarFutureEpoch {
-				exitEpochWrongCount++
-			}
-			if item.Validator.WithdrawableEpoch != params.BeaconConfig().FarFutureEpoch {
-				withdrawEpochWrongCount++
-			}
+	var deposits, lowBalance, wrongExit, wrongWithdraw int
+	for _, v := range validators {
+		key := bytesutil.ToBytes48(v.PublicKey)
+		if _, ok := expected[key]; !ok {
+			continue
+		}
+		delete(expected, key)
+		if v.ActivationEpoch != epoch {
+			continue
+		}
+		deposits++
+		if v.EffectiveBalance < params.BeaconConfig().MaxEffectiveBalance {
+			lowBalance++
+		}
+		if v.ExitEpoch != params.BeaconConfig().FarFutureEpoch {
+			wrongExit++
+		}
+		if v.WithdrawableEpoch != params.BeaconConfig().FarFutureEpoch {
+			wrongWithdraw++
 		}
 	}
-	if depositsInEpoch != churnLimit {
-		return fmt.Errorf("expected %d deposits to be processed in epoch %d, received %d", churnLimit, epoch, depositsInEpoch)
+
+	// Make sure every post-genesis deposit has been proecssed, resulting in a validator.
+	if len(expected) > 0 {
+		return fmt.Errorf("missing %d validators for post-genesis deposits", len(expected))
 	}
 
-	if effBalanceLowCount > 0 {
+	if deposits != churnLimit {
+		return fmt.Errorf("expected %d deposits to be processed in epoch %d, received %d", churnLimit, epoch, deposits)
+	}
+
+	if lowBalance > 0 {
 		return fmt.Errorf(
 			"%d validators did not have genesis validator effective balance of %d",
-			effBalanceLowCount,
+			lowBalance,
 			params.BeaconConfig().MaxEffectiveBalance,
 		)
-	} else if exitEpochWrongCount > 0 {
-		return fmt.Errorf("%d validators did not have an exit epoch of far future epoch", exitEpochWrongCount)
-	} else if withdrawEpochWrongCount > 0 {
-		return fmt.Errorf("%d validators did not have a withdrawable epoch of far future epoch", withdrawEpochWrongCount)
+	} else if wrongExit > 0 {
+		return fmt.Errorf("%d validators did not have an exit epoch of far future epoch", wrongExit)
+	} else if wrongWithdraw > 0 {
+		return fmt.Errorf("%d validators did not have a withdrawable epoch of far future epoch", wrongWithdraw)
 	}
 	return nil
 }
 
-func depositedValidatorsAreActive(conns ...*grpc.ClientConn) error {
+func getAllValidators(c ethpb.BeaconChainClient) ([]*ethpb.Validator, error) {
+	vals := make([]*ethpb.Validator, 0)
+	pageToken := "0"
+	for pageToken != "" {
+		validatorRequest := &ethpb.ListValidatorsRequest{
+			PageSize:  100,
+			PageToken: pageToken,
+		}
+		validators, err := c.ListValidators(context.Background(), validatorRequest)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get validators")
+		}
+		for _, v := range validators.ValidatorList {
+			vals = append(vals, v.Validator)
+		}
+		pageToken = validators.NextPageToken
+		log.WithField("len", len(vals)).WithField("pageToken", pageToken).Info("getAllValidators")
+	}
+	return vals, nil
+}
+
+func depositedValidatorsAreActive(ec e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconChainClient(conn)
-	validatorRequest := &ethpb.ListValidatorsRequest{
-		PageSize:  int32(params.BeaconConfig().MinGenesisActiveValidatorCount),
-		PageToken: "1",
-	}
-	validators, err := client.ListValidators(context.Background(), validatorRequest)
-	if err != nil {
-		return errors.Wrap(err, "failed to get validators")
-	}
-
-	expectedCount := depositValCount
-	receivedCount := uint64(len(validators.ValidatorList))
-	if expectedCount != receivedCount {
-		return fmt.Errorf("expected validator count to be %d, recevied %d", expectedCount, receivedCount)
-	}
 
 	chainHead, err := client.GetChainHead(context.Background(), &emptypb.Empty{})
 	if err != nil {
 		return errors.Wrap(err, "failed to get chain head")
 	}
 
-	inactiveCount, belowBalanceCount := 0, 0
-	for _, item := range validators.ValidatorList {
-		if !corehelpers.IsActiveValidator(item.Validator, chainHead.HeadEpoch) {
-			inactiveCount++
+	vals, err := getAllValidators(client)
+	if err != nil {
+		return errors.Wrap(err, "error retrieving validator list from API")
+	}
+	inactive := 0
+	lowBalance := 0
+	nexits := 0
+	expected := ec.Balances(e2etypes.PostGenesisDepositBatch)
+	nexpected := len(expected)
+	for _, v := range vals {
+		key := bytesutil.ToBytes48(v.PublicKey)
+		if _, ok := expected[key]; !ok {
+			continue // we aren't checking for this validator
 		}
-		if item.Validator.EffectiveBalance < params.BeaconConfig().MaxEffectiveBalance {
-			belowBalanceCount++
+		// ignore voluntary exits when checking balance and active status
+		exited := exitedVals[key]
+		if exited {
+			nexits++
+			delete(expected, key)
+			continue
 		}
+		if !corehelpers.IsActiveValidator(v, chainHead.HeadEpoch) {
+			inactive++
+		}
+		if v.EffectiveBalance < params.BeaconConfig().MaxEffectiveBalance {
+			lowBalance++
+		}
+		delete(expected, key)
+	}
+	if len(expected) > 0 {
+		mk := make([]string, 0)
+		for k := range expected {
+			mk = append(mk, fmt.Sprintf("%#x", k))
+		}
+		return fmt.Errorf("API response missing %d validators, based on deposits; keys=%s", len(expected), strings.Join(mk, ","))
+	}
+	if inactive != 0 || lowBalance != 0 {
+		return fmt.Errorf("active validator set does not match %d total deposited. %d exited, %d inactive, %d low balance", nexpected, nexits, inactive, lowBalance)
 	}
 
-	if inactiveCount > 0 {
-		return fmt.Errorf(
-			"%d validators were not active, expected %d active validators from deposits",
-			inactiveCount,
-			params.BeaconConfig().MinGenesisActiveValidatorCount,
-		)
-	}
-	if belowBalanceCount > 0 {
-		return fmt.Errorf(
-			"%d validators did not have a proper balance, expected %d validators to have 32 ETH",
-			belowBalanceCount,
-			params.BeaconConfig().MinGenesisActiveValidatorCount,
-		)
-	}
 	return nil
 }
 
-func proposeVoluntaryExit(conns ...*grpc.ClientConn) error {
+func proposeVoluntaryExit(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	valClient := ethpb.NewBeaconNodeValidatorClient(conn)
 	beaconClient := ethpb.NewBeaconChainClient(conn)
@@ -292,13 +327,12 @@ func proposeVoluntaryExit(conns ...*grpc.ClientConn) error {
 		return errors.Wrap(err, "could not get chain head")
 	}
 
-	_, privKeys, err := util.DeterministicDepositsAndKeys(params.BeaconConfig().MinGenesisActiveValidatorCount)
+	deposits, privKeys, err := util.DeterministicDepositsAndKeys(params.BeaconConfig().MinGenesisActiveValidatorCount)
 	if err != nil {
 		return err
 	}
 
-	exitedIndex = types.ValidatorIndex(rand.Uint64() % params.BeaconConfig().MinGenesisActiveValidatorCount)
-	valExited = true
+	exitedIndex := types.ValidatorIndex(rand.Uint64() % params.BeaconConfig().MinGenesisActiveValidatorCount)
 
 	voluntaryExit := &ethpb.VoluntaryExit{
 		Epoch:          chainHead.HeadEpoch,
@@ -325,28 +359,33 @@ func proposeVoluntaryExit(conns ...*grpc.ClientConn) error {
 	if _, err = valClient.ProposeExit(ctx, signedExit); err != nil {
 		return errors.Wrap(err, "could not propose exit")
 	}
+	pubk := bytesutil.ToBytes48(deposits[exitedIndex].Data.PublicKey)
+	exitedVals[pubk] = true
+
 	return nil
 }
 
-func validatorIsExited(conns ...*grpc.ClientConn) error {
+func validatorsHaveExited(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconChainClient(conn)
-	validatorRequest := &ethpb.GetValidatorRequest{
-		QueryFilter: &ethpb.GetValidatorRequest_Index{
-			Index: exitedIndex,
-		},
-	}
-	validator, err := client.GetValidator(context.Background(), validatorRequest)
-	if err != nil {
-		return errors.Wrap(err, "failed to get validators")
-	}
-	if validator.ExitEpoch == params.BeaconConfig().FarFutureEpoch {
-		return fmt.Errorf("expected validator %d to be submitted for exit", exitedIndex)
+	for k := range exitedVals {
+		validatorRequest := &ethpb.GetValidatorRequest{
+			QueryFilter: &ethpb.GetValidatorRequest_PublicKey{
+				PublicKey: k[:],
+			},
+		}
+		validator, err := client.GetValidator(context.Background(), validatorRequest)
+		if err != nil {
+			return errors.Wrap(err, "failed to get validators")
+		}
+		if validator.ExitEpoch == params.BeaconConfig().FarFutureEpoch {
+			return fmt.Errorf("expected validator %#x to be submitted for exit", k)
+		}
 	}
 	return nil
 }
 
-func validatorsVoteWithTheMajority(conns ...*grpc.ClientConn) error {
+func validatorsVoteWithTheMajority(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconChainClient(conn)
 	chainHead, err := client.GetChainHead(context.Background(), &emptypb.Empty{})
@@ -412,19 +451,3 @@ func validatorsVoteWithTheMajority(conns ...*grpc.ClientConn) error {
 }
 
 var expectedEth1DataVote []byte
-
-func convertToBlockInterface(obj *ethpb.BeaconBlockContainer) (interfaces.SignedBeaconBlock, error) {
-	if obj.GetPhase0Block() != nil {
-		return blocks.NewSignedBeaconBlock(obj.GetPhase0Block())
-	}
-	if obj.GetAltairBlock() != nil {
-		return blocks.NewSignedBeaconBlock(obj.GetAltairBlock())
-	}
-	if obj.GetBellatrixBlock() != nil {
-		return blocks.NewSignedBeaconBlock(obj.GetBellatrixBlock())
-	}
-	if obj.GetBlindedBellatrixBlock() != nil {
-		return blocks.NewSignedBeaconBlock(obj.GetBlindedBellatrixBlock())
-	}
-	return nil, errors.New("container has no block")
-}

testing/endtoend/evaluators/peers.go

@@ -19,7 +19,7 @@ var PeersCheck = types.Evaluator{
 	Evaluation: peersTest,
 }
 
-func peersTest(conns ...*grpc.ClientConn) error {
+func peersTest(_ types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	debugClient := eth.NewDebugClient(conns[0])
 
 	peerResponses, err := debugClient.ListPeers(context.Background(), &emptypb.Empty{})

testing/endtoend/evaluators/slashing.go

@@ -60,7 +60,7 @@ var SlashedValidatorsLoseBalanceAfterEpoch = func(n types.Epoch) e2eTypes.Evalua
 
 var slashedIndices []uint64
 
-func validatorsSlashed(conns ...*grpc.ClientConn) error {
+func validatorsSlashed(_ e2eTypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	ctx := context.Background()
 	client := eth.NewBeaconChainClient(conn)
@@ -75,7 +75,7 @@ func validatorsSlashed(conns ...*grpc.ClientConn) error {
 	return nil
 }
 
-func validatorsLoseBalance(conns ...*grpc.ClientConn) error {
+func validatorsLoseBalance(_ e2eTypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	ctx := context.Background()
 	client := eth.NewBeaconChainClient(conn)
@@ -106,7 +106,7 @@ func validatorsLoseBalance(conns ...*grpc.ClientConn) error {
 	return nil
 }
 
-func insertDoubleAttestationIntoPool(conns ...*grpc.ClientConn) error {
+func insertDoubleAttestationIntoPool(_ e2eTypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	valClient := eth.NewBeaconNodeValidatorClient(conn)
 	beaconClient := eth.NewBeaconChainClient(conn)
@@ -194,7 +194,7 @@ func insertDoubleAttestationIntoPool(conns ...*grpc.ClientConn) error {
 	return nil
 }
 
-func proposeDoubleBlock(conns ...*grpc.ClientConn) error {
+func proposeDoubleBlock(_ e2eTypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	valClient := eth.NewBeaconNodeValidatorClient(conn)
 	beaconClient := eth.NewBeaconChainClient(conn)

testing/endtoend/evaluators/validator.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/prysmaticlabs/prysm/v3/encoding/bytesutil"
+
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v3/beacon-chain/core/altair"
 	"github.com/prysmaticlabs/prysm/v3/config/params"
@@ -53,7 +55,7 @@ var ValidatorSyncParticipation = types.Evaluator{
 	Evaluation: validatorsSyncParticipation,
 }
 
-func validatorsAreActive(conns ...*grpc.ClientConn) error {
+func validatorsAreActive(_ types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconChainClient(conn)
 	// Balances actually fluctuate but we just want to check initial balance.
@@ -76,7 +78,7 @@ func validatorsAreActive(conns ...*grpc.ClientConn) error {
 	exitEpochWrongCount := 0
 	withdrawEpochWrongCount := 0
 	for _, item := range validators.ValidatorList {
-		if valExited && item.Index == exitedIndex {
+		if exitedVals[bytesutil.ToBytes48(item.Validator.PublicKey)] {
 			continue
 		}
 		if item.Validator.EffectiveBalance < params.BeaconConfig().MaxEffectiveBalance {
@@ -106,7 +108,7 @@ func validatorsAreActive(conns ...*grpc.ClientConn) error {
 }
 
 // validatorsParticipating ensures the validators have an acceptable participation rate.
-func validatorsParticipating(conns ...*grpc.ClientConn) error {
+func validatorsParticipating(_ types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconChainClient(conn)
 	debugClient := ethpbservice.NewBeaconDebugClient(conn)
@@ -167,7 +169,7 @@ func validatorsParticipating(conns ...*grpc.ClientConn) error {
 
 // validatorsSyncParticipation ensures the validators have an acceptable participation rate for
 // sync committee assignments.
-func validatorsSyncParticipation(conns ...*grpc.ClientConn) error {
+func validatorsSyncParticipation(_ types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewNodeClient(conn)
 	altairClient := ethpb.NewBeaconChainClient(conn)

testing/endtoend/helpers/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
     srcs = [
         "epochTimer.go",
         "helpers.go",
+        "keystore.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/v3/testing/endtoend/helpers",
     visibility = ["//testing/endtoend:__subpackages__"],
@@ -16,6 +17,10 @@ go_library(
         "//testing/endtoend/types:go_default_library",
         "//time:go_default_library",
         "//time/slots:go_default_library",
+        "@com_github_ethereum_go_ethereum//accounts/keystore:go_default_library",
+        "@com_github_ethereum_go_ethereum//ethclient:go_default_library",
+        "@com_github_ethereum_go_ethereum//rpc:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@org_golang_google_grpc//:go_default_library",
         "@org_golang_x_sync//errgroup:go_default_library",

testing/endtoend/helpers/helpers.go

@@ -17,6 +17,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
+
 	"github.com/prysmaticlabs/prysm/v3/config/params"
 	eth "github.com/prysmaticlabs/prysm/v3/proto/prysm/v1alpha1"
 	e2e "github.com/prysmaticlabs/prysm/v3/testing/endtoend/params"
@@ -59,6 +62,16 @@ func DeleteAndCreateFile(tmpPath, fileName string) (*os.File, error) {
 	return newFile, nil
 }
 
+// DeleteAndCreatePath replaces DeleteAndCreateFile where a full path is more convenient than dir,file params.
+func DeleteAndCreatePath(fp string) (*os.File, error) {
+	if _, err := os.Stat(fp); os.IsExist(err) {
+		if err = os.Remove(fp); err != nil {
+			return nil, err
+		}
+	}
+	return os.Create(filepath.Clean(fp))
+}
+
 // WaitForTextInFile checks a file every polling interval for the text requested.
 func WaitForTextInFile(file *os.File, text string) error {
 	d := time.Now().Add(maxPollingWaitTime)
@@ -333,3 +346,11 @@ func WaitOnNodes(ctx context.Context, nodes []e2etypes.ComponentRunner, nodesSta
 
 	return g.Wait()
 }
+
+func MinerRPCClient() (*ethclient.Client, error) {
+	client, err := rpc.DialHTTP(e2e.TestParams.Eth1RPCURL(e2e.MinerComponentOffset).String())
+	if err != nil {
+		return nil, err
+	}
+	return ethclient.NewClient(client), nil
+}

testing/endtoend/helpers/keystore.go

@@ -0,0 +1,17 @@
+package helpers
+
+import (
+	"os"
+
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/pkg/errors"
+)
+
+// KeyFromPath should only be used in endtoend tests. It is a simple helper to init a geth keystore.Key from a file.
+func KeyFromPath(path, pw string) (*keystore.Key, error) {
+	jsonb, err := os.ReadFile(path) // #nosec G304 -- for endtoend use only
+	if err != nil {
+		return nil, errors.Wrapf(err, "couldn't read keystore file %s", path)
+	}
+	return keystore.DecryptKey(jsonb, pw)
+}

testing/endtoend/params/BUILD.bazel

@@ -3,7 +3,10 @@ load("@prysm//tools/go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     testonly = True,
-    srcs = ["params.go"],
+    srcs = [
+        "const.go",
+        "params.go",
+    ],
     importpath = "github.com/prysmaticlabs/prysm/v3/testing/endtoend/params",
     visibility = ["//testing/endtoend:__subpackages__"],
     deps = [

testing/endtoend/params/const.go

@@ -0,0 +1,16 @@
+package params
+
+const (
+	// Every EL component has an offset that manages which port it is assigned. The miner always gets offset=0.
+	MinerComponentOffset = 0
+	Eth1StaticFilesPath  = "/testing/endtoend/static-files/eth1"
+	minerKeyFilename     = "UTC--2021-12-22T19-14-08.590377700Z--878705ba3f8bc32fcf7f4caa1a35e72af65cf766"
+	baseELHost           = "127.0.0.1"
+	baseELScheme         = "http"
+	// DepositGasLimit is the gas limit used for all deposit transactions. The exact value probably isn't important
+	// since these are the only transactions in the e2e run.
+	DepositGasLimit = 4000000
+	// SpamTxGasLimit is used for the spam transactions (to/from miner address)
+	// which WaitForBlocks generates in order to advance the EL chain.
+	SpamTxGasLimit = 21000
+)

testing/endtoend/params/params.go

@@ -5,6 +5,8 @@ package params
 import (
 	"errors"
 	"fmt"
+	"net"
+	"net/url"
 	"os"
 	"path"
 	"path/filepath"
@@ -25,6 +27,7 @@ type params struct {
 	LighthouseBeaconNodeCount int
 	ContractAddress           common.Address
 	Ports                     *ports
+	Paths                     *paths
 }
 
 type ports struct {
@@ -49,9 +52,47 @@ type ports struct {
 	JaegerTracingPort               int
 }
 
+type paths struct{}
+
+// Eth1StaticFile abstracts the location of the eth1 static file folder in the e2e directory, so that
+// a relative path can be used.
+// The relative path is specified as a variadic slice of path parts, in the same way as path.Join.
+func (*paths) Eth1StaticFile(rel ...string) string {
+	parts := append([]string{Eth1StaticFilesPath}, rel...)
+	return path.Join(parts...)
+}
+
+// Eth1Runfile returns the full path to a file in the eth1 static directory, within bazel's run context.
+// The relative path is specified as a variadic slice of path parts, in the same style as path.Join.
+func (p *paths) Eth1Runfile(rel ...string) (string, error) {
+	return bazel.Runfile(p.Eth1StaticFile(rel...))
+}
+
+// MinerKeyPath returns the full path to the file containing the miner's cryptographic keys.
+func (p *paths) MinerKeyPath() (string, error) {
+	return p.Eth1Runfile(minerKeyFilename)
+}
+
 // TestParams is the globally accessible var for getting config elements.
 var TestParams *params
 
+// Logfile gives the full path to a file in the bazel test environment log directory.
+// The relative path is specified as a variadic slice of path parts, in the same style as path.Join.
+func (p *params) Logfile(rel ...string) string {
+	return path.Join(append([]string{p.LogPath}, rel...)...)
+}
+
+// Eth1RPCURL gives the full url to use to connect to the given eth1 client's RPC endpoint.
+// The `index` param corresponds to the `index` field of the `eth1.Node` e2e component.
+// These are are off by one compared to corresponding beacon nodes, because the miner is assigned index 0.
+// eg instance the index of the EL instance associated with beacon node index `0` would typically be `1`.
+func (p *params) Eth1RPCURL(index int) *url.URL {
+	return &url.URL{
+		Scheme: baseELScheme,
+		Host:   net.JoinHostPort(baseELHost, fmt.Sprintf("%d", p.Ports.Eth1RPCPort+index)),
+	}
+}
+
 // BootNodeLogFileName is the file name used for the beacon chain node logs.
 var BootNodeLogFileName = "bootnode.log"
 
@@ -70,7 +111,7 @@ var StandardBeaconCount = 2
 // StandardLighthouseNodeCount is a global constant for the count of lighthouse beacon nodes of standard E2E tests.
 var StandardLighthouseNodeCount = 2
 
-// DepositCount is the amount of deposits E2E makes on a separate validator client.
+// DepositCount is the number of deposits the E2E runner should make to evaluate post-genesis deposit processing.
 var DepositCount = uint64(64)
 
 // NumOfExecEngineTxs is the number of transaction sent to the execution engine.

testing/endtoend/types/BUILD.bazel

@@ -3,7 +3,10 @@ load("@prysm//tools/go:def.bzl", "go_library")
 go_library(
     name = "go_default_library",
     testonly = True,
-    srcs = ["types.go"],
+    srcs = [
+        "empty.go",
+        "types.go",
+    ],
     importpath = "github.com/prysmaticlabs/prysm/v3/testing/endtoend/types",
     visibility = ["//testing/endtoend:__subpackages__"],
     deps = [

testing/endtoend/types/empty.go

@@ -0,0 +1,44 @@
+package types
+
+import (
+	"context"
+	"sync"
+)
+
+// EmptyComponent satisfies the component interface.
+// It can be embedded in other types in order to turn them into components.
+type EmptyComponent struct {
+	sync.Mutex
+	startc chan struct{}
+}
+
+func (c *EmptyComponent) Start(context.Context) error {
+	c.chanInit()
+	close(c.startc)
+	return nil
+}
+
+func (c *EmptyComponent) chanInit() {
+	c.Lock()
+	defer c.Unlock()
+	if c.startc == nil {
+		c.startc = make(chan struct{})
+	}
+}
+
+func (c *EmptyComponent) Started() <-chan struct{} {
+	c.chanInit()
+	return c.startc
+}
+
+func (*EmptyComponent) Pause() error {
+	return nil
+}
+
+func (*EmptyComponent) Resume() error {
+	return nil
+}
+
+func (*EmptyComponent) Stop() error {
+	return nil
+}

testing/endtoend/types/types.go

@@ -54,9 +54,33 @@ type E2EConfig struct {
 // Evaluator defines the structure of the evaluators used to
 // conduct the current beacon state during the E2E.
 type Evaluator struct {
-	Name       string
-	Policy     func(currentEpoch types.Epoch) bool
-	Evaluation func(conn ...*grpc.ClientConn) error // A variable amount of conns is allowed to be passed in for evaluations to check all nodes if needed.
+	Name   string
+	Policy func(currentEpoch types.Epoch) bool
+	// Evaluation accepts one or many/all conns, depending on what is needed by the set of evaluators.
+	Evaluation func(ec EvaluationContext, conn ...*grpc.ClientConn) error
+}
+
+// DepositBatch represents a group of deposits that are sent together during an e2e run.
+type DepositBatch int
+
+const (
+	// reserved zero value
+	_ DepositBatch = iota
+	// GenesisDepositBatch deposits are sent to populate the initial set of validators for genesis.
+	GenesisDepositBatch
+	// PostGenesisDepositBatch deposits are sent to test that deposits appear in blocks as expected
+	// and validators become active.
+	PostGenesisDepositBatch
+)
+
+// DepositBalancer represents a type that can sum, by validator, all deposits made in E2E prior to the function call.
+type DepositBalancer interface {
+	Balances(DepositBatch) map[[48]byte]uint64
+}
+
+// EvaluationContext allows for additional data to be provided to evaluators that need extra state.
+type EvaluationContext interface {
+	DepositBalancer
 }
 
 // ComponentRunner defines an interface via which E2E component's configuration, execution and termination is managed.

testing/util/deposits.go

@@ -275,12 +275,14 @@ func DeterministicGenesisState(t testing.TB, numValidators uint64) (state.Beacon
 // DepositTrieFromDeposits takes an array of deposits and returns the deposit trie.
 func DepositTrieFromDeposits(deposits []*ethpb.Deposit) (*trie.SparseMerkleTrie, [][32]byte, error) {
 	encodedDeposits := make([][]byte, len(deposits))
+	roots := make([][32]byte, len(deposits))
 	for i := 0; i < len(encodedDeposits); i++ {
 		hashedDeposit, err := deposits[i].Data.HashTreeRoot()
 		if err != nil {
 			return nil, [][32]byte{}, errors.Wrap(err, "could not tree hash deposit data")
 		}
 		encodedDeposits[i] = hashedDeposit[:]
+		roots[i] = hashedDeposit
 	}
 
 	depositTrie, err := trie.GenerateTrieFromItems(encodedDeposits, params.BeaconConfig().DepositContractTreeDepth)
@@ -288,11 +290,6 @@ func DepositTrieFromDeposits(deposits []*ethpb.Deposit) (*trie.SparseMerkleTrie,
 		return nil, [][32]byte{}, errors.Wrap(err, "Could not generate deposit trie")
 	}
 
-	roots := make([][32]byte, len(deposits))
-	for i, dep := range encodedDeposits {
-		roots[i] = bytesutil.ToBytes32(dep)
-	}
-
 	return depositTrie, roots, nil
 }