Merge branch 'develop' of https://github.com/prysmaticlabs/geth-sharding into trie-debug

* e2e: Provide e2e config yaml to web3signer

* fix build for //testing/endtoend:go_default_test

* Update with web3signer with teku fixes

* buildifier

* Add slasher case for web3signer

* Update testing/endtoend/minimal_e2e_test.go

* Update web3signer to 21.10.6

* Revert "Update web3signer to 21.10.6"

This reverts commit bdf3c408f2.

* Remove slasher part of web3signer e2e tests

* Revert "Remove slasher part of web3signer e2e tests"

This reverts commit 24249802ae.

* fix slasher web3signer test

* fixing build

* updating yaml to match testnet_e2e_config.go

* trying a different order to the e2e test and adding a log

* trying different way to kill process

* handling unhandled error

* testing changes to config WIP

* fixing bazel WIP

* fixing build

* ignoring test for now to test

* fixing bazel

* Test only web3signer e2e

* rolling back some commits to test

* fixing bazel

* trying an updated web3signer version

* changing flag to match version

* trying current version of develop for web3signer

* testing not using the --network property

* addressing build error

* testing config change

* reverting to go bakc to using the network file

* testing adding epochs per sync committee period

* rolling back configs

* removing check to test

* adding log to get sync committee duties and changing bellatrix fork epoch to something large and altair to epoch 1

* fixing bazel

* updating epoch in config file

* fixing more descrepencies between the configurations

* removing un used yaml

* removing goland added duplicates

* reverting using network minimal

* fixing bug

* rolling back some changes

* rolling back some changes

* rolling back changes

* making sure web3signer test doesn't make it to bellatrix fork yet

* reverting changes I did not touch

* undo comment

* Update testing/endtoend/deps.bzl

* Apply suggestions from code review

* rm nl

* fix //testing/endtoend:go_mainnet_test

* Remove unnecessary dep

* fix //testing/endtoend:go_mainnet_test

* addressing review comments

* fixing build and internal conflict

* removing web3signer slasher test as it's unneeded due to the interface nature of key signing, the regular slashing test is enough

* fix: The validator we fetch from the binary can only run before altair, if you add that and the web3signer then these things can never run together as the web3signer sets it to before bellatrix

Co-authored-by: james-prysm <90280386+james-prysm@users.noreply.github.com>
Co-authored-by: James He <james@prysmaticlabs.com>
Co-authored-by: Nishant Das <nishdas93@gmail.com>
Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>

WORKSPACE

@@ -176,7 +176,7 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe
 go_rules_dependencies()
 
 go_register_toolchains(
-    go_version = "1.17.6",
+    go_version = "1.17.9",
     nogo = "@//:nogo",
 )
 

beacon-chain/blockchain/BUILD.bazel

@@ -66,6 +66,7 @@ go_library(
         "//config/params:go_default_library",
         "//crypto/bls:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//math:go_default_library",
         "//monitoring/tracing:go_default_library",
         "//proto/engine/v1:go_default_library",
         "//proto/eth/v1:go_default_library",

beacon-chain/blockchain/chain_info.go

@@ -273,13 +273,13 @@ func (s *Service) CurrentFork() *ethpb.Fork {
 
 // IsCanonical returns true if the input block root is part of the canonical chain.
 func (s *Service) IsCanonical(ctx context.Context, blockRoot [32]byte) (bool, error) {
-	// If the block has been finalized, the block will always be part of the canonical chain.
-	if s.cfg.BeaconDB.IsFinalizedBlock(ctx, blockRoot) {
-		return true, nil
+	// If the block has not been finalized, check fork choice store to see if the block is canonical
+	if s.cfg.ForkChoiceStore.HasNode(blockRoot) {
+		return s.cfg.ForkChoiceStore.IsCanonical(blockRoot), nil
 	}
 
-	// If the block has not been finalized, check fork choice store to see if the block is canonical
-	return s.cfg.ForkChoiceStore.IsCanonical(blockRoot), nil
+	// If the block has been finalized, the block will always be part of the canonical chain.
+	return s.cfg.BeaconDB.IsFinalizedBlock(ctx, blockRoot), nil
 }
 
 // ChainHeads returns all possible chain heads (leaves of fork choice tree).

beacon-chain/blockchain/process_block_helpers.go

@@ -11,6 +11,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/config/params"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	mathutil "github.com/prysmaticlabs/prysm/math"
 	"github.com/prysmaticlabs/prysm/monitoring/tracing"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
@@ -402,10 +403,13 @@ func (s *Service) insertFinalizedDeposits(ctx context.Context, fRoot [32]byte) e
 	// We update the cache up to the last deposit index in the finalized block's state.
 	// We can be confident that these deposits will be included in some block
 	// because the Eth1 follow distance makes such long-range reorgs extremely unlikely.
-	eth1DepositIndex := int64(finalizedState.Eth1Data().DepositCount - 1)
-	s.cfg.DepositCache.InsertFinalizedDeposits(ctx, eth1DepositIndex)
+	eth1DepositIndex, err := mathutil.Int(finalizedState.Eth1DepositIndex())
+	if err != nil {
+		return errors.Wrap(err, "could not cast eth1 deposit index")
+	}
+	s.cfg.DepositCache.InsertFinalizedDeposits(ctx, int64(eth1DepositIndex))
 	// Deposit proofs are only used during state transition and can be safely removed to save space.
-	if err = s.cfg.DepositCache.PruneProofs(ctx, eth1DepositIndex); err != nil {
+	if err = s.cfg.DepositCache.PruneProofs(ctx, int64(eth1DepositIndex)); err != nil {
 		return errors.Wrap(err, "could not prune deposit proofs")
 	}
 	return nil

beacon-chain/blockchain/process_block_test.go

@@ -1516,6 +1516,7 @@ func TestInsertFinalizedDeposits(t *testing.T) {
 	service.store.SetFinalizedCheckpt(&ethpb.Checkpoint{Root: gRoot[:]})
 	gs = gs.Copy()
 	assert.NoError(t, gs.SetEth1Data(&ethpb.Eth1Data{DepositCount: 10}))
+	assert.NoError(t, gs.SetEth1DepositIndex(7))
 	assert.NoError(t, service.cfg.StateGen.SaveState(ctx, [32]byte{'m', 'o', 'c', 'k'}, gs))
 	zeroSig := [96]byte{}
 	for i := uint64(0); i < uint64(4*params.BeaconConfig().SlotsPerEpoch); i++ {
@@ -1529,8 +1530,64 @@ func TestInsertFinalizedDeposits(t *testing.T) {
 	}
 	assert.NoError(t, service.insertFinalizedDeposits(ctx, [32]byte{'m', 'o', 'c', 'k'}))
 	fDeposits := depositCache.FinalizedDeposits(ctx)
-	assert.Equal(t, 9, int(fDeposits.MerkleTrieIndex), "Finalized deposits not inserted correctly")
-	deps := depositCache.AllDeposits(ctx, big.NewInt(109))
+	assert.Equal(t, 7, int(fDeposits.MerkleTrieIndex), "Finalized deposits not inserted correctly")
+	deps := depositCache.AllDeposits(ctx, big.NewInt(107))
+	for _, d := range deps {
+		assert.DeepEqual(t, [][]byte(nil), d.Proof, "Proofs are not empty")
+	}
+}
+
+func TestInsertFinalizedDeposits_MultipleFinalizedRoutines(t *testing.T) {
+	ctx := context.Background()
+	opts := testServiceOptsWithDB(t)
+	depositCache, err := depositcache.New()
+	require.NoError(t, err)
+	opts = append(opts, WithDepositCache(depositCache))
+	service, err := NewService(ctx, opts...)
+	require.NoError(t, err)
+
+	gs, _ := util.DeterministicGenesisState(t, 32)
+	require.NoError(t, service.saveGenesisData(ctx, gs))
+	gBlk, err := service.cfg.BeaconDB.GenesisBlock(ctx)
+	require.NoError(t, err)
+	gRoot, err := gBlk.Block().HashTreeRoot()
+	require.NoError(t, err)
+	service.store.SetFinalizedCheckpt(&ethpb.Checkpoint{Root: gRoot[:]})
+	gs = gs.Copy()
+	assert.NoError(t, gs.SetEth1Data(&ethpb.Eth1Data{DepositCount: 7}))
+	assert.NoError(t, gs.SetEth1DepositIndex(5))
+	assert.NoError(t, service.cfg.StateGen.SaveState(ctx, [32]byte{'m', 'o', 'c', 'k'}, gs))
+	gs2 := gs.Copy()
+	assert.NoError(t, gs2.SetEth1Data(&ethpb.Eth1Data{DepositCount: 15}))
+	assert.NoError(t, gs2.SetEth1DepositIndex(12))
+	assert.NoError(t, service.cfg.StateGen.SaveState(ctx, [32]byte{'m', 'o', 'c', 'k', '2'}, gs2))
+	zeroSig := [96]byte{}
+	for i := uint64(0); i < uint64(4*params.BeaconConfig().SlotsPerEpoch); i++ {
+		root := []byte(strconv.Itoa(int(i)))
+		assert.NoError(t, depositCache.InsertDeposit(ctx, &ethpb.Deposit{Data: &ethpb.Deposit_Data{
+			PublicKey:             bytesutil.FromBytes48([fieldparams.BLSPubkeyLength]byte{}),
+			WithdrawalCredentials: params.BeaconConfig().ZeroHash[:],
+			Amount:                0,
+			Signature:             zeroSig[:],
+		}, Proof: [][]byte{root}}, 100+i, int64(i), bytesutil.ToBytes32(root)))
+	}
+	// Insert 3 deposits before hand.
+	depositCache.InsertFinalizedDeposits(ctx, 2)
+
+	assert.NoError(t, service.insertFinalizedDeposits(ctx, [32]byte{'m', 'o', 'c', 'k'}))
+	fDeposits := depositCache.FinalizedDeposits(ctx)
+	assert.Equal(t, 5, int(fDeposits.MerkleTrieIndex), "Finalized deposits not inserted correctly")
+
+	deps := depositCache.AllDeposits(ctx, big.NewInt(105))
+	for _, d := range deps {
+		assert.DeepEqual(t, [][]byte(nil), d.Proof, "Proofs are not empty")
+	}
+
+	// Insert New Finalized State with higher deposit count.
+	assert.NoError(t, service.insertFinalizedDeposits(ctx, [32]byte{'m', 'o', 'c', 'k', '2'}))
+	fDeposits = depositCache.FinalizedDeposits(ctx)
+	assert.Equal(t, 12, int(fDeposits.MerkleTrieIndex), "Finalized deposits not inserted correctly")
+	deps = depositCache.AllDeposits(ctx, big.NewInt(112))
 	for _, d := range deps {
 		assert.DeepEqual(t, [][]byte(nil), d.Proof, "Proofs are not empty")
 	}

beacon-chain/blockchain/receive_attestation.go

@@ -83,7 +83,8 @@ func (s *Service) VerifyLmdFfgConsistency(ctx context.Context, a *ethpb.Attestat
 func (s *Service) VerifyFinalizedConsistency(ctx context.Context, root []byte) error {
 	// A canonical root implies the root to has an ancestor that aligns with finalized check point.
 	// In this case, we could exit early to save on additional computation.
-	if s.cfg.ForkChoiceStore.IsCanonical(bytesutil.ToBytes32(root)) {
+	blockRoot := bytesutil.ToBytes32(root)
+	if s.cfg.ForkChoiceStore.HasNode(blockRoot) && s.cfg.ForkChoiceStore.IsCanonical(blockRoot) {
 		return nil
 	}
 

beacon-chain/cache/depositcache/deposits_cache.go

@@ -137,6 +137,22 @@ func (dc *DepositCache) InsertFinalizedDeposits(ctx context.Context, eth1Deposit
 
 	depositTrie := dc.finalizedDeposits.Deposits
 	insertIndex := int(dc.finalizedDeposits.MerkleTrieIndex + 1)
+
+	// Don't insert into finalized trie if there is no deposit to
+	// insert.
+	if len(dc.deposits) == 0 {
+		return
+	}
+	// In the event we have less deposits than we need to
+	// finalize we finalize till the index on which we do have it.
+	if len(dc.deposits) <= int(eth1DepositIndex) {
+		eth1DepositIndex = int64(len(dc.deposits)) - 1
+	}
+	// If we finalize to some lower deposit index, we
+	// ignore it.
+	if int(eth1DepositIndex) < insertIndex {
+		return
+	}
 	for _, d := range dc.deposits {
 		if d.Index <= dc.finalizedDeposits.MerkleTrieIndex {
 			continue

beacon-chain/cache/depositcache/deposits_cache_test.go

@@ -459,7 +459,7 @@ func TestFinalizedDeposits_UtilizesPreviouslyCachedDeposits(t *testing.T) {
 			Index: 1,
 		},
 	}
-	newFinalizedDeposit := ethpb.DepositContainer{
+	newFinalizedDeposit := &ethpb.DepositContainer{
 		Deposit: &ethpb.Deposit{
 			Data: &ethpb.Deposit_Data{
 				PublicKey:             bytesutil.PadTo([]byte{2}, 48),
@@ -471,17 +471,17 @@ func TestFinalizedDeposits_UtilizesPreviouslyCachedDeposits(t *testing.T) {
 	}
 	dc.deposits = oldFinalizedDeposits
 	dc.InsertFinalizedDeposits(context.Background(), 1)
-	// Artificially exclude old deposits so that they can only be retrieved from previously finalized deposits.
-	dc.deposits = []*ethpb.DepositContainer{&newFinalizedDeposit}
 
 	dc.InsertFinalizedDeposits(context.Background(), 2)
 
+	dc.deposits = append(dc.deposits, []*ethpb.DepositContainer{newFinalizedDeposit}...)
+
 	cachedDeposits := dc.FinalizedDeposits(context.Background())
 	require.NotNil(t, cachedDeposits, "Deposits not cached")
-	assert.Equal(t, int64(2), cachedDeposits.MerkleTrieIndex)
+	assert.Equal(t, int64(1), cachedDeposits.MerkleTrieIndex)
 
 	var deps [][]byte
-	for _, d := range append(oldFinalizedDeposits, &newFinalizedDeposit) {
+	for _, d := range oldFinalizedDeposits {
 		hash, err := d.Deposit.Data.HashTreeRoot()
 		require.NoError(t, err, "Could not hash deposit data")
 		deps = append(deps, hash[:])
@@ -491,6 +491,140 @@ func TestFinalizedDeposits_UtilizesPreviouslyCachedDeposits(t *testing.T) {
 	assert.Equal(t, trie.HashTreeRoot(), cachedDeposits.Deposits.HashTreeRoot())
 }
 
+func TestFinalizedDeposits_HandleZeroDeposits(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	dc.InsertFinalizedDeposits(context.Background(), 2)
+
+	cachedDeposits := dc.FinalizedDeposits(context.Background())
+	require.NotNil(t, cachedDeposits, "Deposits not cached")
+	assert.Equal(t, int64(-1), cachedDeposits.MerkleTrieIndex)
+}
+
+func TestFinalizedDeposits_HandleSmallerThanExpectedDeposits(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	finalizedDeposits := []*ethpb.DepositContainer{
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{0}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 0,
+		},
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{1}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 1,
+		},
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{2}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 2,
+		},
+	}
+	dc.deposits = finalizedDeposits
+
+	dc.InsertFinalizedDeposits(context.Background(), 5)
+
+	cachedDeposits := dc.FinalizedDeposits(context.Background())
+	require.NotNil(t, cachedDeposits, "Deposits not cached")
+	assert.Equal(t, int64(2), cachedDeposits.MerkleTrieIndex)
+}
+
+func TestFinalizedDeposits_HandleLowerEth1DepositIndex(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	finalizedDeposits := []*ethpb.DepositContainer{
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{0}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 0,
+		},
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{1}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 1,
+		},
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{2}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 2,
+		},
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{3}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 3,
+		},
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{4}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 4,
+		},
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{5}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 5,
+		},
+	}
+	dc.deposits = finalizedDeposits
+
+	dc.InsertFinalizedDeposits(context.Background(), 5)
+
+	// Reinsert finalized deposits with a lower index.
+	dc.InsertFinalizedDeposits(context.Background(), 2)
+
+	cachedDeposits := dc.FinalizedDeposits(context.Background())
+	require.NotNil(t, cachedDeposits, "Deposits not cached")
+	assert.Equal(t, int64(5), cachedDeposits.MerkleTrieIndex)
+}
+
 func TestFinalizedDeposits_InitializedCorrectly(t *testing.T) {
 	dc, err := New()
 	require.NoError(t, err)

beacon-chain/powchain/engine_client.go

@@ -10,9 +10,12 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/holiman/uint256"
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 	pb "github.com/prysmaticlabs/prysm/proto/engine/v1"
+	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 )
 
@@ -49,8 +52,8 @@ type EngineCaller interface {
 	ExchangeTransitionConfiguration(
 		ctx context.Context, cfg *pb.TransitionConfiguration,
 	) error
-	LatestExecutionBlock(ctx context.Context) (*pb.ExecutionBlock, error)
 	ExecutionBlockByHash(ctx context.Context, hash common.Hash) (*pb.ExecutionBlock, error)
+	GetTerminalBlockHash(ctx context.Context) ([]byte, bool, error)
 }
 
 // NewPayload calls the engine_newPayloadV1 method via JSON-RPC.
@@ -174,6 +177,78 @@ func (s *Service) ExchangeTransitionConfiguration(
 	return nil
 }
 
+// GetTerminalBlockHash returns the valid terminal block hash based on total difficulty.
+//
+// Spec code:
+// def get_pow_block_at_terminal_total_difficulty(pow_chain: Dict[Hash32, PowBlock]) -> Optional[PowBlock]:
+//    # `pow_chain` abstractly represents all blocks in the PoW chain
+//    for block in pow_chain:
+//        parent = pow_chain[block.parent_hash]
+//        block_reached_ttd = block.total_difficulty >= TERMINAL_TOTAL_DIFFICULTY
+//        parent_reached_ttd = parent.total_difficulty >= TERMINAL_TOTAL_DIFFICULTY
+//        if block_reached_ttd and not parent_reached_ttd:
+//            return block
+//
+//    return None
+func (s *Service) GetTerminalBlockHash(ctx context.Context) ([]byte, bool, error) {
+	ttd := new(big.Int)
+	ttd.SetString(params.BeaconConfig().TerminalTotalDifficulty, 10)
+	terminalTotalDifficulty, overflows := uint256.FromBig(ttd)
+	if overflows {
+		return nil, false, errors.New("could not convert terminal total difficulty to uint256")
+	}
+	blk, err := s.LatestExecutionBlock(ctx)
+	if err != nil {
+		return nil, false, errors.Wrap(err, "could not get latest execution block")
+	}
+	if blk == nil {
+		return nil, false, errors.New("latest execution block is nil")
+	}
+
+	for {
+		if ctx.Err() != nil {
+			return nil, false, ctx.Err()
+		}
+		currentTotalDifficulty, err := tDStringToUint256(blk.TotalDifficulty)
+		if err != nil {
+			return nil, false, errors.Wrap(err, "could not convert total difficulty to uint256")
+		}
+		blockReachedTTD := currentTotalDifficulty.Cmp(terminalTotalDifficulty) >= 0
+
+		parentHash := bytesutil.ToBytes32(blk.ParentHash)
+		if len(blk.ParentHash) == 0 || parentHash == params.BeaconConfig().ZeroHash {
+			return nil, false, nil
+		}
+		parentBlk, err := s.ExecutionBlockByHash(ctx, parentHash)
+		if err != nil {
+			return nil, false, errors.Wrap(err, "could not get parent execution block")
+		}
+		if parentBlk == nil {
+			return nil, false, errors.New("parent execution block is nil")
+		}
+		if blockReachedTTD {
+			parentTotalDifficulty, err := tDStringToUint256(parentBlk.TotalDifficulty)
+			if err != nil {
+				return nil, false, errors.Wrap(err, "could not convert total difficulty to uint256")
+			}
+			parentReachedTTD := parentTotalDifficulty.Cmp(terminalTotalDifficulty) >= 0
+			if !parentReachedTTD {
+				log.WithFields(logrus.Fields{
+					"number":   blk.Number,
+					"hash":     fmt.Sprintf("%#x", bytesutil.Trunc(blk.Hash)),
+					"td":       blk.TotalDifficulty,
+					"parentTd": parentBlk.TotalDifficulty,
+					"ttd":      terminalTotalDifficulty,
+				}).Info("Retrieved terminal block hash")
+				return blk.Hash, true, nil
+			}
+		} else {
+			return nil, false, nil
+		}
+		blk = parentBlk
+	}
+}
+
 // LatestExecutionBlock fetches the latest execution engine block by calling
 // eth_blockByNumber via JSON-RPC.
 func (s *Service) LatestExecutionBlock(ctx context.Context) (*pb.ExecutionBlock, error) {
@@ -251,3 +326,15 @@ func isTimeout(e error) bool {
 	t, ok := e.(httpTimeoutError)
 	return ok && t.Timeout()
 }
+
+func tDStringToUint256(td string) (*uint256.Int, error) {
+	b, err := hexutil.DecodeBig(td)
+	if err != nil {
+		return nil, err
+	}
+	i, overflows := uint256.FromBig(b)
+	if overflows {
+		return nil, errors.New("total difficulty overflowed")
+	}
+	return i, nil
+}

beacon-chain/powchain/engine_client_test.go

@@ -418,6 +418,155 @@ func TestClient_HTTP(t *testing.T) {
 	})
 }
 
+func TestServer_getPowBlockHashAtTerminalTotalDifficulty(t *testing.T) {
+	tests := []struct {
+		name                  string
+		paramsTd              string
+		currentPowBlock       *pb.ExecutionBlock
+		parentPowBlock        *pb.ExecutionBlock
+		errLatestExecutionBlk error
+		wantTerminalBlockHash []byte
+		wantExists            bool
+		errString             string
+	}{
+		{
+			name:      "config td overflows",
+			paramsTd:  "1115792089237316195423570985008687907853269984665640564039457584007913129638912",
+			errString: "could not convert terminal total difficulty to uint256",
+		},
+		{
+			name:                  "could not get latest execution block",
+			paramsTd:              "1",
+			errLatestExecutionBlk: errors.New("blah"),
+			errString:             "could not get latest execution block",
+		},
+		{
+			name:      "nil latest execution block",
+			paramsTd:  "1",
+			errString: "latest execution block is nil",
+		},
+		{
+			name:     "current execution block invalid TD",
+			paramsTd: "1",
+			currentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'a'},
+				TotalDifficulty: "1115792089237316195423570985008687907853269984665640564039457584007913129638912",
+			},
+			errString: "could not convert total difficulty to uint256",
+		},
+		{
+			name:     "current execution block has zero hash parent",
+			paramsTd: "2",
+			currentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'a'},
+				ParentHash:      params.BeaconConfig().ZeroHash[:],
+				TotalDifficulty: "0x3",
+			},
+		},
+		{
+			name:     "could not get parent block",
+			paramsTd: "2",
+			currentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'a'},
+				ParentHash:      []byte{'b'},
+				TotalDifficulty: "0x3",
+			},
+			errString: "could not get parent execution block",
+		},
+		{
+			name:     "parent execution block invalid TD",
+			paramsTd: "2",
+			currentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'a'},
+				ParentHash:      []byte{'b'},
+				TotalDifficulty: "0x3",
+			},
+			parentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'b'},
+				ParentHash:      []byte{'c'},
+				TotalDifficulty: "1",
+			},
+			errString: "could not convert total difficulty to uint256",
+		},
+		{
+			name:     "happy case",
+			paramsTd: "2",
+			currentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'a'},
+				ParentHash:      []byte{'b'},
+				TotalDifficulty: "0x3",
+			},
+			parentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'b'},
+				ParentHash:      []byte{'c'},
+				TotalDifficulty: "0x1",
+			},
+			wantExists:            true,
+			wantTerminalBlockHash: []byte{'a'},
+		},
+		{
+			name:     "ttd not reached",
+			paramsTd: "3",
+			currentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'a'},
+				ParentHash:      []byte{'b'},
+				TotalDifficulty: "0x2",
+			},
+			parentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'b'},
+				ParentHash:      []byte{'c'},
+				TotalDifficulty: "0x1",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := params.BeaconConfig()
+			cfg.TerminalTotalDifficulty = tt.paramsTd
+			params.OverrideBeaconConfig(cfg)
+			var m map[[32]byte]*pb.ExecutionBlock
+			if tt.parentPowBlock != nil {
+				m = map[[32]byte]*pb.ExecutionBlock{
+					bytesutil.ToBytes32(tt.parentPowBlock.Hash): tt.parentPowBlock,
+				}
+			}
+			client := mocks.EngineClient{
+				ErrLatestExecBlock: tt.errLatestExecutionBlk,
+				ExecutionBlock:     tt.currentPowBlock,
+				BlockByHashMap:     m,
+			}
+			b, e, err := client.GetTerminalBlockHash(context.Background())
+			if tt.errString != "" {
+				require.ErrorContains(t, tt.errString, err)
+			} else {
+				require.NoError(t, err)
+				require.DeepEqual(t, tt.wantExists, e)
+				require.DeepEqual(t, tt.wantTerminalBlockHash, b)
+			}
+		})
+	}
+}
+
+func Test_tDStringToUint256(t *testing.T) {
+	i, err := tDStringToUint256("0x0")
+	require.NoError(t, err)
+	require.DeepEqual(t, uint256.NewInt(0), i)
+
+	i, err = tDStringToUint256("0x10000")
+	require.NoError(t, err)
+	require.DeepEqual(t, uint256.NewInt(65536), i)
+
+	_, err = tDStringToUint256("100")
+	require.ErrorContains(t, "hex string without 0x prefix", err)
+
+	_, err = tDStringToUint256("0xzzzzzz")
+	require.ErrorContains(t, "invalid hex string", err)
+
+	_, err = tDStringToUint256("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" +
+		"FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF")
+	require.ErrorContains(t, "hex number > 256 bits", err)
+}
+
 func TestExchangeTransitionConfiguration(t *testing.T) {
 	fix := fixtures()
 	ctx := context.Background()

beacon-chain/powchain/testing/BUILD.bazel

@@ -26,6 +26,7 @@ go_library(
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
         "@com_github_ethereum_go_ethereum//rpc:go_default_library",
+        "@com_github_holiman_uint256//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
     ],
 )

beacon-chain/powchain/testing/mock_engine_client.go

@@ -2,25 +2,32 @@ package testing
 
 import (
 	"context"
+	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/holiman/uint256"
 	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 	pb "github.com/prysmaticlabs/prysm/proto/engine/v1"
 )
 
 // EngineClient --
 type EngineClient struct {
-	NewPayloadResp        []byte
-	PayloadIDBytes        *pb.PayloadIDBytes
-	ForkChoiceUpdatedResp []byte
-	ExecutionPayload      *pb.ExecutionPayload
-	ExecutionBlock        *pb.ExecutionBlock
-	Err                   error
-	ErrLatestExecBlock    error
-	ErrExecBlockByHash    error
-	ErrForkchoiceUpdated  error
-	ErrNewPayload         error
-	BlockByHashMap        map[[32]byte]*pb.ExecutionBlock
+	NewPayloadResp          []byte
+	PayloadIDBytes          *pb.PayloadIDBytes
+	ForkChoiceUpdatedResp   []byte
+	ExecutionPayload        *pb.ExecutionPayload
+	ExecutionBlock          *pb.ExecutionBlock
+	Err                     error
+	ErrLatestExecBlock      error
+	ErrExecBlockByHash      error
+	ErrForkchoiceUpdated    error
+	ErrNewPayload           error
+	BlockByHashMap          map[[32]byte]*pb.ExecutionBlock
+	TerminalBlockHash       []byte
+	TerminalBlockHashExists bool
 }
 
 // NewPayload --
@@ -58,3 +65,52 @@ func (e *EngineClient) ExecutionBlockByHash(_ context.Context, h common.Hash) (*
 	}
 	return b, e.ErrExecBlockByHash
 }
+
+// GetTerminalBlockHash --
+func (e *EngineClient) GetTerminalBlockHash(ctx context.Context) ([]byte, bool, error) {
+	ttd := new(big.Int)
+	ttd.SetString(params.BeaconConfig().TerminalTotalDifficulty, 10)
+	terminalTotalDifficulty, overflows := uint256.FromBig(ttd)
+	if overflows {
+		return nil, false, errors.New("could not convert terminal total difficulty to uint256")
+	}
+	blk, err := e.LatestExecutionBlock(ctx)
+	if err != nil {
+		return nil, false, errors.Wrap(err, "could not get latest execution block")
+	}
+	if blk == nil {
+		return nil, false, errors.New("latest execution block is nil")
+	}
+
+	for {
+		b, err := hexutil.DecodeBig(blk.TotalDifficulty)
+		if err != nil {
+			return nil, false, errors.Wrap(err, "could not convert total difficulty to uint256")
+		}
+		currentTotalDifficulty, _ := uint256.FromBig(b)
+		blockReachedTTD := currentTotalDifficulty.Cmp(terminalTotalDifficulty) >= 0
+
+		parentHash := bytesutil.ToBytes32(blk.ParentHash)
+		if len(blk.ParentHash) == 0 || parentHash == params.BeaconConfig().ZeroHash {
+			return nil, false, nil
+		}
+		parentBlk, err := e.ExecutionBlockByHash(ctx, parentHash)
+		if err != nil {
+			return nil, false, errors.Wrap(err, "could not get parent execution block")
+		}
+		if blockReachedTTD {
+			b, err := hexutil.DecodeBig(parentBlk.TotalDifficulty)
+			if err != nil {
+				return nil, false, errors.Wrap(err, "could not convert total difficulty to uint256")
+			}
+			parentTotalDifficulty, _ := uint256.FromBig(b)
+			parentReachedTTD := parentTotalDifficulty.Cmp(terminalTotalDifficulty) >= 0
+			if !parentReachedTTD {
+				return blk.Hash, true, nil
+			}
+		} else {
+			return nil, false, nil
+		}
+		blk = parentBlk
+	}
+}

beacon-chain/rpc/prysm/v1alpha1/validator/BUILD.bazel

@@ -77,7 +77,6 @@ go_library(
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_ferranbt_fastssz//:go_default_library",
-        "@com_github_holiman_uint256//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
@@ -162,7 +161,6 @@ go_test(
         "@com_github_d4l3k_messagediff//:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_golang_mock//gomock:go_default_library",
-        "@com_github_holiman_uint256//:go_default_library",
         "@com_github_prysmaticlabs_eth2_types//:go_default_library",
         "@com_github_prysmaticlabs_go_bitfield//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",

beacon-chain/rpc/prysm/v1alpha1/validator/proposer_deposits.go

@@ -159,9 +159,12 @@ func (vs *Server) depositTrie(ctx context.Context, canonicalEth1Data *ethpb.Eth1
 
 	var depositTrie *trie.SparseMerkleTrie
 
+	log.Infof("eth1 data with hash %#x and count %d at height %d", canonicalEth1Data.BlockHash, canonicalEth1Data.DepositCount, canonicalEth1DataHeight.Uint64())
 	finalizedDeposits := vs.DepositFetcher.FinalizedDeposits(ctx)
 	depositTrie = finalizedDeposits.Deposits
+	log.Infof("deposit trie has number of items of %d at index current index %d", depositTrie.NumOfItems(), finalizedDeposits.MerkleTrieIndex)
 	upToEth1DataDeposits := vs.DepositFetcher.NonFinalizedDeposits(ctx, finalizedDeposits.MerkleTrieIndex, canonicalEth1DataHeight)
+	log.Infof("received %d non finalized deposits", len(upToEth1DataDeposits))
 	insertIndex := finalizedDeposits.MerkleTrieIndex + 1
 
 	for _, dep := range upToEth1DataDeposits {
@@ -178,7 +181,20 @@ func (vs *Server) depositTrie(ctx context.Context, canonicalEth1Data *ethpb.Eth1
 	// Log a warning here, as the cached trie is invalid.
 	if !valid {
 		log.Warnf("Cached deposit trie is invalid, rebuilding it now: %v", err)
-		return vs.rebuildDepositTrie(ctx, canonicalEth1Data, canonicalEth1DataHeight)
+		rebuiltTrie, err := vs.rebuildDepositTrie(ctx, canonicalEth1Data, canonicalEth1DataHeight)
+		if err != nil {
+			return nil, err
+		}
+		oldItems := depositTrie.Items()
+		newItems := rebuiltTrie.Items()
+		for i, item := range newItems {
+			if i >= len(oldItems) {
+				continue
+			}
+			if !bytes.Equal(item, oldItems[i]) {
+				log.Warnf("Index %d has a different result: wanted %#x but got %#x", i, item, oldItems[i])
+			}
+		}
 	}
 
 	return depositTrie, nil

beacon-chain/rpc/prysm/v1alpha1/validator/proposer_execution_payload.go

@@ -3,11 +3,7 @@ package validator
 import (
 	"bytes"
 	"context"
-	"fmt"
-	"math/big"
 
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/holiman/uint256"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
@@ -181,79 +177,7 @@ func (vs *Server) getTerminalBlockHashIfExists(ctx context.Context) ([]byte, boo
 		return terminalBlockHash.Bytes(), true, nil
 	}
 
-	return vs.getPowBlockHashAtTerminalTotalDifficulty(ctx)
-}
-
-// This returns the valid terminal block hash based on total difficulty.
-//
-// Spec code:
-// def get_pow_block_at_terminal_total_difficulty(pow_chain: Dict[Hash32, PowBlock]) -> Optional[PowBlock]:
-//    # `pow_chain` abstractly represents all blocks in the PoW chain
-//    for block in pow_chain:
-//        parent = pow_chain[block.parent_hash]
-//        block_reached_ttd = block.total_difficulty >= TERMINAL_TOTAL_DIFFICULTY
-//        parent_reached_ttd = parent.total_difficulty >= TERMINAL_TOTAL_DIFFICULTY
-//        if block_reached_ttd and not parent_reached_ttd:
-//            return block
-//
-//    return None
-func (vs *Server) getPowBlockHashAtTerminalTotalDifficulty(ctx context.Context) ([]byte, bool, error) {
-	ttd := new(big.Int)
-	ttd.SetString(params.BeaconConfig().TerminalTotalDifficulty, 10)
-	terminalTotalDifficulty, overflows := uint256.FromBig(ttd)
-	if overflows {
-		return nil, false, errors.New("could not convert terminal total difficulty to uint256")
-	}
-	blk, err := vs.ExecutionEngineCaller.LatestExecutionBlock(ctx)
-	if err != nil {
-		return nil, false, errors.Wrap(err, "could not get latest execution block")
-	}
-	if blk == nil {
-		return nil, false, errors.New("latest execution block is nil")
-	}
-
-	for {
-		if ctx.Err() != nil {
-			return nil, false, ctx.Err()
-		}
-		currentTotalDifficulty, err := tDStringToUint256(blk.TotalDifficulty)
-		if err != nil {
-			return nil, false, errors.Wrap(err, "could not convert total difficulty to uint256")
-		}
-		blockReachedTTD := currentTotalDifficulty.Cmp(terminalTotalDifficulty) >= 0
-
-		parentHash := bytesutil.ToBytes32(blk.ParentHash)
-		if len(blk.ParentHash) == 0 || parentHash == params.BeaconConfig().ZeroHash {
-			return nil, false, nil
-		}
-		parentBlk, err := vs.ExecutionEngineCaller.ExecutionBlockByHash(ctx, parentHash)
-		if err != nil {
-			return nil, false, errors.Wrap(err, "could not get parent execution block")
-		}
-		if parentBlk == nil {
-			return nil, false, errors.New("parent execution block is nil")
-		}
-		if blockReachedTTD {
-			parentTotalDifficulty, err := tDStringToUint256(parentBlk.TotalDifficulty)
-			if err != nil {
-				return nil, false, errors.Wrap(err, "could not convert total difficulty to uint256")
-			}
-			parentReachedTTD := parentTotalDifficulty.Cmp(terminalTotalDifficulty) >= 0
-			if !parentReachedTTD {
-				log.WithFields(logrus.Fields{
-					"number":   blk.Number,
-					"hash":     fmt.Sprintf("%#x", bytesutil.Trunc(blk.Hash)),
-					"td":       blk.TotalDifficulty,
-					"parentTd": parentBlk.TotalDifficulty,
-					"ttd":      terminalTotalDifficulty,
-				}).Info("Retrieved terminal block hash")
-				return blk.Hash, true, nil
-			}
-		} else {
-			return nil, false, nil
-		}
-		blk = parentBlk
-	}
+	return vs.ExecutionEngineCaller.GetTerminalBlockHash(ctx)
 }
 
 // activationEpochNotReached returns true if activation epoch has not been reach.
@@ -270,18 +194,6 @@ func activationEpochNotReached(slot types.Slot) bool {
 	return false
 }
 
-func tDStringToUint256(td string) (*uint256.Int, error) {
-	b, err := hexutil.DecodeBig(td)
-	if err != nil {
-		return nil, err
-	}
-	i, overflows := uint256.FromBig(b)
-	if overflows {
-		return nil, errors.New("total difficulty overflowed")
-	}
-	return i, nil
-}
-
 func emptyPayload() *enginev1.ExecutionPayload {
 	return &enginev1.ExecutionPayload{
 		ParentHash:    make([]byte, fieldparams.RootLength),

beacon-chain/rpc/prysm/v1alpha1/validator/proposer_execution_payload_test.go

@@ -6,7 +6,6 @@ import (
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/holiman/uint256"
 	types "github.com/prysmaticlabs/eth2-types"
 	chainMock "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
@@ -22,26 +21,6 @@ import (
 	"github.com/prysmaticlabs/prysm/testing/util"
 )
 
-func Test_tDStringToUint256(t *testing.T) {
-	i, err := tDStringToUint256("0x0")
-	require.NoError(t, err)
-	require.DeepEqual(t, uint256.NewInt(0), i)
-
-	i, err = tDStringToUint256("0x10000")
-	require.NoError(t, err)
-	require.DeepEqual(t, uint256.NewInt(65536), i)
-
-	_, err = tDStringToUint256("100")
-	require.ErrorContains(t, "hex string without 0x prefix", err)
-
-	_, err = tDStringToUint256("0xzzzzzz")
-	require.ErrorContains(t, "invalid hex string", err)
-
-	_, err = tDStringToUint256("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" +
-		"FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF")
-	require.ErrorContains(t, "hex number > 256 bits", err)
-}
-
 func TestServer_activationEpochNotReached(t *testing.T) {
 	require.Equal(t, false, activationEpochNotReached(0))
 
@@ -154,137 +133,6 @@ func TestServer_getExecutionPayload(t *testing.T) {
 	}
 }
 
-func TestServer_getPowBlockHashAtTerminalTotalDifficulty(t *testing.T) {
-	tests := []struct {
-		name                  string
-		paramsTd              string
-		currentPowBlock       *pb.ExecutionBlock
-		parentPowBlock        *pb.ExecutionBlock
-		errLatestExecutionBlk error
-		wantTerminalBlockHash []byte
-		wantExists            bool
-		errString             string
-	}{
-		{
-			name:      "config td overflows",
-			paramsTd:  "1115792089237316195423570985008687907853269984665640564039457584007913129638912",
-			errString: "could not convert terminal total difficulty to uint256",
-		},
-		{
-			name:                  "could not get latest execution block",
-			paramsTd:              "1",
-			errLatestExecutionBlk: errors.New("blah"),
-			errString:             "could not get latest execution block",
-		},
-		{
-			name:      "nil latest execution block",
-			paramsTd:  "1",
-			errString: "latest execution block is nil",
-		},
-		{
-			name:     "current execution block invalid TD",
-			paramsTd: "1",
-			currentPowBlock: &pb.ExecutionBlock{
-				Hash:            []byte{'a'},
-				TotalDifficulty: "1115792089237316195423570985008687907853269984665640564039457584007913129638912",
-			},
-			errString: "could not convert total difficulty to uint256",
-		},
-		{
-			name:     "current execution block has zero hash parent",
-			paramsTd: "2",
-			currentPowBlock: &pb.ExecutionBlock{
-				Hash:            []byte{'a'},
-				ParentHash:      params.BeaconConfig().ZeroHash[:],
-				TotalDifficulty: "0x3",
-			},
-		},
-		{
-			name:     "could not get parent block",
-			paramsTd: "2",
-			currentPowBlock: &pb.ExecutionBlock{
-				Hash:            []byte{'a'},
-				ParentHash:      []byte{'b'},
-				TotalDifficulty: "0x3",
-			},
-			errString: "could not get parent execution block",
-		},
-		{
-			name:     "parent execution block invalid TD",
-			paramsTd: "2",
-			currentPowBlock: &pb.ExecutionBlock{
-				Hash:            []byte{'a'},
-				ParentHash:      []byte{'b'},
-				TotalDifficulty: "0x3",
-			},
-			parentPowBlock: &pb.ExecutionBlock{
-				Hash:            []byte{'b'},
-				ParentHash:      []byte{'c'},
-				TotalDifficulty: "1",
-			},
-			errString: "could not convert total difficulty to uint256",
-		},
-		{
-			name:     "happy case",
-			paramsTd: "2",
-			currentPowBlock: &pb.ExecutionBlock{
-				Hash:            []byte{'a'},
-				ParentHash:      []byte{'b'},
-				TotalDifficulty: "0x3",
-			},
-			parentPowBlock: &pb.ExecutionBlock{
-				Hash:            []byte{'b'},
-				ParentHash:      []byte{'c'},
-				TotalDifficulty: "0x1",
-			},
-			wantExists:            true,
-			wantTerminalBlockHash: []byte{'a'},
-		},
-		{
-			name:     "ttd not reached",
-			paramsTd: "3",
-			currentPowBlock: &pb.ExecutionBlock{
-				Hash:            []byte{'a'},
-				ParentHash:      []byte{'b'},
-				TotalDifficulty: "0x2",
-			},
-			parentPowBlock: &pb.ExecutionBlock{
-				Hash:            []byte{'b'},
-				ParentHash:      []byte{'c'},
-				TotalDifficulty: "0x1",
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			cfg := params.BeaconConfig()
-			cfg.TerminalTotalDifficulty = tt.paramsTd
-			params.OverrideBeaconConfig(cfg)
-			var m map[[32]byte]*pb.ExecutionBlock
-			if tt.parentPowBlock != nil {
-				m = map[[32]byte]*pb.ExecutionBlock{
-					bytesutil.ToBytes32(tt.parentPowBlock.Hash): tt.parentPowBlock,
-				}
-			}
-			vs := &Server{
-				ExecutionEngineCaller: &powtesting.EngineClient{
-					ErrLatestExecBlock: tt.errLatestExecutionBlk,
-					ExecutionBlock:     tt.currentPowBlock,
-					BlockByHashMap:     m,
-				},
-			}
-			b, e, err := vs.getPowBlockHashAtTerminalTotalDifficulty(context.Background())
-			if tt.errString != "" {
-				require.ErrorContains(t, tt.errString, err)
-			} else {
-				require.NoError(t, err)
-				require.DeepEqual(t, tt.wantExists, e)
-				require.DeepEqual(t, tt.wantTerminalBlockHash, b)
-			}
-		})
-	}
-}
-
 func TestServer_getTerminalBlockHashIfExists(t *testing.T) {
 	tests := []struct {
 		name                  string

cmd/beacon-chain/BUILD.bazel

@@ -63,10 +63,7 @@ container_image(
 container_bundle(
     name = "image_bundle",
     images = {
-        "gcr.io/prysmaticlabs/prysm/beacon-chain:latest": ":image_with_creation_time",
-        "gcr.io/prysmaticlabs/prysm/beacon-chain:{DOCKER_TAG}": ":image_with_creation_time",
-        "index.docker.io/prysmaticlabs/prysm-beacon-chain:latest": ":image_with_creation_time",
-        "index.docker.io/prysmaticlabs/prysm-beacon-chain:{DOCKER_TAG}": ":image_with_creation_time",
+        "gcr.io/prysmaticlabs/prysm/beacon-chain:trie-debug": ":image_with_creation_time",
     },
     tags = ["manual"],
     visibility = ["//beacon-chain:__pkg__"],

config/params/BUILD.bazel

@@ -48,6 +48,7 @@ go_test(
         "@consensus_spec_tests_mainnet//:test_data",
         "@consensus_spec_tests_minimal//:test_data",
         "@eth2_networks//:configs",
+        "testdata/e2e_config.yaml",
     ],
     gotags = ["develop"],
     race = "on",
@@ -61,3 +62,9 @@ go_test(
         "@io_bazel_rules_go//go/tools/bazel:go_default_library",
     ],
 )
+
+filegroup(
+    name = "custom_configs",
+    srcs = glob(["testdata/*.yaml"]),
+    visibility = ["//testing:__subpackages__"],
+)

config/params/loader_test.go

@@ -18,94 +18,94 @@ import (
 
 var placeholderFields = []string{"UPDATE_TIMEOUT", "INTERVALS_PER_SLOT"}
 
-func TestLoadConfigFileMainnet(t *testing.T) {
+func TestLoadConfigFile(t *testing.T) {
 	// See https://media.githubusercontent.com/media/ethereum/consensus-spec-tests/master/tests/minimal/config/phase0.yaml
-	assertVals := func(name string, fields []string, c1, c2 *params.BeaconChainConfig) {
+	assertVals := func(name string, fields []string, expected, actual *params.BeaconChainConfig) {
 		//  Misc params.
-		assert.Equal(t, c1.MaxCommitteesPerSlot, c2.MaxCommitteesPerSlot, "%s: MaxCommitteesPerSlot", name)
-		assert.Equal(t, c1.TargetCommitteeSize, c2.TargetCommitteeSize, "%s: TargetCommitteeSize", name)
-		assert.Equal(t, c1.MaxValidatorsPerCommittee, c2.MaxValidatorsPerCommittee, "%s: MaxValidatorsPerCommittee", name)
-		assert.Equal(t, c1.MinPerEpochChurnLimit, c2.MinPerEpochChurnLimit, "%s: MinPerEpochChurnLimit", name)
-		assert.Equal(t, c1.ChurnLimitQuotient, c2.ChurnLimitQuotient, "%s: ChurnLimitQuotient", name)
-		assert.Equal(t, c1.ShuffleRoundCount, c2.ShuffleRoundCount, "%s: ShuffleRoundCount", name)
-		assert.Equal(t, c1.MinGenesisActiveValidatorCount, c2.MinGenesisActiveValidatorCount, "%s: MinGenesisActiveValidatorCount", name)
-		assert.Equal(t, c1.MinGenesisTime, c2.MinGenesisTime, "%s: MinGenesisTime", name)
-		assert.Equal(t, c1.HysteresisQuotient, c2.HysteresisQuotient, "%s: HysteresisQuotient", name)
-		assert.Equal(t, c1.HysteresisDownwardMultiplier, c2.HysteresisDownwardMultiplier, "%s: HysteresisDownwardMultiplier", name)
-		assert.Equal(t, c1.HysteresisUpwardMultiplier, c2.HysteresisUpwardMultiplier, "%s: HysteresisUpwardMultiplier", name)
+		assert.Equal(t, expected.MaxCommitteesPerSlot, actual.MaxCommitteesPerSlot, "%s: MaxCommitteesPerSlot", name)
+		assert.Equal(t, expected.TargetCommitteeSize, actual.TargetCommitteeSize, "%s: TargetCommitteeSize", name)
+		assert.Equal(t, expected.MaxValidatorsPerCommittee, actual.MaxValidatorsPerCommittee, "%s: MaxValidatorsPerCommittee", name)
+		assert.Equal(t, expected.MinPerEpochChurnLimit, actual.MinPerEpochChurnLimit, "%s: MinPerEpochChurnLimit", name)
+		assert.Equal(t, expected.ChurnLimitQuotient, actual.ChurnLimitQuotient, "%s: ChurnLimitQuotient", name)
+		assert.Equal(t, expected.ShuffleRoundCount, actual.ShuffleRoundCount, "%s: ShuffleRoundCount", name)
+		assert.Equal(t, expected.MinGenesisActiveValidatorCount, actual.MinGenesisActiveValidatorCount, "%s: MinGenesisActiveValidatorCount", name)
+		assert.Equal(t, expected.MinGenesisTime, actual.MinGenesisTime, "%s: MinGenesisTime", name)
+		assert.Equal(t, expected.HysteresisQuotient, actual.HysteresisQuotient, "%s: HysteresisQuotient", name)
+		assert.Equal(t, expected.HysteresisDownwardMultiplier, actual.HysteresisDownwardMultiplier, "%s: HysteresisDownwardMultiplier", name)
+		assert.Equal(t, expected.HysteresisUpwardMultiplier, actual.HysteresisUpwardMultiplier, "%s: HysteresisUpwardMultiplier", name)
 
 		// Fork Choice params.
-		assert.Equal(t, c1.SafeSlotsToUpdateJustified, c2.SafeSlotsToUpdateJustified, "%s: SafeSlotsToUpdateJustified", name)
+		assert.Equal(t, expected.SafeSlotsToUpdateJustified, actual.SafeSlotsToUpdateJustified, "%s: SafeSlotsToUpdateJustified", name)
 
 		// Validator params.
-		assert.Equal(t, c1.Eth1FollowDistance, c2.Eth1FollowDistance, "%s: Eth1FollowDistance", name)
-		assert.Equal(t, c1.TargetAggregatorsPerCommittee, c2.TargetAggregatorsPerCommittee, "%s: TargetAggregatorsPerCommittee", name)
-		assert.Equal(t, c1.RandomSubnetsPerValidator, c2.RandomSubnetsPerValidator, "%s: RandomSubnetsPerValidator", name)
-		assert.Equal(t, c1.EpochsPerRandomSubnetSubscription, c2.EpochsPerRandomSubnetSubscription, "%s: EpochsPerRandomSubnetSubscription", name)
-		assert.Equal(t, c1.SecondsPerETH1Block, c2.SecondsPerETH1Block, "%s: SecondsPerETH1Block", name)
+		assert.Equal(t, expected.Eth1FollowDistance, actual.Eth1FollowDistance, "%s: Eth1FollowDistance", name)
+		assert.Equal(t, expected.TargetAggregatorsPerCommittee, actual.TargetAggregatorsPerCommittee, "%s: TargetAggregatorsPerCommittee", name)
+		assert.Equal(t, expected.RandomSubnetsPerValidator, actual.RandomSubnetsPerValidator, "%s: RandomSubnetsPerValidator", name)
+		assert.Equal(t, expected.EpochsPerRandomSubnetSubscription, actual.EpochsPerRandomSubnetSubscription, "%s: EpochsPerRandomSubnetSubscription", name)
+		assert.Equal(t, expected.SecondsPerETH1Block, actual.SecondsPerETH1Block, "%s: SecondsPerETH1Block", name)
 
 		// Deposit contract.
-		assert.Equal(t, c1.DepositChainID, c2.DepositChainID, "%s: DepositChainID", name)
-		assert.Equal(t, c1.DepositNetworkID, c2.DepositNetworkID, "%s: DepositNetworkID", name)
-		assert.Equal(t, c1.DepositContractAddress, c2.DepositContractAddress, "%s: DepositContractAddress", name)
+		assert.Equal(t, expected.DepositChainID, actual.DepositChainID, "%s: DepositChainID", name)
+		assert.Equal(t, expected.DepositNetworkID, actual.DepositNetworkID, "%s: DepositNetworkID", name)
+		assert.Equal(t, expected.DepositContractAddress, actual.DepositContractAddress, "%s: DepositContractAddress", name)
 
 		// Gwei values.
-		assert.Equal(t, c1.MinDepositAmount, c2.MinDepositAmount, "%s: MinDepositAmount", name)
-		assert.Equal(t, c1.MaxEffectiveBalance, c2.MaxEffectiveBalance, "%s: MaxEffectiveBalance", name)
-		assert.Equal(t, c1.EjectionBalance, c2.EjectionBalance, "%s: EjectionBalance", name)
-		assert.Equal(t, c1.EffectiveBalanceIncrement, c2.EffectiveBalanceIncrement, "%s: EffectiveBalanceIncrement", name)
+		assert.Equal(t, expected.MinDepositAmount, actual.MinDepositAmount, "%s: MinDepositAmount", name)
+		assert.Equal(t, expected.MaxEffectiveBalance, actual.MaxEffectiveBalance, "%s: MaxEffectiveBalance", name)
+		assert.Equal(t, expected.EjectionBalance, actual.EjectionBalance, "%s: EjectionBalance", name)
+		assert.Equal(t, expected.EffectiveBalanceIncrement, actual.EffectiveBalanceIncrement, "%s: EffectiveBalanceIncrement", name)
 
 		// Initial values.
-		assert.DeepEqual(t, c1.GenesisForkVersion, c2.GenesisForkVersion, "%s: GenesisForkVersion", name)
-		assert.DeepEqual(t, c1.BLSWithdrawalPrefixByte, c2.BLSWithdrawalPrefixByte, "%s: BLSWithdrawalPrefixByte", name)
+		assert.DeepEqual(t, expected.GenesisForkVersion, actual.GenesisForkVersion, "%s: GenesisForkVersion", name)
+		assert.DeepEqual(t, expected.BLSWithdrawalPrefixByte, actual.BLSWithdrawalPrefixByte, "%s: BLSWithdrawalPrefixByte", name)
 
 		// Time parameters.
-		assert.Equal(t, c1.GenesisDelay, c2.GenesisDelay, "%s: GenesisDelay", name)
-		assert.Equal(t, c1.SecondsPerSlot, c2.SecondsPerSlot, "%s: SecondsPerSlot", name)
-		assert.Equal(t, c1.MinAttestationInclusionDelay, c2.MinAttestationInclusionDelay, "%s: MinAttestationInclusionDelay", name)
-		assert.Equal(t, c1.SlotsPerEpoch, c2.SlotsPerEpoch, "%s: SlotsPerEpoch", name)
-		assert.Equal(t, c1.MinSeedLookahead, c2.MinSeedLookahead, "%s: MinSeedLookahead", name)
-		assert.Equal(t, c1.MaxSeedLookahead, c2.MaxSeedLookahead, "%s: MaxSeedLookahead", name)
-		assert.Equal(t, c1.EpochsPerEth1VotingPeriod, c2.EpochsPerEth1VotingPeriod, "%s: EpochsPerEth1VotingPeriod", name)
-		assert.Equal(t, c1.SlotsPerHistoricalRoot, c2.SlotsPerHistoricalRoot, "%s: SlotsPerHistoricalRoot", name)
-		assert.Equal(t, c1.MinValidatorWithdrawabilityDelay, c2.MinValidatorWithdrawabilityDelay, "%s: MinValidatorWithdrawabilityDelay", name)
-		assert.Equal(t, c1.ShardCommitteePeriod, c2.ShardCommitteePeriod, "%s: ShardCommitteePeriod", name)
-		assert.Equal(t, c1.MinEpochsToInactivityPenalty, c2.MinEpochsToInactivityPenalty, "%s: MinEpochsToInactivityPenalty", name)
+		assert.Equal(t, expected.GenesisDelay, actual.GenesisDelay, "%s: GenesisDelay", name)
+		assert.Equal(t, expected.SecondsPerSlot, actual.SecondsPerSlot, "%s: SecondsPerSlot", name)
+		assert.Equal(t, expected.MinAttestationInclusionDelay, actual.MinAttestationInclusionDelay, "%s: MinAttestationInclusionDelay", name)
+		assert.Equal(t, expected.SlotsPerEpoch, actual.SlotsPerEpoch, "%s: SlotsPerEpoch", name)
+		assert.Equal(t, expected.MinSeedLookahead, actual.MinSeedLookahead, "%s: MinSeedLookahead", name)
+		assert.Equal(t, expected.MaxSeedLookahead, actual.MaxSeedLookahead, "%s: MaxSeedLookahead", name)
+		assert.Equal(t, expected.EpochsPerEth1VotingPeriod, actual.EpochsPerEth1VotingPeriod, "%s: EpochsPerEth1VotingPeriod", name)
+		assert.Equal(t, expected.SlotsPerHistoricalRoot, actual.SlotsPerHistoricalRoot, "%s: SlotsPerHistoricalRoot", name)
+		assert.Equal(t, expected.MinValidatorWithdrawabilityDelay, actual.MinValidatorWithdrawabilityDelay, "%s: MinValidatorWithdrawabilityDelay", name)
+		assert.Equal(t, expected.ShardCommitteePeriod, actual.ShardCommitteePeriod, "%s: ShardCommitteePeriod", name)
+		assert.Equal(t, expected.MinEpochsToInactivityPenalty, actual.MinEpochsToInactivityPenalty, "%s: MinEpochsToInactivityPenalty", name)
 
 		// State vector lengths.
-		assert.Equal(t, c1.EpochsPerHistoricalVector, c2.EpochsPerHistoricalVector, "%s: EpochsPerHistoricalVector", name)
-		assert.Equal(t, c1.EpochsPerSlashingsVector, c2.EpochsPerSlashingsVector, "%s: EpochsPerSlashingsVector", name)
-		assert.Equal(t, c1.HistoricalRootsLimit, c2.HistoricalRootsLimit, "%s: HistoricalRootsLimit", name)
-		assert.Equal(t, c1.ValidatorRegistryLimit, c2.ValidatorRegistryLimit, "%s: ValidatorRegistryLimit", name)
+		assert.Equal(t, expected.EpochsPerHistoricalVector, actual.EpochsPerHistoricalVector, "%s: EpochsPerHistoricalVector", name)
+		assert.Equal(t, expected.EpochsPerSlashingsVector, actual.EpochsPerSlashingsVector, "%s: EpochsPerSlashingsVector", name)
+		assert.Equal(t, expected.HistoricalRootsLimit, actual.HistoricalRootsLimit, "%s: HistoricalRootsLimit", name)
+		assert.Equal(t, expected.ValidatorRegistryLimit, actual.ValidatorRegistryLimit, "%s: ValidatorRegistryLimit", name)
 
 		// Reward and penalty quotients.
-		assert.Equal(t, c1.BaseRewardFactor, c2.BaseRewardFactor, "%s: BaseRewardFactor", name)
-		assert.Equal(t, c1.WhistleBlowerRewardQuotient, c2.WhistleBlowerRewardQuotient, "%s: WhistleBlowerRewardQuotient", name)
-		assert.Equal(t, c1.ProposerRewardQuotient, c2.ProposerRewardQuotient, "%s: ProposerRewardQuotient", name)
-		assert.Equal(t, c1.InactivityPenaltyQuotient, c2.InactivityPenaltyQuotient, "%s: InactivityPenaltyQuotient", name)
-		assert.Equal(t, c1.InactivityPenaltyQuotientAltair, c2.InactivityPenaltyQuotientAltair, "%s: InactivityPenaltyQuotientAltair", name)
-		assert.Equal(t, c1.MinSlashingPenaltyQuotient, c2.MinSlashingPenaltyQuotient, "%s: MinSlashingPenaltyQuotient", name)
-		assert.Equal(t, c1.MinSlashingPenaltyQuotientAltair, c2.MinSlashingPenaltyQuotientAltair, "%s: MinSlashingPenaltyQuotientAltair", name)
-		assert.Equal(t, c1.ProportionalSlashingMultiplier, c2.ProportionalSlashingMultiplier, "%s: ProportionalSlashingMultiplier", name)
-		assert.Equal(t, c1.ProportionalSlashingMultiplierAltair, c2.ProportionalSlashingMultiplierAltair, "%s: ProportionalSlashingMultiplierAltair", name)
+		assert.Equal(t, expected.BaseRewardFactor, actual.BaseRewardFactor, "%s: BaseRewardFactor", name)
+		assert.Equal(t, expected.WhistleBlowerRewardQuotient, actual.WhistleBlowerRewardQuotient, "%s: WhistleBlowerRewardQuotient", name)
+		assert.Equal(t, expected.ProposerRewardQuotient, actual.ProposerRewardQuotient, "%s: ProposerRewardQuotient", name)
+		assert.Equal(t, expected.InactivityPenaltyQuotient, actual.InactivityPenaltyQuotient, "%s: InactivityPenaltyQuotient", name)
+		assert.Equal(t, expected.InactivityPenaltyQuotientAltair, actual.InactivityPenaltyQuotientAltair, "%s: InactivityPenaltyQuotientAltair", name)
+		assert.Equal(t, expected.MinSlashingPenaltyQuotient, actual.MinSlashingPenaltyQuotient, "%s: MinSlashingPenaltyQuotient", name)
+		assert.Equal(t, expected.MinSlashingPenaltyQuotientAltair, actual.MinSlashingPenaltyQuotientAltair, "%s: MinSlashingPenaltyQuotientAltair", name)
+		assert.Equal(t, expected.ProportionalSlashingMultiplier, actual.ProportionalSlashingMultiplier, "%s: ProportionalSlashingMultiplier", name)
+		assert.Equal(t, expected.ProportionalSlashingMultiplierAltair, actual.ProportionalSlashingMultiplierAltair, "%s: ProportionalSlashingMultiplierAltair", name)
 
 		// Max operations per block.
-		assert.Equal(t, c1.MaxProposerSlashings, c2.MaxProposerSlashings, "%s: MaxProposerSlashings", name)
-		assert.Equal(t, c1.MaxAttesterSlashings, c2.MaxAttesterSlashings, "%s: MaxAttesterSlashings", name)
-		assert.Equal(t, c1.MaxAttestations, c2.MaxAttestations, "%s: MaxAttestations", name)
-		assert.Equal(t, c1.MaxDeposits, c2.MaxDeposits, "%s: MaxDeposits", name)
-		assert.Equal(t, c1.MaxVoluntaryExits, c2.MaxVoluntaryExits, "%s: MaxVoluntaryExits", name)
+		assert.Equal(t, expected.MaxProposerSlashings, actual.MaxProposerSlashings, "%s: MaxProposerSlashings", name)
+		assert.Equal(t, expected.MaxAttesterSlashings, actual.MaxAttesterSlashings, "%s: MaxAttesterSlashings", name)
+		assert.Equal(t, expected.MaxAttestations, actual.MaxAttestations, "%s: MaxAttestations", name)
+		assert.Equal(t, expected.MaxDeposits, actual.MaxDeposits, "%s: MaxDeposits", name)
+		assert.Equal(t, expected.MaxVoluntaryExits, actual.MaxVoluntaryExits, "%s: MaxVoluntaryExits", name)
 
 		// Signature domains.
-		assert.Equal(t, c1.DomainBeaconProposer, c2.DomainBeaconProposer, "%s: DomainBeaconProposer", name)
-		assert.Equal(t, c1.DomainBeaconAttester, c2.DomainBeaconAttester, "%s: DomainBeaconAttester", name)
-		assert.Equal(t, c1.DomainRandao, c2.DomainRandao, "%s: DomainRandao", name)
-		assert.Equal(t, c1.DomainDeposit, c2.DomainDeposit, "%s: DomainDeposit", name)
-		assert.Equal(t, c1.DomainVoluntaryExit, c2.DomainVoluntaryExit, "%s: DomainVoluntaryExit", name)
-		assert.Equal(t, c1.DomainSelectionProof, c2.DomainSelectionProof, "%s: DomainSelectionProof", name)
-		assert.Equal(t, c1.DomainAggregateAndProof, c2.DomainAggregateAndProof, "%s: DomainAggregateAndProof", name)
+		assert.Equal(t, expected.DomainBeaconProposer, actual.DomainBeaconProposer, "%s: DomainBeaconProposer", name)
+		assert.Equal(t, expected.DomainBeaconAttester, actual.DomainBeaconAttester, "%s: DomainBeaconAttester", name)
+		assert.Equal(t, expected.DomainRandao, actual.DomainRandao, "%s: DomainRandao", name)
+		assert.Equal(t, expected.DomainDeposit, actual.DomainDeposit, "%s: DomainDeposit", name)
+		assert.Equal(t, expected.DomainVoluntaryExit, actual.DomainVoluntaryExit, "%s: DomainVoluntaryExit", name)
+		assert.Equal(t, expected.DomainSelectionProof, actual.DomainSelectionProof, "%s: DomainSelectionProof", name)
+		assert.Equal(t, expected.DomainAggregateAndProof, actual.DomainAggregateAndProof, "%s: DomainAggregateAndProof", name)
 
-		assertYamlFieldsMatch(t, name, fields, c1, c2)
+		assertYamlFieldsMatch(t, name, fields, expected, actual)
 	}
 
 	t.Run("mainnet", func(t *testing.T) {
@@ -129,6 +129,17 @@ func TestLoadConfigFileMainnet(t *testing.T) {
 		fields := fieldsFromYamls(t, append(minimalPresetsFiles, minimalConfigFile))
 		assertVals("minimal", fields, params.MinimalSpecConfig(), params.BeaconConfig())
 	})
+
+	t.Run("e2e", func(t *testing.T) {
+		minimalPresetsFiles := presetsFilePath(t, "minimal")
+		for _, fp := range minimalPresetsFiles {
+			params.LoadChainConfigFile(fp, nil)
+		}
+		configFile := "testdata/e2e_config.yaml"
+		params.LoadChainConfigFile(configFile, nil)
+		fields := fieldsFromYamls(t, append(minimalPresetsFiles, configFile))
+		assertVals("e2e", fields, params.E2ETestConfig(), params.BeaconConfig())
+	})
 }
 
 func TestLoadConfigFile_OverwriteCorrectly(t *testing.T) {

config/params/testdata/e2e_config.yaml

@@ -0,0 +1,118 @@
+# e2e config
+
+# Extends the minimal preset
+PRESET_BASE: 'minimal'
+
+
+
+# Transition
+# ---------------------------------------------------------------
+# TBD, 2**256-2**10 is a placeholder, e2e is 600
+TERMINAL_TOTAL_DIFFICULTY: 600
+# By default, don't use these params
+#TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000
+#TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615
+
+
+
+# Genesis
+# ---------------------------------------------------------------
+# [customized]
+MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 256 # Override for e2e tests
+# Jan 3, 2020
+MIN_GENESIS_TIME: 1578009600
+# Highest byte set to 0x01 to avoid collisions with mainnet versioning
+GENESIS_FORK_VERSION: 0x000000fd
+# [customized] Faster to spin up testnets, but does not give validator reasonable warning time for genesis
+GENESIS_DELAY: 10 # Override for e2e tests
+
+
+# Forking
+# ---------------------------------------------------------------
+# Values provided for illustrative purposes.
+# Individual tests/testnets may set different values.
+
+# Altair
+ALTAIR_FORK_VERSION: 0x010000fd
+ALTAIR_FORK_EPOCH: 6 # Override for e2e
+# Bellatrix
+BELLATRIX_FORK_VERSION: 0x020000fd
+BELLATRIX_FORK_EPOCH: 8
+# Sharding
+SHARDING_FORK_VERSION: 0x030000fd
+SHARDING_FORK_EPOCH: 18446744073709551615
+
+
+# Time parameters
+# ---------------------------------------------------------------
+# [customized] Faster for testing purposes
+SECONDS_PER_SLOT: 10 # Override for e2e tests
+# 14 (estimate from Eth1 mainnet)
+SECONDS_PER_ETH1_BLOCK: 2 # Override for e2e tests
+# 2**8 (= 256) epochs
+MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256
+# [customized] higher frequency of committee turnover and faster time to acceptable voluntary exit
+SHARD_COMMITTEE_PERIOD: 4 # Override for e2e tests
+# [customized] process deposits more quickly, but insecure
+ETH1_FOLLOW_DISTANCE: 4 # Override for e2e tests
+
+
+# Validator cycle
+# ---------------------------------------------------------------
+# 2**2 (= 4)
+INACTIVITY_SCORE_BIAS: 4
+# 2**4 (= 16)
+INACTIVITY_SCORE_RECOVERY_RATE: 16
+# 2**4 * 10**9 (= 16,000,000,000) Gwei
+EJECTION_BALANCE: 16000000000
+# 2**2 (= 4)
+MIN_PER_EPOCH_CHURN_LIMIT: 4
+# [customized] scale queue churn at much lower validator counts for testing
+CHURN_LIMIT_QUOTIENT: 65536
+
+
+# Fork choice
+# ---------------------------------------------------------------
+# 70%
+PROPOSER_SCORE_BOOST: 70
+
+
+# Deposit contract
+# ---------------------------------------------------------------
+# Ethereum Goerli testnet
+DEPOSIT_CHAIN_ID: 1337 # Override for e2e tests
+DEPOSIT_NETWORK_ID: 1337 # Override for e2e tests
+# Configured on a per testnet basis
+DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890
+
+# Updated penalty values
+# ---------------------------------------------------------------
+# 3 * 2**24 (= 50,331,648)
+INACTIVITY_PENALTY_QUOTIENT_ALTAIR: 50331648
+# 2**6 (= 64)
+MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR: 64
+# 2
+PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR: 2
+
+
+# Sync committee
+# ---------------------------------------------------------------
+# [customized]
+SYNC_COMMITTEE_SIZE: 32
+# [customized]
+EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 8
+
+
+# Sync protocol
+# ---------------------------------------------------------------
+# 1
+MIN_SYNC_COMMITTEE_PARTICIPANTS: 1
+
+# Other e2e overrides
+# ---------------------------------------------------------------
+CONFIG_NAME: "end-to-end"
+SLOTS_PER_EPOCH: 6
+EPOCHS_PER_ETH1_VOTING_PERIOD: 2
+MAX_SEED_LOOKAHEAD: 1
+
+

testing/endtoend/BUILD.bazel

@@ -17,6 +17,7 @@ go_test(
         "//:prysm_sh",
         "//cmd/beacon-chain",
         "//cmd/validator",
+        "//config/params:custom_configs",
         "//tools/bootnode",
         "@com_github_ethereum_go_ethereum//cmd/geth",
         "@web3signer",
@@ -35,6 +36,7 @@ go_test(
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/operations/slashings/mock:go_default_library",
         "//beacon-chain/state/stategen/mock:go_default_library",
+        "//build/bazel:go_default_library",
         "//config/params:go_default_library",
         "//crypto/bls:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
@@ -71,6 +73,7 @@ go_test(
         "//:prysm_sh",
         "//cmd/beacon-chain",
         "//cmd/validator",
+        "//config/params:custom_configs",
         "//tools/bootnode",
         "@com_github_ethereum_go_ethereum//cmd/geth",
         "@web3signer",
@@ -89,6 +92,7 @@ go_test(
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/operations/slashings:go_default_library",
         "//beacon-chain/state/stategen:go_default_library",
+        "//build/bazel:go_default_library",
         "//config/params:go_default_library",
         "//crypto/bls:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",

testing/endtoend/components/BUILD.bazel

@@ -54,11 +54,15 @@ go_test(
     name = "go_default_test",
     size = "small",
     srcs = ["web3remotesigner_test.go"],
-    data = ["@web3signer"],
+    data = [
+        "//config/params:custom_configs",
+        "@web3signer",
+    ],
     deps = [
         ":go_default_library",
         "//config/params:go_default_library",
         "//testing/endtoend/params:go_default_library",
         "//testing/require:go_default_library",
+        "@io_bazel_rules_go//go/tools/bazel:go_default_library",
     ],
 )

testing/endtoend/components/validator.go

@@ -151,7 +151,7 @@ func (v *ValidatorNode) Start(ctx context.Context) error {
 		args = append(args, features.E2EValidatorFlags...)
 	}
 	if v.config.UseWeb3RemoteSigner {
-		args = append(args, fmt.Sprintf("--%s=localhost:%d", flags.Web3SignerURLFlag.Name, Web3RemoteSignerPort))
+		args = append(args, fmt.Sprintf("--%s=http://localhost:%d", flags.Web3SignerURLFlag.Name, Web3RemoteSignerPort))
 		// Write the pubkeys as comma seperated hex strings with 0x prefix.
 		// See: https://docs.teku.consensys.net/en/latest/HowTo/External-Signer/Use-External-Signer/
 		_, pubs, err := interop.DeterministicallyGenerateKeys(uint64(offset), uint64(validatorNum))

testing/endtoend/components/web3remotesigner.go

@@ -37,13 +37,16 @@ type rawKeyFile struct {
 }
 
 type Web3RemoteSigner struct {
-	ctx     context.Context
-	started chan struct{}
+	ctx            context.Context
+	started        chan struct{}
+	configFilePath string
+	cmd            *exec.Cmd
 }
 
-func NewWeb3RemoteSigner() *Web3RemoteSigner {
+func NewWeb3RemoteSigner(configFilePath string) *Web3RemoteSigner {
 	return &Web3RemoteSigner{
-		started: make(chan struct{}, 1),
+		started:        make(chan struct{}, 1),
+		configFilePath: configFilePath,
 	}
 }
 
@@ -66,6 +69,12 @@ func (w *Web3RemoteSigner) Start(ctx context.Context) error {
 		return err
 	}
 
+	network := "minimal"
+	if len(w.configFilePath) > 0 {
+		// A file path to yaml config file is acceptable network argument.
+		network = w.configFilePath
+	}
+
 	args := []string{
 		// Global flags
 		fmt.Sprintf("--key-store-path=%s", keystorePath),
@@ -75,13 +84,13 @@ func (w *Web3RemoteSigner) Start(ctx context.Context) error {
 		// Command
 		"eth2",
 		// Command flags
-		"--network=minimal",
+		"--network=" + network,
 		"--slashing-protection-enabled=false", // Otherwise, a postgres DB is required.
-		"--enable-key-manager-api=true",
+		"--key-manager-api-enabled=true",
 	}
 
 	cmd := exec.CommandContext(ctx, binaryPath, args...) // #nosec G204 -- Test code is safe to do this.
-
+	w.cmd = cmd
 	// Write stdout and stderr to log files.
 	stdout, err := os.Create(path.Join(e2e.TestParams.LogPath, "web3signer.stdout.log"))
 	if err != nil {

testing/endtoend/components/web3remotesigner_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/bazelbuild/rules_go/go/tools/bazel"
 	"github.com/prysmaticlabs/prysm/config/params"
 	"github.com/prysmaticlabs/prysm/testing/endtoend/components"
 	e2eparams "github.com/prysmaticlabs/prysm/testing/endtoend/params"
@@ -13,7 +14,11 @@ import (
 
 func TestWeb3RemoteSigner_StartsAndReturnsPublicKeys(t *testing.T) {
 	require.NoError(t, e2eparams.Init(0))
-	wsc := components.NewWeb3RemoteSigner()
+	fp, err := bazel.Runfile("config/params/testdata/e2e_config.yaml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	wsc := components.NewWeb3RemoteSigner(fp)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()

testing/endtoend/deps.bzl

@@ -6,10 +6,11 @@ lighthouse_archive_name = "lighthouse-%s-x86_64-unknown-linux-gnu-portable.tar.g
 def e2e_deps():
     http_archive(
         name = "web3signer",
-        urls = ["https://artifacts.consensys.net/public/web3signer/raw/names/web3signer.tar.gz/versions/21.10.5/web3signer-21.10.5.tar.gz"],
-        sha256 = "d122429f6a310bc555d1281e0b3f4e3ac43a7beec5e5dcf0a0d2416a5984f461",
+        # Built from commit 17d253b which has important unreleased changes.
+        urls = ["https://prysmaticlabs.com/uploads/web3signer-17d253b.tar.gz"],
+        sha256 = "bf450a59a0845c1ce8100b3192c7fec021b565efe8b1ab46bed9f71cb994a6d7",
         build_file = "@prysm//testing/endtoend:web3signer.BUILD",
-        strip_prefix = "web3signer-21.10.5",
+        strip_prefix = "web3signer-develop",
     )
 
     http_archive(

testing/endtoend/endtoend_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/pkg/errors"
 	types "github.com/prysmaticlabs/eth2-types"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/transition"
+	"github.com/prysmaticlabs/prysm/build/bazel"
 	"github.com/prysmaticlabs/prysm/config/params"
 	eth "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/testing/assert"
@@ -89,6 +90,21 @@ func (r *testRunner) run() {
 		})
 	}
 
+	var web3RemoteSigner *components.Web3RemoteSigner
+	if config.UseWeb3RemoteSigner {
+		cfg, err := bazel.Runfile("config/params/testdata/e2e_config.yaml")
+		if err != nil {
+			t.Fatal(err)
+		}
+		web3RemoteSigner = components.NewWeb3RemoteSigner(cfg)
+		g.Go(func() error {
+			if err := web3RemoteSigner.Start(ctx); err != nil {
+				return errors.Wrap(err, "failed to start web3 remote signer")
+			}
+			return nil
+		})
+	}
+
 	// Boot node.
 	bootNode := components.NewBootNode()
 	g.Go(func() error {
@@ -146,18 +162,6 @@ func (r *testRunner) run() {
 		return nil
 	})
 
-	// Web3 remote signer.
-	var web3RemoteSigner *components.Web3RemoteSigner
-	if config.UseWeb3RemoteSigner {
-		web3RemoteSigner = components.NewWeb3RemoteSigner()
-		g.Go(func() error {
-			if err := web3RemoteSigner.Start(ctx); err != nil {
-				return errors.Wrap(err, "failed to start web3 remote signer")
-			}
-			return nil
-		})
-	}
-
 	if multiClientActive {
 		lighthouseNodes = components.NewLighthouseBeaconNodes(config)
 		g.Go(func() error {
@@ -332,7 +336,6 @@ func (r *testRunner) runEvaluators(conns []*grpc.ClientConn, tickingStartTime ti
 func (r *testRunner) testDepositsAndTx(ctx context.Context, g *errgroup.Group,
 	keystorePath string, requiredNodes []e2etypes.ComponentRunner) {
 	minGenesisActiveCount := int(params.BeaconConfig().MinGenesisActiveValidatorCount)
-
 	depositCheckValidator := components.NewValidatorNode(r.config, int(e2e.DepositCount), e2e.TestParams.BeaconNodeCount, minGenesisActiveCount)
 	g.Go(func() error {
 		if err := helpers.ComponentsStarted(ctx, requiredNodes); err != nil {

testing/endtoend/minimal_e2e_test.go

@@ -27,7 +27,6 @@ func TestEndToEnd_MinimalConfig(t *testing.T) {
 }
 
 func TestEndToEnd_MinimalConfig_Web3Signer(t *testing.T) {
-	t.Skip("TODO(9994): Complete web3signer client implementation, currently blocked by https://github.com/ConsenSys/web3signer/issues/494")
 	e2eMinimal(t, &testArgs{
 		usePrysmSh:          false,
 		useWeb3RemoteSigner: true,
@@ -54,6 +53,10 @@ func e2eMinimal(t *testing.T, args *testArgs) {
 		epochsToRun, err = strconv.Atoi(epochStr)
 		require.NoError(t, err)
 	}
+	// TODO(#10053): Web3signer does not support bellatrix yet.
+	if args.useWeb3RemoteSigner {
+		epochsToRun = helpers.BellatrixE2EForkEpoch - 1
+	}
 	if args.usePrysmSh {
 		// If using prysm.sh, run for only 6 epochs.
 		// TODO(#9166): remove this block once v2 changes are live.

testing/spectest/shared/common/forkchoice/service.go

@@ -102,3 +102,7 @@ func (m *engineMock) ExecutionBlockByHash(_ context.Context, hash common.Hash) (
 		Hash:            b.BlockHash,
 	}, nil
 }
+
+func (m *engineMock) GetTerminalBlockHash(context.Context) ([]byte, bool, error) {
+	return nil, false, nil
+}