fix voting period start, skip altair check

due to flaky test re-run behavior, logs from a failed test run are
overwritten by subsequent retries. This makes it difficult to retrieve
logs after the first failed run. It also takes some squinting through
output to find the location of the log file in the first place. This
flag enables logs to be placed in an arbitrary path. Note that bazel
sandboxing generally will force this path to be in the /tmp tree.

beacon-chain/blockchain/execution_engine.go

@@ -1,6 +1,7 @@
 package blockchain
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 
@@ -38,6 +39,7 @@ type notifyForkchoiceUpdateArg struct {
 func (s *Service) notifyForkchoiceUpdate(ctx context.Context, arg *notifyForkchoiceUpdateArg) (*enginev1.PayloadIDBytes, error) {
 	ctx, span := trace.StartSpan(ctx, "blockChain.notifyForkchoiceUpdate")
 	defer span.End()
+	log.Infof("notifyForkchoiceUpdate, fork version=%#x, headRoot=%#x, headState.latest_block_header=%#x", arg.headState.Fork().CurrentVersion, arg.headRoot, arg.headState.LatestBlockHeader().BodyRoot)
 
 	headBlk := arg.headBlock
 	if headBlk == nil || headBlk.IsNil() || headBlk.Body().IsNil() {
@@ -50,7 +52,25 @@ func (s *Service) notifyForkchoiceUpdate(ctx context.Context, arg *notifyForkcho
 		log.WithError(err).Error("Could not determine if head block is execution block")
 		return nil, nil
 	}
+	var eth1BlockHash []byte
 	if !isExecutionBlk {
+		br, err := headBlk.HashTreeRoot()
+		if err != nil {
+			return nil, err
+		}
+
+		// TODO: this is a hack
+		// try to grab the payload header from the state in case the block doesn't have it but the state does
+		sp, err := arg.headState.LatestExecutionPayloadHeader()
+		if err != nil {
+			log.WithError(err).Infof("notifyForkchoiceUpdate, skipping non-execution block with root=%#x", br)
+			return nil, nil
+		}
+		eth1BlockHash = sp.BlockHash()
+		if len(sp.BlockHash()) != 32 || bytes.Equal(eth1BlockHash, params.BeaconConfig().ZeroHash[:]) {
+			log.Infof("notifyForkchoiceUpdate, skipping non-execution block with root=%#x and empty payload header block hash", br)
+			return nil, nil
+		}
 		return nil, nil
 	}
 	headPayload, err := headBlk.Body().Execution()
@@ -58,10 +78,11 @@ func (s *Service) notifyForkchoiceUpdate(ctx context.Context, arg *notifyForkcho
 		log.WithError(err).Error("Could not get execution payload for head block")
 		return nil, nil
 	}
+	eth1BlockHash = headPayload.BlockHash()
 	finalizedHash := s.ForkChoicer().FinalizedPayloadBlockHash()
 	justifiedHash := s.ForkChoicer().JustifiedPayloadBlockHash()
 	fcs := &enginev1.ForkchoiceState{
-		HeadBlockHash:      headPayload.BlockHash(),
+		HeadBlockHash:      eth1BlockHash,
 		SafeBlockHash:      justifiedHash[:],
 		FinalizedBlockHash: finalizedHash[:],
 	}
@@ -80,7 +101,7 @@ func (s *Service) notifyForkchoiceUpdate(ctx context.Context, arg *notifyForkcho
 			forkchoiceUpdatedOptimisticNodeCount.Inc()
 			log.WithFields(logrus.Fields{
 				"headSlot":                  headBlk.Slot(),
-				"headPayloadBlockHash":      fmt.Sprintf("%#x", bytesutil.Trunc(headPayload.BlockHash())),
+				"headPayloadBlockHash":      fmt.Sprintf("%#x", bytesutil.Trunc(eth1BlockHash)),
 				"finalizedPayloadBlockHash": fmt.Sprintf("%#x", bytesutil.Trunc(finalizedHash[:])),
 			}).Info("Called fork choice updated with optimistic block")
 			return payloadID, nil
@@ -145,18 +166,21 @@ func (s *Service) notifyForkchoiceUpdate(ctx context.Context, arg *notifyForkcho
 			return nil, nil
 		}
 	}
+	log.WithField("payloadID", fmt.Sprintf("%#x", payloadID)).WithField("lastValidHash", fmt.Sprintf("%#x", lastValidHash)).Infof("notifyForkchoiceUpdate, no error from ee call")
 	forkchoiceUpdatedValidNodeCount.Inc()
 	if err := s.cfg.ForkChoiceStore.SetOptimisticToValid(ctx, arg.headRoot); err != nil {
-		log.WithError(err).Error("Could not set head root to valid")
+		log.WithField("headRoot", fmt.Sprintf("%#x", arg.headRoot)).WithField("payloadID", fmt.Sprintf("%#x", payloadID)).WithField("lastValidHash", fmt.Sprintf("%#x", lastValidHash)).WithError(err).Error("Could not set head root to valid")
 		return nil, nil
 	}
+	log.Infof("notifyForkchoiceUpdate, set optimistic to valid for root=%#x", arg.headRoot)
 	if hasAttr && payloadID != nil { // If the forkchoice update call has an attribute, update the proposer payload ID cache.
 		var pId [8]byte
 		copy(pId[:], payloadID[:])
 		s.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(nextSlot, proposerId, pId, arg.headRoot)
+		log.Infof("notifyForkchoiceUpdate, SetProposerAndPayloadIDs(nextSlot=%d, proposerId=%d, pId=%#x, arg.headRoot=%#x)", arg.headRoot, proposerId, pId, arg.headRoot)
 	} else if hasAttr && payloadID == nil {
 		log.WithFields(logrus.Fields{
-			"blockHash": fmt.Sprintf("%#x", headPayload.BlockHash()),
+			"blockHash": fmt.Sprintf("%#x", eth1BlockHash),
 			"slot":      headBlk.Slot(),
 		}).Error("Received nil payload ID on VALID engine response")
 	}

beacon-chain/blockchain/head.go

@@ -207,7 +207,7 @@ func (s *Service) setHead(root [32]byte, block interfaces.SignedBeaconBlock, sta
 		block: bCp,
 		state: state.Copy(),
 	}
-	return nil
+	return err
 }
 
 // This sets head view object which is used to track the head slot, root, block and state. The method

beacon-chain/blockchain/log.go

@@ -87,6 +87,7 @@ func logBlockSyncStatus(block interfaces.BeaconBlock, blockRoot [32]byte, justif
 			"version":                   version.String(block.Version()),
 			"sinceSlotStartTime":        prysmTime.Now().Sub(startTime),
 			"chainServiceProcessedTime": prysmTime.Now().Sub(receivedTime),
+			"deposits":                  fmt.Sprintf("%d/%d", len(block.Body().Deposits()), block.Body().Eth1Data().DepositCount),
 		}).Debug("Synced new block")
 	} else {
 		log.WithFields(logrus.Fields{

beacon-chain/blockchain/service.go

@@ -309,7 +309,10 @@ func (s *Service) initializeHeadFromDB(ctx context.Context) error {
 	if err := s.setHead(finalizedRoot, finalizedBlock, finalizedState); err != nil {
 		return errors.Wrap(err, "could not set head")
 	}
-
+	_, err = s.notifyForkchoiceUpdate(s.ctx, &notifyForkchoiceUpdateArg{headState: finalizedState, headRoot: finalizedRoot, headBlock: finalizedBlock.Block()})
+	if err != nil {
+		return errors.Wrap(err, "error calling FCU with finalized state at startup")
+	}
 	return nil
 }
 
@@ -444,6 +447,10 @@ func (s *Service) saveGenesisData(ctx context.Context, genesisState state.Beacon
 	if err := s.setHead(genesisBlkRoot, genesisBlk, genesisState); err != nil {
 		log.WithError(err).Fatal("Could not set head")
 	}
+	_, err = s.notifyForkchoiceUpdate(s.ctx, &notifyForkchoiceUpdateArg{headState: genesisState, headRoot: genesisBlkRoot, headBlock: genesisBlk.Block()})
+	if err != nil {
+		log.WithError(err).Fatal("Could not call FCU with genesis")
+	}
 	return nil
 }
 

beacon-chain/core/altair/epoch_precompute.go

@@ -3,6 +3,8 @@ package altair
 import (
 	"context"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v3/beacon-chain/core/epoch/precompute"
 	"github.com/prysmaticlabs/prysm/v3/beacon-chain/core/helpers"
@@ -29,6 +31,7 @@ func InitializePrecomputeValidators(ctx context.Context, beaconState state.Beaco
 	// This shouldn't happen with a correct beacon state,
 	// but rather be safe to defend against index out of bound panics.
 	if beaconState.NumValidators() != len(inactivityScores) {
+		log.Errorf("NumValidators=%d inactivityScores=%d", beaconState.NumValidators(), len(inactivityScores))
 		return nil, nil, errors.New("num of validators is different than num of inactivity scores")
 	}
 	if err := beaconState.ReadFromEveryValidator(func(idx int, val state.ReadOnlyValidator) error {

beacon-chain/core/blocks/BUILD.bazel

@@ -19,12 +19,7 @@ go_library(
         "withdrawals.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/v3/beacon-chain/core/blocks",
-    visibility = [
-        "//beacon-chain:__subpackages__",
-        "//testing/spectest:__subpackages__",
-        "//testing/util:__pkg__",
-        "//validator:__subpackages__",
-    ],
+    visibility = ["//visibility:public"],
     deps = [
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/signing:go_default_library",

beacon-chain/core/blocks/genesis.go

@@ -3,9 +3,14 @@
 package blocks
 
 import (
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/v3/beacon-chain/state"
 	fieldparams "github.com/prysmaticlabs/prysm/v3/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v3/config/params"
+	"github.com/prysmaticlabs/prysm/v3/consensus-types/blocks"
+	"github.com/prysmaticlabs/prysm/v3/consensus-types/interfaces"
 	"github.com/prysmaticlabs/prysm/v3/encoding/bytesutil"
+	enginev1 "github.com/prysmaticlabs/prysm/v3/proto/engine/v1"
 	ethpb "github.com/prysmaticlabs/prysm/v3/proto/prysm/v1alpha1"
 )
 
@@ -29,3 +34,60 @@ func NewGenesisBlock(stateRoot []byte) *ethpb.SignedBeaconBlock {
 	}
 	return block
 }
+
+var ErrUnrecognizedState = errors.New("uknonwn underlying type for state.BeaconState value")
+
+func NewGenesisBlockForState(root [32]byte, st state.BeaconState) (interfaces.SignedBeaconBlock, error) {
+	ps := st.ToProto()
+	switch ps.(type) {
+	case *ethpb.BeaconState:
+		return blocks.NewSignedBeaconBlock(&ethpb.SignedBeaconBlock{
+			Block: &ethpb.BeaconBlock{
+				ParentRoot: params.BeaconConfig().ZeroHash[:],
+				StateRoot:  root[:],
+				Body: &ethpb.BeaconBlockBody{
+					RandaoReveal: make([]byte, fieldparams.BLSSignatureLength),
+					Eth1Data: &ethpb.Eth1Data{
+						DepositRoot: make([]byte, 32),
+						BlockHash:   make([]byte, 32),
+					},
+					Graffiti: make([]byte, 32),
+				},
+			},
+			Signature: params.BeaconConfig().EmptySignature[:],
+		})
+	case *ethpb.BeaconStateBellatrix:
+		return blocks.NewSignedBeaconBlock(&ethpb.SignedBeaconBlockBellatrix{
+			Block: &ethpb.BeaconBlockBellatrix{
+				ParentRoot: params.BeaconConfig().ZeroHash[:],
+				StateRoot:  root[:],
+				Body: &ethpb.BeaconBlockBodyBellatrix{
+					RandaoReveal: make([]byte, 96),
+					Eth1Data: &ethpb.Eth1Data{
+						DepositRoot: make([]byte, 32),
+						BlockHash:   make([]byte, 32),
+					},
+					Graffiti: make([]byte, 32),
+					SyncAggregate: &ethpb.SyncAggregate{
+						SyncCommitteeBits:      make([]byte, fieldparams.SyncCommitteeLength/8),
+						SyncCommitteeSignature: make([]byte, fieldparams.BLSSignatureLength),
+					},
+					ExecutionPayload: &enginev1.ExecutionPayload{
+						ParentHash:    make([]byte, 32),
+						FeeRecipient:  make([]byte, 20),
+						StateRoot:     make([]byte, 32),
+						ReceiptsRoot:  make([]byte, 32),
+						LogsBloom:     make([]byte, 256),
+						PrevRandao:    make([]byte, 32),
+						BaseFeePerGas: make([]byte, 32),
+						BlockHash:     make([]byte, 32),
+						Transactions:  make([][]byte, 0),
+					},
+				},
+			},
+			Signature: params.BeaconConfig().EmptySignature[:],
+		})
+	default:
+		return nil, ErrUnrecognizedState
+	}
+}

beacon-chain/core/transition/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
         "log.go",
         "skip_slot_cache.go",
         "state.go",
+        "state-bellatrix.go",
         "trailing_slot_state_cache.go",
         "transition.go",
         "transition_no_verify_sig.go",
@@ -36,6 +37,7 @@ go_library(
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/state-native:go_default_library",
         "//beacon-chain/state/stateutil:go_default_library",
+        "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
         "//consensus-types/blocks:go_default_library",
         "//consensus-types/interfaces:go_default_library",
@@ -45,6 +47,7 @@ go_library(
         "//encoding/bytesutil:go_default_library",
         "//math:go_default_library",
         "//monitoring/tracing:go_default_library",
+        "//proto/engine/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//runtime/version:go_default_library",
         "@com_github_pkg_errors//:go_default_library",

beacon-chain/core/transition/state-bellatrix.go

@@ -0,0 +1,283 @@
+package transition
+
+import (
+	"context"
+
+	"github.com/prysmaticlabs/prysm/v3/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/v3/consensus-types/blocks"
+
+	"github.com/prysmaticlabs/prysm/v3/beacon-chain/core/altair"
+
+	enginev1 "github.com/prysmaticlabs/prysm/v3/proto/engine/v1"
+
+	fieldparams "github.com/prysmaticlabs/prysm/v3/config/fieldparams"
+
+	"github.com/pkg/errors"
+	b "github.com/prysmaticlabs/prysm/v3/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/v3/beacon-chain/state"
+	state_native "github.com/prysmaticlabs/prysm/v3/beacon-chain/state/state-native"
+	"github.com/prysmaticlabs/prysm/v3/beacon-chain/state/stateutil"
+	"github.com/prysmaticlabs/prysm/v3/config/params"
+	ethpb "github.com/prysmaticlabs/prysm/v3/proto/prysm/v1alpha1"
+)
+
+// GenesisBeaconStateBellatrix gets called when MinGenesisActiveValidatorCount count of
+// full deposits were made to the deposit contract and the ChainStart log gets emitted.
+//
+// Spec pseudocode definition:
+//
+//	def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32,
+//	                                    eth1_timestamp: uint64,
+//	                                    deposits: Sequence[Deposit]) -> BeaconState:
+//	  fork = Fork(
+//	      previous_version=GENESIS_FORK_VERSION,
+//	      current_version=GENESIS_FORK_VERSION,
+//	      epoch=GENESIS_EPOCH,
+//	  )
+//	  state = BeaconState(
+//	      genesis_time=eth1_timestamp + GENESIS_DELAY,
+//	      fork=fork,
+//	      eth1_data=Eth1Data(block_hash=eth1_block_hash, deposit_count=uint64(len(deposits))),
+//	      latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())),
+//	      randao_mixes=[eth1_block_hash] * EPOCHS_PER_HISTORICAL_VECTOR,  # Seed RANDAO with Eth1 entropy
+//	  )
+//
+//	  # Process deposits
+//	  leaves = list(map(lambda deposit: deposit.data, deposits))
+//	  for index, deposit in enumerate(deposits):
+//	      deposit_data_list = List[DepositData, 2**DEPOSIT_CONTRACT_TREE_DEPTH](*leaves[:index + 1])
+//	      state.eth1_data.deposit_root = hash_tree_root(deposit_data_list)
+//	      process_deposit(state, deposit)
+//
+//	  # Process activations
+//	  for index, validator in enumerate(state.validators):
+//	      balance = state.balances[index]
+//	      validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE)
+//	      if validator.effective_balance == MAX_EFFECTIVE_BALANCE:
+//	          validator.activation_eligibility_epoch = GENESIS_EPOCH
+//	          validator.activation_epoch = GENESIS_EPOCH
+//
+//	  # Set genesis validators root for domain separation and chain versioning
+//	  state.genesis_validators_root = hash_tree_root(state.validators)
+//
+//	  return state
+//
+// This method differs from the spec so as to process deposits beforehand instead of the end of the function.
+func GenesisBeaconStateBellatrix(ctx context.Context, deposits []*ethpb.Deposit, genesisTime uint64, eth1Data *ethpb.Eth1Data, ep *enginev1.ExecutionPayload) (state.BeaconState, error) {
+	st, err := EmptyGenesisStateBellatrix()
+	if err != nil {
+		return nil, err
+	}
+
+	// Process initial deposits.
+	st, err = helpers.UpdateGenesisEth1Data(st, deposits, eth1Data)
+	if err != nil {
+		return nil, err
+	}
+
+	st, err = b.ProcessPreGenesisDeposits(ctx, st, deposits)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not process validator deposits")
+	}
+
+	// After deposits have been processed, overwrite eth1data to what is passed in. This allows us to "pre-mine" validators
+	// without the deposit root and count mismatching the real deposit contract.
+	if err := st.SetEth1Data(eth1Data); err != nil {
+		return nil, err
+	}
+	if err := st.SetEth1DepositIndex(eth1Data.DepositCount); err != nil {
+		return nil, err
+	}
+
+	return OptimizedGenesisBeaconStateBellatrix(genesisTime, st, st.Eth1Data(), ep)
+}
+
+// OptimizedGenesisBeaconState is used to create a state that has already processed deposits. This is to efficiently
+// create a mainnet state at chainstart.
+func OptimizedGenesisBeaconStateBellatrix(genesisTime uint64, preState state.BeaconState, eth1Data *ethpb.Eth1Data, ep *enginev1.ExecutionPayload) (state.BeaconState, error) {
+	if eth1Data == nil {
+		return nil, errors.New("no eth1data provided for genesis state")
+	}
+
+	randaoMixes := make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector)
+	for i := 0; i < len(randaoMixes); i++ {
+		h := make([]byte, 32)
+		copy(h, eth1Data.BlockHash)
+		randaoMixes[i] = h
+	}
+
+	zeroHash := params.BeaconConfig().ZeroHash[:]
+
+	activeIndexRoots := make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector)
+	for i := 0; i < len(activeIndexRoots); i++ {
+		activeIndexRoots[i] = zeroHash
+	}
+
+	blockRoots := make([][]byte, params.BeaconConfig().SlotsPerHistoricalRoot)
+	for i := 0; i < len(blockRoots); i++ {
+		blockRoots[i] = zeroHash
+	}
+
+	stateRoots := make([][]byte, params.BeaconConfig().SlotsPerHistoricalRoot)
+	for i := 0; i < len(stateRoots); i++ {
+		stateRoots[i] = zeroHash
+	}
+
+	slashings := make([]uint64, params.BeaconConfig().EpochsPerSlashingsVector)
+
+	genesisValidatorsRoot, err := stateutil.ValidatorRegistryRoot(preState.Validators())
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not hash tree root genesis validators %v", err)
+	}
+
+	scores, err := preState.InactivityScores()
+	if err != nil {
+		return nil, err
+	}
+	scoresMissing := len(preState.Validators()) - len(scores)
+	if scoresMissing > 0 {
+		for i := 0; i < scoresMissing; i++ {
+			scores = append(scores, 0)
+		}
+	}
+	wep, err := blocks.WrappedExecutionPayload(ep)
+	if err != nil {
+		return nil, err
+	}
+	eph, err := blocks.PayloadToHeader(wep)
+	if err != nil {
+		return nil, err
+	}
+	st := &ethpb.BeaconStateBellatrix{
+		// Misc fields.
+		Slot:                  0,
+		GenesisTime:           genesisTime,
+		GenesisValidatorsRoot: genesisValidatorsRoot[:],
+
+		Fork: &ethpb.Fork{
+			PreviousVersion: params.BeaconConfig().AltairForkVersion,
+			CurrentVersion:  params.BeaconConfig().BellatrixForkVersion,
+			Epoch:           0,
+		},
+
+		// Validator registry fields.
+		Validators: preState.Validators(),
+		Balances:   preState.Balances(),
+
+		// Randomness and committees.
+		RandaoMixes: randaoMixes,
+
+		// Finality.
+		PreviousJustifiedCheckpoint: &ethpb.Checkpoint{
+			Epoch: 0,
+			Root:  params.BeaconConfig().ZeroHash[:],
+		},
+		CurrentJustifiedCheckpoint: &ethpb.Checkpoint{
+			Epoch: 0,
+			Root:  params.BeaconConfig().ZeroHash[:],
+		},
+		JustificationBits: []byte{0},
+		FinalizedCheckpoint: &ethpb.Checkpoint{
+			Epoch: 0,
+			Root:  params.BeaconConfig().ZeroHash[:],
+		},
+
+		HistoricalRoots: [][]byte{},
+		BlockRoots:      blockRoots,
+		StateRoots:      stateRoots,
+		Slashings:       slashings,
+
+		// Eth1 data.
+		Eth1Data:                     eth1Data,
+		Eth1DataVotes:                []*ethpb.Eth1Data{},
+		Eth1DepositIndex:             preState.Eth1DepositIndex(),
+		LatestExecutionPayloadHeader: eph,
+		InactivityScores:             scores,
+	}
+
+	bodyRoot, err := (&ethpb.BeaconBlockBodyBellatrix{
+		RandaoReveal: make([]byte, 96),
+		Eth1Data: &ethpb.Eth1Data{
+			DepositRoot: make([]byte, 32),
+			BlockHash:   make([]byte, 32),
+		},
+		Graffiti: make([]byte, 32),
+		SyncAggregate: &ethpb.SyncAggregate{
+			SyncCommitteeBits:      make([]byte, fieldparams.SyncCommitteeLength/8),
+			SyncCommitteeSignature: make([]byte, fieldparams.BLSSignatureLength),
+		},
+		ExecutionPayload: &enginev1.ExecutionPayload{
+			ParentHash:    make([]byte, 32),
+			FeeRecipient:  make([]byte, 20),
+			StateRoot:     make([]byte, 32),
+			ReceiptsRoot:  make([]byte, 32),
+			LogsBloom:     make([]byte, 256),
+			PrevRandao:    make([]byte, 32),
+			BaseFeePerGas: make([]byte, 32),
+			BlockHash:     make([]byte, 32),
+			Transactions:  make([][]byte, 0),
+		},
+	}).HashTreeRoot()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not hash tree root empty block body")
+	}
+
+	st.LatestBlockHeader = &ethpb.BeaconBlockHeader{
+		ParentRoot: zeroHash,
+		StateRoot:  zeroHash,
+		BodyRoot:   bodyRoot[:],
+	}
+
+	ist, err := state_native.InitializeFromProtoBellatrix(st)
+	if err != nil {
+		return nil, err
+	}
+	sc, err := altair.NextSyncCommittee(context.Background(), ist)
+	if err != nil {
+		return nil, err
+	}
+	if err := ist.SetNextSyncCommittee(sc); err != nil {
+		return nil, err
+	}
+	if err := ist.SetCurrentSyncCommittee(sc); err != nil {
+		return nil, err
+	}
+	return ist, nil
+}
+
+// EmptyGenesisState returns an empty beacon state object.
+func EmptyGenesisStateBellatrix() (state.BeaconState, error) {
+	st := &ethpb.BeaconStateBellatrix{
+		// Misc fields.
+		Slot: 0,
+		Fork: &ethpb.Fork{
+			PreviousVersion: params.BeaconConfig().AltairForkVersion,
+			CurrentVersion:  params.BeaconConfig().BellatrixForkVersion,
+			Epoch:           0,
+		},
+		// Validator registry fields.
+		Validators: []*ethpb.Validator{},
+		Balances:   []uint64{},
+
+		JustificationBits: []byte{0},
+		HistoricalRoots:   [][]byte{},
+
+		// Eth1 data.
+		Eth1Data:         &ethpb.Eth1Data{},
+		Eth1DataVotes:    []*ethpb.Eth1Data{},
+		Eth1DepositIndex: 0,
+		LatestExecutionPayloadHeader: &enginev1.ExecutionPayloadHeader{
+			ParentHash:       make([]byte, 32),
+			FeeRecipient:     make([]byte, 20),
+			StateRoot:        make([]byte, 32),
+			ReceiptsRoot:     make([]byte, 32),
+			LogsBloom:        make([]byte, 256),
+			PrevRandao:       make([]byte, 32),
+			BaseFeePerGas:    make([]byte, 32),
+			BlockHash:        make([]byte, 32),
+			TransactionsRoot: make([]byte, 32),
+		},
+	}
+
+	return state_native.InitializeFromProtoBellatrix(st)
+}

beacon-chain/db/kv/genesis.go

@@ -1,17 +1,14 @@
 package kv
 
 import (
-	"bytes"
 	"context"
-	"fmt"
+
+	"github.com/prysmaticlabs/prysm/v3/encoding/ssz/detect"
 
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v3/beacon-chain/core/blocks"
 	dbIface "github.com/prysmaticlabs/prysm/v3/beacon-chain/db/iface"
 	"github.com/prysmaticlabs/prysm/v3/beacon-chain/state"
-	state_native "github.com/prysmaticlabs/prysm/v3/beacon-chain/state/state-native"
-	"github.com/prysmaticlabs/prysm/v3/config/params"
-	consensusblocks "github.com/prysmaticlabs/prysm/v3/consensus-types/blocks"
 	ethpb "github.com/prysmaticlabs/prysm/v3/proto/prysm/v1alpha1"
 )
 
@@ -19,17 +16,25 @@ import (
 func (s *Store) SaveGenesisData(ctx context.Context, genesisState state.BeaconState) error {
 	stateRoot, err := genesisState.HashTreeRoot(ctx)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "HashTreeRoot")
 	}
-	genesisBlk := blocks.NewGenesisBlock(stateRoot[:])
-	genesisBlkRoot, err := genesisBlk.Block.HashTreeRoot()
+	wsb, err := blocks.NewGenesisBlockForState(stateRoot, genesisState)
+	if err != nil {
+		return errors.Wrap(err, "NewGenesisBlockForState")
+	}
+	genesisBlkRoot, err := wsb.Block().HashTreeRoot()
 	if err != nil {
 		return errors.Wrap(err, "could not get genesis block root")
 	}
-	wsb, err := consensusblocks.NewSignedBeaconBlock(genesisBlk)
-	if err != nil {
-		return errors.Wrap(err, "could not wrap genesis block")
-	}
+	/*
+		lbhr, err := genesisState.LatestBlockHeader().HashTreeRoot()
+		if err != nil {
+			return errors.Wrap(err, "unable to compute HTR of latest block header from genesis state")
+		}
+		if genesisBlkRoot != lbhr {
+			return fmt.Errorf("state.latest_block_header=%#x does not match derived genessis block root=%#x", genesisBlkRoot, lbhr)
+		}
+	*/
 	if err := s.SaveBlock(ctx, wsb); err != nil {
 		return errors.Wrap(err, "could not save genesis block")
 	}
@@ -54,11 +59,11 @@ func (s *Store) SaveGenesisData(ctx context.Context, genesisState state.BeaconSt
 
 // LoadGenesis loads a genesis state from a ssz-serialized byte slice, if no genesis exists already.
 func (s *Store) LoadGenesis(ctx context.Context, sb []byte) error {
-	st := &ethpb.BeaconState{}
-	if err := st.UnmarshalSSZ(sb); err != nil {
+	vu, err := detect.FromState(sb)
+	if err != nil {
 		return err
 	}
-	gs, err := state_native.InitializeFromProtoUnsafePhase0(st)
+	gs, err := vu.UnmarshalBeaconState(sb)
 	if err != nil {
 		return err
 	}
@@ -83,10 +88,6 @@ func (s *Store) LoadGenesis(ctx context.Context, sb []byte) error {
 		return dbIface.ErrExistingGenesisState
 	}
 
-	if !bytes.Equal(gs.Fork().CurrentVersion, params.BeaconConfig().GenesisForkVersion) {
-		return fmt.Errorf("loaded genesis fork version (%#x) does not match config genesis "+
-			"fork version (%#x)", gs.Fork().CurrentVersion, params.BeaconConfig().GenesisForkVersion)
-	}
 	return s.SaveGenesisData(ctx, gs)
 }
 

beacon-chain/execution/block_reader.go

@@ -106,7 +106,11 @@ func (s *Service) BlockByTimestamp(ctx context.Context, time uint64) (*types.Hea
 	latestBlkTime := s.latestEth1Data.BlockTime
 	s.latestEth1DataLock.RUnlock()
 
+	log.WithField("time (req val)", time).WithField("latestBlockTime", latestBlkTime).WithField("latestBlockHeight", s.latestEth1Data.BlockHeight).Info("BlockByTimestamp")
 	if time > latestBlkTime {
+		if latestBlkHeight < params.BeaconConfig().Eth1FollowDistance {
+
+		}
 		return nil, errors.Wrap(errBlockTimeTooLate, fmt.Sprintf("(%d > %d)", time, latestBlkTime))
 	}
 	// Initialize a pointer to eth1 chain's history to start our search from.

beacon-chain/execution/log_processing.go

@@ -288,13 +288,13 @@ func (s *Service) processPastLogs(ctx context.Context) error {
 	headersMap := make(map[uint64]*types.HeaderInfo)
 	rawLogCount, err := s.depositContractCaller.GetDepositCount(&bind.CallOpts{})
 	if err != nil {
-		return err
+		return errors.Wrap(err, "GetDepositCount")
 	}
 	logCount := binary.LittleEndian.Uint64(rawLogCount)
 
 	latestFollowHeight, err := s.followedBlockHeight(ctx)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "followedBlockHeight")
 	}
 
 	batchSize := s.cfg.eth1HeaderReqLimit
@@ -303,7 +303,7 @@ func (s *Service) processPastLogs(ctx context.Context) error {
 	for currentBlockNum < latestFollowHeight {
 		currentBlockNum, batchSize, err = s.processBlockInBatch(ctx, currentBlockNum, latestFollowHeight, batchSize, additiveFactor, logCount, headersMap)
 		if err != nil {
-			return err
+			return errors.Wrapf(err, "processBlockInBatch, num=%d, height=%d", currentBlockNum, latestFollowHeight)
 		}
 	}
 
@@ -313,7 +313,7 @@ func (s *Service) processPastLogs(ctx context.Context) error {
 
 	c, err := s.cfg.beaconDB.FinalizedCheckpoint(ctx)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "FinalizedCheckpoint")
 	}
 	fRoot := bytesutil.ToBytes32(c.Root)
 	// Return if no checkpoint exists yet.
@@ -333,7 +333,7 @@ func (s *Service) processPastLogs(ctx context.Context) error {
 	if isNil || slots.ToEpoch(fState.Slot()) != c.Epoch {
 		fState, err = s.cfg.stateGen.StateByRoot(ctx, fRoot)
 		if err != nil {
-			return err
+			return errors.Wrapf(err, "StateByRoot=%#x", fRoot)
 		}
 	}
 	if fState != nil && !fState.IsNil() && fState.Eth1DepositIndex() > 0 {

beacon-chain/execution/rpc_connection.go

@@ -45,7 +45,7 @@ func (s *Service) setupExecutionClientConnections(ctx context.Context, currEndpo
 		}
 		return errors.Wrap(err, errStr)
 	}
-	s.updateConnectedETH1(true)
+	s.updateConnectedETH1(true, "execution chain connection established")
 	s.runError = nil
 	return nil
 }
@@ -88,13 +88,14 @@ func (s *Service) pollConnectionStatus(ctx context.Context) {
 
 // Forces to retry an execution client connection.
 func (s *Service) retryExecutionClientConnection(ctx context.Context, err error) {
-	s.runError = err
-	s.updateConnectedETH1(false)
+	log.Debugf("retryExecutionClientConnection, err=%s", err.Error())
+	s.runError = errors.Wrap(err, "retryExecutionClientConnection")
+	s.updateConnectedETH1(false, "retryExecutionClientConnection")
 	// Back off for a while before redialing.
 	time.Sleep(backOffPeriod)
 	currClient := s.rpcClient
 	if err := s.setupExecutionClientConnections(ctx, s.cfg.currHttpEndpoint); err != nil {
-		s.runError = err
+		s.runError = errors.Wrap(err, "setupExecutionClientConnections")
 		return
 	}
 	// Close previous client, if connection was successful.

beacon-chain/execution/service.go

@@ -307,7 +307,8 @@ func (s *Service) updateBeaconNodeStats() {
 	s.cfg.beaconNodeStatsUpdater.Update(bs)
 }
 
-func (s *Service) updateConnectedETH1(state bool) {
+func (s *Service) updateConnectedETH1(state bool, reason string) {
+	log.Infof("updateConnectedETH1 - %s", reason)
 	s.connectedETH1 = state
 	s.updateBeaconNodeStats()
 }
@@ -319,10 +320,14 @@ func (s *Service) followedBlockHeight(ctx context.Context) (uint64, error) {
 	latestBlockTime := uint64(0)
 	if s.latestEth1Data.BlockTime > followTime {
 		latestBlockTime = s.latestEth1Data.BlockTime - followTime
+		if s.latestEth1Data.BlockHeight < params.BeaconConfig().Eth1FollowDistance {
+			latestBlockTime = s.latestEth1Data.BlockTime
+		}
 	}
+	log.WithField("distance", params.BeaconConfig().Eth1FollowDistance).WithField("followTime", followTime).WithField("BlockTime", s.latestEth1Data.BlockTime).WithField("BlockHeight", s.latestEth1Data.BlockHeight).WithField("latestBlockTime", latestBlockTime).Info("followedBlockHeight")
 	blk, err := s.BlockByTimestamp(ctx, latestBlockTime)
 	if err != nil {
-		return 0, err
+		return 0, errors.Wrapf(err, "BlockByTimestamp=%d", latestBlockTime)
 	}
 	return blk.Number.Uint64(), nil
 }
@@ -467,21 +472,23 @@ func (s *Service) handleETH1FollowDistance() {
 	}
 	if !s.chainStartData.Chainstarted {
 		if err := s.processChainStartFromBlockNum(ctx, big.NewInt(int64(s.latestEth1Data.LastRequestedBlock))); err != nil {
-			s.runError = err
+			s.runError = errors.Wrap(err, "processChainStartFromBlockNum")
 			log.Error(err)
 			return
 		}
 	}
+
 	// If the last requested block has not changed,
 	// we do not request batched logs as this means there are no new
 	// logs for the powchain service to process. Also it is a potential
 	// failure condition as would mean we have not respected the protocol threshold.
 	if s.latestEth1Data.LastRequestedBlock == s.latestEth1Data.BlockHeight {
-		log.Error("Beacon node is not respecting the follow distance")
+		// TODO this is usually an error but temporarily setting it to warn because its noisy when starting from genesis
+		log.Warn("Beacon node is not respecting the follow distance")
 		return
 	}
 	if err := s.requestBatchedHeadersAndLogs(ctx); err != nil {
-		s.runError = err
+		s.runError = errors.Wrap(err, "requestBatchedHeadersAndLogs")
 		log.Error(err)
 		return
 	}
@@ -511,6 +518,7 @@ func (s *Service) initPOWService() {
 			ctx := s.ctx
 			header, err := s.HeaderByNumber(ctx, nil)
 			if err != nil {
+				err = errors.Wrap(err, "HeaderByNumber")
 				s.retryExecutionClientConnection(ctx, err)
 				errorLogger(err, "Unable to retrieve latest execution client header")
 				continue
@@ -523,6 +531,7 @@ func (s *Service) initPOWService() {
 			s.latestEth1DataLock.Unlock()
 
 			if err := s.processPastLogs(ctx); err != nil {
+				err = errors.Wrap(err, "processPastLogs")
 				s.retryExecutionClientConnection(ctx, err)
 				errorLogger(
 					err,
@@ -532,6 +541,7 @@ func (s *Service) initPOWService() {
 			}
 			// Cache eth1 headers from our voting period.
 			if err := s.cacheHeadersForEth1DataVote(ctx); err != nil {
+				err = errors.Wrap(err, "cacheHeadersForEth1DataVote")
 				s.retryExecutionClientConnection(ctx, err)
 				if errors.Is(err, errBlockTimeTooLate) {
 					log.WithError(err).Debug("Unable to cache headers for execution client votes")
@@ -550,6 +560,7 @@ func (s *Service) initPOWService() {
 				if genHash != [32]byte{} {
 					genHeader, err := s.HeaderByHash(ctx, genHash)
 					if err != nil {
+						err = errors.Wrapf(err, "HeaderByHash, hash=%#x", genHash)
 						s.retryExecutionClientConnection(ctx, err)
 						errorLogger(err, "Unable to retrieve proof-of-stake genesis block data")
 						continue
@@ -558,6 +569,7 @@ func (s *Service) initPOWService() {
 				}
 				s.chainStartData.GenesisBlock = genBlock
 				if err := s.savePowchainData(ctx); err != nil {
+					err = errors.Wrap(err, "savePowchainData")
 					s.retryExecutionClientConnection(ctx, err)
 					errorLogger(err, "Unable to save execution client data")
 					continue
@@ -583,7 +595,7 @@ func (s *Service) run(done <-chan struct{}) {
 			s.isRunning = false
 			s.runError = nil
 			s.rpcClient.Close()
-			s.updateConnectedETH1(false)
+			s.updateConnectedETH1(false, "context canceled in run()")
 			log.Debug("Context closed, exiting goroutine")
 			return
 		case <-s.eth1HeadTicker.C:
@@ -641,11 +653,11 @@ func (s *Service) cacheHeadersForEth1DataVote(ctx context.Context) error {
 	// Find the end block to request from.
 	end, err := s.followedBlockHeight(ctx)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "followedBlockHeight")
 	}
 	start, err := s.determineEarliestVotingBlock(ctx, end)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "determineEarliestVotingBlock=%d", end)
 	}
 	return s.cacheBlockHeaders(start, end)
 }
@@ -677,7 +689,7 @@ func (s *Service) cacheBlockHeaders(start, end uint64) error {
 				}
 				continue
 			}
-			return err
+			return errors.Wrapf(err, "cacheBlockHeaders, start=%d, end=%d", startReq, endReq)
 		}
 	}
 	return nil
@@ -696,12 +708,22 @@ func (s *Service) determineEarliestVotingBlock(ctx context.Context, followBlock
 		}
 		return earliestBlk, nil
 	}
+	if s.latestEth1Data.BlockHeight < params.BeaconConfig().Eth1FollowDistance {
+		return 0, nil
+	}
 	votingTime := slots.VotingPeriodStartTime(genesisTime, currSlot)
 	followBackDist := 2 * params.BeaconConfig().SecondsPerETH1Block * params.BeaconConfig().Eth1FollowDistance
 	if followBackDist > votingTime {
 		return 0, errors.Errorf("invalid genesis time provided. %d > %d", followBackDist, votingTime)
 	}
 	earliestValidTime := votingTime - followBackDist
+	/*
+		if earliestValidTime > s.latestEth1Data.BlockTime {
+			return 0, nil
+		}
+		log.WithField("earliestValidTime", earliestValidTime).Info("calling BlockByTimestamp")
+	*/
+	log.WithField("distance", params.BeaconConfig().Eth1FollowDistance).WithField("earliestValidTime", earliestValidTime).WithField("BlockTime", s.latestEth1Data.BlockTime).WithField("BlockHeight", s.latestEth1Data.BlockHeight).WithField("followBlock", followBlock).Info("determineEarliestVotingBlock")
 	hdr, err := s.BlockByTimestamp(ctx, earliestValidTime)
 	if err != nil {
 		return 0, err

beacon-chain/execution/testing/BUILD.bazel

@@ -4,13 +4,14 @@ go_library(
     name = "go_default_library",
     testonly = True,
     srcs = [
+        "genesis.go",
         "mock_engine_client.go",
         "mock_execution_chain.go",
         "mock_faulty_powchain.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/v3/beacon-chain/execution/testing",
     visibility = [
-        "//beacon-chain:__subpackages__",
+        "//visibility:public",
     ],
     deps = [
         "//async/event:go_default_library",
@@ -26,7 +27,9 @@ go_library(
         "@com_github_ethereum_go_ethereum//accounts/abi/bind/backends:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
+        "@com_github_ethereum_go_ethereum//core:go_default_library",
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
+        "@com_github_ethereum_go_ethereum//params:go_default_library",
         "@com_github_ethereum_go_ethereum//rpc:go_default_library",
         "@com_github_holiman_uint256//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",

beacon-chain/rpc/prysm/v1alpha1/validator/proposer_bellatrix.go

@@ -96,7 +96,7 @@ func (vs *Server) getBellatrixBeaconBlock(ctx context.Context, req *ethpb.BlockR
 	stateRoot, err := vs.computeStateRoot(ctx, wsb)
 	if err != nil {
 		interop.WriteBlockToDisk(wsb, true /*failed*/)
-		return nil, fmt.Errorf("could not compute state root: %v", err)
+		return nil, fmt.Errorf("could not compute state root, %v", err)
 	}
 	blk.StateRoot = stateRoot
 	return &ethpb.GenericBeaconBlock{Block: &ethpb.GenericBeaconBlock_Bellatrix{Bellatrix: blk}}, nil

beacon-chain/rpc/prysm/v1alpha1/validator/proposer_eth1data.go

@@ -2,6 +2,7 @@ package validator
 
 import (
 	"context"
+	"fmt"
 	"math/big"
 
 	"github.com/pkg/errors"
@@ -41,25 +42,38 @@ func (vs *Server) eth1DataMajorityVote(ctx context.Context, beaconState state.Be
 		return vs.mockETH1DataVote(ctx, slot)
 	}
 	if !vs.Eth1InfoFetcher.ExecutionClientConnected() {
-		return vs.randomETH1DataVote(ctx)
+		return vs.randomETH1DataVote(ctx, "ExecutionClientConnected == false")
 	}
 	eth1DataNotification = false
 
-	eth1FollowDistance := params.BeaconConfig().Eth1FollowDistance
-	earliestValidTime := votingPeriodStartTime - 2*params.BeaconConfig().SecondsPerETH1Block*eth1FollowDistance
-	latestValidTime := votingPeriodStartTime - params.BeaconConfig().SecondsPerETH1Block*eth1FollowDistance
+	genesisTime, _ := vs.Eth1InfoFetcher.GenesisExecutionChainInfo()
+	followDistanceSeconds := params.BeaconConfig().Eth1FollowDistance * params.BeaconConfig().SecondsPerETH1Block
+	latestValidTime := votingPeriodStartTime - followDistanceSeconds
+	earliestValidTime := votingPeriodStartTime - 2*followDistanceSeconds
+
+	// Special case for starting from a pre-mined genesis: the eth1 vote should be genesis until the chain has advanced
+	// by ETH1_FOLLOW_DISTANCE. The head state should maintain the same ETH1Data until this condition has passed, so
+	// trust the existing head for the right eth1 vote until we can get a meaningful value from the deposit contract.
+	log.WithField("votingPeriodStartTime", votingPeriodStartTime-genesisTime).WithField("latestValidTime", latestValidTime-genesisTime).WithField("earliestValidTime", earliestValidTime-genesisTime).Info("eth1DataMajorityVote")
+	if latestValidTime < genesisTime+followDistanceSeconds {
+		log.WithField("genesisTime", genesisTime).WithField("latestValidTime", latestValidTime).Info("voting period before genesis + follow distance, return head eth1data")
+		return vs.HeadFetcher.HeadETH1Data(), nil
+	}
 
 	lastBlockByLatestValidTime, err := vs.Eth1BlockFetcher.BlockByTimestamp(ctx, latestValidTime)
 	if err != nil {
 		log.WithError(err).Error("Could not get last block by latest valid time")
-		return vs.randomETH1DataVote(ctx)
+		return vs.randomETH1DataVote(ctx, "error from BlockByTimestamp")
 	}
 	if lastBlockByLatestValidTime.Time < earliestValidTime {
-		return vs.HeadFetcher.HeadETH1Data(), nil
+		ed := vs.HeadFetcher.HeadETH1Data()
+		log.WithField("latestBlock.Time", lastBlockByLatestValidTime.Time).WithField("latestBlock.Height", lastBlockByLatestValidTime.Number).WithFields(logFields(ed)).WithField("condition", "lastBlockByLatestValidTime.Time < earliestValidTime").Info("eth1DataMajorityVote")
+		return ed, nil
 	}
 
 	lastBlockDepositCount, lastBlockDepositRoot := vs.DepositFetcher.DepositsNumberAndRootAtHeight(ctx, lastBlockByLatestValidTime.Number)
 	if lastBlockDepositCount == 0 {
+		log.WithFields(logFields(vs.ChainStartFetcher.ChainStartEth1Data())).WithField("condition", "ChainStartEth1Data").Info("eth1DataMajorityVote")
 		return vs.ChainStartFetcher.ChainStartEth1Data(), nil
 	}
 
@@ -67,17 +81,27 @@ func (vs *Server) eth1DataMajorityVote(ctx context.Context, beaconState state.Be
 		h, err := vs.Eth1BlockFetcher.BlockHashByHeight(ctx, lastBlockByLatestValidTime.Number)
 		if err != nil {
 			log.WithError(err).Error("Could not get hash of last block by latest valid time")
-			return vs.randomETH1DataVote(ctx)
+			return vs.randomETH1DataVote(ctx, "deposit count too high")
 		}
-		return &ethpb.Eth1Data{
+		ed := &ethpb.Eth1Data{
 			BlockHash:    h.Bytes(),
 			DepositCount: lastBlockDepositCount,
 			DepositRoot:  lastBlockDepositRoot[:],
-		}, nil
+		}
+		log.WithFields(logFields(ed)).WithField("condition", "lastBlockDepositCount >= vs.HeadFetcher.HeadETH1Data().DepositCount").Info("eth1DataMajorityVote")
+		return ed, nil
 	}
 	return vs.HeadFetcher.HeadETH1Data(), nil
 }
 
+func logFields(ed *ethpb.Eth1Data) map[string]interface{} {
+	fields := make(map[string]interface{})
+	fields["deposit_root"] = fmt.Sprintf("%#x", ed.DepositRoot)
+	fields["deposit_count"] = ed.DepositCount
+	fields["block_hash"] = fmt.Sprintf("%#x", ed.BlockHash)
+	return fields
+}
+
 func (vs *Server) slotStartTime(slot types.Slot) uint64 {
 	startTime, _ := vs.Eth1InfoFetcher.GenesisExecutionChainInfo()
 	return slots.VotingPeriodStartTime(startTime, slot)
@@ -147,7 +171,8 @@ func (vs *Server) mockETH1DataVote(ctx context.Context, slot types.Slot) (*ethpb
 	}, nil
 }
 
-func (vs *Server) randomETH1DataVote(ctx context.Context) (*ethpb.Eth1Data, error) {
+func (vs *Server) randomETH1DataVote(ctx context.Context, reason string) (*ethpb.Eth1Data, error) {
+	log.WithField("reason", reason).Warn("randomETH1DataVote")
 	if !eth1DataNotification {
 		log.Warn("Beacon Node is no longer connected to an ETH1 chain, so ETH1 data votes are now random.")
 		eth1DataNotification = true

beacon-chain/rpc/prysm/v1alpha1/validator/proposer_phase0.go

@@ -96,12 +96,30 @@ func (vs *Server) buildPhase0BlockData(ctx context.Context, req *ethpb.BlockRequ
 	if err != nil {
 		return nil, fmt.Errorf("could not get head state %v", err)
 	}
+	headRoot, err := head.HashTreeRoot(ctx)
+	if err != nil {
+		return nil, err
+	}
+	lbhr, err := head.LatestBlockHeader().HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	log.WithField("header_parent", fmt.Sprintf("%#x", head.LatestBlockHeader().ParentRoot)).WithField("header_state_root", fmt.Sprintf("%#x", head.LatestBlockHeader().StateRoot)).WithField("slot", head.Slot()).WithField("latest_block_header.root", fmt.Sprintf("%#x", lbhr)).WithField("root", fmt.Sprintf("%#x", headRoot)).Info("buildPhase0BlockData head state")
 
 	head, err = transition.ProcessSlotsUsingNextSlotCache(ctx, head, parentRoot, req.Slot)
 	if err != nil {
 		return nil, fmt.Errorf("could not advance slots to calculate proposer index: %v", err)
 	}
 
+	headRoot, err = head.HashTreeRoot(ctx)
+	if err != nil {
+		return nil, err
+	}
+	lbhr, err = head.LatestBlockHeader().HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	log.WithField("header_parent", fmt.Sprintf("%#x", head.LatestBlockHeader().ParentRoot)).WithField("header_state_root", fmt.Sprintf("%#x", head.LatestBlockHeader().StateRoot)).WithField("slot", head.Slot()).WithField("latest_block_header.root", fmt.Sprintf("%#x", lbhr)).WithField("root", fmt.Sprintf("%#x", headRoot)).Info("buildPhase0BlockData - after process_slots")
 	eth1Data, err := vs.eth1DataMajorityVote(ctx, head)
 	if err != nil {
 		return nil, fmt.Errorf("could not get ETH1 data: %v", err)

beacon-chain/sync/rpc_goodbye.go

@@ -54,7 +54,7 @@ func (s *Service) goodbyeRPCHandler(_ context.Context, msg interface{}, stream l
 
 // disconnectBadPeer checks whether peer is considered bad by some scorer, and tries to disconnect
 // the peer, if that is the case. Additionally, disconnection reason is obtained from scorer.
-func (s *Service) disconnectBadPeer(ctx context.Context, id peer.ID) {
+func (s *Service) disconnectBadPeer(ctx context.Context, id peer.ID, from string) {
 	if !s.cfg.p2p.Peers().IsBad(id) {
 		return
 	}
@@ -63,6 +63,7 @@ func (s *Service) disconnectBadPeer(ctx context.Context, id peer.ID) {
 	if err == nil {
 		goodbyeCode = p2ptypes.GoodbyeCodeBanned
 	}
+	log.Infof("disconnectBadPeer from %s", from)
 	if err := s.sendGoodByeAndDisconnect(ctx, goodbyeCode, id); err != nil {
 		log.WithError(err).Debug("Error when disconnecting with bad peer")
 	}

beacon-chain/sync/rpc_status.go

@@ -47,7 +47,7 @@ func (s *Service) maintainPeerStatuses() {
 				}
 				// Disconnect from peers that are considered bad by any of the registered scorers.
 				if s.cfg.p2p.Peers().IsBad(id) {
-					s.disconnectBadPeer(s.ctx, id)
+					s.disconnectBadPeer(s.ctx, id, "maintainPeerStatuses")
 					return
 				}
 				// If the status hasn't been updated in the recent interval time.
@@ -170,7 +170,7 @@ func (s *Service) sendRPCStatusRequest(ctx context.Context, id peer.ID) error {
 	err = s.validateStatusMessage(ctx, msg)
 	s.cfg.p2p.Peers().Scorers().PeerStatusScorer().SetPeerStatus(id, msg, err)
 	if s.cfg.p2p.Peers().IsBad(id) {
-		s.disconnectBadPeer(s.ctx, id)
+		s.disconnectBadPeer(s.ctx, id, "sendRPCStatusRequest")
 	}
 	return err
 }
@@ -222,6 +222,11 @@ func (s *Service) statusRPCHandler(ctx context.Context, msg interface{}, stream
 			}
 			// Close before disconnecting, and wait for the other end to ack our response.
 			closeStreamAndWait(stream, log)
+			forkDigest, err := s.currentForkDigest()
+			if err != nil {
+				log.WithError(err).Error("failed to compute fork digest")
+			}
+			log.Errorf("fork digest mismatch, expected=%#x, saw=%#x", forkDigest, m.ForkDigest)
 			if err := s.sendGoodByeAndDisconnect(ctx, p2ptypes.GoodbyeCodeWrongNetwork, remotePeer); err != nil {
 				return err
 			}

beacon-chain/sync/subscriber.go

@@ -316,6 +316,7 @@ func (s *Service) subscribeStaticWithSubnets(topic string, validator wrappedVal,
 				return
 			case <-ticker.C():
 				if s.chainStarted.IsSet() && s.cfg.initialSync.Syncing() {
+					log.Debug("(subscribeStaticWithSubnets) chain started set, but in initial sync")
 					continue
 				}
 				valid, err := isDigestValid(digest, genesis, genRoot)
@@ -386,6 +387,7 @@ func (s *Service) subscribeDynamicWithSubnets(
 				return
 			case currentSlot := <-ticker.C():
 				if s.chainStarted.IsSet() && s.cfg.initialSync.Syncing() {
+					log.Debug("(subscribeDynamicWithSubnets) chain started set, but in initial sync")
 					continue
 				}
 				valid, err := isDigestValid(digest, genesis, genRoot)
@@ -516,6 +518,7 @@ func (s *Service) subscribeStaticWithSyncSubnets(topic string, validator wrapped
 				return
 			case <-ticker.C():
 				if s.chainStarted.IsSet() && s.cfg.initialSync.Syncing() {
+					log.Debug("(subscribeStaticWithSyncSubnets) chain started set, but in initial sync")
 					continue
 				}
 				valid, err := isDigestValid(digest, genesis, genRoot)
@@ -585,6 +588,7 @@ func (s *Service) subscribeDynamicWithSyncSubnets(
 				return
 			case currentSlot := <-ticker.C():
 				if s.chainStarted.IsSet() && s.cfg.initialSync.Syncing() {
+					log.Debug("(subscribeDynamicWithSyncSubnets) chain started set, but in initial sync")
 					continue
 				}
 				valid, err := isDigestValid(digest, genesis, genRoot)

config/params/testnet_e2e_config.go

@@ -1,8 +1,8 @@
 package params
 
 const (
-	altairE2EForkEpoch    = 6
-	bellatrixE2EForkEpoch = 8
+	altairE2EForkEpoch    = 0
+	bellatrixE2EForkEpoch = 0
 )
 
 // E2ETestConfig retrieves the configurations made specifically for E2E testing.
@@ -10,6 +10,8 @@ const (
 // WARNING: This config is only for testing, it is not meant for use outside of E2E.
 func E2ETestConfig() *BeaconChainConfig {
 	e2eConfig := MinimalSpecConfig()
+	e2eConfig.DepositContractAddress = "0x4242424242424242424242424242424242424242"
+	e2eConfig.Eth1FollowDistance = 8
 
 	// Misc.
 	e2eConfig.MinGenesisActiveValidatorCount = 256
@@ -21,7 +23,6 @@ func E2ETestConfig() *BeaconChainConfig {
 	e2eConfig.SlotsPerEpoch = 6
 	e2eConfig.SqrRootSlotsPerEpoch = 2
 	e2eConfig.SecondsPerETH1Block = 2
-	e2eConfig.Eth1FollowDistance = 8
 	e2eConfig.EpochsPerEth1VotingPeriod = 2
 	e2eConfig.ShardCommitteePeriod = 4
 	e2eConfig.MaxSeedLookahead = 1
@@ -35,7 +36,7 @@ func E2ETestConfig() *BeaconChainConfig {
 	e2eConfig.BellatrixForkEpoch = bellatrixE2EForkEpoch
 
 	// Terminal Total Difficulty.
-	e2eConfig.TerminalTotalDifficulty = "616"
+	e2eConfig.TerminalTotalDifficulty = "0"
 
 	// Prysm constants.
 	e2eConfig.ConfigName = EndToEndName
@@ -50,6 +51,8 @@ func E2ETestConfig() *BeaconChainConfig {
 
 func E2EMainnetTestConfig() *BeaconChainConfig {
 	e2eConfig := MainnetConfig().Copy()
+	e2eConfig.DepositContractAddress = "0x4242424242424242424242424242424242424242"
+	e2eConfig.Eth1FollowDistance = 8
 
 	// Misc.
 	e2eConfig.MinGenesisActiveValidatorCount = 256
@@ -60,7 +63,6 @@ func E2EMainnetTestConfig() *BeaconChainConfig {
 	e2eConfig.SecondsPerSlot = 6
 	e2eConfig.SqrRootSlotsPerEpoch = 5
 	e2eConfig.SecondsPerETH1Block = 2
-	e2eConfig.Eth1FollowDistance = 8
 	e2eConfig.ShardCommitteePeriod = 4
 
 	// PoW parameters.
@@ -72,7 +74,7 @@ func E2EMainnetTestConfig() *BeaconChainConfig {
 	e2eConfig.BellatrixForkEpoch = bellatrixE2EForkEpoch
 
 	// Terminal Total Difficulty.
-	e2eConfig.TerminalTotalDifficulty = "616"
+	e2eConfig.TerminalTotalDifficulty = "0"
 
 	// Prysm constants.
 	e2eConfig.ConfigName = EndToEndMainnetName

network/forks/fork.go

@@ -2,6 +2,7 @@
 package forks
 
 import (
+	"bytes"
 	"math"
 	"sort"
 	"time"
@@ -165,7 +166,19 @@ func SortedForkVersions(forkSchedule map[[4]byte]types.Epoch) [][4]byte {
 		i++
 	}
 	sort.Slice(sortedVersions, func(a, b int) bool {
-		return forkSchedule[sortedVersions[a]] < forkSchedule[sortedVersions[b]]
+		// va == "version" a, ie the [4]byte version id
+		va, vb := sortedVersions[a], sortedVersions[b]
+		// ea == "epoch" a, ie the types.Epoch corresponding to va
+		ea, eb := forkSchedule[va], forkSchedule[vb]
+		// Try to sort by epochs first, which works fine when epochs are all distinct.
+		// in the case of testnets starting from a given fork, all epochs leading to the fork will be zero.
+		if ea != eb {
+			return ea < eb
+		}
+		// If the epochs are equal, break the tie with a lexicographic comparison of the fork version bytes.
+		// eg 2 versions both with a fork epoch of 0, 0x00000000 would come before 0x01000000.
+		// sort.Slice takes a 'less' func, ie `return a < b`, and when va < vb, bytes.Compare will return -1
+		return bytes.Compare(va[:], vb[:]) < 0
 	})
 	return sortedVersions
 }

runtime/interop/BUILD.bazel

@@ -4,6 +4,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "generate_genesis_state.go",
+        "generate_genesis_state_bellatrix.go",
         "generate_keys.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/v3/runtime/interop",
@@ -18,6 +19,7 @@ go_library(
         "//crypto/bls:go_default_library",
         "//crypto/hash:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//proto/engine/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//time:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
@@ -27,15 +29,17 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
+        "generate_genesis_state_bellatrix_test.go",
         "generate_genesis_state_test.go",
         "generate_keys_test.go",
     ],
     data = [
         "keygen_test_vector.yaml",
     ],
+    embed = [":go_default_library"],
     deps = [
-        ":go_default_library",
         "//beacon-chain/core/transition:go_default_library",
+        "//beacon-chain/state/state-native:go_default_library",
         "//config/params:go_default_library",
         "//container/trie:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",

runtime/interop/generate_genesis_state_bellatrix.go

@@ -0,0 +1,63 @@
+// Package interop contains deterministic utilities for generating
+// genesis states and keys.
+package interop
+
+import (
+	"context"
+
+	enginev1 "github.com/prysmaticlabs/prysm/v3/proto/engine/v1"
+
+	"github.com/pkg/errors"
+	coreState "github.com/prysmaticlabs/prysm/v3/beacon-chain/core/transition"
+	statenative "github.com/prysmaticlabs/prysm/v3/beacon-chain/state/state-native"
+	"github.com/prysmaticlabs/prysm/v3/config/params"
+	"github.com/prysmaticlabs/prysm/v3/container/trie"
+	ethpb "github.com/prysmaticlabs/prysm/v3/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/v3/time"
+)
+
+// GenerateGenesisStateBellatrix deterministically given a genesis time and number of validators.
+// If a genesis time of 0 is supplied it is set to the current time.
+func GenerateGenesisStateBellatrix(ctx context.Context, genesisTime, numValidators uint64, ep *enginev1.ExecutionPayload, ed *ethpb.Eth1Data) (*ethpb.BeaconStateBellatrix, []*ethpb.Deposit, error) {
+	privKeys, pubKeys, err := DeterministicallyGenerateKeys(0 /*startIndex*/, numValidators)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "could not deterministically generate keys for %d validators", numValidators)
+	}
+	depositDataItems, depositDataRoots, err := DepositDataFromKeys(privKeys, pubKeys)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "could not generate deposit data from keys")
+	}
+	return GenerateGenesisStateBellatrixFromDepositData(ctx, genesisTime, depositDataItems, depositDataRoots, ep, ed)
+}
+
+// GenerateGenesisStateBellatrixFromDepositData creates a genesis state given a list of
+// deposit data items and their corresponding roots.
+func GenerateGenesisStateBellatrixFromDepositData(
+	ctx context.Context, genesisTime uint64, depositData []*ethpb.Deposit_Data, depositDataRoots [][]byte, ep *enginev1.ExecutionPayload, e1d *ethpb.Eth1Data,
+) (*ethpb.BeaconStateBellatrix, []*ethpb.Deposit, error) {
+	t, err := trie.GenerateTrieFromItems(depositDataRoots, params.BeaconConfig().DepositContractTreeDepth)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "could not generate Merkle trie for deposit proofs")
+	}
+	deposits, err := GenerateDepositsFromData(depositData, t)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "could not generate deposits from the deposit data provided")
+	}
+	if genesisTime == 0 {
+		genesisTime = uint64(time.Now().Unix())
+	}
+	beaconState, err := coreState.GenesisBeaconStateBellatrix(ctx, deposits, genesisTime, e1d, ep)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "could not generate genesis state")
+	}
+	bsi := beaconState.ToProtoUnsafe()
+	pbb, ok := bsi.(*ethpb.BeaconStateBellatrix)
+	if !ok {
+		return nil, nil, errors.New("unexpected BeaconState version")
+	}
+	pbState, err := statenative.ProtobufBeaconStateBellatrix(pbb)
+	if err != nil {
+		return nil, nil, err
+	}
+	return pbState, deposits, nil
+}

runtime/interop/generate_genesis_state_bellatrix_test.go

@@ -0,0 +1,27 @@
+package interop
+
+import (
+	"context"
+	"testing"
+
+	state_native "github.com/prysmaticlabs/prysm/v3/beacon-chain/state/state-native"
+	"github.com/prysmaticlabs/prysm/v3/config/params"
+	"github.com/prysmaticlabs/prysm/v3/container/trie"
+	"github.com/prysmaticlabs/prysm/v3/testing/require"
+)
+
+func TestGenerateGenesisStateBellatrix(t *testing.T) {
+	g, _, err := GenerateGenesisStateBellatrix(context.Background(), 0, params.BeaconConfig().MinGenesisActiveValidatorCount)
+	require.NoError(t, err)
+
+	tr, err := trie.NewTrie(params.BeaconConfig().DepositContractTreeDepth)
+	require.NoError(t, err)
+	dr, err := tr.HashTreeRoot()
+	require.NoError(t, err)
+	g.Eth1Data.DepositRoot = dr[:]
+	g.Eth1Data.BlockHash = make([]byte, 32)
+	st, err := state_native.InitializeFromProtoUnsafeBellatrix(g)
+	require.NoError(t, err)
+	_, err = st.MarshalSSZ()
+	require.NoError(t, err)
+}

testing/endtoend/component_handler_test.go

@@ -2,6 +2,9 @@ package endtoend
 
 import (
 	"context"
+	"fmt"
+	"os"
+	"strconv"
 	"testing"
 	"time"
 
@@ -220,6 +223,37 @@ func (c *componentHandler) required() []e2etypes.ComponentRunner {
 	return requiredComponents
 }
 
+func (c *componentHandler) printPIDs(logger func(string, ...interface{})) {
+	msg := "\nPID of components. Attach a debugger... if you dare!\n\n"
+
+	msg = "This test PID: " + strconv.Itoa(os.Getpid()) + " (parent=" + strconv.Itoa(os.Getppid()) + ")\n"
+
+	// Beacon chain nodes
+	msg += fmt.Sprintf("Beacon chain nodes: %v\n", PIDsFromMultiComponentRunner(c.beaconNodes))
+	// Validator nodes
+	msg += fmt.Sprintf("Validators: %v\n", PIDsFromMultiComponentRunner(c.validatorNodes))
+	// ETH1 nodes
+	msg += fmt.Sprintf("ETH1 nodes: %v\n", PIDsFromMultiComponentRunner(c.eth1Nodes))
+
+	logger(msg)
+}
+
+func PIDsFromMultiComponentRunner(runner e2etypes.MultipleComponentRunners) []int {
+	var pids []int
+
+	for i := 0; true; i++ {
+		c, err := runner.ComponentAtIndex(i)
+		if c == nil || err != nil {
+			break
+		}
+		p := c.UnderlyingProcess()
+		if p != nil {
+			pids = append(pids, p.Pid)
+		}
+	}
+	return pids
+}
+
 func appendDebugEndpoints(cfg *e2etypes.E2EConfig) {
 	debug := []string{
 		"--enable-debug-rpc-endpoints",

testing/endtoend/components/BUILD.bazel

@@ -20,15 +20,23 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/v3/testing/endtoend/components",
     visibility = ["//testing/endtoend:__subpackages__"],
     deps = [
+        "//beacon-chain/core/blocks:go_default_library",
+        "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/state-native:go_default_library",
         "//cmd:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",
+        "//cmd/beacon-chain/sync/genesis:go_default_library",
         "//cmd/validator/flags:go_default_library",
         "//config/features:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
         "//config/validator/service:go_default_library",
+        "//container/trie:go_default_library",
         "//crypto/bls:go_default_library",
+        "//encoding/bytesutil:go_default_library",
         "//io/file:go_default_library",
+        "//proto/engine/v1:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
         "//runtime/interop:go_default_library",
         "//testing/endtoend/helpers:go_default_library",
         "//testing/endtoend/params:go_default_library",

testing/endtoend/components/beacon_node.go

@@ -12,6 +12,22 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/prysmaticlabs/prysm/v3/beacon-chain/core/blocks"
+
+	ethpb "github.com/prysmaticlabs/prysm/v3/proto/prysm/v1alpha1"
+
+	fieldparams "github.com/prysmaticlabs/prysm/v3/config/fieldparams"
+	"github.com/prysmaticlabs/prysm/v3/encoding/bytesutil"
+	enginev1 "github.com/prysmaticlabs/prysm/v3/proto/engine/v1"
+
+	"github.com/prysmaticlabs/prysm/v3/beacon-chain/state"
+	state_native "github.com/prysmaticlabs/prysm/v3/beacon-chain/state/state-native"
+
+	"github.com/prysmaticlabs/prysm/v3/cmd/beacon-chain/sync/genesis"
+	"github.com/prysmaticlabs/prysm/v3/container/trie"
+	"github.com/prysmaticlabs/prysm/v3/io/file"
+	"github.com/prysmaticlabs/prysm/v3/runtime/interop"
+
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
 	"github.com/pkg/errors"
 	cmdshared "github.com/prysmaticlabs/prysm/v3/cmd"
@@ -166,6 +182,109 @@ func NewBeaconNode(config *e2etypes.E2EConfig, index int, enr string) *BeaconNod
 	}
 }
 
+func (node *BeaconNode) generateGenesis(ctx context.Context) (state.BeaconState, error) {
+	if e2e.TestParams.Eth1GenesisBlock == nil {
+		return nil, errors.New("Cannot construct bellatrix block, e2e.TestParams.Eth1GenesisBlock == nil")
+	}
+	gb := e2e.TestParams.Eth1GenesisBlock
+
+	// so the DepositRoot in the BeaconState should be set to the HTR of an empty deposit trie.
+	t, err := trie.NewTrie(params.BeaconConfig().DepositContractTreeDepth)
+	if err != nil {
+		return nil, err
+	}
+	dr, err := t.HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	e1d := &ethpb.Eth1Data{
+		DepositRoot:  dr[:],
+		DepositCount: 0,
+		BlockHash:    gb.Hash().Bytes(),
+	}
+
+	payload := &enginev1.ExecutionPayload{
+		ParentHash:    gb.ParentHash().Bytes(),
+		FeeRecipient:  gb.Coinbase().Bytes(),
+		StateRoot:     gb.Root().Bytes(),
+		ReceiptsRoot:  gb.ReceiptHash().Bytes(),
+		LogsBloom:     gb.Bloom().Bytes(),
+		PrevRandao:    params.BeaconConfig().ZeroHash[:],
+		BlockNumber:   gb.NumberU64(),
+		GasLimit:      gb.GasLimit(),
+		GasUsed:       gb.GasUsed(),
+		Timestamp:     gb.Time(),
+		ExtraData:     gb.Extra()[:32],
+		BaseFeePerGas: bytesutil.PadTo(bytesutil.ReverseByteOrder(gb.BaseFee().Bytes()), fieldparams.RootLength),
+		BlockHash:     gb.Hash().Bytes(),
+		Transactions:  make([][]byte, 0),
+	}
+	genesis, _, err := interop.GenerateGenesisStateBellatrix(ctx, e2e.TestParams.CLGenesisTime, params.BeaconConfig().MinGenesisActiveValidatorCount, payload, e1d)
+	if err != nil {
+		return nil, err
+	}
+	sr, err := genesis.HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	lbhr, err := genesis.LatestBlockHeader.HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	si, err := state_native.InitializeFromProtoUnsafeBellatrix(genesis)
+	if err != nil {
+		return nil, err
+	}
+	genb, err := blocks.NewGenesisBlockForState(sr, si)
+	if err != nil {
+		return nil, err
+	}
+	gbr, err := genb.Block().HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	log.WithField("el_block_time", gb.Time()).
+		WithField("cl_genesis_time", genesis.GenesisTime).
+		WithField("state_root", fmt.Sprintf("%#x", sr)).
+		WithField("latest_block_header_root", fmt.Sprintf("%#x", lbhr)).
+		WithField("latest_block_header_state_root", fmt.Sprintf("%#x", genesis.LatestBlockHeader.StateRoot)).
+		WithField("latest_block_header_parent_root", fmt.Sprintf("%#x", genesis.LatestBlockHeader.ParentRoot)).
+		WithField("latest_block_header_body_root", fmt.Sprintf("%#x", genesis.LatestBlockHeader.BodyRoot)).
+		WithField("derived_block_root", fmt.Sprintf("%#x", gbr)).
+		WithField("el_block_root", fmt.Sprintf("%#x", genesis.Eth1Data.BlockHash)).
+		Info("genesis eth1 data")
+	return si, nil
+}
+
+func (node *BeaconNode) saveGenesis(ctx context.Context) (string, error) {
+	// The deposit contract starts with an empty trie, we use the BeaconState to "pre-mine" the validator registry,
+	g, err := node.generateGenesis(ctx)
+	if err != nil {
+		return "", err
+	}
+
+	root, err := g.HashTreeRoot(ctx)
+	if err != nil {
+		return "", err
+	}
+	lbhr, err := g.LatestBlockHeader().HashTreeRoot()
+	if err != nil {
+		return "", err
+	}
+	log.WithField("fork_version", g.Fork().CurrentVersion).WithField("latest_block_header.root", fmt.Sprintf("%#x", lbhr)).WithField("root", fmt.Sprintf("%#x", root)).Infof("BeaconState infoz")
+
+	genesisBytes, err := g.MarshalSSZ()
+	if err != nil {
+		return "", err
+	}
+	genesisDir := path.Join(e2e.TestParams.TestPath, fmt.Sprintf("genesis/%d", node.index))
+	if err := file.MkdirAll(genesisDir); err != nil {
+		return "", err
+	}
+	genesisPath := path.Join(genesisDir, "genesis.ssz")
+	return genesisPath, file.WriteFile(genesisPath, genesisBytes)
+}
+
 // Start starts a fresh beacon node, connecting to all passed in beacon nodes.
 func (node *BeaconNode) Start(ctx context.Context) error {
 	binaryPath, found := bazel.FindBinary("cmd/beacon-chain", "beacon-chain")
@@ -191,10 +310,16 @@ func (node *BeaconNode) Start(ctx context.Context) error {
 		jwtPath = path.Join(e2e.TestParams.TestPath, "eth1data/miner/")
 	}
 	jwtPath = path.Join(jwtPath, "geth/jwtsecret")
+
+	genesisPath, err := node.saveGenesis(ctx)
+	if err != nil {
+		return err
+	}
 	args := []string{
+		fmt.Sprintf("--%s=%s", genesis.StatePath.Name, genesisPath),
 		fmt.Sprintf("--%s=%s/eth2-beacon-node-%d", cmdshared.DataDirFlag.Name, e2e.TestParams.TestPath, index),
 		fmt.Sprintf("--%s=%s", cmdshared.LogFileName.Name, stdOutFile.Name()),
-		fmt.Sprintf("--%s=%s", flags.DepositContractFlag.Name, e2e.TestParams.ContractAddress.Hex()),
+		fmt.Sprintf("--%s=%s", flags.DepositContractFlag.Name, params.BeaconConfig().DepositContractAddress),
 		fmt.Sprintf("--%s=%d", flags.RPCPort.Name, e2e.TestParams.Ports.PrysmBeaconNodeRPCPort+index),
 		fmt.Sprintf("--%s=http://127.0.0.1:%d", flags.ExecutionEngineEndpoint.Name, e2e.TestParams.Ports.Eth1ProxyPort+index),
 		fmt.Sprintf("--%s=%s", flags.ExecutionJWTSecretFlag.Name, jwtPath),
@@ -290,3 +415,7 @@ func (node *BeaconNode) Resume() error {
 func (node *BeaconNode) Stop() error {
 	return node.cmd.Process.Kill()
 }
+
+func (node *BeaconNode) UnderlyingProcess() *os.Process {
+	return node.cmd.Process
+}

testing/endtoend/components/boot_node.go

@@ -9,6 +9,8 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/prysmaticlabs/prysm/v3/config/params"
+
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
 	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/helpers"
 	e2e "github.com/prysmaticlabs/prysm/v3/testing/endtoend/params"
@@ -54,6 +56,7 @@ func (node *BootNode) Start(ctx context.Context) error {
 		fmt.Sprintf("--log-file=%s", stdOutFile.Name()),
 		fmt.Sprintf("--discv5-port=%d", e2e.TestParams.Ports.BootNodePort),
 		fmt.Sprintf("--metrics-port=%d", e2e.TestParams.Ports.BootNodeMetricsPort),
+		fmt.Sprintf("--fork-version=%#x", params.BeaconConfig().BellatrixForkVersion),
 		"--debug",
 	}
 

testing/endtoend/components/eth1/BUILD.bazel

@@ -15,9 +15,9 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/v3/testing/endtoend/components/eth1",
     visibility = ["//testing/endtoend:__subpackages__"],
     deps = [
+        "//beacon-chain/execution/testing:go_default_library",
         "//config/params:go_default_library",
         "//contracts/deposit:go_default_library",
-        "//contracts/deposit/mock:go_default_library",
         "//crypto/rand:go_default_library",
         "//encoding/bytesutil:go_default_library",
         "//io/file:go_default_library",

testing/endtoend/components/eth1/depositor.go

@@ -4,9 +4,14 @@ import (
 	"context"
 	"fmt"
 	"math/big"
+	"os"
 	"sync"
 	"time"
 
+	log "github.com/sirupsen/logrus"
+
+	"github.com/ethereum/go-ethereum/common"
+
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	gethtypes "github.com/ethereum/go-ethereum/core/types"
@@ -142,6 +147,11 @@ type SentDeposit struct {
 // (using 2 transactions for partial deposits) and then uses WaitForBlocks (which spams the miner node with transactions
 // to and from its own address) to advance the chain until it has moved forward ETH1_FOLLOW_DISTANCE blocks.
 func (d *Depositor) SendAndMine(ctx context.Context, offset, nvals int, batch types.DepositBatch, partial bool) error {
+	balance, err := d.Client.BalanceAt(ctx, d.Key.Address, nil)
+	if err != nil {
+		return err
+	}
+	log.WithField("balance", balance.String()).WithField("account", d.Key.Address.Hex()).Info("SendAndMine balance check")
 	// This is the "Send" part of the function. Compute deposits for `nvals` validators,
 	// with half of those deposits being split over 2 transactions if the `partial` flag is true,
 	// and throwing away any validators before `offset`.
@@ -210,7 +220,8 @@ func (d *Depositor) txops(ctx context.Context) (*bind.TransactOpts, error) {
 // DepositContract is a special-purpose client for calling the deposit contract.
 func (d *Depositor) contractDepositor() (*contracts.DepositContract, error) {
 	if d.cd == nil {
-		contract, err := contracts.NewDepositContract(e2e.TestParams.ContractAddress, d.Client)
+		addr := common.HexToAddress(params.BeaconConfig().DepositContractAddress)
+		contract, err := contracts.NewDepositContract(addr, d.Client)
 		if err != nil {
 			return nil, err
 		}
@@ -218,3 +229,7 @@ func (d *Depositor) contractDepositor() (*contracts.DepositContract, error) {
 	}
 	return d.cd, nil
 }
+
+func (d *Depositor) UnderlyingProcess() *os.Process {
+	return nil // No subprocess for this component.
+}

testing/endtoend/components/eth1/helpers.go

@@ -47,6 +47,7 @@ func WaitForBlocks(web3 *ethclient.Client, key *keystore.Key, blocksToWait uint6
 	finishBlock := block.NumberU64() + blocksToWait
 
 	for block.NumberU64() <= finishBlock {
+		//log.Infof("waiting for block number %d, last saw %d", finishBlock, block.NumberU64())
 		spamTX := types.NewTransaction(nonce, key.Address, big.NewInt(0), params.SpamTxGasLimit, big.NewInt(1e6), []byte{})
 		signed, err := types.SignTx(spamTX, types.NewEIP155Signer(chainID), key.PrivateKey)
 		if err != nil {

testing/endtoend/components/eth1/miner.go

@@ -3,13 +3,14 @@ package eth1
 import (
 	"context"
 	"fmt"
-	"math/big"
 	"os"
 	"os/exec"
 	"path"
 	"strings"
 	"syscall"
-	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/prysmaticlabs/prysm/v3/beacon-chain/execution/testing"
 
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
@@ -17,8 +18,8 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v3/config/params"
-	contracts "github.com/prysmaticlabs/prysm/v3/contracts/deposit/mock"
-	io "github.com/prysmaticlabs/prysm/v3/io/file"
+	contracts "github.com/prysmaticlabs/prysm/v3/contracts/deposit"
+	"github.com/prysmaticlabs/prysm/v3/io/file"
 	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/helpers"
 	e2e "github.com/prysmaticlabs/prysm/v3/testing/endtoend/params"
 	e2etypes "github.com/prysmaticlabs/prysm/v3/testing/endtoend/types"
@@ -86,21 +87,24 @@ func (m *Miner) initAttempt(ctx context.Context, attempt int) (*os.File, error)
 		return nil, errors.New("go-ethereum binary not found")
 	}
 
-	staticGenesis, err := e2e.TestParams.Paths.Eth1Runfile("genesis.json")
+	gethJsonPath := path.Join(path.Dir(binaryPath), "genesis.json")
+	gen := testing.GethTestnetGenesis(e2e.TestParams.Eth1GenesisTime, params.BeaconConfig())
+	log.Infof("eth1 miner genesis timestamp=%d", e2e.TestParams.Eth1GenesisTime)
+	b, err := testing.TerribleMarshalHack(gen, params.BeaconConfig().DepositContractAddress, testing.DefaultMinerAddress)
 	if err != nil {
 		return nil, err
 	}
-	genesisPath := path.Join(path.Dir(binaryPath), "genesis.json")
-	if err := io.CopyFile(staticGenesis, genesisPath); err != nil {
-		return nil, errors.Wrapf(err, "error copying %s to %s", staticGenesis, genesisPath)
+	if err := file.WriteFile(gethJsonPath, b); err != nil {
+		return nil, err
 	}
 
-	initCmd := exec.CommandContext(
-		ctx,
-		binaryPath,
-		"init",
-		fmt.Sprintf("--datadir=%s", m.DataDir()),
-		genesisPath) // #nosec G204 -- Safe
+	// write the same thing to the logs dir for inspection
+	gethJsonLogPath := e2e.TestParams.Logfile("genesis.json")
+	if err := file.WriteFile(gethJsonLogPath, b); err != nil {
+		return nil, err
+	}
+
+	initCmd := exec.CommandContext(ctx, binaryPath, "init", fmt.Sprintf("--datadir=%s", m.DataDir()), gethJsonPath)
 
 	// redirect stderr to a log file
 	initFile, err := helpers.DeleteAndCreatePath(e2e.TestParams.Logfile("eth1-init_miner.log"))
@@ -139,6 +143,7 @@ func (m *Miner) initAttempt(ctx context.Context, attempt int) (*os.File, error)
 		"--ws.origins=\"*\"",
 		"--ipcdisable",
 		"--verbosity=4",
+		"--vmdebug",
 		"--mine",
 		fmt.Sprintf("--unlock=%s", EthAddress),
 		"--allow-insecure-unlock",
@@ -151,10 +156,10 @@ func (m *Miner) initAttempt(ctx context.Context, attempt int) (*os.File, error)
 	if err != nil {
 		return nil, err
 	}
-	if err = io.CopyFile(keystorePath, m.DataDir("keystore", minerFile)); err != nil {
+	if err = file.CopyFile(keystorePath, m.DataDir("keystore", minerFile)); err != nil {
 		return nil, errors.Wrapf(err, "error copying %s to %s", keystorePath, m.DataDir("keystore", minerFile))
 	}
-	err = io.WriteFile(pwFile, []byte(KeystorePassword))
+	err = file.WriteFile(pwFile, []byte(KeystorePassword))
 	if err != nil {
 		return nil, err
 	}
@@ -170,14 +175,16 @@ func (m *Miner) initAttempt(ctx context.Context, attempt int) (*os.File, error)
 	if err = runCmd.Start(); err != nil {
 		return nil, fmt.Errorf("failed to start eth1 chain: %w", err)
 	}
-	// check logs for common issues that prevent the EL miner from starting up.
-	if err = helpers.WaitForTextInFile(minerLog, "Commit new sealing work"); err != nil {
-		kerr := runCmd.Process.Kill()
-		if kerr != nil {
-			log.WithError(kerr).Error("error sending kill to failed miner command process")
+	/*
+		// check logs for common issues that prevent the EL miner from starting up.
+		if err = helpers.WaitForTextInFile(minerLog, "Commit new sealing work"); err != nil {
+			kerr := runCmd.Process.Kill()
+			if kerr != nil {
+				log.WithError(kerr).Error("error sending kill to failed miner command process")
+			}
+			return nil, fmt.Errorf("mining log not found, this means the eth1 chain had issues starting: %w", err)
 		}
-		return nil, fmt.Errorf("mining log not found, this means the eth1 chain had issues starting: %w", err)
-	}
+	*/
 	if err = helpers.WaitForTextInFile(minerLog, "Started P2P networking"); err != nil {
 		kerr := runCmd.Process.Kill()
 		if kerr != nil {
@@ -220,54 +227,33 @@ func (m *Miner) Start(ctx context.Context) error {
 		return fmt.Errorf("failed to connect to ipc: %w", err)
 	}
 	web3 := ethclient.NewClient(client)
-	keystorePath, err := e2e.TestParams.Paths.MinerKeyPath()
+	block, err := web3.BlockByNumber(ctx, nil)
 	if err != nil {
 		return err
 	}
-	// this is the key for the miner account. miner account balance is pre-mined in genesis.json.
-	key, err := helpers.KeyFromPath(keystorePath, KeystorePassword)
+	log.Infof("genesis block timestamp=%d", block.Time())
+	eth1BlockHash := block.Hash()
+	e2e.TestParams.Eth1GenesisBlock = block
+	log.Infof("miner says genesis block root=%#x", eth1BlockHash)
+	cAddr := common.HexToAddress(params.BeaconConfig().DepositContractAddress)
+	code, err := web3.CodeAt(ctx, cAddr, nil)
 	if err != nil {
 		return err
 	}
-	// Waiting for the blocks to advance by eth1follow to prevent issues reading the chain.
-	// Note that WaitForBlocks spams transfer transactions (to and from the miner's address) in order to advance.
-	if err = WaitForBlocks(web3, key, params.BeaconConfig().Eth1FollowDistance); err != nil {
-		return fmt.Errorf("unable to advance chain: %w", err)
-	}
-
-	// Time to deploy the contract using the miner's key.
-	txOpts, err := bind.NewKeyedTransactorWithChainID(key.PrivateKey, big.NewInt(NetworkId))
+	log.Infof("contract code size = %d", len(code))
+	depositContractCaller, err := contracts.NewDepositContractCaller(cAddr, web3)
 	if err != nil {
 		return err
 	}
-	nonce, err := web3.PendingNonceAt(ctx, key.Address)
+	dCount, err := depositContractCaller.GetDepositCount(&bind.CallOpts{})
 	if err != nil {
+		log.Error("failed to call get_deposit_count method of deposit contract")
 		return err
 	}
-	txOpts.Nonce = big.NewInt(0).SetUint64(nonce)
-	txOpts.Context = ctx
-	contractAddr, tx, _, err := contracts.DeployDepositContract(txOpts, web3)
-	if err != nil {
-		return fmt.Errorf("failed to deploy deposit contract: %w", err)
-	}
-	e2e.TestParams.ContractAddress = contractAddr
-
-	// Wait for contract to mine.
-	for pending := true; pending; _, pending, err = web3.TransactionByHash(ctx, tx.Hash()) {
-		if err != nil {
-			return err
-		}
-		time.Sleep(timeGapPerTX)
-	}
-
-	// Advancing the blocks another eth1follow distance to prevent issues reading the chain.
-	if err = WaitForBlocks(web3, key, params.BeaconConfig().Eth1FollowDistance); err != nil {
-		return fmt.Errorf("unable to advance chain: %w", err)
-	}
+	log.Infof("deposit contract count=%d", dCount)
 
 	// Mark node as ready.
 	close(m.started)
-
 	return m.cmd.Wait()
 }
 

testing/endtoend/components/eth1/node.go

@@ -10,6 +10,10 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/prysmaticlabs/prysm/v3/beacon-chain/execution/testing"
+	"github.com/prysmaticlabs/prysm/v3/config/params"
+	"github.com/prysmaticlabs/prysm/v3/io/file"
+
 	log "github.com/sirupsen/logrus"
 
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
@@ -53,12 +57,31 @@ func (node *Node) Start(ctx context.Context) error {
 		}
 	}
 
+	if err := file.MkdirAll(eth1Path); err != nil {
+		return err
+	}
+	gethJsonPath := path.Join(eth1Path, "genesis.json")
+
+	gen := testing.GethTestnetGenesis(e2e.TestParams.Eth1GenesisTime, params.BeaconConfig())
+	b, err := testing.TerribleMarshalHack(gen, params.BeaconConfig().DepositContractAddress, testing.DefaultMinerAddress)
+	if err != nil {
+		return err
+	}
+
+	if err := file.WriteFile(gethJsonPath, b); err != nil {
+		return err
+	}
+	copyPath := path.Join(e2e.TestParams.LogPath, "eth1-genesis.json")
+	if err := file.WriteFile(copyPath, b); err != nil {
+		return err
+	}
+
 	initCmd := exec.CommandContext(
 		ctx,
 		binaryPath,
 		"init",
 		fmt.Sprintf("--datadir=%s", eth1Path),
-		binaryPath[:strings.LastIndex(binaryPath, "/")]+"/genesis.json") // #nosec G204 -- Safe
+		gethJsonPath)
 	initFile, err := helpers.DeleteAndCreateFile(e2e.TestParams.LogPath, "eth1-init_"+strconv.Itoa(node.index)+".log")
 	if err != nil {
 		return err
@@ -92,6 +115,7 @@ func (node *Node) Start(ctx context.Context) error {
 		"--ws.origins=\"*\"",
 		"--ipcdisable",
 		"--verbosity=4",
+		"--vmdebug",
 		"--syncmode=full",
 		fmt.Sprintf("--txpool.locals=%s", EthAddress),
 	}
@@ -151,3 +175,7 @@ func (node *Node) Resume() error {
 func (node *Node) Stop() error {
 	return node.cmd.Process.Kill()
 }
+
+func (node *Node) UnderlyingProcess() *os.Process {
+	return node.cmd.Process
+}

testing/endtoend/components/lighthouse_beacon.go

@@ -261,7 +261,7 @@ func (node *LighthouseBeaconNode) createTestnetDir(index int) (string, error) {
 	configPath := filepath.Join(testNetDir, "config.yaml")
 	rawYaml := params.E2EMainnetConfigYaml()
 	// Add in deposit contract in yaml
-	depContractStr := fmt.Sprintf("\nDEPOSIT_CONTRACT_ADDRESS: %#x", e2e.TestParams.ContractAddress)
+	depContractStr := fmt.Sprintf("\nDEPOSIT_CONTRACT_ADDRESS: %#x", params.BeaconConfig().DepositContractAddress)
 	rawYaml = append(rawYaml, []byte(depContractStr)...)
 
 	if err := file.MkdirAll(testNetDir); err != nil {

testing/endtoend/components/lighthouse_validator.go

@@ -19,6 +19,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v3/runtime/interop"
 	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/helpers"
 	e2e "github.com/prysmaticlabs/prysm/v3/testing/endtoend/params"
+	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/types"
 	e2etypes "github.com/prysmaticlabs/prysm/v3/testing/endtoend/types"
 	"github.com/prysmaticlabs/prysm/v3/validator/keymanager"
 	keystorev4 "github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4"
@@ -247,6 +248,8 @@ func (v *LighthouseValidatorNode) Stop() error {
 	return v.cmd.Process.Kill()
 }
 
+var _ types.ComponentRunner = &KeystoreGenerator{}
+
 type KeystoreGenerator struct {
 	started chan struct{}
 }
@@ -361,3 +364,7 @@ func setupKeystores(valClientIdx, startIdx, numOfKeys int) (string, error) {
 	}
 	return testNetDir, nil
 }
+
+func (k *KeystoreGenerator) UnderlyingProcess() *os.Process {
+	return nil // No subprocess for this component.
+}

testing/endtoend/components/tracing_sink.go

@@ -14,8 +14,11 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/helpers"
 	e2e "github.com/prysmaticlabs/prysm/v3/testing/endtoend/params"
+	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/types"
 )
 
+var _ types.ComponentRunner = &TracingSink{}
+
 // TracingSink to capture HTTP requests from opentracing pushes. This is meant
 // to capture all opentracing spans from Prysm during an end-to-end test. Spans
 // are normally sent to a jaeger (https://www.jaegertracing.io/docs/1.25/getting-started/)
@@ -146,3 +149,7 @@ func captureRequest(f io.Writer, r *http.Request) error {
 	}
 	return nil
 }
+
+func (ts *TracingSink) UnderlyingProcess() *os.Process {
+	return nil // No subprocess for this component.
+}

testing/endtoend/components/validator.go

@@ -298,6 +298,10 @@ func (v *ValidatorNode) Stop() error {
 	return v.cmd.Process.Kill()
 }
 
+func (v *ValidatorNode) UnderlyingProcess() *os.Process {
+	return v.cmd.Process
+}
+
 func createProposerSettingsPath(pubkeys []string, validatorIndex int) (string, error) {
 	testNetDir := e2e.TestParams.TestPath + fmt.Sprintf("/proposer-settings/validator_%d", validatorIndex)
 	configPath := filepath.Join(testNetDir, "config.json")

testing/endtoend/components/web3remotesigner.go

@@ -261,7 +261,7 @@ func (w *Web3RemoteSigner) createTestnetDir() (string, error) {
 	configPath := filepath.Join(testNetDir, "config.yaml")
 	rawYaml := params.E2ETestConfigYaml()
 	// Add in deposit contract in yaml
-	depContractStr := fmt.Sprintf("\nDEPOSIT_CONTRACT_ADDRESS: %#x", e2e.TestParams.ContractAddress)
+	depContractStr := fmt.Sprintf("\nDEPOSIT_CONTRACT_ADDRESS: %#x", params.BeaconConfig().DepositContractAddress)
 	rawYaml = append(rawYaml, []byte(depContractStr)...)
 
 	if err := file.MkdirAll(testNetDir); err != nil {
@@ -273,3 +273,7 @@ func (w *Web3RemoteSigner) createTestnetDir() (string, error) {
 
 	return configPath, nil
 }
+
+func (w *Web3RemoteSigner) UnderlyingProcess() *os.Process {
+	return w.cmd.Process
+}

testing/endtoend/endtoend_test.go

@@ -85,7 +85,7 @@ func (r *testRunner) runBase(runEvents []runEvent) {
 			return errors.Wrap(err, "eth1Miner component never started - cannot send deposits")
 		}
 		// refactored send and mine goes here
-		minGenesisActiveCount := int(params.BeaconConfig().MinGenesisActiveValidatorCount)
+		//minGenesisActiveCount := int(params.BeaconConfig().MinGenesisActiveValidatorCount)
 		keyPath, err := e2e.TestParams.Paths.MinerKeyPath()
 		if err != nil {
 			return errors.Wrap(err, "error getting miner key file from bazel static files")
@@ -99,9 +99,11 @@ func (r *testRunner) runBase(runEvents []runEvent) {
 			return errors.Wrap(err, "failed to initialize a client to connect to the miner EL node")
 		}
 		r.depositor = &eth1.Depositor{Key: key, Client: client, NetworkId: big.NewInt(eth1.NetworkId)}
-		if err := r.depositor.SendAndMine(r.comHandler.ctx, 0, minGenesisActiveCount, e2etypes.GenesisDepositBatch, true); err != nil {
-			return errors.Wrap(err, "failed to send and mine deposits")
-		}
+		/*
+			if err := r.depositor.SendAndMine(r.comHandler.ctx, 0, minGenesisActiveCount, e2etypes.GenesisDepositBatch, true); err != nil {
+				return errors.Wrap(err, "failed to send and mine deposits")
+			}
+		*/
 		if err := r.depositor.Start(r.comHandler.ctx); err != nil {
 			return errors.Wrap(err, "depositor.Start failed")
 		}
@@ -448,6 +450,8 @@ func (r *testRunner) defaultEndToEndRun() error {
 		return errors.Wrap(err, "components take too long to start")
 	}
 
+	r.comHandler.printPIDs(t.Logf)
+
 	// Since defer unwraps in LIFO order, parent context will be closed only after logs are written.
 	defer helpers.LogOutput(t)
 	if config.UsePprof {
@@ -550,6 +554,8 @@ func (r *testRunner) scenarioRun() error {
 		return errors.Wrap(err, "components take too long to start")
 	}
 
+	r.comHandler.printPIDs(t.Logf)
+
 	// Since defer unwraps in LIFO order, parent context will be closed only after logs are written.
 	defer helpers.LogOutput(t)
 	if config.UsePprof {

testing/endtoend/evaluators/BUILD.bazel

@@ -40,6 +40,7 @@ go_library(
         "//proto/eth/v2:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//runtime/interop:go_default_library",
+        "//runtime/version:go_default_library",
         "//testing/endtoend/components:go_default_library",
         "//testing/endtoend/helpers:go_default_library",
         "//testing/endtoend/params:go_default_library",

testing/endtoend/evaluators/fork.go

@@ -4,8 +4,11 @@ import (
 	"context"
 	"time"
 
+	"github.com/prysmaticlabs/prysm/v3/runtime/version"
+
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v3/consensus-types/blocks"
+	ptypes "github.com/prysmaticlabs/prysm/v3/consensus-types/primitives"
 	ethpb "github.com/prysmaticlabs/prysm/v3/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/helpers"
 	"github.com/prysmaticlabs/prysm/v3/testing/endtoend/policies"
@@ -15,11 +18,19 @@ import (
 )
 
 var streamDeadline = 1 * time.Minute
+var startingFork = version.Bellatrix
 
 // AltairForkTransition ensures that the Altair hard fork has occurred successfully.
 var AltairForkTransition = types.Evaluator{
-	Name:       "altair_fork_transition_%d",
-	Policy:     policies.OnEpoch(helpers.AltairE2EForkEpoch),
+	Name: "altair_fork_transition_%d",
+	Policy: func(e ptypes.Epoch) bool {
+		altair := policies.OnEpoch(helpers.AltairE2EForkEpoch)
+		// TODO modify policies to take an end to end config
+		if startingFork == version.Phase0 {
+			return altair(e)
+		}
+		return false
+	},
 	Evaluation: altairForkOccurs,
 }
 

testing/endtoend/evaluators/operations.go

@@ -34,7 +34,7 @@ var churnLimit = 4
 var depositValCount = e2e.DepositCount
 
 // Deposits should be processed in twice the length of the epochs per eth1 voting period.
-var depositsInBlockStart = types.Epoch(math.Floor(float64(params.E2ETestConfig().EpochsPerEth1VotingPeriod) * 2))
+var depositsInBlockStart = params.E2ETestConfig().EpochsPerEth1VotingPeriod * 2
 
 // deposits included + finalization + MaxSeedLookahead for activation.
 var depositActivationStartEpoch = depositsInBlockStart + 2 + params.E2ETestConfig().MaxSeedLookahead
@@ -155,7 +155,11 @@ func verifyGraffitiInBlocks(_ e2etypes.EvaluationContext, conns ...*grpc.ClientC
 	if err != nil {
 		return errors.Wrap(err, "failed to get chain head")
 	}
-	req := &ethpb.ListBlocksRequest{QueryFilter: &ethpb.ListBlocksRequest_Epoch{Epoch: chainHead.HeadEpoch.Sub(1)}}
+	begin := chainHead.HeadEpoch
+	if begin > 0 {
+		begin = begin.Sub(1)
+	}
+	req := &ethpb.ListBlocksRequest{QueryFilter: &ethpb.ListBlocksRequest_Epoch{Epoch: begin}}
 	blks, err := client.ListBeaconBlocks(context.Background(), req)
 	if err != nil {
 		return errors.Wrap(err, "failed to get blocks from beacon-chain")
@@ -393,12 +397,17 @@ func validatorsVoteWithTheMajority(_ e2etypes.EvaluationContext, conns ...*grpc.
 		return errors.Wrap(err, "failed to get chain head")
 	}
 
-	req := &ethpb.ListBlocksRequest{QueryFilter: &ethpb.ListBlocksRequest_Epoch{Epoch: chainHead.HeadEpoch.Sub(1)}}
+	begin := chainHead.HeadEpoch
+	if begin > 0 {
+		begin = begin.Sub(1)
+	}
+	req := &ethpb.ListBlocksRequest{QueryFilter: &ethpb.ListBlocksRequest_Epoch{Epoch: begin}}
 	blks, err := client.ListBeaconBlocks(context.Background(), req)
 	if err != nil {
 		return errors.Wrap(err, "failed to get blocks from beacon-chain")
 	}
 
+	slotsPerVotingPeriod := params.E2ETestConfig().SlotsPerEpoch.Mul(uint64(params.E2ETestConfig().EpochsPerEth1VotingPeriod))
 	for _, blk := range blks.BlockContainers {
 		var slot types.Slot
 		var vote []byte
@@ -407,22 +416,25 @@ func validatorsVoteWithTheMajority(_ e2etypes.EvaluationContext, conns ...*grpc.
 			b := blk.GetPhase0Block().Block
 			slot = b.Slot
 			vote = b.Body.Eth1Data.BlockHash
+			log.WithField("slot", slot).WithField("vote", fmt.Sprintf("%#x", vote)).Warn("phase0 block vote")
 		case *ethpb.BeaconBlockContainer_AltairBlock:
 			b := blk.GetAltairBlock().Block
 			slot = b.Slot
 			vote = b.Body.Eth1Data.BlockHash
+			log.WithField("slot", slot).WithField("vote", fmt.Sprintf("%#x", vote)).Warn("altair block vote")
 		case *ethpb.BeaconBlockContainer_BellatrixBlock:
 			b := blk.GetBellatrixBlock().Block
 			slot = b.Slot
 			vote = b.Body.Eth1Data.BlockHash
+			log.WithField("slot", slot).WithField("vote", fmt.Sprintf("%#x", vote)).Warn("bellatrix block vote")
 		case *ethpb.BeaconBlockContainer_BlindedBellatrixBlock:
 			b := blk.GetBlindedBellatrixBlock().Block
 			slot = b.Slot
 			vote = b.Body.Eth1Data.BlockHash
+			log.WithField("slot", slot).WithField("vote", fmt.Sprintf("%#x", vote)).Warn("blinded bellatrix block vote")
 		default:
 			return errors.New("block neither phase0,altair or bellatrix")
 		}
-		slotsPerVotingPeriod := params.E2ETestConfig().SlotsPerEpoch.Mul(uint64(params.E2ETestConfig().EpochsPerEth1VotingPeriod))
 
 		// We treat epoch 1 differently from other epoch for two reasons:
 		// - this evaluator is not executed for epoch 0 so we have to calculate the first slot differently

testing/endtoend/params/BUILD.bazel

@@ -8,10 +8,10 @@ go_library(
         "params.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/v3/testing/endtoend/params",
-    visibility = ["//testing/endtoend:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [
         "//io/file:go_default_library",
-        "@com_github_ethereum_go_ethereum//common:go_default_library",
+        "@com_github_ethereum_go_ethereum//core/types:go_default_library",
         "@io_bazel_rules_go//go/tools/bazel:go_default_library",
     ],
 )

testing/endtoend/params/params.go

@@ -12,9 +12,11 @@ import (
 	"path/filepath"
 	"strconv"
 	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/core/types"
 
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/prysmaticlabs/prysm/v3/io/file"
 )
 
@@ -25,9 +27,12 @@ type params struct {
 	TestShardIndex            int
 	BeaconNodeCount           int
 	LighthouseBeaconNodeCount int
-	ContractAddress           common.Address
 	Ports                     *ports
 	Paths                     *paths
+	Eth1GenesisBlock          *types.Block
+	StartTime                 time.Time
+	CLGenesisTime             uint64
+	Eth1GenesisTime           uint64
 }
 
 type ports struct {
@@ -148,19 +153,33 @@ const (
 	ValidatorMetricsPort = ValidatorGatewayPort + portSpan
 
 	JaegerTracingPort = 9150
+
+	StartupBufferSecs = 5
 )
 
+func logDir() string {
+	wTime := func(p string) string {
+		return path.Join(p, time.Now().Format("20060102/150405"))
+	}
+	path, ok := os.LookupEnv("E2E_LOG_PATH")
+	if ok {
+		return wTime(path)
+	}
+	path, _ = os.LookupEnv("TEST_UNDECLARED_OUTPUTS_DIR")
+	return wTime(path)
+}
+
 // Init initializes the E2E config, properly handling test sharding.
 func Init(t *testing.T, beaconNodeCount int) error {
-	testPath := bazel.TestTmpDir()
-	logPath, ok := os.LookupEnv("TEST_UNDECLARED_OUTPUTS_DIR")
-	if !ok {
-		return errors.New("expected TEST_UNDECLARED_OUTPUTS_DIR to be defined")
+	d := logDir()
+	if d == "" {
+		return errors.New("unable to determine log directory, no value for E2E_LOG_PATH or TEST_UNDECLARED_OUTPUTS_DIR")
 	}
-	logPath = path.Join(logPath, t.Name())
+	logPath := path.Join(d, t.Name())
 	if err := file.MkdirAll(logPath); err != nil {
 		return err
 	}
+	testPath := bazel.TestTmpDir()
 	testTotalShardsStr, ok := os.LookupEnv("TEST_TOTAL_SHARDS")
 	if !ok {
 		testTotalShardsStr = "1"
@@ -185,12 +204,22 @@ func Init(t *testing.T, beaconNodeCount int) error {
 		return err
 	}
 
+	//cfg := cfgparams.BeaconConfig()
+	now := time.Now()
+	clGenTime := uint64(now.Unix()) + StartupBufferSecs
+	//epochSecs := cfg.SecondsPerSlot * uint64(cfg.SlotsPerEpoch)
+	// TODO: support starting from any fork, make the genesis offset variable
+	//forkOffset := epochSecs * uint64(cfg.CapellaForkEpoch)
+
 	TestParams = &params{
 		TestPath:        filepath.Join(testPath, fmt.Sprintf("shard-%d", testShardIndex)),
 		LogPath:         logPath,
 		TestShardIndex:  testShardIndex,
 		BeaconNodeCount: beaconNodeCount,
 		Ports:           testPorts,
+		StartTime:       now,
+		CLGenesisTime:   clGenTime,
+		Eth1GenesisTime: clGenTime,
 	}
 	return nil
 }
@@ -234,6 +263,12 @@ func InitMultiClient(t *testing.T, beaconNodeCount int, lighthouseNodeCount int)
 		return err
 	}
 
+	//cfg := cfgparams.BeaconConfig()
+	now := time.Now()
+	clGenTime := uint64(now.Unix()) + StartupBufferSecs
+	//epochSecs := cfg.SecondsPerSlot * uint64(cfg.SlotsPerEpoch)
+	// TODO: support starting from any fork, make the genesis offset variable
+	//forkOffset := epochSecs * uint64(cfg.CapellaForkEpoch)
 	TestParams = &params{
 		TestPath:                  filepath.Join(testPath, fmt.Sprintf("shard-%d", testShardIndex)),
 		LogPath:                   logPath,
@@ -241,6 +276,9 @@ func InitMultiClient(t *testing.T, beaconNodeCount int, lighthouseNodeCount int)
 		BeaconNodeCount:           beaconNodeCount,
 		LighthouseBeaconNodeCount: lighthouseNodeCount,
 		Ports:                     testPorts,
+		StartTime:                 now,
+		CLGenesisTime:             clGenTime,
+		Eth1GenesisTime:           clGenTime,
 	}
 	return nil
 }

testing/endtoend/static-files/eth1/genesis.json

@@ -11,8 +11,8 @@
     "istanbulBlock": 0,
     "berlinBlock": 0,
     "londonBlock": 0,
-    "mergeForkBlock": 308,
-    "terminalTotalDifficulty": 616,
+    "mergeNetsplitBlock": 308,
+    "terminalTotalDifficulty": 0,
     "clique": {
       "period": 2,
       "epoch": 30000
@@ -29,4 +29,4 @@
   "mixhash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
   "parentHash" : "0x0000000000000000000000000000000000000000000000000000000000000000",
   "timestamp" : "0x00"
-}
+}

testing/endtoend/types/types.go

@@ -4,6 +4,7 @@ package types
 
 import (
 	"context"
+	"os"
 
 	types "github.com/prysmaticlabs/prysm/v3/consensus-types/primitives"
 	"google.golang.org/grpc"
@@ -95,6 +96,8 @@ type ComponentRunner interface {
 	Resume() error
 	// Stop stops a component.
 	Stop() error
+	// UnderlyingProcess is the underlying process, once started.
+	UnderlyingProcess() *os.Process
 }
 
 type MultipleComponentRunners interface {

testing/util/bellatrix_state.go

@@ -134,6 +134,12 @@ func buildGenesisBeaconStateBellatrix(genesisTime uint64, preState state.BeaconS
 	if err != nil {
 		return nil, err
 	}
+	scoresMissing := len(preState.Validators()) - len(scores)
+	if scoresMissing > 0 {
+		for i := 0; i < scoresMissing; i++ {
+			scores = append(scores, 0)
+		}
+	}
 	st := &ethpb.BeaconStateBellatrix{
 		// Misc fields.
 		Slot:                  0,
@@ -240,5 +246,16 @@ func buildGenesisBeaconStateBellatrix(genesisTime uint64, preState state.BeaconS
 		TransactionsRoot: make([]byte, 32),
 	}
 
-	return state_native.InitializeFromProtoBellatrix(st)
+	bs, err := state_native.InitializeFromProtoBellatrix(st)
+	if err != nil {
+		return nil, err
+	}
+	is, err := bs.InactivityScores()
+	if err != nil {
+		return nil, err
+	}
+	if bs.NumValidators() != len(is) {
+		panic("inactivity score mismatch with num vals")
+	}
+	return bs, nil
 }

tools/bootnode/bootnode.go

@@ -193,8 +193,12 @@ func createLocalNode(privKey *ecdsa.PrivateKey, ipAddr net.IP, port int) (*enode
 		external = ipAddr
 	}
 	fVersion := params.BeaconConfig().GenesisForkVersion
-	if *forkVersion != "" {
-		fVersion, err = hex.DecodeString(*forkVersion)
+	fvs := *forkVersion
+	if fvs != "" {
+		if len(fvs) == 10 {
+			fvs = fvs[2:]
+		}
+		fVersion, err = hex.DecodeString(fvs)
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not retrieve fork version")
 		}