comment out committe caching for subsequent epoch

api/client/beacon/checkpoint.go

@@ -40,15 +40,15 @@ func (od *OriginData) CheckpointString() string {
 // SaveBlock saves the downloaded block to a unique file in the given path.
 // For readability and collision avoidance, the file name includes: type, config name, slot and root
 func (od *OriginData) SaveBlock(dir string) (string, error) {
-	blockPath := path.Join(dir, fname("state", od.cf, od.st.Slot(), od.wsd.BlockRoot))
-	return blockPath, file.WriteFile(blockPath, od.sb)
+	blockPath := path.Join(dir, fname("block", od.cf, od.b.Block().Slot(), od.wsd.BlockRoot))
+	return blockPath, file.WriteFile(blockPath, od.BlockBytes())
 }
 
 // SaveState saves the downloaded state to a unique file in the given path.
 // For readability and collision avoidance, the file name includes: type, config name, slot and root
 func (od *OriginData) SaveState(dir string) (string, error) {
 	statePath := path.Join(dir, fname("state", od.cf, od.st.Slot(), od.wsd.StateRoot))
-	return statePath, file.WriteFile(statePath, od.sb)
+	return statePath, file.WriteFile(statePath, od.StateBytes())
 }
 
 // StateBytes returns the ssz-encoded bytes of the downloaded BeaconState value.

beacon-chain/blockchain/optimistic_sync.go

@@ -46,31 +46,15 @@ func (s *Service) notifyForkchoiceUpdate(ctx context.Context, headState state.Be
 	if err != nil {
 		return nil, errors.Wrap(err, "could not get execution payload")
 	}
-	finalizedBlock, err := s.cfg.BeaconDB.Block(ctx, s.ensureRootNotZeros(finalizedRoot))
+	finalizedHash, err := s.getFinalizedPayloadHash(ctx, finalizedRoot)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not get finalized block")
-	}
-	if finalizedBlock == nil || finalizedBlock.IsNil() {
-		finalizedBlock = s.getInitSyncBlock(s.ensureRootNotZeros(finalizedRoot))
-		if finalizedBlock == nil || finalizedBlock.IsNil() {
-			return nil, errors.Errorf("finalized block with root %#x does not exist in the db or our cache", s.ensureRootNotZeros(finalizedRoot))
-		}
-	}
-	var finalizedHash []byte
-	if blocks.IsPreBellatrixVersion(finalizedBlock.Block().Version()) {
-		finalizedHash = params.BeaconConfig().ZeroHash[:]
-	} else {
-		payload, err := finalizedBlock.Block().Body().ExecutionPayload()
-		if err != nil {
-			return nil, errors.Wrap(err, "could not get finalized block execution payload")
-		}
-		finalizedHash = payload.BlockHash
+		return nil, errors.Wrap(err, "could not get finalized payload hash")
 	}
 
 	fcs := &enginev1.ForkchoiceState{
 		HeadBlockHash:      headPayload.BlockHash,
 		SafeBlockHash:      headPayload.BlockHash,
-		FinalizedBlockHash: finalizedHash,
+		FinalizedBlockHash: finalizedHash[:],
 	}
 
 	nextSlot := s.CurrentSlot() + 1 // Cache payload ID for next slot proposer.
@@ -84,9 +68,9 @@ func (s *Service) notifyForkchoiceUpdate(ctx context.Context, headState state.Be
 		switch err {
 		case powchain.ErrAcceptedSyncingPayloadStatus:
 			log.WithFields(logrus.Fields{
-				"headSlot":      headBlk.Slot(),
-				"headHash":      fmt.Sprintf("%#x", bytesutil.Trunc(headPayload.BlockHash)),
-				"finalizedHash": fmt.Sprintf("%#x", bytesutil.Trunc(finalizedHash)),
+				"headSlot":                  headBlk.Slot(),
+				"headPayloadBlockHash":      fmt.Sprintf("%#x", bytesutil.Trunc(headPayload.BlockHash)),
+				"finalizedPayloadBlockHash": fmt.Sprintf("%#x", bytesutil.Trunc(finalizedHash[:])),
 			}).Info("Called fork choice updated with optimistic block")
 			return payloadID, nil
 		default:
@@ -104,6 +88,55 @@ func (s *Service) notifyForkchoiceUpdate(ctx context.Context, headState state.Be
 	return payloadID, nil
 }
 
+// getFinalizedPayloadHash returns the finalized payload hash for the given finalized block root.
+// It checks the following in order:
+//   1. The finalized block exists in db
+//   2. The finalized block exists in initial sync block cache
+//   3. The finalized block is the weak subjectivity block and exists in db
+// Error is returned if the finalized block is not found from above.
+func (s *Service) getFinalizedPayloadHash(ctx context.Context, finalizedRoot [32]byte) ([32]byte, error) {
+	b, err := s.cfg.BeaconDB.Block(ctx, s.ensureRootNotZeros(finalizedRoot))
+	if err != nil {
+		return [32]byte{}, errors.Wrap(err, "could not get finalized block")
+	}
+	if b != nil {
+		return getPayloadHash(b.Block())
+	}
+
+	b = s.getInitSyncBlock(finalizedRoot)
+	if b != nil {
+		return getPayloadHash(b.Block())
+	}
+
+	r, err := s.cfg.BeaconDB.OriginCheckpointBlockRoot(ctx)
+	if err != nil {
+		return [32]byte{}, errors.Wrap(err, "could not get finalized block")
+	}
+	b, err = s.cfg.BeaconDB.Block(ctx, r)
+	if err != nil {
+		return [32]byte{}, errors.Wrap(err, "could not get finalized block")
+	}
+	if b != nil {
+		return getPayloadHash(b.Block())
+	}
+
+	return [32]byte{}, errors.Errorf("finalized block with root %#x does not exist in the db or our cache", s.ensureRootNotZeros(finalizedRoot))
+}
+
+// getPayloadHash returns the payload hash for the input given block.
+// zeros are returned if the block is older than bellatrix.
+func getPayloadHash(b block.BeaconBlock) ([32]byte, error) {
+	if blocks.IsPreBellatrixVersion(b.Version()) {
+		return params.BeaconConfig().ZeroHash, nil
+	}
+
+	payload, err := b.Body().ExecutionPayload()
+	if err != nil {
+		return [32]byte{}, errors.Wrap(err, "could not get finalized block execution payload")
+	}
+	return bytesutil.ToBytes32(payload.BlockHash), nil
+}
+
 // notifyForkchoiceUpdate signals execution engine on a new payload.
 // It returns true if the EL has returned VALID for the block
 func (s *Service) notifyNewPayload(ctx context.Context, preStateVersion, postStateVersion int,

beacon-chain/blockchain/optimistic_sync_test.go

@@ -792,3 +792,74 @@ func TestService_removeInvalidBlockAndState(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, false, has)
 }
+
+func TestService_getFinalizedPayloadHash(t *testing.T) {
+	ctx := context.Background()
+	beaconDB := testDB.SetupDB(t)
+	opts := []Option{
+		WithDatabase(beaconDB),
+		WithStateGen(stategen.New(beaconDB)),
+	}
+	service, err := NewService(ctx, opts...)
+	require.NoError(t, err)
+
+	// Use the block in DB
+	b := util.NewBeaconBlockBellatrix()
+	b.Block.Body.ExecutionPayload.BlockHash = bytesutil.PadTo([]byte("hi"), 32)
+	blk, err := wrapper.WrappedSignedBeaconBlock(b)
+	require.NoError(t, err)
+	r, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, service.cfg.BeaconDB.SaveBlock(ctx, blk))
+	h, err := service.getFinalizedPayloadHash(ctx, r)
+	require.NoError(t, err)
+	require.Equal(t, bytesutil.ToBytes32(b.Block.Body.ExecutionPayload.BlockHash), h)
+
+	// Use the block in init sync cache
+	b = util.NewBeaconBlockBellatrix()
+	b.Block.Body.ExecutionPayload.BlockHash = bytesutil.PadTo([]byte("hello"), 32)
+	blk, err = wrapper.WrappedSignedBeaconBlock(b)
+	require.NoError(t, err)
+	r, err = b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	service.initSyncBlocks[r] = blk
+	h, err = service.getFinalizedPayloadHash(ctx, r)
+	require.NoError(t, err)
+	require.Equal(t, bytesutil.ToBytes32(b.Block.Body.ExecutionPayload.BlockHash), h)
+
+	// Use the weak subjectivity sync block
+	b = util.NewBeaconBlockBellatrix()
+	b.Block.Body.ExecutionPayload.BlockHash = bytesutil.PadTo([]byte("howdy"), 32)
+	blk, err = wrapper.WrappedSignedBeaconBlock(b)
+	require.NoError(t, err)
+	r, err = b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, service.cfg.BeaconDB.SaveBlock(ctx, blk))
+	require.NoError(t, service.cfg.BeaconDB.SaveOriginCheckpointBlockRoot(ctx, r))
+	h, err = service.getFinalizedPayloadHash(ctx, r)
+	require.NoError(t, err)
+	require.Equal(t, bytesutil.ToBytes32(b.Block.Body.ExecutionPayload.BlockHash), h)
+
+	// None of the above should error
+	require.NoError(t, service.cfg.BeaconDB.SaveOriginCheckpointBlockRoot(ctx, [32]byte{'a'}))
+	_, err = service.getFinalizedPayloadHash(ctx, [32]byte{'a'})
+	require.ErrorContains(t, "does not exist in the db or our cache", err)
+}
+
+func TestService_getPayloadHash(t *testing.T) {
+	// Pre-bellatrix
+	blk, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlock())
+	require.NoError(t, err)
+	h, err := getPayloadHash(blk.Block())
+	require.NoError(t, err)
+	require.Equal(t, [32]byte{}, h)
+
+	// Post bellatrix
+	b := util.NewBeaconBlockBellatrix()
+	b.Block.Body.ExecutionPayload.BlockHash = bytesutil.PadTo([]byte("hi"), 32)
+	blk, err = wrapper.WrappedSignedBeaconBlock(b)
+	require.NoError(t, err)
+	h, err = getPayloadHash(blk.Block())
+	require.NoError(t, err)
+	require.Equal(t, bytesutil.ToBytes32(bytesutil.PadTo([]byte("hi"), 32)), h)
+}

beacon-chain/blockchain/process_block.go

@@ -506,6 +506,7 @@ func (s *Service) handleEpochBoundary(ctx context.Context, postState state.Beaco
 
 	if postState.Slot()+1 == s.nextEpochBoundarySlot {
 		// Update caches for the next epoch at epoch boundary slot - 1.
+		log.Infof("UpdateCommitteeCache from handleEpochBoundary (postState.Slot()+1 == s.nextEpochBoundarySlot), slot=%d, epoch=%d", postState.Slot(), coreTime.CurrentEpoch(postState))
 		if err := helpers.UpdateCommitteeCache(postState, coreTime.NextEpoch(postState)); err != nil {
 			return err
 		}
@@ -514,6 +515,7 @@ func (s *Service) handleEpochBoundary(ctx context.Context, postState state.Beaco
 		if err != nil {
 			return err
 		}
+		log.Info("UpdateProposerIndicesInCache from handleEpochBoundary (postState.Slot()+1 == s.nextEpochBoundarySlot)")
 		if err := helpers.UpdateProposerIndicesInCache(ctx, copied); err != nil {
 			return err
 		}
@@ -529,9 +531,12 @@ func (s *Service) handleEpochBoundary(ctx context.Context, postState state.Beaco
 
 		// Update caches at epoch boundary slot.
 		// The following updates have short cut to return nil cheaply if fulfilled during boundary slot - 1.
+		log.Info("UpdateCommitteeCache from handleEpochBoundary (postState.Slot() >= s.nextEpochBoundarySlot)")
 		if err := helpers.UpdateCommitteeCache(postState, coreTime.CurrentEpoch(postState)); err != nil {
 			return err
 		}
+
+		log.Info("UpdateProposerIndicesInCache from handleEpochBoundary (postState.Slot() >= s.nextEpochBoundarySlot)")
 		if err := helpers.UpdateProposerIndicesInCache(ctx, postState); err != nil {
 			return err
 		}

beacon-chain/blockchain/service.go

@@ -440,9 +440,11 @@ func (s *Service) initializeBeaconChain(
 	s.cfg.ChainStartFetcher.ClearPreGenesisData()
 
 	// Update committee shuffled indices for genesis epoch.
+	log.Infof("UpdateCommitteeCache from initializeBeaconChain, slot=%d", genesisState.Slot())
 	if err := helpers.UpdateCommitteeCache(genesisState, 0 /* genesis epoch */); err != nil {
 		return nil, err
 	}
+	log.Info("UpdateProposerIndicesInCache from initializeBeaconChain")
 	if err := helpers.UpdateProposerIndicesInCache(ctx, genesisState); err != nil {
 		return nil, err
 	}

beacon-chain/core/helpers/beacon_committee.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	log "github.com/sirupsen/logrus"
 	"sort"
 
 	"github.com/pkg/errors"
@@ -286,40 +287,43 @@ func ShuffledIndices(s state.ReadOnlyBeaconState, epoch types.Epoch) ([]types.Va
 // UpdateCommitteeCache gets called at the beginning of every epoch to cache the committee shuffled indices
 // list with committee index and epoch number. It caches the shuffled indices for current epoch and next epoch.
 func UpdateCommitteeCache(state state.ReadOnlyBeaconState, epoch types.Epoch) error {
-	for _, e := range []types.Epoch{epoch, epoch + 1} {
-		seed, err := Seed(state, e, params.BeaconConfig().DomainBeaconAttester)
-		if err != nil {
-			return err
-		}
-		if committeeCache.HasEntry(string(seed[:])) {
-			return nil
-		}
-
-		shuffledIndices, err := ShuffledIndices(state, e)
-		if err != nil {
-			return err
-		}
-
-		count := SlotCommitteeCount(uint64(len(shuffledIndices)))
-
-		// Store the sorted indices as well as shuffled indices. In current spec,
-		// sorted indices is required to retrieve proposer index. This is also
-		// used for failing verify signature fallback.
-		sortedIndices := make([]types.ValidatorIndex, len(shuffledIndices))
-		copy(sortedIndices, shuffledIndices)
-		sort.Slice(sortedIndices, func(i, j int) bool {
-			return sortedIndices[i] < sortedIndices[j]
-		})
-
-		if err := committeeCache.AddCommitteeShuffledList(&cache.Committees{
-			ShuffledIndices: shuffledIndices,
-			CommitteeCount:  uint64(params.BeaconConfig().SlotsPerEpoch.Mul(count)),
-			Seed:            seed,
-			SortedIndices:   sortedIndices,
-		}); err != nil {
-			return err
-		}
+	//for _, e := range []types.Epoch{epoch, epoch + 1} {
+	seed, err := Seed(state, epoch, params.BeaconConfig().DomainBeaconAttester)
+	if err != nil {
+		return err
 	}
+	log.Infof("computed seed=%#x for slot=%d", seed, state.Slot())
+	if committeeCache.HasEntry(string(seed[:])) {
+		log.Infof("UpdateCommitteeCache: seed=%#x already in cache at slot=%d", seed, state.Slot())
+		return nil
+	}
+
+	shuffledIndices, err := ShuffledIndices(state, epoch)
+	if err != nil {
+		return err
+	}
+
+	count := SlotCommitteeCount(uint64(len(shuffledIndices)))
+
+	// Store the sorted indices as well as shuffled indices. In current spec,
+	// sorted indices is required to retrieve proposer index. This is also
+	// used for failing verify signature fallback.
+	sortedIndices := make([]types.ValidatorIndex, len(shuffledIndices))
+	copy(sortedIndices, shuffledIndices)
+	sort.Slice(sortedIndices, func(i, j int) bool {
+		return sortedIndices[i] < sortedIndices[j]
+	})
+
+	log.Infof("UpdateCommitteeCache: epoch=%d, state.slot=%d, indices=%v, seed=%#x", epoch, state.Slot(), sortedIndices, seed)
+	if err := committeeCache.AddCommitteeShuffledList(&cache.Committees{
+		ShuffledIndices: shuffledIndices,
+		CommitteeCount:  uint64(params.BeaconConfig().SlotsPerEpoch.Mul(count)),
+		Seed:            seed,
+		SortedIndices:   sortedIndices,
+	}); err != nil {
+		return err
+	}
+	//}
 
 	return nil
 }
@@ -363,6 +367,7 @@ func UpdateProposerIndicesInCache(ctx context.Context, state state.ReadOnlyBeaco
 	if err != nil {
 		return err
 	}
+	log.Infof("UpdateProposerIndicesInCache: state.slot=%d, slot=%d, root=%#x, indices=%v", state.Slot(), s, r, indices)
 	return proposerIndicesCache.AddProposerIndices(&cache.ProposerIndices{
 		BlockRoot:       bytesutil.ToBytes32(r),
 		ProposerIndices: proposerIndices,

beacon-chain/core/helpers/randao.go

@@ -7,6 +7,7 @@ import (
 	"github.com/prysmaticlabs/prysm/crypto/bls"
 	"github.com/prysmaticlabs/prysm/crypto/hash"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	log "github.com/sirupsen/logrus"
 )
 
 // Seed returns the randao seed used for shuffling of a given epoch.
@@ -33,6 +34,8 @@ func Seed(state state.ReadOnlyBeaconState, epoch types.Epoch, domain [bls.Domain
 
 	seed32 := hash.Hash(seed)
 
+	log.Infof("seed computation params for slot=%d: domain=%#x, epoch=%d, randaoMix=%#x", state.Slot(), domain, epoch, randaoMix)
+
 	return seed32, nil
 }
 

beacon-chain/core/helpers/validators.go

@@ -3,6 +3,8 @@ package helpers
 import (
 	"bytes"
 	"context"
+	"fmt"
+	"github.com/prysmaticlabs/prysm/time/slots"
 
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
@@ -15,7 +17,6 @@ import (
 	"github.com/prysmaticlabs/prysm/crypto/hash"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
-	"github.com/prysmaticlabs/prysm/time/slots"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -88,11 +89,27 @@ func ActiveValidatorIndices(ctx context.Context, s state.ReadOnlyBeaconState, ep
 	if err != nil {
 		return nil, errors.Wrap(err, "could not get seed")
 	}
+	var ci []types.ValidatorIndex
+	if s.Slot() == 78 {
+		if err := s.ReadFromEveryValidator(func(idx int, val state.ReadOnlyValidator) error {
+			if IsActiveValidatorUsingTrie(val, epoch) {
+				ci = append(ci, types.ValidatorIndex(idx))
+			}
+			return nil
+		}); err != nil {
+			log.Errorf("got error doing double-check validator index computation=%v", err)
+			return nil, err
+		}
+	}
 	activeIndices, err := committeeCache.ActiveIndices(ctx, seed)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not interface with committee cache")
 	}
 	if activeIndices != nil {
+		if s.Slot() == 78 {
+			log.Infof("double check indices for 78, len=%d, low=%d, high=%d", len(ci), ci[0], ci[len(ci)-1])
+		}
+		log.Infof("found indices in cache for slot=%d, len=%d, low=%d, high=%d", s.Slot(), len(activeIndices), activeIndices[0], activeIndices[len(activeIndices)-1])
 		return activeIndices, nil
 	}
 
@@ -106,6 +123,7 @@ func ActiveValidatorIndices(ctx context.Context, s state.ReadOnlyBeaconState, ep
 				return nil, errors.New("nil active indices")
 			}
 			CommitteeCacheInProgressHit.Inc()
+			log.Infof("found indices in in-progress cache for slot=%d, len=%d, low=%d, high=%d", s.Slot(), len(activeIndices), activeIndices[0], activeIndices[len(activeIndices)-1])
 			return activeIndices, nil
 		}
 		return nil, errors.Wrap(err, "could not mark committee cache as in progress")
@@ -126,9 +144,16 @@ func ActiveValidatorIndices(ctx context.Context, s state.ReadOnlyBeaconState, ep
 		return nil, err
 	}
 
+	log.Infof("computed indices slot=%d, len=%d, low=%d, high=%d", s.Slot(), len(indices), indices[0], indices[len(indices)-1])
+	log.Infof("UpdateCommitteeCache from ActiveValidatorIndices, slot=%d", s.Slot())
 	if err := UpdateCommitteeCache(s, epoch); err != nil {
 		return nil, errors.Wrap(err, "could not update committee cache")
 	}
+	/*
+	if err := UpdateProposerIndicesInCache(ctx, s); err != nil {
+		return nil, errors.Wrap(err, "failed to update proposer indices cache in ActiveValidatorIndices")
+	}
+	 */
 
 	return indices, nil
 }
@@ -175,6 +200,7 @@ func ActiveValidatorCount(ctx context.Context, s state.ReadOnlyBeaconState, epoc
 		return 0, err
 	}
 
+	log.Infof("UpdateCommitteeCache from ActiveValidatorCount, slot=%d", s.Slot())
 	if err := UpdateCommitteeCache(s, epoch); err != nil {
 		return 0, errors.Wrap(err, "could not update committee cache")
 	}
@@ -249,6 +275,7 @@ func BeaconProposerIndex(ctx context.Context, state state.ReadOnlyBeaconState) (
 				}
 				return proposerIndices[state.Slot()%params.BeaconConfig().SlotsPerEpoch], nil
 			}
+			log.Info("UpdateProposerIndicesInCache from BeaconProposerIndex")
 			if err := UpdateProposerIndicesInCache(ctx, state); err != nil {
 				return 0, errors.Wrap(err, "could not update committee cache")
 			}
@@ -259,15 +286,19 @@ func BeaconProposerIndex(ctx context.Context, state state.ReadOnlyBeaconState) (
 	if err != nil {
 		return 0, errors.Wrap(err, "could not generate seed")
 	}
+	fmt.Printf("BeaconProposerIndex:seed=%#x", seed)
 
 	seedWithSlot := append(seed[:], bytesutil.Bytes8(uint64(state.Slot()))...)
+	fmt.Printf("BeaconProposerIndex:seedWithSlot=%#x", seed)
 	seedWithSlotHash := hash.Hash(seedWithSlot)
+	fmt.Printf("BeaconProposerIndex:seedWithSlotHash=%#x", seed)
 
 	indices, err := ActiveValidatorIndices(ctx, state, e)
 	if err != nil {
 		return 0, errors.Wrap(err, "could not get active indices")
 	}
 
+	log.Infof("validator index length=%d, low=%d, high=%d", len(indices), indices[0], indices[len(indices)-1])
 	return ComputeProposerIndex(state, indices, seedWithSlotHash)
 }
 

beacon-chain/core/transition/transition.go

@@ -138,6 +138,7 @@ func ProcessSlotsUsingNextSlotCache(
 	ctx, span := trace.StartSpan(ctx, "core.state.ProcessSlotsUsingNextSlotCache")
 	defer span.End()
 
+	/*
 	// Check whether the parent state has been advanced by 1 slot in next slot cache.
 	nextSlotState, err := NextSlotState(ctx, parentRoot)
 	if err != nil {
@@ -148,6 +149,11 @@ func ProcessSlotsUsingNextSlotCache(
 	// We replace next slot state with parent state.
 	if cachedStateExists {
 		parentState = nextSlotState
+		root, err := parentState.HashTreeRoot(ctx)
+		if err != nil {
+			log.Errorf("weird, got an error calling HTR for the state=%v where root should be=%#x", err, parentRoot)
+		}
+		log.Infof("found state in NextSlotCache at slot=%d with root=%#x (parentRoot=%#x)", parentState.Slot(), root, parentRoot)
 	}
 
 	// In the event our cached state has advanced our
@@ -155,9 +161,12 @@ func ProcessSlotsUsingNextSlotCache(
 	if cachedStateExists && parentState.Slot() == slot {
 		return parentState, nil
 	}
+
+	 */
+	log.Infof("process_slots being called up to slot=%d where state.slot=%d", slot, parentState.Slot())
 	// Since next slot cache only advances state by 1 slot,
 	// we check if there's more slots that need to process.
-	parentState, err = ProcessSlots(ctx, parentState, slot)
+	parentState, err := ProcessSlots(ctx, parentState, slot)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not process slots")
 	}

beacon-chain/core/transition/transition_no_verify_sig.go

@@ -282,7 +282,11 @@ func ProcessBlockForStateRoot(
 	state, err = b.ProcessBlockHeaderNoVerify(ctx, state, blk.Slot(), blk.ProposerIndex(), blk.ParentRoot(), bodyRoot[:])
 	if err != nil {
 		tracing.AnnotateError(span, err)
-		return nil, errors.Wrap(err, "could not process block header")
+		r, err := signed.Block().HashTreeRoot()
+		if err != nil {
+			return nil, errors.Wrap(err, "could not process block header, also failed to compute its htr")
+		}
+		return nil, errors.Wrapf(err, "could not process block header, state slot=%d, root=%#x", state.Slot(), r)
 	}
 
 	enabled, err := b.IsExecutionEnabled(state, blk.Body())

beacon-chain/db/iface/interface.go

@@ -105,6 +105,7 @@ type HeadAccessDatabase interface {
 
 	// initialization method needed for origin checkpoint sync
 	SaveOrigin(ctx context.Context, serState, serBlock []byte) error
+	SaveOriginCheckpointBlockRoot(ctx context.Context, blockRoot [32]byte) error
 	SaveBackfillBlockRoot(ctx context.Context, blockRoot [32]byte) error
 }
 

beacon-chain/db/kv/finalized_block_roots.go

@@ -23,7 +23,7 @@ var previousFinalizedCheckpointKey = []byte("previous-finalized-checkpoint")
 var containerFinalizedButNotCanonical = []byte("recent block needs reindexing to determine canonical")
 
 // The finalized block roots index tracks beacon blocks which are finalized in the canonical chain.
-// The finalized checkpoint contains the the epoch which was finalized and the highest beacon block
+// The finalized checkpoint contains the epoch which was finalized and the highest beacon block
 // root where block.slot <= start_slot(epoch). As a result, we cannot index the finalized canonical
 // beacon block chain using the finalized root alone as this would exclude all other blocks in the
 // finalized epoch from being indexed as "final and canonical".
@@ -75,7 +75,7 @@ func (s *Store) updateFinalizedBlockRoots(ctx context.Context, tx *bolt.Tx, chec
 	// Walk up the ancestry chain until we reach a block root present in the finalized block roots
 	// index bucket or genesis block root.
 	for {
-		if bytes.Equal(root, genesisRoot) || bytes.Equal(root, initCheckpointRoot) {
+		if bytes.Equal(root, genesisRoot) {
 			break
 		}
 
@@ -105,6 +105,12 @@ func (s *Store) updateFinalizedBlockRoots(ctx context.Context, tx *bolt.Tx, chec
 			return err
 		}
 
+		// breaking here allows the initial checkpoint root to be correctly inserted,
+		// but stops the loop from trying to search for its parent.
+		if bytes.Equal(root, initCheckpointRoot) {
+			break
+		}
+
 		// Found parent, loop exit condition.
 		if parentBytes := bkt.Get(block.ParentRoot()); parentBytes != nil {
 			parent := &ethpb.FinalizedBlockRootContainer{}

beacon-chain/db/kv/wss_test.go

@@ -42,4 +42,8 @@ func TestSaveOrigin(t *testing.T) {
 	cbb, err := scb.MarshalSSZ()
 	require.NoError(t, err)
 	require.NoError(t, db.SaveOrigin(ctx, csb, cbb))
+
+	broot, err := scb.Block().HashTreeRoot()
+	require.NoError(t, err)
+	require.Equal(t, true, db.IsFinalizedBlock(ctx, broot))
 }

beacon-chain/sync/initial-sync/blocks_fetcher_utils.go

@@ -45,7 +45,7 @@ func (f *blocksFetcher) nonSkippedSlotAfter(ctx context.Context, slot types.Slot
 
 	// Exit early if no peers with epoch higher than our known head are found.
 	if targetEpoch <= headEpoch {
-		return 0, errSlotIsTooHigh
+		return 0, errors.Wrapf(errSlotIsTooHigh, "no peers with epoch higher than our known head, peer epoch=%d, head=%d", targetEpoch, headEpoch)
 	}
 
 	// Transform peer list to avoid eclipsing (filter, shuffle, trim).

beacon-chain/sync/initial-sync/blocks_queue.go

@@ -2,7 +2,7 @@ package initialsync
 
 import (
 	"context"
-	"errors"
+	"github.com/pkg/errors"
 	"time"
 
 	"github.com/libp2p/go-libp2p-core/peer"
@@ -285,7 +285,7 @@ func (q *blocksQueue) onScheduleEvent(ctx context.Context) eventHandlerFn {
 		}
 		if m.start > q.highestExpectedSlot {
 			m.setState(stateSkipped)
-			return m.state, errSlotIsTooHigh
+			return m.state, errors.Wrapf(errSlotIsTooHigh, "slot=%d", m.start)
 		}
 		blocksPerRequest := q.blocksFetcher.blocksPerSecond
 		if err := q.blocksFetcher.scheduleRequest(ctx, m.start, blocksPerRequest); err != nil {

beacon-chain/sync/rpc_status.go

@@ -3,6 +3,7 @@ package sync
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"sync"
 	"time"
 
@@ -307,7 +308,7 @@ func (s *Service) validateStatusMessage(ctx context.Context, msg *pb.Status) err
 		return nil
 	}
 	if !s.cfg.beaconDB.IsFinalizedBlock(ctx, bytesutil.ToBytes32(msg.FinalizedRoot)) {
-		return p2ptypes.ErrInvalidFinalizedRoot
+		return errors.Wrap(p2ptypes.ErrInvalidFinalizedRoot, fmt.Sprintf("root=%#x", msg.FinalizedRoot))
 	}
 	blk, err := s.cfg.beaconDB.Block(ctx, bytesutil.ToBytes32(msg.FinalizedRoot))
 	if err != nil {

beacon-chain/sync/validate_aggregate_proof.go

@@ -97,10 +97,15 @@ func (s *Service) validateAggregateAndProof(ctx context.Context, pid peer.ID, ms
 		return pubsub.ValidationIgnore, nil
 	}
 	// Check that the block being voted on isn't invalid.
-	if s.hasBadBlock(bytesutil.ToBytes32(m.Message.Aggregate.Data.BeaconBlockRoot)) ||
-		s.hasBadBlock(bytesutil.ToBytes32(m.Message.Aggregate.Data.Target.Root)) ||
-		s.hasBadBlock(bytesutil.ToBytes32(m.Message.Aggregate.Data.Source.Root)) {
-		return pubsub.ValidationReject, errors.New("bad block referenced in attestation data")
+	errBadBlockRef := errors.New("bad block referenced in attestation data")
+	if s.hasBadBlock(bytesutil.ToBytes32(m.Message.Aggregate.Data.BeaconBlockRoot)) {
+		return pubsub.ValidationReject, errors.Wrapf(errBadBlockRef, "block=BeaconBlockRoot, root=%#x", m.Message.Aggregate.Data.BeaconBlockRoot)
+	}
+	if s.hasBadBlock(bytesutil.ToBytes32(m.Message.Aggregate.Data.Target.Root)) {
+		return pubsub.ValidationReject, errors.Wrapf(errBadBlockRef, "block=Target, root=%#x", m.Message.Aggregate.Data.Target.Root)
+	}
+	if s.hasBadBlock(bytesutil.ToBytes32(m.Message.Aggregate.Data.Source.Root)) {
+		return pubsub.ValidationReject, errors.Wrapf(errBadBlockRef, "block=Source, root=%#x", m.Message.Aggregate.Data.Source.Root)
 	}
 
 	// Verify aggregate attestation has not already been seen via aggregate gossip, within a block, or through the creation locally.

beacon-chain/sync/validate_beacon_blocks.go

@@ -189,6 +189,8 @@ func (s *Service) validateBeaconBlockPubSub(ctx context.Context, pid peer.ID, ms
 	return pubsub.ValidationAccept, nil
 }
 
+var errIncorrectProposerIndex = errors.New("incorrect proposer index")
+
 func (s *Service) validateBeaconBlock(ctx context.Context, blk block.SignedBeaconBlock, blockRoot [32]byte) error {
 	ctx, span := trace.StartSpan(ctx, "sync.validateBeaconBlock")
 	defer span.End()
@@ -220,13 +222,19 @@ func (s *Service) validateBeaconBlock(ctx context.Context, blk block.SignedBeaco
 	if err != nil {
 		return err
 	}
+	sRoot, err := parentState.HashTreeRoot(ctx)
+	if err != nil {
+		log.Errorf("that's weird, htr fail")
+	}
+	log.Infof("validating block with slot=%d, state.slot=%d, block_root=%#x, state_root=%#x", blk.Block().Slot(), parentState.Slot(), blockRoot, sRoot)
 	idx, err := helpers.BeaconProposerIndex(ctx, parentState)
 	if err != nil {
 		return err
 	}
+	log.Infof("got BeaconProposerIndex=%d, block proposer index=%d", idx, blk.Block().ProposerIndex())
 	if blk.Block().ProposerIndex() != idx {
 		s.setBadBlock(ctx, blockRoot)
-		return errors.New("incorrect proposer index")
+		return errors.Wrapf(errIncorrectProposerIndex, "state slot=%d, root=%#x, block_root=%#x", parentState.Slot(), sRoot, blockRoot)
 	}
 
 	if err = s.validateBellatrixBeaconBlock(ctx, parentState, blk.Block()); err != nil {

config/params/config_utils_develop.go

@@ -26,6 +26,15 @@ func BeaconConfig() *BeaconChainConfig {
 func OverrideBeaconConfig(c *BeaconChainConfig) {
 	beaconConfigLock.Lock()
 	defer beaconConfigLock.Unlock()
+	c.InitializeForkSchedule()
+	name, ok := reverseConfigNames[c.ConfigName]
+	// if name collides with an existing config name, override it, because the fork versions probably conflict
+	if !ok {
+		// otherwise define it as the special "Dynamic" name, ie for a config loaded from a file at runtime
+		name = Dynamic
+	}
+	KnownConfigs[name] = func() *BeaconChainConfig { return c }
+	rebuildKnownForkVersions()
 	beaconConfig = c
 }
 

config/params/config_utils_prod.go

@@ -19,6 +19,15 @@ func BeaconConfig() *BeaconChainConfig {
 // OverrideBeaconConfig(c). Any subsequent calls to params.BeaconConfig() will
 // return this new configuration.
 func OverrideBeaconConfig(c *BeaconChainConfig) {
+	c.InitializeForkSchedule()
+	name, ok := reverseConfigNames[c.ConfigName]
+	// if name collides with an existing config name, override it, because the fork versions probably conflict
+	if !ok {
+		// otherwise define it as the special "Dynamic" name, ie for a config loaded from a file at runtime
+		name = Dynamic
+	}
+	KnownConfigs[name] = func() *BeaconChainConfig { return c }
+	rebuildKnownForkVersions()
 	beaconConfig = c
 }
 

config/params/loader.go

@@ -70,7 +70,6 @@ func LoadChainConfigFile(chainConfigFileName string, conf *BeaconChainConfig) {
 	// recompute SqrRootSlotsPerEpoch constant to handle non-standard values of SlotsPerEpoch
 	conf.SqrRootSlotsPerEpoch = types.Slot(math.IntegerSquareRoot(uint64(conf.SlotsPerEpoch)))
 	log.Debugf("Config file values: %+v", conf)
-	conf.InitializeForkSchedule()
 	OverrideBeaconConfig(conf)
 }
 

config/params/values.go

@@ -15,6 +15,7 @@ const (
 	Pyrmont
 	Prater
 	EndToEndMainnet
+	Dynamic
 )
 
 // ConfigName enum describes the type of known network in use.
@@ -36,7 +37,9 @@ var ConfigNames = map[ConfigName]string{
 	Pyrmont:         "pyrmont",
 	Prater:          "prater",
 	EndToEndMainnet: "end-to-end-mainnet",
+	Dynamic:         "dynamic",
 }
+var reverseConfigNames map[string]ConfigName
 
 // KnownConfigs provides an index of all known BeaconChainConfig values.
 var KnownConfigs = map[ConfigName]func() *BeaconChainConfig{
@@ -63,6 +66,18 @@ func ConfigForVersion(version [fieldparams.VersionLength]byte) (*BeaconChainConf
 }
 
 func init() {
+	rebuildKnownForkVersions()
+	buildReverseConfigName()
+}
+
+func buildReverseConfigName() {
+	reverseConfigNames = make(map[string]ConfigName)
+	for cn, s := range ConfigNames {
+		reverseConfigNames[s] = cn
+	}
+}
+
+func rebuildKnownForkVersions() {
 	knownForkVersions = make(map[[fieldparams.VersionLength]byte]ConfigName)
 	for n, cfunc := range KnownConfigs {
 		cfg := cfunc()

deps.bzl

@@ -1274,6 +1274,12 @@ def prysm_deps():
         sum = "h1:utua3L2IbQJmauC5IXdEA547bcoU5dozgQAfc8Onsg4=",
         version = "v0.0.0-20181222135242-d2cdd8c08219",
     )
+    go_repository(
+        name = "com_github_gomarkdown_markdown",
+        importpath = "github.com/gomarkdown/markdown",
+        sum = "h1:YVvt637ygnOO9qjLBVmPOvrUmCz/i8YECSu/8UlOQW0=",
+        version = "v0.0.0-20220310201231-552c6011c0b8",
+    )
 
     go_repository(
         name = "com_github_google_btree",

encoding/ssz/detect/configfork.go

@@ -81,40 +81,40 @@ func FromForkVersion(cv [fieldparams.VersionLength]byte) (*VersionedUnmarshaler,
 // UnmarshalBeaconState uses internal knowledge in the VersionedUnmarshaler to pick the right concrete BeaconState type,
 // then Unmarshal()s the type and returns an instance of state.BeaconState if successful.
 func (cf *VersionedUnmarshaler) UnmarshalBeaconState(marshaled []byte) (s state.BeaconState, err error) {
-	forkName := version.String(cf.Fork)
+	info := fmt.Sprintf("fork=%s, config=%s", version.String(cf.Fork), cf.Config.ConfigName)
 	switch fork := cf.Fork; fork {
 	case version.Phase0:
 		st := &ethpb.BeaconState{}
 		err = st.UnmarshalSSZ(marshaled)
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to unmarshal state, detected fork=%s", forkName)
+			return nil, errors.Wrapf(err, "failed to unmarshal state, detected info=%s", info)
 		}
 		s, err = v1.InitializeFromProtoUnsafe(st)
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to init state trie from state, detected fork=%s", forkName)
+			return nil, errors.Wrapf(err, "failed to init state trie from state, detected info=%s", info)
 		}
 	case version.Altair:
 		st := &ethpb.BeaconStateAltair{}
 		err = st.UnmarshalSSZ(marshaled)
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to unmarshal state, detected fork=%s", forkName)
+			return nil, errors.Wrapf(err, "failed to unmarshal state, detected info=%s", info)
 		}
 		s, err = v2.InitializeFromProtoUnsafe(st)
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to init state trie from state, detected fork=%s", forkName)
+			return nil, errors.Wrapf(err, "failed to init state trie from state, detected info=%s", info)
 		}
 	case version.Bellatrix:
 		st := &ethpb.BeaconStateBellatrix{}
 		err = st.UnmarshalSSZ(marshaled)
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to unmarshal state, detected fork=%s", forkName)
+			return nil, errors.Wrapf(err, "failed to unmarshal state, detected info=%s", info)
 		}
 		s, err = v3.InitializeFromProtoUnsafe(st)
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to init state trie from state, detected fork=%s", forkName)
+			return nil, errors.Wrapf(err, "failed to init state trie from state, detected info=%s", info)
 		}
 	default:
-		return nil, fmt.Errorf("unable to initialize BeaconState for fork version=%s", forkName)
+		return nil, fmt.Errorf("unable to initialize BeaconState for info=%s", info)
 	}
 
 	return s, nil

go.mod

@@ -255,6 +255,7 @@ require (
 	github.com/go-logr/logr v0.2.1 // indirect
 	github.com/go-ole/go-ole v1.2.5 // indirect
 	github.com/go-playground/validator/v10 v10.10.0
+	github.com/gomarkdown/markdown v0.0.0-20220310201231-552c6011c0b8
 	github.com/peterh/liner v1.2.0 // indirect
 	github.com/prometheus/tsdb v0.10.0 // indirect
 	github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220303211031-f753e083138c

go.sum

@@ -447,6 +447,8 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
+github.com/gomarkdown/markdown v0.0.0-20220310201231-552c6011c0b8 h1:YVvt637ygnOO9qjLBVmPOvrUmCz/i8YECSu/8UlOQW0=
+github.com/gomarkdown/markdown v0.0.0-20220310201231-552c6011c0b8/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=

testing/endtoend/BUILD.bazel

@@ -7,6 +7,7 @@ go_test(
     size = "large",
     testonly = True,
     srcs = [
+        "checkpoint_sync_test.go",
         "endtoend_test.go",
         "minimal_e2e_test.go",
         "minimal_slashing_e2e_test.go",
@@ -30,6 +31,7 @@ go_test(
         "requires-network",
     ],
     deps = [
+        "//api/client/beacon:go_default_library",
         "//beacon-chain/blockchain/testing:go_default_library",
         "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
@@ -37,10 +39,15 @@ go_test(
         "//beacon-chain/state/stategen/mock:go_default_library",
         "//config/params:go_default_library",
         "//crypto/bls:go_default_library",
+        "//encoding/bytesutil:go_default_library",
+        "//io/file:go_default_library",
+        "//proto/eth/service:go_default_library",
+        "//proto/eth/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/endtoend/components:go_default_library",
         "//testing/endtoend/components/eth1:go_default_library",
+        "//testing/endtoend/e2ez:go_default_library",
         "//testing/endtoend/evaluators:go_default_library",
         "//testing/endtoend/helpers:go_default_library",
         "//testing/endtoend/params:go_default_library",
@@ -53,6 +60,8 @@ go_test(
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
         "@org_golang_google_grpc//:go_default_library",
+        "@org_golang_google_grpc//codes:go_default_library",
+        "@org_golang_google_grpc//status:go_default_library",
         "@org_golang_google_protobuf//types/known/emptypb:go_default_library",
         "@org_golang_x_sync//errgroup:go_default_library",
     ],
@@ -84,6 +93,7 @@ go_test(
         "requires-network",
     ],
     deps = [
+        "//api/client/beacon:go_default_library",
         "//beacon-chain/blockchain/testing:go_default_library",
         "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
@@ -91,6 +101,11 @@ go_test(
         "//beacon-chain/state/stategen:go_default_library",
         "//config/params:go_default_library",
         "//crypto/bls:go_default_library",
+        "//encoding/bytesutil:go_default_library",
+        "//io/file:go_default_library",
+        "//proto/eth/service:go_default_library",
+        "//proto/eth/v1:go_default_library",
+        "//proto/eth/v2:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/endtoend/components:go_default_library",
@@ -102,11 +117,14 @@ go_test(
         "//testing/require:go_default_library",
         "//testing/slasher/simulator:go_default_library",
         "//testing/util:go_default_library",
+        "//time/slots:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prysmaticlabs_eth2_types//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
         "@org_golang_google_grpc//:go_default_library",
+        "@org_golang_google_grpc//codes:go_default_library",
+        "@org_golang_google_grpc//status:go_default_library",
         "@org_golang_google_protobuf//types/known/emptypb:go_default_library",
         "@org_golang_x_sync//errgroup:go_default_library",
     ],

testing/endtoend/checkpoint_sync_test.go

@@ -0,0 +1,93 @@
+package endtoend
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"testing"
+
+	e2types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/config/params"
+	ev "github.com/prysmaticlabs/prysm/testing/endtoend/evaluators"
+	e2eParams "github.com/prysmaticlabs/prysm/testing/endtoend/params"
+	"github.com/prysmaticlabs/prysm/testing/endtoend/types"
+	"github.com/prysmaticlabs/prysm/testing/require"
+)
+
+// This test customizes the minimal config in order to artificially shorten the weak subjectivity period
+// so that the state used will not be genesis despite there only being 10 epochs of history.
+func TestCheckpointSync_CustomConfig(t *testing.T) {
+	// Run for 10 epochs if not in long-running to confirm long-running has no issues.
+	var err error
+	epochsToRun := 10
+	epochStr, longRunning := os.LookupEnv("E2E_EPOCHS")
+	if longRunning {
+		epochsToRun, err = strconv.Atoi(epochStr)
+		require.NoError(t, err)
+	}
+
+	cfg := params.E2ETestConfig()
+	cfg.BellatrixForkEpoch = 10000
+	// setting this to 1 should change the weak subjectivity computation,
+	// so the computed weak subjectivity checkpoint will just be a few epochs before head
+	cfg.MinValidatorWithdrawabilityDelay = e2types.Epoch(epochsToRun / 2)
+	cfg.SlotsPerEpoch = 6
+	cfg.SecondsPerSlot = 6
+	params.OverrideBeaconConfig(cfg)
+	require.NoError(t, e2eParams.Init(e2eParams.StandardBeaconCount))
+
+	seed := 0
+	seedStr, isValid := os.LookupEnv("E2E_SEED")
+	if isValid {
+		seed, err = strconv.Atoi(seedStr)
+		require.NoError(t, err)
+	}
+	tracingPort := e2eParams.TestParams.Ports.JaegerTracingPort
+	tracingEndpoint := fmt.Sprintf("127.0.0.1:%d", tracingPort)
+	evals := []types.Evaluator{
+		ev.PeersConnect,
+		ev.HealthzCheck,
+		ev.MetricsCheck,
+		ev.ValidatorsAreActive,
+		ev.ValidatorsParticipatingAtEpoch(2),
+		ev.FinalizationOccurs(3),
+		ev.PeersCheck,
+		ev.ProcessesDepositsInBlocks,
+		ev.VerifyBlockGraffiti,
+		ev.ActivatesDepositedValidators,
+		ev.DepositedValidatorsAreActive,
+		ev.ProposeVoluntaryExit,
+		ev.ValidatorHasExited,
+		ev.ValidatorsVoteWithTheMajority,
+		ev.ColdStateCheckpoint,
+		ev.ForkTransition,
+		ev.APIMiddlewareVerifyIntegrity,
+		ev.APIGatewayV1Alpha1VerifyIntegrity,
+		ev.FinishedSyncing,
+		ev.AllNodesHaveSameHead,
+		ev.ValidatorSyncParticipation,
+	}
+	testConfig := &types.E2EConfig{
+		BeaconFlags: []string{
+			fmt.Sprintf("--slots-per-archive-point=%d", params.BeaconConfig().SlotsPerEpoch*16),
+			fmt.Sprintf("--tracing-endpoint=http://%s", tracingEndpoint),
+			"--enable-tracing",
+			"--trace-sample-fraction=1.0",
+		},
+		ValidatorFlags:      []string{},
+		EpochsToRun:         uint64(epochsToRun),
+		TestSync:            true,
+		TestFeature:         true,
+		TestDeposits:        true,
+		UseFixedPeerIDs:     true,
+		UsePrysmShValidator: false,
+		UsePprof:            !longRunning,
+		TracingSinkEndpoint: tracingEndpoint,
+		Evaluators:          evals,
+		Seed:                int64(seed),
+		BeaconChainConfig:   cfg,
+		LeaveRunning: false,
+	}
+
+	newTestRunner(t, testConfig).run()
+}

testing/endtoend/components/BUILD.bazel

@@ -31,6 +31,7 @@ go_library(
         "//io/file:go_default_library",
         "//runtime/interop:go_default_library",
         "//testing/endtoend/components/eth1:go_default_library",
+        "//testing/endtoend/e2ez:go_default_library",
         "//testing/endtoend/helpers:go_default_library",
         "//testing/endtoend/params:go_default_library",
         "//testing/endtoend/types:go_default_library",

testing/endtoend/components/beacon_node.go

@@ -3,20 +3,24 @@
 package components
 
 import (
+	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
+	"text/template"
 
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
+	"github.com/pkg/errors"
 	cmdshared "github.com/prysmaticlabs/prysm/cmd"
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/flags"
 	"github.com/prysmaticlabs/prysm/config/features"
 	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/testing/endtoend/e2ez"
 	"github.com/prysmaticlabs/prysm/testing/endtoend/helpers"
 	e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params"
 	e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types"
@@ -24,7 +28,6 @@ import (
 
 var _ e2etypes.ComponentRunner = (*BeaconNode)(nil)
 var _ e2etypes.ComponentRunner = (*BeaconNodeSet)(nil)
-var _ e2etypes.BeaconNodeSet = (*BeaconNodeSet)(nil)
 
 // BeaconNodeSet represents set of beacon nodes.
 type BeaconNodeSet struct {
@@ -33,19 +36,37 @@ type BeaconNodeSet struct {
 	enr     string
 	ids     []string
 	started chan struct{}
-}
-
-// SetENR assigns ENR to the set of beacon nodes.
-func (s *BeaconNodeSet) SetENR(enr string) {
-	s.enr = enr
+	nodes   []*BeaconNode
+	flags   []string
+	zp *e2ez.Server
 }
 
 // NewBeaconNodes creates and returns a set of beacon nodes.
-func NewBeaconNodes(config *e2etypes.E2EConfig) *BeaconNodeSet {
-	return &BeaconNodeSet{
+func NewBeaconNodes(config *e2etypes.E2EConfig, enr string, flags []string, zp *e2ez.Server) *BeaconNodeSet {
+	// Create beacon nodes.
+	nodes := make([]*BeaconNode, e2e.TestParams.BeaconNodeCount)
+	for i := 0; i < e2e.TestParams.BeaconNodeCount; i++ {
+		nodes[i] = NewBeaconNode(i, enr, flags, config)
+		zp.HandleZPages(nodes[i])
+	}
+
+	bns := &BeaconNodeSet{
 		config:  config,
 		started: make(chan struct{}, 1),
+		nodes:   nodes,
+		enr:     enr,
+		flags:   flags,
+		zp: zp,
 	}
+	zp.HandleZPages(bns)
+	return bns
+}
+
+func (s *BeaconNodeSet) AddBeaconNode(index int, flags []string) *BeaconNode {
+	bn := NewBeaconNode(index, s.enr, flags, s.config)
+	s.nodes = append(s.nodes, bn)
+	s.zp.HandleZPages(bn)
+	return bn
 }
 
 // Start starts all the beacon nodes in set.
@@ -54,10 +75,9 @@ func (s *BeaconNodeSet) Start(ctx context.Context) error {
 		return errors.New("empty ENR")
 	}
 
-	// Create beacon nodes.
-	nodes := make([]e2etypes.ComponentRunner, e2e.TestParams.BeaconNodeCount)
-	for i := 0; i < e2e.TestParams.BeaconNodeCount; i++ {
-		nodes[i] = NewBeaconNode(s.config, i, s.enr)
+	nodes := make([]e2etypes.ComponentRunner, len(s.nodes))
+	for i, n := range s.nodes {
+		nodes[i] = n
 	}
 
 	// Wait for all nodes to finish their job (blocking).
@@ -74,6 +94,23 @@ func (s *BeaconNodeSet) Start(ctx context.Context) error {
 	})
 }
 
+func (s *BeaconNodeSet) ZPath() string {
+	return "/beacon-nodes"
+}
+
+func (s *BeaconNodeSet) ZMarkdown() (string, error) {
+	tmpl := `
+%d beacon nodes
+---------------
+
+%s`
+	nodeList := ""
+	for _, node := range s.nodes {
+		nodeList = nodeList + fmt.Sprintf("\n - [beacon node #%d](%s)", node.index, node.ZPath())
+	}
+	return fmt.Sprintf(tmpl, len(s.nodes), nodeList), nil
+}
+
 // Started checks whether beacon node set is started and all nodes are ready to be queried.
 func (s *BeaconNodeSet) Started() <-chan struct{} {
 	return s.started
@@ -85,45 +122,94 @@ type BeaconNode struct {
 	config  *e2etypes.E2EConfig
 	started chan struct{}
 	index   int
+	flags   []string
 	enr     string
 	peerID  string
 }
 
+func (node *BeaconNode) ZPath() string {
+	return fmt.Sprintf("/beacon-node/%d", node.index)
+}
+
+var bnzm = template.Must(template.New("BeaconNode.ZMarkdown").Parse("" +
+	"beacon node {{.Index}}\n" +
+	"--------------\n\n" +
+	"```\n" +
+	"{{.StartCmd}}" +
+	"```\n\n" +
+	"http addr={{.HTTPAddr}}\n\n" +
+	"grpc addr={{.GRPCAddr}}\n\n" +
+	"db path={{.DBPath}}\n\n" +
+	"log path={{.LogPath}}\n\n" +
+	"stdout path={{.StdoutPath}}\n\n" +
+	"stderr path={{.StderrPath}}\n\n"))
+
+func (node *BeaconNode) ZMarkdown() (string, error) {
+	bin, args, err := node.startCommand()
+	if err != nil {
+		return "", err
+	}
+	cmd := path.Join(bin, args[0])
+	for _, a := range args {
+		cmd += fmt.Sprintf("\n%s \\", a)
+	}
+
+	buf := bytes.NewBuffer(nil)
+	err = bnzm.Execute(buf, struct {
+		Index      int
+		StartCmd   string
+		DBPath     string
+		LogPath    string
+		StdoutPath string
+		StderrPath string
+		HTTPAddr   string
+		GRPCAddr   string
+	}{
+		Index:      node.index,
+		StartCmd:   cmd,
+		DBPath:     node.dbPath(),
+		LogPath:    node.logPath(),
+		StdoutPath: node.stdoutPath(),
+		StderrPath: node.stderrPath(),
+		HTTPAddr:   node.httpAddr(),
+		GRPCAddr:   node.grpcAddr(),
+	})
+	return buf.String(), err
+}
+
+var _ e2ez.ZPage = &BeaconNode{}
+
 // NewBeaconNode creates and returns a beacon node.
-func NewBeaconNode(config *e2etypes.E2EConfig, index int, enr string) *BeaconNode {
+func NewBeaconNode(index int, enr string, flags []string, config *e2etypes.E2EConfig) *BeaconNode {
 	return &BeaconNode{
 		config:  config,
 		index:   index,
 		enr:     enr,
 		started: make(chan struct{}, 1),
+		flags:   flags,
 	}
 }
 
-// Start starts a fresh beacon node, connecting to all passed in beacon nodes.
-func (node *BeaconNode) Start(ctx context.Context) error {
+func (node *BeaconNode) startCommand() (string, []string, error) {
 	binaryPath, found := bazel.FindBinary("cmd/beacon-chain", "beacon-chain")
 	if !found {
 		log.Info(binaryPath)
-		return errors.New("beacon chain binary not found")
+		return "", []string{}, errors.New("beacon chain binary not found")
 	}
-
 	config, index, enr := node.config, node.index, node.enr
-	stdOutFile, err := helpers.DeleteAndCreateFile(e2e.TestParams.LogPath, fmt.Sprintf(e2e.BeaconNodeLogFileName, index))
-	if err != nil {
-		return err
-	}
-	expectedNumOfPeers := e2e.TestParams.BeaconNodeCount + e2e.TestParams.LighthouseBeaconNodeCount - 1
+	expectedNumOfPeers := e2e.TestParams.BeaconNodeCount + e2e.TestParams.LighthouseBeaconNodeCount
 	if node.config.TestSync {
 		expectedNumOfPeers += 1
 	}
+	expectedNumOfPeers += 10
 	jwtPath := path.Join(e2e.TestParams.TestPath, "eth1data/"+strconv.Itoa(node.index)+"/")
 	if index == 0 {
 		jwtPath = path.Join(e2e.TestParams.TestPath, "eth1data/miner/")
 	}
 	jwtPath = path.Join(jwtPath, "geth/jwtsecret")
 	args := []string{
-		fmt.Sprintf("--%s=%s/eth2-beacon-node-%d", cmdshared.DataDirFlag.Name, e2e.TestParams.TestPath, index),
-		fmt.Sprintf("--%s=%s", cmdshared.LogFileName.Name, stdOutFile.Name()),
+		fmt.Sprintf("--%s=%s", cmdshared.DataDirFlag.Name, node.dbPath()),
+		fmt.Sprintf("--%s=%s", cmdshared.LogFileName.Name, node.logPath()),
 		fmt.Sprintf("--%s=%s", flags.DepositContractFlag.Name, e2e.TestParams.ContractAddress.Hex()),
 		fmt.Sprintf("--%s=%d", flags.RPCPort.Name, e2e.TestParams.Ports.PrysmBeaconNodeRPCPort+index),
 		fmt.Sprintf("--%s=http://127.0.0.1:%d", flags.HTTPWeb3ProviderFlag.Name, e2e.TestParams.Ports.Eth1RPCPort+index),
@@ -139,8 +225,9 @@ func (node *BeaconNode) Start(ctx context.Context) error {
 		fmt.Sprintf("--%s=%d", cmdshared.RPCMaxPageSizeFlag.Name, params.BeaconConfig().MinGenesisActiveValidatorCount),
 		fmt.Sprintf("--%s=%s", cmdshared.BootstrapNode.Name, enr),
 		fmt.Sprintf("--%s=%s", cmdshared.VerbosityFlag.Name, "debug"),
+		fmt.Sprintf("--%s=%s", cmdshared.ChainConfigFileFlag.Name, node.config.BeaconChainConfigPath()),
+		"--slots-per-archive-point=1",
 		"--" + cmdshared.ForceClearDB.Name,
-		"--" + cmdshared.E2EConfigFlag.Name,
 		"--" + cmdshared.AcceptTosFlag.Name,
 		"--" + flags.EnableDebugRPCEndpoints.Name,
 	}
@@ -153,14 +240,55 @@ func (node *BeaconNode) Start(ctx context.Context) error {
 		args = append(args, features.E2EBeaconChainFlags...)
 	}
 	args = append(args, config.BeaconFlags...)
+	args = append(args, node.flags...)
 
-	cmd := exec.CommandContext(ctx, binaryPath, args...) // #nosec G204 -- Safe
-	// Write stdout and stderr to log files.
-	stdout, err := os.Create(path.Join(e2e.TestParams.LogPath, fmt.Sprintf("beacon_node_%d_stdout.log", index)))
+	return binaryPath, args, nil
+}
+
+func (node *BeaconNode) dbPath() string {
+	return fmt.Sprintf("%s/eth2-beacon-node-%d", e2e.TestParams.TestPath, node.index)
+}
+
+func (node *BeaconNode) logPath() string {
+	return filepath.Clean(path.Join(e2e.TestParams.LogPath, fmt.Sprintf(e2e.BeaconNodeLogFileName, node.index)))
+}
+
+func (node *BeaconNode) stdoutPath() string {
+	return path.Join(e2e.TestParams.LogPath, fmt.Sprintf("beacon_node_%d_stdout.log", node.index))
+}
+
+func (node *BeaconNode) stderrPath() string {
+	return path.Join(e2e.TestParams.LogPath, fmt.Sprintf("beacon_node_%d_stderr.log", node.index))
+}
+
+func (node *BeaconNode) httpAddr() string {
+	port := e2e.TestParams.Ports.PrysmBeaconNodeGatewayPort + node.index
+	return fmt.Sprintf("http://localhost:%d", port)
+}
+
+func (node *BeaconNode) grpcAddr() string {
+	port := e2e.TestParams.Ports.PrysmBeaconNodeRPCPort + node.index
+	return fmt.Sprintf("localhost:%d", port)
+}
+
+// Start starts a fresh beacon node, connecting to all passed in beacon nodes.
+func (node *BeaconNode) Start(ctx context.Context) error {
+	stdOutFile, err := helpers.DeleteAndCreateFile(node.logPath(), "")
 	if err != nil {
 		return err
 	}
-	stderr, err := os.Create(path.Join(e2e.TestParams.LogPath, fmt.Sprintf("beacon_node_%d_stderr.log", index)))
+
+	bin, args, err := node.startCommand()
+	if err != nil {
+		return errors.Wrap(err, "filed to generate start command")
+	}
+	cmd := exec.CommandContext(ctx, bin, args...) // #nosec G204 -- Safe
+	// Write stdout and stderr to log files.
+	stdout, err := os.Create(node.stdoutPath())
+	if err != nil {
+		return err
+	}
+	stderr, err := os.Create(node.stderrPath())
 	if err != nil {
 		return err
 	}
@@ -174,16 +302,16 @@ func (node *BeaconNode) Start(ctx context.Context) error {
 	}()
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
-	log.Infof("Starting beacon chain %d with flags: %s", index, strings.Join(args[2:], " "))
+	log.Infof("Starting beacon chain %d with flags: %s", node.index, strings.Join(args[2:], " "))
 	if err = cmd.Start(); err != nil {
 		return fmt.Errorf("failed to start beacon node: %w", err)
 	}
 
 	if err = helpers.WaitForTextInFile(stdOutFile, "gRPC server listening on port"); err != nil {
-		return fmt.Errorf("could not find multiaddr for node %d, this means the node had issues starting: %w", index, err)
+		return fmt.Errorf("could not find multiaddr for node %d, this means the node had issues starting: %w", node.index, err)
 	}
 
-	if config.UseFixedPeerIDs {
+	if node.config.UseFixedPeerIDs {
 		peerId, err := helpers.FindFollowingTextInFile(stdOutFile, "Running node with peer id of ")
 		if err != nil {
 			return fmt.Errorf("could not find peer id: %w", err)
@@ -201,3 +329,7 @@ func (node *BeaconNode) Start(ctx context.Context) error {
 func (node *BeaconNode) Started() <-chan struct{} {
 	return node.started
 }
+
+func (node *BeaconNode) Index() int {
+	return node.index
+}

testing/endtoend/components/boot_node.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
+	"github.com/prysmaticlabs/prysm/config/params"
 	"github.com/prysmaticlabs/prysm/testing/endtoend/helpers"
 	e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params"
 	e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types"
@@ -48,10 +49,12 @@ func (node *BootNode) Start(ctx context.Context) error {
 		return err
 	}
 
+	cfg := params.BeaconConfig()
 	args := []string{
 		fmt.Sprintf("--log-file=%s", stdOutFile.Name()),
 		fmt.Sprintf("--discv5-port=%d", e2e.TestParams.Ports.BootNodePort),
 		fmt.Sprintf("--metrics-port=%d", e2e.TestParams.Ports.BootNodeMetricsPort),
+		fmt.Sprintf("--fork-version=%#x", cfg.GenesisForkVersion),
 		"--debug",
 	}
 

testing/endtoend/components/lighthouse_beacon.go

@@ -1,6 +1,7 @@
 package components
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -10,10 +11,12 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"text/template"
 
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
 	"github.com/prysmaticlabs/prysm/config/params"
 	"github.com/prysmaticlabs/prysm/io/file"
+	"github.com/prysmaticlabs/prysm/testing/endtoend/e2ez"
 	"github.com/prysmaticlabs/prysm/testing/endtoend/helpers"
 	e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params"
 	e2etypes "github.com/prysmaticlabs/prysm/testing/endtoend/types"
@@ -21,7 +24,6 @@ import (
 
 var _ e2etypes.ComponentRunner = (*LighthouseBeaconNode)(nil)
 var _ e2etypes.ComponentRunner = (*LighthouseBeaconNodeSet)(nil)
-var _ e2etypes.BeaconNodeSet = (*LighthouseBeaconNodeSet)(nil)
 
 // LighthouseBeaconNodeSet represents set of lighthouse beacon nodes.
 type LighthouseBeaconNodeSet struct {
@@ -29,19 +31,26 @@ type LighthouseBeaconNodeSet struct {
 	config  *e2etypes.E2EConfig
 	enr     string
 	started chan struct{}
-}
-
-// SetENR assigns ENR to the set of beacon nodes.
-func (s *LighthouseBeaconNodeSet) SetENR(enr string) {
-	s.enr = enr
+	nodes   []*LighthouseBeaconNode
+	zp *e2ez.Server
 }
 
 // NewLighthouseBeaconNodes creates and returns a set of lighthouse beacon nodes.
-func NewLighthouseBeaconNodes(config *e2etypes.E2EConfig) *LighthouseBeaconNodeSet {
-	return &LighthouseBeaconNodeSet{
+func NewLighthouseBeaconNodes(config *e2etypes.E2EConfig, enr string, zp *e2ez.Server) *LighthouseBeaconNodeSet {
+	nodes := make([]*LighthouseBeaconNode, e2e.TestParams.LighthouseBeaconNodeCount)
+	for i := 0; i < e2e.TestParams.LighthouseBeaconNodeCount; i++ {
+		nodes[i] = NewLighthouseBeaconNode(config, i, enr)
+		zp.HandleZPages(nodes[i])
+	}
+	bns := &LighthouseBeaconNodeSet{
 		config:  config,
 		started: make(chan struct{}, 1),
+		enr:     enr,
+		nodes:   nodes,
+		zp: zp,
 	}
+	zp.HandleZPages(bns)
+	return bns
 }
 
 // Start starts all the beacon nodes in set.
@@ -51,9 +60,9 @@ func (s *LighthouseBeaconNodeSet) Start(ctx context.Context) error {
 	}
 
 	// Create beacon nodes.
-	nodes := make([]e2etypes.ComponentRunner, e2e.TestParams.LighthouseBeaconNodeCount)
+	nodes := make([]e2etypes.ComponentRunner, len(s.nodes))
 	for i := 0; i < e2e.TestParams.LighthouseBeaconNodeCount; i++ {
-		nodes[i] = NewLighthouseBeaconNode(s.config, i, s.enr)
+		nodes[i] = s.nodes[i]
 	}
 
 	// Wait for all nodes to finish their job (blocking).
@@ -69,6 +78,23 @@ func (s *LighthouseBeaconNodeSet) Started() <-chan struct{} {
 	return s.started
 }
 
+func (s *LighthouseBeaconNodeSet) ZPath() string {
+	return "/lh-beacon-nodes"
+}
+
+func (s *LighthouseBeaconNodeSet) ZMarkdown() (string, error) {
+	tmpl := `
+%d beacon nodes
+---------------
+
+%s`
+	nodeList := ""
+	for _, node := range s.nodes {
+		nodeList = nodeList + fmt.Sprintf("\n - [beacon node #%d](%s)", node.index, node.ZPath())
+	}
+	return fmt.Sprintf(tmpl, len(s.nodes), nodeList), nil
+}
+
 // LighthouseBeaconNode represents a lighthouse beacon node.
 type LighthouseBeaconNode struct {
 	e2etypes.ComponentRunner
@@ -88,39 +114,51 @@ func NewLighthouseBeaconNode(config *e2etypes.E2EConfig, index int, enr string)
 	}
 }
 
-// Start starts a fresh beacon node, connecting to all passed in beacon nodes.
-func (node *LighthouseBeaconNode) Start(ctx context.Context) error {
+func (node *LighthouseBeaconNode) dbPath() string {
+	return fmt.Sprintf("%s/lighthouse-beacon-node-%d", e2e.TestParams.TestPath, node.index)
+}
+
+func (node *LighthouseBeaconNode) stdoutPath() string {
+	return path.Join(e2e.TestParams.LogPath, fmt.Sprintf("lighthouse_beacon_node_%d_stdout.log", node.index))
+}
+
+func (node *LighthouseBeaconNode) stderrPath() string {
+	return path.Join(e2e.TestParams.LogPath, fmt.Sprintf("lighthouse_beacon_node_%d_stderr.log", node.index))
+}
+
+func (node *LighthouseBeaconNode) httpPort() int {
+	return e2e.TestParams.Ports.LighthouseBeaconNodeHTTPPort + node.index
+}
+
+func (node *LighthouseBeaconNode) startCommand() (string, []string, error) {
 	binaryPath, found := bazel.FindBinary("external/lighthouse", "lighthouse")
 	if !found {
 		log.Info(binaryPath)
 		log.Error("beacon chain binary not found")
 	}
-
-	_, index, _ := node.config, node.index, node.enr
-	testDir, err := node.createTestnetDir(index)
+	testDir, err := node.createTestnetDir(node.index)
 	if err != nil {
-		return err
+		return "", []string{}, err
 	}
-
 	prysmNodeCount := e2e.TestParams.BeaconNodeCount
 	jwtPath := path.Join(e2e.TestParams.TestPath, "eth1data/"+strconv.Itoa(node.index+prysmNodeCount)+"/")
 	jwtPath = path.Join(jwtPath, "geth/jwtsecret")
 	args := []string{
 		"beacon_node",
-		fmt.Sprintf("--datadir=%s/lighthouse-beacon-node-%d", e2e.TestParams.TestPath, index),
+		fmt.Sprintf("--datadir=%s", node.dbPath()),
 		fmt.Sprintf("--testnet-dir=%s", testDir),
 		"--staking",
 		"--enr-address=127.0.0.1",
-		fmt.Sprintf("--enr-udp-port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeP2PPort+index),
-		fmt.Sprintf("--enr-tcp-port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeP2PPort+index),
-		fmt.Sprintf("--port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeP2PPort+index),
-		fmt.Sprintf("--http-port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeHTTPPort+index),
+		fmt.Sprintf("--enr-udp-port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeP2PPort+node.index),
+		fmt.Sprintf("--enr-tcp-port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeP2PPort+node.index),
+		fmt.Sprintf("--port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeP2PPort+node.index),
+		fmt.Sprintf("--http-port=%d", node.httpPort()),
 		fmt.Sprintf("--target-peers=%d", 10),
-		fmt.Sprintf("--eth1-endpoints=http://127.0.0.1:%d", e2e.TestParams.Ports.Eth1RPCPort+prysmNodeCount+index),
-		fmt.Sprintf("--execution-endpoints=http://127.0.0.1:%d", e2e.TestParams.Ports.Eth1AuthRPCPort+prysmNodeCount+index),
+		fmt.Sprintf("--eth1-endpoints=http://127.0.0.1:%d", e2e.TestParams.Ports.Eth1RPCPort+prysmNodeCount+node.index),
+		fmt.Sprintf("--execution-endpoints=http://127.0.0.1:%d", e2e.TestParams.Ports.Eth1AuthRPCPort+prysmNodeCount+node.index),
 		fmt.Sprintf("--jwt-secrets=%s", jwtPath),
 		fmt.Sprintf("--boot-nodes=%s", node.enr),
-		fmt.Sprintf("--metrics-port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeMetricsPort+index),
+		fmt.Sprintf("--metrics-port=%d", e2e.TestParams.Ports.LighthouseBeaconNodeMetricsPort+node.index),
 		"--metrics",
 		"--http",
 		"--http-allow-sync-stalled",
@@ -133,13 +171,22 @@ func (node *LighthouseBeaconNode) Start(ctx context.Context) error {
 		args = append(args,
 			fmt.Sprintf("--trusted-peers=%s", flagVal))
 	}
-	cmd := exec.CommandContext(ctx, binaryPath, args...) /* #nosec G204 */
-	// Write stdout and stderr to log files.
-	stdout, err := os.Create(path.Join(e2e.TestParams.LogPath, fmt.Sprintf("lighthouse_beacon_node_%d_stdout.log", index)))
+	return binaryPath, args, nil
+}
+
+// Start starts a fresh beacon node, connecting to all passed in beacon nodes.
+func (node *LighthouseBeaconNode) Start(ctx context.Context) error {
+	binaryPath, args, err := node.startCommand()
 	if err != nil {
 		return err
 	}
-	stderr, err := os.Create(path.Join(e2e.TestParams.LogPath, fmt.Sprintf("lighthouse_beacon_node_%d_stderr.log", index)))
+	cmd := exec.CommandContext(ctx, binaryPath, args...) /* #nosec G204 */
+	// Write stdout and stderr to log files.
+	stdout, err := os.Create(node.stdoutPath())
+	if err != nil {
+		return err
+	}
+	stderr, err := os.Create(node.stderrPath())
 	if err != nil {
 		return err
 	}
@@ -153,13 +200,13 @@ func (node *LighthouseBeaconNode) Start(ctx context.Context) error {
 	}()
 	cmd.Stdout = stdout
 	cmd.Stderr = stderr
-	log.Infof("Starting lighthouse beacon chain %d with flags: %s", index, strings.Join(args[2:], " "))
+	log.Infof("Starting lighthouse beacon chain %d with flags: %s", node.index, strings.Join(args[2:], " "))
 	if err = cmd.Start(); err != nil {
 		return fmt.Errorf("failed to start beacon node: %w", err)
 	}
 
 	if err = helpers.WaitForTextInFile(stderr, "Configured for network"); err != nil {
-		return fmt.Errorf("could not find initialization for node %d, this means the node had issues starting: %w", index, err)
+		return fmt.Errorf("could not find initialization for node %d, this means the node had issues starting: %w", node.index, err)
 	}
 
 	// Mark node as ready.
@@ -176,11 +223,11 @@ func (node *LighthouseBeaconNode) Started() <-chan struct{} {
 func (node *LighthouseBeaconNode) createTestnetDir(index int) (string, error) {
 	testNetDir := e2e.TestParams.TestPath + fmt.Sprintf("/lighthouse-testnet-%d", index)
 	configPath := filepath.Join(testNetDir, "config.yaml")
-	rawYaml := params.E2EMainnetConfigYaml()
+
+	rawYaml := params.ConfigToYaml(node.config.BeaconChainConfig)
 	// Add in deposit contract in yaml
 	depContractStr := fmt.Sprintf("\nDEPOSIT_CONTRACT_ADDRESS: %#x", e2e.TestParams.ContractAddress)
 	rawYaml = append(rawYaml, []byte(depContractStr)...)
-
 	if err := file.MkdirAll(testNetDir); err != nil {
 		return "", err
 	}
@@ -196,3 +243,49 @@ func (node *LighthouseBeaconNode) createTestnetDir(index int) (string, error) {
 	deployYaml := []byte("0")
 	return testNetDir, file.WriteFile(deployPath, deployYaml)
 }
+
+func (node *LighthouseBeaconNode) ZPath() string {
+	return fmt.Sprintf("/lh-beacon-node/%d", node.index)
+}
+
+var lbnzm = template.Must(template.New("BeaconNode.ZMarkdown").Parse("" +
+	"beacon node {{.Index}}\n" +
+	"--------------\n\n" +
+	"http addr={{.HTTPAddr}}\n\n" +
+	"db path={{.DBPath}}\n\n" +
+	"stdout path={{.StdoutPath}}\n\n" +
+	"stderr path={{.StderrPath}}\n\n" +
+	"```\n" +
+	"{{.StartCmd}}" +
+	"```\n\n"))
+
+func (node *LighthouseBeaconNode) ZMarkdown() (string, error) {
+	bin, args, err := node.startCommand()
+	if err != nil {
+		return "", err
+	}
+	cmd := path.Join(bin, args[0])
+	for _, a := range args {
+		cmd += fmt.Sprintf("\n%s \\", a)
+	}
+
+	buf := bytes.NewBuffer(nil)
+	err = lbnzm.Execute(buf, struct {
+		Index      int
+		StartCmd   string
+		DBPath     string
+		StdoutPath string
+		StderrPath string
+		HTTPAddr   string
+	}{
+		Index:      node.index,
+		StartCmd:   cmd,
+		DBPath:     node.dbPath(),
+		StdoutPath: node.stdoutPath(),
+		StderrPath: node.stderrPath(),
+		HTTPAddr:   fmt.Sprintf("http://localhost:%d", node.httpPort()),
+	})
+	return buf.String(), err
+}
+
+var _ e2ez.ZPage = &LighthouseBeaconNode{}

testing/endtoend/components/validator.go

@@ -142,10 +142,11 @@ func (v *ValidatorNode) Start(ctx context.Context) error {
 		fmt.Sprintf("--%s=localhost:%d", flags.BeaconRPCProviderFlag.Name, beaconRPCPort),
 		fmt.Sprintf("--%s=%s", flags.GrpcHeadersFlag.Name, "dummy=value,foo=bar"), // Sending random headers shouldn't break anything.
 		fmt.Sprintf("--%s=%s", cmdshared.VerbosityFlag.Name, "debug"),
+		fmt.Sprintf("--%s=%s", cmdshared.ChainConfigFileFlag.Name, v.config.BeaconChainConfigPath()),
 		"--" + cmdshared.ForceClearDB.Name,
-		"--" + cmdshared.E2EConfigFlag.Name,
 		"--" + cmdshared.AcceptTosFlag.Name,
 	}
+
 	// Only apply e2e flags to the current branch. New flags may not exist in previous release.
 	if !v.config.UsePrysmShValidator {
 		args = append(args, features.E2EValidatorFlags...)

testing/endtoend/e2ez/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["server.go"],
+    importpath = "github.com/prysmaticlabs/prysm/testing/endtoend/e2ez",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_gomarkdown_markdown//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+    ],
+)

testing/endtoend/e2ez/server.go

@@ -0,0 +1,113 @@
+package e2ez
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+
+	"github.com/gomarkdown/markdown"
+	log "github.com/sirupsen/logrus"
+)
+
+// Server is an http server that serves all zpages.
+// if zpages register using the HandleMarkdown method, their responses will be transformed from markdown
+// to html before being streamed back to the client.
+type Server struct {
+	handler *http.ServeMux
+	ec      chan error
+}
+
+// NewServer should be used to instantiate a Server, so that it can set up the internal http.Handler
+// and http.Server values.
+func NewServer() *Server {
+	return &Server{
+		handler: http.NewServeMux(),
+		ec:      make(chan error),
+	}
+}
+
+// ListenAndServe just starts the underlying http.Server using the provided addr.
+// This method does not use a goroutine and will block, call it in a goroutine
+// if you do not want the caller to block.
+func (s *Server) ListenAndServe(ctx context.Context, addr string) {
+	srv := &http.Server{
+		Addr:    addr,
+		Handler: s.handler,
+	}
+	go func() {
+		if err := srv.ListenAndServe(); err != nil {
+			s.ec <- err
+		}
+	}()
+	for {
+		select {
+		case err := <-s.ec:
+			log.Error(err)
+		case <-ctx.Done():
+			err := srv.Shutdown(ctx)
+			if err != nil {
+				log.Error(err)
+			}
+		}
+	}
+}
+
+// HandleMarkdown mirrors http.HandleFunc. It wraps the given handler in a "middleware" enclosure that assumes
+// a successful response body is a markdown document, translating the markdown to an html page.
+func (s *Server) HandleMarkdown(pattern string, handler func(http.ResponseWriter, *http.Request)) {
+	s.handler.HandleFunc(pattern, handleMarkdown(handler, s.ec))
+}
+
+// HandleZPage allows any type that implements the minimal ZPage interface to
+// handle requests for information about itself.
+func (s *Server) HandleZPages(zps ...ZPage) {
+	for i := 0; i < len(zps); i++ {
+		z := zps[i]
+		f := func(rw http.ResponseWriter, req *http.Request) {
+			md, err := z.ZMarkdown()
+			if err != nil {
+				rw.WriteHeader(http.StatusInternalServerError)
+				s.ec <- err
+				return
+			}
+			rw.Header().Add("Content-Type", "text/html")
+			rw.WriteHeader(http.StatusOK)
+			_, err = rw.Write(markdown.ToHTML([]byte(md), nil, nil))
+			if err != nil {
+				s.ec <- err
+				return
+			}
+		}
+		s.handler.HandleFunc(z.ZPath(), f)
+	}
+}
+
+type markdownResponseWriter struct {
+	http.ResponseWriter
+	buf *bytes.Buffer
+}
+
+func (w *markdownResponseWriter) Write(i []byte) (int, error) {
+	return w.buf.Write(i)
+}
+
+func handleMarkdown(wrapped http.HandlerFunc, ec chan error) http.HandlerFunc {
+	return func(rw http.ResponseWriter, req *http.Request) {
+		w := &markdownResponseWriter{ResponseWriter: rw, buf: bytes.NewBuffer(nil)}
+		wrapped(w, req)
+		hb := markdown.ToHTML(w.buf.Bytes(), nil, nil)
+		_, err := rw.Write(hb)
+		if err != nil {
+			ec <- err
+			return
+		}
+	}
+}
+
+// ZPage allows a type to generate a markdown zpage without consideration for http server semantics.
+// ZPath() is used to claim a path for itself in the zpage namespace, ZMarkdown returns the markdown
+// value to translate to HTML.
+type ZPage interface {
+	ZPath() string
+	ZMarkdown() (string, error)
+}

testing/endtoend/endtoend_test.go

@@ -8,20 +8,28 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"os/signal"
 	"path"
 	"strings"
 	"sync"
+	"syscall"
 	"testing"
 	"time"
 
 	"github.com/pkg/errors"
 	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/api/client/beacon"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/transition"
 	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	"github.com/prysmaticlabs/prysm/io/file"
+	"github.com/prysmaticlabs/prysm/proto/eth/service"
+	v1 "github.com/prysmaticlabs/prysm/proto/eth/v1"
 	eth "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/testing/assert"
 	"github.com/prysmaticlabs/prysm/testing/endtoend/components"
 	"github.com/prysmaticlabs/prysm/testing/endtoend/components/eth1"
+	"github.com/prysmaticlabs/prysm/testing/endtoend/e2ez"
 	ev "github.com/prysmaticlabs/prysm/testing/endtoend/evaluators"
 	"github.com/prysmaticlabs/prysm/testing/endtoend/helpers"
 	e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params"
@@ -30,6 +38,8 @@ import (
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/emptypb"
 )
 
@@ -48,37 +58,84 @@ func init() {
 
 // testRunner abstracts E2E test configuration and running.
 type testRunner struct {
-	t      *testing.T
-	config *e2etypes.E2EConfig
+	t        *testing.T
+	config   *e2etypes.E2EConfig
+	z        *e2ez.Server
+	ctx      context.Context
+	doneChan context.CancelFunc
+	group    *errgroup.Group
 }
 
 // newTestRunner creates E2E test runner.
 func newTestRunner(t *testing.T, config *e2etypes.E2EConfig) *testRunner {
+	ctx, done := context.WithCancel(context.Background())
+	g, ctx := errgroup.WithContext(ctx)
 	return &testRunner{
-		t:      t,
-		config: config,
+		t:        t,
+		config:   config,
+		z:        e2ez.NewServer(),
+		ctx:      ctx,
+		doneChan: done,
+		group:    g,
 	}
 }
 
+type zPageMenu struct{}
+
+func (z *zPageMenu) ZPath() string {
+	return "/"
+}
+
+func (z *zPageMenu) ZMarkdown() (string, error) {
+	return `
+e2e admin
+===========
+
+- [prysm beacon nodes](/beacon-nodes)
+- [lh beacon nodes](/lh-beacon-nodes)
+`, nil
+}
+
+func (z *zPageMenu) ZChildren() []e2ez.ZPage {
+	return []e2ez.ZPage{}
+}
+
 // run executes configured E2E test.
 func (r *testRunner) run() {
 	t, config := r.t, r.config
+	err := config.WriteBeaconChainConfig()
+	if err != nil {
+		t.Fatalf("failed to write BeaconChainConfig to bazel sandbox")
+	}
+
 	t.Logf("Shard index: %d\n", e2e.TestParams.TestShardIndex)
 	t.Logf("Starting time: %s\n", time.Now().String())
 	t.Logf("Log Path: %s\n", e2e.TestParams.LogPath)
 
+	if e2e.TestParams.ZPageAddr == "" {
+		e2e.TestParams.ZPageAddr = ":8080"
+	}
+
+	// we need debug turned on and max ssz payload bumped up when running checkpoint sync teste2e.TestParams.BeaconNodeCounts
+	if config.TestSync {
+		config.BeaconFlags = appendDebugEndpoints(config.BeaconFlags)
+	}
 	minGenesisActiveCount := int(params.BeaconConfig().MinGenesisActiveValidatorCount)
 	multiClientActive := e2e.TestParams.LighthouseBeaconNodeCount > 0
 	var keyGen, lighthouseValidatorNodes e2etypes.ComponentRunner
 	var lighthouseNodes *components.LighthouseBeaconNodeSet
 
-	ctx, done := context.WithCancel(context.Background())
-	g, ctx := errgroup.WithContext(ctx)
+	ctx := r.ctx
+	g := r.group
+	done := r.doneChan
 
 	tracingSink := components.NewTracingSink(config.TracingSinkEndpoint)
 	g.Go(func() error {
 		return tracingSink.Start(ctx)
 	})
+	zp := e2ez.NewServer()
+	zp.HandleZPages(&zPageMenu{})
+	go zp.ListenAndServe(ctx, e2e.TestParams.ZPageAddr)
 
 	if multiClientActive {
 		keyGen = components.NewKeystoreGenerator()
@@ -88,6 +145,25 @@ func (r *testRunner) run() {
 			return keyGen.Start(ctx)
 		})
 	}
+	// wait on this channel if LeaveRunning is specified
+	lrChan := make(chan struct{})
+	g.Go(func() error {
+		sigc := make(chan os.Signal, 1)
+		signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM)
+		defer signal.Stop(sigc)
+		if !r.config.LeaveRunning {
+			close(lrChan)
+			return nil
+		}
+		// if LeaveRunning flag has been set, cause this goroutine to block until
+		// the context is canceled or signint/sigterm is received.
+		select {
+		case <-sigc:
+			close(lrChan)
+			log.Info("got sigint/term in LeaveRunning keepalive routine")
+			return nil
+		}
+	})
 
 	// Boot node.
 	bootNode := components.NewBootNode()
@@ -133,19 +209,6 @@ func (r *testRunner) run() {
 		return nil
 	})
 
-	// Beacon nodes.
-	beaconNodes := components.NewBeaconNodes(config)
-	g.Go(func() error {
-		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Nodes, bootNode}); err != nil {
-			return errors.Wrap(err, "beacon nodes require ETH1 and boot node to run")
-		}
-		beaconNodes.SetENR(bootNode.ENR())
-		if err := beaconNodes.Start(ctx); err != nil {
-			return errors.Wrap(err, "failed to start beacon nodes")
-		}
-		return nil
-	})
-
 	// Web3 remote signer.
 	var web3RemoteSigner *components.Web3RemoteSigner
 	if config.UseWeb3RemoteSigner {
@@ -158,13 +221,27 @@ func (r *testRunner) run() {
 		})
 	}
 
+	// Beacon nodes.
+	if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{bootNode}); err != nil {
+		t.Fatal(err, errors.Wrap(err, "beacon nodes require ETH1 and boot node to run"))
+	}
+	beaconNodes := components.NewBeaconNodes(config, bootNode.ENR(), config.BeaconFlags, zp)
+	g.Go(func() error {
+		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Nodes, bootNode}); err != nil {
+			t.Fatal(err, errors.Wrap(err, "beacon nodes require ETH1 and boot node to run"))
+		}
+		if err := beaconNodes.Start(ctx); err != nil {
+			return errors.Wrap(err, "failed to start beacon nodes")
+		}
+		return nil
+	})
+
 	if multiClientActive {
-		lighthouseNodes = components.NewLighthouseBeaconNodes(config)
+		if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Nodes, bootNode, beaconNodes}); err != nil {
+			t.Fatal(errors.Wrap(err, "lighthouse beacon nodes require ETH1 and boot node to run"))
+		}
+		lighthouseNodes = components.NewLighthouseBeaconNodes(config, bootNode.ENR(), zp)
 		g.Go(func() error {
-			if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{eth1Nodes, bootNode, beaconNodes}); err != nil {
-				return errors.Wrap(err, "lighthouse beacon nodes require ETH1 and boot node to run")
-			}
-			lighthouseNodes.SetENR(bootNode.ENR())
 			if err := lighthouseNodes.Start(ctx); err != nil {
 				return errors.Wrap(err, "failed to start lighthouse beacon nodes")
 			}
@@ -205,7 +282,12 @@ func (r *testRunner) run() {
 	g.Go(func() error {
 		// When everything is done, cancel parent context (will stop all spawned nodes).
 		defer func() {
-			log.Info("All E2E evaluations are finished, cleaning up")
+			log.Info("All E2E evaluations are finished.")
+			if config.LeaveRunning {
+				log.Info("LeaveRunning flag set, services won't shut down until ctrl+c received.")
+				return
+			}
+			log.Info("Canceling context to clean up.")
 			done()
 		}()
 
@@ -254,21 +336,29 @@ func (r *testRunner) run() {
 		require.NoError(t, err)
 		tickingStartTime := helpers.EpochTickerStartTime(genesis)
 
+		index := e2e.TestParams.BeaconNodeCount + e2e.TestParams.LighthouseBeaconNodeCount
 		// Run assigned evaluators.
 		if err := r.runEvaluators(conns, tickingStartTime); err != nil {
 			return errors.Wrap(err, "one or more evaluators failed")
 		}
-
-		// If requested, run sync test.
-		if !config.TestSync {
-			return nil
-		}
-		if err := r.testBeaconChainSync(ctx, g, conns, tickingStartTime, bootNode.ENR(), eth1Miner.ENR()); err != nil {
+		if err := r.testBeaconChainSync(ctx, g, index, conns, tickingStartTime, beaconNodes, eth1Miner.ENR()); err != nil {
 			return errors.Wrap(err, "beacon chain sync test failed")
 		}
+		index += 1
 		if err := r.testDoppelGangerProtection(ctx); err != nil {
 			return errors.Wrap(err, "doppel ganger protection check failed")
 		}
+
+		// If requested, run sync test.
+		if config.TestSync {
+			httpEndpoints := helpers.BeaconAPIHostnames(e2e.TestParams.BeaconNodeCount)
+			menr := eth1Miner.ENR()
+			if err := r.testCheckpointSync(index, conns, beaconNodes, httpEndpoints[0],  menr); err != nil {
+				return errors.Wrap(err, "checkpoint sync test failed")
+			}
+			index += 1
+		}
+
 		return nil
 	})
 
@@ -277,10 +367,21 @@ func (r *testRunner) run() {
 		if strings.Contains(err.Error(), "signal: killed") {
 			return
 		}
+		if config.LeaveRunning {
+			<-lrChan
+		}
 		t.Fatalf("E2E test ended in error: %v", err)
 	}
 }
 
+func appendDebugEndpoints(flags []string) []string {
+	debugFlags := []string{
+		"--enable-debug-rpc-endpoints",
+		"--grpc-max-msg-size=65568081",
+	}
+	return append(flags, debugFlags...)
+}
+
 // waitForChainStart allows to wait up until beacon nodes are started.
 func (r *testRunner) waitForChainStart() {
 	// Sleep depending on the count of validators, as generating the genesis state could take some time.
@@ -365,11 +466,115 @@ func (r *testRunner) testTxGeneration(ctx context.Context, g *errgroup.Group, ke
 	})
 }
 
+func (r *testRunner) waitForMatchingHead(ctx context.Context, check, ref *grpc.ClientConn) error {
+	// sleep hack copied from testBeaconChainSync
+	// Sleep a second for every 4 blocks that need to be synced for the newly started node.
+	secondsPerEpoch := uint64(params.BeaconConfig().SlotsPerEpoch.Mul(params.BeaconConfig().SecondsPerSlot))
+	extraSecondsToSync := (r.config.EpochsToRun)*secondsPerEpoch + uint64(params.BeaconConfig().SlotsPerEpoch.Div(4).Mul(r.config.EpochsToRun))
+	deadline := time.Now().Add(time.Second*time.Duration(extraSecondsToSync))
+	ctx, cancel := context.WithDeadline(ctx, deadline)
+	defer cancel()
+	checkClient := service.NewBeaconChainClient(check)
+	refClient := service.NewBeaconChainClient(ref)
+	for {
+		select {
+		case <-ctx.Done():
+			// deadline ensures that the test eventually exits when beacon node fails to sync in a resonable timeframe
+			return fmt.Errorf("deadline exceeded waiting for known good block to appear in checkpoint-synced node")
+		default:
+			cResp, err := checkClient.GetBlockRoot(ctx, &v1.BlockRequest{BlockId: []byte("head")})
+			if err != nil {
+				errStatus, ok := status.FromError(err)
+				// in the happy path we expect NotFound results until the node has synced
+				if ok && errStatus.Code() == codes.NotFound {
+					continue
+				}
+				return fmt.Errorf("error requesting head from 'check' beacon node")
+			}
+			rResp, err := refClient.GetBlockRoot(ctx, &v1.BlockRequest{BlockId: []byte("head")})
+			if err != nil {
+				return errors.Wrap(err, "unexpected error requesting head block root from 'ref' beacon node")
+			}
+			if bytesutil.ToBytes32(cResp.Data.Root) == bytesutil.ToBytes32(rResp.Data.Root) {
+				return nil
+			}
+		}
+	}
+}
+
+func (r *testRunner) testCheckpointSync(i int, conns []*grpc.ClientConn, nodes *components.BeaconNodeSet, bnAPI, minerEnr string) error {
+	ethNode := eth1.NewNode(i, minerEnr)
+	r.group.Go(func() error {
+		return ethNode.Start(r.ctx)
+	})
+	if err := helpers.ComponentsStarted(r.ctx, []e2etypes.ComponentRunner{ethNode}); err != nil {
+		return fmt.Errorf("sync beacon node not ready: %w", err)
+	}
+
+	client, err := beacon.NewClient(bnAPI)
+	if err != nil {
+		return err
+	}
+
+	od, err := beacon.DownloadOriginData(r.ctx, client)
+	if err != nil {
+		return err
+	}
+	blockPath, err := od.SaveBlock(e2e.TestParams.TestPath)
+	if err != nil {
+		return err
+	}
+	statePath, err := od.SaveState(e2e.TestParams.TestPath)
+	if err != nil {
+		return err
+	}
+	gb, err := client.GetState(r.ctx, beacon.IdGenesis)
+	if err != nil {
+		return err
+	}
+	genPath := path.Join(e2e.TestParams.TestPath, "genesis.ssz")
+	err = file.WriteFile(genPath, gb)
+	if err != nil {
+		return err
+	}
+
+	flags := append([]string{}, r.config.BeaconFlags...)
+	flags = append(flags, fmt.Sprintf("--weak-subjectivity-checkpoint=%s", od.CheckpointString()))
+	flags = append(flags, fmt.Sprintf("--checkpoint-state=%s", statePath))
+	flags = append(flags, fmt.Sprintf("--checkpoint-block=%s", blockPath))
+	flags = append(flags, fmt.Sprintf("--genesis-state=%s", genPath))
+
+	// zero-indexed, so next value would be len of list
+	cpsyncer := nodes.AddBeaconNode(i, flags)
+	r.group.Go(func() error {
+		return cpsyncer.Start(r.ctx)
+	})
+	if err := helpers.ComponentsStarted(r.ctx, []e2etypes.ComponentRunner{cpsyncer}); err != nil {
+		return fmt.Errorf("checkpoint sync beacon node not ready: %w", err)
+	}
+	c, err := grpc.Dial(fmt.Sprintf("127.0.0.1:%d", e2e.TestParams.Ports.PrysmBeaconNodeRPCPort+i), grpc.WithInsecure())
+	require.NoError(r.t, err, "Failed to dial")
+
+	// this is so that the syncEvaluators checks can run on the checkpoint sync'd node
+	conns = append(conns, c)
+	err = r.waitForMatchingHead(r.ctx, c, conns[0])
+	if err != nil {
+		return err
+	}
+
+	syncEvaluators := []e2etypes.Evaluator{ev.FinishedSyncing, ev.AllNodesHaveSameHead}
+	for _, evaluator := range syncEvaluators {
+		r.t.Run(evaluator.Name, func(t *testing.T) {
+			assert.NoError(t, evaluator.Evaluation(conns...), "Evaluation failed for sync node")
+		})
+	}
+	return nil
+}
+
 // testBeaconChainSync creates another beacon node, and tests whether it can sync to head using previous nodes.
-func (r *testRunner) testBeaconChainSync(ctx context.Context, g *errgroup.Group,
-	conns []*grpc.ClientConn, tickingStartTime time.Time, bootnodeEnr, minerEnr string) error {
-	t, config := r.t, r.config
-	index := e2e.TestParams.BeaconNodeCount + e2e.TestParams.LighthouseBeaconNodeCount
+func (r *testRunner) testBeaconChainSync(ctx context.Context, g *errgroup.Group, index int,
+	conns []*grpc.ClientConn, tickingStartTime time.Time, beaconNodes *components.BeaconNodeSet, minerEnr string) error {
+	t := r.t
 	ethNode := eth1.NewNode(index, minerEnr)
 	g.Go(func() error {
 		return ethNode.Start(ctx)
@@ -377,7 +582,9 @@ func (r *testRunner) testBeaconChainSync(ctx context.Context, g *errgroup.Group,
 	if err := helpers.ComponentsStarted(ctx, []e2etypes.ComponentRunner{ethNode}); err != nil {
 		return fmt.Errorf("sync beacon node not ready: %w", err)
 	}
-	syncBeaconNode := components.NewBeaconNode(config, index, bootnodeEnr)
+
+	//syncBeaconNode := components.NewBeaconNode(index, bootnodeEnr, r.config.BeaconFlags, config)
+	syncBeaconNode := beaconNodes.AddBeaconNode(index, r.config.BeaconFlags)
 	g.Go(func() error {
 		return syncBeaconNode.Start(ctx)
 	})
@@ -390,17 +597,17 @@ func (r *testRunner) testBeaconChainSync(ctx context.Context, g *errgroup.Group,
 
 	// Sleep a second for every 4 blocks that need to be synced for the newly started node.
 	secondsPerEpoch := uint64(params.BeaconConfig().SlotsPerEpoch.Mul(params.BeaconConfig().SecondsPerSlot))
-	extraSecondsToSync := (config.EpochsToRun)*secondsPerEpoch + uint64(params.BeaconConfig().SlotsPerEpoch.Div(4).Mul(config.EpochsToRun))
+	extraSecondsToSync := (r.config.EpochsToRun)*secondsPerEpoch + uint64(params.BeaconConfig().SlotsPerEpoch.Div(4).Mul(r.config.EpochsToRun))
 	waitForSync := tickingStartTime.Add(time.Duration(extraSecondsToSync) * time.Second)
 	time.Sleep(time.Until(waitForSync))
 
 	syncLogFile, err := os.Open(path.Join(e2e.TestParams.LogPath, fmt.Sprintf(e2e.BeaconNodeLogFileName, index)))
-	require.NoError(t, err)
-	defer helpers.LogErrorOutput(t, syncLogFile, "beacon chain node", index)
-	t.Run("sync completed", func(t *testing.T) {
+	require.NoError(r.t, err)
+	defer helpers.LogErrorOutput(r.t, syncLogFile, "beacon chain node", index)
+	r.t.Run("sync completed", func(t *testing.T) {
 		assert.NoError(t, helpers.WaitForTextInFile(syncLogFile, "Synced up to"), "Failed to sync")
 	})
-	if t.Failed() {
+	if r.t.Failed() {
 		return errors.New("cannot sync beacon node")
 	}
 
@@ -408,7 +615,7 @@ func (r *testRunner) testBeaconChainSync(ctx context.Context, g *errgroup.Group,
 	time.Sleep(time.Duration(params.BeaconConfig().SecondsPerSlot) * time.Second)
 	syncEvaluators := []e2etypes.Evaluator{ev.FinishedSyncing, ev.AllNodesHaveSameHead}
 	for _, evaluator := range syncEvaluators {
-		t.Run(evaluator.Name, func(t *testing.T) {
+		r.t.Run(evaluator.Name, func(t *testing.T) {
 			assert.NoError(t, evaluator.Evaluation(conns...), "Evaluation failed for sync node")
 		})
 	}

testing/endtoend/helpers/helpers.go

@@ -270,6 +270,15 @@ func NewLocalConnections(ctx context.Context, numConns int) ([]*grpc.ClientConn,
 	}, nil
 }
 
+func BeaconAPIHostnames(numConns int) []string {
+	hostnames := make([]string, 0)
+	for i := 0; i < numConns; i++ {
+		port := e2e.TestParams.Ports.PrysmBeaconNodeGatewayPort + i
+		hostnames = append(hostnames, fmt.Sprintf("127.0.0.1:%d", port))
+	}
+	return hostnames
+}
+
 // ComponentsStarted checks, sequentially, each provided component, blocks until all of the components are ready.
 func ComponentsStarted(ctx context.Context, comps []e2etypes.ComponentRunner) error {
 	for _, comp := range comps {

testing/endtoend/mainnet_e2e_test.go

@@ -19,7 +19,8 @@ func TestEndToEnd_MainnetConfig(t *testing.T) {
 }
 
 func e2eMainnet(t *testing.T, usePrysmSh bool) {
-	params.UseE2EMainnetConfig()
+	cfg := params.E2EMainnetTestConfig()
+	params.OverrideBeaconConfig(cfg)
 	require.NoError(t, e2eParams.InitMultiClient(e2eParams.StandardBeaconCount, e2eParams.StandardLighthouseNodeCount))
 
 	// Run for 10 epochs if not in long-running to confirm long-running has no issues.
@@ -69,7 +70,7 @@ func e2eMainnet(t *testing.T, usePrysmSh bool) {
 		},
 		ValidatorFlags:          []string{},
 		EpochsToRun:             uint64(epochsToRun),
-		TestSync:                true,
+		TestSync:                false,
 		TestFeature:             true,
 		TestDeposits:            true,
 		UseFixedPeerIDs:         true,
@@ -79,6 +80,7 @@ func e2eMainnet(t *testing.T, usePrysmSh bool) {
 		TracingSinkEndpoint:     tracingEndpoint,
 		Evaluators:              evals,
 		Seed:                    int64(seed),
+		BeaconChainConfig:       cfg,
 	}
 
 	newTestRunner(t, testConfig).run()

testing/endtoend/minimal_e2e_test.go

@@ -43,7 +43,8 @@ func TestEndToEnd_MinimalConfig_ValidatorAtCurrentRelease(t *testing.T) {
 }
 
 func e2eMinimal(t *testing.T, args *testArgs) {
-	params.UseE2EConfig()
+	cfg := params.E2ETestConfig()
+	params.OverrideBeaconConfig(cfg)
 	require.NoError(t, e2eParams.Init(e2eParams.StandardBeaconCount))
 
 	// Run for 10 epochs if not in long-running to confirm long-running has no issues.
@@ -99,7 +100,7 @@ func e2eMinimal(t *testing.T, args *testArgs) {
 		},
 		ValidatorFlags:      []string{},
 		EpochsToRun:         uint64(epochsToRun),
-		TestSync:            true,
+		TestSync:            false,
 		TestFeature:         true,
 		TestDeposits:        true,
 		UsePrysmShValidator: args.usePrysmSh,
@@ -108,6 +109,7 @@ func e2eMinimal(t *testing.T, args *testArgs) {
 		TracingSinkEndpoint: tracingEndpoint,
 		Evaluators:          evals,
 		Seed:                int64(seed),
+		BeaconChainConfig:   cfg,
 	}
 
 	newTestRunner(t, testConfig).run()

testing/endtoend/minimal_slashing_e2e_test.go

@@ -12,7 +12,8 @@ import (
 )
 
 func TestEndToEnd_Slasher_MinimalConfig(t *testing.T) {
-	params.UseE2EConfig()
+	cfg := params.E2ETestConfig()
+	params.OverrideBeaconConfig(cfg)
 	require.NoError(t, e2eParams.Init(e2eParams.StandardBeaconCount))
 
 	tracingPort := e2eParams.TestParams.Ports.JaegerTracingPort
@@ -36,6 +37,7 @@ func TestEndToEnd_Slasher_MinimalConfig(t *testing.T) {
 			ev.InjectDoubleBlockOnEpoch(2),
 		},
 		TracingSinkEndpoint: tracingEndpoint,
+		BeaconChainConfig:   cfg,
 	}
 
 	newTestRunner(t, testConfig).run()

testing/endtoend/params/params.go

@@ -22,6 +22,7 @@ type params struct {
 	LighthouseBeaconNodeCount int
 	ContractAddress           common.Address
 	Ports                     *ports
+	ZPageAddr                 string
 }
 
 type ports struct {

testing/endtoend/types/BUILD.bazel

@@ -7,6 +7,9 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/testing/endtoend/types",
     visibility = ["//testing/endtoend:__subpackages__"],
     deps = [
+        "//config/params:go_default_library",
+        "//io/file:go_default_library",
+        "//testing/endtoend/params:go_default_library",
         "@com_github_prysmaticlabs_eth2_types//:go_default_library",
         "@org_golang_google_grpc//:go_default_library",
     ],

testing/endtoend/types/types.go

@@ -4,8 +4,13 @@ package types
 
 import (
 	"context"
+	"fmt"
+	"path"
 
 	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/io/file"
+	e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params"
 	"google.golang.org/grpc"
 )
 
@@ -26,6 +31,21 @@ type E2EConfig struct {
 	BeaconFlags             []string
 	ValidatorFlags          []string
 	PeerIDs                 []string
+	BeaconChainConfig       *params.BeaconChainConfig
+	LeaveRunning bool
+}
+
+// BeaconChainConfigPath determines the canonical path to the yaml-encoded BeaconChainConfig
+// written by WriteBeaconChainConfig. Used by components to load a non-standard config in tests.
+func (cfg *E2EConfig) BeaconChainConfigPath() string {
+	fname := fmt.Sprintf("beacon-chain-config_%s.yaml", cfg.BeaconChainConfig.ConfigName)
+	return path.Join(e2e.TestParams.LogPath, fname)
+}
+
+// WriteBeaconChainConfig writes the yaml encoding of the BeaconChainConfig struct member
+// to a file at the path specified by BeaconChainConfigPath.
+func (cfg *E2EConfig) WriteBeaconChainConfig() error {
+	return file.WriteFile(cfg.BeaconChainConfigPath(), params.ConfigToYaml(cfg.BeaconChainConfig))
 }
 
 // Evaluator defines the structure of the evaluators used to
@@ -43,11 +63,3 @@ type ComponentRunner interface {
 	// Started checks whether an underlying component is started and ready to be queried.
 	Started() <-chan struct{}
 }
-
-// BeaconNodeSet defines an interface for an object that fulfills the duties
-// of a group of beacon nodes.
-type BeaconNodeSet interface {
-	ComponentRunner
-	// SetENR provides the relevant bootnode's enr to the beacon nodes.
-	SetENR(enr string)
-}

tools/bootnode/bootnode.go

@@ -192,8 +192,12 @@ func createLocalNode(privKey *ecdsa.PrivateKey, ipAddr net.IP, port int) (*enode
 		external = ipAddr
 	}
 	fVersion := params.BeaconConfig().GenesisForkVersion
-	if *forkVersion != "" {
-		fVersion, err = hex.DecodeString(*forkVersion)
+	fv := *forkVersion
+	if fv != "" {
+		if fv[0:2] == "0x" {
+			fv = fv[2:]
+		}
+		fVersion, err = hex.DecodeString(fv)
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not retrieve fork version")
 		}