Retrieving state by slot to always apply canonical block at slot, when available (#10255)

* new stategen.StateReplayer/ReplayerBuilder to give more fine-grained
  control of replaying state+block history
* all rpc/api methods updated to use the new interface, return post-state

Co-authored-by: Kasey Kirkham <kasey@users.noreply.github.com>
Co-authored-by: Radosław Kapka <rkapka@wp.pl>
Co-authored-by: terence tsao <terence@prysmaticlabs.com>

beacon-chain/core/helpers/block.go

@@ -11,18 +11,22 @@ import (
 	"github.com/prysmaticlabs/prysm/time/slots"
 )
 
+var ErrNilSignedBeaconBlock = errors.New("signed beacon block can't be nil")
+var ErrNilBeaconBlock = errors.New("beacon block can't be nil")
+var ErrNilBeaconBlockBody = errors.New("beacon block body can't be nil")
+
 // BeaconBlockIsNil checks if any composite field of input signed beacon block is nil.
 // Access to these nil fields will result in run time panic,
 // it is recommended to run these checks as first line of defense.
 func BeaconBlockIsNil(b block.SignedBeaconBlock) error {
 	if b == nil || b.IsNil() {
-		return errors.New("signed beacon block can't be nil")
+		return ErrNilSignedBeaconBlock
 	}
 	if b.Block().IsNil() {
-		return errors.New("beacon block can't be nil")
+		return ErrNilBeaconBlock
 	}
 	if b.Block().Body().IsNil() {
-		return errors.New("beacon block body can't be nil")
+		return ErrNilBeaconBlockBody
 	}
 	return nil
 }

beacon-chain/core/transition/transition_no_verify_sig.go

@@ -137,16 +137,6 @@ func CalculateStateRoot(
 	if err != nil {
 		return [32]byte{}, errors.Wrap(err, "could not process block")
 	}
-	if signed.Version() == version.Altair || signed.Version() == version.Bellatrix {
-		sa, err := signed.Block().Body().SyncAggregate()
-		if err != nil {
-			return [32]byte{}, err
-		}
-		state, err = altair.ProcessSyncAggregate(ctx, state, sa)
-		if err != nil {
-			return [32]byte{}, err
-		}
-	}
 
 	return state.HashTreeRoot(ctx)
 }
@@ -183,16 +173,6 @@ func ProcessBlockNoVerifyAnySig(
 	if err != nil {
 		return nil, nil, err
 	}
-	if signed.Version() == version.Altair || signed.Version() == version.Bellatrix {
-		sa, err := signed.Block().Body().SyncAggregate()
-		if err != nil {
-			return nil, nil, err
-		}
-		state, err = altair.ProcessSyncAggregate(ctx, state, sa)
-		if err != nil {
-			return nil, nil, err
-		}
-	}
 
 	bSet, err := b.BlockSignatureBatch(state, blk.ProposerIndex(), signed.Signature(), blk.HashTreeRoot)
 	if err != nil {
@@ -340,6 +320,19 @@ func ProcessBlockForStateRoot(
 		return nil, errors.Wrap(err, "could not process block operation")
 	}
 
+	if signed.Block().Version() == version.Phase0 {
+		return state, nil
+	}
+
+	sa, err := signed.Block().Body().SyncAggregate()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get sync aggregate from block")
+	}
+	state, err = altair.ProcessSyncAggregate(ctx, state, sa)
+	if err != nil {
+		return nil, errors.Wrap(err, "process_sync_aggregate failed")
+	}
+
 	return state, nil
 }
 

beacon-chain/db/errors.go

@@ -7,3 +7,9 @@ import "github.com/prysmaticlabs/prysm/beacon-chain/db/kv"
 // i/o error. This variable copies the value in the kv package to the same scope as the Database interfaces,
 // so that it is available to code paths that do not interact directly with the kv package.
 var ErrNotFound = kv.ErrNotFound
+
+// ErrNotFoundState wraps ErrNotFound for an error specific to a state not being found in the database.
+var ErrNotFoundState = kv.ErrNotFoundState
+
+// ErrNotFoundOriginBlockRoot wraps ErrNotFound for an error specific to the origin block root.
+var ErrNotFoundOriginBlockRoot = kv.ErrNotFoundOriginBlockRoot

beacon-chain/db/iface/interface.go

@@ -32,6 +32,7 @@ type ReadOnlyDatabase interface {
 	HighestSlotBlocksBelow(ctx context.Context, slot types.Slot) ([]block.SignedBeaconBlock, error)
 	// State related methods.
 	State(ctx context.Context, blockRoot [32]byte) (state.BeaconState, error)
+	StateOrError(ctx context.Context, blockRoot [32]byte) (state.BeaconState, error)
 	GenesisState(ctx context.Context) (state.BeaconState, error)
 	HasState(ctx context.Context, blockRoot [32]byte) bool
 	StateSummary(ctx context.Context, blockRoot [32]byte) (*ethpb.StateSummary, error)

beacon-chain/db/kv/BUILD.bazel

@@ -80,7 +80,6 @@ go_test(
         "checkpoint_test.go",
         "deposit_contract_test.go",
         "encoding_test.go",
-        "error_test.go",
         "finalized_block_roots_test.go",
         "genesis_test.go",
         "init_test.go",

beacon-chain/db/kv/error.go

@@ -1,6 +1,6 @@
 package kv
 
-import "errors"
+import "github.com/pkg/errors"
 
 // errDeleteFinalized is raised when we attempt to delete a finalized block/state
 var errDeleteFinalized = errors.New("cannot delete finalized block or state")
@@ -8,42 +8,7 @@ var errDeleteFinalized = errors.New("cannot delete finalized block or state")
 // ErrNotFound can be used directly, or as a wrapped DBError, whenever a db method needs to
 // indicate that a value couldn't be found.
 var ErrNotFound = errors.New("not found in db")
+var ErrNotFoundState = errors.Wrap(ErrNotFound, "state not found")
 
 // ErrNotFoundOriginBlockRoot is an error specifically for the origin block root getter
-var ErrNotFoundOriginBlockRoot = WrapDBError(ErrNotFound, "OriginBlockRoot")
-
-// WrapDBError wraps an error in a DBError. See commentary on DBError for more context.
-func WrapDBError(e error, outer string) error {
-	return DBError{
-		Wraps: e,
-		Outer: errors.New(outer),
-	}
-}
-
-// DBError implements the Error method so that it can be asserted as an error.
-// The Unwrap method supports error wrapping, enabling it to be used with errors.Is/As.
-// The primary use case is to make it simple for database methods to return errors
-// that wrap ErrNotFound, allowing calling code to check for "not found" errors
-// like: `error.Is(err, ErrNotFound)`. This is intended to improve error handling
-// in db lookup methods that need to differentiate between a missing value and some
-// other database error. for more background see:
-// https://go.dev/blog/go1.13-errors
-type DBError struct {
-	Wraps error
-	Outer error
-}
-
-// Error satisfies the error interface, so that DBErrors can be used anywhere that
-// expects an `error`.
-func (e DBError) Error() string {
-	es := e.Outer.Error()
-	if e.Wraps != nil {
-		es += ": " + e.Wraps.Error()
-	}
-	return es
-}
-
-// Unwrap is used by the errors package Is and As methods.
-func (e DBError) Unwrap() error {
-	return e.Wraps
-}
+var ErrNotFoundOriginBlockRoot = errors.Wrap(ErrNotFound, "OriginBlockRoot")

beacon-chain/db/kv/error_test.go

@@ -1,24 +0,0 @@
-package kv
-
-import (
-	"errors"
-	"testing"
-)
-
-func TestWrappedSentinelError(t *testing.T) {
-	e := ErrNotFoundOriginBlockRoot
-	if !errors.Is(e, ErrNotFoundOriginBlockRoot) {
-		t.Error("expected that a copy of ErrNotFoundOriginBlockRoot would have an is-a relationship")
-	}
-
-	outer := errors.New("wrapped error")
-	e2 := DBError{Wraps: ErrNotFoundOriginBlockRoot, Outer: outer}
-	if !errors.Is(e2, ErrNotFoundOriginBlockRoot) {
-		t.Error("expected that errors.Is would know DBError wraps ErrNotFoundOriginBlockRoot")
-	}
-
-	// test that the innermost not found error is detected
-	if !errors.Is(e2, ErrNotFound) {
-		t.Error("expected that errors.Is would know ErrNotFoundOriginBlockRoot wraps ErrNotFound")
-	}
-}

beacon-chain/db/kv/state.go

@@ -3,6 +3,7 @@ package kv
 import (
 	"bytes"
 	"context"
+	"fmt"
 
 	"github.com/golang/snappy"
 	"github.com/pkg/errors"
@@ -46,6 +47,19 @@ func (s *Store) State(ctx context.Context, blockRoot [32]byte) (state.BeaconStat
 	return s.unmarshalState(ctx, enc, valEntries)
 }
 
+// StateOrError is just like State(), except it only returns a non-error response
+// if the requested state is found in the database.
+func (s *Store) StateOrError(ctx context.Context, blockRoot [32]byte) (state.BeaconState, error) {
+	st, err := s.State(ctx, blockRoot)
+	if err != nil {
+		return nil, err
+	}
+	if st == nil || st.IsNil() {
+		return nil, errors.Wrap(ErrNotFoundState, fmt.Sprintf("no state with blockroot=%#x", blockRoot))
+	}
+	return st, nil
+}
+
 // GenesisState returns the genesis state in beacon chain.
 func (s *Store) GenesisState(ctx context.Context) (state.BeaconState, error) {
 	ctx, span := trace.StartSpan(ctx, "BeaconDB.GenesisState")

beacon-chain/db/kv/state_test.go

@@ -21,6 +21,12 @@ import (
 	bolt "go.etcd.io/bbolt"
 )
 
+func TestStateNil(t *testing.T) {
+	db := setupDB(t)
+	_, err := db.StateOrError(context.Background(), [32]byte{})
+	require.ErrorIs(t, err, ErrNotFoundState)
+}
+
 func TestState_CanSaveRetrieve(t *testing.T) {
 	db := setupDB(t)
 

beacon-chain/rpc/eth/beacon/sync_committee_test.go

@@ -212,6 +212,10 @@ func (m *futureSyncMockFetcher) StateRoot(context.Context, []byte) ([]byte, erro
 	return m.BeaconStateRoot, nil
 }
 
+func (m *futureSyncMockFetcher) StateBySlot(context.Context, types.Slot) (state.BeaconState, error) {
+	return m.BeaconState, nil
+}
+
 func TestListSyncCommitteesFuture(t *testing.T) {
 	ctx := context.Background()
 	st, _ := util.DeterministicGenesisStateAltair(t, params.BeaconConfig().SyncCommitteeSize)

beacon-chain/rpc/prysm/v1alpha1/beacon/BUILD.bazel

@@ -83,6 +83,7 @@ go_test(
     shard_count = 4,
     deps = [
         "//beacon-chain/blockchain/testing:go_default_library",
+        "//beacon-chain/core/altair:go_default_library",
         "//beacon-chain/core/epoch/precompute:go_default_library",
         "//beacon-chain/core/feed:go_default_library",
         "//beacon-chain/core/feed/block:go_default_library",
@@ -91,6 +92,7 @@ go_test(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/signing:go_default_library",
         "//beacon-chain/core/time:go_default_library",
+        "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/operations/attestations:go_default_library",

beacon-chain/rpc/prysm/v1alpha1/beacon/assignments.go

@@ -2,6 +2,7 @@ package beacon
 
 import (
 	"context"
+	"fmt"
 	"strconv"
 
 	types "github.com/prysmaticlabs/eth2-types"
@@ -58,9 +59,11 @@ func (bs *Server) ListValidatorAssignments(
 	if err != nil {
 		return nil, err
 	}
-	requestedState, err := bs.StateGen.StateBySlot(ctx, startSlot)
+	requestedState, err := bs.ReplayerBuilder.ForSlot(startSlot).ReplayBlocks(ctx)
 	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Could not retrieve archived state for epoch %d: %v", requestedEpoch, err)
+		msg := fmt.Sprintf("could not replay all blocks from the closest stored state (at slot %d) "+
+			"to the requested epoch (%d) - %v", startSlot, requestedEpoch, err)
+		return nil, status.Error(codes.Internal, msg)
 	}
 
 	// Filter out assignments by public keys.

beacon-chain/rpc/prysm/v1alpha1/beacon/assignments_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	dbTest "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	mockstategen "github.com/prysmaticlabs/prysm/beacon-chain/state/stategen/mock"
 	"github.com/prysmaticlabs/prysm/cmd"
 	"github.com/prysmaticlabs/prysm/config/params"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
@@ -24,13 +25,13 @@ import (
 )
 
 func TestServer_ListAssignments_CannotRequestFutureEpoch(t *testing.T) {
-
 	db := dbTest.SetupDB(t)
 	ctx := context.Background()
 	bs := &Server{
 		BeaconDB:           db,
 		GenesisTimeFetcher: &mock.ChainService{},
 	}
+	addDefaultReplayerBuilder(bs, db)
 
 	wanted := errNoEpochInfoError
 	_, err := bs.ListValidatorAssignments(
@@ -45,7 +46,6 @@ func TestServer_ListAssignments_CannotRequestFutureEpoch(t *testing.T) {
 }
 
 func TestServer_ListAssignments_NoResults(t *testing.T) {
-
 	db := dbTest.SetupDB(t)
 	ctx := context.Background()
 	st, err := util.NewBeaconState()
@@ -62,6 +62,7 @@ func TestServer_ListAssignments_NoResults(t *testing.T) {
 		BeaconDB:           db,
 		GenesisTimeFetcher: &mock.ChainService{},
 		StateGen:           stategen.New(db),
+		ReplayerBuilder:    mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(st)),
 	}
 	wanted := &ethpb.ValidatorAssignments{
 		Assignments:   make([]*ethpb.ValidatorAssignments_CommitteeAssignment, 0),
@@ -101,8 +102,8 @@ func TestServer_ListAssignments_Pagination_InputOutOfRange(t *testing.T) {
 		})
 	}
 
-	blk := util.NewBeaconBlock().Block
-	blockRoot, err := blk.HashTreeRoot()
+	blk := util.NewBeaconBlock()
+	blockRoot, err := blk.Block.HashTreeRoot()
 	require.NoError(t, err)
 
 	s, err := util.NewBeaconState()
@@ -123,6 +124,7 @@ func TestServer_ListAssignments_Pagination_InputOutOfRange(t *testing.T) {
 		},
 		GenesisTimeFetcher: &mock.ChainService{},
 		StateGen:           stategen.New(db),
+		ReplayerBuilder:    mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(s)),
 	}
 
 	wanted := fmt.Sprintf("page start %d >= list %d", 500, count)
@@ -176,8 +178,8 @@ func TestServer_ListAssignments_Pagination_DefaultPageSize_NoArchive(t *testing.
 		}
 	}
 
-	blk := util.NewBeaconBlock().Block
-	blockRoot, err := blk.HashTreeRoot()
+	b := util.NewBeaconBlock()
+	blockRoot, err := b.Block.HashTreeRoot()
 	require.NoError(t, err)
 
 	s, err := util.NewBeaconState()
@@ -198,6 +200,7 @@ func TestServer_ListAssignments_Pagination_DefaultPageSize_NoArchive(t *testing.
 		},
 		GenesisTimeFetcher: &mock.ChainService{},
 		StateGen:           stategen.New(db),
+		ReplayerBuilder:    mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(s)),
 	}
 
 	res, err := bs.ListValidatorAssignments(context.Background(), &ethpb.ListValidatorAssignmentsRequest{
@@ -246,8 +249,8 @@ func TestServer_ListAssignments_FilterPubkeysIndices_NoPagination(t *testing.T)
 		validators = append(validators, val)
 	}
 
-	blk := util.NewBeaconBlock().Block
-	blockRoot, err := blk.HashTreeRoot()
+	b := util.NewBeaconBlock()
+	blockRoot, err := b.Block.HashTreeRoot()
 	require.NoError(t, err)
 	s, err := util.NewBeaconState()
 	require.NoError(t, err)
@@ -264,6 +267,7 @@ func TestServer_ListAssignments_FilterPubkeysIndices_NoPagination(t *testing.T)
 		},
 		GenesisTimeFetcher: &mock.ChainService{},
 		StateGen:           stategen.New(db),
+		ReplayerBuilder:    mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(s)),
 	}
 
 	pubKey1 := make([]byte, params.BeaconConfig().BLSPubkeyLength)
@@ -315,13 +319,16 @@ func TestServer_ListAssignments_CanFilterPubkeysIndices_WithPagination(t *testin
 		validators = append(validators, val)
 	}
 
-	blk := util.NewBeaconBlock().Block
-	blockRoot, err := blk.HashTreeRoot()
+	b := util.NewBeaconBlock()
+	blockRoot, err := b.Block.HashTreeRoot()
 	require.NoError(t, err)
 	s, err := util.NewBeaconState()
 	require.NoError(t, err)
+	w, err := wrapper.WrappedSignedBeaconBlock(b)
+	require.NoError(t, err)
 	require.NoError(t, s.SetValidators(validators))
 	require.NoError(t, db.SaveState(ctx, s, blockRoot))
+	require.NoError(t, db.SaveBlock(ctx, w))
 	require.NoError(t, db.SaveGenesisBlockRoot(ctx, blockRoot))
 
 	bs := &Server{
@@ -335,6 +342,8 @@ func TestServer_ListAssignments_CanFilterPubkeysIndices_WithPagination(t *testin
 		StateGen:           stategen.New(db),
 	}
 
+	addDefaultReplayerBuilder(bs, db)
+
 	req := &ethpb.ListValidatorAssignmentsRequest{Indices: []types.ValidatorIndex{1, 2, 3, 4, 5, 6}, PageSize: 2, PageToken: "1"}
 	res, err := bs.ListValidatorAssignments(context.Background(), req)
 	require.NoError(t, err)

beacon-chain/rpc/prysm/v1alpha1/beacon/blocks.go

@@ -2,6 +2,7 @@ package beacon
 
 import (
 	"context"
+	"fmt"
 	"strconv"
 
 	"github.com/pkg/errors"
@@ -478,10 +479,12 @@ func (bs *Server) GetWeakSubjectivityCheckpoint(ctx context.Context, _ *emptypb.
 		return nil, status.Error(codes.Internal, "Could not get weak subjectivity slot")
 	}
 
-	wsState, err := bs.StateGen.StateBySlot(ctx, wsSlot)
+	wsState, err := bs.ReplayerBuilder.ForSlot(wsSlot).ReplayBlocks(ctx)
 	if err != nil {
-		return nil, status.Error(codes.Internal, "Could not get weak subjectivity state")
+		msg := fmt.Sprintf("error replaying blocks for state at slot %d: %v", wsSlot, err)
+		return nil, status.Error(codes.Internal, msg)
 	}
+
 	stateRoot, err := wsState.HashTreeRoot(ctx)
 	if err != nil {
 		return nil, status.Error(codes.Internal, "Could not get weak subjectivity state root")

beacon-chain/rpc/prysm/v1alpha1/beacon/blocks_test.go

@@ -12,9 +12,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	blockfeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/block"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	dbTest "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	v1 "github.com/prysmaticlabs/prysm/beacon-chain/state/v1"
 	"github.com/prysmaticlabs/prysm/cmd"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
@@ -728,77 +726,6 @@ func TestServer_StreamBlocksVerified_OnHeadUpdated(t *testing.T) {
 	<-exitRoutine
 }
 
-func TestServer_GetWeakSubjectivityCheckpoint(t *testing.T) {
-	params.SetupTestConfigCleanup(t)
-	params.OverrideBeaconConfig(params.MainnetConfig())
-
-	db := dbTest.SetupDB(t)
-	ctx := context.Background()
-
-	// Beacon state.
-	beaconState, err := util.NewBeaconState()
-	require.NoError(t, err)
-	require.NoError(t, beaconState.SetSlot(10))
-
-	// Active validator set is used for computing the weak subjectivity period.
-	numVals := 256 // Works with params.BeaconConfig().MinGenesisActiveValidatorCount as well, but takes longer.
-	validators := make([]*ethpb.Validator, numVals)
-	balances := make([]uint64, len(validators))
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{
-			PublicKey:             make([]byte, params.BeaconConfig().BLSPubkeyLength),
-			WithdrawalCredentials: make([]byte, 32),
-			EffectiveBalance:      28 * 1e9,
-			ExitEpoch:             params.BeaconConfig().FarFutureEpoch,
-		}
-		balances[i] = validators[i].EffectiveBalance
-	}
-	require.NoError(t, beaconState.SetValidators(validators))
-	require.NoError(t, beaconState.SetBalances(balances))
-
-	// Genesis block.
-	genesisBlock := util.NewBeaconBlock()
-	genesisBlockRoot, err := genesisBlock.Block.HashTreeRoot()
-	require.NoError(t, err)
-	require.NoError(t, db.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(genesisBlock)))
-	require.NoError(t, db.SaveState(ctx, beaconState, genesisBlockRoot))
-	require.NoError(t, db.SaveGenesisBlockRoot(ctx, genesisBlockRoot))
-
-	// Finalized checkpoint.
-	finalizedEpoch := types.Epoch(1020)
-	require.NoError(t, beaconState.SetSlot(types.Slot(finalizedEpoch.Mul(uint64(params.BeaconConfig().SlotsPerEpoch)))))
-	require.NoError(t, beaconState.SetCurrentJustifiedCheckpoint(&ethpb.Checkpoint{
-		Epoch: finalizedEpoch - 1,
-		Root:  bytesutil.PadTo([]byte{'A'}, fieldparams.RootLength),
-	}))
-	require.NoError(t, beaconState.SetFinalizedCheckpoint(&ethpb.Checkpoint{
-		Epoch: finalizedEpoch,
-		Root:  bytesutil.PadTo([]byte{'B'}, fieldparams.RootLength),
-	}))
-
-	chainService := &chainMock.ChainService{State: beaconState}
-	server := &Server{
-		Ctx:           ctx,
-		BlockNotifier: chainService.BlockNotifier(),
-		HeadFetcher:   chainService,
-		BeaconDB:      db,
-		StateGen:      stategen.New(db),
-	}
-
-	wsEpoch, err := helpers.ComputeWeakSubjectivityPeriod(context.Background(), beaconState)
-	require.NoError(t, err)
-
-	c, err := server.GetWeakSubjectivityCheckpoint(ctx, &emptypb.Empty{})
-	require.NoError(t, err)
-	e := finalizedEpoch - (finalizedEpoch % wsEpoch)
-	require.Equal(t, e, c.Epoch)
-	wsState, err := server.StateGen.StateBySlot(ctx, params.BeaconConfig().SlotsPerEpoch.Mul(uint64(e)))
-	require.NoError(t, err)
-	sRoot, err := wsState.HashTreeRoot(ctx)
-	require.NoError(t, err)
-	require.DeepEqual(t, sRoot[:], c.StateRoot)
-}
-
 func TestServer_ListBeaconBlocks_NoResults(t *testing.T) {
 	db := dbTest.SetupDB(t)
 	ctx := context.Background()

beacon-chain/rpc/prysm/v1alpha1/beacon/committees.go

@@ -74,9 +74,9 @@ func (bs *Server) retrieveCommitteesForEpoch(
 	if err != nil {
 		return nil, nil, err
 	}
-	requestedState, err := bs.StateGen.StateBySlot(ctx, startSlot)
+	requestedState, err := bs.ReplayerBuilder.ForSlot(startSlot).ReplayBlocks(ctx)
 	if err != nil {
-		return nil, nil, status.Errorf(codes.Internal, "Could not get state: %v", err)
+		return nil, nil, status.Errorf(codes.Internal, "error replaying blocks for state at slot %d: %v", startSlot, err)
 	}
 	seed, err := helpers.Seed(requestedState, epoch, params.BeaconConfig().DomainBeaconAttester)
 	if err != nil {

beacon-chain/rpc/prysm/v1alpha1/beacon/committees_test.go

@@ -3,6 +3,7 @@ package beacon
 import (
 	"context"
 	"encoding/binary"
+	"math"
 	"testing"
 	"time"
 
@@ -11,6 +12,7 @@ import (
 	dbTest "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	mockstategen "github.com/prysmaticlabs/prysm/beacon-chain/state/stategen/mock"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
@@ -48,6 +50,8 @@ func TestServer_ListBeaconCommittees_CurrentEpoch(t *testing.T) {
 	require.NoError(t, db.SaveGenesisBlockRoot(ctx, gRoot))
 	require.NoError(t, db.SaveState(ctx, headState, gRoot))
 
+	bs.ReplayerBuilder = mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(headState))
+
 	activeIndices, err := helpers.ActiveValidatorIndices(ctx, headState, 0)
 	require.NoError(t, err)
 	attesterSeed, err := helpers.Seed(headState, 0, params.BeaconConfig().DomainBeaconAttester)
@@ -69,6 +73,13 @@ func TestServer_ListBeaconCommittees_CurrentEpoch(t *testing.T) {
 	}
 }
 
+func addDefaultReplayerBuilder(s *Server, h stategen.HistoryAccessor) {
+	cc := &mockstategen.MockCanonicalChecker{Is: true, Err: nil}
+	cs := &mockstategen.MockCurrentSlotter{Slot: math.MaxUint64 - 1}
+	s.ReplayerBuilder = stategen.NewCanonicalBuilder(h, cc, cs)
+}
+
+// TODO: test failure
 func TestServer_ListBeaconCommittees_PreviousEpoch(t *testing.T) {
 	params.SetupTestConfigCleanup(t)
 	params.OverrideBeaconConfig(params.MainnetConfig())
@@ -87,12 +98,13 @@ func TestServer_ListBeaconCommittees_PreviousEpoch(t *testing.T) {
 	require.NoError(t, headState.SetRandaoMixes(mixes))
 	require.NoError(t, headState.SetSlot(params.BeaconConfig().SlotsPerEpoch))
 
-	b := util.NewBeaconBlock()
-	require.NoError(t, db.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
-	gRoot, err := b.Block.HashTreeRoot()
+	b, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlock())
+	require.NoError(t, wrapper.SetBlockSlot(b, headState.Slot()))
+	require.NoError(t, err)
+	require.NoError(t, db.SaveBlock(ctx, b))
+	gRoot, err := b.Block().HashTreeRoot()
 	require.NoError(t, err)
 	require.NoError(t, db.SaveState(ctx, headState, gRoot))
-	require.NoError(t, db.SaveGenesisBlockRoot(ctx, gRoot))
 
 	offset := int64(headState.Slot().Mul(params.BeaconConfig().SecondsPerSlot))
 	m := &mock.ChainService{
@@ -104,6 +116,7 @@ func TestServer_ListBeaconCommittees_PreviousEpoch(t *testing.T) {
 		GenesisTimeFetcher: m,
 		StateGen:           stategen.New(db),
 	}
+	addDefaultReplayerBuilder(bs, db)
 
 	activeIndices, err := helpers.ActiveValidatorIndices(ctx, headState, 1)
 	require.NoError(t, err)

beacon-chain/rpc/prysm/v1alpha1/beacon/server.go

@@ -45,4 +45,5 @@ type Server struct {
 	CollectedAttestationsBuffer chan []*ethpb.Attestation
 	StateGen                    stategen.StateManager
 	SyncChecker                 sync.Checker
+	ReplayerBuilder             stategen.ReplayerBuilder
 }

beacon-chain/rpc/prysm/v1alpha1/beacon/validators.go

@@ -2,6 +2,7 @@ package beacon
 
 import (
 	"context"
+	"fmt"
 	"sort"
 	"strconv"
 
@@ -64,9 +65,9 @@ func (bs *Server) ListValidatorBalances(
 	if err != nil {
 		return nil, err
 	}
-	requestedState, err := bs.StateGen.StateBySlot(ctx, startSlot)
+	requestedState, err := bs.ReplayerBuilder.ForSlot(startSlot).ReplayBlocks(ctx)
 	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Could not get state: %v", err)
+		return nil, status.Error(codes.Internal, fmt.Sprintf("error replaying blocks for state at slot %d: %v", startSlot, err))
 	}
 
 	vals := requestedState.Validators()
@@ -219,7 +220,10 @@ func (bs *Server) ListValidators(
 		if err != nil {
 			return nil, err
 		}
-		reqState, err = bs.StateGen.StateBySlot(ctx, s)
+		reqState, err = bs.ReplayerBuilder.ForSlot(s).ReplayBlocks(ctx)
+		if err != nil {
+			return nil, status.Error(codes.Internal, fmt.Sprintf("error replaying blocks for state at slot %d: %v", s, err))
+		}
 	} else {
 		reqState, err = bs.HeadFetcher.HeadState(ctx)
 	}
@@ -411,9 +415,9 @@ func (bs *Server) GetValidatorActiveSetChanges(
 	if err != nil {
 		return nil, err
 	}
-	requestedState, err := bs.StateGen.StateBySlot(ctx, s)
+	requestedState, err := bs.ReplayerBuilder.ForSlot(s).ReplayBlocks(ctx)
 	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Could not get state: %v", err)
+		return nil, status.Error(codes.Internal, fmt.Sprintf("error replaying blocks for state at slot %d: %v", s, err))
 	}
 
 	activeValidatorCount, err := helpers.ActiveValidatorCount(ctx, requestedState, coreTime.CurrentEpoch(requestedState))
@@ -493,31 +497,24 @@ func (bs *Server) GetValidatorParticipation(
 			requestedEpoch,
 		)
 	}
-
-	// Get current slot state for current epoch attestations.
-	startSlot, err := slots.EpochStart(requestedEpoch)
-	if err != nil {
-		return nil, err
-	}
 	// Use the last slot of requested epoch to obtain current and previous epoch attestations.
 	// This ensures that we don't miss previous attestations when input requested epochs.
-	startSlot += params.BeaconConfig().SlotsPerEpoch - 1
-	// The start slot should be a canonical slot.
-	canonical, err := bs.isSlotCanonical(ctx, startSlot)
+	startSlot, err := slots.EpochEnd(requestedEpoch)
 	if err != nil {
 		return nil, err
 	}
-	// Keep looking back until there's a canonical slot.
-	for i := int(startSlot - 1); !canonical && i >= 0; i-- {
-		canonical, err = bs.isSlotCanonical(ctx, types.Slot(i))
-		if err != nil {
-			return nil, err
-		}
-		startSlot = types.Slot(i)
+	// Get as close as we can to the end of the current epoch without going past the curent slot.
+	// The above check ensures a future *epoch* isn't requested, but the end slot of the requested epoch could still
+	// be past the current slot. In that case, use the current slot as the best approximation of the requested epoch.
+	// Replayer will make sure the slot ultimately used is canonical.
+	if startSlot > currentSlot {
+		startSlot = currentSlot
 	}
-	beaconState, err := bs.StateGen.StateBySlot(ctx, startSlot)
+
+	// ReplayerBuilder ensures that a canonical chain is followed to the slot
+	beaconState, err := bs.ReplayerBuilder.ForSlot(startSlot).ReplayBlocks(ctx)
 	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Could not get state: %v", err)
+		return nil, status.Error(codes.Internal, fmt.Sprintf("error replaying blocks for state at slot %d: %v", startSlot, err))
 	}
 	var v []*precompute.Validator
 	var b *precompute.Balance
@@ -834,9 +831,9 @@ func (bs *Server) GetIndividualVotes(
 	if err != nil {
 		return nil, err
 	}
-	requestedState, err := bs.StateGen.StateBySlot(ctx, s)
+	st, err := bs.ReplayerBuilder.ForSlot(s).ReplayBlocks(ctx)
 	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Could not retrieve archived state for epoch %d: %v", req.Epoch, err)
+		return nil, status.Errorf(codes.Internal, "failed to replay blocks for state at epoch %d: %v", req.Epoch, err)
 	}
 	// Track filtered validators to prevent duplication in the response.
 	filtered := map[types.ValidatorIndex]bool{}
@@ -844,7 +841,7 @@ func (bs *Server) GetIndividualVotes(
 	votes := make([]*ethpb.IndividualVotesRespond_IndividualVote, 0, len(req.Indices)+len(req.PublicKeys))
 	// Filter out assignments by public keys.
 	for _, pubKey := range req.PublicKeys {
-		index, ok := requestedState.ValidatorIndexByPubkey(bytesutil.ToBytes48(pubKey))
+		index, ok := st.ValidatorIndexByPubkey(bytesutil.ToBytes48(pubKey))
 		if !ok {
 			votes = append(votes, &ethpb.IndividualVotesRespond_IndividualVote{PublicKey: pubKey, ValidatorIndex: types.ValidatorIndex(^uint64(0))})
 			continue
@@ -864,27 +861,27 @@ func (bs *Server) GetIndividualVotes(
 
 	var v []*precompute.Validator
 	var bal *precompute.Balance
-	switch requestedState.Version() {
+	switch st.Version() {
 	case version.Phase0:
-		v, bal, err = precompute.New(ctx, requestedState)
+		v, bal, err = precompute.New(ctx, st)
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "Could not set up pre compute instance: %v", err)
 		}
-		v, _, err = precompute.ProcessAttestations(ctx, requestedState, v, bal)
+		v, _, err = precompute.ProcessAttestations(ctx, st, v, bal)
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "Could not pre compute attestations: %v", err)
 		}
 	case version.Altair:
-		v, bal, err = altair.InitializePrecomputeValidators(ctx, requestedState)
+		v, bal, err = altair.InitializePrecomputeValidators(ctx, st)
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "Could not set up altair pre compute instance: %v", err)
 		}
-		v, _, err = altair.ProcessEpochParticipation(ctx, requestedState, bal, v)
+		v, _, err = altair.ProcessEpochParticipation(ctx, st, bal, v)
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "Could not pre compute attestations: %v", err)
 		}
 	default:
-		return nil, status.Errorf(codes.Internal, "Invalid state type retrieved with a version of %d", requestedState.Version())
+		return nil, status.Errorf(codes.Internal, "Invalid state type retrieved with a version of %d", st.Version())
 	}
 
 	for _, index := range filteredIndices {
@@ -892,7 +889,7 @@ func (bs *Server) GetIndividualVotes(
 			votes = append(votes, &ethpb.IndividualVotesRespond_IndividualVote{ValidatorIndex: index})
 			continue
 		}
-		val, err := requestedState.ValidatorAtIndexReadOnly(index)
+		val, err := st.ValidatorAtIndexReadOnly(index)
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "Could not retrieve validator: %v", err)
 
@@ -923,37 +920,6 @@ func (bs *Server) GetIndividualVotes(
 	}, nil
 }
 
-// isSlotCanonical returns true if the input slot has a canonical block in the chain,
-// if the input slot has a skip block, false is returned,
-// if the input slot has more than one block, an error is returned.
-func (bs *Server) isSlotCanonical(ctx context.Context, slot types.Slot) (bool, error) {
-	if slot == 0 {
-		return true, nil
-	}
-
-	hasBlockRoots, roots, err := bs.BeaconDB.BlockRootsBySlot(ctx, slot)
-	if err != nil {
-		return false, err
-	}
-	if !hasBlockRoots {
-		return false, nil
-	}
-
-	// Loop through all roots in slot, and
-	// check which one is canonical.
-	for _, rt := range roots {
-		canonical, err := bs.CanonicalFetcher.IsCanonical(ctx, rt)
-		if err != nil {
-			return false, err
-		}
-		if canonical {
-			return true, nil
-		}
-
-	}
-	return false, nil
-}
-
 // Determines whether a validator has already exited.
 func validatorHasExited(validator *ethpb.Validator, currentEpoch types.Epoch) bool {
 	farFutureEpoch := params.BeaconConfig().FarFutureEpoch

beacon-chain/rpc/prysm/v1alpha1/beacon/validators_test.go

@@ -12,9 +12,11 @@ import (
 	types "github.com/prysmaticlabs/eth2-types"
 	"github.com/prysmaticlabs/go-bitfield"
 	mock "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/altair"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	coreTime "github.com/prysmaticlabs/prysm/beacon-chain/core/time"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/transition"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
 	dbTest "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
@@ -115,6 +117,8 @@ func TestServer_ListValidatorBalances_NoResults(t *testing.T) {
 	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, gRoot))
 	require.NoError(t, beaconDB.SaveState(ctx, headState, gRoot))
 
+	bs.ReplayerBuilder = mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(headState))
+
 	wanted := &ethpb.ValidatorBalances{
 		Balances:      make([]*ethpb.ValidatorBalances_Balance, 0),
 		TotalSize:     int32(0),
@@ -172,6 +176,7 @@ func TestServer_ListValidatorBalances_DefaultResponse_NoArchive(t *testing.T) {
 		HeadFetcher: &mock.ChainService{
 			State: st,
 		},
+		ReplayerBuilder: mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(st)),
 	}
 	res, err := bs.ListValidatorBalances(
 		ctx,
@@ -200,6 +205,7 @@ func TestServer_ListValidatorBalances_PaginationOutOfRange(t *testing.T) {
 		HeadFetcher: &mock.ChainService{
 			State: headState,
 		},
+		ReplayerBuilder: mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(headState)),
 	}
 
 	wanted := fmt.Sprintf("page start %d >= list %d", 200, len(headState.Balances()))
@@ -248,6 +254,7 @@ func TestServer_ListValidatorBalances_Pagination_Default(t *testing.T) {
 		HeadFetcher: &mock.ChainService{
 			State: headState,
 		},
+		ReplayerBuilder: mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(headState)),
 	}
 
 	tests := []struct {
@@ -331,6 +338,7 @@ func TestServer_ListValidatorBalances_Pagination_CustomPageSizes(t *testing.T) {
 		HeadFetcher: &mock.ChainService{
 			State: headState,
 		},
+		ReplayerBuilder: mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(headState)),
 	}
 
 	tests := []struct {
@@ -398,6 +406,7 @@ func TestServer_ListValidatorBalances_OutOfRange(t *testing.T) {
 		HeadFetcher: &mock.ChainService{
 			State: headState,
 		},
+		ReplayerBuilder: mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(headState)),
 	}
 
 	req := &ethpb.ListValidatorBalancesRequest{Indices: []types.ValidatorIndex{types.ValidatorIndex(1)}, QueryFilter: &ethpb.ListValidatorBalancesRequest_Epoch{Epoch: 0}}
@@ -660,6 +669,7 @@ func TestServer_ListValidatorBalances_UnknownValidatorInResponse(t *testing.T) {
 		HeadFetcher: &mock.ChainService{
 			State: headState,
 		},
+		ReplayerBuilder: mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(headState)),
 	}
 
 	nonExistentPubKey := [32]byte{8}
@@ -1009,47 +1019,41 @@ func TestServer_ListValidators_DefaultPageSize(t *testing.T) {
 }
 
 func TestServer_ListValidators_FromOldEpoch(t *testing.T) {
-	beaconDB := dbTest.SetupDB(t)
+	params.OverrideBeaconConfig(params.MainnetConfig())
+	transition.SkipSlotCache.Disable()
+
 	ctx := context.Background()
+	slot := types.Slot(0)
+	epochs := 10
+	numVals := uint64(10)
 
-	numEpochs := types.Epoch(30)
-	validators := make([]*ethpb.Validator, numEpochs)
-	for i := types.Epoch(0); i < numEpochs; i++ {
-		validators[i] = &ethpb.Validator{
-			ActivationEpoch:       i,
-			PublicKey:             make([]byte, 48),
-			WithdrawalCredentials: make([]byte, 32),
-		}
-	}
-	want := make([]*ethpb.Validators_ValidatorContainer, len(validators))
-	for i := 0; i < len(validators); i++ {
-		want[i] = &ethpb.Validators_ValidatorContainer{
-			Index:     types.ValidatorIndex(i),
-			Validator: validators[i],
-		}
-	}
-
-	st, err := util.NewBeaconState()
-	require.NoError(t, err)
-	require.NoError(t, st.SetSlot(20*params.BeaconConfig().SlotsPerEpoch))
-	require.NoError(t, st.SetValidators(validators))
 	b := util.NewBeaconBlock()
-	require.NoError(t, beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
-	gRoot, err := b.Block.HashTreeRoot()
+	b.Block.Slot = slot
+	sb, err := wrapper.WrappedSignedBeaconBlock(b)
 	require.NoError(t, err)
-	require.NoError(t, beaconDB.SaveState(ctx, st, gRoot))
-	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, gRoot))
+
+	r, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	st, _ := util.DeterministicGenesisState(t, numVals)
+	require.NoError(t, st.SetSlot(slot))
+	require.Equal(t, int(numVals), len(st.Validators()))
+
+	beaconDB := dbTest.SetupDB(t)
+	require.NoError(t, beaconDB.SaveBlock(ctx, sb))
+	require.NoError(t, beaconDB.SaveState(ctx, st, r))
+	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, r))
+
 	secondsPerEpoch := params.BeaconConfig().SecondsPerSlot * uint64(params.BeaconConfig().SlotsPerEpoch)
 	bs := &Server{
 		HeadFetcher: &mock.ChainService{
 			State: st,
 		},
 		GenesisTimeFetcher: &mock.ChainService{
-			// We are in epoch 30
-			Genesis: time.Now().Add(time.Duration(-1*int64(30*secondsPerEpoch)) * time.Second),
+			Genesis: time.Now().Add(time.Duration(-1*int64(uint64(epochs)*secondsPerEpoch)) * time.Second),
 		},
-		StateGen: stategen.New(beaconDB),
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
 	req := &ethpb.ListValidatorsRequest{
 		QueryFilter: &ethpb.ListValidatorsRequest_Genesis{
@@ -1058,16 +1062,26 @@ func TestServer_ListValidators_FromOldEpoch(t *testing.T) {
 	}
 	res, err := bs.ListValidators(context.Background(), req)
 	require.NoError(t, err)
-	assert.Equal(t, 30, len(res.ValidatorList))
+	assert.Equal(t, epochs, len(res.ValidatorList))
 
+	vals := st.Validators()
+	want := make([]*ethpb.Validators_ValidatorContainer, 0)
+	for i, v := range vals {
+		want = append(want, &ethpb.Validators_ValidatorContainer{
+			Index:     types.ValidatorIndex(i),
+			Validator: v,
+		})
+	}
 	req = &ethpb.ListValidatorsRequest{
 		QueryFilter: &ethpb.ListValidatorsRequest_Epoch{
-			Epoch: 20,
+			Epoch: 10,
 		},
 	}
 	res, err = bs.ListValidators(context.Background(), req)
 	require.NoError(t, err)
-	assert.DeepSSZEqual(t, want, res.ValidatorList, "Incorrect number of validators")
+
+	require.Equal(t, len(want), len(res.ValidatorList), "incorrect number of validators")
+	assert.DeepSSZEqual(t, want, res.ValidatorList, "mismatch in validator values")
 }
 
 func TestServer_ListValidators_ProcessHeadStateSlots(t *testing.T) {
@@ -1270,8 +1284,8 @@ func TestServer_GetValidatorActiveSetChanges(t *testing.T) {
 			FinalizedCheckPoint: &ethpb.Checkpoint{Epoch: 0, Root: make([]byte, fieldparams.RootLength)},
 		},
 		GenesisTimeFetcher: &mock.ChainService{},
-		StateGen:           stategen.New(beaconDB),
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 	res, err := bs.GetValidatorActiveSetChanges(ctx, &ethpb.GetValidatorActiveSetChangesRequest{
 		QueryFilter: &ethpb.GetValidatorActiveSetChangesRequest_Genesis{Genesis: true},
 	})
@@ -1489,43 +1503,6 @@ func TestServer_GetValidatorParticipation_CannotRequestFutureEpoch(t *testing.T)
 	assert.ErrorContains(t, wanted, err)
 }
 
-func TestServer_GetValidatorParticipation_UnknownState(t *testing.T) {
-	beaconDB := dbTest.SetupDB(t)
-
-	ctx := context.Background()
-	headState, err := util.NewBeaconState()
-	require.NoError(t, err)
-	require.NoError(t, headState.SetSlot(0))
-	epoch := types.Epoch(50)
-	slots := params.BeaconConfig().SlotsPerEpoch.Mul(uint64(epoch))
-	mockStateGen := &mockstategen.MockStateManager{
-		StatesBySlot: map[types.Slot]state.BeaconState{
-			0: (*v1.BeaconState)(nil),
-		},
-	}
-	bs := &Server{
-		BeaconDB: beaconDB,
-		HeadFetcher: &mock.ChainService{
-			State: headState,
-		},
-		GenesisTimeFetcher: &mock.ChainService{
-			Genesis: time.Now().Add(time.Duration(-1*int64(slots)) * time.Second),
-		},
-		StateGen: mockStateGen,
-	}
-
-	wanted := "Could not set up pre compute instance: failed to initialize precompute: nil inner state"
-	_, err = bs.GetValidatorParticipation(
-		ctx,
-		&ethpb.GetValidatorParticipationRequest{
-			QueryFilter: &ethpb.GetValidatorParticipationRequest_Epoch{
-				Epoch: 1,
-			},
-		},
-	)
-	assert.ErrorContains(t, wanted, err)
-}
-
 func TestServer_GetValidatorParticipation_CurrentAndPrevEpoch(t *testing.T) {
 	helpers.ClearCache()
 	beaconDB := dbTest.SetupDB(t)
@@ -1552,7 +1529,7 @@ func TestServer_GetValidatorParticipation_CurrentAndPrevEpoch(t *testing.T) {
 	}}
 	headState, err := util.NewBeaconState()
 	require.NoError(t, err)
-	require.NoError(t, headState.SetSlot(2*params.BeaconConfig().SlotsPerEpoch-1))
+	require.NoError(t, headState.SetSlot(16))
 	require.NoError(t, headState.SetValidators(validators))
 	require.NoError(t, headState.SetBalances(balances))
 	require.NoError(t, headState.AppendCurrentEpochAttestations(atts[0]))
@@ -1567,6 +1544,7 @@ func TestServer_GetValidatorParticipation_CurrentAndPrevEpoch(t *testing.T) {
 	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, bRoot))
 	require.NoError(t, err)
 	require.NoError(t, beaconDB.SaveState(ctx, headState, bRoot))
+	require.NoError(t, beaconDB.SaveState(ctx, headState, params.BeaconConfig().ZeroHash))
 
 	m := &mock.ChainService{State: headState}
 	offset := int64(params.BeaconConfig().SlotsPerEpoch.Mul(params.BeaconConfig().SecondsPerSlot))
@@ -1584,6 +1562,7 @@ func TestServer_GetValidatorParticipation_CurrentAndPrevEpoch(t *testing.T) {
 		},
 		FinalizationFetcher: &mock.ChainService{FinalizedCheckPoint: &ethpb.Checkpoint{Epoch: 100}},
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
 	res, err := bs.GetValidatorParticipation(ctx, &ethpb.GetValidatorParticipationRequest{QueryFilter: &ethpb.GetValidatorParticipationRequest_Epoch{Epoch: 1}})
 	require.NoError(t, err)
@@ -1632,7 +1611,7 @@ func TestServer_GetValidatorParticipation_OrphanedUntilGenesis(t *testing.T) {
 	}}
 	headState, err := util.NewBeaconState()
 	require.NoError(t, err)
-	require.NoError(t, headState.SetSlot(2*params.BeaconConfig().SlotsPerEpoch-1))
+	require.NoError(t, headState.SetSlot(0))
 	require.NoError(t, headState.SetValidators(validators))
 	require.NoError(t, headState.SetBalances(balances))
 	require.NoError(t, headState.AppendCurrentEpochAttestations(atts[0]))
@@ -1641,11 +1620,10 @@ func TestServer_GetValidatorParticipation_OrphanedUntilGenesis(t *testing.T) {
 	b := util.NewBeaconBlock()
 	require.NoError(t, beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
 	bRoot, err := b.Block.HashTreeRoot()
-	require.NoError(t, beaconDB.SaveStateSummary(ctx, &ethpb.StateSummary{Root: bRoot[:]}))
-	require.NoError(t, beaconDB.SaveStateSummary(ctx, &ethpb.StateSummary{Root: params.BeaconConfig().ZeroHash[:]}))
 	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, bRoot))
 	require.NoError(t, err)
 	require.NoError(t, beaconDB.SaveState(ctx, headState, bRoot))
+	require.NoError(t, beaconDB.SaveState(ctx, headState, params.BeaconConfig().ZeroHash))
 
 	m := &mock.ChainService{State: headState}
 	offset := int64(params.BeaconConfig().SlotsPerEpoch.Mul(params.BeaconConfig().SecondsPerSlot))
@@ -1663,6 +1641,7 @@ func TestServer_GetValidatorParticipation_OrphanedUntilGenesis(t *testing.T) {
 		},
 		FinalizationFetcher: &mock.ChainService{FinalizedCheckPoint: &ethpb.Checkpoint{Epoch: 100}},
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
 	res, err := bs.GetValidatorParticipation(ctx, &ethpb.GetValidatorParticipationRequest{QueryFilter: &ethpb.GetValidatorParticipationRequest_Epoch{Epoch: 1}})
 	require.NoError(t, err)
@@ -1688,32 +1667,36 @@ func TestServer_GetValidatorParticipation_CurrentAndPrevEpochAltair(t *testing.T
 	beaconDB := dbTest.SetupDB(t)
 	params.SetupTestConfigCleanup(t)
 	params.OverrideBeaconConfig(params.MainnetConfig())
+	transition.SkipSlotCache.Disable()
 
 	ctx := context.Background()
 	validatorCount := uint64(32)
 
+	genState, _ := util.DeterministicGenesisStateAltair(t, validatorCount)
+	c, err := altair.NextSyncCommittee(ctx, genState)
+	require.NoError(t, err)
+	require.NoError(t, genState.SetCurrentSyncCommittee(c))
+
 	bits := make([]byte, validatorCount)
 	for i := range bits {
 		bits[i] = 0xff
 	}
-	headState, _ := util.DeterministicGenesisStateAltair(t, validatorCount)
-	require.NoError(t, headState.SetSlot(2*params.BeaconConfig().SlotsPerEpoch-1))
-	require.NoError(t, headState.SetCurrentParticipationBits(bits))
-	require.NoError(t, headState.SetPreviousParticipationBits(bits))
+	require.NoError(t, genState.SetCurrentParticipationBits(bits))
+	require.NoError(t, genState.SetPreviousParticipationBits(bits))
 
-	b := util.NewBeaconBlockAltair()
-	b.Block.Slot = 16
-	ab, err := wrapper.WrappedAltairSignedBeaconBlock(b)
+	gsr, err := genState.HashTreeRoot(ctx)
 	require.NoError(t, err)
-	require.NoError(t, beaconDB.SaveBlock(ctx, ab))
-	bRoot, err := b.Block.HashTreeRoot()
-	require.NoError(t, beaconDB.SaveStateSummary(ctx, &ethpb.StateSummary{Root: bRoot[:]}))
-	require.NoError(t, beaconDB.SaveStateSummary(ctx, &ethpb.StateSummary{Root: params.BeaconConfig().ZeroHash[:]}))
-	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, bRoot))
+	gb, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlockAltair())
+	require.NoError(t, wrapper.SetBlockStateRoot(gb, gsr))
+	require.NoError(t, err)
+	gRoot, err := gb.Block().HashTreeRoot()
 	require.NoError(t, err)
-	require.NoError(t, beaconDB.SaveState(ctx, headState, bRoot))
 
-	m := &mock.ChainService{State: headState}
+	require.NoError(t, beaconDB.SaveState(ctx, genState, gRoot))
+	require.NoError(t, beaconDB.SaveBlock(ctx, gb))
+	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, gRoot))
+
+	m := &mock.ChainService{State: genState}
 	offset := int64(params.BeaconConfig().SlotsPerEpoch.Mul(params.BeaconConfig().SecondsPerSlot))
 	bs := &Server{
 		BeaconDB:    beaconDB,
@@ -1722,15 +1705,11 @@ func TestServer_GetValidatorParticipation_CurrentAndPrevEpochAltair(t *testing.T
 		GenesisTimeFetcher: &mock.ChainService{
 			Genesis: prysmTime.Now().Add(time.Duration(-1*offset) * time.Second),
 		},
-		CanonicalFetcher: &mock.ChainService{
-			CanonicalRoots: map[[32]byte]bool{
-				bRoot: true,
-			},
-		},
 		FinalizationFetcher: &mock.ChainService{FinalizedCheckPoint: &ethpb.Checkpoint{Epoch: 100}},
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
-	res, err := bs.GetValidatorParticipation(ctx, &ethpb.GetValidatorParticipationRequest{QueryFilter: &ethpb.GetValidatorParticipationRequest_Epoch{Epoch: 1}})
+	res, err := bs.GetValidatorParticipation(ctx, &ethpb.GetValidatorParticipationRequest{QueryFilter: &ethpb.GetValidatorParticipationRequest_Epoch{Epoch: 0}})
 	require.NoError(t, err)
 
 	wanted := &ethpb.ValidatorParticipation{
@@ -1747,6 +1726,24 @@ func TestServer_GetValidatorParticipation_CurrentAndPrevEpochAltair(t *testing.T
 	}
 	assert.DeepEqual(t, true, res.Finalized, "Incorrect validator participation respond")
 	assert.DeepEqual(t, wanted, res.Participation, "Incorrect validator participation respond")
+
+	res, err = bs.GetValidatorParticipation(ctx, &ethpb.GetValidatorParticipationRequest{QueryFilter: &ethpb.GetValidatorParticipationRequest_Epoch{Epoch: 1}})
+	require.NoError(t, err)
+
+	wanted = &ethpb.ValidatorParticipation{
+		GlobalParticipationRate:          1,
+		VotedEther:                       validatorCount * params.BeaconConfig().MaxEffectiveBalance,
+		EligibleEther:                    validatorCount * params.BeaconConfig().MaxEffectiveBalance,
+		CurrentEpochActiveGwei:           validatorCount * params.BeaconConfig().MaxEffectiveBalance,
+		CurrentEpochAttestingGwei:        params.BeaconConfig().EffectiveBalanceIncrement, // Empty because after one epoch, current participation rotates to previous
+		CurrentEpochTargetAttestingGwei:  params.BeaconConfig().EffectiveBalanceIncrement,
+		PreviousEpochActiveGwei:          validatorCount * params.BeaconConfig().MaxEffectiveBalance,
+		PreviousEpochAttestingGwei:       validatorCount * params.BeaconConfig().MaxEffectiveBalance,
+		PreviousEpochTargetAttestingGwei: validatorCount * params.BeaconConfig().MaxEffectiveBalance,
+		PreviousEpochHeadAttestingGwei:   validatorCount * params.BeaconConfig().MaxEffectiveBalance,
+	}
+	assert.DeepEqual(t, true, res.Finalized, "Incorrect validator participation respond")
+	assert.DeepEqual(t, wanted, res.Participation, "Incorrect validator participation respond")
 }
 
 func TestGetValidatorPerformance_Syncing(t *testing.T) {
@@ -2070,6 +2067,7 @@ func BenchmarkListValidatorBalances(b *testing.B) {
 			State: headState,
 		},
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
 	req := &ethpb.ListValidatorBalancesRequest{PageSize: 100}
 	b.StartTimer()
@@ -2114,15 +2112,16 @@ func TestServer_GetIndividualVotes_ValidatorsDontExist(t *testing.T) {
 	beaconDB := dbTest.SetupDB(t)
 	ctx := context.Background()
 
+	var slot types.Slot = 0
 	validators := uint64(64)
 	stateWithValidators, _ := util.DeterministicGenesisState(t, validators)
 	beaconState, err := util.NewBeaconState()
 	require.NoError(t, err)
 	require.NoError(t, beaconState.SetValidators(stateWithValidators.Validators()))
-	require.NoError(t, beaconState.SetSlot(params.BeaconConfig().SlotsPerEpoch))
+	require.NoError(t, beaconState.SetSlot(slot))
 
 	b := util.NewBeaconBlock()
-	b.Block.Slot = params.BeaconConfig().SlotsPerEpoch
+	b.Block.Slot = slot
 	require.NoError(t, beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
 	gRoot, err := b.Block.HashTreeRoot()
 	require.NoError(t, err)
@@ -2134,6 +2133,7 @@ func TestServer_GetIndividualVotes_ValidatorsDontExist(t *testing.T) {
 		StateGen:           gen,
 		GenesisTimeFetcher: &mock.ChainService{},
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
 	// Test non exist public key.
 	res, err := bs.GetIndividualVotes(ctx, &ethpb.IndividualVotesRequest{
@@ -2192,7 +2192,6 @@ func TestServer_GetIndividualVotes_Working(t *testing.T) {
 	beaconState, err := util.NewBeaconState()
 	require.NoError(t, err)
 	require.NoError(t, beaconState.SetValidators(stateWithValidators.Validators()))
-	require.NoError(t, beaconState.SetSlot(params.BeaconConfig().SlotsPerEpoch))
 
 	bf := bitfield.NewBitlist(validators / uint64(params.BeaconConfig().SlotsPerEpoch))
 	att1 := util.NewAttestation()
@@ -2218,7 +2217,7 @@ func TestServer_GetIndividualVotes_Working(t *testing.T) {
 	require.NoError(t, err)
 
 	b := util.NewBeaconBlock()
-	b.Block.Slot = params.BeaconConfig().SlotsPerEpoch
+	b.Block.Slot = 0
 	require.NoError(t, beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
 	gRoot, err := b.Block.HashTreeRoot()
 	require.NoError(t, err)
@@ -2230,6 +2229,7 @@ func TestServer_GetIndividualVotes_Working(t *testing.T) {
 		StateGen:           gen,
 		GenesisTimeFetcher: &mock.ChainService{},
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
 	res, err := bs.GetIndividualVotes(ctx, &ethpb.IndividualVotesRequest{
 		Indices: []types.ValidatorIndex{0, 1},
@@ -2268,9 +2268,10 @@ func TestServer_GetIndividualVotes_WorkingAltair(t *testing.T) {
 	beaconDB := dbTest.SetupDB(t)
 	ctx := context.Background()
 
+	var slot types.Slot = 0
 	validators := uint64(32)
 	beaconState, _ := util.DeterministicGenesisStateAltair(t, validators)
-	require.NoError(t, beaconState.SetSlot(params.BeaconConfig().SlotsPerEpoch))
+	require.NoError(t, beaconState.SetSlot(slot))
 
 	pb, err := beaconState.CurrentEpochParticipation()
 	require.NoError(t, err)
@@ -2281,7 +2282,7 @@ func TestServer_GetIndividualVotes_WorkingAltair(t *testing.T) {
 	require.NoError(t, beaconState.SetPreviousParticipationBits(pb))
 
 	b := util.NewBeaconBlock()
-	b.Block.Slot = params.BeaconConfig().SlotsPerEpoch
+	b.Block.Slot = slot
 	require.NoError(t, beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
 	gRoot, err := b.Block.HashTreeRoot()
 	require.NoError(t, err)
@@ -2293,6 +2294,7 @@ func TestServer_GetIndividualVotes_WorkingAltair(t *testing.T) {
 		StateGen:           gen,
 		GenesisTimeFetcher: &mock.ChainService{},
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
 	res, err := bs.GetIndividualVotes(ctx, &ethpb.IndividualVotesRequest{
 		Indices: []types.ValidatorIndex{0, 1},
@@ -2378,6 +2380,7 @@ func TestServer_GetIndividualVotes_AltairEndOfEpoch(t *testing.T) {
 		StateGen:           gen,
 		GenesisTimeFetcher: &mock.ChainService{},
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
 	res, err := bs.GetIndividualVotes(ctx, &ethpb.IndividualVotesRequest{
 		Indices: []types.ValidatorIndex{0, 1},
@@ -2499,88 +2502,3 @@ func Test_validatorStatus(t *testing.T) {
 		})
 	}
 }
-
-func TestServer_isSlotCanonical(t *testing.T) {
-	beaconDB := dbTest.SetupDB(t)
-	ctx := context.Background()
-	var roots [][32]byte
-	cRoots := map[[32]byte]bool{}
-	for i := 1; i < 100; i++ {
-		b := util.NewBeaconBlock()
-		b.Block.Slot = types.Slot(i)
-		require.NoError(t, beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
-		br, err := b.Block.HashTreeRoot()
-		require.NoError(t, err)
-		if i%2 == 0 {
-			cRoots[br] = true
-		}
-		roots = append(roots, br)
-	}
-
-	bs := &Server{
-		BeaconDB: beaconDB,
-		CanonicalFetcher: &mock.ChainService{
-			CanonicalRoots: cRoots,
-		},
-	}
-
-	for i := range roots {
-		slot := types.Slot(i + 1)
-		c, err := bs.isSlotCanonical(ctx, slot)
-		require.NoError(t, err)
-		if slot%2 == 0 {
-			require.Equal(t, true, c)
-		} else {
-			require.Equal(t, false, c)
-		}
-	}
-}
-
-func TestServer_isSlotCanonical_MultipleBlocks(t *testing.T) {
-	beaconDB := dbTest.SetupDB(t)
-	ctx := context.Background()
-	var roots [][32]byte
-	cRoots := map[[32]byte]bool{}
-	for i := 1; i < 100; i++ {
-		b := util.NewBeaconBlock()
-		b.Block.Slot = types.Slot(i)
-		require.NoError(t, beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
-		br, err := b.Block.HashTreeRoot()
-		require.NoError(t, err)
-		if i%2 == 0 {
-			cRoots[br] = true
-			// Save a block in the same slot
-			b = util.NewBeaconBlock()
-			b.Block.Slot = types.Slot(i)
-			b.Block.ProposerIndex = 100
-			require.NoError(t, beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
-		}
-		roots = append(roots, br)
-	}
-
-	bs := &Server{
-		BeaconDB: beaconDB,
-		CanonicalFetcher: &mock.ChainService{
-			CanonicalRoots: cRoots,
-		},
-	}
-
-	for i := range roots {
-		slot := types.Slot(i + 1)
-		c, err := bs.isSlotCanonical(ctx, slot)
-		require.NoError(t, err)
-		if slot%2 == 0 {
-			require.Equal(t, true, c)
-		} else {
-			require.Equal(t, false, c)
-		}
-	}
-}
-
-func TestServer_isSlotCanonicalForSlot0(t *testing.T) {
-	ctx := context.Background()
-	bs := &Server{}
-	c, err := bs.isSlotCanonical(ctx, 0)
-	require.NoError(t, err)
-	require.Equal(t, true, c)
-}

beacon-chain/rpc/prysm/v1alpha1/debug/BUILD.bazel

@@ -50,7 +50,9 @@ go_test(
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/forkchoice/protoarray:go_default_library",
         "//beacon-chain/p2p/testing:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/stategen:go_default_library",
+        "//beacon-chain/state/stategen/mock:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",

beacon-chain/rpc/prysm/v1alpha1/debug/server.go

@@ -30,6 +30,7 @@ type Server struct {
 	HeadFetcher        blockchain.HeadFetcher
 	PeerManager        p2p.PeerManager
 	PeersFetcher       p2p.PeersProvider
+	ReplayerBuilder    stategen.ReplayerBuilder
 }
 
 // SetLoggingLevel of a beacon node according to a request type,

beacon-chain/rpc/prysm/v1alpha1/debug/state.go

@@ -2,6 +2,7 @@ package debug
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 	pbrpc "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
@@ -28,10 +29,11 @@ func (ds *Server) GetBeaconState(
 			)
 		}
 
-		st, err := ds.StateGen.StateBySlot(ctx, q.Slot)
+		st, err := ds.ReplayerBuilder.ForSlot(q.Slot).ReplayBlocks(ctx)
 		if err != nil {
-			return nil, status.Errorf(codes.Internal, "Could not compute state by slot: %v", err)
+			return nil, status.Error(codes.Internal, fmt.Sprintf("error replaying blocks for state at slot %d: %v", q.Slot, err))
 		}
+
 		encoded, err := st.MarshalSSZ()
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "Could not ssz encode beacon state: %v", err)

beacon-chain/rpc/prysm/v1alpha1/debug/state_test.go

@@ -2,12 +2,15 @@ package debug
 
 import (
 	"context"
+	"math"
 	"testing"
 
 	types "github.com/prysmaticlabs/eth2-types"
 	mock "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
 	dbTest "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	mockstategen "github.com/prysmaticlabs/prysm/beacon-chain/state/stategen/mock"
 	pbrpc "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
 	"github.com/prysmaticlabs/prysm/testing/assert"
@@ -15,6 +18,16 @@ import (
 	"github.com/prysmaticlabs/prysm/testing/util"
 )
 
+func addReplayerBuilder(s *Server, h stategen.HistoryAccessor, is bool, canonErr error, currSlot types.Slot) {
+	cc := &mockstategen.MockCanonicalChecker{Is: is, Err: canonErr}
+	cs := &mockstategen.MockCurrentSlotter{Slot: currSlot}
+	s.ReplayerBuilder = stategen.NewCanonicalBuilder(h, cc, cs)
+}
+
+func addDefaultReplayerBuilder(s *Server, h stategen.HistoryAccessor) {
+	addReplayerBuilder(s, h, true, nil, math.MaxUint64-1)
+}
+
 func TestServer_GetBeaconState(t *testing.T) {
 	db := dbTest.SetupDB(t)
 	ctx := context.Background()
@@ -34,6 +47,7 @@ func TestServer_GetBeaconState(t *testing.T) {
 		StateGen:           gen,
 		GenesisTimeFetcher: &mock.ChainService{},
 	}
+	addDefaultReplayerBuilder(bs, db)
 	_, err = bs.GetBeaconState(ctx, &pbrpc.BeaconStateRequest{})
 	assert.ErrorContains(t, "Need to specify either a block root or slot to request state", err)
 	req := &pbrpc.BeaconStateRequest{
@@ -46,16 +60,44 @@ func TestServer_GetBeaconState(t *testing.T) {
 	wanted, err := st.MarshalSSZ()
 	require.NoError(t, err)
 	assert.DeepEqual(t, wanted, res.Encoded)
+
+	req = &pbrpc.BeaconStateRequest{
+		QueryFilter: &pbrpc.BeaconStateRequest_Slot{
+			Slot: st.Slot(),
+		},
+	}
+	wanted, err = st.MarshalSSZ()
+	require.NoError(t, err)
+	res, err = bs.GetBeaconState(ctx, req)
+	require.NoError(t, err)
+	resState := &pbrpc.BeaconState{}
+	err = resState.UnmarshalSSZ(res.Encoded)
+	require.NoError(t, err)
+	assert.Equal(t, resState.Slot, st.Slot())
+	assert.DeepEqual(t, wanted, res.Encoded)
+
+	// request a slot after the state
+	// note that if the current slot were <= slot+1, this would fail
+	// but the mock stategen.CurrentSlotter gives a current slot far in the future
+	// so this acts like requesting a state at a skipped slot
 	req = &pbrpc.BeaconStateRequest{
 		QueryFilter: &pbrpc.BeaconStateRequest_Slot{
 			Slot: slot + 1,
 		},
 	}
-	require.NoError(t, st.SetSlot(slot+1))
-	wanted, err = st.MarshalSSZ()
+	state := state.BeaconState(st)
+	// since we are requesting a state at a skipped slot, use the same method as stategen
+	// to advance to the pre-state for the subsequent slot
+	state, err = stategen.ReplayProcessSlots(ctx, state, slot+1)
+	require.NoError(t, err)
+	wanted, err = state.MarshalSSZ()
 	require.NoError(t, err)
 	res, err = bs.GetBeaconState(ctx, req)
 	require.NoError(t, err)
+	resState = &pbrpc.BeaconState{}
+	err = resState.UnmarshalSSZ(res.Encoded)
+	require.NoError(t, err)
+	assert.Equal(t, resState.Slot, state.Slot())
 	assert.DeepEqual(t, wanted, res.Encoded)
 }
 

beacon-chain/rpc/prysm/v1alpha1/validator/server.go

@@ -64,6 +64,7 @@ type Server struct {
 	PendingDepositsFetcher depositcache.PendingDepositsFetcher
 	OperationNotifier      opfeed.Notifier
 	StateGen               stategen.StateManager
+	ReplayerBuilder        stategen.ReplayerBuilder
 	BeaconDB               db.HeadAccessDatabase
 	ExecutionEngineCaller  enginev1.Caller
 }

beacon-chain/rpc/prysm/v1alpha1/validator/status.go

@@ -8,7 +8,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/signing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/time"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/transition"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
@@ -362,11 +361,8 @@ func (vs *Server) retrieveAfterEpochTransition(ctx context.Context, epoch types.
 	if err != nil {
 		return nil, err
 	}
-	retState, err := vs.StateGen.StateBySlot(ctx, endSlot)
-	if err != nil {
-		return nil, err
-	}
-	return transition.ProcessSlots(ctx, retState, retState.Slot()+1)
+	// replay to first slot of following epoch
+	return vs.ReplayerBuilder.ForSlot(endSlot).ReplayToSlot(ctx, endSlot+1)
 }
 
 func checkValidatorsAreRecent(headEpoch types.Epoch, req *ethpb.DoppelGangerRequest) (bool, *ethpb.DoppelGangerResponse) {

beacon-chain/rpc/prysm/v1alpha1/validator/status_test.go

@@ -961,8 +961,7 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 			name:    "normal doppelganger request",
 			wantErr: false,
 			svSetup: func(t *testing.T) (*Server, *ethpb.DoppelGangerRequest, *ethpb.DoppelGangerResponse) {
-				mockGen := mockstategen.NewMockService()
-				hs, ps, os, keys := createStateSetup(t, 4, mockGen)
+				hs, ps, os, keys, builder := createStateSetup(t, 4)
 				// Previous Epoch State
 				for i := 0; i < 3; i++ {
 					bal, err := ps.BalanceAtIndex(types.ValidatorIndex(i))
@@ -978,11 +977,11 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 					assert.NoError(t, os.UpdateBalancesAtIndex(types.ValidatorIndex(i), bal+2000000000))
 				}
 				vs := &Server{
-					StateGen: mockGen,
 					HeadFetcher: &mockChain.ChainService{
 						State: hs,
 					},
-					SyncChecker: &mockSync.Sync{IsSyncing: false},
+					SyncChecker:     &mockSync.Sync{IsSyncing: false},
+					ReplayerBuilder: builder,
 				}
 				request := &ethpb.DoppelGangerRequest{
 					ValidatorRequests: make([]*ethpb.DoppelGangerRequest_ValidatorRequest, 0),
@@ -1006,9 +1005,7 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 			name:    "doppelganger exists current epoch",
 			wantErr: false,
 			svSetup: func(t *testing.T) (*Server, *ethpb.DoppelGangerRequest, *ethpb.DoppelGangerResponse) {
-				mockGen := mockstategen.NewMockService()
-
-				hs, ps, os, keys := createStateSetup(t, 4, mockGen)
+				hs, ps, os, keys, builder := createStateSetup(t, 4)
 				// Previous Epoch State
 				for i := 0; i < 2; i++ {
 					bal, err := ps.BalanceAtIndex(types.ValidatorIndex(i))
@@ -1034,11 +1031,11 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 				assert.NoError(t, os.UpdateBalancesAtIndex(types.ValidatorIndex(2), bal-1000000000))
 
 				vs := &Server{
-					StateGen: mockGen,
 					HeadFetcher: &mockChain.ChainService{
 						State: hs,
 					},
-					SyncChecker: &mockSync.Sync{IsSyncing: false},
+					SyncChecker:     &mockSync.Sync{IsSyncing: false},
+					ReplayerBuilder: builder,
 				}
 				request := &ethpb.DoppelGangerRequest{
 					ValidatorRequests: make([]*ethpb.DoppelGangerRequest_ValidatorRequest, 0),
@@ -1073,9 +1070,7 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 			name:    "doppelganger exists previous epoch",
 			wantErr: false,
 			svSetup: func(t *testing.T) (*Server, *ethpb.DoppelGangerRequest, *ethpb.DoppelGangerResponse) {
-				mockGen := mockstategen.NewMockService()
-
-				hs, ps, os, keys := createStateSetup(t, 4, mockGen)
+				hs, ps, os, keys, builder := createStateSetup(t, 4)
 				// Previous Epoch State
 				for i := 0; i < 2; i++ {
 					bal, err := ps.BalanceAtIndex(types.ValidatorIndex(i))
@@ -1101,11 +1096,11 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 				assert.NoError(t, os.UpdateBalancesAtIndex(types.ValidatorIndex(2), bal-2000000000))
 
 				vs := &Server{
-					StateGen: mockGen,
 					HeadFetcher: &mockChain.ChainService{
 						State: hs,
 					},
-					SyncChecker: &mockSync.Sync{IsSyncing: false},
+					SyncChecker:     &mockSync.Sync{IsSyncing: false},
+					ReplayerBuilder: builder,
 				}
 				request := &ethpb.DoppelGangerRequest{
 					ValidatorRequests: make([]*ethpb.DoppelGangerRequest_ValidatorRequest, 0),
@@ -1140,9 +1135,7 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 			name:    "multiple doppelganger exists",
 			wantErr: false,
 			svSetup: func(t *testing.T) (*Server, *ethpb.DoppelGangerRequest, *ethpb.DoppelGangerResponse) {
-				mockGen := mockstategen.NewMockService()
-
-				hs, ps, os, keys := createStateSetup(t, 4, mockGen)
+				hs, ps, os, keys, builder := createStateSetup(t, 4)
 				// Previous Epoch State
 				for i := 10; i < 15; i++ {
 					bal, err := ps.BalanceAtIndex(types.ValidatorIndex(i))
@@ -1160,11 +1153,11 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 				}
 
 				vs := &Server{
-					StateGen: mockGen,
 					HeadFetcher: &mockChain.ChainService{
 						State: hs,
 					},
-					SyncChecker: &mockSync.Sync{IsSyncing: false},
+					SyncChecker:     &mockSync.Sync{IsSyncing: false},
+					ReplayerBuilder: builder,
 				}
 				request := &ethpb.DoppelGangerRequest{
 					ValidatorRequests: make([]*ethpb.DoppelGangerRequest_ValidatorRequest, 0),
@@ -1190,16 +1183,14 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 			name:    "attesters are too recent",
 			wantErr: false,
 			svSetup: func(t *testing.T) (*Server, *ethpb.DoppelGangerRequest, *ethpb.DoppelGangerResponse) {
-				mockGen := mockstategen.NewMockService()
-
-				hs, _, _, keys := createStateSetup(t, 4, mockGen)
+				hs, _, _, keys, _ := createStateSetup(t, 4)
 
 				vs := &Server{
-					StateGen: nil,
 					HeadFetcher: &mockChain.ChainService{
 						State: hs,
 					},
-					SyncChecker: &mockSync.Sync{IsSyncing: false},
+					SyncChecker:     &mockSync.Sync{IsSyncing: false},
+					ReplayerBuilder: nil,
 				}
 				request := &ethpb.DoppelGangerRequest{
 					ValidatorRequests: make([]*ethpb.DoppelGangerRequest_ValidatorRequest, 0),
@@ -1237,8 +1228,9 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 	}
 }
 
-func createStateSetup(t *testing.T, head types.Epoch, mockgen *mockstategen.MockStateManager) (state.BeaconState,
-	state.BeaconState, state.BeaconState, []bls.SecretKey) {
+func createStateSetup(t *testing.T, head types.Epoch) (state.BeaconState,
+	state.BeaconState, state.BeaconState, []bls.SecretKey, *mockstategen.MockReplayerBuilder) {
+	rb := &mockstategen.MockReplayerBuilder{}
 	gs, keys := util.DeterministicGenesisState(t, 64)
 	hs := gs.Copy()
 	// Head State
@@ -1267,9 +1259,8 @@ func createStateSetup(t *testing.T, head types.Epoch, mockgen *mockstategen.Mock
 			ProposerIndex:  10,
 		}
 		assert.NoError(t, hs.AppendCurrentEpochAttestations(pendingAtt))
-
 	}
-	mockgen.StatesBySlot[headSlot] = hs
+	rb.SetMockState(hs)
 
 	// Previous Epoch State
 	prevEpoch := headEpoch - 1
@@ -1299,9 +1290,8 @@ func createStateSetup(t *testing.T, head types.Epoch, mockgen *mockstategen.Mock
 			ProposerIndex:  10,
 		}
 		assert.NoError(t, ps.AppendCurrentEpochAttestations(pendingAtt))
-
 	}
-	mockgen.StatesBySlot[prevSlot] = ps
+	rb.SetMockState(ps)
 
 	// Older Epoch State
 	olderEpoch := prevEpoch - 1
@@ -1335,8 +1325,7 @@ func createStateSetup(t *testing.T, head types.Epoch, mockgen *mockstategen.Mock
 			ProposerIndex:  10,
 		}
 		assert.NoError(t, os.AppendCurrentEpochAttestations(pendingAtt))
-
 	}
-	mockgen.StatesBySlot[olderSlot] = os
-	return hs, ps, os, keys
+	rb.SetMockState(os)
+	return hs, ps, os, keys, rb
 }

beacon-chain/rpc/service.go

@@ -125,6 +125,10 @@ func NewService(ctx context.Context, cfg *Config) *Service {
 	}
 }
 
+// paranoid build time check to ensure ChainInfoFetcher implements required interfaces
+var _ stategen.CanonicalChecker = blockchain.ChainInfoFetcher(nil)
+var _ stategen.CurrentSlotter = blockchain.ChainInfoFetcher(nil)
+
 // Start the gRPC server.
 func (s *Service) Start() {
 	address := fmt.Sprintf("%s:%s", s.cfg.Host, s.cfg.Port)
@@ -169,6 +173,12 @@ func (s *Service) Start() {
 	}
 	s.grpcServer = grpc.NewServer(opts...)
 
+	var stateCache stategen.CachedGetter
+	if s.cfg.StateGen != nil {
+		stateCache = s.cfg.StateGen.CombinedCache()
+	}
+	withCache := stategen.WithCache(stateCache)
+
 	validatorServer := &validatorv1alpha1.Server{
 		Ctx:                    s.ctx,
 		AttestationCache:       cache.NewAttestationCache(),
@@ -194,6 +204,7 @@ func (s *Service) Start() {
 		SlashingsPool:          s.cfg.SlashingsPool,
 		StateGen:               s.cfg.StateGen,
 		SyncCommitteePool:      s.cfg.SyncCommitteeObjectPool,
+		ReplayerBuilder:        stategen.NewCanonicalBuilder(s.cfg.BeaconDB, s.cfg.ChainInfoFetcher, s.cfg.ChainInfoFetcher, withCache),
 		ExecutionEngineCaller:  s.cfg.ExecutionEngineCaller,
 		BeaconDB:               s.cfg.BeaconDB,
 	}
@@ -210,6 +221,7 @@ func (s *Service) Start() {
 			ChainInfoFetcher:   s.cfg.ChainInfoFetcher,
 			GenesisTimeFetcher: s.cfg.GenesisTimeFetcher,
 			StateGenService:    s.cfg.StateGen,
+			ReplayerBuilder:    stategen.NewCanonicalBuilder(s.cfg.BeaconDB, s.cfg.ChainInfoFetcher, s.cfg.ChainInfoFetcher, withCache),
 		},
 		SyncCommitteePool: s.cfg.SyncCommitteeObjectPool,
 	}
@@ -259,6 +271,7 @@ func (s *Service) Start() {
 		SyncChecker:                 s.cfg.SyncService,
 		ReceivedAttestationsBuffer:  make(chan *ethpbv1alpha1.Attestation, attestationBufferSize),
 		CollectedAttestationsBuffer: make(chan []*ethpbv1alpha1.Attestation, attestationBufferSize),
+		ReplayerBuilder:             stategen.NewCanonicalBuilder(s.cfg.BeaconDB, s.cfg.ChainInfoFetcher, s.cfg.ChainInfoFetcher, withCache),
 	}
 	beaconChainServerV1 := &beacon.Server{
 		BeaconDB:           s.cfg.BeaconDB,
@@ -301,6 +314,7 @@ func (s *Service) Start() {
 			HeadFetcher:        s.cfg.HeadFetcher,
 			PeerManager:        s.cfg.PeerManager,
 			PeersFetcher:       s.cfg.PeersFetcher,
+			ReplayerBuilder:    stategen.NewCanonicalBuilder(s.cfg.BeaconDB, s.cfg.ChainInfoFetcher, s.cfg.ChainInfoFetcher, withCache),
 		}
 		debugServerV1 := &debug.Server{
 			BeaconDB:    s.cfg.BeaconDB,
@@ -310,6 +324,7 @@ func (s *Service) Start() {
 				ChainInfoFetcher:   s.cfg.ChainInfoFetcher,
 				GenesisTimeFetcher: s.cfg.GenesisTimeFetcher,
 				StateGenService:    s.cfg.StateGen,
+				ReplayerBuilder:    stategen.NewCanonicalBuilder(s.cfg.BeaconDB, s.cfg.ChainInfoFetcher, s.cfg.ChainInfoFetcher, withCache),
 			},
 		}
 		ethpbv1alpha1.RegisterDebugServer(s.grpcServer, debugServer)

beacon-chain/rpc/statefetcher/BUILD.bazel

@@ -14,6 +14,7 @@ go_library(
         "//encoding/bytesutil:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prysmaticlabs_eth2_types//:go_default_library",
+        "@io_opencensus_go//trace:go_default_library",
     ],
 )
 
@@ -24,6 +25,7 @@ go_test(
     deps = [
         "//beacon-chain/blockchain/testing:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
+        "//beacon-chain/state/stategen:go_default_library",
         "//beacon-chain/state/stategen/mock:go_default_library",
         "//config/params:go_default_library",
         "//encoding/bytesutil:go_default_library",

beacon-chain/rpc/statefetcher/fetcher.go

@@ -15,8 +15,13 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	"go.opencensus.io/trace"
 )
 
+var ErrNoAncestorForBlock = errors.New("could not find an ancestor state for block")
+var ErrNoCanonicalBlockForSlot = errors.New("none of the blocks found in the db slot index are canonical")
+var ErrInvalidDBBlock = errors.New("invalid block found in database")
+
 // StateIdParseError represents an error scenario where a state ID could not be parsed.
 type StateIdParseError struct {
 	message string
@@ -72,6 +77,7 @@ func (e *StateRootNotFoundError) Error() string {
 type Fetcher interface {
 	State(ctx context.Context, stateId []byte) (state.BeaconState, error)
 	StateRoot(ctx context.Context, stateId []byte) ([]byte, error)
+	StateBySlot(ctx context.Context, slot types.Slot) (state.BeaconState, error)
 }
 
 // StateProvider is a real implementation of Fetcher.
@@ -80,6 +86,7 @@ type StateProvider struct {
 	ChainInfoFetcher   blockchain.ChainInfoFetcher
 	GenesisTimeFetcher blockchain.TimeFetcher
 	StateGenService    stategen.StateManager
+	ReplayerBuilder    stategen.ReplayerBuilder
 }
 
 // State returns the BeaconState for a given identifier. The identifier can be one of:
@@ -129,7 +136,7 @@ func (p *StateProvider) State(ctx context.Context, stateId []byte) (state.Beacon
 				e := NewStateIdParseError(parseErr)
 				return nil, &e
 			}
-			s, err = p.stateBySlot(ctx, types.Slot(slotNumber))
+			s, err = p.StateBySlot(ctx, types.Slot(slotNumber))
 		}
 	}
 
@@ -187,16 +194,21 @@ func (p *StateProvider) stateByHex(ctx context.Context, stateId []byte) (state.B
 	return nil, &stateNotFoundErr
 }
 
-func (p *StateProvider) stateBySlot(ctx context.Context, slot types.Slot) (state.BeaconState, error) {
-	currentSlot := p.GenesisTimeFetcher.CurrentSlot()
-	if slot > currentSlot {
-		return nil, errors.New("slot cannot be in the future")
-	}
-	state, err := p.StateGenService.StateBySlot(ctx, slot)
+// StateBySlot returns the post-state for the requested slot. To generate the state, it uses the
+// most recent canonical state prior to the target slot, and all canonical blocks
+// between the found state's slot and the target slot.
+// process_blocks is applied for all canonical blocks, and process_slots is called for any skipped
+// slots, or slots following the most recent canonical block up to and including the target slot.
+func (p *StateProvider) StateBySlot(ctx context.Context, target types.Slot) (state.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "statefetcher.StateBySlot")
+	defer span.End()
+
+	st, err := p.ReplayerBuilder.ForSlot(target).ReplayBlocks(ctx)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not get state")
+		msg := fmt.Sprintf("error while replaying history to slot=%d", target)
+		return nil, errors.Wrap(err, msg)
 	}
-	return state, nil
+	return st, nil
 }
 
 func (p *StateProvider) headStateRoot(ctx context.Context) ([]byte, error) {
@@ -204,7 +216,7 @@ func (p *StateProvider) headStateRoot(ctx context.Context) ([]byte, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "could not get head block")
 	}
-	if err := helpers.BeaconBlockIsNil(b); err != nil {
+	if err = helpers.BeaconBlockIsNil(b); err != nil {
 		return nil, err
 	}
 	return b.Block().StateRoot(), nil

beacon-chain/rpc/statefetcher/fetcher_test.go

@@ -7,6 +7,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	types "github.com/prysmaticlabs/eth2-types"
 	chainMock "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
@@ -151,12 +153,14 @@ func TestGetState(t *testing.T) {
 	})
 
 	t.Run("slot", func(t *testing.T) {
-		stateGen := mockstategen.NewMockService()
-		stateGen.StatesBySlot[headSlot] = newBeaconState
-
 		p := StateProvider{
 			GenesisTimeFetcher: &chainMock.ChainService{Slot: &headSlot},
-			StateGenService:    stateGen,
+			ChainInfoFetcher: &chainMock.ChainService{
+				CanonicalRoots: map[[32]byte]bool{
+					bytesutil.ToBytes32(newBeaconState.LatestBlockHeader().ParentRoot): true,
+				},
+			},
+			ReplayerBuilder: mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(newBeaconState)),
 		}
 
 		s, err := p.State(ctx, []byte(strconv.FormatUint(uint64(headSlot), 10)))
@@ -166,14 +170,16 @@ func TestGetState(t *testing.T) {
 		assert.Equal(t, stateRoot, sRoot)
 	})
 
+	rb := mockstategen.NewMockReplayerBuilder(mockstategen.WithStateError(1, stategen.ErrFutureSlotRequested))
 	t.Run("slot_too_big", func(t *testing.T) {
 		p := StateProvider{
 			GenesisTimeFetcher: &chainMock.ChainService{
 				Genesis: time.Now(),
 			},
+			ReplayerBuilder: rb,
 		}
 		_, err := p.State(ctx, []byte(strconv.FormatUint(1, 10)))
-		assert.ErrorContains(t, "slot cannot be in the future", err)
+		assert.ErrorContains(t, "cannot replay to future slots", err)
 	})
 
 	t.Run("invalid_state", func(t *testing.T) {

beacon-chain/rpc/testutil/mock_state_fetcher.go

@@ -3,6 +3,7 @@ package testutil
 import (
 	"context"
 
+	types "github.com/prysmaticlabs/eth2-types"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 )
 
@@ -21,3 +22,7 @@ func (m *MockFetcher) State(context.Context, []byte) (state.BeaconState, error)
 func (m *MockFetcher) StateRoot(context.Context, []byte) ([]byte, error) {
 	return m.BeaconStateRoot, nil
 }
+
+func (m *MockFetcher) StateBySlot(context.Context, types.Slot) (state.BeaconState, error) {
+	return m.BeaconState, nil
+}

beacon-chain/state/stategen/BUILD.bazel

@@ -3,6 +3,7 @@ load("@prysm//tools/go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
+        "cacher.go",
         "epoch_boundary_state_cache.go",
         "errors.go",
         "getter.go",
@@ -11,16 +12,12 @@ go_library(
         "metrics.go",
         "migrate.go",
         "replay.go",
+        "replayer.go",
         "service.go",
         "setter.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/state/stategen",
-    visibility = [
-        "//beacon-chain:__subpackages__",
-        "//testing/endtoend:__subpackages__",
-        "//testing/slasher/simulator:__pkg__",
-        "//testing/spectest:__subpackages__",
-    ],
+    visibility = ["//visibility:public"],
     deps = [
         "//beacon-chain/core/altair:go_default_library",
         "//beacon-chain/core/execution:go_default_library",
@@ -33,6 +30,7 @@ go_library(
         "//cache/lru:go_default_library",
         "//config/params:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//monitoring/tracing:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//proto/prysm/v1alpha1/block:go_default_library",
         "//runtime/version:go_default_library",
@@ -56,13 +54,17 @@ go_test(
         "hot_state_cache_test.go",
         "init_test.go",
         "migrate_test.go",
+        "mock_test.go",
         "replay_test.go",
+        "replayer_test.go",
         "service_test.go",
         "setter_test.go",
     ],
     embed = [":go_default_library"],
     deps = [
         "//beacon-chain/core/blocks:go_default_library",
+        "//beacon-chain/core/helpers:go_default_library",
+        "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/state:go_default_library",
@@ -71,11 +73,13 @@ go_test(
         "//encoding/bytesutil:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//proto/prysm/v1alpha1/block:go_default_library",
+        "//proto/prysm/v1alpha1/block/mock:go_default_library",
         "//proto/prysm/v1alpha1/wrapper:go_default_library",
         "//runtime/version:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/require:go_default_library",
         "//testing/util:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
         "@com_github_prysmaticlabs_eth2_types//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
         "@org_golang_google_protobuf//proto:go_default_library",

beacon-chain/state/stategen/cacher.go

@@ -0,0 +1,32 @@
+package stategen
+
+import (
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+)
+
+var ErrNotInCache = errors.New("state not found in cache")
+
+type CachedGetter interface {
+	ByRoot([32]byte) (state.BeaconState, error)
+}
+
+type CombinedCache struct {
+	getters []CachedGetter
+}
+
+func (c CombinedCache) ByRoot(root [32]byte) (state.BeaconState, error) {
+	for _, getter := range c.getters {
+		st, err := getter.ByRoot(root)
+		if err == nil {
+			return st, nil
+		}
+		if errors.Is(err, ErrNotInCache) {
+			continue
+		}
+		return nil, err
+	}
+	return nil, ErrNotInCache
+}
+
+var _ CachedGetter = &CombinedCache{}

beacon-chain/state/stategen/epoch_boundary_state_cache.go

@@ -66,6 +66,18 @@ func newBoundaryStateCache() *epochBoundaryState {
 	}
 }
 
+// ByRoot satisfies the CachedGetter interface
+func (e *epochBoundaryState) ByRoot(r [32]byte) (state.BeaconState, error) {
+	rsi, ok, err := e.getByRoot(r)
+	if err != nil {
+		return nil, err
+	}
+	if !ok {
+		return nil, ErrNotInCache
+	}
+	return rsi.state, nil
+}
+
 // get epoch boundary state by its block root. Returns copied state in state info object if exists. Otherwise returns nil.
 func (e *epochBoundaryState) getByRoot(r [32]byte) (*rootStateInfo, bool, error) {
 	e.lock.RLock()

beacon-chain/state/stategen/getter.go

@@ -84,7 +84,7 @@ func (s *State) StateByRootInitialSync(ctx context.Context, blockRoot [32]byte)
 		return cachedInfo.state, nil
 	}
 
-	startState, err := s.lastAncestorState(ctx, blockRoot)
+	startState, err := s.LastAncestorState(ctx, blockRoot)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not get ancestor state")
 	}
@@ -185,7 +185,7 @@ func (s *State) loadStateByRoot(ctx context.Context, blockRoot [32]byte) (state.
 
 	// Since the requested state is not in caches, start replaying using the last available ancestor state which is
 	// retrieved using input block's parent root.
-	startState, err := s.lastAncestorState(ctx, blockRoot)
+	startState, err := s.LastAncestorState(ctx, blockRoot)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not get ancestor state")
 	}
@@ -230,7 +230,7 @@ func (s *State) loadStateBySlot(ctx context.Context, slot types.Slot) (state.Bea
 	}
 
 	if lastValidSlot < slot {
-		replayStartState, err = processSlotsStateGen(ctx, replayStartState, slot)
+		replayStartState, err = ReplayProcessSlots(ctx, replayStartState, slot)
 		if err != nil {
 			return nil, err
 		}
@@ -246,8 +246,8 @@ func (s *State) loadStateBySlot(ctx context.Context, slot types.Slot) (state.Bea
 // 1.) block parent state is the last finalized state
 // 2.) block parent state is the epoch boundary state and exists in epoch boundary cache.
 // 3.) block parent state is in DB.
-func (s *State) lastAncestorState(ctx context.Context, root [32]byte) (state.BeaconState, error) {
-	ctx, span := trace.StartSpan(ctx, "stateGen.lastAncestorState")
+func (s *State) LastAncestorState(ctx context.Context, root [32]byte) (state.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "stateGen.LastAncestorState")
 	defer span.End()
 
 	if s.isFinalizedRoot(root) && s.finalizedState() != nil {
@@ -304,3 +304,14 @@ func (s *State) lastAncestorState(ctx context.Context, root [32]byte) (state.Bea
 		}
 	}
 }
+
+func (s *State) CombinedCache() *CombinedCache {
+	getters := make([]CachedGetter, 0)
+	if s.hotStateCache != nil {
+		getters = append(getters, s.hotStateCache)
+	}
+	if s.epochBoundaryStateCache != nil {
+		getters = append(getters, s.epochBoundaryStateCache)
+	}
+	return &CombinedCache{getters: getters}
+}

beacon-chain/state/stategen/getter_test.go

@@ -491,7 +491,7 @@ func TestLastAncestorState_CanGetUsingDB(t *testing.T) {
 	require.NoError(t, service.beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b3)))
 	require.NoError(t, service.beaconDB.SaveState(ctx, b1State, r1))
 
-	lastState, err := service.lastAncestorState(ctx, r3)
+	lastState, err := service.LastAncestorState(ctx, r3)
 	require.NoError(t, err)
 	assert.Equal(t, b1State.Slot(), lastState.Slot(), "Did not get wanted state")
 }
@@ -531,7 +531,7 @@ func TestLastAncestorState_CanGetUsingCache(t *testing.T) {
 	require.NoError(t, service.beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b3)))
 	service.hotStateCache.put(r1, b1State)
 
-	lastState, err := service.lastAncestorState(ctx, r3)
+	lastState, err := service.LastAncestorState(ctx, r3)
 	require.NoError(t, err)
 	assert.Equal(t, b1State.Slot(), lastState.Slot(), "Did not get wanted state")
 }

beacon-chain/state/stategen/hot_state_cache.go

@@ -52,6 +52,14 @@ func (c *hotStateCache) get(root [32]byte) state.BeaconState {
 	return nil
 }
 
+func (c *hotStateCache) ByRoot(root [32]byte) (state.BeaconState, error) {
+	st := c.get(root)
+	if st == nil {
+		return nil, ErrNotInCache
+	}
+	return st, nil
+}
+
 // GetWithoutCopy returns a non-copied cached response via input block root.
 func (c *hotStateCache) getWithoutCopy(root [32]byte) state.BeaconState {
 	c.lock.RLock()

beacon-chain/state/stategen/mock/BUILD.bazel

@@ -3,11 +3,15 @@ load("@prysm//tools/go:def.bzl", "go_library")
 go_library(
     name = "go_default_library",
     testonly = True,
-    srcs = ["mock.go"],
+    srcs = [
+        "mock.go",
+        "replayer.go",
+    ],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/state/stategen/mock",
     visibility = ["//visibility:public"],
     deps = [
         "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/stategen:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//proto/prysm/v1alpha1/block:go_default_library",
         "@com_github_prysmaticlabs_eth2_types//:go_default_library",

beacon-chain/state/stategen/mock/replayer.go

@@ -0,0 +1,91 @@
+package mock
+
+import (
+	"context"
+
+	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+)
+
+func NewMockReplayerBuilder(opt ...MockReplayerBuilderOption) *MockReplayerBuilder {
+	b := &MockReplayerBuilder{}
+	for _, o := range opt {
+		o(b)
+	}
+	return b
+}
+
+type MockReplayerBuilderOption func(*MockReplayerBuilder)
+
+func WithMockState(s state.BeaconState) MockReplayerBuilderOption {
+	return func(b *MockReplayerBuilder) {
+		b.SetMockState(s)
+	}
+}
+
+func WithStateError(s types.Slot, e error) MockReplayerBuilderOption {
+	return func(b *MockReplayerBuilder) {
+		b.SetMockSlotError(s, e)
+	}
+}
+
+type MockReplayerBuilder struct {
+	forSlot map[types.Slot]*MockReplayer
+}
+
+func (b *MockReplayerBuilder) SetMockState(s state.BeaconState) {
+	if b.forSlot == nil {
+		b.forSlot = make(map[types.Slot]*MockReplayer)
+	}
+	b.forSlot[s.Slot()] = &MockReplayer{State: s}
+}
+
+func (b *MockReplayerBuilder) SetMockSlotError(s types.Slot, e error) {
+	if b.forSlot == nil {
+		b.forSlot = make(map[types.Slot]*MockReplayer)
+	}
+	b.forSlot[s] = &MockReplayer{Err: e}
+}
+
+func (b *MockReplayerBuilder) ForSlot(target types.Slot) stategen.Replayer {
+	return b.forSlot[target]
+}
+
+var _ stategen.ReplayerBuilder = &MockReplayerBuilder{}
+
+type MockReplayer struct {
+	State state.BeaconState
+	Err   error
+}
+
+func (m *MockReplayer) ReplayBlocks(_ context.Context) (state.BeaconState, error) {
+	return m.State, m.Err
+}
+
+func (m *MockReplayer) ReplayToSlot(_ context.Context, _ types.Slot) (state.BeaconState, error) {
+	return m.State, m.Err
+}
+
+var _ stategen.Replayer = &MockReplayer{}
+
+type MockCanonicalChecker struct {
+	Is  bool
+	Err error
+}
+
+func (m *MockCanonicalChecker) IsCanonical(_ context.Context, _ [32]byte) (bool, error) {
+	return m.Is, m.Err
+}
+
+var _ stategen.CanonicalChecker = &MockCanonicalChecker{}
+
+type MockCurrentSlotter struct {
+	Slot types.Slot
+}
+
+func (c *MockCurrentSlotter) CurrentSlot() types.Slot {
+	return c.Slot
+}
+
+var _ stategen.CurrentSlotter = &MockCurrentSlotter{}

beacon-chain/state/stategen/mock_test.go

@@ -0,0 +1,282 @@
+package stategen
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"testing"
+
+	"github.com/pkg/errors"
+	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/transition"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
+	"github.com/prysmaticlabs/prysm/testing/require"
+	"github.com/prysmaticlabs/prysm/testing/util"
+)
+
+func TestMockHistoryStates(t *testing.T) {
+	ctx := context.Background()
+	var begin, middle, end types.Slot = 100, 150, 155
+	specs := []mockHistorySpec{
+		{slot: begin},
+		{slot: middle, savedState: true},
+		{slot: end, canonicalBlock: true},
+	}
+	hist := newMockHistory(t, specs, end+1)
+	// we should have 2 "saved" states, genesis and "middle" (savedState == true)
+	require.Equal(t, 2, len(hist.states))
+	genesisRoot := hist.slotMap[0]
+	st, err := hist.StateOrError(ctx, genesisRoot)
+	require.NoError(t, err)
+	require.DeepEqual(t, hist.states[genesisRoot], st)
+	require.Equal(t, types.Slot(0), st.Slot())
+
+	shouldExist, err := hist.StateOrError(ctx, hist.slotMap[middle])
+	require.NoError(t, err)
+	require.DeepEqual(t, hist.states[hist.slotMap[middle]], shouldExist)
+	require.Equal(t, middle, shouldExist.Slot())
+
+	cantExist, err := hist.StateOrError(ctx, hist.slotMap[end])
+	require.ErrorIs(t, err, db.ErrNotFoundState)
+	require.Equal(t, nil, cantExist)
+
+	cantExist, err = hist.StateOrError(ctx, hist.slotMap[begin])
+	require.ErrorIs(t, err, db.ErrNotFoundState)
+	require.Equal(t, nil, cantExist)
+}
+
+func TestMockHistoryParentRoot(t *testing.T) {
+	ctx := context.Background()
+	var begin, middle, end types.Slot = 100, 150, 155
+	specs := []mockHistorySpec{
+		{slot: begin},
+		{slot: middle, savedState: true},
+		{slot: end, canonicalBlock: true},
+	}
+	hist := newMockHistory(t, specs, end+1)
+	endRoot := hist.slotMap[end]
+	endBlock, err := hist.Block(ctx, endRoot)
+	require.NoError(t, err)
+	// middle should be the parent of end, compare the middle root to endBlock's parent root
+	require.Equal(t, hist.slotMap[middle], bytesutil.ToBytes32(endBlock.Block().ParentRoot()))
+}
+
+type mockHistorySpec struct {
+	slot           types.Slot
+	savedState     bool
+	canonicalBlock bool
+}
+
+type mockHistory struct {
+	blocks                         map[[32]byte]block.SignedBeaconBlock
+	slotMap                        map[types.Slot][32]byte
+	slotIndex                      slotList
+	canonical                      map[[32]byte]bool
+	states                         map[[32]byte]state.BeaconState
+	hiddenStates                   map[[32]byte]state.BeaconState
+	current                        types.Slot
+	overrideHighestSlotBlocksBelow func(context.Context, types.Slot) ([]block.SignedBeaconBlock, error)
+}
+
+type slotList []types.Slot
+
+func (m slotList) Len() int {
+	return len(m)
+}
+
+func (m slotList) Less(i, j int) bool {
+	return m[i] < m[j]
+}
+
+func (m slotList) Swap(i, j int) {
+	m[i], m[j] = m[j], m[i]
+}
+
+var errFallThroughOverride = errors.New("override yielding control back to real HighestSlotBlocksBelow")
+
+func (m *mockHistory) HighestSlotBlocksBelow(_ context.Context, slot types.Slot) ([]block.SignedBeaconBlock, error) {
+	if m.overrideHighestSlotBlocksBelow != nil {
+		s, err := m.overrideHighestSlotBlocksBelow(context.Background(), slot)
+		if !errors.Is(err, errFallThroughOverride) {
+			return s, err
+		}
+	}
+	if len(m.slotIndex) == 0 && len(m.slotMap) > 0 {
+		for k := range m.slotMap {
+			m.slotIndex = append(m.slotIndex, k)
+		}
+		sort.Sort(sort.Reverse(m.slotIndex))
+	}
+	for _, s := range m.slotIndex {
+		if s < slot {
+			return []block.SignedBeaconBlock{m.blocks[m.slotMap[s]]}, nil
+		}
+	}
+	return []block.SignedBeaconBlock{}, nil
+}
+
+var errGenesisBlockNotFound = errors.New("canonical genesis block not found in db")
+
+func (m *mockHistory) GenesisBlock(_ context.Context) (block.SignedBeaconBlock, error) {
+	genesisRoot, ok := m.slotMap[0]
+	if !ok {
+		return nil, errGenesisBlockNotFound
+	}
+	return m.blocks[genesisRoot], nil
+}
+
+func (m *mockHistory) Block(_ context.Context, blockRoot [32]byte) (block.SignedBeaconBlock, error) {
+	if b, ok := m.blocks[blockRoot]; ok {
+		return b, nil
+	}
+	return nil, nil
+}
+
+func (m *mockHistory) StateOrError(_ context.Context, blockRoot [32]byte) (state.BeaconState, error) {
+	if s, ok := m.states[blockRoot]; ok {
+		return s.Copy(), nil
+	}
+	return nil, db.ErrNotFoundState
+}
+
+func (m *mockHistory) IsCanonical(_ context.Context, blockRoot [32]byte) (bool, error) {
+	canon, ok := m.canonical[blockRoot]
+	return ok && canon, nil
+}
+
+func (m *mockHistory) CurrentSlot() types.Slot {
+	return m.current
+}
+
+func (h *mockHistory) addBlock(root [32]byte, b block.SignedBeaconBlock, canon bool) {
+	h.blocks[root] = b
+	h.slotMap[b.Block().Slot()] = root
+	h.canonical[root] = canon
+}
+
+func (h *mockHistory) addState(root [32]byte, s state.BeaconState) {
+	h.states[root] = s
+}
+
+func (h *mockHistory) hideState(root [32]byte, s state.BeaconState) {
+	h.hiddenStates[root] = s
+}
+
+func (h *mockHistory) validateRoots() error {
+	uniqParentRoots := make(map[[32]byte]types.Slot)
+	for s, root := range h.slotMap {
+		b := h.blocks[root]
+		htr, err := b.Block().HashTreeRoot()
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("error computing htr for block at slot %d", s))
+		}
+		if htr != root {
+			return fmt.Errorf("htr mismatch, expected=%#x, actual=%#x", root, htr)
+		}
+		if ps, ok := uniqParentRoots[htr]; ok {
+			return fmt.Errorf("duplicate parent_root %#x seen at slots %d, %d", htr, ps, s)
+		}
+		uniqParentRoots[htr] = s
+	}
+	return nil
+}
+
+func newMockHistory(t *testing.T, hist []mockHistorySpec, current types.Slot) *mockHistory {
+	ctx := context.Background()
+	mh := &mockHistory{
+		blocks:       map[[32]byte]block.SignedBeaconBlock{},
+		canonical:    map[[32]byte]bool{},
+		states:       map[[32]byte]state.BeaconState{},
+		hiddenStates: map[[32]byte]state.BeaconState{},
+		slotMap:      map[types.Slot][32]byte{},
+		slotIndex:    slotList{},
+		current:      current,
+	}
+
+	// genesis state for history
+	gs, _ := util.DeterministicGenesisState(t, 32)
+	gsr, err := gs.HashTreeRoot(ctx)
+	require.NoError(t, err)
+
+	// generate new genesis block using the root of the deterministic state
+	gb, err := wrapper.WrappedSignedBeaconBlock(blocks.NewGenesisBlock(gsr[:]))
+	require.NoError(t, err)
+	pr, err := gb.Block().HashTreeRoot()
+	require.NoError(t, err)
+
+	// add genesis block as canonical
+	mh.addBlock(pr, gb, true)
+	// add genesis state, indexed by unapplied genesis block - genesis block is never really processed...
+	mh.addState(pr, gs.Copy())
+
+	ps := gs.Copy()
+	for _, spec := range hist {
+		// call process_slots and process_block separately, because process_slots updates values used in randao mix
+		// which influences proposer_index.
+		s, err := ReplayProcessSlots(ctx, ps, spec.slot)
+		require.NoError(t, err)
+
+		// create proposer block, setting values in the order seen in the validator.md spec
+		b, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlock())
+		require.NoError(t, err)
+
+		// set slot to mock history spec value
+		require.NoError(t, wrapper.SetBlockSlot(b, spec.slot))
+
+		// set the correct proposer_index in the "proposal" block
+		// so that it will pass validation in process_block. important that we do this
+		// after process_slots!
+		idx, err := helpers.BeaconProposerIndex(ctx, s)
+		require.NoError(t, err)
+		require.NoError(t, wrapper.SetProposerIndex(b, idx))
+
+		// set parent root
+		require.NoError(t, wrapper.SetBlockParentRoot(b, pr))
+
+		// now do process_block
+		s, err = transition.ProcessBlockForStateRoot(ctx, s, b)
+		require.NoError(t, err)
+
+		sr, err := s.HashTreeRoot(ctx)
+		require.NoError(t, err)
+		err = wrapper.SetBlockStateRoot(b, sr)
+		require.NoError(t, err)
+
+		pr, err = b.Block().HashTreeRoot()
+		require.NoError(t, err)
+		if spec.savedState {
+			mh.addState(pr, s)
+		} else {
+			mh.hideState(pr, s)
+		}
+		mh.addBlock(pr, b, spec.canonicalBlock)
+		ps = s.Copy()
+	}
+
+	require.NoError(t, mh.validateRoots())
+	return mh
+}
+
+var _ HistoryAccessor = &mockHistory{}
+var _ CanonicalChecker = &mockHistory{}
+var _ CurrentSlotter = &mockHistory{}
+
+type mockCachedGetter struct {
+	cache map[[32]byte]state.BeaconState
+}
+
+func (m mockCachedGetter) ByRoot(root [32]byte) (state.BeaconState, error) {
+	st, ok := m.cache[root]
+	if !ok {
+		return nil, ErrNotInCache
+	}
+	return st, nil
+}
+
+var _ CachedGetter = &mockCachedGetter{}

beacon-chain/state/stategen/replay.go

@@ -10,11 +10,12 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/altair"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/execution"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	prysmTime "github.com/prysmaticlabs/prysm/beacon-chain/core/time"
+	prysmtime "github.com/prysmaticlabs/prysm/beacon-chain/core/time"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/transition"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	"github.com/prysmaticlabs/prysm/monitoring/tracing"
 	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
 	"github.com/prysmaticlabs/prysm/runtime/version"
 	"github.com/sirupsen/logrus"
@@ -60,7 +61,7 @@ func (_ *State) ReplayBlocks(
 
 	// If there is skip slots at the end.
 	if targetSlot > state.Slot() {
-		state, err = processSlotsStateGen(ctx, state, targetSlot)
+		state, err = ReplayProcessSlots(ctx, state, targetSlot)
 		if err != nil {
 			return nil, err
 		}
@@ -151,7 +152,7 @@ func executeStateTransitionStateGen(
 
 	// Execute per slots transition.
 	// Given this is for state gen, a node uses the version process slots without skip slots cache.
-	state, err = processSlotsStateGen(ctx, state, signed.Block().Slot())
+	state, err = ReplayProcessSlots(ctx, state, signed.Block().Slot())
 	if err != nil {
 		return nil, errors.Wrap(err, "could not process slot")
 	}
@@ -163,22 +164,14 @@ func executeStateTransitionStateGen(
 	if err != nil {
 		return nil, errors.Wrap(err, "could not process block")
 	}
-	if signed.Version() == version.Phase0 {
-		return state, nil
-	}
-
-	sa, err := signed.Block().Body().SyncAggregate()
-	if err != nil {
-		return nil, err
-	}
-	return altair.ProcessSyncAggregate(ctx, state, sa)
+	return state, nil
 }
 
-// processSlotsStateGen to process old slots for state gen usages.
+// ReplayProcessSlots to process old slots for state gen usages.
 // There's no skip slot cache involved given state gen only works with already stored block and state in DB.
 // WARNING: This method should not be used for future slot.
-func processSlotsStateGen(ctx context.Context, state state.BeaconState, slot types.Slot) (state.BeaconState, error) {
-	ctx, span := trace.StartSpan(ctx, "stategen.ProcessSlotsStateGen")
+func ReplayProcessSlots(ctx context.Context, state state.BeaconState, slot types.Slot) (state.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "stategen.ReplayProcessSlots")
 	defer span.End()
 	if state == nil || state.IsNil() {
 		return nil, errUnknownState
@@ -199,35 +192,41 @@ func processSlotsStateGen(ctx context.Context, state state.BeaconState, slot typ
 		if err != nil {
 			return nil, errors.Wrap(err, "could not process slot")
 		}
-		if prysmTime.CanProcessEpoch(state) {
+		if prysmtime.CanProcessEpoch(state) {
 			switch state.Version() {
 			case version.Phase0:
 				state, err = transition.ProcessEpochPrecompute(ctx, state)
 				if err != nil {
+					tracing.AnnotateError(span, err)
 					return nil, errors.Wrap(err, "could not process epoch with optimizations")
 				}
 			case version.Altair, version.Bellatrix:
 				state, err = altair.ProcessEpoch(ctx, state)
 				if err != nil {
-					return nil, errors.Wrap(err, "could not process epoch with optimization")
+					tracing.AnnotateError(span, err)
+					return nil, errors.Wrap(err, "could not process epoch")
 				}
 			default:
 				return nil, errors.New("beacon state should have a version")
 			}
 		}
 		if err := state.SetSlot(state.Slot() + 1); err != nil {
-			return nil, err
+			tracing.AnnotateError(span, err)
+			return nil, errors.Wrap(err, "failed to increment state slot")
 		}
 
-		if prysmTime.CanUpgradeToAltair(state.Slot()) {
+		if prysmtime.CanUpgradeToAltair(state.Slot()) {
 			state, err = altair.UpgradeToAltair(ctx, state)
 			if err != nil {
+				tracing.AnnotateError(span, err)
 				return nil, err
 			}
 		}
-		if prysmTime.CanUpgradeToBellatrix(state.Slot()) {
+
+		if prysmtime.CanUpgradeToBellatrix(state.Slot()) {
 			state, err = execution.UpgradeToBellatrix(ctx, state)
 			if err != nil {
+				tracing.AnnotateError(span, err)
 				return nil, err
 			}
 		}

beacon-chain/state/stategen/replayer.go

@@ -0,0 +1,387 @@
+package stategen
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
+	"github.com/sirupsen/logrus"
+	"go.opencensus.io/trace"
+)
+
+var ErrFutureSlotRequested = errors.New("cannot replay to future slots")
+var ErrNoCanonicalBlockForSlot = errors.New("none of the blocks found in the db slot index are canonical")
+var ErrNoBlocksBelowSlot = errors.New("no blocks found in db below slot")
+var ErrInvalidDBBlock = errors.New("invalid block found in database")
+
+// HistoryAccessor describes the minimum set of database methods needed to support the ReplayerBuilder.
+type HistoryAccessor interface {
+	HighestSlotBlocksBelow(ctx context.Context, slot types.Slot) ([]block.SignedBeaconBlock, error)
+	GenesisBlock(ctx context.Context) (block.SignedBeaconBlock, error)
+	Block(ctx context.Context, blockRoot [32]byte) (block.SignedBeaconBlock, error)
+	StateOrError(ctx context.Context, blockRoot [32]byte) (state.BeaconState, error)
+}
+
+// CanonicalChecker determines whether the given block root is canonical.
+// In practice this should be satisfied by a type that uses the fork choice store.
+type CanonicalChecker interface {
+	IsCanonical(ctx context.Context, blockRoot [32]byte) (bool, error)
+}
+
+// CurrentSlotter provides the current Slot.
+type CurrentSlotter interface {
+	CurrentSlot() types.Slot
+}
+
+// Replayer encapsulates database query and replay logic. It can be constructed via a ReplayerBuilder.
+type Replayer interface {
+	// ReplayBlocks replays the blocks the Replayer knows about based on Builder params
+	ReplayBlocks(ctx context.Context) (state.BeaconState, error)
+	// ReplayToSlot invokes ReplayBlocks under the hood,
+	// but then also runs process_slots to advance the state past the root or slot used in the builder.
+	// For example, if you wanted the state to be at the target slot, but only integrating blocks up to
+	// slot-1, you could request Builder.ForSlot(slot-1).ReplayToSlot(slot)
+	ReplayToSlot(ctx context.Context, target types.Slot) (state.BeaconState, error)
+}
+
+var _ Replayer = &stateReplayer{}
+
+type stateReplayer struct {
+	s           state.BeaconState
+	descendants []block.SignedBeaconBlock
+	target      types.Slot
+	method      retrievalMethod
+	chainer     chainer
+}
+
+// ReplayBlocks applies all the blocks that were accumulated when building the Replayer.
+// This method relies on the correctness of the code that constructed the Replayer data.
+func (rs *stateReplayer) ReplayBlocks(ctx context.Context) (state.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "stateGen.stateReplayer.ReplayBlocks")
+	defer span.End()
+
+	var s state.BeaconState
+	var descendants []block.SignedBeaconBlock
+	var err error
+	switch rs.method {
+	case forSlot:
+		s, descendants, err = rs.chainer.chainForSlot(ctx, rs.target)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, errors.New("Replayer initialized using unknown state retrieval method")
+	}
+
+	start := time.Now()
+	diff, err := rs.target.SafeSubSlot(s.Slot())
+	if err != nil {
+		msg := fmt.Sprintf("error subtracting state.slot %d from replay target slot %d", s.Slot(), rs.target)
+		return nil, errors.Wrap(err, msg)
+	}
+	log.WithFields(logrus.Fields{
+		"startSlot": s.Slot(),
+		"endSlot":   rs.target,
+		"diff":      diff,
+	}).Debug("Replaying canonical blocks from most recent state")
+
+	for _, b := range descendants {
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+		s, err = executeStateTransitionStateGen(ctx, s, b)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if rs.target > s.Slot() {
+		s, err = ReplayProcessSlots(ctx, s, rs.target)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	duration := time.Since(start)
+	log.WithFields(logrus.Fields{
+		"duration": duration,
+	}).Debug("Finished calling process_blocks on all blocks in ReplayBlocks")
+	return s, nil
+}
+
+// ReplayToSlot invokes ReplayBlocks under the hood,
+// but then also runs process_slots to advance the state past the root or slot used in the builder.
+// for example, if you wanted the state to be at the target slot, but only integrating blocks up to
+// slot-1, you could request Builder.ForSlot(slot-1).ReplayToSlot(slot)
+func (rs *stateReplayer) ReplayToSlot(ctx context.Context, replayTo types.Slot) (state.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "stateGen.stateReplayer.ReplayToSlot")
+	defer span.End()
+
+	s, err := rs.ReplayBlocks(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to ReplayBlocks")
+	}
+
+	if replayTo > s.Slot() {
+		start := time.Now()
+		log.WithFields(logrus.Fields{
+			"startSlot": s.Slot(),
+			"endSlot":   replayTo,
+			"diff":      replayTo - s.Slot(),
+		}).Debug("calling process_slots on remaining slots")
+
+		if replayTo > s.Slot() {
+			// err will be handled after the bookend log
+			s, err = ReplayProcessSlots(ctx, s, replayTo)
+		}
+
+		duration := time.Since(start)
+		log.WithFields(logrus.Fields{
+			"duration": duration,
+		}).Debug("time spent in process_slots")
+		if err != nil {
+			return nil, errors.Wrap(err, fmt.Sprintf("ReplayToSlot failed to seek to slot %d after applying blocks", replayTo))
+		}
+	}
+	return s, nil
+}
+
+// ReplayerBuilder creates a Replayer that can be used to obtain a state at a specified slot or root
+// (only ForSlot implemented so far).
+// See documentation on Replayer for more on how to use this to obtain pre/post-block states
+type ReplayerBuilder interface {
+	// ForSlot creates a builder that will create a state that includes blocks up to and including the requested slot
+	// The resulting Replayer will always yield a state with .Slot=target; if there are skipped blocks
+	// between the highest canonical block in the db and the target, the replayer will fast-forward past the intervening
+	// slots via process_slots.
+	ForSlot(target types.Slot) Replayer
+}
+
+func WithCache(c CachedGetter) CanonicalBuilderOption {
+	return func(b *CanonicalBuilder) {
+		b.chainer.useCache(c)
+	}
+}
+
+type CanonicalBuilderOption func(*CanonicalBuilder)
+
+// NewCanonicalBuilder handles initializing the default concrete ReplayerBuilder implementation.
+func NewCanonicalBuilder(h HistoryAccessor, c CanonicalChecker, cs CurrentSlotter, opts ...CanonicalBuilderOption) *CanonicalBuilder {
+	b := &CanonicalBuilder{
+		chainer: &canonicalChainer{
+			h:  h,
+			c:  c,
+			cs: cs,
+		},
+	}
+	for _, o := range opts {
+		o(b)
+	}
+	return b
+}
+
+type retrievalMethod int
+
+const (
+	forSlot retrievalMethod = iota
+)
+
+// CanonicalBuilder builds a Replayer that uses a combination of database queries and a
+// CanonicalChecker (which should usually be a fork choice store implementing an IsCanonical method)
+// to determine the canonical chain and apply it to generate the desired state.
+type CanonicalBuilder struct {
+	chainer chainer
+}
+
+var _ ReplayerBuilder = &CanonicalBuilder{}
+
+func (r *CanonicalBuilder) ForSlot(target types.Slot) Replayer {
+	return &stateReplayer{chainer: r.chainer, method: forSlot, target: target}
+}
+
+// chainer is responsible for supplying the chain components necessary to rebuild a state,
+// namely a starting BeaconState and all available blocks from the starting state up to and including the target slot
+type chainer interface {
+	chainForSlot(ctx context.Context, target types.Slot) (state.BeaconState, []block.SignedBeaconBlock, error)
+	useCache(c CachedGetter)
+}
+
+type canonicalChainer struct {
+	h     HistoryAccessor
+	c     CanonicalChecker
+	cs    CurrentSlotter
+	cache CachedGetter
+}
+
+var _ chainer = &canonicalChainer{}
+
+func (c *canonicalChainer) useCache(cache CachedGetter) {
+	c.cache = cache
+}
+
+// ChainForSlot creates a value that satisfies the Replayer interface via db queries
+// and the stategen transition helper methods. This implementation uses the following algorithm:
+// - find the highest canonical block <= the target slot
+// - starting with this block, recursively search backwards for a stored state, and accumulate intervening blocks
+func (c *canonicalChainer) chainForSlot(ctx context.Context, target types.Slot) (state.BeaconState, []block.SignedBeaconBlock, error) {
+	ctx, span := trace.StartSpan(ctx, "canonicalChainer.chainForSlot")
+	defer span.End()
+	currentSlot := c.cs.CurrentSlot()
+	if target > currentSlot {
+		return nil, nil, errors.Wrap(ErrFutureSlotRequested, fmt.Sprintf("requested=%d, current=%d", target, currentSlot))
+	}
+	_, b, err := c.canonicalBlockForSlot(ctx, target)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, fmt.Sprintf("unable to find replay data for slot=%d", target))
+	}
+	s, descendants, err := c.ancestorChain(ctx, b)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to query for ancestor and descendant blocks")
+	}
+
+	return s, descendants, nil
+}
+
+// canonicalBlockForSlot uses HighestSlotBlocksBelow(target+1) and the CanonicalChecker
+// to find the highest canonical block available to replay to the given slot.
+func (c *canonicalChainer) canonicalBlockForSlot(ctx context.Context, target types.Slot) ([32]byte, block.SignedBeaconBlock, error) {
+	for target > 0 {
+		if ctx.Err() != nil {
+			return [32]byte{}, nil, errors.Wrap(ctx.Err(), "context canceled during canonicalBlockForSlot")
+		}
+		hbs, err := c.h.HighestSlotBlocksBelow(ctx, target+1)
+		if err != nil {
+			return [32]byte{}, nil, errors.Wrap(err, fmt.Sprintf("error finding highest block w/ slot <= %d", target))
+		}
+		if len(hbs) == 0 {
+			return [32]byte{}, nil, errors.Wrap(ErrNoBlocksBelowSlot, fmt.Sprintf("slot=%d", target))
+		}
+		r, b, err := c.bestForSlot(ctx, hbs)
+		if err == nil {
+			// we found a valid, canonical block!
+			return r, b, nil
+		}
+
+		// we found a block, but it wasn't considered canonical - keep looking
+		if errors.Is(err, ErrNoCanonicalBlockForSlot) {
+			// break once we've seen slot 0 (and prevent underflow)
+			if hbs[0].Block().Slot() == 0 {
+				break
+			}
+			target = hbs[0].Block().Slot() - 1
+			continue
+		}
+		return [32]byte{}, nil, err
+	}
+	b, err := c.h.GenesisBlock(ctx)
+	if err != nil {
+		return [32]byte{}, nil, errors.Wrap(err, "db error while retrieving genesis block")
+	}
+	root, _, err := c.bestForSlot(ctx, []block.SignedBeaconBlock{b})
+	if err != nil {
+		return [32]byte{}, nil, errors.Wrap(err, "problem retrieving genesis block")
+	}
+	return root, b, nil
+}
+
+// bestForSlot encapsulates several messy realities of the underlying db code, looping through multiple blocks,
+// performing null/validity checks, and using CanonicalChecker to only pick canonical blocks.
+func (c *canonicalChainer) bestForSlot(ctx context.Context, hbs []block.SignedBeaconBlock) ([32]byte, block.SignedBeaconBlock, error) {
+	for _, b := range hbs {
+		if helpers.BeaconBlockIsNil(b) != nil {
+			continue
+		}
+		root, err := b.Block().HashTreeRoot()
+		if err != nil {
+			// use this error message to wrap a sentinel error for error type matching
+			wrapped := errors.Wrap(ErrInvalidDBBlock, err.Error())
+			msg := fmt.Sprintf("could not compute hash_tree_root for block at slot=%d", b.Block().Slot())
+			return [32]byte{}, nil, errors.Wrap(wrapped, msg)
+		}
+		canon, err := c.c.IsCanonical(ctx, root)
+		if err != nil {
+			return [32]byte{}, nil, errors.Wrap(err, "replayer could not check if block is canonical")
+		}
+		if canon {
+			return root, b, nil
+		}
+	}
+	return [32]byte{}, nil, errors.Wrap(ErrNoCanonicalBlockForSlot, "no good block for slot")
+}
+
+func (c *canonicalChainer) getState(ctx context.Context, root [32]byte) (state.BeaconState, error) {
+	if c.cache != nil {
+		st, err := c.cache.ByRoot(root)
+		if err == nil {
+			return st, nil
+		}
+		if !errors.Is(err, ErrNotInCache) {
+			return nil, errors.Wrap(err, "error reading from state cache during state replay")
+		}
+	}
+	return c.h.StateOrError(ctx, root)
+}
+
+// ancestorChain works backwards through the chain lineage, accumulating blocks and checking for a saved state.
+// If it finds a saved state that the tail block was descended from, it returns this state and
+// all blocks in the lineage, including the tail block. Blocks are returned in ascending order.
+// Note that this function assumes that the tail is a canonical block, and therefore assumes that
+// all ancestors are also canonical.
+func (c *canonicalChainer) ancestorChain(ctx context.Context, tail block.SignedBeaconBlock) (state.BeaconState, []block.SignedBeaconBlock, error) {
+	ctx, span := trace.StartSpan(ctx, "canonicalChainer.ancestorChain")
+	defer span.End()
+	chain := make([]block.SignedBeaconBlock, 0)
+	for {
+		if err := ctx.Err(); err != nil {
+			msg := fmt.Sprintf("context canceled while finding ancestors of block at slot %d", tail.Block().Slot())
+			return nil, nil, errors.Wrap(err, msg)
+		}
+		b := tail.Block()
+		// compute hash_tree_root of current block and try to look up the corresponding state
+		root, err := b.HashTreeRoot()
+		if err != nil {
+			msg := fmt.Sprintf("could not compute htr for descendant block at slot=%d", b.Slot())
+			return nil, nil, errors.Wrap(err, msg)
+		}
+		st, err := c.getState(ctx, root)
+		// err == nil, we've got a real state - the job is done!
+		// Note: in cases where there are skipped slots we could find a state that is a descendant
+		// of the block we are searching for. We don't want to return a future block, so in this case
+		// we keep working backwards.
+		if err == nil && st.Slot() == b.Slot() {
+			// we found the state by the root of the head, meaning it has already been applied.
+			// we only want to return the blocks descended from it.
+			reverseChain(chain)
+			return st, chain, nil
+		}
+		// ErrNotFoundState errors are fine, but other errors mean something is wrong with the db
+		if err != nil && !errors.Is(err, db.ErrNotFoundState) {
+			return nil, nil, errors.Wrap(err, fmt.Sprintf("error querying database for state w/ block root = %#x", root))
+		}
+		parent, err := c.h.Block(ctx, bytesutil.ToBytes32(b.ParentRoot()))
+		if err != nil {
+			msg := fmt.Sprintf("db error when retrieving parent of block at slot=%d by root=%#x", b.Slot(), b.ParentRoot())
+			return nil, nil, errors.Wrap(err, msg)
+		}
+		if helpers.BeaconBlockIsNil(parent) != nil {
+			msg := fmt.Sprintf("unable to retrieve parent of block at slot=%d by root=%#x", b.Slot(), b.ParentRoot())
+			return nil, nil, errors.Wrap(db.ErrNotFound, msg)
+		}
+		chain = append(chain, tail)
+		tail = parent
+	}
+}
+
+func reverseChain(c []block.SignedBeaconBlock) {
+	last := len(c) - 1
+	swaps := (last + 1) / 2
+	for i := 0; i < swaps; i++ {
+		c[i], c[last-i] = c[last-i], c[i]
+	}
+}

beacon-chain/state/stategen/replayer_test.go

@@ -0,0 +1,716 @@
+package stategen
+
+import (
+	"context"
+	"encoding/binary"
+	"fmt"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+
+	"github.com/pkg/errors"
+	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block/mock"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
+	"github.com/prysmaticlabs/prysm/testing/require"
+)
+
+func headerFromBlock(b block.SignedBeaconBlock) (*ethpb.BeaconBlockHeader, error) {
+	bodyRoot, err := b.Block().Body().HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	return &ethpb.BeaconBlockHeader{
+		Slot:          b.Block().Slot(),
+		StateRoot:     b.Block().StateRoot(),
+		ProposerIndex: b.Block().ProposerIndex(),
+		BodyRoot:      bodyRoot[:],
+		ParentRoot:    b.Block().ParentRoot(),
+	}, nil
+}
+
+func TestReplayBlocks(t *testing.T) {
+	ctx := context.Background()
+	var zero, one, two, three, four, five types.Slot = 50, 51, 150, 151, 152, 200
+	specs := []mockHistorySpec{
+		{slot: zero},
+		{slot: one, savedState: true},
+		{slot: two},
+		{slot: three},
+		{slot: four},
+		{slot: five, canonicalBlock: true},
+	}
+
+	hist := newMockHistory(t, specs, five+1)
+	bld := NewCanonicalBuilder(hist, hist, hist)
+	st, err := bld.ForSlot(five).ReplayBlocks(ctx)
+	require.NoError(t, err)
+	expected := hist.hiddenStates[hist.slotMap[five]]
+	expectedHTR, err := expected.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualHTR, err := st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	expectedLBH := expected.LatestBlockHeader()
+	actualLBH := st.LatestBlockHeader()
+	require.Equal(t, expectedLBH.Slot, actualLBH.Slot)
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.ParentRoot), bytesutil.ToBytes32(actualLBH.ParentRoot))
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.StateRoot), bytesutil.ToBytes32(actualLBH.StateRoot))
+	require.Equal(t, expectedLBH.ProposerIndex, actualLBH.ProposerIndex)
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.BodyRoot), bytesutil.ToBytes32(actualLBH.BodyRoot))
+	require.Equal(t, expectedHTR, actualHTR)
+
+	st, err = bld.ForSlot(one).ReplayBlocks(ctx)
+	require.NoError(t, err)
+	expected = hist.states[hist.slotMap[one]]
+
+	// no canonical blocks in between, so latest block process_block_header will be for genesis
+	expectedLBH, err = headerFromBlock(hist.blocks[hist.slotMap[0]])
+	require.NoError(t, err)
+	actualLBH = st.LatestBlockHeader()
+	require.Equal(t, expectedLBH.Slot, actualLBH.Slot)
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.ParentRoot), bytesutil.ToBytes32(actualLBH.ParentRoot))
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.StateRoot), bytesutil.ToBytes32(actualLBH.StateRoot))
+	require.Equal(t, expectedLBH.ProposerIndex, actualLBH.ProposerIndex)
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.BodyRoot), bytesutil.ToBytes32(actualLBH.BodyRoot))
+
+	require.Equal(t, expected.Slot(), st.Slot())
+	// NOTE: HTR is not compared, because process_block is not called for non-canonical blocks,
+	// so there are multiple differences compared to the "db" state that applies all blocks
+}
+
+func TestReplayToSlot(t *testing.T) {
+	ctx := context.Background()
+	var zero, one, two, three, four, five types.Slot = 50, 51, 150, 151, 152, 200
+	specs := []mockHistorySpec{
+		{slot: zero},
+		{slot: one, savedState: true},
+		{slot: two},
+		{slot: three},
+		{slot: four},
+		{slot: five, canonicalBlock: true},
+	}
+
+	// first case tests that ReplayToSlot is equivalent to ReplayBlocks
+	hist := newMockHistory(t, specs, five+1)
+	bld := NewCanonicalBuilder(hist, hist, hist)
+
+	st, err := bld.ForSlot(five).ReplayToSlot(ctx, five)
+	require.NoError(t, err)
+	expected := hist.hiddenStates[hist.slotMap[five]]
+	expectedHTR, err := expected.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualHTR, err := st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	expectedLBH := expected.LatestBlockHeader()
+	actualLBH := st.LatestBlockHeader()
+	require.Equal(t, expectedLBH.Slot, actualLBH.Slot)
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.ParentRoot), bytesutil.ToBytes32(actualLBH.ParentRoot))
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.StateRoot), bytesutil.ToBytes32(actualLBH.StateRoot))
+	require.Equal(t, expectedLBH.ProposerIndex, actualLBH.ProposerIndex)
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.BodyRoot), bytesutil.ToBytes32(actualLBH.BodyRoot))
+	require.Equal(t, expectedHTR, actualHTR)
+
+	st, err = bld.ForSlot(five).ReplayToSlot(ctx, five+100)
+	require.NoError(t, err)
+	require.Equal(t, five+100, st.Slot())
+	expectedLBH, err = headerFromBlock(hist.blocks[hist.slotMap[five]])
+	require.NoError(t, err)
+	actualLBH = st.LatestBlockHeader()
+	require.Equal(t, expectedLBH.Slot, actualLBH.Slot)
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.ParentRoot), bytesutil.ToBytes32(actualLBH.ParentRoot))
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.StateRoot), bytesutil.ToBytes32(actualLBH.StateRoot))
+	require.Equal(t, expectedLBH.ProposerIndex, actualLBH.ProposerIndex)
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.BodyRoot), bytesutil.ToBytes32(actualLBH.BodyRoot))
+}
+
+// happy path tests
+func TestCanonicalBlockForSlotHappy(t *testing.T) {
+	ctx := context.Background()
+	var begin, middle, end types.Slot = 100, 150, 155
+	specs := []mockHistorySpec{
+		{slot: begin},
+		{slot: middle, savedState: true},
+		{slot: end, canonicalBlock: true},
+	}
+	hist := newMockHistory(t, specs, end+1)
+	cc := canonicalChainer{h: hist, c: hist, cs: hist}
+
+	// since only the end block and genesis are canonical, once the slot drops below
+	// end, we should always get genesis
+	cases := []struct {
+		slot    types.Slot
+		highest types.Slot
+		canon   types.Slot
+		name    string
+	}{
+		{slot: hist.current, highest: end, canon: end, name: "slot > end"},
+		{slot: end, highest: end, canon: end, name: "slot == end"},
+		{slot: end - 1, highest: middle, canon: 0, name: "middle < slot < end"},
+		{slot: middle, highest: middle, canon: 0, name: "slot == middle"},
+		{slot: middle - 1, highest: begin, canon: 0, name: "begin < slot < middle"},
+		{slot: begin, highest: begin, canon: 0, name: "slot == begin"},
+		{slot: begin - 1, highest: 0, canon: 0, name: "genesis < slot < begin"},
+		{slot: 0, highest: 0, canon: 0, name: "slot == genesis"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			bs, err := hist.HighestSlotBlocksBelow(ctx, c.slot+1)
+			require.NoError(t, err)
+			require.Equal(t, len(bs), 1)
+			r, err := bs[0].Block().HashTreeRoot()
+			require.NoError(t, err)
+			require.Equal(t, hist.slotMap[c.highest], r)
+			cr, _, err := cc.canonicalBlockForSlot(ctx, c.slot)
+			require.NoError(t, err)
+			require.Equal(t, hist.slotMap[c.canon], cr)
+		})
+	}
+}
+
+func TestCanonicalBlockForSlotNonHappy(t *testing.T) {
+	ctx := context.Background()
+	var begin, middle, end types.Slot = 100, 150, 155
+	specs := []mockHistorySpec{
+		{slot: begin},
+		{slot: middle, savedState: true},
+		{slot: end, canonicalBlock: true},
+	}
+	hist := newMockHistory(t, specs, end+1)
+
+	slotOrderObserved := make([]types.Slot, 0)
+	derp := errors.New("HighestSlotBlocksBelow don't work")
+	// since only the end block and genesis are canonical, once the slot drops below
+	// end, we should always get genesis
+	cases := []struct {
+		name              string
+		slot              types.Slot
+		canon             CanonicalChecker
+		overrideHighest   func(context.Context, types.Slot) ([]block.SignedBeaconBlock, error)
+		slotOrderExpected []types.Slot
+		err               error
+		root              [32]byte
+	}{
+		{
+			name: "HigestSlotBlocksBelow not called for genesis",
+			overrideHighest: func(_ context.Context, _ types.Slot) ([]block.SignedBeaconBlock, error) {
+				return nil, derp
+			},
+			root: hist.slotMap[0],
+		},
+		{
+			name: "wrapped error from HigestSlotBlocksBelow returned",
+			err:  derp,
+			overrideHighest: func(_ context.Context, _ types.Slot) ([]block.SignedBeaconBlock, error) {
+				return nil, derp
+			},
+			slot: end,
+		},
+		{
+			name: "HigestSlotBlocksBelow empty list",
+			err:  ErrNoBlocksBelowSlot,
+			overrideHighest: func(_ context.Context, _ types.Slot) ([]block.SignedBeaconBlock, error) {
+				return []block.SignedBeaconBlock{}, nil
+			},
+			slot: end,
+		},
+		{
+			name:  "HigestSlotBlocksBelow no canonical",
+			err:   ErrNoCanonicalBlockForSlot,
+			canon: &mockCanonicalChecker{is: false},
+			slot:  end,
+		},
+		{
+			name: "slot ordering correct - only genesis canonical",
+			canon: &mockCanonicalChecker{isCanon: func(root [32]byte) (bool, error) {
+				if root == hist.slotMap[0] {
+					return true, nil
+				}
+				return false, nil
+			}},
+			overrideHighest: func(_ context.Context, s types.Slot) ([]block.SignedBeaconBlock, error) {
+				slotOrderObserved = append(slotOrderObserved, s)
+				// this allows the mock HighestSlotBlocksBelow to continue to execute now that we've recorded
+				// the slot in our channel
+				return nil, errFallThroughOverride
+			},
+			slotOrderExpected: []types.Slot{156, 155, 150, 100},
+			slot:              end,
+			root:              hist.slotMap[0],
+		},
+		{
+			name: "slot ordering correct - slot 100 canonical",
+			canon: &mockCanonicalChecker{isCanon: func(root [32]byte) (bool, error) {
+				if root == hist.slotMap[100] {
+					return true, nil
+				}
+				return false, nil
+			}},
+			overrideHighest: func(_ context.Context, s types.Slot) ([]block.SignedBeaconBlock, error) {
+				slotOrderObserved = append(slotOrderObserved, s)
+				// this allows the mock HighestSlotBlocksBelow to continue to execute now that we've recorded
+				// the slot in our channel
+				return nil, errFallThroughOverride
+			},
+			slotOrderExpected: []types.Slot{156, 155, 150},
+			slot:              end,
+			root:              hist.slotMap[100],
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			var canon CanonicalChecker = hist
+			if c.canon != nil {
+				canon = c.canon
+			}
+			cc := canonicalChainer{h: hist, c: canon, cs: hist}
+			hist.overrideHighestSlotBlocksBelow = c.overrideHighest
+			r, _, err := cc.canonicalBlockForSlot(ctx, c.slot)
+			if c.err == nil {
+				require.NoError(t, err)
+			} else {
+				require.ErrorIs(t, err, c.err)
+			}
+			if len(c.slotOrderExpected) > 0 {
+				require.Equal(t, len(c.slotOrderExpected), len(slotOrderObserved), "HighestSlotBlocksBelow not called the expected number of times")
+				for i := range c.slotOrderExpected {
+					require.Equal(t, c.slotOrderExpected[i], slotOrderObserved[i])
+				}
+			}
+			if c.root != [32]byte{} {
+				require.Equal(t, c.root, r)
+			}
+			slotOrderObserved = make([]types.Slot, 0)
+		})
+	}
+}
+
+type mockCurrentSlotter struct {
+	Slot types.Slot
+}
+
+func (c *mockCurrentSlotter) CurrentSlot() types.Slot {
+	return c.Slot
+}
+
+var _ CurrentSlotter = &mockCurrentSlotter{}
+
+func TestCanonicalChainerFuture(t *testing.T) {
+	r := &canonicalChainer{
+		cs: &mockCurrentSlotter{Slot: 0},
+	}
+	_, _, err := r.chainForSlot(context.Background(), 1)
+	require.ErrorIs(t, err, ErrFutureSlotRequested)
+}
+
+func TestAncestorChainCache(t *testing.T) {
+	ctx := context.Background()
+	var begin, middle, end types.Slot = 100, 150, 155
+	specs := []mockHistorySpec{
+		{slot: begin, canonicalBlock: true},
+		{slot: middle, canonicalBlock: true},
+		{slot: end, canonicalBlock: true},
+	}
+	hist := newMockHistory(t, specs, end+1)
+	cc := &canonicalChainer{h: hist, c: hist, cs: hist}
+
+	// should only contain the genesis block
+	require.Equal(t, 1, len(hist.states))
+
+	endBlock := hist.blocks[hist.slotMap[end]]
+	st, bs, err := cc.ancestorChain(ctx, endBlock)
+	require.NoError(t, err)
+	require.Equal(t, 3, len(bs))
+	expectedHTR, err := hist.states[hist.slotMap[0]].HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualHTR, err := st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, expectedHTR, actualHTR)
+
+	// now populate the cache, we should get the cached state instead of genesis
+	cc.cache = &mockCachedGetter{
+		cache: map[[32]byte]state.BeaconState{
+			hist.slotMap[end]: hist.hiddenStates[hist.slotMap[end]],
+		},
+	}
+	st, bs, err = cc.ancestorChain(ctx, endBlock)
+	require.NoError(t, err)
+	require.Equal(t, 0, len(bs))
+	expectedHTR, err = hist.hiddenStates[hist.slotMap[end]].HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualHTR, err = st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, expectedHTR, actualHTR)
+
+	// populate cache with a different state for good measure
+	cc.cache = &mockCachedGetter{
+		cache: map[[32]byte]state.BeaconState{
+			hist.slotMap[begin]: hist.hiddenStates[hist.slotMap[begin]],
+		},
+	}
+	st, bs, err = cc.ancestorChain(ctx, endBlock)
+	require.NoError(t, err)
+	require.Equal(t, 2, len(bs))
+	expectedHTR, err = hist.hiddenStates[hist.slotMap[begin]].HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualHTR, err = st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, expectedHTR, actualHTR)
+
+	// rebuild history w/ last state saved, make sure we get that instead of cache
+	specs[2].savedState = true
+	hist = newMockHistory(t, specs, end+1)
+	cc = &canonicalChainer{h: hist, c: hist, cs: hist}
+	cc.cache = &mockCachedGetter{
+		cache: map[[32]byte]state.BeaconState{
+			hist.slotMap[begin]: hist.hiddenStates[hist.slotMap[begin]],
+		},
+	}
+	st, bs, err = cc.ancestorChain(ctx, endBlock)
+	require.NoError(t, err)
+	require.Equal(t, 0, len(bs))
+	expectedHTR, err = hist.states[hist.slotMap[end]].HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualHTR, err = st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, expectedHTR, actualHTR)
+}
+
+func TestAncestorChainOK(t *testing.T) {
+	ctx := context.Background()
+	var begin, middle, end types.Slot = 100, 150, 155
+	specs := []mockHistorySpec{
+		{slot: begin},
+		{slot: middle, savedState: true},
+		{slot: end, canonicalBlock: true},
+	}
+	hist := newMockHistory(t, specs, end+1)
+	cc := &canonicalChainer{h: hist, c: hist, cs: hist}
+
+	endBlock := hist.blocks[hist.slotMap[end]]
+	st, bs, err := cc.ancestorChain(ctx, endBlock)
+	require.NoError(t, err)
+
+	// middle is the most recent slot where savedState == true
+	require.Equal(t, 1, len(bs))
+	require.DeepEqual(t, endBlock, bs[0])
+	expectedHTR, err := hist.states[hist.slotMap[middle]].HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualHTR, err := st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, expectedHTR, actualHTR)
+
+	middleBlock := hist.blocks[hist.slotMap[middle]]
+	st, bs, err = cc.ancestorChain(ctx, middleBlock)
+	require.NoError(t, err)
+	actualHTR, err = st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, 0, len(bs))
+	require.Equal(t, expectedHTR, actualHTR)
+}
+
+func TestChainForSlot(t *testing.T) {
+	ctx := context.Background()
+	var zero, one, two, three types.Slot = 50, 51, 150, 151
+	specs := []mockHistorySpec{
+		{slot: zero, canonicalBlock: true, savedState: true},
+		{slot: one, canonicalBlock: true},
+		{slot: two},
+		{slot: three, canonicalBlock: true},
+	}
+	hist := newMockHistory(t, specs, three+10)
+	cc := &canonicalChainer{h: hist, c: hist, cs: hist}
+	firstNonGenesisRoot := hist.slotMap[zero]
+	nonGenesisStateRoot, err := hist.states[firstNonGenesisRoot].HashTreeRoot(ctx)
+	require.NoError(t, err)
+
+	cases := []struct {
+		name       string
+		slot       types.Slot
+		stateRoot  [32]byte
+		blockRoots [][32]byte
+	}{
+		{
+			name:       "above latest slot (but before current slot)",
+			slot:       three + 1,
+			stateRoot:  nonGenesisStateRoot,
+			blockRoots: [][32]byte{hist.slotMap[one], hist.slotMap[two], hist.slotMap[three]},
+		},
+		{
+			name:       "last canonical slot - two treated as canonical because it is parent of three",
+			slot:       three,
+			stateRoot:  nonGenesisStateRoot,
+			blockRoots: [][32]byte{hist.slotMap[one], hist.slotMap[two], hist.slotMap[three]},
+		},
+		{
+			name:       "non-canonical slot skipped",
+			slot:       two,
+			stateRoot:  nonGenesisStateRoot,
+			blockRoots: [][32]byte{hist.slotMap[one]},
+		},
+		{
+			name:       "first canonical slot",
+			slot:       one,
+			stateRoot:  nonGenesisStateRoot,
+			blockRoots: [][32]byte{hist.slotMap[one]},
+		},
+		{
+			name:       "slot at saved state",
+			slot:       zero,
+			stateRoot:  nonGenesisStateRoot,
+			blockRoots: [][32]byte{},
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			st, blocks, err := cc.chainForSlot(ctx, c.slot)
+			require.NoError(t, err)
+			actualStRoot, err := st.HashTreeRoot(ctx)
+			require.NoError(t, err)
+			require.Equal(t, c.stateRoot, actualStRoot)
+			require.Equal(t, len(c.blockRoots), len(blocks))
+			for i, b := range blocks {
+				root, err := b.Block().HashTreeRoot()
+				require.NoError(t, err)
+				require.Equal(t, c.blockRoots[i], root)
+			}
+		})
+	}
+}
+
+func TestAncestorChainOrdering(t *testing.T) {
+	ctx := context.Background()
+	var zero, one, two, three, four, five types.Slot = 50, 51, 150, 151, 152, 200
+	specs := []mockHistorySpec{
+		{slot: zero},
+		{slot: one, savedState: true},
+		{slot: two},
+		{slot: three},
+		{slot: four},
+		{slot: five},
+	}
+
+	hist := newMockHistory(t, specs, five+1)
+	endRoot := hist.slotMap[specs[len(specs)-1].slot]
+	endBlock := hist.blocks[endRoot]
+
+	cc := &canonicalChainer{h: hist, c: hist, cs: hist}
+	st, bs, err := cc.ancestorChain(ctx, endBlock)
+	require.NoError(t, err)
+	expectedRoot, err := hist.states[hist.slotMap[one]].HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualRoot, err := st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, expectedRoot, actualRoot)
+	// we asked for the chain leading up to five
+	// one has the savedState. one is applied to the savedState, so it should be omitted
+	// that means we should get two, three, four, five (length of 4)
+	require.Equal(t, 4, len(bs))
+	for i, slot := range []types.Slot{two, three, four, five} {
+		require.Equal(t, slot, bs[i].Block().Slot(), fmt.Sprintf("wrong value at index %d", i))
+	}
+
+	// do the same query, but with the final state saved
+	// we should just get the final state w/o block to apply
+	specs[5].savedState = true
+	hist = newMockHistory(t, specs, five+1)
+	endRoot = hist.slotMap[specs[len(specs)-1].slot]
+	endBlock = hist.blocks[endRoot]
+
+	cc = &canonicalChainer{h: hist, c: hist, cs: hist}
+	st, bs, err = cc.ancestorChain(ctx, endBlock)
+	require.NoError(t, err)
+	expectedRoot, err = hist.states[endRoot].HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualRoot, err = st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, expectedRoot, actualRoot)
+	require.Equal(t, 0, len(bs))
+
+	// slice off the last element for an odd size list (to cover odd/even in the reverseChain func)
+	specs = specs[:len(specs)-1]
+	require.Equal(t, 5, len(specs))
+	hist = newMockHistory(t, specs, five+1)
+
+	cc = &canonicalChainer{h: hist, c: hist, cs: hist}
+	endRoot = hist.slotMap[specs[len(specs)-1].slot]
+	endBlock = hist.blocks[endRoot]
+	st, bs, err = cc.ancestorChain(ctx, endBlock)
+	require.NoError(t, err)
+	expectedRoot, err = hist.states[hist.slotMap[one]].HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualRoot, err = st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, expectedRoot, actualRoot)
+	require.Equal(t, 3, len(bs))
+	for i, slot := range []types.Slot{two, three, four} {
+		require.Equal(t, slot, bs[i].Block().Slot(), fmt.Sprintf("wrong value at index %d", i))
+	}
+}
+
+func TestBestForSlot(t *testing.T) {
+	nilBlock, err := wrapper.WrappedSignedBeaconBlock(&ethpb.SignedBeaconBlock{})
+	require.NoError(t, err)
+	nilBody, err := wrapper.WrappedSignedBeaconBlock(&ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{}})
+	require.NoError(t, err)
+	derp := errors.New("fake hash tree root method no hash good")
+	badHTR := &mock.SignedBeaconBlock{BeaconBlock: &mock.BeaconBlock{HtrErr: derp, BeaconBlockBody: &mock.BeaconBlockBody{}}}
+	var goodHTR [32]byte
+	copy(goodHTR[:], []byte{23})
+	var betterHTR [32]byte
+	copy(betterHTR[:], []byte{42})
+	good := &mock.SignedBeaconBlock{BeaconBlock: &mock.BeaconBlock{BeaconBlockBody: &mock.BeaconBlockBody{}, Htr: goodHTR}}
+	better := &mock.SignedBeaconBlock{BeaconBlock: &mock.BeaconBlock{BeaconBlockBody: &mock.BeaconBlockBody{}, Htr: betterHTR}}
+
+	cases := []struct {
+		name   string
+		err    error
+		blocks []block.SignedBeaconBlock
+		best   block.SignedBeaconBlock
+		root   [32]byte
+		cc     CanonicalChecker
+	}{
+		{
+			name:   "empty list",
+			err:    ErrNoCanonicalBlockForSlot,
+			blocks: []block.SignedBeaconBlock{},
+		},
+		{
+			name:   "empty SignedBeaconBlock",
+			err:    ErrNoCanonicalBlockForSlot,
+			blocks: []block.SignedBeaconBlock{nil},
+		},
+		{
+			name:   "empty BeaconBlock",
+			err:    ErrNoCanonicalBlockForSlot,
+			blocks: []block.SignedBeaconBlock{nilBlock},
+		},
+		{
+			name:   "empty BeaconBlockBody",
+			err:    ErrNoCanonicalBlockForSlot,
+			blocks: []block.SignedBeaconBlock{nilBody},
+		},
+		{
+			name:   "bad HTR",
+			err:    ErrInvalidDBBlock,
+			blocks: []block.SignedBeaconBlock{badHTR},
+		},
+		{
+			name:   "IsCanonical fail",
+			blocks: []block.SignedBeaconBlock{good, better},
+			cc:     &mockCanonicalChecker{is: true, err: derp},
+			err:    derp,
+		},
+		{
+			name:   "all non-canonical",
+			err:    ErrNoCanonicalBlockForSlot,
+			blocks: []block.SignedBeaconBlock{good, better},
+			cc:     &mockCanonicalChecker{is: false},
+		},
+		{
+			name:   "one canonical",
+			blocks: []block.SignedBeaconBlock{good},
+			cc:     &mockCanonicalChecker{is: true},
+			root:   goodHTR,
+			best:   good,
+		},
+		{
+			name:   "all canonical",
+			blocks: []block.SignedBeaconBlock{better, good},
+			cc:     &mockCanonicalChecker{is: true},
+			root:   betterHTR,
+			best:   better,
+		},
+		{
+			name:   "first wins",
+			blocks: []block.SignedBeaconBlock{good, better},
+			cc:     &mockCanonicalChecker{is: true},
+			root:   goodHTR,
+			best:   good,
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			chk := CanonicalChecker(&mockCanonicalChecker{is: true})
+			if c.cc != nil {
+				chk = c.cc
+			}
+			cc := &canonicalChainer{c: chk}
+			r, b, err := cc.bestForSlot(context.Background(), c.blocks)
+			if c.err == nil {
+				require.NoError(t, err)
+				require.DeepEqual(t, c.best, b)
+				require.Equal(t, c.root, r)
+			} else {
+				require.ErrorIs(t, err, c.err)
+			}
+		})
+	}
+}
+
+type mockCanonicalChecker struct {
+	isCanon func([32]byte) (bool, error)
+	is      bool
+	err     error
+}
+
+func (m *mockCanonicalChecker) IsCanonical(_ context.Context, root [32]byte) (bool, error) {
+	if m.isCanon != nil {
+		return m.isCanon(root)
+	}
+	return m.is, m.err
+}
+
+func TestReverseChain(t *testing.T) {
+	// test 0,1,2,3 elements to handle: zero case; single element; even number; odd number
+	for i := 0; i < 4; i++ {
+		t.Run(fmt.Sprintf("reverseChain with %d elements", i), func(t *testing.T) {
+			actual := mockBlocks(i, incrFwd)
+			expected := mockBlocks(i, incrBwd)
+			reverseChain(actual)
+			if len(actual) != len(expected) {
+				t.Errorf("different list lengths")
+			}
+			for i := 0; i < len(actual); i++ {
+				sblockA, ok := actual[i].(*mock.SignedBeaconBlock)
+				require.Equal(t, true, ok)
+				blockA, ok := sblockA.BeaconBlock.(*mock.BeaconBlock)
+				require.Equal(t, true, ok)
+				sblockE, ok := expected[i].(*mock.SignedBeaconBlock)
+				require.Equal(t, true, ok)
+				blockE, ok := sblockE.BeaconBlock.(*mock.BeaconBlock)
+				require.Equal(t, true, ok)
+				require.Equal(t, blockA.Htr, blockE.Htr)
+			}
+		})
+	}
+}
+
+func incrBwd(n int, c chan uint32) {
+	for i := n - 1; i >= 0; i-- {
+		c <- uint32(i)
+	}
+	close(c)
+}
+
+func incrFwd(n int, c chan uint32) {
+	for i := 0; i < n; i++ {
+		c <- uint32(i)
+	}
+	close(c)
+}
+
+func mockBlocks(n int, iter func(int, chan uint32)) []block.SignedBeaconBlock {
+	bchan := make(chan uint32)
+	go iter(n, bchan)
+	mb := make([]block.SignedBeaconBlock, 0)
+	for i := range bchan {
+		h := [32]byte{}
+		binary.LittleEndian.PutUint32(h[:], i)
+		b := &mock.SignedBeaconBlock{BeaconBlock: &mock.BeaconBlock{BeaconBlockBody: &mock.BeaconBlockBody{}, Htr: h}}
+		mb = append(mb, b)
+	}
+	return mb
+}

proto/prysm/v1alpha1/block/mock/BUILD.bazel

@@ -0,0 +1,17 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["block.go"],
+    importpath = "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block/mock",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//proto/engine/v1:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
+        "//proto/prysm/v1alpha1/block:go_default_library",
+        "//proto/prysm/v1alpha1/validator-client:go_default_library",
+        "@com_github_ferranbt_fastssz//:go_default_library",
+        "@com_github_prysmaticlabs_eth2_types//:go_default_library",
+        "@org_golang_google_protobuf//proto:go_default_library",
+    ],
+)

proto/prysm/v1alpha1/block/mock/block.go

@@ -0,0 +1,200 @@
+package mock
+
+import (
+	ssz "github.com/ferranbt/fastssz"
+	types "github.com/prysmaticlabs/eth2-types"
+	enginev1 "github.com/prysmaticlabs/prysm/proto/engine/v1"
+	eth "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
+	validatorpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/validator-client"
+	"google.golang.org/protobuf/proto"
+)
+
+type SignedBeaconBlock struct {
+	BeaconBlock block.BeaconBlock
+}
+
+func (SignedBeaconBlock) PbGenericBlock() (*eth.GenericSignedBeaconBlock, error) {
+	panic("implement me")
+}
+
+func (m SignedBeaconBlock) Block() block.BeaconBlock {
+	return m.BeaconBlock
+}
+
+func (SignedBeaconBlock) Signature() []byte {
+	panic("implement me")
+}
+
+func (m SignedBeaconBlock) IsNil() bool {
+	return m.BeaconBlock == nil || m.Block().IsNil()
+}
+
+func (SignedBeaconBlock) Copy() block.SignedBeaconBlock {
+	panic("implement me")
+}
+
+func (SignedBeaconBlock) Proto() proto.Message {
+	panic("implement me")
+}
+
+func (SignedBeaconBlock) PbPhase0Block() (*eth.SignedBeaconBlock, error) {
+	panic("implement me")
+}
+
+func (SignedBeaconBlock) PbAltairBlock() (*eth.SignedBeaconBlockAltair, error) {
+	panic("implement me")
+}
+
+func (SignedBeaconBlock) PbBellatrixBlock() (*eth.SignedBeaconBlockBellatrix, error) {
+	panic("implement me")
+}
+
+func (SignedBeaconBlock) MarshalSSZTo(_ []byte) ([]byte, error) {
+	panic("implement me")
+}
+
+func (SignedBeaconBlock) MarshalSSZ() ([]byte, error) {
+	panic("implement me")
+}
+
+func (SignedBeaconBlock) SizeSSZ() int {
+	panic("implement me")
+}
+
+func (SignedBeaconBlock) UnmarshalSSZ(_ []byte) error {
+	panic("implement me")
+}
+
+func (SignedBeaconBlock) Version() int {
+	panic("implement me")
+}
+
+func (SignedBeaconBlock) Header() (*eth.SignedBeaconBlockHeader, error) {
+	panic("implement me")
+}
+
+type BeaconBlock struct {
+	Htr             [32]byte
+	HtrErr          error
+	BeaconBlockBody block.BeaconBlockBody
+	BlockSlot       types.Slot
+}
+
+func (BeaconBlock) AsSignRequestObject() validatorpb.SignRequestObject {
+	panic("implement me")
+}
+
+func (m BeaconBlock) HashTreeRoot() ([32]byte, error) {
+	return m.Htr, m.HtrErr
+}
+
+func (m BeaconBlock) Slot() types.Slot {
+	return m.BlockSlot
+}
+
+func (BeaconBlock) ProposerIndex() types.ValidatorIndex {
+	panic("implement me")
+}
+
+func (BeaconBlock) ParentRoot() []byte {
+	panic("implement me")
+}
+
+func (BeaconBlock) StateRoot() []byte {
+	panic("implement me")
+}
+
+func (m BeaconBlock) Body() block.BeaconBlockBody {
+	return m.BeaconBlockBody
+}
+
+func (BeaconBlock) IsNil() bool {
+	return false
+}
+
+func (BeaconBlock) Proto() proto.Message {
+	panic("implement me")
+}
+
+func (BeaconBlock) MarshalSSZTo(_ []byte) ([]byte, error) {
+	panic("implement me")
+}
+
+func (BeaconBlock) MarshalSSZ() ([]byte, error) {
+	panic("implement me")
+}
+
+func (BeaconBlock) SizeSSZ() int {
+	panic("implement me")
+}
+
+func (BeaconBlock) UnmarshalSSZ(_ []byte) error {
+	panic("implement me")
+}
+
+func (BeaconBlock) HashTreeRootWith(_ *ssz.Hasher) error {
+	panic("implement me")
+}
+
+func (BeaconBlock) Version() int {
+	panic("implement me")
+}
+
+type BeaconBlockBody struct{}
+
+func (BeaconBlockBody) RandaoReveal() []byte {
+	panic("implement me")
+}
+
+func (BeaconBlockBody) Eth1Data() *eth.Eth1Data {
+	panic("implement me")
+}
+
+func (BeaconBlockBody) Graffiti() []byte {
+	panic("implement me")
+}
+
+func (BeaconBlockBody) ProposerSlashings() []*eth.ProposerSlashing {
+	panic("implement me")
+}
+
+func (BeaconBlockBody) AttesterSlashings() []*eth.AttesterSlashing {
+	panic("implement me")
+}
+
+func (BeaconBlockBody) Attestations() []*eth.Attestation {
+	panic("implement me")
+}
+
+func (BeaconBlockBody) Deposits() []*eth.Deposit {
+	panic("implement me")
+}
+
+func (BeaconBlockBody) VoluntaryExits() []*eth.SignedVoluntaryExit {
+	panic("implement me")
+}
+
+func (BeaconBlockBody) SyncAggregate() (*eth.SyncAggregate, error) {
+	panic("implement me")
+}
+
+func (BeaconBlockBody) IsNil() bool {
+	return false
+}
+
+func (BeaconBlockBody) HashTreeRoot() ([32]byte, error) {
+	panic("implement me")
+}
+
+func (BeaconBlockBody) Proto() proto.Message {
+	panic("implement me")
+}
+
+func (BeaconBlockBody) ExecutionPayload() (*enginev1.ExecutionPayload, error) {
+	panic("implement me")
+}
+
+var _ block.SignedBeaconBlock = &SignedBeaconBlock{}
+var _ block.BeaconBlock = &BeaconBlock{}
+var _ block.BeaconBlockBody = &BeaconBlockBody{}

proto/prysm/v1alpha1/wrapper/BUILD.bazel

@@ -8,6 +8,7 @@ go_library(
         "beacon_block_bellatrix.go",
         "beacon_block_phase0.go",
         "metadata.go",
+        "mutator.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper",
     visibility = ["//visibility:public"],

proto/prysm/v1alpha1/wrapper/mutator.go

@@ -0,0 +1,77 @@
+package wrapper
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+	types "github.com/prysmaticlabs/eth2-types"
+	eth "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
+	"github.com/prysmaticlabs/prysm/runtime/version"
+)
+
+type BlockMutator struct {
+	Phase0    func(beaconBlock *eth.SignedBeaconBlock)
+	Altair    func(beaconBlock *eth.SignedBeaconBlockAltair)
+	Bellatrix func(beaconBlock *eth.SignedBeaconBlockBellatrix)
+}
+
+func (m BlockMutator) Apply(b block.SignedBeaconBlock) error {
+	switch b.Version() {
+	case version.Phase0:
+		bb, err := b.PbPhase0Block()
+		if err != nil {
+			return err
+		}
+		m.Phase0(bb)
+		return nil
+	case version.Altair:
+		bb, err := b.PbAltairBlock()
+		if err != nil {
+			return err
+		}
+		m.Altair(bb)
+		return nil
+	case version.Bellatrix:
+		bb, err := b.PbBellatrixBlock()
+		if err != nil {
+			return err
+		}
+		m.Bellatrix(bb)
+		return nil
+	}
+	msg := fmt.Sprintf("version %d = %s", b.Version(), version.String(b.Version()))
+	return errors.Wrap(ErrUnsupportedSignedBeaconBlock, msg)
+}
+
+func SetBlockStateRoot(b block.SignedBeaconBlock, sr [32]byte) error {
+	return BlockMutator{
+		Phase0:    func(bb *eth.SignedBeaconBlock) { bb.Block.StateRoot = sr[:] },
+		Altair:    func(bb *eth.SignedBeaconBlockAltair) { bb.Block.StateRoot = sr[:] },
+		Bellatrix: func(bb *eth.SignedBeaconBlockBellatrix) { bb.Block.StateRoot = sr[:] },
+	}.Apply(b)
+}
+
+func SetBlockParentRoot(b block.SignedBeaconBlock, pr [32]byte) error {
+	return BlockMutator{
+		Phase0:    func(bb *eth.SignedBeaconBlock) { bb.Block.ParentRoot = pr[:] },
+		Altair:    func(bb *eth.SignedBeaconBlockAltair) { bb.Block.ParentRoot = pr[:] },
+		Bellatrix: func(bb *eth.SignedBeaconBlockBellatrix) { bb.Block.ParentRoot = pr[:] },
+	}.Apply(b)
+}
+
+func SetBlockSlot(b block.SignedBeaconBlock, s types.Slot) error {
+	return BlockMutator{
+		Phase0:    func(bb *eth.SignedBeaconBlock) { bb.Block.Slot = s },
+		Altair:    func(bb *eth.SignedBeaconBlockAltair) { bb.Block.Slot = s },
+		Bellatrix: func(bb *eth.SignedBeaconBlockBellatrix) { bb.Block.Slot = s },
+	}.Apply(b)
+}
+
+func SetProposerIndex(b block.SignedBeaconBlock, idx types.ValidatorIndex) error {
+	return BlockMutator{
+		Phase0:    func(bb *eth.SignedBeaconBlock) { bb.Block.ProposerIndex = idx },
+		Altair:    func(bb *eth.SignedBeaconBlockAltair) { bb.Block.ProposerIndex = idx },
+		Bellatrix: func(bb *eth.SignedBeaconBlockBellatrix) { bb.Block.ProposerIndex = idx },
+	}.Apply(b)
+}

testing/util/BUILD.bazel

@@ -44,9 +44,11 @@ go_library(
         "//proto/eth/v1:go_default_library",
         "//proto/eth/v2:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
+        "//proto/prysm/v1alpha1/block:go_default_library",
         "//proto/prysm/v1alpha1/wrapper:go_default_library",
         "//runtime/interop:go_default_library",
         "//runtime/version:go_default_library",
+        "//testing/require:go_default_library",
         "//time/slots:go_default_library",
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_pkg_errors//:go_default_library",

testing/util/state.go

@@ -2,6 +2,10 @@ package util
 
 import (
 	"fmt"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
+	"github.com/prysmaticlabs/prysm/testing/require"
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	types "github.com/prysmaticlabs/eth2-types"
@@ -54,8 +58,26 @@ func FillRootsNaturalOptBellatrix(state *ethpb.BeaconStateBellatrix) error {
 	return nil
 }
 
+func WithStateSlot(slot types.Slot) NewBeaconStateOption {
+	return func(st *ethpb.BeaconState) error {
+		st.Slot = slot
+		return nil
+	}
+}
+
+func WithLatestHeaderFromBlock(t *testing.T, b block.SignedBeaconBlock) NewBeaconStateOption {
+	return func(st *ethpb.BeaconState) error {
+		sh, err := b.Header()
+		require.NoError(t, err)
+		st.LatestBlockHeader = sh.Header
+		return nil
+	}
+}
+
+type NewBeaconStateOption func(state *ethpb.BeaconState) error
+
 // NewBeaconState creates a beacon state with minimum marshalable fields.
-func NewBeaconState(options ...func(state *ethpb.BeaconState) error) (state.BeaconState, error) {
+func NewBeaconState(options ...NewBeaconStateOption) (state.BeaconState, error) {
 	seed := &ethpb.BeaconState{
 		BlockRoots:                 filledByteSlice2D(uint64(params.MainnetConfig().SlotsPerHistoricalRoot), 32),
 		StateRoots:                 filledByteSlice2D(uint64(params.MainnetConfig().SlotsPerHistoricalRoot), 32),