Fix test

the replayer code already ensures that the block chosen for replay is
canonical, so searching backwards for the canonical block in the rpc
package is redundant.

WORKSPACE

@@ -183,7 +183,7 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe
 go_rules_dependencies()
 
 go_register_toolchains(
-    go_version = "1.16.4",
+    go_version = "1.17.6",
     nogo = "@//:nogo",
 )
 
@@ -349,9 +349,9 @@ filegroup(
     visibility = ["//visibility:public"],
 )
 """,
-    sha256 = "f196fe4367c2d2d01d36565c0dc6eecfa4f03adba1fc03a61d62953fce606e1f",
+    sha256 = "4797a7e594a5b1f4c1c8080701613f3ee451b01ec0861499ea7d9b60877a6b23",
     urls = [
-        "https://github.com/prysmaticlabs/prysm-web-ui/releases/download/v1.0.2/prysm-web-ui.tar.gz",
+        "https://github.com/prysmaticlabs/prysm-web-ui/releases/download/v1.0.3/prysm-web-ui.tar.gz",
     ],
 )
 

api/gateway/apimiddleware/api_middleware.go

@@ -3,6 +3,7 @@ package apimiddleware
 import (
 	"net/http"
 	"reflect"
+	"time"
 
 	"github.com/gorilla/mux"
 )
@@ -14,6 +15,7 @@ import (
 type ApiProxyMiddleware struct {
 	GatewayAddress  string
 	EndpointCreator EndpointFactory
+	Timeout         time.Duration
 	router          *mux.Router
 }
 
@@ -120,7 +122,7 @@ func (m *ApiProxyMiddleware) WithMiddleware(path string) http.HandlerFunc {
 			WriteError(w, errJson, nil)
 			return
 		}
-		grpcResp, errJson := ProxyRequest(req)
+		grpcResp, errJson := m.ProxyRequest(req)
 		if errJson != nil {
 			WriteError(w, errJson, nil)
 			return

api/gateway/apimiddleware/process_request.go

@@ -5,10 +5,10 @@ import (
 	"encoding/json"
 	"io"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/api/grpc"
@@ -75,11 +75,14 @@ func (m *ApiProxyMiddleware) PrepareRequestForProxying(endpoint Endpoint, req *h
 }
 
 // ProxyRequest proxies the request to grpc-gateway.
-func ProxyRequest(req *http.Request) (*http.Response, ErrorJson) {
+func (m *ApiProxyMiddleware) ProxyRequest(req *http.Request) (*http.Response, ErrorJson) {
 	// We do not use http.DefaultClient because it does not have any timeout.
-	netClient := &http.Client{Timeout: time.Minute * 2}
+	netClient := &http.Client{Timeout: m.Timeout}
 	grpcResp, err := netClient.Do(req)
 	if err != nil {
+		if err, ok := err.(net.Error); ok && err.Timeout() {
+			return nil, TimeoutError()
+		}
 		return nil, InternalServerErrorWithMessage(err, "could not proxy request")
 	}
 	if grpcResp == nil {
@@ -111,9 +114,14 @@ func HandleGrpcResponseError(errJson ErrorJson, resp *http.Response, respBody []
 				w.Header().Set(h, v)
 			}
 		}
-		// Set code to HTTP code because unmarshalled body contained gRPC code.
-		errJson.SetCode(resp.StatusCode)
-		WriteError(w, errJson, resp.Header)
+		// Handle gRPC timeout.
+		if resp.StatusCode == http.StatusGatewayTimeout {
+			WriteError(w, TimeoutError(), resp.Header)
+		} else {
+			// Set code to HTTP code because unmarshalled body contained gRPC code.
+			errJson.SetCode(resp.StatusCode)
+			WriteError(w, errJson, resp.Header)
+		}
 	}
 	return responseHasError, nil
 }

api/gateway/apimiddleware/structs.go

@@ -41,6 +41,13 @@ func InternalServerError(err error) *DefaultErrorJson {
 	}
 }
 
+func TimeoutError() *DefaultErrorJson {
+	return &DefaultErrorJson{
+		Message: "Request timeout",
+		Code:    http.StatusRequestTimeout,
+	}
+}
+
 // StatusCode returns the error's underlying error code.
 func (e *DefaultErrorJson) StatusCode() int {
 	return e.Code

api/gateway/gateway.go

@@ -52,6 +52,7 @@ type config struct {
 	muxHandler                   MuxHandler
 	pbHandlers                   []*PbMux
 	router                       *mux.Router
+	timeout                      time.Duration
 }
 
 // Gateway is the gRPC gateway to serve HTTP JSON traffic as a proxy and forward it to the gRPC server.
@@ -248,6 +249,7 @@ func (g *Gateway) registerApiMiddleware() {
 	g.proxy = &apimiddleware.ApiProxyMiddleware{
 		GatewayAddress:  g.cfg.gatewayAddr,
 		EndpointCreator: g.cfg.apiMiddlewareEndpointFactory,
+		Timeout:         g.cfg.timeout,
 	}
 	log.Info("Starting API middleware")
 	g.proxy.Run(g.cfg.router)

api/gateway/options.go

@@ -1,7 +1,10 @@
 package gateway
 
 import (
+	"time"
+
 	"github.com/gorilla/mux"
+	gwruntime "github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/prysmaticlabs/prysm/api/gateway/apimiddleware"
 )
 
@@ -79,3 +82,12 @@ func WithApiMiddleware(endpointFactory apimiddleware.EndpointFactory) Option {
 		return nil
 	}
 }
+
+// WithTimeout allows changing the timeout value for API calls.
+func WithTimeout(seconds uint64) Option {
+	return func(g *Gateway) error {
+		g.cfg.timeout = time.Second * time.Duration(seconds)
+		gwruntime.DefaultContextTimeout = time.Second * time.Duration(seconds)
+		return nil
+	}
+}

beacon-chain/blockchain/BUILD.bazel

@@ -56,6 +56,7 @@ go_library(
         "//beacon-chain/operations/voluntaryexits:go_default_library",
         "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
+        "//beacon-chain/powchain/engine-api-client/v1:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/stategen:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",
@@ -65,6 +66,7 @@ go_library(
         "//crypto/bls:go_default_library",
         "//encoding/bytesutil:go_default_library",
         "//monitoring/tracing:go_default_library",
+        "//proto/engine/v1:go_default_library",
         "//proto/eth/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//proto/prysm/v1alpha1/attestation:go_default_library",
@@ -73,6 +75,8 @@ go_library(
         "//time:go_default_library",
         "//time/slots:go_default_library",
         "@com_github_emicklei_dot//:go_default_library",
+        "@com_github_ethereum_go_ethereum//common:go_default_library",
+        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_holiman_uint256//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
@@ -104,6 +108,7 @@ go_test(
         "init_test.go",
         "log_test.go",
         "metrics_test.go",
+        "mock_engine_test.go",
         "mock_test.go",
         "optimistic_sync_test.go",
         "pow_block_test.go",

beacon-chain/blockchain/chain_info.go

@@ -13,6 +13,7 @@ import (
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
+	"github.com/prysmaticlabs/prysm/time/slots"
 	"go.opencensus.io/trace"
 )
 
@@ -334,6 +335,10 @@ func (s *Service) HeadValidatorIndexToPublicKey(_ context.Context, index types.V
 func (s *Service) IsOptimistic(ctx context.Context) (bool, error) {
 	s.headLock.RLock()
 	defer s.headLock.RUnlock()
+	if slots.ToEpoch(s.CurrentSlot()) < params.BeaconConfig().BellatrixForkEpoch {
+		return false, nil
+	}
+
 	return s.cfg.ForkChoiceStore.Optimistic(ctx, s.head.root, s.head.slot)
 }
 

beacon-chain/blockchain/chain_info_test.go

@@ -357,6 +357,11 @@ func TestService_HeadValidatorIndexToPublicKeyNil(t *testing.T) {
 }
 
 func TestService_IsOptimistic(t *testing.T) {
+	params.SetupTestConfigCleanup(t)
+	cfg := params.BeaconConfig()
+	cfg.BellatrixForkEpoch = 0
+	params.OverrideBeaconConfig(cfg)
+
 	ctx := context.Background()
 	c := &Service{cfg: &config{ForkChoiceStore: protoarray.New(0, 0, [32]byte{})}, head: &head{slot: 101, root: [32]byte{'b'}}}
 	require.NoError(t, c.cfg.ForkChoiceStore.ProcessBlock(ctx, 100, [32]byte{'a'}, [32]byte{}, [32]byte{}, 0, 0))
@@ -367,6 +372,14 @@ func TestService_IsOptimistic(t *testing.T) {
 	require.Equal(t, true, opt)
 }
 
+func TestService_IsOptimisticBeforeBellatrix(t *testing.T) {
+	ctx := context.Background()
+	c := &Service{genesisTime: time.Now()}
+	opt, err := c.IsOptimistic(ctx)
+	require.NoError(t, err)
+	require.Equal(t, false, opt)
+}
+
 func TestService_IsOptimisticForRoot(t *testing.T) {
 	ctx := context.Background()
 	c := &Service{cfg: &config{ForkChoiceStore: protoarray.New(0, 0, [32]byte{})}, head: &head{slot: 101, root: [32]byte{'b'}}}

beacon-chain/blockchain/mock_engine_test.go

@@ -0,0 +1,49 @@
+package blockchain
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/pkg/errors"
+	enginev1 "github.com/prysmaticlabs/prysm/proto/engine/v1"
+)
+
+type mockEngineService struct {
+	newPayloadError error
+	forkchoiceError error
+	blks            map[[32]byte]*enginev1.ExecutionBlock
+}
+
+func (m *mockEngineService) NewPayload(context.Context, *enginev1.ExecutionPayload) ([]byte, error) {
+	return nil, m.newPayloadError
+}
+
+func (m *mockEngineService) ForkchoiceUpdated(context.Context, *enginev1.ForkchoiceState, *enginev1.PayloadAttributes) (*enginev1.PayloadIDBytes, []byte, error) {
+	return nil, nil, m.forkchoiceError
+}
+
+func (*mockEngineService) GetPayloadV1(
+	_ context.Context, _ enginev1.PayloadIDBytes,
+) *enginev1.ExecutionPayload {
+	return nil
+}
+
+func (*mockEngineService) GetPayload(context.Context, [8]byte) (*enginev1.ExecutionPayload, error) {
+	return nil, nil
+}
+
+func (*mockEngineService) ExchangeTransitionConfiguration(context.Context, *enginev1.TransitionConfiguration) error {
+	return nil
+}
+
+func (*mockEngineService) LatestExecutionBlock(context.Context) (*enginev1.ExecutionBlock, error) {
+	return nil, nil
+}
+
+func (m *mockEngineService) ExecutionBlockByHash(_ context.Context, hash common.Hash) (*enginev1.ExecutionBlock, error) {
+	blk, ok := m.blks[common.BytesToHash(hash.Bytes())]
+	if !ok {
+		return nil, errors.New("block not found")
+	}
+	return blk, nil
+}

beacon-chain/blockchain/optimistic_sync.go

@@ -2,15 +2,140 @@ package blockchain
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/pkg/errors"
 	types "github.com/prysmaticlabs/eth2-types"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	v1 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/config/params"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	enginev1 "github.com/prysmaticlabs/prysm/proto/engine/v1"
 	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
+	"github.com/prysmaticlabs/prysm/runtime/version"
+	"github.com/sirupsen/logrus"
 )
 
+// notifyForkchoiceUpdate signals execution engine the fork choice updates. Execution engine should:
+// 1. Re-organizes the execution payload chain and corresponding state to make head_block_hash the head.
+// 2. Applies finality to the execution state: it irreversibly persists the chain of all execution payloads and corresponding state, up to and including finalized_block_hash.
+func (s *Service) notifyForkchoiceUpdate(ctx context.Context, headBlk block.BeaconBlock, finalizedRoot [32]byte) (*enginev1.PayloadIDBytes, error) {
+	if headBlk == nil || headBlk.IsNil() || headBlk.Body().IsNil() {
+		return nil, errors.New("nil head block")
+	}
+	// Must not call fork choice updated until the transition conditions are met on the Pow network.
+	if isPreBellatrix(headBlk.Version()) {
+		return nil, nil
+	}
+	isExecutionBlk, err := blocks.ExecutionBlock(headBlk.Body())
+	if err != nil {
+		return nil, errors.Wrap(err, "could not determine if block is execution block")
+	}
+	if !isExecutionBlk {
+		return nil, nil
+	}
+	headPayload, err := headBlk.Body().ExecutionPayload()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get execution payload")
+	}
+	finalizedBlock, err := s.cfg.BeaconDB.Block(ctx, s.ensureRootNotZeros(finalizedRoot))
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get finalized block")
+	}
+	var finalizedHash []byte
+	if isPreBellatrix(finalizedBlock.Block().Version()) {
+		finalizedHash = params.BeaconConfig().ZeroHash[:]
+	} else {
+		payload, err := finalizedBlock.Block().Body().ExecutionPayload()
+		if err != nil {
+			return nil, errors.Wrap(err, "could not get finalized block execution payload")
+		}
+		finalizedHash = payload.BlockHash
+	}
+
+	fcs := &enginev1.ForkchoiceState{
+		HeadBlockHash:      headPayload.BlockHash,
+		SafeBlockHash:      headPayload.BlockHash,
+		FinalizedBlockHash: finalizedHash,
+	}
+
+	// payload attribute is only required when requesting payload, here we are just updating fork choice, so it is nil.
+	payloadID, _, err := s.cfg.ExecutionEngineCaller.ForkchoiceUpdated(ctx, fcs, nil /*payload attribute*/)
+	if err != nil {
+		switch err {
+		case v1.ErrAcceptedSyncingPayloadStatus:
+			log.WithFields(logrus.Fields{
+				"headSlot":      headBlk.Slot(),
+				"headHash":      fmt.Sprintf("%#x", bytesutil.Trunc(headPayload.BlockHash)),
+				"finalizedHash": fmt.Sprintf("%#x", bytesutil.Trunc(finalizedHash)),
+			}).Info("Called fork choice updated with optimistic block")
+			return payloadID, nil
+		default:
+			return nil, errors.Wrap(err, "could not notify forkchoice update from execution engine")
+		}
+	}
+	return payloadID, nil
+}
+
+// notifyForkchoiceUpdate signals execution engine on a new payload
+func (s *Service) notifyNewPayload(ctx context.Context, preState, postState state.BeaconState, blk block.SignedBeaconBlock) error {
+	if preState == nil || postState == nil {
+		return errors.New("pre and post states must not be nil")
+	}
+	// Execution payload is only supported in Bellatrix and beyond.
+	if isPreBellatrix(postState.Version()) {
+		return nil
+	}
+	if err := helpers.BeaconBlockIsNil(blk); err != nil {
+		return err
+	}
+	body := blk.Block().Body()
+	enabled, err := blocks.ExecutionEnabled(postState, blk.Block().Body())
+	if err != nil {
+		return errors.Wrap(err, "could not determine if execution is enabled")
+	}
+	if !enabled {
+		return nil
+	}
+	payload, err := body.ExecutionPayload()
+	if err != nil {
+		return errors.Wrap(err, "could not get execution payload")
+	}
+	_, err = s.cfg.ExecutionEngineCaller.NewPayload(ctx, payload)
+	if err != nil {
+		switch err {
+		case v1.ErrAcceptedSyncingPayloadStatus:
+			log.WithFields(logrus.Fields{
+				"slot":      postState.Slot(),
+				"blockHash": fmt.Sprintf("%#x", bytesutil.Trunc(payload.BlockHash)),
+			}).Info("Called new payload with optimistic block")
+			return nil
+		default:
+			return errors.Wrap(err, "could not validate execution payload from execution engine")
+		}
+	}
+
+	// During the transition event, the transition block should be verified for sanity.
+	if isPreBellatrix(preState.Version()) {
+		return nil
+	}
+	atTransition, err := blocks.MergeTransitionBlock(preState, body)
+	if err != nil {
+		return errors.Wrap(err, "could not check if merge block is terminal")
+	}
+	if !atTransition {
+		return nil
+	}
+	return s.validateMergeBlock(ctx, blk)
+}
+
+// isPreBellatrix returns true if input version is before bellatrix fork.
+func isPreBellatrix(v int) bool {
+	return v == version.Phase0 || v == version.Altair
+}
+
 // optimisticCandidateBlock returns true if this block can be optimistically synced.
 //
 // Spec pseudocode definition:

beacon-chain/blockchain/optimistic_sync_test.go

@@ -7,10 +7,13 @@ import (
 
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	engine "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	v1 "github.com/prysmaticlabs/prysm/proto/engine/v1"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
 	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
@@ -19,6 +22,305 @@ import (
 	"github.com/prysmaticlabs/prysm/time/slots"
 )
 
+func Test_NotifyForkchoiceUpdate(t *testing.T) {
+	ctx := context.Background()
+	beaconDB := testDB.SetupDB(t)
+	altairBlk, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlockAltair())
+	require.NoError(t, err)
+	altairBlkRoot, err := altairBlk.Block().HashTreeRoot()
+	require.NoError(t, err)
+	bellatrixBlk, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlockBellatrix())
+	require.NoError(t, err)
+	bellatrixBlkRoot, err := bellatrixBlk.Block().HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, beaconDB.SaveBlock(ctx, altairBlk))
+	require.NoError(t, beaconDB.SaveBlock(ctx, bellatrixBlk))
+
+	fcs := protoarray.New(0, 0, [32]byte{'a'})
+	opts := []Option{
+		WithDatabase(beaconDB),
+		WithStateGen(stategen.New(beaconDB)),
+		WithForkChoiceStore(fcs),
+	}
+	service, err := NewService(ctx, opts...)
+	require.NoError(t, err)
+
+	tests := []struct {
+		name             string
+		blk              block.BeaconBlock
+		finalizedRoot    [32]byte
+		newForkchoiceErr error
+		errString        string
+	}{
+		{
+			name:      "nil block",
+			errString: "nil head block",
+		},
+		{
+			name: "phase0 block",
+			blk: func() block.BeaconBlock {
+				b, err := wrapper.WrappedBeaconBlock(&ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}})
+				require.NoError(t, err)
+				return b
+			}(),
+		},
+		{
+			name: "altair block",
+			blk: func() block.BeaconBlock {
+				b, err := wrapper.WrappedBeaconBlock(&ethpb.BeaconBlockAltair{Body: &ethpb.BeaconBlockBodyAltair{}})
+				require.NoError(t, err)
+				return b
+			}(),
+		},
+		{
+			name: "not execution block",
+			blk: func() block.BeaconBlock {
+				b, err := wrapper.WrappedBeaconBlock(&ethpb.BeaconBlockBellatrix{
+					Body: &ethpb.BeaconBlockBodyBellatrix{
+						ExecutionPayload: &v1.ExecutionPayload{
+							ParentHash:    make([]byte, fieldparams.RootLength),
+							FeeRecipient:  make([]byte, fieldparams.FeeRecipientLength),
+							StateRoot:     make([]byte, fieldparams.RootLength),
+							ReceiptsRoot:  make([]byte, fieldparams.RootLength),
+							LogsBloom:     make([]byte, fieldparams.LogsBloomLength),
+							PrevRandao:    make([]byte, fieldparams.RootLength),
+							BaseFeePerGas: make([]byte, fieldparams.RootLength),
+							BlockHash:     make([]byte, fieldparams.RootLength),
+						},
+					},
+				})
+				require.NoError(t, err)
+				return b
+			}(),
+		},
+		{
+			name: "happy case: finalized root is altair block",
+			blk: func() block.BeaconBlock {
+				b, err := wrapper.WrappedBeaconBlock(&ethpb.BeaconBlockBellatrix{
+					Body: &ethpb.BeaconBlockBodyBellatrix{
+						ExecutionPayload: &v1.ExecutionPayload{},
+					},
+				})
+				require.NoError(t, err)
+				return b
+			}(),
+			finalizedRoot: altairBlkRoot,
+		},
+		{
+			name: "happy case: finalized root is bellatrix block",
+			blk: func() block.BeaconBlock {
+				b, err := wrapper.WrappedBeaconBlock(&ethpb.BeaconBlockBellatrix{
+					Body: &ethpb.BeaconBlockBodyBellatrix{
+						ExecutionPayload: &v1.ExecutionPayload{},
+					},
+				})
+				require.NoError(t, err)
+				return b
+			}(),
+			finalizedRoot: bellatrixBlkRoot,
+		},
+		{
+			name: "forkchoice updated with optimistic block",
+			blk: func() block.BeaconBlock {
+				b, err := wrapper.WrappedBeaconBlock(&ethpb.BeaconBlockBellatrix{
+					Body: &ethpb.BeaconBlockBodyBellatrix{
+						ExecutionPayload: &v1.ExecutionPayload{},
+					},
+				})
+				require.NoError(t, err)
+				return b
+			}(),
+			newForkchoiceErr: engine.ErrAcceptedSyncingPayloadStatus,
+			finalizedRoot:    bellatrixBlkRoot,
+		},
+		{
+			name: "forkchoice updated with invalid block",
+			blk: func() block.BeaconBlock {
+				b, err := wrapper.WrappedBeaconBlock(&ethpb.BeaconBlockBellatrix{
+					Body: &ethpb.BeaconBlockBodyBellatrix{
+						ExecutionPayload: &v1.ExecutionPayload{},
+					},
+				})
+				require.NoError(t, err)
+				return b
+			}(),
+			newForkchoiceErr: engine.ErrInvalidPayloadStatus,
+			finalizedRoot:    bellatrixBlkRoot,
+			errString:        "could not notify forkchoice update from execution engine: payload status is INVALID",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			engine := &mockEngineService{forkchoiceError: tt.newForkchoiceErr}
+			service.cfg.ExecutionEngineCaller = engine
+			_, err := service.notifyForkchoiceUpdate(ctx, tt.blk, tt.finalizedRoot)
+			if tt.errString != "" {
+				require.ErrorContains(t, tt.errString, err)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
+func Test_NotifyNewPayload(t *testing.T) {
+	cfg := params.BeaconConfig()
+	cfg.TerminalTotalDifficulty = "2"
+	params.OverrideBeaconConfig(cfg)
+
+	ctx := context.Background()
+	beaconDB := testDB.SetupDB(t)
+	fcs := protoarray.New(0, 0, [32]byte{'a'})
+	opts := []Option{
+		WithDatabase(beaconDB),
+		WithStateGen(stategen.New(beaconDB)),
+		WithForkChoiceStore(fcs),
+	}
+	phase0State, _ := util.DeterministicGenesisState(t, 1)
+	altairState, _ := util.DeterministicGenesisStateAltair(t, 1)
+	bellatrixState, _ := util.DeterministicGenesisStateBellatrix(t, 2)
+	blk := &ethpb.SignedBeaconBlockBellatrix{
+		Block: &ethpb.BeaconBlockBellatrix{
+			Body: &ethpb.BeaconBlockBodyBellatrix{
+				ExecutionPayload: &v1.ExecutionPayload{},
+			},
+		},
+	}
+	bellatrixBlk, err := wrapper.WrappedSignedBeaconBlock(blk)
+	require.NoError(t, err)
+	service, err := NewService(ctx, opts...)
+	require.NoError(t, err)
+
+	tests := []struct {
+		name          string
+		preState      state.BeaconState
+		postState     state.BeaconState
+		blk           block.SignedBeaconBlock
+		newPayloadErr error
+		errString     string
+	}{
+		{
+			name:      "phase 0 post state",
+			postState: phase0State,
+			preState:  phase0State,
+		},
+		{
+			name:      "altair post state",
+			postState: altairState,
+			preState:  altairState,
+		},
+		{
+			name:      "nil states",
+			errString: "pre and post states must not be nil",
+		},
+		{
+			name:      "nil beacon block",
+			postState: bellatrixState,
+			preState:  bellatrixState,
+			errString: "signed beacon block can't be nil",
+		},
+		{
+			name:          "new payload with optimistic block",
+			postState:     bellatrixState,
+			preState:      bellatrixState,
+			blk:           bellatrixBlk,
+			newPayloadErr: engine.ErrAcceptedSyncingPayloadStatus,
+		},
+		{
+			name:          "new payload with invalid block",
+			postState:     bellatrixState,
+			preState:      bellatrixState,
+			blk:           bellatrixBlk,
+			newPayloadErr: engine.ErrInvalidPayloadStatus,
+			errString:     "could not validate execution payload from execution engine: payload status is INVALID",
+		},
+		{
+			name:      "altair pre state",
+			postState: bellatrixState,
+			preState:  altairState,
+			blk:       bellatrixBlk,
+		},
+		{
+			name:      "could not get merge block",
+			postState: bellatrixState,
+			preState:  bellatrixState,
+			blk:       bellatrixBlk,
+			errString: "could not get merge block parent hash and total difficulty",
+		},
+		{
+			name:      "not at merge transition",
+			postState: bellatrixState,
+			preState:  bellatrixState,
+			blk: func() block.SignedBeaconBlock {
+				blk := &ethpb.SignedBeaconBlockBellatrix{
+					Block: &ethpb.BeaconBlockBellatrix{
+						Body: &ethpb.BeaconBlockBodyBellatrix{
+							ExecutionPayload: &v1.ExecutionPayload{
+								ParentHash:    make([]byte, fieldparams.RootLength),
+								FeeRecipient:  make([]byte, fieldparams.FeeRecipientLength),
+								StateRoot:     make([]byte, fieldparams.RootLength),
+								ReceiptsRoot:  make([]byte, fieldparams.RootLength),
+								LogsBloom:     make([]byte, fieldparams.LogsBloomLength),
+								PrevRandao:    make([]byte, fieldparams.RootLength),
+								BaseFeePerGas: make([]byte, fieldparams.RootLength),
+								BlockHash:     make([]byte, fieldparams.RootLength),
+							},
+						},
+					},
+				}
+				b, err := wrapper.WrappedSignedBeaconBlock(blk)
+				require.NoError(t, err)
+				return b
+			}(),
+		},
+		{
+			name:      "could not get merge block",
+			postState: bellatrixState,
+			preState:  bellatrixState,
+			blk:       bellatrixBlk,
+			errString: "could not get merge block parent hash and total difficulty",
+		},
+		{
+			name:      "happy case",
+			postState: bellatrixState,
+			preState:  bellatrixState,
+			blk: func() block.SignedBeaconBlock {
+				blk := &ethpb.SignedBeaconBlockBellatrix{
+					Block: &ethpb.BeaconBlockBellatrix{
+						Body: &ethpb.BeaconBlockBodyBellatrix{
+							ExecutionPayload: &v1.ExecutionPayload{
+								ParentHash: bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength),
+							},
+						},
+					},
+				}
+				b, err := wrapper.WrappedSignedBeaconBlock(blk)
+				require.NoError(t, err)
+				return b
+			}(),
+		},
+	}
+	for _, tt := range tests {
+		engine := &mockEngineService{newPayloadError: tt.newPayloadErr, blks: map[[32]byte]*v1.ExecutionBlock{}}
+		engine.blks[[32]byte{'a'}] = &v1.ExecutionBlock{
+			ParentHash:      bytesutil.PadTo([]byte{'b'}, fieldparams.RootLength),
+			TotalDifficulty: "0x2",
+		}
+		engine.blks[[32]byte{'b'}] = &v1.ExecutionBlock{
+			ParentHash:      bytesutil.PadTo([]byte{'3'}, fieldparams.RootLength),
+			TotalDifficulty: "0x1",
+		}
+		service.cfg.ExecutionEngineCaller = engine
+		err := service.notifyNewPayload(ctx, tt.preState, tt.postState, tt.blk)
+		if tt.errString != "" {
+			require.ErrorContains(t, tt.errString, err)
+		} else {
+			require.NoError(t, err)
+		}
+	}
+}
+
 func Test_IsOptimisticCandidateBlock(t *testing.T) {
 	params.SetupTestConfigCleanup(t)
 	params.OverrideBeaconConfig(params.MainnetConfig())
@@ -115,7 +417,7 @@ func Test_IsOptimisticCandidateBlock(t *testing.T) {
 				blk.Block.Body.ExecutionPayload.StateRoot = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
 				blk.Block.Body.ExecutionPayload.ReceiptsRoot = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
 				blk.Block.Body.ExecutionPayload.LogsBloom = bytesutil.PadTo([]byte{'a'}, fieldparams.LogsBloomLength)
-				blk.Block.Body.ExecutionPayload.Random = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
+				blk.Block.Body.ExecutionPayload.PrevRandao = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
 				blk.Block.Body.ExecutionPayload.BaseFeePerGas = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
 				blk.Block.Body.ExecutionPayload.BlockHash = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
 				wr, err := wrapper.WrappedBellatrixSignedBeaconBlock(blk)

beacon-chain/blockchain/pow_block.go

@@ -1,20 +1,132 @@
 package blockchain
 
 import (
+	"bytes"
+	"context"
+	"fmt"
 	"math/big"
 
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/holiman/uint256"
 	"github.com/pkg/errors"
+	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	enginev1 "github.com/prysmaticlabs/prysm/proto/engine/v1"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
+	"github.com/prysmaticlabs/prysm/time/slots"
+	"github.com/sirupsen/logrus"
 )
 
-// validates terminal pow block by comparing own total difficulty with parent's total difficulty.
+// validateMergeBlock validates terminal block hash in the event of manual overrides before checking for total difficulty.
+//
+// def validate_merge_block(block: BeaconBlock) -> None:
+//    if TERMINAL_BLOCK_HASH != Hash32():
+//        # If `TERMINAL_BLOCK_HASH` is used as an override, the activation epoch must be reached.
+//        assert compute_epoch_at_slot(block.slot) >= TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH
+//        assert block.body.execution_payload.parent_hash == TERMINAL_BLOCK_HASH
+//        return
+//
+//    pow_block = get_pow_block(block.body.execution_payload.parent_hash)
+//    # Check if `pow_block` is available
+//    assert pow_block is not None
+//    pow_parent = get_pow_block(pow_block.parent_hash)
+//    # Check if `pow_parent` is available
+//    assert pow_parent is not None
+//    # Check if `pow_block` is a valid terminal PoW block
+//    assert is_valid_terminal_pow_block(pow_block, pow_parent)
+func (s *Service) validateMergeBlock(ctx context.Context, b block.SignedBeaconBlock) error {
+	if err := helpers.BeaconBlockIsNil(b); err != nil {
+		return err
+	}
+	payload, err := b.Block().Body().ExecutionPayload()
+	if err != nil {
+		return err
+	}
+	if payload == nil {
+		return errors.New("nil execution payload")
+	}
+	if err := validateTerminalBlockHash(b.Block().Slot(), payload); err != nil {
+		return errors.Wrap(err, "could not validate terminal block hash")
+	}
+	mergeBlockParentHash, mergeBlockTD, err := s.getBlkParentHashAndTD(ctx, payload.ParentHash)
+	if err != nil {
+		return errors.Wrap(err, "could not get merge block parent hash and total difficulty")
+	}
+	_, mergeBlockParentTD, err := s.getBlkParentHashAndTD(ctx, mergeBlockParentHash)
+	if err != nil {
+		return errors.Wrap(err, "could not get merge parent block total difficulty")
+	}
+	valid, err := validateTerminalBlockDifficulties(mergeBlockTD, mergeBlockParentTD)
+	if err != nil {
+		return err
+	}
+	if !valid {
+		return fmt.Errorf("invalid TTD, configTTD: %s, currentTTD: %s, parentTTD: %s",
+			params.BeaconConfig().TerminalTotalDifficulty, mergeBlockTD, mergeBlockParentTD)
+	}
+
+	log.WithFields(logrus.Fields{
+		"slot":                            b.Block().Slot(),
+		"mergeBlockHash":                  common.BytesToHash(payload.ParentHash).String(),
+		"mergeBlockParentHash":            common.BytesToHash(mergeBlockParentHash).String(),
+		"terminalTotalDifficulty":         params.BeaconConfig().TerminalTotalDifficulty,
+		"mergeBlockTotalDifficulty":       mergeBlockTD,
+		"mergeBlockParentTotalDifficulty": mergeBlockParentTD,
+	}).Info("Validated terminal block")
+
+	return nil
+}
+
+// getBlkParentHashAndTD retrieves the parent hash and total difficulty of the given block.
+func (s *Service) getBlkParentHashAndTD(ctx context.Context, blkHash []byte) ([]byte, *uint256.Int, error) {
+	blk, err := s.cfg.ExecutionEngineCaller.ExecutionBlockByHash(ctx, common.BytesToHash(blkHash))
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "could not get pow block")
+	}
+	if blk == nil {
+		return nil, nil, errors.New("pow block is nil")
+	}
+	blkTDBig, err := hexutil.DecodeBig(blk.TotalDifficulty)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "could not decode merge block total difficulty")
+	}
+	blkTDUint256, overflows := uint256.FromBig(blkTDBig)
+	if overflows {
+		return nil, nil, errors.New("total difficulty overflows")
+	}
+	return blk.ParentHash, blkTDUint256, nil
+}
+
+// validateTerminalBlockHash validates if the merge block is a valid terminal PoW block.
+// spec code:
+// if TERMINAL_BLOCK_HASH != Hash32():
+//        # If `TERMINAL_BLOCK_HASH` is used as an override, the activation epoch must be reached.
+//        assert compute_epoch_at_slot(block.slot) >= TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH
+//        assert block.body.execution_payload.parent_hash == TERMINAL_BLOCK_HASH
+//        return
+func validateTerminalBlockHash(blkSlot types.Slot, payload *enginev1.ExecutionPayload) error {
+	if bytesutil.ToBytes32(params.BeaconConfig().TerminalBlockHash.Bytes()) == [32]byte{} {
+		return nil
+	}
+	if params.BeaconConfig().TerminalBlockHashActivationEpoch > slots.ToEpoch(blkSlot) {
+		return errors.New("terminal block hash activation epoch not reached")
+	}
+	if !bytes.Equal(payload.ParentHash, params.BeaconConfig().TerminalBlockHash.Bytes()) {
+		return errors.New("parent hash does not match terminal block hash")
+	}
+	return nil
+}
+
+// validateTerminalBlockDifficulties validates terminal pow block by comparing own total difficulty with parent's total difficulty.
 //
 // def is_valid_terminal_pow_block(block: PowBlock, parent: PowBlock) -> bool:
 //    is_total_difficulty_reached = block.total_difficulty >= TERMINAL_TOTAL_DIFFICULTY
 //    is_parent_total_difficulty_valid = parent.total_difficulty < TERMINAL_TOTAL_DIFFICULTY
 //    return is_total_difficulty_reached and is_parent_total_difficulty_valid
-func validTerminalPowBlock(currentDifficulty *uint256.Int, parentDifficulty *uint256.Int) (bool, error) {
+func validateTerminalBlockDifficulties(currentDifficulty *uint256.Int, parentDifficulty *uint256.Int) (bool, error) {
 	b, ok := new(big.Int).SetString(params.BeaconConfig().TerminalTotalDifficulty, 10)
 	if !ok {
 		return false, errors.New("failed to parse terminal total difficulty")

beacon-chain/blockchain/pow_block_test.go

@@ -1,12 +1,21 @@
 package blockchain
 
 import (
+	"context"
 	"fmt"
 	"math/big"
 	"testing"
 
 	"github.com/holiman/uint256"
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	enginev1 "github.com/prysmaticlabs/prysm/proto/engine/v1"
+	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
 	"github.com/prysmaticlabs/prysm/testing/require"
 )
 
@@ -66,10 +75,10 @@ func Test_validTerminalPowBlock(t *testing.T) {
 			cfg := params.BeaconConfig()
 			cfg.TerminalTotalDifficulty = fmt.Sprint(tt.ttd)
 			params.OverrideBeaconConfig(cfg)
-			got, err := validTerminalPowBlock(tt.currentDifficulty, tt.parentDifficulty)
+			got, err := validateTerminalBlockDifficulties(tt.currentDifficulty, tt.parentDifficulty)
 			require.NoError(t, err)
 			if got != tt.want {
-				t.Errorf("validTerminalPowBlock() = %v, want %v", got, tt.want)
+				t.Errorf("validateTerminalBlockDifficulties() = %v, want %v", got, tt.want)
 			}
 		})
 	}
@@ -87,7 +96,115 @@ func Test_validTerminalPowBlockSpecConfig(t *testing.T) {
 	parent, of := uint256.FromBig(i)
 	require.Equal(t, of, false)
 
-	got, err := validTerminalPowBlock(current, parent)
+	got, err := validateTerminalBlockDifficulties(current, parent)
 	require.NoError(t, err)
 	require.Equal(t, true, got)
 }
+
+func Test_validateMergeBlock(t *testing.T) {
+	cfg := params.BeaconConfig()
+	cfg.TerminalTotalDifficulty = "2"
+	params.OverrideBeaconConfig(cfg)
+
+	ctx := context.Background()
+	beaconDB := testDB.SetupDB(t)
+	fcs := protoarray.New(0, 0, [32]byte{'a'})
+	opts := []Option{
+		WithDatabase(beaconDB),
+		WithStateGen(stategen.New(beaconDB)),
+		WithForkChoiceStore(fcs),
+	}
+	service, err := NewService(ctx, opts...)
+	require.NoError(t, err)
+
+	engine := &mockEngineService{blks: map[[32]byte]*enginev1.ExecutionBlock{}}
+	service.cfg.ExecutionEngineCaller = engine
+	engine.blks[[32]byte{'a'}] = &enginev1.ExecutionBlock{
+		ParentHash:      bytesutil.PadTo([]byte{'b'}, fieldparams.RootLength),
+		TotalDifficulty: "0x2",
+	}
+	engine.blks[[32]byte{'b'}] = &enginev1.ExecutionBlock{
+		ParentHash:      bytesutil.PadTo([]byte{'3'}, fieldparams.RootLength),
+		TotalDifficulty: "0x1",
+	}
+	blk := &ethpb.SignedBeaconBlockBellatrix{
+		Block: &ethpb.BeaconBlockBellatrix{
+			Slot: 1,
+			Body: &ethpb.BeaconBlockBodyBellatrix{
+				ExecutionPayload: &enginev1.ExecutionPayload{
+					ParentHash: bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength),
+				},
+			},
+		},
+	}
+	b, err := wrapper.WrappedSignedBeaconBlock(blk)
+	require.NoError(t, err)
+	require.NoError(t, service.validateMergeBlock(ctx, b))
+
+	cfg.TerminalTotalDifficulty = "1"
+	params.OverrideBeaconConfig(cfg)
+	require.ErrorContains(t, "invalid TTD, configTTD: 1, currentTTD: 2, parentTTD: 1", service.validateMergeBlock(ctx, b))
+}
+
+func Test_getBlkParentHashAndTD(t *testing.T) {
+	ctx := context.Background()
+	beaconDB := testDB.SetupDB(t)
+	fcs := protoarray.New(0, 0, [32]byte{'a'})
+	opts := []Option{
+		WithDatabase(beaconDB),
+		WithStateGen(stategen.New(beaconDB)),
+		WithForkChoiceStore(fcs),
+	}
+	service, err := NewService(ctx, opts...)
+	require.NoError(t, err)
+
+	engine := &mockEngineService{blks: map[[32]byte]*enginev1.ExecutionBlock{}}
+	service.cfg.ExecutionEngineCaller = engine
+	h := [32]byte{'a'}
+	p := [32]byte{'b'}
+	td := "0x1"
+	engine.blks[h] = &enginev1.ExecutionBlock{
+		ParentHash:      p[:],
+		TotalDifficulty: td,
+	}
+	parentHash, totalDifficulty, err := service.getBlkParentHashAndTD(ctx, h[:])
+	require.NoError(t, err)
+	require.Equal(t, p, bytesutil.ToBytes32(parentHash))
+	require.Equal(t, td, totalDifficulty.String())
+
+	_, _, err = service.getBlkParentHashAndTD(ctx, []byte{'c'})
+	require.ErrorContains(t, "could not get pow block: block not found", err)
+
+	engine.blks[h] = nil
+	_, _, err = service.getBlkParentHashAndTD(ctx, h[:])
+	require.ErrorContains(t, "pow block is nil", err)
+
+	engine.blks[h] = &enginev1.ExecutionBlock{
+		ParentHash:      p[:],
+		TotalDifficulty: "1",
+	}
+	_, _, err = service.getBlkParentHashAndTD(ctx, h[:])
+	require.ErrorContains(t, "could not decode merge block total difficulty: hex string without 0x prefix", err)
+
+	engine.blks[h] = &enginev1.ExecutionBlock{
+		ParentHash:      p[:],
+		TotalDifficulty: "0XFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
+	}
+	_, _, err = service.getBlkParentHashAndTD(ctx, h[:])
+	require.ErrorContains(t, "could not decode merge block total difficulty: hex number > 256 bits", err)
+}
+
+func Test_validateTerminalBlockHash(t *testing.T) {
+	require.NoError(t, validateTerminalBlockHash(1, &enginev1.ExecutionPayload{}))
+
+	cfg := params.BeaconConfig()
+	cfg.TerminalBlockHash = [32]byte{0x01}
+	params.OverrideBeaconConfig(cfg)
+	require.ErrorContains(t, "terminal block hash activation epoch not reached", validateTerminalBlockHash(1, &enginev1.ExecutionPayload{}))
+
+	cfg.TerminalBlockHashActivationEpoch = 0
+	params.OverrideBeaconConfig(cfg)
+	require.ErrorContains(t, "parent hash does not match terminal block hash", validateTerminalBlockHash(1, &enginev1.ExecutionPayload{}))
+
+	require.NoError(t, validateTerminalBlockHash(1, &enginev1.ExecutionPayload{ParentHash: cfg.TerminalBlockHash.Bytes()}))
+}

beacon-chain/blockchain/service.go

@@ -27,6 +27,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
+	enginev1 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/flags"
@@ -83,6 +84,7 @@ type config struct {
 	SlasherAttestationsFeed *event.Feed
 	WeakSubjectivityCheckpt *ethpb.Checkpoint
 	FinalizedStateAtStartUp state.BeaconState
+	ExecutionEngineCaller   enginev1.Caller
 }
 
 // NewService instantiates a new block service instance that will

beacon-chain/blockchain/testing/mock.go

@@ -32,34 +32,35 @@ var ErrNilState = errors.New("nil state")
 
 // ChainService defines the mock interface for testing
 type ChainService struct {
-	State                       state.BeaconState
-	Root                        []byte
-	Block                       block.SignedBeaconBlock
+	Optimistic                  bool
+	ValidAttestation            bool
+	ValidatorsRoot              [32]byte
+	PublicKey                   [fieldparams.BLSPubkeyLength]byte
 	FinalizedCheckPoint         *ethpb.Checkpoint
 	CurrentJustifiedCheckPoint  *ethpb.Checkpoint
 	PreviousJustifiedCheckPoint *ethpb.Checkpoint
-	BlocksReceived              []block.SignedBeaconBlock
+	Slot                        *types.Slot // Pointer because 0 is a useful value, so checking against it can be incorrect.
 	Balance                     *precompute.Balance
-	Genesis                     time.Time
-	ValidatorsRoot              [32]byte
+	ForkChoiceStore             *protoarray.Store
 	CanonicalRoots              map[[32]byte]bool
 	Fork                        *ethpb.Fork
 	ETH1Data                    *ethpb.Eth1Data
+	InitSyncBlockRoots          map[[32]byte]bool
 	DB                          db.Database
+	State                       state.BeaconState
+	Block                       block.SignedBeaconBlock
+	VerifyBlkDescendantErr      error
 	stateNotifier               statefeed.Notifier
+	BlocksReceived              []block.SignedBeaconBlock
+	SyncCommitteeIndices        []types.CommitteeIndex
 	blockNotifier               blockfeed.Notifier
 	opNotifier                  opfeed.Notifier
-	ValidAttestation            bool
-	ForkChoiceStore             *protoarray.Store
-	VerifyBlkDescendantErr      error
-	Slot                        *types.Slot // Pointer because 0 is a useful value, so checking against it can be incorrect.
-	SyncCommitteeIndices        []types.CommitteeIndex
+	Root                        []byte
 	SyncCommitteeDomain         []byte
 	SyncSelectionProofDomain    []byte
 	SyncContributionProofDomain []byte
-	PublicKey                   [fieldparams.BLSPubkeyLength]byte
 	SyncCommitteePubkeys        [][]byte
-	InitSyncBlockRoots          map[[32]byte]bool
+	Genesis                     time.Time
 }
 
 // StateNotifier mocks the same method in the chain service.
@@ -442,7 +443,7 @@ func (s *ChainService) HeadSyncContributionProofDomain(_ context.Context, _ type
 
 // IsOptimistic mocks the same method in the chain service.
 func (s *ChainService) IsOptimistic(_ context.Context) (bool, error) {
-	return false, nil
+	return s.Optimistic, nil
 }
 
 // IsOptimisticForRoot mocks the same method in the chain service.

beacon-chain/core/blocks/payload.go

@@ -124,8 +124,8 @@ func ValidatePayload(st state.BeaconState, payload *enginev1.ExecutionPayload) e
 		return err
 	}
 
-	if !bytes.Equal(payload.Random, random) {
-		return errors.New("incorrect random")
+	if !bytes.Equal(payload.PrevRandao, random) {
+		return errors.New("incorrect prev randao")
 	}
 	t, err := slots.ToTime(st.GenesisTime(), st.Slot())
 	if err != nil {
@@ -201,7 +201,7 @@ func PayloadToHeader(payload *enginev1.ExecutionPayload) (*ethpb.ExecutionPayloa
 		StateRoot:        bytesutil.SafeCopyBytes(payload.StateRoot),
 		ReceiptRoot:      bytesutil.SafeCopyBytes(payload.ReceiptsRoot),
 		LogsBloom:        bytesutil.SafeCopyBytes(payload.LogsBloom),
-		Random:           bytesutil.SafeCopyBytes(payload.Random),
+		PrevRandao:       bytesutil.SafeCopyBytes(payload.PrevRandao),
 		BlockNumber:      payload.BlockNumber,
 		GasLimit:         payload.GasLimit,
 		GasUsed:          payload.GasUsed,
@@ -229,7 +229,7 @@ func isEmptyPayload(p *enginev1.ExecutionPayload) bool {
 	if !bytes.Equal(p.LogsBloom, make([]byte, fieldparams.LogsBloomLength)) {
 		return false
 	}
-	if !bytes.Equal(p.Random, make([]byte, fieldparams.RootLength)) {
+	if !bytes.Equal(p.PrevRandao, make([]byte, fieldparams.RootLength)) {
 		return false
 	}
 	if !bytes.Equal(p.BaseFeePerGas, make([]byte, fieldparams.RootLength)) {
@@ -275,7 +275,7 @@ func isEmptyHeader(h *ethpb.ExecutionPayloadHeader) bool {
 	if !bytes.Equal(h.LogsBloom, make([]byte, fieldparams.LogsBloomLength)) {
 		return false
 	}
-	if !bytes.Equal(h.Random, make([]byte, fieldparams.RootLength)) {
+	if !bytes.Equal(h.PrevRandao, make([]byte, fieldparams.RootLength)) {
 		return false
 	}
 	if !bytes.Equal(h.BaseFeePerGas, make([]byte, fieldparams.RootLength)) {

beacon-chain/core/blocks/payload_test.go

@@ -78,7 +78,7 @@ func Test_MergeComplete(t *testing.T) {
 			name: "has random",
 			payload: func() *ethpb.ExecutionPayloadHeader {
 				h := emptyPayloadHeader()
-				h.Random = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
+				h.PrevRandao = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
 				return h
 			}(),
 			want: true,
@@ -246,7 +246,7 @@ func Test_MergeBlock(t *testing.T) {
 			name: "empty header, payload has random",
 			payload: func() *enginev1.ExecutionPayload {
 				p := emptyPayload()
-				p.Random = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
+				p.PrevRandao = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
 				return p
 			}(),
 			header: emptyPayloadHeader(),
@@ -520,21 +520,21 @@ func Test_ValidatePayload(t *testing.T) {
 			name: "validate passes",
 			payload: func() *enginev1.ExecutionPayload {
 				h := emptyPayload()
-				h.Random = random
+				h.PrevRandao = random
 				h.Timestamp = uint64(ts.Unix())
 				return h
 			}(), err: nil,
 		},
 		{
-			name:    "incorrect random",
+			name:    "incorrect prev randao",
 			payload: emptyPayload(),
-			err:     errors.New("incorrect random"),
+			err:     errors.New("incorrect prev randao"),
 		},
 		{
 			name: "incorrect timestamp",
 			payload: func() *enginev1.ExecutionPayload {
 				h := emptyPayload()
-				h.Random = random
+				h.PrevRandao = random
 				h.Timestamp = 1
 				return h
 			}(),
@@ -568,21 +568,21 @@ func Test_ProcessPayload(t *testing.T) {
 			name: "process passes",
 			payload: func() *enginev1.ExecutionPayload {
 				h := emptyPayload()
-				h.Random = random
+				h.PrevRandao = random
 				h.Timestamp = uint64(ts.Unix())
 				return h
 			}(), err: nil,
 		},
 		{
-			name:    "incorrect random",
+			name:    "incorrect prev randao",
 			payload: emptyPayload(),
-			err:     errors.New("incorrect random"),
+			err:     errors.New("incorrect prev randao"),
 		},
 		{
 			name: "incorrect timestamp",
 			payload: func() *enginev1.ExecutionPayload {
 				h := emptyPayload()
-				h.Random = random
+				h.PrevRandao = random
 				h.Timestamp = 1
 				return h
 			}(),
@@ -621,7 +621,7 @@ func Test_PayloadToHeader(t *testing.T) {
 	p.StateRoot = b
 	p.ReceiptsRoot = b
 	p.LogsBloom = b
-	p.Random = b
+	p.PrevRandao = b
 	p.ExtraData = b
 	p.BaseFeePerGas = b
 	p.BlockHash = b
@@ -635,7 +635,7 @@ func Test_PayloadToHeader(t *testing.T) {
 	require.DeepSSZEqual(t, h.StateRoot, make([]byte, fieldparams.RootLength))
 	require.DeepSSZEqual(t, h.ReceiptRoot, make([]byte, fieldparams.RootLength))
 	require.DeepSSZEqual(t, h.LogsBloom, make([]byte, fieldparams.LogsBloomLength))
-	require.DeepSSZEqual(t, h.Random, make([]byte, fieldparams.RootLength))
+	require.DeepSSZEqual(t, h.PrevRandao, make([]byte, fieldparams.RootLength))
 	require.DeepSSZEqual(t, h.ExtraData, make([]byte, 0))
 	require.DeepSSZEqual(t, h.BaseFeePerGas, make([]byte, fieldparams.RootLength))
 	require.DeepSSZEqual(t, h.BlockHash, make([]byte, fieldparams.RootLength))
@@ -663,7 +663,7 @@ func emptyPayloadHeader() *ethpb.ExecutionPayloadHeader {
 		StateRoot:        make([]byte, fieldparams.RootLength),
 		ReceiptRoot:      make([]byte, fieldparams.RootLength),
 		LogsBloom:        make([]byte, fieldparams.LogsBloomLength),
-		Random:           make([]byte, fieldparams.RootLength),
+		PrevRandao:       make([]byte, fieldparams.RootLength),
 		BaseFeePerGas:    make([]byte, fieldparams.RootLength),
 		BlockHash:        make([]byte, fieldparams.RootLength),
 		TransactionsRoot: make([]byte, fieldparams.RootLength),
@@ -678,7 +678,7 @@ func emptyPayload() *enginev1.ExecutionPayload {
 		StateRoot:     make([]byte, fieldparams.RootLength),
 		ReceiptsRoot:  make([]byte, fieldparams.RootLength),
 		LogsBloom:     make([]byte, fieldparams.LogsBloomLength),
-		Random:        make([]byte, fieldparams.RootLength),
+		PrevRandao:    make([]byte, fieldparams.RootLength),
 		BaseFeePerGas: make([]byte, fieldparams.RootLength),
 		BlockHash:     make([]byte, fieldparams.RootLength),
 		Transactions:  make([][]byte, 0),

beacon-chain/core/execution/upgrade.go

@@ -71,7 +71,7 @@ func UpgradeToBellatrix(ctx context.Context, state state.BeaconState) (state.Bea
 			StateRoot:        make([]byte, 32),
 			ReceiptRoot:      make([]byte, 32),
 			LogsBloom:        make([]byte, 256),
-			Random:           make([]byte, 32),
+			PrevRandao:       make([]byte, 32),
 			BlockNumber:      0,
 			GasLimit:         0,
 			GasUsed:          0,

beacon-chain/core/execution/upgrade_test.go

@@ -67,7 +67,7 @@ func TestUpgradeToBellatrix(t *testing.T) {
 		StateRoot:        make([]byte, 32),
 		ReceiptRoot:      make([]byte, 32),
 		LogsBloom:        make([]byte, 256),
-		Random:           make([]byte, 32),
+		PrevRandao:       make([]byte, 32),
 		BlockNumber:      0,
 		GasLimit:         0,
 		GasUsed:          0,

beacon-chain/core/helpers/block.go

@@ -11,18 +11,22 @@ import (
 	"github.com/prysmaticlabs/prysm/time/slots"
 )
 
+var ErrNilSignedBeaconBlock = errors.New("signed beacon block can't be nil")
+var ErrNilBeaconBlock = errors.New("beacon block can't be nil")
+var ErrNilBeaconBlockBody = errors.New("beacon block body can't be nil")
+
 // BeaconBlockIsNil checks if any composite field of input signed beacon block is nil.
 // Access to these nil fields will result in run time panic,
 // it is recommended to run these checks as first line of defense.
 func BeaconBlockIsNil(b block.SignedBeaconBlock) error {
 	if b == nil || b.IsNil() {
-		return errors.New("signed beacon block can't be nil")
+		return ErrNilSignedBeaconBlock
 	}
 	if b.Block().IsNil() {
-		return errors.New("beacon block can't be nil")
+		return ErrNilBeaconBlock
 	}
 	if b.Block().Body().IsNil() {
-		return errors.New("beacon block body can't be nil")
+		return ErrNilBeaconBlockBody
 	}
 	return nil
 }

beacon-chain/core/transition/bellatrix_transition_no_verify_sig_test.go

@@ -244,7 +244,7 @@ func createFullBellatrixBlockWithOperations(t *testing.T) (state.BeaconState,
 					StateRoot:     make([]byte, fieldparams.RootLength),
 					ReceiptsRoot:  make([]byte, fieldparams.RootLength),
 					LogsBloom:     make([]byte, fieldparams.LogsBloomLength),
-					Random:        make([]byte, fieldparams.RootLength),
+					PrevRandao:    make([]byte, fieldparams.RootLength),
 					BaseFeePerGas: make([]byte, fieldparams.RootLength),
 					BlockHash:     make([]byte, fieldparams.RootLength),
 					Transactions:  make([][]byte, 0),

beacon-chain/core/transition/transition_no_verify_sig.go

@@ -137,16 +137,6 @@ func CalculateStateRoot(
 	if err != nil {
 		return [32]byte{}, errors.Wrap(err, "could not process block")
 	}
-	if signed.Version() == version.Altair || signed.Version() == version.Bellatrix {
-		sa, err := signed.Block().Body().SyncAggregate()
-		if err != nil {
-			return [32]byte{}, err
-		}
-		state, err = altair.ProcessSyncAggregate(ctx, state, sa)
-		if err != nil {
-			return [32]byte{}, err
-		}
-	}
 
 	return state.HashTreeRoot(ctx)
 }
@@ -183,16 +173,6 @@ func ProcessBlockNoVerifyAnySig(
 	if err != nil {
 		return nil, nil, err
 	}
-	if signed.Version() == version.Altair || signed.Version() == version.Bellatrix {
-		sa, err := signed.Block().Body().SyncAggregate()
-		if err != nil {
-			return nil, nil, err
-		}
-		state, err = altair.ProcessSyncAggregate(ctx, state, sa)
-		if err != nil {
-			return nil, nil, err
-		}
-	}
 
 	bSet, err := b.BlockSignatureBatch(state, blk.ProposerIndex(), signed.Signature(), blk.HashTreeRoot)
 	if err != nil {
@@ -340,6 +320,19 @@ func ProcessBlockForStateRoot(
 		return nil, errors.Wrap(err, "could not process block operation")
 	}
 
+	if signed.Block().Version() == version.Phase0 {
+		return state, nil
+	}
+
+	sa, err := signed.Block().Body().SyncAggregate()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get sync aggregate from block")
+	}
+	state, err = altair.ProcessSyncAggregate(ctx, state, sa)
+	if err != nil {
+		return nil, errors.Wrap(err, "process_sync_aggregate failed")
+	}
+
 	return state, nil
 }
 

beacon-chain/db/errors.go

@@ -7,3 +7,9 @@ import "github.com/prysmaticlabs/prysm/beacon-chain/db/kv"
 // i/o error. This variable copies the value in the kv package to the same scope as the Database interfaces,
 // so that it is available to code paths that do not interact directly with the kv package.
 var ErrNotFound = kv.ErrNotFound
+
+// ErrNotFoundState wraps ErrNotFound for an error specific to a state not being found in the database.
+var ErrNotFoundState = kv.ErrNotFoundState
+
+// ErrNotFoundOriginBlockRoot wraps ErrNotFound for an error specific to the origin block root.
+var ErrNotFoundOriginBlockRoot = kv.ErrNotFoundOriginBlockRoot

beacon-chain/db/iface/interface.go

@@ -33,6 +33,7 @@ type ReadOnlyDatabase interface {
 	ValidatedTips(ctx context.Context) (map[[32]byte]types.Slot, error)
 	// State related methods.
 	State(ctx context.Context, blockRoot [32]byte) (state.BeaconState, error)
+	StateOrError(ctx context.Context, blockRoot [32]byte) (state.BeaconState, error)
 	GenesisState(ctx context.Context) (state.BeaconState, error)
 	HasState(ctx context.Context, blockRoot [32]byte) bool
 	StateSummary(ctx context.Context, blockRoot [32]byte) (*ethpb.StateSummary, error)

beacon-chain/db/kv/BUILD.bazel

@@ -81,7 +81,6 @@ go_test(
         "checkpoint_test.go",
         "deposit_contract_test.go",
         "encoding_test.go",
-        "error_test.go",
         "finalized_block_roots_test.go",
         "genesis_test.go",
         "init_test.go",

beacon-chain/db/kv/error.go

@@ -1,6 +1,6 @@
 package kv
 
-import "errors"
+import "github.com/pkg/errors"
 
 // errDeleteFinalized is raised when we attempt to delete a finalized block/state
 var errDeleteFinalized = errors.New("cannot delete finalized block or state")
@@ -8,42 +8,7 @@ var errDeleteFinalized = errors.New("cannot delete finalized block or state")
 // ErrNotFound can be used directly, or as a wrapped DBError, whenever a db method needs to
 // indicate that a value couldn't be found.
 var ErrNotFound = errors.New("not found in db")
+var ErrNotFoundState = errors.Wrap(ErrNotFound, "state not found")
 
 // ErrNotFoundOriginBlockRoot is an error specifically for the origin block root getter
-var ErrNotFoundOriginBlockRoot = WrapDBError(ErrNotFound, "OriginBlockRoot")
-
-// WrapDBError wraps an error in a DBError. See commentary on DBError for more context.
-func WrapDBError(e error, outer string) error {
-	return DBError{
-		Wraps: e,
-		Outer: errors.New(outer),
-	}
-}
-
-// DBError implements the Error method so that it can be asserted as an error.
-// The Unwrap method supports error wrapping, enabling it to be used with errors.Is/As.
-// The primary use case is to make it simple for database methods to return errors
-// that wrap ErrNotFound, allowing calling code to check for "not found" errors
-// like: `error.Is(err, ErrNotFound)`. This is intended to improve error handling
-// in db lookup methods that need to differentiate between a missing value and some
-// other database error. for more background see:
-// https://go.dev/blog/go1.13-errors
-type DBError struct {
-	Wraps error
-	Outer error
-}
-
-// Error satisfies the error interface, so that DBErrors can be used anywhere that
-// expects an `error`.
-func (e DBError) Error() string {
-	es := e.Outer.Error()
-	if e.Wraps != nil {
-		es += ": " + e.Wraps.Error()
-	}
-	return es
-}
-
-// Unwrap is used by the errors package Is and As methods.
-func (e DBError) Unwrap() error {
-	return e.Wraps
-}
+var ErrNotFoundOriginBlockRoot = errors.Wrap(ErrNotFound, "OriginBlockRoot")

beacon-chain/db/kv/error_test.go

@@ -1,24 +0,0 @@
-package kv
-
-import (
-	"errors"
-	"testing"
-)
-
-func TestWrappedSentinelError(t *testing.T) {
-	e := ErrNotFoundOriginBlockRoot
-	if !errors.Is(e, ErrNotFoundOriginBlockRoot) {
-		t.Error("expected that a copy of ErrNotFoundOriginBlockRoot would have an is-a relationship")
-	}
-
-	outer := errors.New("wrapped error")
-	e2 := DBError{Wraps: ErrNotFoundOriginBlockRoot, Outer: outer}
-	if !errors.Is(e2, ErrNotFoundOriginBlockRoot) {
-		t.Error("expected that errors.Is would know DBError wraps ErrNotFoundOriginBlockRoot")
-	}
-
-	// test that the innermost not found error is detected
-	if !errors.Is(e2, ErrNotFound) {
-		t.Error("expected that errors.Is would know ErrNotFoundOriginBlockRoot wraps ErrNotFound")
-	}
-}

beacon-chain/db/kv/state.go

@@ -3,6 +3,7 @@ package kv
 import (
 	"bytes"
 	"context"
+	"fmt"
 
 	"github.com/golang/snappy"
 	"github.com/pkg/errors"
@@ -46,6 +47,19 @@ func (s *Store) State(ctx context.Context, blockRoot [32]byte) (state.BeaconStat
 	return s.unmarshalState(ctx, enc, valEntries)
 }
 
+// StateOrError is just like State(), except it only returns a non-error response
+// if the requested state is found in the database.
+func (s *Store) StateOrError(ctx context.Context, blockRoot [32]byte) (state.BeaconState, error) {
+	st, err := s.State(ctx, blockRoot)
+	if err != nil {
+		return nil, err
+	}
+	if st == nil || st.IsNil() {
+		return nil, errors.Wrap(ErrNotFoundState, fmt.Sprintf("no state with blockroot=%#x", blockRoot))
+	}
+	return st, nil
+}
+
 // GenesisState returns the genesis state in beacon chain.
 func (s *Store) GenesisState(ctx context.Context) (state.BeaconState, error) {
 	ctx, span := trace.StartSpan(ctx, "BeaconDB.GenesisState")

beacon-chain/db/kv/state_test.go

@@ -21,6 +21,12 @@ import (
 	bolt "go.etcd.io/bbolt"
 )
 
+func TestStateNil(t *testing.T) {
+	db := setupDB(t)
+	_, err := db.StateOrError(context.Background(), [32]byte{})
+	require.ErrorIs(t, err, ErrNotFoundState)
+}
+
 func TestState_CanSaveRetrieve(t *testing.T) {
 	db := setupDB(t)
 

beacon-chain/forkchoice/protoarray/optimistic_sync.go

@@ -47,6 +47,7 @@ func (f *ForkChoice) Optimistic(ctx context.Context, root [32]byte, slot types.S
 	f.syncedTips.RLock()
 	_, ok := f.syncedTips.validatedTips[root]
 	if ok {
+		f.syncedTips.RUnlock()
 		return false, nil
 	}
 	f.syncedTips.RUnlock()

beacon-chain/forkchoice/protoarray/optimistic_sync_test.go

@@ -200,6 +200,10 @@ func TestOptimistic(t *testing.T) {
 	op, err = f.Optimistic(ctx, nodeK.root, nodeK.slot)
 	require.NoError(t, err)
 	require.Equal(t, op, true)
+
+	// request a write Lock to synced Tips regression #10289
+	f.syncedTips.Lock()
+	defer f.syncedTips.Unlock()
 }
 
 // This tests the algorithm to update syncedTips

beacon-chain/node/node.go

@@ -834,6 +834,7 @@ func (b *BeaconNode) registerGRPCGateway() error {
 	selfCert := b.cliCtx.String(flags.CertFlag.Name)
 	maxCallSize := b.cliCtx.Uint64(cmd.GrpcMaxCallRecvMsgSizeFlag.Name)
 	httpModules := b.cliCtx.String(flags.HTTPModules.Name)
+	timeout := b.cliCtx.Int(cmd.ApiTimeoutFlag.Name)
 	if enableDebugRPCEndpoints {
 		maxCallSize = uint64(math.Max(float64(maxCallSize), debugGrpcMaxMsgSize))
 	}
@@ -855,6 +856,7 @@ func (b *BeaconNode) registerGRPCGateway() error {
 		apigateway.WithRemoteCert(selfCert),
 		apigateway.WithMaxCallRecvMsgSize(maxCallSize),
 		apigateway.WithAllowedOrigins(allowedOrigins),
+		apigateway.WithTimeout(uint64(timeout)),
 	}
 	if flags.EnableHTTPEthAPI(httpModules) {
 		opts = append(opts, apigateway.WithApiMiddleware(&apimiddleware.BeaconEndpointFactory{}))

beacon-chain/powchain/BUILD.bazel

@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "block_cache.go",
         "block_reader.go",
+        "check_transition_config.go",
         "deposit.go",
         "log.go",
         "log_processing.go",
@@ -44,6 +45,7 @@ go_library(
         "//monitoring/tracing:go_default_library",
         "//network:go_default_library",
         "//network/authorization:go_default_library",
+        "//proto/engine/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//time:go_default_library",
         "//time/slots:go_default_library",
@@ -54,6 +56,7 @@ go_library(
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
         "@com_github_ethereum_go_ethereum//ethclient:go_default_library",
         "@com_github_ethereum_go_ethereum//rpc:go_default_library",
+        "@com_github_holiman_uint256//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
@@ -69,6 +72,7 @@ go_test(
     srcs = [
         "block_cache_test.go",
         "block_reader_test.go",
+        "check_transition_config_test.go",
         "deposit_test.go",
         "init_test.go",
         "log_processing_test.go",
@@ -87,6 +91,8 @@ go_test(
         "//beacon-chain/core/signing:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
+        "//beacon-chain/powchain/engine-api-client/v1:go_default_library",
+        "//beacon-chain/powchain/engine-api-client/v1/mocks:go_default_library",
         "//beacon-chain/powchain/testing:go_default_library",
         "//beacon-chain/powchain/types:go_default_library",
         "//beacon-chain/state/stategen:go_default_library",

beacon-chain/powchain/check_transition_config.go

@@ -0,0 +1,82 @@
+package powchain
+
+import (
+	"context"
+	"errors"
+	"math/big"
+	"time"
+
+	"github.com/holiman/uint256"
+	v1 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
+	"github.com/prysmaticlabs/prysm/config/params"
+	pb "github.com/prysmaticlabs/prysm/proto/engine/v1"
+)
+
+var (
+	checkTransitionPollingInterval = time.Second * 10
+	configMismatchLog              = "Configuration mismatch between your execution client and Prysm. " +
+		"Please check your execution client and restart it with the proper configuration. If this is not done, " +
+		"your node will not be able to complete the proof-of-stake transition"
+)
+
+// Checks the transition configuration between Prysm and the connected execution node to ensure
+// there are no differences in terminal block difficulty and block hash.
+// If there are any discrepancies, we must log errors to ensure users can resolve
+//the problem and be ready for the merge transition.
+func (s *Service) checkTransitionConfiguration(ctx context.Context) {
+	if s.engineAPIClient == nil {
+		return
+	}
+	i := new(big.Int)
+	i.SetString(params.BeaconConfig().TerminalTotalDifficulty, 10)
+	ttd := new(uint256.Int)
+	ttd.SetFromBig(i)
+	cfg := &pb.TransitionConfiguration{
+		TerminalTotalDifficulty: ttd.Hex(),
+		TerminalBlockHash:       params.BeaconConfig().TerminalBlockHash[:],
+		TerminalBlockNumber:     big.NewInt(0).Bytes(), // A value of 0 is recommended in the request.
+	}
+	err := s.engineAPIClient.ExchangeTransitionConfiguration(ctx, cfg)
+	if err != nil {
+		if errors.Is(err, v1.ErrConfigMismatch) {
+			log.WithError(err).Fatal(configMismatchLog)
+		}
+		log.WithError(err).Error("Could not check configuration values between execution and consensus client")
+	}
+
+	// We poll the execution client to see if the transition configuration has changed.
+	// This serves as a heartbeat to ensure the execution client and Prysm are ready for the
+	// Bellatrix hard-fork transition.
+	ticker := time.NewTicker(checkTransitionPollingInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			err = s.engineAPIClient.ExchangeTransitionConfiguration(ctx, cfg)
+			s.handleExchangeConfigurationError(err)
+		}
+	}
+}
+
+// We check if there is a configuration mismatch error between the execution client
+// and the Prysm beacon node. If so, we need to log errors in the node as it cannot successfully
+// complete the merge transition for the Bellatrix hard fork.
+func (s *Service) handleExchangeConfigurationError(err error) {
+	if err == nil {
+		// If there is no error in checking the exchange configuration error, we clear
+		// the run error of the service if we had previously set it to ErrConfigMismatch.
+		if errors.Is(s.runError, v1.ErrConfigMismatch) {
+			s.runError = nil
+		}
+		return
+	}
+	// If the error is a configuration mismatch, we set a runtime error in the service.
+	if errors.Is(err, v1.ErrConfigMismatch) {
+		s.runError = err
+		log.WithError(err).Error(configMismatchLog)
+		return
+	}
+	log.WithError(err).Error("Could not check configuration values between execution and consensus client")
+}

beacon-chain/powchain/check_transition_config_test.go

@@ -0,0 +1,59 @@
+package powchain
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	v1 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1/mocks"
+	"github.com/prysmaticlabs/prysm/testing/require"
+	logTest "github.com/sirupsen/logrus/hooks/test"
+)
+
+func Test_checkTransitionConfiguration(t *testing.T) {
+	ctx := context.Background()
+	hook := logTest.NewGlobal()
+
+	m := &mocks.EngineClient{}
+	m.Err = errors.New("something went wrong")
+
+	srv := &Service{}
+	srv.engineAPIClient = m
+	checkTransitionPollingInterval = time.Millisecond
+	ctx, cancel := context.WithCancel(ctx)
+	go srv.checkTransitionConfiguration(ctx)
+	<-time.After(100 * time.Millisecond)
+	cancel()
+	require.LogsContain(t, hook, "Could not check configuration values")
+}
+
+func TestService_handleExchangeConfigurationError(t *testing.T) {
+	hook := logTest.NewGlobal()
+	t.Run("clears existing service error", func(t *testing.T) {
+		srv := &Service{isRunning: true}
+		srv.runError = v1.ErrConfigMismatch
+		srv.handleExchangeConfigurationError(nil)
+		require.Equal(t, true, srv.Status() == nil)
+	})
+	t.Run("does not clear existing service error if wrong kind", func(t *testing.T) {
+		srv := &Service{isRunning: true}
+		err := errors.New("something else went wrong")
+		srv.runError = err
+		srv.handleExchangeConfigurationError(nil)
+		require.ErrorIs(t, err, srv.Status())
+	})
+	t.Run("sets service error on config mismatch", func(t *testing.T) {
+		srv := &Service{isRunning: true}
+		srv.handleExchangeConfigurationError(v1.ErrConfigMismatch)
+		require.Equal(t, v1.ErrConfigMismatch, srv.Status())
+		require.LogsContain(t, hook, configMismatchLog)
+	})
+	t.Run("does not set service error if unrelated problem", func(t *testing.T) {
+		srv := &Service{isRunning: true}
+		srv.handleExchangeConfigurationError(errors.New("foo"))
+		require.Equal(t, true, srv.Status() == nil)
+		require.LogsContain(t, hook, "Could not check configuration values")
+	})
+}

beacon-chain/powchain/engine-api-client/v1/BUILD.bazel

@@ -3,6 +3,7 @@ load("@prysm//tools/go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
+        "auth.go",
         "client.go",
         "errors.go",
         "options.go",
@@ -13,16 +14,22 @@ go_library(
         "//config/params:go_default_library",
         "//proto/engine/v1:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
+        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_ethereum_go_ethereum//rpc:go_default_library",
+        "@com_github_golang_jwt_jwt_v4//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
     ],
 )
 
 go_test(
     name = "go_default_test",
-    srcs = ["client_test.go"],
+    srcs = [
+        "auth_test.go",
+        "client_test.go",
+    ],
     embed = [":go_default_library"],
     deps = [
+        "//beacon-chain/powchain/engine-api-client/v1/mocks:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
         "//encoding/bytesutil:go_default_library",
@@ -30,6 +37,8 @@ go_test(
         "//testing/require:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//rpc:go_default_library",
+        "@com_github_golang_jwt_jwt_v4//:go_default_library",
+        "@com_github_holiman_uint256//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@org_golang_google_protobuf//proto:go_default_library",
     ],

beacon-chain/powchain/engine-api-client/v1/auth.go

@@ -0,0 +1,44 @@
+package v1
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/golang-jwt/jwt/v4"
+	"github.com/pkg/errors"
+)
+
+// This creates a custom HTTP transport which we can attach to our HTTP client
+// in order to inject JWT auth strings into our HTTP request headers. Authentication
+// is required when interacting with an Ethereum engine API server via HTTP, and JWT
+// is chosen as the scheme of choice.
+// For more details on the requirements of authentication when using the engine API, see
+// the specification here: https://github.com/ethereum/execution-apis/blob/main/src/engine/authentication.md
+//
+// To use this transport, initialize a new &http.Client{} from the standard library
+// and set the Transport field to &jwtTransport{} with values
+// http.DefaultTransport and a JWT secret.
+type jwtTransport struct {
+	underlyingTransport http.RoundTripper
+	jwtSecret           []byte
+}
+
+// RoundTrip ensures our transport implements http.RoundTripper interface from the
+// standard library. When used as the transport for an HTTP client, the code below
+// will run every time our client makes an HTTP request. This is used to inject
+// an JWT bearer token in the Authorization request header of every outgoing request
+// our HTTP client makes.
+func (t *jwtTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
+		// Required claim for engine API auth. "iat" stands for issued at
+		// and it must be a unix timestamp that is +/- 5 seconds from the current
+		// timestamp at the moment the server verifies this value.
+		"iat": time.Now().Unix(),
+	})
+	tokenString, err := token.SignedString(t.jwtSecret)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not produce signed JWT token")
+	}
+	req.Header.Set("Authorization", "Bearer "+tokenString)
+	return t.underlyingTransport.RoundTrip(req)
+}

beacon-chain/powchain/engine-api-client/v1/auth_test.go

@@ -0,0 +1,53 @@
+package v1
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/golang-jwt/jwt/v4"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	"github.com/prysmaticlabs/prysm/testing/require"
+)
+
+func TestJWTAuthTransport(t *testing.T) {
+	secret := bytesutil.PadTo([]byte("foo"), 32)
+	authTransport := &jwtTransport{
+		underlyingTransport: http.DefaultTransport,
+		jwtSecret:           secret,
+	}
+	client := &http.Client{
+		Timeout:   DefaultTimeout,
+		Transport: authTransport,
+	}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		reqToken := r.Header.Get("Authorization")
+		splitToken := strings.Split(reqToken, "Bearer")
+		// The format should be `Bearer ${token}`.
+		require.Equal(t, 2, len(splitToken))
+		reqToken = strings.TrimSpace(splitToken[1])
+		token, err := jwt.Parse(reqToken, func(token *jwt.Token) (interface{}, error) {
+			// We should be doing HMAC signing.
+			_, ok := token.Method.(*jwt.SigningMethodHMAC)
+			require.Equal(t, true, ok)
+			return secret, nil
+		})
+		require.NoError(t, err)
+		require.Equal(t, true, token.Valid)
+		claims, ok := token.Claims.(jwt.MapClaims)
+		require.Equal(t, true, ok)
+		item, ok := claims["iat"]
+		require.Equal(t, true, ok)
+		iat, ok := item.(float64)
+		require.Equal(t, true, ok)
+		issuedAt := time.Unix(int64(iat), 0)
+		// The claims should have an "iat" field (issued at) that is at most, 5 seconds ago.
+		since := time.Since(issuedAt)
+		require.Equal(t, true, since <= time.Second*5)
+	}))
+	defer srv.Close()
+	_, err := client.Get(srv.URL)
+	require.NoError(t, err)
+}

beacon-chain/powchain/engine-api-client/v1/client.go

@@ -6,11 +6,13 @@ package v1
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"math/big"
 	"net/url"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/config/params"
@@ -41,17 +43,17 @@ type ForkchoiceUpdatedResponse struct {
 	PayloadId *pb.PayloadIDBytes `json:"payloadId"`
 }
 
-// EngineCaller defines a client that can interact with an Ethereum
+// Caller defines a client that can interact with an Ethereum
 // execution node's engine service via JSON-RPC.
-type EngineCaller interface {
-	NewPayload(ctx context.Context, payload *pb.ExecutionPayload) (*pb.PayloadStatus, error)
+type Caller interface {
+	NewPayload(ctx context.Context, payload *pb.ExecutionPayload) ([]byte, error)
 	ForkchoiceUpdated(
 		ctx context.Context, state *pb.ForkchoiceState, attrs *pb.PayloadAttributes,
-	) (*ForkchoiceUpdatedResponse, error)
+	) (*pb.PayloadIDBytes, []byte, error)
 	GetPayload(ctx context.Context, payloadId [8]byte) (*pb.ExecutionPayload, error)
 	ExchangeTransitionConfiguration(
 		ctx context.Context, cfg *pb.TransitionConfiguration,
-	) (*pb.TransitionConfiguration, error)
+	) error
 	LatestExecutionBlock(ctx context.Context) (*pb.ExecutionBlock, error)
 	ExecutionBlockByHash(ctx context.Context, hash common.Hash) (*pb.ExecutionBlock, error)
 }
@@ -73,6 +75,11 @@ func New(ctx context.Context, endpoint string, opts ...Option) (*Client, error)
 	c := &Client{
 		cfg: defaultConfig(),
 	}
+	for _, opt := range opts {
+		if err := opt(c); err != nil {
+			return nil, err
+		}
+	}
 	switch u.Scheme {
 	case "http", "https":
 		c.rpc, err = rpc.DialHTTPWithClient(endpoint, c.cfg.httpClient)
@@ -84,28 +91,59 @@ func New(ctx context.Context, endpoint string, opts ...Option) (*Client, error)
 	if err != nil {
 		return nil, err
 	}
-	for _, opt := range opts {
-		if err := opt(c); err != nil {
-			return nil, err
-		}
-	}
 	return c, nil
 }
 
 // NewPayload calls the engine_newPayloadV1 method via JSON-RPC.
-func (c *Client) NewPayload(ctx context.Context, payload *pb.ExecutionPayload) (*pb.PayloadStatus, error) {
+func (c *Client) NewPayload(ctx context.Context, payload *pb.ExecutionPayload) ([]byte, error) {
 	result := &pb.PayloadStatus{}
 	err := c.rpc.CallContext(ctx, result, NewPayloadMethod, payload)
-	return result, handleRPCError(err)
+	if err != nil {
+		return nil, handleRPCError(err)
+	}
+
+	switch result.Status {
+	case pb.PayloadStatus_INVALID_BLOCK_HASH:
+		return nil, fmt.Errorf("could not validate block hash: %v", result.ValidationError)
+	case pb.PayloadStatus_INVALID_TERMINAL_BLOCK:
+		return nil, fmt.Errorf("could not satisfy terminal block condition: %v", result.ValidationError)
+	case pb.PayloadStatus_ACCEPTED, pb.PayloadStatus_SYNCING:
+		return nil, ErrAcceptedSyncingPayloadStatus
+	case pb.PayloadStatus_INVALID:
+		return result.LatestValidHash, ErrInvalidPayloadStatus
+	case pb.PayloadStatus_VALID:
+		return result.LatestValidHash, nil
+	default:
+		return nil, ErrUnknownPayloadStatus
+	}
 }
 
 // ForkchoiceUpdated calls the engine_forkchoiceUpdatedV1 method via JSON-RPC.
 func (c *Client) ForkchoiceUpdated(
 	ctx context.Context, state *pb.ForkchoiceState, attrs *pb.PayloadAttributes,
-) (*ForkchoiceUpdatedResponse, error) {
+) (*pb.PayloadIDBytes, []byte, error) {
 	result := &ForkchoiceUpdatedResponse{}
 	err := c.rpc.CallContext(ctx, result, ForkchoiceUpdatedMethod, state, attrs)
-	return result, handleRPCError(err)
+	if err != nil {
+		return nil, nil, handleRPCError(err)
+	}
+
+	if result.Status == nil {
+		return nil, nil, ErrNilResponse
+	}
+	resp := result.Status
+	switch resp.Status {
+	case pb.PayloadStatus_INVALID_TERMINAL_BLOCK:
+		return nil, nil, fmt.Errorf("could not satisfy terminal block condition: %v", resp.ValidationError)
+	case pb.PayloadStatus_SYNCING:
+		return nil, nil, ErrAcceptedSyncingPayloadStatus
+	case pb.PayloadStatus_INVALID:
+		return nil, resp.LatestValidHash, ErrInvalidPayloadStatus
+	case pb.PayloadStatus_VALID:
+		return result.PayloadId, resp.LatestValidHash, nil
+	default:
+		return nil, nil, ErrUnknownPayloadStatus
+	}
 }
 
 // GetPayload calls the engine_getPayloadV1 method via JSON-RPC.
@@ -118,35 +156,39 @@ func (c *Client) GetPayload(ctx context.Context, payloadId [8]byte) (*pb.Executi
 // ExchangeTransitionConfiguration calls the engine_exchangeTransitionConfigurationV1 method via JSON-RPC.
 func (c *Client) ExchangeTransitionConfiguration(
 	ctx context.Context, cfg *pb.TransitionConfiguration,
-) (*pb.TransitionConfiguration, error) {
-	// Terminal block number should be set to 0
+) error {
+	// We set terminal block number to 0 as the parameter is not set on the consensus layer.
 	zeroBigNum := big.NewInt(0)
 	cfg.TerminalBlockNumber = zeroBigNum.Bytes()
 	result := &pb.TransitionConfiguration{}
 	if err := c.rpc.CallContext(ctx, result, ExchangeTransitionConfigurationMethod, cfg); err != nil {
-		return nil, handleRPCError(err)
+		return handleRPCError(err)
 	}
 	// We surface an error to the user if local configuration settings mismatch
 	// according to the response from the execution node.
 	cfgTerminalHash := params.BeaconConfig().TerminalBlockHash[:]
 	if !bytes.Equal(cfgTerminalHash, result.TerminalBlockHash) {
-		return nil, errors.Wrapf(
-			ErrMismatchTerminalBlockHash,
+		return errors.Wrapf(
+			ErrConfigMismatch,
 			"got %#x from execution node, wanted %#x",
 			result.TerminalBlockHash,
 			cfgTerminalHash,
 		)
 	}
 	ttdCfg := params.BeaconConfig().TerminalTotalDifficulty
-	if ttdCfg != result.TerminalTotalDifficulty {
-		return nil, errors.Wrapf(
-			ErrMismatchTerminalTotalDiff,
+	ttdResult, err := hexutil.DecodeBig(result.TerminalTotalDifficulty)
+	if err != nil {
+		return errors.Wrap(err, "could not decode received terminal total difficulty")
+	}
+	if ttdResult.String() != ttdCfg {
+		return errors.Wrapf(
+			ErrConfigMismatch,
 			"got %s from execution node, wanted %s",
-			result.TerminalTotalDifficulty,
+			ttdResult.String(),
 			ttdCfg,
 		)
 	}
-	return result, nil
+	return nil
 }
 
 // LatestExecutionBlock fetches the latest execution engine block by calling

beacon-chain/powchain/engine-api-client/v1/client_test.go

@@ -13,7 +13,9 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/holiman/uint256"
 	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1/mocks"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
@@ -22,7 +24,10 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
-var _ = EngineCaller(&Client{})
+var (
+	_ = Caller(&Client{})
+	_ = Caller(&mocks.EngineClient{})
+)
 
 func TestClient_IPC(t *testing.T) {
 	server := newTestIPCServer(t)
@@ -45,35 +50,25 @@ func TestClient_IPC(t *testing.T) {
 	t.Run(ForkchoiceUpdatedMethod, func(t *testing.T) {
 		want, ok := fix["ForkchoiceUpdatedResponse"].(*ForkchoiceUpdatedResponse)
 		require.Equal(t, true, ok)
-		resp, err := client.ForkchoiceUpdated(ctx, &pb.ForkchoiceState{}, &pb.PayloadAttributes{})
+		payloadID, validHash, err := client.ForkchoiceUpdated(ctx, &pb.ForkchoiceState{}, &pb.PayloadAttributes{})
 		require.NoError(t, err)
-		require.DeepEqual(t, want.Status, resp.Status)
-		require.DeepEqual(t, want.PayloadId, resp.PayloadId)
+		require.DeepEqual(t, want.Status.LatestValidHash, validHash)
+		require.DeepEqual(t, want.PayloadId, payloadID)
 	})
 	t.Run(NewPayloadMethod, func(t *testing.T) {
-		want, ok := fix["PayloadStatus"].(*pb.PayloadStatus)
+		want, ok := fix["ValidPayloadStatus"].(*pb.PayloadStatus)
 		require.Equal(t, true, ok)
 		req, ok := fix["ExecutionPayload"].(*pb.ExecutionPayload)
 		require.Equal(t, true, ok)
-		resp, err := client.NewPayload(ctx, req)
+		latestValidHash, err := client.NewPayload(ctx, req)
 		require.NoError(t, err)
-		require.DeepEqual(t, want, resp)
-	})
-	t.Run(NewPayloadMethod, func(t *testing.T) {
-		want, ok := fix["PayloadStatus"].(*pb.PayloadStatus)
-		require.Equal(t, true, ok)
-		req, ok := fix["ExecutionPayload"].(*pb.ExecutionPayload)
-		require.Equal(t, true, ok)
-		resp, err := client.NewPayload(ctx, req)
-		require.NoError(t, err)
-		require.DeepEqual(t, want, resp)
+		require.DeepEqual(t, bytesutil.ToBytes32(want.LatestValidHash), bytesutil.ToBytes32(latestValidHash))
 	})
 	t.Run(ExchangeTransitionConfigurationMethod, func(t *testing.T) {
 		want, ok := fix["TransitionConfiguration"].(*pb.TransitionConfiguration)
 		require.Equal(t, true, ok)
-		resp, err := client.ExchangeTransitionConfiguration(ctx, want)
+		err := client.ExchangeTransitionConfiguration(ctx, want)
 		require.NoError(t, err)
-		require.DeepEqual(t, want, resp)
 	})
 	t.Run(ExecutionBlockByNumberMethod, func(t *testing.T) {
 		want, ok := fix["ExecutionBlock"].(*pb.ExecutionBlock)
@@ -138,7 +133,7 @@ func TestClient_HTTP(t *testing.T) {
 		require.NoError(t, err)
 		require.DeepEqual(t, want, resp)
 	})
-	t.Run(ForkchoiceUpdatedMethod, func(t *testing.T) {
+	t.Run(ForkchoiceUpdatedMethod+" VALID status", func(t *testing.T) {
 		forkChoiceState := &pb.ForkchoiceState{
 			HeadBlockHash:      []byte("head"),
 			SafeBlockHash:      []byte("safe"),
@@ -146,98 +141,174 @@ func TestClient_HTTP(t *testing.T) {
 		}
 		payloadAttributes := &pb.PayloadAttributes{
 			Timestamp:             1,
-			Random:                []byte("random"),
+			PrevRandao:            []byte("random"),
 			SuggestedFeeRecipient: []byte("suggestedFeeRecipient"),
 		}
 		want, ok := fix["ForkchoiceUpdatedResponse"].(*ForkchoiceUpdatedResponse)
 		require.Equal(t, true, ok)
-
-		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			w.Header().Set("Content-Type", "application/json")
-			defer func() {
-				require.NoError(t, r.Body.Close())
-			}()
-			enc, err := ioutil.ReadAll(r.Body)
-			require.NoError(t, err)
-			jsonRequestString := string(enc)
-
-			forkChoiceStateReq, err := json.Marshal(forkChoiceState)
-			require.NoError(t, err)
-			payloadAttrsReq, err := json.Marshal(payloadAttributes)
-			require.NoError(t, err)
-
-			// We expect the JSON string RPC request contains the right arguments.
-			require.Equal(t, true, strings.Contains(
-				jsonRequestString, string(forkChoiceStateReq),
-			))
-			require.Equal(t, true, strings.Contains(
-				jsonRequestString, string(payloadAttrsReq),
-			))
-			resp := map[string]interface{}{
-				"jsonrpc": "2.0",
-				"id":      1,
-				"result":  want,
-			}
-			err = json.NewEncoder(w).Encode(resp)
-			require.NoError(t, err)
-		}))
-		defer srv.Close()
-
-		rpcClient, err := rpc.DialHTTP(srv.URL)
-		require.NoError(t, err)
-		defer rpcClient.Close()
-
-		client := &Client{}
-		client.rpc = rpcClient
+		client := forkchoiceUpdateSetup(t, forkChoiceState, payloadAttributes, want)
 
 		// We call the RPC method via HTTP and expect a proper result.
-		resp, err := client.ForkchoiceUpdated(ctx, forkChoiceState, payloadAttributes)
+		payloadID, validHash, err := client.ForkchoiceUpdated(ctx, forkChoiceState, payloadAttributes)
 		require.NoError(t, err)
-		require.DeepEqual(t, want.Status, resp.Status)
-		require.DeepEqual(t, want.PayloadId, resp.PayloadId)
+		require.DeepEqual(t, want.Status.LatestValidHash, validHash)
+		require.DeepEqual(t, want.PayloadId, payloadID)
 	})
-	t.Run(NewPayloadMethod, func(t *testing.T) {
+	t.Run(ForkchoiceUpdatedMethod+" SYNCING status", func(t *testing.T) {
+		forkChoiceState := &pb.ForkchoiceState{
+			HeadBlockHash:      []byte("head"),
+			SafeBlockHash:      []byte("safe"),
+			FinalizedBlockHash: []byte("finalized"),
+		}
+		payloadAttributes := &pb.PayloadAttributes{
+			Timestamp:             1,
+			PrevRandao:            []byte("random"),
+			SuggestedFeeRecipient: []byte("suggestedFeeRecipient"),
+		}
+		want, ok := fix["ForkchoiceUpdatedSyncingResponse"].(*ForkchoiceUpdatedResponse)
+		require.Equal(t, true, ok)
+		client := forkchoiceUpdateSetup(t, forkChoiceState, payloadAttributes, want)
+
+		// We call the RPC method via HTTP and expect a proper result.
+		payloadID, validHash, err := client.ForkchoiceUpdated(ctx, forkChoiceState, payloadAttributes)
+		require.ErrorIs(t, err, ErrAcceptedSyncingPayloadStatus)
+		require.DeepEqual(t, (*pb.PayloadIDBytes)(nil), payloadID)
+		require.DeepEqual(t, []byte(nil), validHash)
+	})
+	t.Run(ForkchoiceUpdatedMethod+" INVALID status", func(t *testing.T) {
+		forkChoiceState := &pb.ForkchoiceState{
+			HeadBlockHash:      []byte("head"),
+			SafeBlockHash:      []byte("safe"),
+			FinalizedBlockHash: []byte("finalized"),
+		}
+		payloadAttributes := &pb.PayloadAttributes{
+			Timestamp:             1,
+			PrevRandao:            []byte("random"),
+			SuggestedFeeRecipient: []byte("suggestedFeeRecipient"),
+		}
+		want, ok := fix["ForkchoiceUpdatedInvalidResponse"].(*ForkchoiceUpdatedResponse)
+		require.Equal(t, true, ok)
+		client := forkchoiceUpdateSetup(t, forkChoiceState, payloadAttributes, want)
+
+		// We call the RPC method via HTTP and expect a proper result.
+		payloadID, validHash, err := client.ForkchoiceUpdated(ctx, forkChoiceState, payloadAttributes)
+		require.ErrorIs(t, err, ErrInvalidPayloadStatus)
+		require.DeepEqual(t, (*pb.PayloadIDBytes)(nil), payloadID)
+		require.DeepEqual(t, want.Status.LatestValidHash, validHash)
+	})
+	t.Run(ForkchoiceUpdatedMethod+" UNKNOWN status", func(t *testing.T) {
+		forkChoiceState := &pb.ForkchoiceState{
+			HeadBlockHash:      []byte("head"),
+			SafeBlockHash:      []byte("safe"),
+			FinalizedBlockHash: []byte("finalized"),
+		}
+		payloadAttributes := &pb.PayloadAttributes{
+			Timestamp:             1,
+			PrevRandao:            []byte("random"),
+			SuggestedFeeRecipient: []byte("suggestedFeeRecipient"),
+		}
+		want, ok := fix["ForkchoiceUpdatedAcceptedResponse"].(*ForkchoiceUpdatedResponse)
+		require.Equal(t, true, ok)
+		client := forkchoiceUpdateSetup(t, forkChoiceState, payloadAttributes, want)
+
+		// We call the RPC method via HTTP and expect a proper result.
+		payloadID, validHash, err := client.ForkchoiceUpdated(ctx, forkChoiceState, payloadAttributes)
+		require.ErrorIs(t, err, ErrUnknownPayloadStatus)
+		require.DeepEqual(t, (*pb.PayloadIDBytes)(nil), payloadID)
+		require.DeepEqual(t, []byte(nil), validHash)
+	})
+	t.Run(ForkchoiceUpdatedMethod+" INVALID_TERMINAL_BLOCK status", func(t *testing.T) {
+		forkChoiceState := &pb.ForkchoiceState{
+			HeadBlockHash:      []byte("head"),
+			SafeBlockHash:      []byte("safe"),
+			FinalizedBlockHash: []byte("finalized"),
+		}
+		payloadAttributes := &pb.PayloadAttributes{
+			Timestamp:             1,
+			PrevRandao:            []byte("random"),
+			SuggestedFeeRecipient: []byte("suggestedFeeRecipient"),
+		}
+		want, ok := fix["ForkchoiceUpdatedInvalidTerminalBlockResponse"].(*ForkchoiceUpdatedResponse)
+		require.Equal(t, true, ok)
+		client := forkchoiceUpdateSetup(t, forkChoiceState, payloadAttributes, want)
+
+		// We call the RPC method via HTTP and expect a proper result.
+		payloadID, validHash, err := client.ForkchoiceUpdated(ctx, forkChoiceState, payloadAttributes)
+		require.ErrorContains(t, "could not satisfy terminal block condition", err)
+		require.DeepEqual(t, (*pb.PayloadIDBytes)(nil), payloadID)
+		require.DeepEqual(t, []byte(nil), validHash)
+	})
+	t.Run(NewPayloadMethod+" VALID status", func(t *testing.T) {
 		execPayload, ok := fix["ExecutionPayload"].(*pb.ExecutionPayload)
 		require.Equal(t, true, ok)
-		want, ok := fix["PayloadStatus"].(*pb.PayloadStatus)
+		want, ok := fix["ValidPayloadStatus"].(*pb.PayloadStatus)
 		require.Equal(t, true, ok)
-		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			w.Header().Set("Content-Type", "application/json")
-			defer func() {
-				require.NoError(t, r.Body.Close())
-			}()
-			enc, err := ioutil.ReadAll(r.Body)
-			require.NoError(t, err)
-			jsonRequestString := string(enc)
-
-			reqArg, err := json.Marshal(execPayload)
-			require.NoError(t, err)
-
-			// We expect the JSON string RPC request contains the right arguments.
-			require.Equal(t, true, strings.Contains(
-				jsonRequestString, string(reqArg),
-			))
-			resp := map[string]interface{}{
-				"jsonrpc": "2.0",
-				"id":      1,
-				"result":  want,
-			}
-			err = json.NewEncoder(w).Encode(resp)
-			require.NoError(t, err)
-		}))
-		defer srv.Close()
-
-		rpcClient, err := rpc.DialHTTP(srv.URL)
-		require.NoError(t, err)
-		defer rpcClient.Close()
-
-		client := &Client{}
-		client.rpc = rpcClient
+		client := newPayloadSetup(t, want, execPayload)
 
 		// We call the RPC method via HTTP and expect a proper result.
 		resp, err := client.NewPayload(ctx, execPayload)
 		require.NoError(t, err)
-		require.DeepEqual(t, want, resp)
+		require.DeepEqual(t, want.LatestValidHash, resp)
+	})
+	t.Run(NewPayloadMethod+" SYNCING status", func(t *testing.T) {
+		execPayload, ok := fix["ExecutionPayload"].(*pb.ExecutionPayload)
+		require.Equal(t, true, ok)
+		want, ok := fix["SyncingStatus"].(*pb.PayloadStatus)
+		require.Equal(t, true, ok)
+		client := newPayloadSetup(t, want, execPayload)
+
+		// We call the RPC method via HTTP and expect a proper result.
+		resp, err := client.NewPayload(ctx, execPayload)
+		require.ErrorIs(t, ErrAcceptedSyncingPayloadStatus, err)
+		require.DeepEqual(t, []uint8(nil), resp)
+	})
+	t.Run(NewPayloadMethod+" INVALID_BLOCK_HASH status", func(t *testing.T) {
+		execPayload, ok := fix["ExecutionPayload"].(*pb.ExecutionPayload)
+		require.Equal(t, true, ok)
+		want, ok := fix["InvalidBlockHashStatus"].(*pb.PayloadStatus)
+		require.Equal(t, true, ok)
+		client := newPayloadSetup(t, want, execPayload)
+
+		// We call the RPC method via HTTP and expect a proper result.
+		resp, err := client.NewPayload(ctx, execPayload)
+		require.ErrorContains(t, "could not validate block hash", err)
+		require.DeepEqual(t, []uint8(nil), resp)
+	})
+	t.Run(NewPayloadMethod+" INVALID_TERMINAL_BLOCK status", func(t *testing.T) {
+		execPayload, ok := fix["ExecutionPayload"].(*pb.ExecutionPayload)
+		require.Equal(t, true, ok)
+		want, ok := fix["InvalidTerminalBlockStatus"].(*pb.PayloadStatus)
+		require.Equal(t, true, ok)
+		client := newPayloadSetup(t, want, execPayload)
+
+		// We call the RPC method via HTTP and expect a proper result.
+		resp, err := client.NewPayload(ctx, execPayload)
+		require.ErrorContains(t, "could not satisfy terminal block condition", err)
+		require.DeepEqual(t, []uint8(nil), resp)
+	})
+	t.Run(NewPayloadMethod+" INVALID status", func(t *testing.T) {
+		execPayload, ok := fix["ExecutionPayload"].(*pb.ExecutionPayload)
+		require.Equal(t, true, ok)
+		want, ok := fix["InvalidStatus"].(*pb.PayloadStatus)
+		require.Equal(t, true, ok)
+		client := newPayloadSetup(t, want, execPayload)
+
+		// We call the RPC method via HTTP and expect a proper result.
+		resp, err := client.NewPayload(ctx, execPayload)
+		require.ErrorIs(t, ErrInvalidPayloadStatus, err)
+		require.DeepEqual(t, want.LatestValidHash, resp)
+	})
+	t.Run(NewPayloadMethod+" UNKNOWN status", func(t *testing.T) {
+		execPayload, ok := fix["ExecutionPayload"].(*pb.ExecutionPayload)
+		require.Equal(t, true, ok)
+		want, ok := fix["UnknownStatus"].(*pb.PayloadStatus)
+		require.Equal(t, true, ok)
+		client := newPayloadSetup(t, want, execPayload)
+
+		// We call the RPC method via HTTP and expect a proper result.
+		resp, err := client.NewPayload(ctx, execPayload)
+		require.ErrorIs(t, ErrUnknownPayloadStatus, err)
+		require.DeepEqual(t, []uint8(nil), resp)
 	})
 	t.Run(ExecutionBlockByNumberMethod, func(t *testing.T) {
 		want, ok := fix["ExecutionBlock"].(*pb.ExecutionBlock)
@@ -304,9 +375,8 @@ func TestClient_HTTP(t *testing.T) {
 		client.rpc = rpcClient
 
 		// We call the RPC method via HTTP and expect a proper result.
-		resp, err := client.ExchangeTransitionConfiguration(ctx, want)
+		err = client.ExchangeTransitionConfiguration(ctx, want)
 		require.NoError(t, err)
-		require.DeepEqual(t, want, resp)
 	})
 	t.Run(ExecutionBlockByHashMethod, func(t *testing.T) {
 		arg := common.BytesToHash([]byte("foo"))
@@ -382,8 +452,8 @@ func TestExchangeTransitionConfiguration(t *testing.T) {
 		client := &Client{}
 		client.rpc = rpcClient
 
-		_, err = client.ExchangeTransitionConfiguration(ctx, request)
-		require.Equal(t, true, errors.Is(err, ErrMismatchTerminalBlockHash))
+		err = client.ExchangeTransitionConfiguration(ctx, request)
+		require.Equal(t, true, errors.Is(err, ErrConfigMismatch))
 	})
 	t.Run("wrong terminal total difficulty", func(t *testing.T) {
 		request, ok := fix["TransitionConfiguration"].(*pb.TransitionConfiguration)
@@ -398,7 +468,7 @@ func TestExchangeTransitionConfiguration(t *testing.T) {
 			}()
 
 			// Change the terminal block hash.
-			resp.TerminalTotalDifficulty = "bar"
+			resp.TerminalTotalDifficulty = "0x1"
 			respJSON := map[string]interface{}{
 				"jsonrpc": "2.0",
 				"id":      1,
@@ -415,8 +485,8 @@ func TestExchangeTransitionConfiguration(t *testing.T) {
 		client := &Client{}
 		client.rpc = rpcClient
 
-		_, err = client.ExchangeTransitionConfiguration(ctx, request)
-		require.Equal(t, true, errors.Is(err, ErrMismatchTerminalTotalDiff))
+		err = client.ExchangeTransitionConfiguration(ctx, request)
+		require.Equal(t, true, errors.Is(err, ErrConfigMismatch))
 	})
 }
 
@@ -533,7 +603,7 @@ func fixtures() map[string]interface{} {
 		StateRoot:     foo[:],
 		ReceiptsRoot:  foo[:],
 		LogsBloom:     baz,
-		Random:        foo[:],
+		PrevRandao:    foo[:],
 		BlockNumber:   1,
 		GasLimit:      1,
 		GasUsed:       1,
@@ -563,7 +633,7 @@ func fixtures() map[string]interface{} {
 		ReceiptsRoot:     receiptsRoot,
 		LogsBloom:        logsBloom,
 		Difficulty:       bytesutil.PadTo([]byte("1"), fieldparams.RootLength),
-		TotalDifficulty:  bytesutil.PadTo([]byte("2"), fieldparams.RootLength),
+		TotalDifficulty:  "2",
 		GasLimit:         3,
 		GasUsed:          4,
 		Timestamp:        5,
@@ -574,7 +644,7 @@ func fixtures() map[string]interface{} {
 		Uncles:           [][]byte{foo[:]},
 	}
 	status := &pb.PayloadStatus{
-		Status:          pb.PayloadStatus_ACCEPTED,
+		Status:          pb.PayloadStatus_VALID,
 		LatestValidHash: foo[:],
 		ValidationError: "",
 	}
@@ -583,17 +653,86 @@ func fixtures() map[string]interface{} {
 		Status:    status,
 		PayloadId: &id,
 	}
+	forkChoiceSyncingResp := &ForkchoiceUpdatedResponse{
+		Status: &pb.PayloadStatus{
+			Status:          pb.PayloadStatus_SYNCING,
+			LatestValidHash: nil,
+		},
+		PayloadId: &id,
+	}
+	forkChoiceInvalidTerminalBlockResp := &ForkchoiceUpdatedResponse{
+		Status: &pb.PayloadStatus{
+			Status:          pb.PayloadStatus_INVALID_TERMINAL_BLOCK,
+			LatestValidHash: nil,
+		},
+		PayloadId: &id,
+	}
+	forkChoiceAcceptedResp := &ForkchoiceUpdatedResponse{
+		Status: &pb.PayloadStatus{
+			Status:          pb.PayloadStatus_ACCEPTED,
+			LatestValidHash: nil,
+		},
+		PayloadId: &id,
+	}
+	forkChoiceInvalidResp := &ForkchoiceUpdatedResponse{
+		Status: &pb.PayloadStatus{
+			Status:          pb.PayloadStatus_INVALID,
+			LatestValidHash: []byte("latestValidHash"),
+		},
+		PayloadId: &id,
+	}
+	b, _ := new(big.Int).SetString(params.BeaconConfig().TerminalTotalDifficulty, 10)
+	ttd, _ := uint256.FromBig(b)
 	transitionCfg := &pb.TransitionConfiguration{
 		TerminalBlockHash:       params.BeaconConfig().TerminalBlockHash[:],
-		TerminalTotalDifficulty: params.BeaconConfig().TerminalTotalDifficulty,
+		TerminalTotalDifficulty: ttd.Hex(),
 		TerminalBlockNumber:     big.NewInt(0).Bytes(),
 	}
+	validStatus := &pb.PayloadStatus{
+		Status:          pb.PayloadStatus_VALID,
+		LatestValidHash: foo[:],
+		ValidationError: "",
+	}
+	inValidBlockHashStatus := &pb.PayloadStatus{
+		Status:          pb.PayloadStatus_INVALID_BLOCK_HASH,
+		LatestValidHash: nil,
+	}
+	inValidTerminalBlockStatus := &pb.PayloadStatus{
+		Status:          pb.PayloadStatus_INVALID_TERMINAL_BLOCK,
+		LatestValidHash: nil,
+	}
+	acceptedStatus := &pb.PayloadStatus{
+		Status:          pb.PayloadStatus_ACCEPTED,
+		LatestValidHash: nil,
+	}
+	syncingStatus := &pb.PayloadStatus{
+		Status:          pb.PayloadStatus_SYNCING,
+		LatestValidHash: nil,
+	}
+	invalidStatus := &pb.PayloadStatus{
+		Status:          pb.PayloadStatus_INVALID,
+		LatestValidHash: foo[:],
+	}
+	unknownStatus := &pb.PayloadStatus{
+		Status:          pb.PayloadStatus_UNKNOWN,
+		LatestValidHash: foo[:],
+	}
 	return map[string]interface{}{
-		"ExecutionBlock":            executionBlock,
-		"ExecutionPayload":          executionPayloadFixture,
-		"PayloadStatus":             status,
-		"ForkchoiceUpdatedResponse": forkChoiceResp,
-		"TransitionConfiguration":   transitionCfg,
+		"ExecutionBlock":                                executionBlock,
+		"ExecutionPayload":                              executionPayloadFixture,
+		"ValidPayloadStatus":                            validStatus,
+		"InvalidBlockHashStatus":                        inValidBlockHashStatus,
+		"InvalidTerminalBlockStatus":                    inValidTerminalBlockStatus,
+		"AcceptedStatus":                                acceptedStatus,
+		"SyncingStatus":                                 syncingStatus,
+		"InvalidStatus":                                 invalidStatus,
+		"UnknownStatus":                                 unknownStatus,
+		"ForkchoiceUpdatedResponse":                     forkChoiceResp,
+		"ForkchoiceUpdatedSyncingResponse":              forkChoiceSyncingResp,
+		"ForkchoiceUpdatedInvalidTerminalBlockResponse": forkChoiceInvalidTerminalBlockResp,
+		"ForkchoiceUpdatedAcceptedResponse":             forkChoiceAcceptedResp,
+		"ForkchoiceUpdatedInvalidResponse":              forkChoiceInvalidResp,
+		"TransitionConfiguration":                       transitionCfg,
 	}
 }
 
@@ -653,6 +792,7 @@ func (*testEngineService) ForkchoiceUpdatedV1(
 	if !ok {
 		panic("not found")
 	}
+	item.Status.Status = pb.PayloadStatus_VALID
 	return item
 }
 
@@ -660,9 +800,83 @@ func (*testEngineService) NewPayloadV1(
 	_ context.Context, _ *pb.ExecutionPayload,
 ) *pb.PayloadStatus {
 	fix := fixtures()
-	item, ok := fix["PayloadStatus"].(*pb.PayloadStatus)
+	item, ok := fix["ValidPayloadStatus"].(*pb.PayloadStatus)
 	if !ok {
 		panic("not found")
 	}
 	return item
 }
+
+func forkchoiceUpdateSetup(t *testing.T, fcs *pb.ForkchoiceState, att *pb.PayloadAttributes, res *ForkchoiceUpdatedResponse) *Client {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		defer func() {
+			require.NoError(t, r.Body.Close())
+		}()
+		enc, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		jsonRequestString := string(enc)
+
+		forkChoiceStateReq, err := json.Marshal(fcs)
+		require.NoError(t, err)
+		payloadAttrsReq, err := json.Marshal(att)
+		require.NoError(t, err)
+
+		// We expect the JSON string RPC request contains the right arguments.
+		require.Equal(t, true, strings.Contains(
+			jsonRequestString, string(forkChoiceStateReq),
+		))
+		require.Equal(t, true, strings.Contains(
+			jsonRequestString, string(payloadAttrsReq),
+		))
+		resp := map[string]interface{}{
+			"jsonrpc": "2.0",
+			"id":      1,
+			"result":  res,
+		}
+		err = json.NewEncoder(w).Encode(resp)
+		require.NoError(t, err)
+	}))
+
+	rpcClient, err := rpc.DialHTTP(srv.URL)
+	require.NoError(t, err)
+
+	client := &Client{}
+	client.rpc = rpcClient
+
+	return client
+}
+
+func newPayloadSetup(t *testing.T, status *pb.PayloadStatus, payload *pb.ExecutionPayload) *Client {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		defer func() {
+			require.NoError(t, r.Body.Close())
+		}()
+		enc, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		jsonRequestString := string(enc)
+
+		reqArg, err := json.Marshal(payload)
+		require.NoError(t, err)
+
+		// We expect the JSON string RPC request contains the right arguments.
+		require.Equal(t, true, strings.Contains(
+			jsonRequestString, string(reqArg),
+		))
+		resp := map[string]interface{}{
+			"jsonrpc": "2.0",
+			"id":      1,
+			"result":  status,
+		}
+		err = json.NewEncoder(w).Encode(resp)
+		require.NoError(t, err)
+	}))
+
+	rpcClient, err := rpc.DialHTTP(srv.URL)
+	require.NoError(t, err)
+
+	client := &Client{}
+	client.rpc = rpcClient
+	return client
+}

beacon-chain/powchain/engine-api-client/v1/errors.go

@@ -17,12 +17,23 @@ var (
 	ErrServer = errors.New("client error while processing request")
 	// ErrUnknownPayload corresponds to JSON-RPC code -32001.
 	ErrUnknownPayload = errors.New("payload does not exist or is not available")
+	// ErrUnknownPayloadStatus when the payload status is unknown.
+	ErrUnknownPayloadStatus = errors.New("unknown payload status")
 	// ErrUnsupportedScheme for unsupported URL schemes.
 	ErrUnsupportedScheme = errors.New("unsupported url scheme, only http(s) and ipc are supported")
+	// ErrConfigMismatch when the execution node's terminal total difficulty or
+	// terminal block hash received via the API mismatches Prysm's configuration value.
+	ErrConfigMismatch = errors.New("execution client configuration mismatch")
 	// ErrMismatchTerminalBlockHash when the terminal block hash value received via
 	// the API mismatches Prysm's configuration value.
 	ErrMismatchTerminalBlockHash = errors.New("terminal block hash mismatch")
 	// ErrMismatchTerminalTotalDiff when the terminal total difficulty value received via
 	// the API mismatches Prysm's configuration value.
 	ErrMismatchTerminalTotalDiff = errors.New("terminal total difficulty mismatch")
+	// ErrAcceptedSyncingPayloadStatus when the status of the payload is syncing or accepted.
+	ErrAcceptedSyncingPayloadStatus = errors.New("payload status is SYNCING or ACCEPTED")
+	// ErrInvalidPayloadStatus when the status of the payload is invalid.
+	ErrInvalidPayloadStatus = errors.New("payload status is INVALID")
+	// ErrNilResponse when the response is nil.
+	ErrNilResponse = errors.New("nil response")
 )

beacon-chain/powchain/engine-api-client/v1/mocks/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["client.go"],
+    importpath = "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1/mocks",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//proto/engine/v1:go_default_library",
+        "@com_github_ethereum_go_ethereum//common:go_default_library",
+    ],
+)

beacon-chain/powchain/engine-api-client/v1/mocks/client.go

@@ -0,0 +1,50 @@
+package mocks
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/common"
+	pb "github.com/prysmaticlabs/prysm/proto/engine/v1"
+)
+
+// EngineClient --
+type EngineClient struct {
+	NewPayloadResp        []byte
+	PayloadIDBytes        *pb.PayloadIDBytes
+	ForkChoiceUpdatedResp []byte
+	ExecutionPayload      *pb.ExecutionPayload
+	Err                   error
+	ExecutionBlock        *pb.ExecutionBlock
+}
+
+// NewPayload --
+func (e *EngineClient) NewPayload(_ context.Context, _ *pb.ExecutionPayload) ([]byte, error) {
+	return e.NewPayloadResp, nil
+}
+
+// ForkchoiceUpdated --
+func (e *EngineClient) ForkchoiceUpdated(
+	_ context.Context, _ *pb.ForkchoiceState, _ *pb.PayloadAttributes,
+) (*pb.PayloadIDBytes, []byte, error) {
+	return e.PayloadIDBytes, e.ForkChoiceUpdatedResp, nil
+}
+
+// GetPayload --
+func (e *EngineClient) GetPayload(_ context.Context, _ [8]byte) (*pb.ExecutionPayload, error) {
+	return e.ExecutionPayload, nil
+}
+
+// ExchangeTransitionConfiguration --
+func (e *EngineClient) ExchangeTransitionConfiguration(_ context.Context, _ *pb.TransitionConfiguration) error {
+	return e.Err
+}
+
+// LatestExecutionBlock --
+func (e *EngineClient) LatestExecutionBlock(_ context.Context) (*pb.ExecutionBlock, error) {
+	return e.ExecutionBlock, nil
+}
+
+// ExecutionBlockByHash --
+func (e *EngineClient) ExecutionBlockByHash(_ context.Context, _ common.Hash) (*pb.ExecutionBlock, error) {
+	return e.ExecutionBlock, nil
+}

beacon-chain/powchain/engine-api-client/v1/options.go

@@ -19,11 +19,21 @@ func defaultConfig() *config {
 	}
 }
 
-// WithHTTPClient allows setting a custom HTTP client
-// for the API connection.
-func WithHTTPClient(httpClient *http.Client) Option {
+// WithJWTSecret allows setting a JWT secret for authenticating
+// the client via HTTP connections.
+func WithJWTSecret(secret []byte) Option {
 	return func(c *Client) error {
-		c.cfg.httpClient = httpClient
+		if len(secret) == 0 {
+			return nil
+		}
+		authTransport := &jwtTransport{
+			underlyingTransport: http.DefaultTransport,
+			jwtSecret:           secret,
+		}
+		c.cfg.httpClient = &http.Client{
+			Timeout:   DefaultTimeout,
+			Transport: authTransport,
+		}
 		return nil
 	}
 }

beacon-chain/powchain/options.go

@@ -40,6 +40,14 @@ func WithExecutionEndpoint(endpoint string) Option {
 	}
 }
 
+// WithExecutionClientJWTSecret for authenticating the execution node JSON-RPC endpoint.
+func WithExecutionClientJWTSecret(jwtSecret []byte) Option {
+	return func(s *Service) error {
+		s.cfg.executionEndpointJWTSecret = jwtSecret
+		return nil
+	}
+}
+
 // WithDepositContractAddress for the deposit contract.
 func WithDepositContractAddress(addr common.Address) Option {
 	return func(s *Service) error {

beacon-chain/powchain/service.go

@@ -127,17 +127,18 @@ type RPCClient interface {
 
 // config defines a config struct for dependencies into the service.
 type config struct {
-	depositContractAddr     common.Address
-	beaconDB                db.HeadAccessDatabase
-	depositCache            *depositcache.DepositCache
-	stateNotifier           statefeed.Notifier
-	stateGen                *stategen.State
-	eth1HeaderReqLimit      uint64
-	beaconNodeStatsUpdater  BeaconNodeStatsUpdater
-	httpEndpoints           []network.Endpoint
-	executionEndpoint       string
-	currHttpEndpoint        network.Endpoint
-	finalizedStateAtStartup state.BeaconState
+	depositContractAddr        common.Address
+	beaconDB                   db.HeadAccessDatabase
+	depositCache               *depositcache.DepositCache
+	stateNotifier              statefeed.Notifier
+	stateGen                   *stategen.State
+	eth1HeaderReqLimit         uint64
+	beaconNodeStatsUpdater     BeaconNodeStatsUpdater
+	httpEndpoints              []network.Endpoint
+	executionEndpoint          string
+	executionEndpointJWTSecret []byte
+	currHttpEndpoint           network.Endpoint
+	finalizedStateAtStartup    state.BeaconState
 }
 
 // Service fetches important information about the canonical
@@ -156,7 +157,7 @@ type Service struct {
 	headTicker              *time.Ticker
 	httpLogger              bind.ContractFilterer
 	eth1DataFetcher         RPCDataFetcher
-	engineAPIClient         *engine.Client
+	engineAPIClient         engine.Caller
 	rpcClient               RPCClient
 	headerCache             *headerCache // cache to store block hash/block height.
 	latestEth1Data          *ethpb.LatestETH1Data
@@ -216,6 +217,9 @@ func NewService(ctx context.Context, opts ...Option) (*Service, error) {
 		return nil, errors.Wrap(err, "unable to initialize engine API client")
 	}
 
+	// Check transition configuration for the engine API client in the background.
+	go s.checkTransitionConfiguration(ctx)
+
 	if err := s.ensureValidPowchainData(ctx); err != nil {
 		return nil, errors.Wrap(err, "unable to validate powchain data")
 	}
@@ -298,15 +302,12 @@ func (s *Service) Status() error {
 		return nil
 	}
 	// get error from run function
-	if s.runError != nil {
-		return s.runError
-	}
-	return nil
+	return s.runError
 }
 
 // EngineAPIClient returns the associated engine API client to interact
 // with an execution node via JSON-RPC.
-func (s *Service) EngineAPIClient() *engine.Client {
+func (s *Service) EngineAPIClient() engine.Caller {
 	return s.engineAPIClient
 }
 
@@ -1058,7 +1059,10 @@ func (s *Service) initializeEngineAPIClient(ctx context.Context) error {
 	if s.cfg.executionEndpoint == "" {
 		return nil
 	}
-	client, err := engine.New(ctx, s.cfg.executionEndpoint)
+	opts := []engine.Option{
+		engine.WithJWTSecret(s.cfg.executionEndpointJWTSecret),
+	}
+	client, err := engine.New(ctx, s.cfg.executionEndpoint, opts...)
 	if err != nil {
 		return err
 	}

beacon-chain/rpc/apimiddleware/custom_handlers.go

@@ -74,7 +74,7 @@ func handleGetSSZ(
 		apimiddleware.WriteError(w, errJson, nil)
 		return true
 	}
-	grpcResponse, errJson := apimiddleware.ProxyRequest(req)
+	grpcResponse, errJson := m.ProxyRequest(req)
 	if errJson != nil {
 		apimiddleware.WriteError(w, errJson, nil)
 		return true

beacon-chain/rpc/eth/beacon/sync_committee_test.go

@@ -212,6 +212,10 @@ func (m *futureSyncMockFetcher) StateRoot(context.Context, []byte) ([]byte, erro
 	return m.BeaconStateRoot, nil
 }
 
+func (m *futureSyncMockFetcher) StateBySlot(context.Context, types.Slot) (state.BeaconState, error) {
+	return m.BeaconState, nil
+}
+
 func TestListSyncCommitteesFuture(t *testing.T) {
 	ctx := context.Background()
 	st, _ := util.DeterministicGenesisStateAltair(t, params.BeaconConfig().SyncCommitteeSize)

beacon-chain/rpc/prysm/v1alpha1/beacon/BUILD.bazel

@@ -91,6 +91,7 @@ go_test(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/signing:go_default_library",
         "//beacon-chain/core/time:go_default_library",
+        "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/operations/attestations:go_default_library",

beacon-chain/rpc/prysm/v1alpha1/beacon/assignments.go

@@ -2,6 +2,7 @@ package beacon
 
 import (
 	"context"
+	"fmt"
 	"strconv"
 
 	types "github.com/prysmaticlabs/eth2-types"
@@ -58,9 +59,11 @@ func (bs *Server) ListValidatorAssignments(
 	if err != nil {
 		return nil, err
 	}
-	requestedState, err := bs.StateGen.StateBySlot(ctx, startSlot)
+	requestedState, err := bs.ReplayerBuilder.ForSlot(startSlot).ReplayBlocks(ctx)
 	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Could not retrieve archived state for epoch %d: %v", requestedEpoch, err)
+		msg := fmt.Sprintf("could not replay all blocks from the closest stored state (at slot %d) "+
+			"to the requested epoch (%d) - %v", startSlot, requestedEpoch, err)
+		return nil, status.Error(codes.Internal, msg)
 	}
 
 	// Filter out assignments by public keys.

beacon-chain/rpc/prysm/v1alpha1/beacon/assignments_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	dbTest "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	mockstategen "github.com/prysmaticlabs/prysm/beacon-chain/state/stategen/mock"
 	"github.com/prysmaticlabs/prysm/cmd"
 	"github.com/prysmaticlabs/prysm/config/params"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
@@ -24,13 +25,13 @@ import (
 )
 
 func TestServer_ListAssignments_CannotRequestFutureEpoch(t *testing.T) {
-
 	db := dbTest.SetupDB(t)
 	ctx := context.Background()
 	bs := &Server{
 		BeaconDB:           db,
 		GenesisTimeFetcher: &mock.ChainService{},
 	}
+	addDefaultReplayerBuilder(bs, db)
 
 	wanted := errNoEpochInfoError
 	_, err := bs.ListValidatorAssignments(
@@ -45,7 +46,6 @@ func TestServer_ListAssignments_CannotRequestFutureEpoch(t *testing.T) {
 }
 
 func TestServer_ListAssignments_NoResults(t *testing.T) {
-
 	db := dbTest.SetupDB(t)
 	ctx := context.Background()
 	st, err := util.NewBeaconState()
@@ -62,6 +62,7 @@ func TestServer_ListAssignments_NoResults(t *testing.T) {
 		BeaconDB:           db,
 		GenesisTimeFetcher: &mock.ChainService{},
 		StateGen:           stategen.New(db),
+		ReplayerBuilder:    mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(st)),
 	}
 	wanted := &ethpb.ValidatorAssignments{
 		Assignments:   make([]*ethpb.ValidatorAssignments_CommitteeAssignment, 0),
@@ -101,8 +102,8 @@ func TestServer_ListAssignments_Pagination_InputOutOfRange(t *testing.T) {
 		})
 	}
 
-	blk := util.NewBeaconBlock().Block
-	blockRoot, err := blk.HashTreeRoot()
+	blk := util.NewBeaconBlock()
+	blockRoot, err := blk.Block.HashTreeRoot()
 	require.NoError(t, err)
 
 	s, err := util.NewBeaconState()
@@ -123,6 +124,7 @@ func TestServer_ListAssignments_Pagination_InputOutOfRange(t *testing.T) {
 		},
 		GenesisTimeFetcher: &mock.ChainService{},
 		StateGen:           stategen.New(db),
+		ReplayerBuilder:    mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(s)),
 	}
 
 	wanted := fmt.Sprintf("page start %d >= list %d", 500, count)
@@ -176,8 +178,8 @@ func TestServer_ListAssignments_Pagination_DefaultPageSize_NoArchive(t *testing.
 		}
 	}
 
-	blk := util.NewBeaconBlock().Block
-	blockRoot, err := blk.HashTreeRoot()
+	b := util.NewBeaconBlock()
+	blockRoot, err := b.Block.HashTreeRoot()
 	require.NoError(t, err)
 
 	s, err := util.NewBeaconState()
@@ -198,6 +200,7 @@ func TestServer_ListAssignments_Pagination_DefaultPageSize_NoArchive(t *testing.
 		},
 		GenesisTimeFetcher: &mock.ChainService{},
 		StateGen:           stategen.New(db),
+		ReplayerBuilder:    mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(s)),
 	}
 
 	res, err := bs.ListValidatorAssignments(context.Background(), &ethpb.ListValidatorAssignmentsRequest{
@@ -246,8 +249,8 @@ func TestServer_ListAssignments_FilterPubkeysIndices_NoPagination(t *testing.T)
 		validators = append(validators, val)
 	}
 
-	blk := util.NewBeaconBlock().Block
-	blockRoot, err := blk.HashTreeRoot()
+	b := util.NewBeaconBlock()
+	blockRoot, err := b.Block.HashTreeRoot()
 	require.NoError(t, err)
 	s, err := util.NewBeaconState()
 	require.NoError(t, err)
@@ -264,6 +267,7 @@ func TestServer_ListAssignments_FilterPubkeysIndices_NoPagination(t *testing.T)
 		},
 		GenesisTimeFetcher: &mock.ChainService{},
 		StateGen:           stategen.New(db),
+		ReplayerBuilder:    mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(s)),
 	}
 
 	pubKey1 := make([]byte, params.BeaconConfig().BLSPubkeyLength)
@@ -315,13 +319,16 @@ func TestServer_ListAssignments_CanFilterPubkeysIndices_WithPagination(t *testin
 		validators = append(validators, val)
 	}
 
-	blk := util.NewBeaconBlock().Block
-	blockRoot, err := blk.HashTreeRoot()
+	b := util.NewBeaconBlock()
+	blockRoot, err := b.Block.HashTreeRoot()
 	require.NoError(t, err)
 	s, err := util.NewBeaconState()
 	require.NoError(t, err)
+	w, err := wrapper.WrappedSignedBeaconBlock(b)
+	require.NoError(t, err)
 	require.NoError(t, s.SetValidators(validators))
 	require.NoError(t, db.SaveState(ctx, s, blockRoot))
+	require.NoError(t, db.SaveBlock(ctx, w))
 	require.NoError(t, db.SaveGenesisBlockRoot(ctx, blockRoot))
 
 	bs := &Server{
@@ -335,6 +342,8 @@ func TestServer_ListAssignments_CanFilterPubkeysIndices_WithPagination(t *testin
 		StateGen:           stategen.New(db),
 	}
 
+	addDefaultReplayerBuilder(bs, db)
+
 	req := &ethpb.ListValidatorAssignmentsRequest{Indices: []types.ValidatorIndex{1, 2, 3, 4, 5, 6}, PageSize: 2, PageToken: "1"}
 	res, err := bs.ListValidatorAssignments(context.Background(), req)
 	require.NoError(t, err)

beacon-chain/rpc/prysm/v1alpha1/beacon/blocks.go

@@ -2,6 +2,7 @@ package beacon
 
 import (
 	"context"
+	"fmt"
 	"strconv"
 
 	"github.com/pkg/errors"
@@ -478,10 +479,12 @@ func (bs *Server) GetWeakSubjectivityCheckpoint(ctx context.Context, _ *emptypb.
 		return nil, status.Error(codes.Internal, "Could not get weak subjectivity slot")
 	}
 
-	wsState, err := bs.StateGen.StateBySlot(ctx, wsSlot)
+	wsState, err := bs.ReplayerBuilder.ForSlot(wsSlot).ReplayBlocks(ctx)
 	if err != nil {
-		return nil, status.Error(codes.Internal, "Could not get weak subjectivity state")
+		msg := fmt.Sprintf("error replaying blocks for state at slot %d: %v", wsSlot, err)
+		return nil, status.Error(codes.Internal, msg)
 	}
+
 	stateRoot, err := wsState.HashTreeRoot(ctx)
 	if err != nil {
 		return nil, status.Error(codes.Internal, "Could not get weak subjectivity state root")

beacon-chain/rpc/prysm/v1alpha1/beacon/blocks_test.go

@@ -12,9 +12,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	blockfeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/block"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	dbTest "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	v1 "github.com/prysmaticlabs/prysm/beacon-chain/state/v1"
 	"github.com/prysmaticlabs/prysm/cmd"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
@@ -728,77 +726,6 @@ func TestServer_StreamBlocksVerified_OnHeadUpdated(t *testing.T) {
 	<-exitRoutine
 }
 
-func TestServer_GetWeakSubjectivityCheckpoint(t *testing.T) {
-	params.SetupTestConfigCleanup(t)
-	params.OverrideBeaconConfig(params.MainnetConfig())
-
-	db := dbTest.SetupDB(t)
-	ctx := context.Background()
-
-	// Beacon state.
-	beaconState, err := util.NewBeaconState()
-	require.NoError(t, err)
-	require.NoError(t, beaconState.SetSlot(10))
-
-	// Active validator set is used for computing the weak subjectivity period.
-	numVals := 256 // Works with params.BeaconConfig().MinGenesisActiveValidatorCount as well, but takes longer.
-	validators := make([]*ethpb.Validator, numVals)
-	balances := make([]uint64, len(validators))
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{
-			PublicKey:             make([]byte, params.BeaconConfig().BLSPubkeyLength),
-			WithdrawalCredentials: make([]byte, 32),
-			EffectiveBalance:      28 * 1e9,
-			ExitEpoch:             params.BeaconConfig().FarFutureEpoch,
-		}
-		balances[i] = validators[i].EffectiveBalance
-	}
-	require.NoError(t, beaconState.SetValidators(validators))
-	require.NoError(t, beaconState.SetBalances(balances))
-
-	// Genesis block.
-	genesisBlock := util.NewBeaconBlock()
-	genesisBlockRoot, err := genesisBlock.Block.HashTreeRoot()
-	require.NoError(t, err)
-	require.NoError(t, db.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(genesisBlock)))
-	require.NoError(t, db.SaveState(ctx, beaconState, genesisBlockRoot))
-	require.NoError(t, db.SaveGenesisBlockRoot(ctx, genesisBlockRoot))
-
-	// Finalized checkpoint.
-	finalizedEpoch := types.Epoch(1020)
-	require.NoError(t, beaconState.SetSlot(types.Slot(finalizedEpoch.Mul(uint64(params.BeaconConfig().SlotsPerEpoch)))))
-	require.NoError(t, beaconState.SetCurrentJustifiedCheckpoint(&ethpb.Checkpoint{
-		Epoch: finalizedEpoch - 1,
-		Root:  bytesutil.PadTo([]byte{'A'}, fieldparams.RootLength),
-	}))
-	require.NoError(t, beaconState.SetFinalizedCheckpoint(&ethpb.Checkpoint{
-		Epoch: finalizedEpoch,
-		Root:  bytesutil.PadTo([]byte{'B'}, fieldparams.RootLength),
-	}))
-
-	chainService := &chainMock.ChainService{State: beaconState}
-	server := &Server{
-		Ctx:           ctx,
-		BlockNotifier: chainService.BlockNotifier(),
-		HeadFetcher:   chainService,
-		BeaconDB:      db,
-		StateGen:      stategen.New(db),
-	}
-
-	wsEpoch, err := helpers.ComputeWeakSubjectivityPeriod(context.Background(), beaconState)
-	require.NoError(t, err)
-
-	c, err := server.GetWeakSubjectivityCheckpoint(ctx, &emptypb.Empty{})
-	require.NoError(t, err)
-	e := finalizedEpoch - (finalizedEpoch % wsEpoch)
-	require.Equal(t, e, c.Epoch)
-	wsState, err := server.StateGen.StateBySlot(ctx, params.BeaconConfig().SlotsPerEpoch.Mul(uint64(e)))
-	require.NoError(t, err)
-	sRoot, err := wsState.HashTreeRoot(ctx)
-	require.NoError(t, err)
-	require.DeepEqual(t, sRoot[:], c.StateRoot)
-}
-
 func TestServer_ListBeaconBlocks_NoResults(t *testing.T) {
 	db := dbTest.SetupDB(t)
 	ctx := context.Background()

beacon-chain/rpc/prysm/v1alpha1/beacon/committees.go

@@ -74,9 +74,9 @@ func (bs *Server) retrieveCommitteesForEpoch(
 	if err != nil {
 		return nil, nil, err
 	}
-	requestedState, err := bs.StateGen.StateBySlot(ctx, startSlot)
+	requestedState, err := bs.ReplayerBuilder.ForSlot(startSlot).ReplayBlocks(ctx)
 	if err != nil {
-		return nil, nil, status.Errorf(codes.Internal, "Could not get state: %v", err)
+		return nil, nil, status.Errorf(codes.Internal, "error replaying blocks for state at slot %d: %v", startSlot, err)
 	}
 	seed, err := helpers.Seed(requestedState, epoch, params.BeaconConfig().DomainBeaconAttester)
 	if err != nil {

beacon-chain/rpc/prysm/v1alpha1/beacon/committees_test.go

@@ -3,6 +3,7 @@ package beacon
 import (
 	"context"
 	"encoding/binary"
+	"math"
 	"testing"
 	"time"
 
@@ -11,6 +12,7 @@ import (
 	dbTest "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	mockstategen "github.com/prysmaticlabs/prysm/beacon-chain/state/stategen/mock"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
@@ -48,6 +50,8 @@ func TestServer_ListBeaconCommittees_CurrentEpoch(t *testing.T) {
 	require.NoError(t, db.SaveGenesisBlockRoot(ctx, gRoot))
 	require.NoError(t, db.SaveState(ctx, headState, gRoot))
 
+	bs.ReplayerBuilder = mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(headState))
+
 	activeIndices, err := helpers.ActiveValidatorIndices(ctx, headState, 0)
 	require.NoError(t, err)
 	attesterSeed, err := helpers.Seed(headState, 0, params.BeaconConfig().DomainBeaconAttester)
@@ -69,6 +73,13 @@ func TestServer_ListBeaconCommittees_CurrentEpoch(t *testing.T) {
 	}
 }
 
+func addDefaultReplayerBuilder(s *Server, h stategen.HistoryAccessor) {
+	cc := &mockstategen.MockCanonicalChecker{Is: true, Err: nil}
+	cs := &mockstategen.MockCurrentSlotter{Slot: math.MaxUint64 - 1}
+	s.ReplayerBuilder = stategen.NewCanonicalBuilder(h, cc, cs)
+}
+
+// TODO: test failure
 func TestServer_ListBeaconCommittees_PreviousEpoch(t *testing.T) {
 	params.SetupTestConfigCleanup(t)
 	params.OverrideBeaconConfig(params.MainnetConfig())
@@ -87,12 +98,13 @@ func TestServer_ListBeaconCommittees_PreviousEpoch(t *testing.T) {
 	require.NoError(t, headState.SetRandaoMixes(mixes))
 	require.NoError(t, headState.SetSlot(params.BeaconConfig().SlotsPerEpoch))
 
-	b := util.NewBeaconBlock()
-	require.NoError(t, db.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
-	gRoot, err := b.Block.HashTreeRoot()
+	b, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlock())
+	require.NoError(t, wrapper.SetBlockSlot(b, headState.Slot()))
+	require.NoError(t, err)
+	require.NoError(t, db.SaveBlock(ctx, b))
+	gRoot, err := b.Block().HashTreeRoot()
 	require.NoError(t, err)
 	require.NoError(t, db.SaveState(ctx, headState, gRoot))
-	require.NoError(t, db.SaveGenesisBlockRoot(ctx, gRoot))
 
 	offset := int64(headState.Slot().Mul(params.BeaconConfig().SecondsPerSlot))
 	m := &mock.ChainService{
@@ -104,6 +116,7 @@ func TestServer_ListBeaconCommittees_PreviousEpoch(t *testing.T) {
 		GenesisTimeFetcher: m,
 		StateGen:           stategen.New(db),
 	}
+	addDefaultReplayerBuilder(bs, db)
 
 	activeIndices, err := helpers.ActiveValidatorIndices(ctx, headState, 1)
 	require.NoError(t, err)

beacon-chain/rpc/prysm/v1alpha1/beacon/server.go

@@ -45,4 +45,5 @@ type Server struct {
 	CollectedAttestationsBuffer chan []*ethpb.Attestation
 	StateGen                    stategen.StateManager
 	SyncChecker                 sync.Checker
+	ReplayerBuilder             stategen.ReplayerBuilder
 }

beacon-chain/rpc/prysm/v1alpha1/beacon/validators.go

@@ -2,6 +2,7 @@ package beacon
 
 import (
 	"context"
+	"fmt"
 	"sort"
 	"strconv"
 
@@ -64,9 +65,9 @@ func (bs *Server) ListValidatorBalances(
 	if err != nil {
 		return nil, err
 	}
-	requestedState, err := bs.StateGen.StateBySlot(ctx, startSlot)
+	requestedState, err := bs.ReplayerBuilder.ForSlot(startSlot).ReplayBlocks(ctx)
 	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Could not get state: %v", err)
+		return nil, status.Error(codes.Internal, fmt.Sprintf("error replaying blocks for state at slot %d: %v", startSlot, err))
 	}
 
 	vals := requestedState.Validators()
@@ -219,7 +220,10 @@ func (bs *Server) ListValidators(
 		if err != nil {
 			return nil, err
 		}
-		reqState, err = bs.StateGen.StateBySlot(ctx, s)
+		reqState, err = bs.ReplayerBuilder.ForSlot(s).ReplayBlocks(ctx)
+		if err != nil {
+			return nil, status.Error(codes.Internal, fmt.Sprintf("error replaying blocks for state at slot %d: %v", s, err))
+		}
 	} else {
 		reqState, err = bs.HeadFetcher.HeadState(ctx)
 	}
@@ -411,9 +415,9 @@ func (bs *Server) GetValidatorActiveSetChanges(
 	if err != nil {
 		return nil, err
 	}
-	requestedState, err := bs.StateGen.StateBySlot(ctx, s)
+	requestedState, err := bs.ReplayerBuilder.ForSlot(s).ReplayBlocks(ctx)
 	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Could not get state: %v", err)
+		return nil, status.Error(codes.Internal, fmt.Sprintf("error replaying blocks for state at slot %d: %v", s, err))
 	}
 
 	activeValidatorCount, err := helpers.ActiveValidatorCount(ctx, requestedState, coreTime.CurrentEpoch(requestedState))
@@ -502,22 +506,11 @@ func (bs *Server) GetValidatorParticipation(
 	// Use the last slot of requested epoch to obtain current and previous epoch attestations.
 	// This ensures that we don't miss previous attestations when input requested epochs.
 	startSlot += params.BeaconConfig().SlotsPerEpoch - 1
-	// The start slot should be a canonical slot.
-	canonical, err := bs.isSlotCanonical(ctx, startSlot)
+
+	// ReplayerBuilder insures that a canonical chain is followed to the slot
+	beaconState, err := bs.ReplayerBuilder.ForSlot(startSlot).ReplayBlocks(ctx)
 	if err != nil {
-		return nil, err
-	}
-	// Keep looking back until there's a canonical slot.
-	for i := int(startSlot - 1); !canonical && i >= 0; i-- {
-		canonical, err = bs.isSlotCanonical(ctx, types.Slot(i))
-		if err != nil {
-			return nil, err
-		}
-		startSlot = types.Slot(i)
-	}
-	beaconState, err := bs.StateGen.StateBySlot(ctx, startSlot)
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Could not get state: %v", err)
+		return nil, status.Error(codes.Internal, fmt.Sprintf("error replaying blocks for state at slot %d: %v", startSlot, err))
 	}
 	var v []*precompute.Validator
 	var b *precompute.Balance
@@ -834,9 +827,9 @@ func (bs *Server) GetIndividualVotes(
 	if err != nil {
 		return nil, err
 	}
-	requestedState, err := bs.StateGen.StateBySlot(ctx, s)
+	st, err := bs.ReplayerBuilder.ForSlot(s).ReplayBlocks(ctx)
 	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Could not retrieve archived state for epoch %d: %v", req.Epoch, err)
+		return nil, status.Errorf(codes.Internal, "failed to replay blocks for state at epoch %d: %v", req.Epoch, err)
 	}
 	// Track filtered validators to prevent duplication in the response.
 	filtered := map[types.ValidatorIndex]bool{}
@@ -844,7 +837,7 @@ func (bs *Server) GetIndividualVotes(
 	votes := make([]*ethpb.IndividualVotesRespond_IndividualVote, 0, len(req.Indices)+len(req.PublicKeys))
 	// Filter out assignments by public keys.
 	for _, pubKey := range req.PublicKeys {
-		index, ok := requestedState.ValidatorIndexByPubkey(bytesutil.ToBytes48(pubKey))
+		index, ok := st.ValidatorIndexByPubkey(bytesutil.ToBytes48(pubKey))
 		if !ok {
 			votes = append(votes, &ethpb.IndividualVotesRespond_IndividualVote{PublicKey: pubKey, ValidatorIndex: types.ValidatorIndex(^uint64(0))})
 			continue
@@ -864,27 +857,27 @@ func (bs *Server) GetIndividualVotes(
 
 	var v []*precompute.Validator
 	var bal *precompute.Balance
-	switch requestedState.Version() {
+	switch st.Version() {
 	case version.Phase0:
-		v, bal, err = precompute.New(ctx, requestedState)
+		v, bal, err = precompute.New(ctx, st)
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "Could not set up pre compute instance: %v", err)
 		}
-		v, _, err = precompute.ProcessAttestations(ctx, requestedState, v, bal)
+		v, _, err = precompute.ProcessAttestations(ctx, st, v, bal)
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "Could not pre compute attestations: %v", err)
 		}
 	case version.Altair:
-		v, bal, err = altair.InitializePrecomputeValidators(ctx, requestedState)
+		v, bal, err = altair.InitializePrecomputeValidators(ctx, st)
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "Could not set up altair pre compute instance: %v", err)
 		}
-		v, _, err = altair.ProcessEpochParticipation(ctx, requestedState, bal, v)
+		v, _, err = altair.ProcessEpochParticipation(ctx, st, bal, v)
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "Could not pre compute attestations: %v", err)
 		}
 	default:
-		return nil, status.Errorf(codes.Internal, "Invalid state type retrieved with a version of %d", requestedState.Version())
+		return nil, status.Errorf(codes.Internal, "Invalid state type retrieved with a version of %d", st.Version())
 	}
 
 	for _, index := range filteredIndices {
@@ -892,7 +885,7 @@ func (bs *Server) GetIndividualVotes(
 			votes = append(votes, &ethpb.IndividualVotesRespond_IndividualVote{ValidatorIndex: index})
 			continue
 		}
-		val, err := requestedState.ValidatorAtIndexReadOnly(index)
+		val, err := st.ValidatorAtIndexReadOnly(index)
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "Could not retrieve validator: %v", err)
 
@@ -923,37 +916,6 @@ func (bs *Server) GetIndividualVotes(
 	}, nil
 }
 
-// isSlotCanonical returns true if the input slot has a canonical block in the chain,
-// if the input slot has a skip block, false is returned,
-// if the input slot has more than one block, an error is returned.
-func (bs *Server) isSlotCanonical(ctx context.Context, slot types.Slot) (bool, error) {
-	if slot == 0 {
-		return true, nil
-	}
-
-	hasBlockRoots, roots, err := bs.BeaconDB.BlockRootsBySlot(ctx, slot)
-	if err != nil {
-		return false, err
-	}
-	if !hasBlockRoots {
-		return false, nil
-	}
-
-	// Loop through all roots in slot, and
-	// check which one is canonical.
-	for _, rt := range roots {
-		canonical, err := bs.CanonicalFetcher.IsCanonical(ctx, rt)
-		if err != nil {
-			return false, err
-		}
-		if canonical {
-			return true, nil
-		}
-
-	}
-	return false, nil
-}
-
 // Determines whether a validator has already exited.
 func validatorHasExited(validator *ethpb.Validator, currentEpoch types.Epoch) bool {
 	farFutureEpoch := params.BeaconConfig().FarFutureEpoch

beacon-chain/rpc/prysm/v1alpha1/beacon/validators_test.go

@@ -12,9 +12,11 @@ import (
 	types "github.com/prysmaticlabs/eth2-types"
 	"github.com/prysmaticlabs/go-bitfield"
 	mock "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/altair"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	coreTime "github.com/prysmaticlabs/prysm/beacon-chain/core/time"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/transition"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
 	dbTest "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
@@ -115,6 +117,8 @@ func TestServer_ListValidatorBalances_NoResults(t *testing.T) {
 	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, gRoot))
 	require.NoError(t, beaconDB.SaveState(ctx, headState, gRoot))
 
+	bs.ReplayerBuilder = mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(headState))
+
 	wanted := &ethpb.ValidatorBalances{
 		Balances:      make([]*ethpb.ValidatorBalances_Balance, 0),
 		TotalSize:     int32(0),
@@ -172,6 +176,7 @@ func TestServer_ListValidatorBalances_DefaultResponse_NoArchive(t *testing.T) {
 		HeadFetcher: &mock.ChainService{
 			State: st,
 		},
+		ReplayerBuilder: mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(st)),
 	}
 	res, err := bs.ListValidatorBalances(
 		ctx,
@@ -200,6 +205,7 @@ func TestServer_ListValidatorBalances_PaginationOutOfRange(t *testing.T) {
 		HeadFetcher: &mock.ChainService{
 			State: headState,
 		},
+		ReplayerBuilder: mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(headState)),
 	}
 
 	wanted := fmt.Sprintf("page start %d >= list %d", 200, len(headState.Balances()))
@@ -248,6 +254,7 @@ func TestServer_ListValidatorBalances_Pagination_Default(t *testing.T) {
 		HeadFetcher: &mock.ChainService{
 			State: headState,
 		},
+		ReplayerBuilder: mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(headState)),
 	}
 
 	tests := []struct {
@@ -331,6 +338,7 @@ func TestServer_ListValidatorBalances_Pagination_CustomPageSizes(t *testing.T) {
 		HeadFetcher: &mock.ChainService{
 			State: headState,
 		},
+		ReplayerBuilder: mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(headState)),
 	}
 
 	tests := []struct {
@@ -398,6 +406,7 @@ func TestServer_ListValidatorBalances_OutOfRange(t *testing.T) {
 		HeadFetcher: &mock.ChainService{
 			State: headState,
 		},
+		ReplayerBuilder: mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(headState)),
 	}
 
 	req := &ethpb.ListValidatorBalancesRequest{Indices: []types.ValidatorIndex{types.ValidatorIndex(1)}, QueryFilter: &ethpb.ListValidatorBalancesRequest_Epoch{Epoch: 0}}
@@ -660,6 +669,7 @@ func TestServer_ListValidatorBalances_UnknownValidatorInResponse(t *testing.T) {
 		HeadFetcher: &mock.ChainService{
 			State: headState,
 		},
+		ReplayerBuilder: mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(headState)),
 	}
 
 	nonExistentPubKey := [32]byte{8}
@@ -1009,47 +1019,41 @@ func TestServer_ListValidators_DefaultPageSize(t *testing.T) {
 }
 
 func TestServer_ListValidators_FromOldEpoch(t *testing.T) {
-	beaconDB := dbTest.SetupDB(t)
+	params.OverrideBeaconConfig(params.MainnetConfig())
+	transition.SkipSlotCache.Disable()
+
 	ctx := context.Background()
+	slot := types.Slot(0)
+	epochs := 10
+	numVals := uint64(10)
 
-	numEpochs := types.Epoch(30)
-	validators := make([]*ethpb.Validator, numEpochs)
-	for i := types.Epoch(0); i < numEpochs; i++ {
-		validators[i] = &ethpb.Validator{
-			ActivationEpoch:       i,
-			PublicKey:             make([]byte, 48),
-			WithdrawalCredentials: make([]byte, 32),
-		}
-	}
-	want := make([]*ethpb.Validators_ValidatorContainer, len(validators))
-	for i := 0; i < len(validators); i++ {
-		want[i] = &ethpb.Validators_ValidatorContainer{
-			Index:     types.ValidatorIndex(i),
-			Validator: validators[i],
-		}
-	}
-
-	st, err := util.NewBeaconState()
-	require.NoError(t, err)
-	require.NoError(t, st.SetSlot(20*params.BeaconConfig().SlotsPerEpoch))
-	require.NoError(t, st.SetValidators(validators))
 	b := util.NewBeaconBlock()
-	require.NoError(t, beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
-	gRoot, err := b.Block.HashTreeRoot()
+	b.Block.Slot = slot
+	sb, err := wrapper.WrappedSignedBeaconBlock(b)
 	require.NoError(t, err)
-	require.NoError(t, beaconDB.SaveState(ctx, st, gRoot))
-	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, gRoot))
+
+	r, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	st, _ := util.DeterministicGenesisState(t, numVals)
+	require.NoError(t, st.SetSlot(slot))
+	require.Equal(t, int(numVals), len(st.Validators()))
+
+	beaconDB := dbTest.SetupDB(t)
+	require.NoError(t, beaconDB.SaveBlock(ctx, sb))
+	require.NoError(t, beaconDB.SaveState(ctx, st, r))
+	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, r))
+
 	secondsPerEpoch := params.BeaconConfig().SecondsPerSlot * uint64(params.BeaconConfig().SlotsPerEpoch)
 	bs := &Server{
 		HeadFetcher: &mock.ChainService{
 			State: st,
 		},
 		GenesisTimeFetcher: &mock.ChainService{
-			// We are in epoch 30
-			Genesis: time.Now().Add(time.Duration(-1*int64(30*secondsPerEpoch)) * time.Second),
+			Genesis: time.Now().Add(time.Duration(-1*int64(uint64(epochs)*secondsPerEpoch)) * time.Second),
 		},
-		StateGen: stategen.New(beaconDB),
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
 	req := &ethpb.ListValidatorsRequest{
 		QueryFilter: &ethpb.ListValidatorsRequest_Genesis{
@@ -1058,16 +1062,26 @@ func TestServer_ListValidators_FromOldEpoch(t *testing.T) {
 	}
 	res, err := bs.ListValidators(context.Background(), req)
 	require.NoError(t, err)
-	assert.Equal(t, 30, len(res.ValidatorList))
+	assert.Equal(t, epochs, len(res.ValidatorList))
 
+	vals := st.Validators()
+	want := make([]*ethpb.Validators_ValidatorContainer, 0)
+	for i, v := range vals {
+		want = append(want, &ethpb.Validators_ValidatorContainer{
+			Index:     types.ValidatorIndex(i),
+			Validator: v,
+		})
+	}
 	req = &ethpb.ListValidatorsRequest{
 		QueryFilter: &ethpb.ListValidatorsRequest_Epoch{
-			Epoch: 20,
+			Epoch: 10,
 		},
 	}
 	res, err = bs.ListValidators(context.Background(), req)
 	require.NoError(t, err)
-	assert.DeepSSZEqual(t, want, res.ValidatorList, "Incorrect number of validators")
+
+	require.Equal(t, len(want), len(res.ValidatorList), "incorrect number of validators")
+	assert.DeepSSZEqual(t, want, res.ValidatorList, "mismatch in validator values")
 }
 
 func TestServer_ListValidators_ProcessHeadStateSlots(t *testing.T) {
@@ -1270,8 +1284,8 @@ func TestServer_GetValidatorActiveSetChanges(t *testing.T) {
 			FinalizedCheckPoint: &ethpb.Checkpoint{Epoch: 0, Root: make([]byte, fieldparams.RootLength)},
 		},
 		GenesisTimeFetcher: &mock.ChainService{},
-		StateGen:           stategen.New(beaconDB),
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 	res, err := bs.GetValidatorActiveSetChanges(ctx, &ethpb.GetValidatorActiveSetChangesRequest{
 		QueryFilter: &ethpb.GetValidatorActiveSetChangesRequest_Genesis{Genesis: true},
 	})
@@ -1489,43 +1503,6 @@ func TestServer_GetValidatorParticipation_CannotRequestFutureEpoch(t *testing.T)
 	assert.ErrorContains(t, wanted, err)
 }
 
-func TestServer_GetValidatorParticipation_UnknownState(t *testing.T) {
-	beaconDB := dbTest.SetupDB(t)
-
-	ctx := context.Background()
-	headState, err := util.NewBeaconState()
-	require.NoError(t, err)
-	require.NoError(t, headState.SetSlot(0))
-	epoch := types.Epoch(50)
-	slots := params.BeaconConfig().SlotsPerEpoch.Mul(uint64(epoch))
-	mockStateGen := &mockstategen.MockStateManager{
-		StatesBySlot: map[types.Slot]state.BeaconState{
-			0: (*v1.BeaconState)(nil),
-		},
-	}
-	bs := &Server{
-		BeaconDB: beaconDB,
-		HeadFetcher: &mock.ChainService{
-			State: headState,
-		},
-		GenesisTimeFetcher: &mock.ChainService{
-			Genesis: time.Now().Add(time.Duration(-1*int64(slots)) * time.Second),
-		},
-		StateGen: mockStateGen,
-	}
-
-	wanted := "Could not set up pre compute instance: failed to initialize precompute: nil inner state"
-	_, err = bs.GetValidatorParticipation(
-		ctx,
-		&ethpb.GetValidatorParticipationRequest{
-			QueryFilter: &ethpb.GetValidatorParticipationRequest_Epoch{
-				Epoch: 1,
-			},
-		},
-	)
-	assert.ErrorContains(t, wanted, err)
-}
-
 func TestServer_GetValidatorParticipation_CurrentAndPrevEpoch(t *testing.T) {
 	helpers.ClearCache()
 	beaconDB := dbTest.SetupDB(t)
@@ -1552,7 +1529,7 @@ func TestServer_GetValidatorParticipation_CurrentAndPrevEpoch(t *testing.T) {
 	}}
 	headState, err := util.NewBeaconState()
 	require.NoError(t, err)
-	require.NoError(t, headState.SetSlot(2*params.BeaconConfig().SlotsPerEpoch-1))
+	require.NoError(t, headState.SetSlot(16))
 	require.NoError(t, headState.SetValidators(validators))
 	require.NoError(t, headState.SetBalances(balances))
 	require.NoError(t, headState.AppendCurrentEpochAttestations(atts[0]))
@@ -1567,6 +1544,7 @@ func TestServer_GetValidatorParticipation_CurrentAndPrevEpoch(t *testing.T) {
 	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, bRoot))
 	require.NoError(t, err)
 	require.NoError(t, beaconDB.SaveState(ctx, headState, bRoot))
+	require.NoError(t, beaconDB.SaveState(ctx, headState, params.BeaconConfig().ZeroHash))
 
 	m := &mock.ChainService{State: headState}
 	offset := int64(params.BeaconConfig().SlotsPerEpoch.Mul(params.BeaconConfig().SecondsPerSlot))
@@ -1584,6 +1562,7 @@ func TestServer_GetValidatorParticipation_CurrentAndPrevEpoch(t *testing.T) {
 		},
 		FinalizationFetcher: &mock.ChainService{FinalizedCheckPoint: &ethpb.Checkpoint{Epoch: 100}},
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
 	res, err := bs.GetValidatorParticipation(ctx, &ethpb.GetValidatorParticipationRequest{QueryFilter: &ethpb.GetValidatorParticipationRequest_Epoch{Epoch: 1}})
 	require.NoError(t, err)
@@ -1632,7 +1611,7 @@ func TestServer_GetValidatorParticipation_OrphanedUntilGenesis(t *testing.T) {
 	}}
 	headState, err := util.NewBeaconState()
 	require.NoError(t, err)
-	require.NoError(t, headState.SetSlot(2*params.BeaconConfig().SlotsPerEpoch-1))
+	require.NoError(t, headState.SetSlot(0))
 	require.NoError(t, headState.SetValidators(validators))
 	require.NoError(t, headState.SetBalances(balances))
 	require.NoError(t, headState.AppendCurrentEpochAttestations(atts[0]))
@@ -1641,11 +1620,10 @@ func TestServer_GetValidatorParticipation_OrphanedUntilGenesis(t *testing.T) {
 	b := util.NewBeaconBlock()
 	require.NoError(t, beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
 	bRoot, err := b.Block.HashTreeRoot()
-	require.NoError(t, beaconDB.SaveStateSummary(ctx, &ethpb.StateSummary{Root: bRoot[:]}))
-	require.NoError(t, beaconDB.SaveStateSummary(ctx, &ethpb.StateSummary{Root: params.BeaconConfig().ZeroHash[:]}))
 	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, bRoot))
 	require.NoError(t, err)
 	require.NoError(t, beaconDB.SaveState(ctx, headState, bRoot))
+	require.NoError(t, beaconDB.SaveState(ctx, headState, params.BeaconConfig().ZeroHash))
 
 	m := &mock.ChainService{State: headState}
 	offset := int64(params.BeaconConfig().SlotsPerEpoch.Mul(params.BeaconConfig().SecondsPerSlot))
@@ -1663,6 +1641,7 @@ func TestServer_GetValidatorParticipation_OrphanedUntilGenesis(t *testing.T) {
 		},
 		FinalizationFetcher: &mock.ChainService{FinalizedCheckPoint: &ethpb.Checkpoint{Epoch: 100}},
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
 	res, err := bs.GetValidatorParticipation(ctx, &ethpb.GetValidatorParticipationRequest{QueryFilter: &ethpb.GetValidatorParticipationRequest_Epoch{Epoch: 1}})
 	require.NoError(t, err)
@@ -1688,32 +1667,36 @@ func TestServer_GetValidatorParticipation_CurrentAndPrevEpochAltair(t *testing.T
 	beaconDB := dbTest.SetupDB(t)
 	params.SetupTestConfigCleanup(t)
 	params.OverrideBeaconConfig(params.MainnetConfig())
+	transition.SkipSlotCache.Disable()
 
 	ctx := context.Background()
 	validatorCount := uint64(32)
 
+	genState, _ := util.DeterministicGenesisStateAltair(t, validatorCount)
+	c, err := altair.NextSyncCommittee(ctx, genState)
+	require.NoError(t, err)
+	require.NoError(t, genState.SetCurrentSyncCommittee(c))
+
 	bits := make([]byte, validatorCount)
 	for i := range bits {
 		bits[i] = 0xff
 	}
-	headState, _ := util.DeterministicGenesisStateAltair(t, validatorCount)
-	require.NoError(t, headState.SetSlot(2*params.BeaconConfig().SlotsPerEpoch-1))
-	require.NoError(t, headState.SetCurrentParticipationBits(bits))
-	require.NoError(t, headState.SetPreviousParticipationBits(bits))
+	require.NoError(t, genState.SetCurrentParticipationBits(bits))
+	require.NoError(t, genState.SetPreviousParticipationBits(bits))
 
-	b := util.NewBeaconBlockAltair()
-	b.Block.Slot = 16
-	ab, err := wrapper.WrappedAltairSignedBeaconBlock(b)
+	gsr, err := genState.HashTreeRoot(ctx)
 	require.NoError(t, err)
-	require.NoError(t, beaconDB.SaveBlock(ctx, ab))
-	bRoot, err := b.Block.HashTreeRoot()
-	require.NoError(t, beaconDB.SaveStateSummary(ctx, &ethpb.StateSummary{Root: bRoot[:]}))
-	require.NoError(t, beaconDB.SaveStateSummary(ctx, &ethpb.StateSummary{Root: params.BeaconConfig().ZeroHash[:]}))
-	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, bRoot))
+	gb, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlockAltair())
+	require.NoError(t, wrapper.SetBlockStateRoot(gb, gsr))
+	require.NoError(t, err)
+	gRoot, err := gb.Block().HashTreeRoot()
 	require.NoError(t, err)
-	require.NoError(t, beaconDB.SaveState(ctx, headState, bRoot))
 
-	m := &mock.ChainService{State: headState}
+	require.NoError(t, beaconDB.SaveState(ctx, genState, gRoot))
+	require.NoError(t, beaconDB.SaveBlock(ctx, gb))
+	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, gRoot))
+
+	m := &mock.ChainService{State: genState}
 	offset := int64(params.BeaconConfig().SlotsPerEpoch.Mul(params.BeaconConfig().SecondsPerSlot))
 	bs := &Server{
 		BeaconDB:    beaconDB,
@@ -1722,15 +1705,11 @@ func TestServer_GetValidatorParticipation_CurrentAndPrevEpochAltair(t *testing.T
 		GenesisTimeFetcher: &mock.ChainService{
 			Genesis: prysmTime.Now().Add(time.Duration(-1*offset) * time.Second),
 		},
-		CanonicalFetcher: &mock.ChainService{
-			CanonicalRoots: map[[32]byte]bool{
-				bRoot: true,
-			},
-		},
 		FinalizationFetcher: &mock.ChainService{FinalizedCheckPoint: &ethpb.Checkpoint{Epoch: 100}},
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
-	res, err := bs.GetValidatorParticipation(ctx, &ethpb.GetValidatorParticipationRequest{QueryFilter: &ethpb.GetValidatorParticipationRequest_Epoch{Epoch: 1}})
+	res, err := bs.GetValidatorParticipation(ctx, &ethpb.GetValidatorParticipationRequest{QueryFilter: &ethpb.GetValidatorParticipationRequest_Epoch{Epoch: 0}})
 	require.NoError(t, err)
 
 	wanted := &ethpb.ValidatorParticipation{
@@ -1747,6 +1726,24 @@ func TestServer_GetValidatorParticipation_CurrentAndPrevEpochAltair(t *testing.T
 	}
 	assert.DeepEqual(t, true, res.Finalized, "Incorrect validator participation respond")
 	assert.DeepEqual(t, wanted, res.Participation, "Incorrect validator participation respond")
+
+	res, err = bs.GetValidatorParticipation(ctx, &ethpb.GetValidatorParticipationRequest{QueryFilter: &ethpb.GetValidatorParticipationRequest_Epoch{Epoch: 1}})
+	require.NoError(t, err)
+
+	wanted = &ethpb.ValidatorParticipation{
+		GlobalParticipationRate:          1,
+		VotedEther:                       validatorCount * params.BeaconConfig().MaxEffectiveBalance,
+		EligibleEther:                    validatorCount * params.BeaconConfig().MaxEffectiveBalance,
+		CurrentEpochActiveGwei:           validatorCount * params.BeaconConfig().MaxEffectiveBalance,
+		CurrentEpochAttestingGwei:        params.BeaconConfig().EffectiveBalanceIncrement, // Empty because after one epoch, current participation rotates to previous
+		CurrentEpochTargetAttestingGwei:  params.BeaconConfig().EffectiveBalanceIncrement,
+		PreviousEpochActiveGwei:          validatorCount * params.BeaconConfig().MaxEffectiveBalance,
+		PreviousEpochAttestingGwei:       validatorCount * params.BeaconConfig().MaxEffectiveBalance,
+		PreviousEpochTargetAttestingGwei: validatorCount * params.BeaconConfig().MaxEffectiveBalance,
+		PreviousEpochHeadAttestingGwei:   validatorCount * params.BeaconConfig().MaxEffectiveBalance,
+	}
+	assert.DeepEqual(t, true, res.Finalized, "Incorrect validator participation respond")
+	assert.DeepEqual(t, wanted, res.Participation, "Incorrect validator participation respond")
 }
 
 func TestGetValidatorPerformance_Syncing(t *testing.T) {
@@ -2070,6 +2067,7 @@ func BenchmarkListValidatorBalances(b *testing.B) {
 			State: headState,
 		},
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
 	req := &ethpb.ListValidatorBalancesRequest{PageSize: 100}
 	b.StartTimer()
@@ -2114,15 +2112,16 @@ func TestServer_GetIndividualVotes_ValidatorsDontExist(t *testing.T) {
 	beaconDB := dbTest.SetupDB(t)
 	ctx := context.Background()
 
+	var slot types.Slot = 0
 	validators := uint64(64)
 	stateWithValidators, _ := util.DeterministicGenesisState(t, validators)
 	beaconState, err := util.NewBeaconState()
 	require.NoError(t, err)
 	require.NoError(t, beaconState.SetValidators(stateWithValidators.Validators()))
-	require.NoError(t, beaconState.SetSlot(params.BeaconConfig().SlotsPerEpoch))
+	require.NoError(t, beaconState.SetSlot(slot))
 
 	b := util.NewBeaconBlock()
-	b.Block.Slot = params.BeaconConfig().SlotsPerEpoch
+	b.Block.Slot = slot
 	require.NoError(t, beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
 	gRoot, err := b.Block.HashTreeRoot()
 	require.NoError(t, err)
@@ -2134,6 +2133,7 @@ func TestServer_GetIndividualVotes_ValidatorsDontExist(t *testing.T) {
 		StateGen:           gen,
 		GenesisTimeFetcher: &mock.ChainService{},
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
 	// Test non exist public key.
 	res, err := bs.GetIndividualVotes(ctx, &ethpb.IndividualVotesRequest{
@@ -2230,6 +2230,7 @@ func TestServer_GetIndividualVotes_Working(t *testing.T) {
 		StateGen:           gen,
 		GenesisTimeFetcher: &mock.ChainService{},
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
 	res, err := bs.GetIndividualVotes(ctx, &ethpb.IndividualVotesRequest{
 		Indices: []types.ValidatorIndex{0, 1},
@@ -2268,9 +2269,10 @@ func TestServer_GetIndividualVotes_WorkingAltair(t *testing.T) {
 	beaconDB := dbTest.SetupDB(t)
 	ctx := context.Background()
 
+	var slot types.Slot = 0
 	validators := uint64(32)
 	beaconState, _ := util.DeterministicGenesisStateAltair(t, validators)
-	require.NoError(t, beaconState.SetSlot(params.BeaconConfig().SlotsPerEpoch))
+	require.NoError(t, beaconState.SetSlot(slot))
 
 	pb, err := beaconState.CurrentEpochParticipation()
 	require.NoError(t, err)
@@ -2281,7 +2283,7 @@ func TestServer_GetIndividualVotes_WorkingAltair(t *testing.T) {
 	require.NoError(t, beaconState.SetPreviousParticipationBits(pb))
 
 	b := util.NewBeaconBlock()
-	b.Block.Slot = params.BeaconConfig().SlotsPerEpoch
+	b.Block.Slot = slot
 	require.NoError(t, beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
 	gRoot, err := b.Block.HashTreeRoot()
 	require.NoError(t, err)
@@ -2293,6 +2295,7 @@ func TestServer_GetIndividualVotes_WorkingAltair(t *testing.T) {
 		StateGen:           gen,
 		GenesisTimeFetcher: &mock.ChainService{},
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
 	res, err := bs.GetIndividualVotes(ctx, &ethpb.IndividualVotesRequest{
 		Indices: []types.ValidatorIndex{0, 1},
@@ -2378,6 +2381,7 @@ func TestServer_GetIndividualVotes_AltairEndOfEpoch(t *testing.T) {
 		StateGen:           gen,
 		GenesisTimeFetcher: &mock.ChainService{},
 	}
+	addDefaultReplayerBuilder(bs, beaconDB)
 
 	res, err := bs.GetIndividualVotes(ctx, &ethpb.IndividualVotesRequest{
 		Indices: []types.ValidatorIndex{0, 1},
@@ -2499,88 +2503,3 @@ func Test_validatorStatus(t *testing.T) {
 		})
 	}
 }
-
-func TestServer_isSlotCanonical(t *testing.T) {
-	beaconDB := dbTest.SetupDB(t)
-	ctx := context.Background()
-	var roots [][32]byte
-	cRoots := map[[32]byte]bool{}
-	for i := 1; i < 100; i++ {
-		b := util.NewBeaconBlock()
-		b.Block.Slot = types.Slot(i)
-		require.NoError(t, beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
-		br, err := b.Block.HashTreeRoot()
-		require.NoError(t, err)
-		if i%2 == 0 {
-			cRoots[br] = true
-		}
-		roots = append(roots, br)
-	}
-
-	bs := &Server{
-		BeaconDB: beaconDB,
-		CanonicalFetcher: &mock.ChainService{
-			CanonicalRoots: cRoots,
-		},
-	}
-
-	for i := range roots {
-		slot := types.Slot(i + 1)
-		c, err := bs.isSlotCanonical(ctx, slot)
-		require.NoError(t, err)
-		if slot%2 == 0 {
-			require.Equal(t, true, c)
-		} else {
-			require.Equal(t, false, c)
-		}
-	}
-}
-
-func TestServer_isSlotCanonical_MultipleBlocks(t *testing.T) {
-	beaconDB := dbTest.SetupDB(t)
-	ctx := context.Background()
-	var roots [][32]byte
-	cRoots := map[[32]byte]bool{}
-	for i := 1; i < 100; i++ {
-		b := util.NewBeaconBlock()
-		b.Block.Slot = types.Slot(i)
-		require.NoError(t, beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
-		br, err := b.Block.HashTreeRoot()
-		require.NoError(t, err)
-		if i%2 == 0 {
-			cRoots[br] = true
-			// Save a block in the same slot
-			b = util.NewBeaconBlock()
-			b.Block.Slot = types.Slot(i)
-			b.Block.ProposerIndex = 100
-			require.NoError(t, beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
-		}
-		roots = append(roots, br)
-	}
-
-	bs := &Server{
-		BeaconDB: beaconDB,
-		CanonicalFetcher: &mock.ChainService{
-			CanonicalRoots: cRoots,
-		},
-	}
-
-	for i := range roots {
-		slot := types.Slot(i + 1)
-		c, err := bs.isSlotCanonical(ctx, slot)
-		require.NoError(t, err)
-		if slot%2 == 0 {
-			require.Equal(t, true, c)
-		} else {
-			require.Equal(t, false, c)
-		}
-	}
-}
-
-func TestServer_isSlotCanonicalForSlot0(t *testing.T) {
-	ctx := context.Background()
-	bs := &Server{}
-	c, err := bs.isSlotCanonical(ctx, 0)
-	require.NoError(t, err)
-	require.Equal(t, true, c)
-}

beacon-chain/rpc/prysm/v1alpha1/debug/BUILD.bazel

@@ -50,7 +50,9 @@ go_test(
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/forkchoice/protoarray:go_default_library",
         "//beacon-chain/p2p/testing:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/stategen:go_default_library",
+        "//beacon-chain/state/stategen/mock:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",

beacon-chain/rpc/prysm/v1alpha1/debug/server.go

@@ -30,6 +30,7 @@ type Server struct {
 	HeadFetcher        blockchain.HeadFetcher
 	PeerManager        p2p.PeerManager
 	PeersFetcher       p2p.PeersProvider
+	ReplayerBuilder    stategen.ReplayerBuilder
 }
 
 // SetLoggingLevel of a beacon node according to a request type,

beacon-chain/rpc/prysm/v1alpha1/debug/state.go

@@ -2,6 +2,7 @@ package debug
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 	pbrpc "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
@@ -28,10 +29,11 @@ func (ds *Server) GetBeaconState(
 			)
 		}
 
-		st, err := ds.StateGen.StateBySlot(ctx, q.Slot)
+		st, err := ds.ReplayerBuilder.ForSlot(q.Slot).ReplayBlocks(ctx)
 		if err != nil {
-			return nil, status.Errorf(codes.Internal, "Could not compute state by slot: %v", err)
+			return nil, status.Error(codes.Internal, fmt.Sprintf("error replaying blocks for state at slot %d: %v", q.Slot, err))
 		}
+
 		encoded, err := st.MarshalSSZ()
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "Could not ssz encode beacon state: %v", err)

beacon-chain/rpc/prysm/v1alpha1/debug/state_test.go

@@ -2,12 +2,15 @@ package debug
 
 import (
 	"context"
+	"math"
 	"testing"
 
 	types "github.com/prysmaticlabs/eth2-types"
 	mock "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
 	dbTest "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	mockstategen "github.com/prysmaticlabs/prysm/beacon-chain/state/stategen/mock"
 	pbrpc "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
 	"github.com/prysmaticlabs/prysm/testing/assert"
@@ -15,6 +18,16 @@ import (
 	"github.com/prysmaticlabs/prysm/testing/util"
 )
 
+func addReplayerBuilder(s *Server, h stategen.HistoryAccessor, is bool, canonErr error, currSlot types.Slot) {
+	cc := &mockstategen.MockCanonicalChecker{Is: is, Err: canonErr}
+	cs := &mockstategen.MockCurrentSlotter{Slot: currSlot}
+	s.ReplayerBuilder = stategen.NewCanonicalBuilder(h, cc, cs)
+}
+
+func addDefaultReplayerBuilder(s *Server, h stategen.HistoryAccessor) {
+	addReplayerBuilder(s, h, true, nil, math.MaxUint64-1)
+}
+
 func TestServer_GetBeaconState(t *testing.T) {
 	db := dbTest.SetupDB(t)
 	ctx := context.Background()
@@ -34,6 +47,7 @@ func TestServer_GetBeaconState(t *testing.T) {
 		StateGen:           gen,
 		GenesisTimeFetcher: &mock.ChainService{},
 	}
+	addDefaultReplayerBuilder(bs, db)
 	_, err = bs.GetBeaconState(ctx, &pbrpc.BeaconStateRequest{})
 	assert.ErrorContains(t, "Need to specify either a block root or slot to request state", err)
 	req := &pbrpc.BeaconStateRequest{
@@ -46,16 +60,44 @@ func TestServer_GetBeaconState(t *testing.T) {
 	wanted, err := st.MarshalSSZ()
 	require.NoError(t, err)
 	assert.DeepEqual(t, wanted, res.Encoded)
+
+	req = &pbrpc.BeaconStateRequest{
+		QueryFilter: &pbrpc.BeaconStateRequest_Slot{
+			Slot: st.Slot(),
+		},
+	}
+	wanted, err = st.MarshalSSZ()
+	require.NoError(t, err)
+	res, err = bs.GetBeaconState(ctx, req)
+	require.NoError(t, err)
+	resState := &pbrpc.BeaconState{}
+	err = resState.UnmarshalSSZ(res.Encoded)
+	require.NoError(t, err)
+	assert.Equal(t, resState.Slot, st.Slot())
+	assert.DeepEqual(t, wanted, res.Encoded)
+
+	// request a slot after the state
+	// note that if the current slot were <= slot+1, this would fail
+	// but the mock stategen.CurrentSlotter gives a current slot far in the future
+	// so this acts like requesting a state at a skipped slot
 	req = &pbrpc.BeaconStateRequest{
 		QueryFilter: &pbrpc.BeaconStateRequest_Slot{
 			Slot: slot + 1,
 		},
 	}
-	require.NoError(t, st.SetSlot(slot+1))
-	wanted, err = st.MarshalSSZ()
+	state := state.BeaconState(st)
+	// since we are requesting a state at a skipped slot, use the same method as stategen
+	// to advance to the pre-state for the subsequent slot
+	state, err = stategen.ReplayProcessSlots(ctx, state, slot+1)
+	require.NoError(t, err)
+	wanted, err = state.MarshalSSZ()
 	require.NoError(t, err)
 	res, err = bs.GetBeaconState(ctx, req)
 	require.NoError(t, err)
+	resState = &pbrpc.BeaconState{}
+	err = resState.UnmarshalSSZ(res.Encoded)
+	require.NoError(t, err)
+	assert.Equal(t, resState.Slot, state.Slot())
 	assert.DeepEqual(t, wanted, res.Encoded)
 }
 

beacon-chain/rpc/prysm/v1alpha1/validator/proposer_bellatrix.go

@@ -39,7 +39,7 @@ func (vs *Server) getBellatrixBeaconBlock(ctx context.Context, req *ethpb.BlockR
 				StateRoot:     make([]byte, fieldparams.RootLength),
 				ReceiptsRoot:  make([]byte, fieldparams.RootLength),
 				LogsBloom:     make([]byte, fieldparams.LogsBloomLength),
-				Random:        make([]byte, fieldparams.RootLength),
+				PrevRandao:    make([]byte, fieldparams.RootLength),
 				BaseFeePerGas: make([]byte, fieldparams.RootLength),
 				BlockHash:     make([]byte, fieldparams.RootLength),
 			}, // TODO(9853) Insert real execution payload.

beacon-chain/rpc/prysm/v1alpha1/validator/proposer_test.go

@@ -209,116 +209,79 @@ func TestProposer_GetBlock_AddsUnaggregatedAtts(t *testing.T) {
 	assert.Equal(t, false, hasUnaggregatedAtt, "Expected block to not have unaggregated attestation")
 }
 
-func TestProposer_ProposeBlock_Phase0_OK(t *testing.T) {
-	db := dbutil.SetupDB(t)
-	ctx := context.Background()
-	params.SetupTestConfigCleanup(t)
-	params.OverrideBeaconConfig(params.MainnetConfig())
-
-	genesis := util.NewBeaconBlock()
-	require.NoError(t, db.SaveBlock(context.Background(), wrapper.WrappedPhase0SignedBeaconBlock(genesis)), "Could not save genesis block")
-
-	numDeposits := uint64(64)
-	beaconState, _ := util.DeterministicGenesisState(t, numDeposits)
-	bsRoot, err := beaconState.HashTreeRoot(ctx)
-	require.NoError(t, err)
-	genesisRoot, err := genesis.Block.HashTreeRoot()
-	require.NoError(t, err)
-	require.NoError(t, db.SaveState(ctx, beaconState, genesisRoot), "Could not save genesis state")
-
-	c := &mock.ChainService{Root: bsRoot[:], State: beaconState}
-	proposerServer := &Server{
-		ChainStartFetcher: &mockPOW.POWChain{},
-		Eth1InfoFetcher:   &mockPOW.POWChain{},
-		Eth1BlockFetcher:  &mockPOW.POWChain{},
-		BlockReceiver:     c,
-		HeadFetcher:       c,
-		BlockNotifier:     c.BlockNotifier(),
-		P2P:               mockp2p.NewTestP2P(t),
+func TestProposer_ProposeBlock_OK(t *testing.T) {
+	tests := []struct {
+		name  string
+		block func([32]byte) *ethpb.GenericSignedBeaconBlock
+	}{
+		{
+			name: "phase0",
+			block: func(parent [32]byte) *ethpb.GenericSignedBeaconBlock {
+				blockToPropose := util.NewBeaconBlock()
+				blockToPropose.Block.Slot = 5
+				blockToPropose.Block.ParentRoot = parent[:]
+				blk := &ethpb.GenericSignedBeaconBlock_Phase0{Phase0: blockToPropose}
+				return &ethpb.GenericSignedBeaconBlock{Block: blk}
+			},
+		},
+		{
+			name: "altair",
+			block: func(parent [32]byte) *ethpb.GenericSignedBeaconBlock {
+				blockToPropose := util.NewBeaconBlockAltair()
+				blockToPropose.Block.Slot = 5
+				blockToPropose.Block.ParentRoot = parent[:]
+				blk := &ethpb.GenericSignedBeaconBlock_Altair{Altair: blockToPropose}
+				return &ethpb.GenericSignedBeaconBlock{Block: blk}
+			},
+		},
+		{
+			name: "bellatrix",
+			block: func(parent [32]byte) *ethpb.GenericSignedBeaconBlock {
+				blockToPropose := util.NewBeaconBlockBellatrix()
+				blockToPropose.Block.Slot = 5
+				blockToPropose.Block.ParentRoot = parent[:]
+				blk := &ethpb.GenericSignedBeaconBlock_Bellatrix{Bellatrix: blockToPropose}
+				return &ethpb.GenericSignedBeaconBlock{Block: blk}
+			},
+		},
 	}
-	req := util.NewBeaconBlock()
-	req.Block.Slot = 5
-	req.Block.ParentRoot = bsRoot[:]
-	require.NoError(t, db.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(req)))
-	blk := &ethpb.GenericSignedBeaconBlock_Phase0{Phase0: req}
-	_, err = proposerServer.ProposeBeaconBlock(context.Background(), &ethpb.GenericSignedBeaconBlock{Block: blk})
-	assert.NoError(t, err, "Could not propose block correctly")
-}
 
-func TestProposer_ProposeBlock_Altair_OK(t *testing.T) {
-	db := dbutil.SetupDB(t)
-	ctx := context.Background()
-	params.SetupTestConfigCleanup(t)
-	params.OverrideBeaconConfig(params.MainnetConfig())
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			db := dbutil.SetupDB(t)
+			ctx := context.Background()
+			params.SetupTestConfigCleanup(t)
+			params.OverrideBeaconConfig(params.MainnetConfig())
 
-	genesis := util.NewBeaconBlock()
-	require.NoError(t, db.SaveBlock(context.Background(), wrapper.WrappedPhase0SignedBeaconBlock(genesis)), "Could not save genesis block")
+			genesis := util.NewBeaconBlock()
+			require.NoError(t, db.SaveBlock(context.Background(), wrapper.WrappedPhase0SignedBeaconBlock(genesis)), "Could not save genesis block")
 
-	numDeposits := uint64(64)
-	beaconState, _ := util.DeterministicGenesisStateAltair(t, numDeposits)
-	bsRoot, err := beaconState.HashTreeRoot(ctx)
-	require.NoError(t, err)
-	genesisRoot, err := genesis.Block.HashTreeRoot()
-	require.NoError(t, err)
-	require.NoError(t, db.SaveState(ctx, beaconState, genesisRoot), "Could not save genesis state")
+			numDeposits := uint64(64)
+			beaconState, _ := util.DeterministicGenesisState(t, numDeposits)
+			bsRoot, err := beaconState.HashTreeRoot(ctx)
+			require.NoError(t, err)
+			genesisRoot, err := genesis.Block.HashTreeRoot()
+			require.NoError(t, err)
+			require.NoError(t, db.SaveState(ctx, beaconState, genesisRoot), "Could not save genesis state")
 
-	c := &mock.ChainService{Root: bsRoot[:], State: beaconState}
-	proposerServer := &Server{
-		ChainStartFetcher: &mockPOW.POWChain{},
-		Eth1InfoFetcher:   &mockPOW.POWChain{},
-		Eth1BlockFetcher:  &mockPOW.POWChain{},
-		BlockReceiver:     c,
-		HeadFetcher:       c,
-		BlockNotifier:     c.BlockNotifier(),
-		P2P:               mockp2p.NewTestP2P(t),
+			c := &mock.ChainService{Root: bsRoot[:], State: beaconState}
+			proposerServer := &Server{
+				ChainStartFetcher: &mockPOW.POWChain{},
+				Eth1InfoFetcher:   &mockPOW.POWChain{},
+				Eth1BlockFetcher:  &mockPOW.POWChain{},
+				BlockReceiver:     c,
+				HeadFetcher:       c,
+				BlockNotifier:     c.BlockNotifier(),
+				P2P:               mockp2p.NewTestP2P(t),
+			}
+			blockToPropose := tt.block(bsRoot)
+			res, err := proposerServer.ProposeBeaconBlock(context.Background(), blockToPropose)
+			assert.NoError(t, err, "Could not propose block correctly")
+			if res == nil || len(res.BlockRoot) == 0 {
+				t.Error("No block root was returned")
+			}
+		})
 	}
-	blockToPropose := util.NewBeaconBlockAltair()
-	blockToPropose.Block.Slot = 5
-	blockToPropose.Block.ParentRoot = bsRoot[:]
-	blk := &ethpb.GenericSignedBeaconBlock_Altair{Altair: blockToPropose}
-	wrapped, err := wrapper.WrappedAltairSignedBeaconBlock(blockToPropose)
-	require.NoError(t, err)
-	require.NoError(t, db.SaveBlock(ctx, wrapped))
-	_, err = proposerServer.ProposeBeaconBlock(context.Background(), &ethpb.GenericSignedBeaconBlock{Block: blk})
-	assert.NoError(t, err, "Could not propose block correctly")
-}
-
-func TestProposer_ProposeBlock_Bellatrix_OK(t *testing.T) {
-	db := dbutil.SetupDB(t)
-	ctx := context.Background()
-	params.SetupTestConfigCleanup(t)
-	params.OverrideBeaconConfig(params.MainnetConfig())
-
-	genesis := util.NewBeaconBlock()
-	require.NoError(t, db.SaveBlock(context.Background(), wrapper.WrappedPhase0SignedBeaconBlock(genesis)), "Could not save genesis block")
-
-	numDeposits := uint64(64)
-	beaconState, _ := util.DeterministicGenesisStateBellatrix(t, numDeposits)
-	bsRoot, err := beaconState.HashTreeRoot(ctx)
-	require.NoError(t, err)
-	genesisRoot, err := genesis.Block.HashTreeRoot()
-	require.NoError(t, err)
-	require.NoError(t, db.SaveState(ctx, beaconState, genesisRoot), "Could not save genesis state")
-
-	c := &mock.ChainService{Root: bsRoot[:], State: beaconState}
-	proposerServer := &Server{
-		ChainStartFetcher: &mockPOW.POWChain{},
-		Eth1InfoFetcher:   &mockPOW.POWChain{},
-		Eth1BlockFetcher:  &mockPOW.POWChain{},
-		BlockReceiver:     c,
-		HeadFetcher:       c,
-		BlockNotifier:     c.BlockNotifier(),
-		P2P:               mockp2p.NewTestP2P(t),
-	}
-	blockToPropose := util.NewBeaconBlockBellatrix()
-	blockToPropose.Block.Slot = 5
-	blockToPropose.Block.ParentRoot = bsRoot[:]
-	blk := &ethpb.GenericSignedBeaconBlock_Bellatrix{Bellatrix: blockToPropose}
-	wrapped, err := wrapper.WrappedSignedBeaconBlock(blockToPropose)
-	require.NoError(t, err)
-	require.NoError(t, db.SaveBlock(ctx, wrapped))
-	_, err = proposerServer.ProposeBeaconBlock(context.Background(), &ethpb.GenericSignedBeaconBlock{Block: blk})
-	assert.NoError(t, err, "Could not propose block correctly")
 }
 
 func TestProposer_ComputeStateRoot_OK(t *testing.T) {
@@ -2305,7 +2268,7 @@ func TestProposer_GetBeaconBlock_BellatrixEpoch(t *testing.T) {
 					StateRoot:     make([]byte, fieldparams.RootLength),
 					ReceiptsRoot:  make([]byte, fieldparams.RootLength),
 					LogsBloom:     make([]byte, fieldparams.LogsBloomLength),
-					Random:        make([]byte, fieldparams.RootLength),
+					PrevRandao:    make([]byte, fieldparams.RootLength),
 					BaseFeePerGas: make([]byte, fieldparams.RootLength),
 					BlockHash:     make([]byte, fieldparams.RootLength),
 				},

beacon-chain/rpc/prysm/v1alpha1/validator/server.go

@@ -62,6 +62,7 @@ type Server struct {
 	PendingDepositsFetcher depositcache.PendingDepositsFetcher
 	OperationNotifier      opfeed.Notifier
 	StateGen               stategen.StateManager
+	ReplayerBuilder        stategen.ReplayerBuilder
 }
 
 // WaitForActivation checks if a validator public key exists in the active validator registry of the current

beacon-chain/rpc/prysm/v1alpha1/validator/status.go

@@ -8,7 +8,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/signing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/time"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/transition"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
@@ -338,11 +337,8 @@ func (vs *Server) retrieveAfterEpochTransition(ctx context.Context, epoch types.
 	if err != nil {
 		return nil, err
 	}
-	retState, err := vs.StateGen.StateBySlot(ctx, endSlot)
-	if err != nil {
-		return nil, err
-	}
-	return transition.ProcessSlots(ctx, retState, retState.Slot()+1)
+	// replay to first slot of following epoch
+	return vs.ReplayerBuilder.ForSlot(endSlot).ReplayToSlot(ctx, endSlot+1)
 }
 
 func checkValidatorsAreRecent(headEpoch types.Epoch, req *ethpb.DoppelGangerRequest) (bool, *ethpb.DoppelGangerResponse) {

beacon-chain/rpc/prysm/v1alpha1/validator/status_test.go

@@ -937,8 +937,7 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 			name:    "normal doppelganger request",
 			wantErr: false,
 			svSetup: func(t *testing.T) (*Server, *ethpb.DoppelGangerRequest, *ethpb.DoppelGangerResponse) {
-				mockGen := mockstategen.NewMockService()
-				hs, ps, os, keys := createStateSetup(t, 4, mockGen)
+				hs, ps, os, keys, builder := createStateSetup(t, 4)
 				// Previous Epoch State
 				for i := 0; i < 3; i++ {
 					bal, err := ps.BalanceAtIndex(types.ValidatorIndex(i))
@@ -954,11 +953,11 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 					assert.NoError(t, os.UpdateBalancesAtIndex(types.ValidatorIndex(i), bal+2000000000))
 				}
 				vs := &Server{
-					StateGen: mockGen,
 					HeadFetcher: &mockChain.ChainService{
 						State: hs,
 					},
-					SyncChecker: &mockSync.Sync{IsSyncing: false},
+					SyncChecker:     &mockSync.Sync{IsSyncing: false},
+					ReplayerBuilder: builder,
 				}
 				request := &ethpb.DoppelGangerRequest{
 					ValidatorRequests: make([]*ethpb.DoppelGangerRequest_ValidatorRequest, 0),
@@ -982,9 +981,7 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 			name:    "doppelganger exists current epoch",
 			wantErr: false,
 			svSetup: func(t *testing.T) (*Server, *ethpb.DoppelGangerRequest, *ethpb.DoppelGangerResponse) {
-				mockGen := mockstategen.NewMockService()
-
-				hs, ps, os, keys := createStateSetup(t, 4, mockGen)
+				hs, ps, os, keys, builder := createStateSetup(t, 4)
 				// Previous Epoch State
 				for i := 0; i < 2; i++ {
 					bal, err := ps.BalanceAtIndex(types.ValidatorIndex(i))
@@ -1010,11 +1007,11 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 				assert.NoError(t, os.UpdateBalancesAtIndex(types.ValidatorIndex(2), bal-1000000000))
 
 				vs := &Server{
-					StateGen: mockGen,
 					HeadFetcher: &mockChain.ChainService{
 						State: hs,
 					},
-					SyncChecker: &mockSync.Sync{IsSyncing: false},
+					SyncChecker:     &mockSync.Sync{IsSyncing: false},
+					ReplayerBuilder: builder,
 				}
 				request := &ethpb.DoppelGangerRequest{
 					ValidatorRequests: make([]*ethpb.DoppelGangerRequest_ValidatorRequest, 0),
@@ -1049,9 +1046,7 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 			name:    "doppelganger exists previous epoch",
 			wantErr: false,
 			svSetup: func(t *testing.T) (*Server, *ethpb.DoppelGangerRequest, *ethpb.DoppelGangerResponse) {
-				mockGen := mockstategen.NewMockService()
-
-				hs, ps, os, keys := createStateSetup(t, 4, mockGen)
+				hs, ps, os, keys, builder := createStateSetup(t, 4)
 				// Previous Epoch State
 				for i := 0; i < 2; i++ {
 					bal, err := ps.BalanceAtIndex(types.ValidatorIndex(i))
@@ -1077,11 +1072,11 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 				assert.NoError(t, os.UpdateBalancesAtIndex(types.ValidatorIndex(2), bal-2000000000))
 
 				vs := &Server{
-					StateGen: mockGen,
 					HeadFetcher: &mockChain.ChainService{
 						State: hs,
 					},
-					SyncChecker: &mockSync.Sync{IsSyncing: false},
+					SyncChecker:     &mockSync.Sync{IsSyncing: false},
+					ReplayerBuilder: builder,
 				}
 				request := &ethpb.DoppelGangerRequest{
 					ValidatorRequests: make([]*ethpb.DoppelGangerRequest_ValidatorRequest, 0),
@@ -1116,9 +1111,7 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 			name:    "multiple doppelganger exists",
 			wantErr: false,
 			svSetup: func(t *testing.T) (*Server, *ethpb.DoppelGangerRequest, *ethpb.DoppelGangerResponse) {
-				mockGen := mockstategen.NewMockService()
-
-				hs, ps, os, keys := createStateSetup(t, 4, mockGen)
+				hs, ps, os, keys, builder := createStateSetup(t, 4)
 				// Previous Epoch State
 				for i := 10; i < 15; i++ {
 					bal, err := ps.BalanceAtIndex(types.ValidatorIndex(i))
@@ -1136,11 +1129,11 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 				}
 
 				vs := &Server{
-					StateGen: mockGen,
 					HeadFetcher: &mockChain.ChainService{
 						State: hs,
 					},
-					SyncChecker: &mockSync.Sync{IsSyncing: false},
+					SyncChecker:     &mockSync.Sync{IsSyncing: false},
+					ReplayerBuilder: builder,
 				}
 				request := &ethpb.DoppelGangerRequest{
 					ValidatorRequests: make([]*ethpb.DoppelGangerRequest_ValidatorRequest, 0),
@@ -1166,16 +1159,14 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 			name:    "attesters are too recent",
 			wantErr: false,
 			svSetup: func(t *testing.T) (*Server, *ethpb.DoppelGangerRequest, *ethpb.DoppelGangerResponse) {
-				mockGen := mockstategen.NewMockService()
-
-				hs, _, _, keys := createStateSetup(t, 4, mockGen)
+				hs, _, _, keys, _ := createStateSetup(t, 4)
 
 				vs := &Server{
-					StateGen: nil,
 					HeadFetcher: &mockChain.ChainService{
 						State: hs,
 					},
-					SyncChecker: &mockSync.Sync{IsSyncing: false},
+					SyncChecker:     &mockSync.Sync{IsSyncing: false},
+					ReplayerBuilder: nil,
 				}
 				request := &ethpb.DoppelGangerRequest{
 					ValidatorRequests: make([]*ethpb.DoppelGangerRequest_ValidatorRequest, 0),
@@ -1213,8 +1204,9 @@ func TestServer_CheckDoppelGanger(t *testing.T) {
 	}
 }
 
-func createStateSetup(t *testing.T, head types.Epoch, mockgen *mockstategen.MockStateManager) (state.BeaconState,
-	state.BeaconState, state.BeaconState, []bls.SecretKey) {
+func createStateSetup(t *testing.T, head types.Epoch) (state.BeaconState,
+	state.BeaconState, state.BeaconState, []bls.SecretKey, *mockstategen.MockReplayerBuilder) {
+	rb := &mockstategen.MockReplayerBuilder{}
 	gs, keys := util.DeterministicGenesisState(t, 64)
 	hs := gs.Copy()
 	// Head State
@@ -1243,9 +1235,8 @@ func createStateSetup(t *testing.T, head types.Epoch, mockgen *mockstategen.Mock
 			ProposerIndex:  10,
 		}
 		assert.NoError(t, hs.AppendCurrentEpochAttestations(pendingAtt))
-
 	}
-	mockgen.StatesBySlot[headSlot] = hs
+	rb.SetMockState(hs)
 
 	// Previous Epoch State
 	prevEpoch := headEpoch - 1
@@ -1275,9 +1266,8 @@ func createStateSetup(t *testing.T, head types.Epoch, mockgen *mockstategen.Mock
 			ProposerIndex:  10,
 		}
 		assert.NoError(t, ps.AppendCurrentEpochAttestations(pendingAtt))
-
 	}
-	mockgen.StatesBySlot[prevSlot] = ps
+	rb.SetMockState(ps)
 
 	// Older Epoch State
 	olderEpoch := prevEpoch - 1
@@ -1311,8 +1301,7 @@ func createStateSetup(t *testing.T, head types.Epoch, mockgen *mockstategen.Mock
 			ProposerIndex:  10,
 		}
 		assert.NoError(t, os.AppendCurrentEpochAttestations(pendingAtt))
-
 	}
-	mockgen.StatesBySlot[olderSlot] = os
-	return hs, ps, os, keys
+	rb.SetMockState(os)
+	return hs, ps, os, keys, rb
 }

beacon-chain/rpc/service.go

@@ -123,6 +123,10 @@ func NewService(ctx context.Context, cfg *Config) *Service {
 	}
 }
 
+// paranoid build time check to ensure ChainInfoFetcher implements required interfaces
+var _ stategen.CanonicalChecker = blockchain.ChainInfoFetcher(nil)
+var _ stategen.CurrentSlotter = blockchain.ChainInfoFetcher(nil)
+
 // Start the gRPC server.
 func (s *Service) Start() {
 	address := fmt.Sprintf("%s:%s", s.cfg.Host, s.cfg.Port)
@@ -167,6 +171,13 @@ func (s *Service) Start() {
 	}
 	s.grpcServer = grpc.NewServer(opts...)
 
+	var stateCache stategen.CachedGetter
+	// s.cfg.StateGen is often nil in tests...
+	if s.cfg.StateGen != nil {
+		stateCache = s.cfg.StateGen.CombinedCache()
+	}
+	withCache := stategen.WithCache(stateCache)
+
 	validatorServer := &validatorv1alpha1.Server{
 		Ctx:                    s.ctx,
 		AttestationCache:       cache.NewAttestationCache(),
@@ -192,6 +203,7 @@ func (s *Service) Start() {
 		SlashingsPool:          s.cfg.SlashingsPool,
 		StateGen:               s.cfg.StateGen,
 		SyncCommitteePool:      s.cfg.SyncCommitteeObjectPool,
+		ReplayerBuilder:        stategen.NewCanonicalBuilder(s.cfg.BeaconDB, s.cfg.ChainInfoFetcher, s.cfg.ChainInfoFetcher, withCache),
 	}
 	validatorServerV1 := &validator.Server{
 		HeadFetcher:      s.cfg.HeadFetcher,
@@ -206,6 +218,7 @@ func (s *Service) Start() {
 			ChainInfoFetcher:   s.cfg.ChainInfoFetcher,
 			GenesisTimeFetcher: s.cfg.GenesisTimeFetcher,
 			StateGenService:    s.cfg.StateGen,
+			ReplayerBuilder:    stategen.NewCanonicalBuilder(s.cfg.BeaconDB, s.cfg.ChainInfoFetcher, s.cfg.ChainInfoFetcher, withCache),
 		},
 		SyncCommitteePool: s.cfg.SyncCommitteeObjectPool,
 	}
@@ -255,6 +268,7 @@ func (s *Service) Start() {
 		SyncChecker:                 s.cfg.SyncService,
 		ReceivedAttestationsBuffer:  make(chan *ethpbv1alpha1.Attestation, attestationBufferSize),
 		CollectedAttestationsBuffer: make(chan []*ethpbv1alpha1.Attestation, attestationBufferSize),
+		ReplayerBuilder:             stategen.NewCanonicalBuilder(s.cfg.BeaconDB, s.cfg.ChainInfoFetcher, s.cfg.ChainInfoFetcher, withCache),
 	}
 	beaconChainServerV1 := &beacon.Server{
 		BeaconDB:           s.cfg.BeaconDB,
@@ -297,6 +311,7 @@ func (s *Service) Start() {
 			HeadFetcher:        s.cfg.HeadFetcher,
 			PeerManager:        s.cfg.PeerManager,
 			PeersFetcher:       s.cfg.PeersFetcher,
+			ReplayerBuilder:    stategen.NewCanonicalBuilder(s.cfg.BeaconDB, s.cfg.ChainInfoFetcher, s.cfg.ChainInfoFetcher, withCache),
 		}
 		debugServerV1 := &debug.Server{
 			BeaconDB:    s.cfg.BeaconDB,
@@ -306,6 +321,7 @@ func (s *Service) Start() {
 				ChainInfoFetcher:   s.cfg.ChainInfoFetcher,
 				GenesisTimeFetcher: s.cfg.GenesisTimeFetcher,
 				StateGenService:    s.cfg.StateGen,
+				ReplayerBuilder:    stategen.NewCanonicalBuilder(s.cfg.BeaconDB, s.cfg.ChainInfoFetcher, s.cfg.ChainInfoFetcher, withCache),
 			},
 		}
 		ethpbv1alpha1.RegisterDebugServer(s.grpcServer, debugServer)

beacon-chain/rpc/statefetcher/BUILD.bazel

@@ -14,6 +14,7 @@ go_library(
         "//encoding/bytesutil:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prysmaticlabs_eth2_types//:go_default_library",
+        "@io_opencensus_go//trace:go_default_library",
     ],
 )
 
@@ -24,6 +25,7 @@ go_test(
     deps = [
         "//beacon-chain/blockchain/testing:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
+        "//beacon-chain/state/stategen:go_default_library",
         "//beacon-chain/state/stategen/mock:go_default_library",
         "//config/params:go_default_library",
         "//encoding/bytesutil:go_default_library",

beacon-chain/rpc/statefetcher/fetcher.go

@@ -15,8 +15,13 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	"go.opencensus.io/trace"
 )
 
+var ErrNoAncestorForBlock = errors.New("could not find an ancestor state for block")
+var ErrNoCanonicalBlockForSlot = errors.New("none of the blocks found in the db slot index are canonical")
+var ErrInvalidDBBlock = errors.New("invalid block found in database")
+
 // StateIdParseError represents an error scenario where a state ID could not be parsed.
 type StateIdParseError struct {
 	message string
@@ -72,6 +77,7 @@ func (e *StateRootNotFoundError) Error() string {
 type Fetcher interface {
 	State(ctx context.Context, stateId []byte) (state.BeaconState, error)
 	StateRoot(ctx context.Context, stateId []byte) ([]byte, error)
+	StateBySlot(ctx context.Context, slot types.Slot) (state.BeaconState, error)
 }
 
 // StateProvider is a real implementation of Fetcher.
@@ -80,6 +86,7 @@ type StateProvider struct {
 	ChainInfoFetcher   blockchain.ChainInfoFetcher
 	GenesisTimeFetcher blockchain.TimeFetcher
 	StateGenService    stategen.StateManager
+	ReplayerBuilder    stategen.ReplayerBuilder
 }
 
 // State returns the BeaconState for a given identifier. The identifier can be one of:
@@ -129,7 +136,7 @@ func (p *StateProvider) State(ctx context.Context, stateId []byte) (state.Beacon
 				e := NewStateIdParseError(parseErr)
 				return nil, &e
 			}
-			s, err = p.stateBySlot(ctx, types.Slot(slotNumber))
+			s, err = p.StateBySlot(ctx, types.Slot(slotNumber))
 		}
 	}
 
@@ -187,16 +194,21 @@ func (p *StateProvider) stateByHex(ctx context.Context, stateId []byte) (state.B
 	return nil, &stateNotFoundErr
 }
 
-func (p *StateProvider) stateBySlot(ctx context.Context, slot types.Slot) (state.BeaconState, error) {
-	currentSlot := p.GenesisTimeFetcher.CurrentSlot()
-	if slot > currentSlot {
-		return nil, errors.New("slot cannot be in the future")
-	}
-	state, err := p.StateGenService.StateBySlot(ctx, slot)
+// StateBySlot returns the post-state for the requested slot. To generate the state, it uses the
+// most recent canonical state prior to the target slot, and all canonical blocks
+// between the found state's slot and the target slot.
+// process_blocks is applied for all canonical blocks, and process_slots is called for any skipped
+// slots, or slots following the most recent canonical block up to and including the target slot.
+func (p *StateProvider) StateBySlot(ctx context.Context, target types.Slot) (state.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "statefetcher.StateBySlot")
+	defer span.End()
+
+	st, err := p.ReplayerBuilder.ForSlot(target).ReplayBlocks(ctx)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not get state")
+		msg := fmt.Sprintf("error while replaying history to slot=%d", target)
+		return nil, errors.Wrap(err, msg)
 	}
-	return state, nil
+	return st, nil
 }
 
 func (p *StateProvider) headStateRoot(ctx context.Context) ([]byte, error) {
@@ -204,7 +216,7 @@ func (p *StateProvider) headStateRoot(ctx context.Context) ([]byte, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "could not get head block")
 	}
-	if err := helpers.BeaconBlockIsNil(b); err != nil {
+	if err = helpers.BeaconBlockIsNil(b); err != nil {
 		return nil, err
 	}
 	return b.Block().StateRoot(), nil

beacon-chain/rpc/statefetcher/fetcher_test.go

@@ -7,6 +7,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	types "github.com/prysmaticlabs/eth2-types"
 	chainMock "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
@@ -151,12 +153,14 @@ func TestGetState(t *testing.T) {
 	})
 
 	t.Run("slot", func(t *testing.T) {
-		stateGen := mockstategen.NewMockService()
-		stateGen.StatesBySlot[headSlot] = newBeaconState
-
 		p := StateProvider{
 			GenesisTimeFetcher: &chainMock.ChainService{Slot: &headSlot},
-			StateGenService:    stateGen,
+			ChainInfoFetcher: &chainMock.ChainService{
+				CanonicalRoots: map[[32]byte]bool{
+					bytesutil.ToBytes32(newBeaconState.LatestBlockHeader().ParentRoot): true,
+				},
+			},
+			ReplayerBuilder: mockstategen.NewMockReplayerBuilder(mockstategen.WithMockState(newBeaconState)),
 		}
 
 		s, err := p.State(ctx, []byte(strconv.FormatUint(uint64(headSlot), 10)))
@@ -166,14 +170,16 @@ func TestGetState(t *testing.T) {
 		assert.Equal(t, stateRoot, sRoot)
 	})
 
+	rb := mockstategen.NewMockReplayerBuilder(mockstategen.WithStateError(1, stategen.ErrFutureSlotRequested))
 	t.Run("slot_too_big", func(t *testing.T) {
 		p := StateProvider{
 			GenesisTimeFetcher: &chainMock.ChainService{
 				Genesis: time.Now(),
 			},
+			ReplayerBuilder: rb,
 		}
 		_, err := p.State(ctx, []byte(strconv.FormatUint(1, 10)))
-		assert.ErrorContains(t, "slot cannot be in the future", err)
+		assert.ErrorContains(t, "cannot replay to future slots", err)
 	})
 
 	t.Run("invalid_state", func(t *testing.T) {

beacon-chain/rpc/testutil/mock_state_fetcher.go

@@ -3,6 +3,7 @@ package testutil
 import (
 	"context"
 
+	types "github.com/prysmaticlabs/eth2-types"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 )
 
@@ -21,3 +22,7 @@ func (m *MockFetcher) State(context.Context, []byte) (state.BeaconState, error)
 func (m *MockFetcher) StateRoot(context.Context, []byte) ([]byte, error) {
 	return m.BeaconStateRoot, nil
 }
+
+func (m *MockFetcher) StateBySlot(context.Context, types.Slot) (state.BeaconState, error) {
+	return m.BeaconState, nil
+}

beacon-chain/state/fieldtrie/field_trie_test.go

@@ -20,7 +20,7 @@ func TestFieldTrie_NewTrie(t *testing.T) {
 	// 5 represents the enum value of state roots
 	trie, err := fieldtrie.NewFieldTrie(5, stateTypes.BasicArray, newState.StateRoots(), uint64(params.BeaconConfig().SlotsPerHistoricalRoot))
 	require.NoError(t, err)
-	root, err := stateutil.RootsArrayHashTreeRoot(newState.StateRoots(), uint64(params.BeaconConfig().SlotsPerHistoricalRoot), "StateRoots")
+	root, err := stateutil.RootsArrayHashTreeRoot(newState.StateRoots(), uint64(params.BeaconConfig().SlotsPerHistoricalRoot))
 	require.NoError(t, err)
 	newRoot, err := trie.TrieRoot()
 	require.NoError(t, err)

beacon-chain/state/state-native/v1/field_roots.go

@@ -4,15 +4,11 @@ import (
 	"context"
 
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
-	"github.com/prysmaticlabs/prysm/config/features"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 )
 
 // computeFieldRoots returns the hash tree root computations of every field in
 // the beacon state as a list of 32 byte roots.
 func computeFieldRoots(ctx context.Context, state *ethpb.BeaconState) ([][]byte, error) {
-	if features.Get().EnableSSZCache {
-		return stateutil.CachedHasher.ComputeFieldRootsWithHasherPhase0(ctx, state)
-	}
-	return stateutil.NocachedHasher.ComputeFieldRootsWithHasherPhase0(ctx, state)
+	return stateutil.ComputeFieldRootsWithHasherPhase0(ctx, state)
 }

beacon-chain/state/state-native/v1/state_trie_test.go

@@ -258,3 +258,21 @@ func TestBeaconState_AppendValidator_DoesntMutateCopy(t *testing.T) {
 	_, ok := st1.ValidatorIndexByPubkey(bytesutil.ToBytes48(val.PublicKey))
 	assert.Equal(t, false, ok, "Expected no validator index to be present in st1 for the newly inserted pubkey")
 }
+
+func BenchmarkBeaconState(b *testing.B) {
+	testState, _ := util.DeterministicGenesisState(b, 16000)
+	pbState, err := v1.ProtobufBeaconState(testState.InnerStateUnsafe())
+	require.NoError(b, err)
+
+	b.Run("Vectorized SHA256", func(b *testing.B) {
+		st, err := v1.InitializeFromProtoUnsafe(pbState)
+		require.NoError(b, err)
+		_, err = st.HashTreeRoot(context.Background())
+		assert.NoError(b, err)
+	})
+
+	b.Run("Current SHA256", func(b *testing.B) {
+		_, err := pbState.HashTreeRoot()
+		require.NoError(b, err)
+	})
+}

beacon-chain/state/state-native/v2/field_roots.go

@@ -4,15 +4,11 @@ import (
 	"context"
 
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
-	"github.com/prysmaticlabs/prysm/config/features"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 )
 
 // computeFieldRoots returns the hash tree root computations of every field in
 // the beacon state as a list of 32 byte roots.
 func computeFieldRoots(ctx context.Context, state *ethpb.BeaconStateAltair) ([][]byte, error) {
-	if features.Get().EnableSSZCache {
-		return stateutil.CachedHasher.ComputeFieldRootsWithHasherAltair(ctx, state)
-	}
-	return stateutil.NocachedHasher.ComputeFieldRootsWithHasherAltair(ctx, state)
+	return stateutil.ComputeFieldRootsWithHasherAltair(ctx, state)
 }

beacon-chain/state/state-native/v3/field_roots.go

@@ -4,15 +4,11 @@ import (
 	"context"
 
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
-	"github.com/prysmaticlabs/prysm/config/features"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 )
 
 // computeFieldRoots returns the hash tree root computations of every field in
 // the beacon state as a list of 32 byte roots.
 func computeFieldRoots(ctx context.Context, state *ethpb.BeaconStateBellatrix) ([][]byte, error) {
-	if features.Get().EnableSSZCache {
-		return stateutil.CachedHasher.ComputeFieldRootsWithHasherBellatrix(ctx, state)
-	}
-	return stateutil.NocachedHasher.ComputeFieldRootsWithHasherBellatrix(ctx, state)
+	return stateutil.ComputeFieldRootsWithHasherBellatrix(ctx, state)
 }

beacon-chain/state/state-native/v3/setters_test.go

@@ -61,7 +61,7 @@ func TestBeaconState_AppendBalanceWithTrie(t *testing.T) {
 		StateRoot:        make([]byte, fieldparams.RootLength),
 		ReceiptRoot:      make([]byte, fieldparams.RootLength),
 		LogsBloom:        make([]byte, 256),
-		Random:           make([]byte, fieldparams.RootLength),
+		PrevRandao:       make([]byte, fieldparams.RootLength),
 		BaseFeePerGas:    make([]byte, fieldparams.RootLength),
 		BlockHash:        make([]byte, fieldparams.RootLength),
 		TransactionsRoot: make([]byte, fieldparams.RootLength),

beacon-chain/state/stategen/BUILD.bazel

@@ -3,6 +3,7 @@ load("@prysm//tools/go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
+        "cacher.go",
         "epoch_boundary_state_cache.go",
         "errors.go",
         "getter.go",
@@ -11,16 +12,12 @@ go_library(
         "metrics.go",
         "migrate.go",
         "replay.go",
+        "replayer.go",
         "service.go",
         "setter.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/state/stategen",
-    visibility = [
-        "//beacon-chain:__subpackages__",
-        "//testing/endtoend:__subpackages__",
-        "//testing/slasher/simulator:__pkg__",
-        "//testing/spectest:__subpackages__",
-    ],
+    visibility = ["//visibility:public"],
     deps = [
         "//beacon-chain/core/altair:go_default_library",
         "//beacon-chain/core/execution:go_default_library",
@@ -33,6 +30,7 @@ go_library(
         "//cache/lru:go_default_library",
         "//config/params:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//monitoring/tracing:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//proto/prysm/v1alpha1/block:go_default_library",
         "//runtime/version:go_default_library",
@@ -56,13 +54,17 @@ go_test(
         "hot_state_cache_test.go",
         "init_test.go",
         "migrate_test.go",
+        "mock_test.go",
         "replay_test.go",
+        "replayer_test.go",
         "service_test.go",
         "setter_test.go",
     ],
     embed = [":go_default_library"],
     deps = [
         "//beacon-chain/core/blocks:go_default_library",
+        "//beacon-chain/core/helpers:go_default_library",
+        "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/state:go_default_library",
@@ -71,11 +73,13 @@ go_test(
         "//encoding/bytesutil:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//proto/prysm/v1alpha1/block:go_default_library",
+        "//proto/prysm/v1alpha1/block/mock:go_default_library",
         "//proto/prysm/v1alpha1/wrapper:go_default_library",
         "//runtime/version:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/require:go_default_library",
         "//testing/util:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
         "@com_github_prysmaticlabs_eth2_types//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
         "@org_golang_google_protobuf//proto:go_default_library",

beacon-chain/state/stategen/cacher.go

@@ -0,0 +1,32 @@
+package stategen
+
+import (
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+)
+
+var ErrNotInCache = errors.New("state not found in cache")
+
+type CachedGetter interface {
+	ByRoot([32]byte) (state.BeaconState, error)
+}
+
+type CombinedCache struct {
+	getters []CachedGetter
+}
+
+func (c CombinedCache) ByRoot(root [32]byte) (state.BeaconState, error) {
+	for _, getter := range c.getters {
+		st, err := getter.ByRoot(root)
+		if err == nil {
+			return st, nil
+		}
+		if errors.Is(err, ErrNotInCache) {
+			continue
+		}
+		return nil, err
+	}
+	return nil, ErrNotInCache
+}
+
+var _ CachedGetter = &CombinedCache{}

beacon-chain/state/stategen/epoch_boundary_state_cache.go

@@ -66,6 +66,18 @@ func newBoundaryStateCache() *epochBoundaryState {
 	}
 }
 
+// ByRoot satisfies the CachedGetter interface
+func (e *epochBoundaryState) ByRoot(r [32]byte) (state.BeaconState, error) {
+	rsi, ok, err := e.getByRoot(r)
+	if err != nil {
+		return nil, err
+	}
+	if !ok {
+		return nil, ErrNotInCache
+	}
+	return rsi.state, nil
+}
+
 // get epoch boundary state by its block root. Returns copied state in state info object if exists. Otherwise returns nil.
 func (e *epochBoundaryState) getByRoot(r [32]byte) (*rootStateInfo, bool, error) {
 	e.lock.RLock()

beacon-chain/state/stategen/getter.go

@@ -84,7 +84,7 @@ func (s *State) StateByRootInitialSync(ctx context.Context, blockRoot [32]byte)
 		return cachedInfo.state, nil
 	}
 
-	startState, err := s.lastAncestorState(ctx, blockRoot)
+	startState, err := s.LastAncestorState(ctx, blockRoot)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not get ancestor state")
 	}
@@ -185,7 +185,7 @@ func (s *State) loadStateByRoot(ctx context.Context, blockRoot [32]byte) (state.
 
 	// Since the requested state is not in caches, start replaying using the last available ancestor state which is
 	// retrieved using input block's parent root.
-	startState, err := s.lastAncestorState(ctx, blockRoot)
+	startState, err := s.LastAncestorState(ctx, blockRoot)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not get ancestor state")
 	}
@@ -230,7 +230,7 @@ func (s *State) loadStateBySlot(ctx context.Context, slot types.Slot) (state.Bea
 	}
 
 	if lastValidSlot < slot {
-		replayStartState, err = processSlotsStateGen(ctx, replayStartState, slot)
+		replayStartState, err = ReplayProcessSlots(ctx, replayStartState, slot)
 		if err != nil {
 			return nil, err
 		}
@@ -246,8 +246,8 @@ func (s *State) loadStateBySlot(ctx context.Context, slot types.Slot) (state.Bea
 // 1.) block parent state is the last finalized state
 // 2.) block parent state is the epoch boundary state and exists in epoch boundary cache.
 // 3.) block parent state is in DB.
-func (s *State) lastAncestorState(ctx context.Context, root [32]byte) (state.BeaconState, error) {
-	ctx, span := trace.StartSpan(ctx, "stateGen.lastAncestorState")
+func (s *State) LastAncestorState(ctx context.Context, root [32]byte) (state.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "stateGen.LastAncestorState")
 	defer span.End()
 
 	if s.isFinalizedRoot(root) && s.finalizedState() != nil {
@@ -304,3 +304,14 @@ func (s *State) lastAncestorState(ctx context.Context, root [32]byte) (state.Bea
 		}
 	}
 }
+
+func (s *State) CombinedCache() *CombinedCache {
+	getters := make([]CachedGetter, 0)
+	if s.hotStateCache != nil {
+		getters = append(getters, s.hotStateCache)
+	}
+	if s.epochBoundaryStateCache != nil {
+		getters = append(getters, s.epochBoundaryStateCache)
+	}
+	return &CombinedCache{getters: getters}
+}

beacon-chain/state/stategen/getter_test.go

@@ -491,7 +491,7 @@ func TestLastAncestorState_CanGetUsingDB(t *testing.T) {
 	require.NoError(t, service.beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b3)))
 	require.NoError(t, service.beaconDB.SaveState(ctx, b1State, r1))
 
-	lastState, err := service.lastAncestorState(ctx, r3)
+	lastState, err := service.LastAncestorState(ctx, r3)
 	require.NoError(t, err)
 	assert.Equal(t, b1State.Slot(), lastState.Slot(), "Did not get wanted state")
 }
@@ -531,7 +531,7 @@ func TestLastAncestorState_CanGetUsingCache(t *testing.T) {
 	require.NoError(t, service.beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b3)))
 	service.hotStateCache.put(r1, b1State)
 
-	lastState, err := service.lastAncestorState(ctx, r3)
+	lastState, err := service.LastAncestorState(ctx, r3)
 	require.NoError(t, err)
 	assert.Equal(t, b1State.Slot(), lastState.Slot(), "Did not get wanted state")
 }

beacon-chain/state/stategen/hot_state_cache.go

@@ -52,6 +52,14 @@ func (c *hotStateCache) get(root [32]byte) state.BeaconState {
 	return nil
 }
 
+func (c *hotStateCache) ByRoot(root [32]byte) (state.BeaconState, error) {
+	st := c.get(root)
+	if st == nil {
+		return nil, ErrNotInCache
+	}
+	return st, nil
+}
+
 // GetWithoutCopy returns a non-copied cached response via input block root.
 func (c *hotStateCache) getWithoutCopy(root [32]byte) state.BeaconState {
 	c.lock.RLock()

beacon-chain/state/stategen/mock/BUILD.bazel

@@ -3,11 +3,15 @@ load("@prysm//tools/go:def.bzl", "go_library")
 go_library(
     name = "go_default_library",
     testonly = True,
-    srcs = ["mock.go"],
+    srcs = [
+        "mock.go",
+        "replayer.go",
+    ],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/state/stategen/mock",
     visibility = ["//visibility:public"],
     deps = [
         "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/stategen:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//proto/prysm/v1alpha1/block:go_default_library",
         "@com_github_prysmaticlabs_eth2_types//:go_default_library",

beacon-chain/state/stategen/mock/replayer.go

@@ -0,0 +1,91 @@
+package mock
+
+import (
+	"context"
+
+	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+)
+
+func NewMockReplayerBuilder(opt ...MockReplayerBuilderOption) *MockReplayerBuilder {
+	b := &MockReplayerBuilder{}
+	for _, o := range opt {
+		o(b)
+	}
+	return b
+}
+
+type MockReplayerBuilderOption func(*MockReplayerBuilder)
+
+func WithMockState(s state.BeaconState) MockReplayerBuilderOption {
+	return func(b *MockReplayerBuilder) {
+		b.SetMockState(s)
+	}
+}
+
+func WithStateError(s types.Slot, e error) MockReplayerBuilderOption {
+	return func(b *MockReplayerBuilder) {
+		b.SetMockSlotError(s, e)
+	}
+}
+
+type MockReplayerBuilder struct {
+	forSlot map[types.Slot]*MockReplayer
+}
+
+func (b *MockReplayerBuilder) SetMockState(s state.BeaconState) {
+	if b.forSlot == nil {
+		b.forSlot = make(map[types.Slot]*MockReplayer)
+	}
+	b.forSlot[s.Slot()] = &MockReplayer{State: s}
+}
+
+func (b *MockReplayerBuilder) SetMockSlotError(s types.Slot, e error) {
+	if b.forSlot == nil {
+		b.forSlot = make(map[types.Slot]*MockReplayer)
+	}
+	b.forSlot[s] = &MockReplayer{Err: e}
+}
+
+func (b *MockReplayerBuilder) ForSlot(target types.Slot) stategen.Replayer {
+	return b.forSlot[target]
+}
+
+var _ stategen.ReplayerBuilder = &MockReplayerBuilder{}
+
+type MockReplayer struct {
+	State state.BeaconState
+	Err   error
+}
+
+func (m *MockReplayer) ReplayBlocks(_ context.Context) (state.BeaconState, error) {
+	return m.State, m.Err
+}
+
+func (m *MockReplayer) ReplayToSlot(_ context.Context, _ types.Slot) (state.BeaconState, error) {
+	return m.State, m.Err
+}
+
+var _ stategen.Replayer = &MockReplayer{}
+
+type MockCanonicalChecker struct {
+	Is  bool
+	Err error
+}
+
+func (m *MockCanonicalChecker) IsCanonical(_ context.Context, _ [32]byte) (bool, error) {
+	return m.Is, m.Err
+}
+
+var _ stategen.CanonicalChecker = &MockCanonicalChecker{}
+
+type MockCurrentSlotter struct {
+	Slot types.Slot
+}
+
+func (c *MockCurrentSlotter) CurrentSlot() types.Slot {
+	return c.Slot
+}
+
+var _ stategen.CurrentSlotter = &MockCurrentSlotter{}

beacon-chain/state/stategen/mock_test.go

@@ -0,0 +1,282 @@
+package stategen
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"testing"
+
+	"github.com/pkg/errors"
+	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/transition"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
+	"github.com/prysmaticlabs/prysm/testing/require"
+	"github.com/prysmaticlabs/prysm/testing/util"
+)
+
+func TestMockHistoryStates(t *testing.T) {
+	ctx := context.Background()
+	var begin, middle, end types.Slot = 100, 150, 155
+	specs := []mockHistorySpec{
+		{slot: begin},
+		{slot: middle, savedState: true},
+		{slot: end, canonicalBlock: true},
+	}
+	hist := newMockHistory(t, specs, end+1)
+	// we should have 2 "saved" states, genesis and "middle" (savedState == true)
+	require.Equal(t, 2, len(hist.states))
+	genesisRoot := hist.slotMap[0]
+	st, err := hist.StateOrError(ctx, genesisRoot)
+	require.NoError(t, err)
+	require.DeepEqual(t, hist.states[genesisRoot], st)
+	require.Equal(t, types.Slot(0), st.Slot())
+
+	shouldExist, err := hist.StateOrError(ctx, hist.slotMap[middle])
+	require.NoError(t, err)
+	require.DeepEqual(t, hist.states[hist.slotMap[middle]], shouldExist)
+	require.Equal(t, middle, shouldExist.Slot())
+
+	cantExist, err := hist.StateOrError(ctx, hist.slotMap[end])
+	require.ErrorIs(t, err, db.ErrNotFoundState)
+	require.Equal(t, nil, cantExist)
+
+	cantExist, err = hist.StateOrError(ctx, hist.slotMap[begin])
+	require.ErrorIs(t, err, db.ErrNotFoundState)
+	require.Equal(t, nil, cantExist)
+}
+
+func TestMockHistoryParentRoot(t *testing.T) {
+	ctx := context.Background()
+	var begin, middle, end types.Slot = 100, 150, 155
+	specs := []mockHistorySpec{
+		{slot: begin},
+		{slot: middle, savedState: true},
+		{slot: end, canonicalBlock: true},
+	}
+	hist := newMockHistory(t, specs, end+1)
+	endRoot := hist.slotMap[end]
+	endBlock, err := hist.Block(ctx, endRoot)
+	require.NoError(t, err)
+	// middle should be the parent of end, compare the middle root to endBlock's parent root
+	require.Equal(t, hist.slotMap[middle], bytesutil.ToBytes32(endBlock.Block().ParentRoot()))
+}
+
+type mockHistorySpec struct {
+	slot           types.Slot
+	savedState     bool
+	canonicalBlock bool
+}
+
+type mockHistory struct {
+	blocks                         map[[32]byte]block.SignedBeaconBlock
+	slotMap                        map[types.Slot][32]byte
+	slotIndex                      slotList
+	canonical                      map[[32]byte]bool
+	states                         map[[32]byte]state.BeaconState
+	hiddenStates                   map[[32]byte]state.BeaconState
+	current                        types.Slot
+	overrideHighestSlotBlocksBelow func(context.Context, types.Slot) ([]block.SignedBeaconBlock, error)
+}
+
+type slotList []types.Slot
+
+func (m slotList) Len() int {
+	return len(m)
+}
+
+func (m slotList) Less(i, j int) bool {
+	return m[i] < m[j]
+}
+
+func (m slotList) Swap(i, j int) {
+	m[i], m[j] = m[j], m[i]
+}
+
+var errFallThroughOverride = errors.New("override yielding control back to real HighestSlotBlocksBelow")
+
+func (m *mockHistory) HighestSlotBlocksBelow(_ context.Context, slot types.Slot) ([]block.SignedBeaconBlock, error) {
+	if m.overrideHighestSlotBlocksBelow != nil {
+		s, err := m.overrideHighestSlotBlocksBelow(context.Background(), slot)
+		if !errors.Is(err, errFallThroughOverride) {
+			return s, err
+		}
+	}
+	if len(m.slotIndex) == 0 && len(m.slotMap) > 0 {
+		for k := range m.slotMap {
+			m.slotIndex = append(m.slotIndex, k)
+		}
+		sort.Sort(sort.Reverse(m.slotIndex))
+	}
+	for _, s := range m.slotIndex {
+		if s < slot {
+			return []block.SignedBeaconBlock{m.blocks[m.slotMap[s]]}, nil
+		}
+	}
+	return []block.SignedBeaconBlock{}, nil
+}
+
+var errGenesisBlockNotFound = errors.New("canonical genesis block not found in db")
+
+func (m *mockHistory) GenesisBlock(_ context.Context) (block.SignedBeaconBlock, error) {
+	genesisRoot, ok := m.slotMap[0]
+	if !ok {
+		return nil, errGenesisBlockNotFound
+	}
+	return m.blocks[genesisRoot], nil
+}
+
+func (m *mockHistory) Block(_ context.Context, blockRoot [32]byte) (block.SignedBeaconBlock, error) {
+	if b, ok := m.blocks[blockRoot]; ok {
+		return b, nil
+	}
+	return nil, nil
+}
+
+func (m *mockHistory) StateOrError(_ context.Context, blockRoot [32]byte) (state.BeaconState, error) {
+	if s, ok := m.states[blockRoot]; ok {
+		return s.Copy(), nil
+	}
+	return nil, db.ErrNotFoundState
+}
+
+func (m *mockHistory) IsCanonical(_ context.Context, blockRoot [32]byte) (bool, error) {
+	canon, ok := m.canonical[blockRoot]
+	return ok && canon, nil
+}
+
+func (m *mockHistory) CurrentSlot() types.Slot {
+	return m.current
+}
+
+func (h *mockHistory) addBlock(root [32]byte, b block.SignedBeaconBlock, canon bool) {
+	h.blocks[root] = b
+	h.slotMap[b.Block().Slot()] = root
+	h.canonical[root] = canon
+}
+
+func (h *mockHistory) addState(root [32]byte, s state.BeaconState) {
+	h.states[root] = s
+}
+
+func (h *mockHistory) hideState(root [32]byte, s state.BeaconState) {
+	h.hiddenStates[root] = s
+}
+
+func (h *mockHistory) validateRoots() error {
+	uniqParentRoots := make(map[[32]byte]types.Slot)
+	for s, root := range h.slotMap {
+		b := h.blocks[root]
+		htr, err := b.Block().HashTreeRoot()
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("error computing htr for block at slot %d", s))
+		}
+		if htr != root {
+			return fmt.Errorf("htr mismatch, expected=%#x, actual=%#x", root, htr)
+		}
+		if ps, ok := uniqParentRoots[htr]; ok {
+			return fmt.Errorf("duplicate parent_root %#x seen at slots %d, %d", htr, ps, s)
+		}
+		uniqParentRoots[htr] = s
+	}
+	return nil
+}
+
+func newMockHistory(t *testing.T, hist []mockHistorySpec, current types.Slot) *mockHistory {
+	ctx := context.Background()
+	mh := &mockHistory{
+		blocks:       map[[32]byte]block.SignedBeaconBlock{},
+		canonical:    map[[32]byte]bool{},
+		states:       map[[32]byte]state.BeaconState{},
+		hiddenStates: map[[32]byte]state.BeaconState{},
+		slotMap:      map[types.Slot][32]byte{},
+		slotIndex:    slotList{},
+		current:      current,
+	}
+
+	// genesis state for history
+	gs, _ := util.DeterministicGenesisState(t, 32)
+	gsr, err := gs.HashTreeRoot(ctx)
+	require.NoError(t, err)
+
+	// generate new genesis block using the root of the deterministic state
+	gb, err := wrapper.WrappedSignedBeaconBlock(blocks.NewGenesisBlock(gsr[:]))
+	require.NoError(t, err)
+	pr, err := gb.Block().HashTreeRoot()
+	require.NoError(t, err)
+
+	// add genesis block as canonical
+	mh.addBlock(pr, gb, true)
+	// add genesis state, indexed by unapplied genesis block - genesis block is never really processed...
+	mh.addState(pr, gs.Copy())
+
+	ps := gs.Copy()
+	for _, spec := range hist {
+		// call process_slots and process_block separately, because process_slots updates values used in randao mix
+		// which influences proposer_index.
+		s, err := ReplayProcessSlots(ctx, ps, spec.slot)
+		require.NoError(t, err)
+
+		// create proposer block, setting values in the order seen in the validator.md spec
+		b, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlock())
+		require.NoError(t, err)
+
+		// set slot to mock history spec value
+		require.NoError(t, wrapper.SetBlockSlot(b, spec.slot))
+
+		// set the correct proposer_index in the "proposal" block
+		// so that it will pass validation in process_block. important that we do this
+		// after process_slots!
+		idx, err := helpers.BeaconProposerIndex(ctx, s)
+		require.NoError(t, err)
+		require.NoError(t, wrapper.SetProposerIndex(b, idx))
+
+		// set parent root
+		require.NoError(t, wrapper.SetBlockParentRoot(b, pr))
+
+		// now do process_block
+		s, err = transition.ProcessBlockForStateRoot(ctx, s, b)
+		require.NoError(t, err)
+
+		sr, err := s.HashTreeRoot(ctx)
+		require.NoError(t, err)
+		err = wrapper.SetBlockStateRoot(b, sr)
+		require.NoError(t, err)
+
+		pr, err = b.Block().HashTreeRoot()
+		require.NoError(t, err)
+		if spec.savedState {
+			mh.addState(pr, s)
+		} else {
+			mh.hideState(pr, s)
+		}
+		mh.addBlock(pr, b, spec.canonicalBlock)
+		ps = s.Copy()
+	}
+
+	require.NoError(t, mh.validateRoots())
+	return mh
+}
+
+var _ HistoryAccessor = &mockHistory{}
+var _ CanonicalChecker = &mockHistory{}
+var _ CurrentSlotter = &mockHistory{}
+
+type mockCachedGetter struct {
+	cache map[[32]byte]state.BeaconState
+}
+
+func (m mockCachedGetter) ByRoot(root [32]byte) (state.BeaconState, error) {
+	st, ok := m.cache[root]
+	if !ok {
+		return nil, ErrNotInCache
+	}
+	return st, nil
+}
+
+var _ CachedGetter = &mockCachedGetter{}

beacon-chain/state/stategen/replay.go

@@ -10,11 +10,12 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/altair"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/execution"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	prysmTime "github.com/prysmaticlabs/prysm/beacon-chain/core/time"
+	prysmtime "github.com/prysmaticlabs/prysm/beacon-chain/core/time"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/transition"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	"github.com/prysmaticlabs/prysm/monitoring/tracing"
 	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
 	"github.com/prysmaticlabs/prysm/runtime/version"
 	"github.com/sirupsen/logrus"
@@ -60,7 +61,7 @@ func (_ *State) ReplayBlocks(
 
 	// If there is skip slots at the end.
 	if targetSlot > state.Slot() {
-		state, err = processSlotsStateGen(ctx, state, targetSlot)
+		state, err = ReplayProcessSlots(ctx, state, targetSlot)
 		if err != nil {
 			return nil, err
 		}
@@ -151,7 +152,7 @@ func executeStateTransitionStateGen(
 
 	// Execute per slots transition.
 	// Given this is for state gen, a node uses the version process slots without skip slots cache.
-	state, err = processSlotsStateGen(ctx, state, signed.Block().Slot())
+	state, err = ReplayProcessSlots(ctx, state, signed.Block().Slot())
 	if err != nil {
 		return nil, errors.Wrap(err, "could not process slot")
 	}
@@ -163,22 +164,14 @@ func executeStateTransitionStateGen(
 	if err != nil {
 		return nil, errors.Wrap(err, "could not process block")
 	}
-	if signed.Version() == version.Phase0 {
-		return state, nil
-	}
-
-	sa, err := signed.Block().Body().SyncAggregate()
-	if err != nil {
-		return nil, err
-	}
-	return altair.ProcessSyncAggregate(ctx, state, sa)
+	return state, nil
 }
 
-// processSlotsStateGen to process old slots for state gen usages.
+// ReplayProcessSlots to process old slots for state gen usages.
 // There's no skip slot cache involved given state gen only works with already stored block and state in DB.
 // WARNING: This method should not be used for future slot.
-func processSlotsStateGen(ctx context.Context, state state.BeaconState, slot types.Slot) (state.BeaconState, error) {
-	ctx, span := trace.StartSpan(ctx, "stategen.ProcessSlotsStateGen")
+func ReplayProcessSlots(ctx context.Context, state state.BeaconState, slot types.Slot) (state.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "stategen.ReplayProcessSlots")
 	defer span.End()
 	if state == nil || state.IsNil() {
 		return nil, errUnknownState
@@ -199,35 +192,41 @@ func processSlotsStateGen(ctx context.Context, state state.BeaconState, slot typ
 		if err != nil {
 			return nil, errors.Wrap(err, "could not process slot")
 		}
-		if prysmTime.CanProcessEpoch(state) {
+		if prysmtime.CanProcessEpoch(state) {
 			switch state.Version() {
 			case version.Phase0:
 				state, err = transition.ProcessEpochPrecompute(ctx, state)
 				if err != nil {
+					tracing.AnnotateError(span, err)
 					return nil, errors.Wrap(err, "could not process epoch with optimizations")
 				}
 			case version.Altair, version.Bellatrix:
 				state, err = altair.ProcessEpoch(ctx, state)
 				if err != nil {
-					return nil, errors.Wrap(err, "could not process epoch with optimization")
+					tracing.AnnotateError(span, err)
+					return nil, errors.Wrap(err, "could not process epoch")
 				}
 			default:
 				return nil, errors.New("beacon state should have a version")
 			}
 		}
 		if err := state.SetSlot(state.Slot() + 1); err != nil {
-			return nil, err
+			tracing.AnnotateError(span, err)
+			return nil, errors.Wrap(err, "failed to increment state slot")
 		}
 
-		if prysmTime.CanUpgradeToAltair(state.Slot()) {
+		if prysmtime.CanUpgradeToAltair(state.Slot()) {
 			state, err = altair.UpgradeToAltair(ctx, state)
 			if err != nil {
+				tracing.AnnotateError(span, err)
 				return nil, err
 			}
 		}
-		if prysmTime.CanUpgradeToBellatrix(state.Slot()) {
+
+		if prysmtime.CanUpgradeToBellatrix(state.Slot()) {
 			state, err = execution.UpgradeToBellatrix(ctx, state)
 			if err != nil {
+				tracing.AnnotateError(span, err)
 				return nil, err
 			}
 		}

beacon-chain/state/stategen/replayer.go

@@ -0,0 +1,387 @@
+package stategen
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
+	"github.com/sirupsen/logrus"
+	"go.opencensus.io/trace"
+)
+
+var ErrFutureSlotRequested = errors.New("cannot replay to future slots")
+var ErrNoCanonicalBlockForSlot = errors.New("none of the blocks found in the db slot index are canonical")
+var ErrNoBlocksBelowSlot = errors.New("no blocks found in db below slot")
+var ErrInvalidDBBlock = errors.New("invalid block found in database")
+
+// HistoryAccessor describes the minimum set of database methods needed to support the ReplayerBuilder.
+type HistoryAccessor interface {
+	HighestSlotBlocksBelow(ctx context.Context, slot types.Slot) ([]block.SignedBeaconBlock, error)
+	GenesisBlock(ctx context.Context) (block.SignedBeaconBlock, error)
+	Block(ctx context.Context, blockRoot [32]byte) (block.SignedBeaconBlock, error)
+	StateOrError(ctx context.Context, blockRoot [32]byte) (state.BeaconState, error)
+}
+
+// CanonicalChecker determines whether the given block root is canonical.
+// In practice this should be satisfied by a type that uses the fork choice store.
+type CanonicalChecker interface {
+	IsCanonical(ctx context.Context, blockRoot [32]byte) (bool, error)
+}
+
+// CurrentSlotter provides the current Slot.
+type CurrentSlotter interface {
+	CurrentSlot() types.Slot
+}
+
+// Replayer encapsulates database query and replay logic. It can be constructed via a StateReplayerBuilder.
+type Replayer interface {
+	// ReplayBlocks replays the blocks the Replayer knows about based on Builder params
+	ReplayBlocks(ctx context.Context) (state.BeaconState, error)
+	// ReplayToSlot invokes ReplayBlocks under the hood,
+	// but then also runs process_slots to advance the state past the root or slot used in the builder.
+	// For example, if you wanted the state to be at the target slot, but only integrating blocks up to
+	// slot-1, you could request Builder.ForSlot(slot-1).ReplayToSlot(slot)
+	ReplayToSlot(ctx context.Context, target types.Slot) (state.BeaconState, error)
+}
+
+var _ Replayer = &stateReplayer{}
+
+type stateReplayer struct {
+	s           state.BeaconState
+	descendants []block.SignedBeaconBlock
+	target      types.Slot
+	method      retrievalMethod
+	chainer     chainer
+}
+
+// ReplayBlocks applies all the blocks that were accumulated when building the Replayer.
+// This method relies on the correctness of the code that constructed the Replayer data.
+func (rs *stateReplayer) ReplayBlocks(ctx context.Context) (state.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "stateGen.stateReplayer.ReplayBlocks")
+	defer span.End()
+
+	var s state.BeaconState
+	var descendants []block.SignedBeaconBlock
+	var err error
+	switch rs.method {
+	case forSlot:
+		s, descendants, err = rs.chainer.chainForSlot(ctx, rs.target)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, errors.New("Replayer initialized using unknown state retrieval method")
+	}
+
+	start := time.Now()
+	diff, err := rs.target.SafeSubSlot(s.Slot())
+	if err != nil {
+		msg := fmt.Sprintf("error subtracting state.slot %d from replay target slot %d", s.Slot(), rs.target)
+		return nil, errors.Wrap(err, msg)
+	}
+	log.WithFields(logrus.Fields{
+		"startSlot": s.Slot(),
+		"endSlot":   rs.target,
+		"diff":      diff,
+	}).Debug("Replaying canonical blocks from most recent state")
+
+	for _, b := range descendants {
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+		s, err = executeStateTransitionStateGen(ctx, s, b)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if rs.target > s.Slot() {
+		s, err = ReplayProcessSlots(ctx, s, rs.target)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	duration := time.Since(start)
+	log.WithFields(logrus.Fields{
+		"duration": duration,
+	}).Debug("Finished calling process_blocks on all blocks in ReplayBlocks")
+	return s, nil
+}
+
+// ReplayToSlot invokes ReplayBlocks under the hood,
+// but then also runs process_slots to advance the state past the root or slot used in the builder.
+// for example, if you wanted the state to be at the target slot, but only integrating blocks up to
+// slot-1, you could request Builder.ForSlot(slot-1).ReplayToSlot(slot)
+func (rs *stateReplayer) ReplayToSlot(ctx context.Context, replayTo types.Slot) (state.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "stateGen.stateReplayer.ReplayToSlot")
+	defer span.End()
+
+	s, err := rs.ReplayBlocks(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to ReplayBlocks")
+	}
+
+	if replayTo > s.Slot() {
+		start := time.Now()
+		log.WithFields(logrus.Fields{
+			"startSlot": s.Slot(),
+			"endSlot":   replayTo,
+			"diff":      replayTo - s.Slot(),
+		}).Debug("calling process_slots on remaining slots")
+
+		if replayTo > s.Slot() {
+			// err will be handled after the bookend log
+			s, err = ReplayProcessSlots(ctx, s, replayTo)
+		}
+
+		duration := time.Since(start)
+		log.WithFields(logrus.Fields{
+			"duration": duration,
+		}).Debug("time spent in process_slots")
+		if err != nil {
+			return nil, errors.Wrap(err, fmt.Sprintf("ReplayToSlot failed to seek to slot %d after applying blocks", replayTo))
+		}
+	}
+	return s, nil
+}
+
+// ReplayerBuilder creates a Replayer that can be used to obtain a state at a specified slot or root
+// (only ForSlot implemented so far).
+// See documentation on Replayer for more on how to use this to obtain pre/post-block states
+type ReplayerBuilder interface {
+	// ForSlot creates a builder that will create a state that includes blocks up to and including the requested slot
+	// The resulting Replayer will always yield a state with .Slot=target; if there are skipped blocks
+	// between the highest canonical block in the db and the target, the replayer will fast-forward past the intervening
+	// slots via process_slots.
+	ForSlot(target types.Slot) Replayer
+}
+
+func WithCache(c CachedGetter) CanonicalBuilderOption {
+	return func(b *CanonicalBuilder) {
+		b.chainer.useCache(c)
+	}
+}
+
+type CanonicalBuilderOption func(*CanonicalBuilder)
+
+// NewCanonicalBuilder handles initializing the default concrete ReplayerBuilder implementation.
+func NewCanonicalBuilder(h HistoryAccessor, c CanonicalChecker, cs CurrentSlotter, opts ...CanonicalBuilderOption) *CanonicalBuilder {
+	b := &CanonicalBuilder{
+		chainer: &canonicalChainer{
+			h:  h,
+			c:  c,
+			cs: cs,
+		},
+	}
+	for _, o := range opts {
+		o(b)
+	}
+	return b
+}
+
+type retrievalMethod int
+
+const (
+	forSlot retrievalMethod = iota
+)
+
+// CanonicalBuilder builds a Replayer that uses a combination of database queries and a
+// CanonicalChecker (which should usually be a fork choice store implementing an IsCanonical method)
+// to determine the canonical chain and apply it to generate the desired state.
+type CanonicalBuilder struct {
+	chainer chainer
+}
+
+var _ ReplayerBuilder = &CanonicalBuilder{}
+
+func (r *CanonicalBuilder) ForSlot(target types.Slot) Replayer {
+	return &stateReplayer{chainer: r.chainer, method: forSlot, target: target}
+}
+
+// chainer is responsible for supplying the chain components necessary to rebuild a state,
+// namely a starting BeaconState and all available blocks from the starting state up to and including the target slot
+type chainer interface {
+	chainForSlot(ctx context.Context, target types.Slot) (state.BeaconState, []block.SignedBeaconBlock, error)
+	useCache(c CachedGetter)
+}
+
+type canonicalChainer struct {
+	h     HistoryAccessor
+	c     CanonicalChecker
+	cs    CurrentSlotter
+	cache CachedGetter
+}
+
+var _ chainer = &canonicalChainer{}
+
+func (c *canonicalChainer) useCache(cache CachedGetter) {
+	c.cache = cache
+}
+
+// ChainForSlot creates a value that satisfies the Replayer interface via db queries
+// and the stategen transition helper methods. This implementation uses the following algorithm:
+// - find the highest canonical block <= the target slot
+// - starting with this block, recursively search backwards for a stored state, and accumulate intervening blocks
+func (c *canonicalChainer) chainForSlot(ctx context.Context, target types.Slot) (state.BeaconState, []block.SignedBeaconBlock, error) {
+	ctx, span := trace.StartSpan(ctx, "canonicalChainer.chainForSlot")
+	defer span.End()
+	currentSlot := c.cs.CurrentSlot()
+	if target > currentSlot {
+		return nil, nil, errors.Wrap(ErrFutureSlotRequested, fmt.Sprintf("requested=%d, current=%d", target, currentSlot))
+	}
+	_, b, err := c.canonicalBlockForSlot(ctx, target)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, fmt.Sprintf("unable to find replay data for slot=%d", target))
+	}
+	s, descendants, err := c.ancestorChain(ctx, b)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to query for ancestor and descendant blocks")
+	}
+
+	return s, descendants, nil
+}
+
+// canonicalBlockForSlot uses HighestSlotBlocksBelow(target+1) and the CanonicalChecker
+// to find the highest canonical block available to replay to the given slot.
+func (c *canonicalChainer) canonicalBlockForSlot(ctx context.Context, target types.Slot) ([32]byte, block.SignedBeaconBlock, error) {
+	for target > 0 {
+		if ctx.Err() != nil {
+			return [32]byte{}, nil, errors.Wrap(ctx.Err(), "context canceled during canonicalBlockForSlot")
+		}
+		hbs, err := c.h.HighestSlotBlocksBelow(ctx, target+1)
+		if err != nil {
+			return [32]byte{}, nil, errors.Wrap(err, fmt.Sprintf("error finding highest block w/ slot <= %d", target))
+		}
+		if len(hbs) == 0 {
+			return [32]byte{}, nil, errors.Wrap(ErrNoBlocksBelowSlot, fmt.Sprintf("slot=%d", target))
+		}
+		r, b, err := c.bestForSlot(ctx, hbs)
+		if err == nil {
+			// we found a valid, canonical block!
+			return r, b, nil
+		}
+
+		// we found a block, but it wasn't considered canonical - keep looking
+		if errors.Is(err, ErrNoCanonicalBlockForSlot) {
+			// break once we've seen slot 0 (and prevent underflow)
+			if hbs[0].Block().Slot() == 0 {
+				break
+			}
+			target = hbs[0].Block().Slot() - 1
+			continue
+		}
+		return [32]byte{}, nil, err
+	}
+	b, err := c.h.GenesisBlock(ctx)
+	if err != nil {
+		return [32]byte{}, nil, errors.Wrap(err, "db error while retrieving genesis block")
+	}
+	root, _, err := c.bestForSlot(ctx, []block.SignedBeaconBlock{b})
+	if err != nil {
+		return [32]byte{}, nil, errors.Wrap(err, "problem retrieving genesis block")
+	}
+	return root, b, nil
+}
+
+// bestForSlot encapsulates several messy realities of the underlying db code, looping through multiple blocks,
+// performing null/validity checks, and using CanonicalChecker to only pick canonical blocks.
+func (c *canonicalChainer) bestForSlot(ctx context.Context, hbs []block.SignedBeaconBlock) ([32]byte, block.SignedBeaconBlock, error) {
+	for _, b := range hbs {
+		if helpers.BeaconBlockIsNil(b) != nil {
+			continue
+		}
+		root, err := b.Block().HashTreeRoot()
+		if err != nil {
+			// use this error message to wrap a sentinel error for error type matching
+			wrapped := errors.Wrap(ErrInvalidDBBlock, err.Error())
+			msg := fmt.Sprintf("could not compute hash_tree_root for block at slot=%d", b.Block().Slot())
+			return [32]byte{}, nil, errors.Wrap(wrapped, msg)
+		}
+		canon, err := c.c.IsCanonical(ctx, root)
+		if err != nil {
+			return [32]byte{}, nil, errors.Wrap(err, "replayer could not check if block is canonical")
+		}
+		if canon {
+			return root, b, nil
+		}
+	}
+	return [32]byte{}, nil, errors.Wrap(ErrNoCanonicalBlockForSlot, "no good block for slot")
+}
+
+func (c *canonicalChainer) getState(ctx context.Context, root [32]byte) (state.BeaconState, error) {
+	if c.cache != nil {
+		st, err := c.cache.ByRoot(root)
+		if err == nil {
+			return st, nil
+		}
+		if !errors.Is(err, ErrNotInCache) {
+			return nil, errors.Wrap(err, "error reading from state cache during state replay")
+		}
+	}
+	return c.h.StateOrError(ctx, root)
+}
+
+// ancestorChain works backwards through the chain lineage, accumulating blocks and checking for a saved state.
+// If it finds a saved state that the tail block was descended from, it returns this state and
+// all blocks in the lineage, including the tail block. Blocks are returned in ascending order.
+// Note that this function assumes that the tail is a canonical block, and therefore assumes that
+// all ancestors are also canonical.
+func (c *canonicalChainer) ancestorChain(ctx context.Context, tail block.SignedBeaconBlock) (state.BeaconState, []block.SignedBeaconBlock, error) {
+	ctx, span := trace.StartSpan(ctx, "canonicalChainer.ancestorChain")
+	defer span.End()
+	chain := make([]block.SignedBeaconBlock, 0)
+	for {
+		if err := ctx.Err(); err != nil {
+			msg := fmt.Sprintf("context canceled while finding ancestors of block at slot %d", tail.Block().Slot())
+			return nil, nil, errors.Wrap(err, msg)
+		}
+		b := tail.Block()
+		// compute hash_tree_root of current block and try to look up the corresponding state
+		root, err := b.HashTreeRoot()
+		if err != nil {
+			msg := fmt.Sprintf("could not compute htr for descendant block at slot=%d", b.Slot())
+			return nil, nil, errors.Wrap(err, msg)
+		}
+		st, err := c.getState(ctx, root)
+		// err == nil, we've got a real state - the job is done!
+		// Note: in cases where there are skipped slots we could find a state that is a descendant
+		// of the block we are searching for. We don't want to return a future block, so in this case
+		// we keep working backwards.
+		if err == nil && st.Slot() == b.Slot() {
+			// we found the state by the root of the head, meaning it has already been applied.
+			// we only want to return the blocks descended from it.
+			reverseChain(chain)
+			return st, chain, nil
+		}
+		// ErrNotFoundState errors are fine, but other errors mean something is wrong with the db
+		if err != nil && !errors.Is(err, db.ErrNotFoundState) {
+			return nil, nil, errors.Wrap(err, fmt.Sprintf("error querying database for state w/ block root = %#x", root))
+		}
+		parent, err := c.h.Block(ctx, bytesutil.ToBytes32(b.ParentRoot()))
+		if err != nil {
+			msg := fmt.Sprintf("db error when retrieving parent of block at slot=%d by root=%#x", b.Slot(), b.ParentRoot())
+			return nil, nil, errors.Wrap(err, msg)
+		}
+		if helpers.BeaconBlockIsNil(parent) != nil {
+			msg := fmt.Sprintf("unable to retrieve parent of block at slot=%d by root=%#x", b.Slot(), b.ParentRoot())
+			return nil, nil, errors.Wrap(db.ErrNotFound, msg)
+		}
+		chain = append(chain, tail)
+		tail = parent
+	}
+}
+
+func reverseChain(c []block.SignedBeaconBlock) {
+	last := len(c) - 1
+	swaps := (last + 1) / 2
+	for i := 0; i < swaps; i++ {
+		c[i], c[last-i] = c[last-i], c[i]
+	}
+}

beacon-chain/state/stategen/replayer_test.go

@@ -0,0 +1,716 @@
+package stategen
+
+import (
+	"context"
+	"encoding/binary"
+	"fmt"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+
+	"github.com/pkg/errors"
+	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block/mock"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
+	"github.com/prysmaticlabs/prysm/testing/require"
+)
+
+func headerFromBlock(b block.SignedBeaconBlock) (*ethpb.BeaconBlockHeader, error) {
+	bodyRoot, err := b.Block().Body().HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	return &ethpb.BeaconBlockHeader{
+		Slot:          b.Block().Slot(),
+		StateRoot:     b.Block().StateRoot(),
+		ProposerIndex: b.Block().ProposerIndex(),
+		BodyRoot:      bodyRoot[:],
+		ParentRoot:    b.Block().ParentRoot(),
+	}, nil
+}
+
+func TestReplayBlocks(t *testing.T) {
+	ctx := context.Background()
+	var zero, one, two, three, four, five types.Slot = 50, 51, 150, 151, 152, 200
+	specs := []mockHistorySpec{
+		{slot: zero},
+		{slot: one, savedState: true},
+		{slot: two},
+		{slot: three},
+		{slot: four},
+		{slot: five, canonicalBlock: true},
+	}
+
+	hist := newMockHistory(t, specs, five+1)
+	bld := NewCanonicalBuilder(hist, hist, hist)
+	st, err := bld.ForSlot(five).ReplayBlocks(ctx)
+	require.NoError(t, err)
+	expected := hist.hiddenStates[hist.slotMap[five]]
+	expectedHTR, err := expected.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualHTR, err := st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	expectedLBH := expected.LatestBlockHeader()
+	actualLBH := st.LatestBlockHeader()
+	require.Equal(t, expectedLBH.Slot, actualLBH.Slot)
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.ParentRoot), bytesutil.ToBytes32(actualLBH.ParentRoot))
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.StateRoot), bytesutil.ToBytes32(actualLBH.StateRoot))
+	require.Equal(t, expectedLBH.ProposerIndex, actualLBH.ProposerIndex)
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.BodyRoot), bytesutil.ToBytes32(actualLBH.BodyRoot))
+	require.Equal(t, expectedHTR, actualHTR)
+
+	st, err = bld.ForSlot(one).ReplayBlocks(ctx)
+	require.NoError(t, err)
+	expected = hist.states[hist.slotMap[one]]
+
+	// no canonical blocks in between, so latest block process_block_header will be for genesis
+	expectedLBH, err = headerFromBlock(hist.blocks[hist.slotMap[0]])
+	require.NoError(t, err)
+	actualLBH = st.LatestBlockHeader()
+	require.Equal(t, expectedLBH.Slot, actualLBH.Slot)
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.ParentRoot), bytesutil.ToBytes32(actualLBH.ParentRoot))
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.StateRoot), bytesutil.ToBytes32(actualLBH.StateRoot))
+	require.Equal(t, expectedLBH.ProposerIndex, actualLBH.ProposerIndex)
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.BodyRoot), bytesutil.ToBytes32(actualLBH.BodyRoot))
+
+	require.Equal(t, expected.Slot(), st.Slot())
+	// NOTE: HTR is not compared, because process_block is not called for non-canonical blocks,
+	// so there are multiple differences compared to the "db" state that applies all blocks
+}
+
+func TestReplayToSlot(t *testing.T) {
+	ctx := context.Background()
+	var zero, one, two, three, four, five types.Slot = 50, 51, 150, 151, 152, 200
+	specs := []mockHistorySpec{
+		{slot: zero},
+		{slot: one, savedState: true},
+		{slot: two},
+		{slot: three},
+		{slot: four},
+		{slot: five, canonicalBlock: true},
+	}
+
+	// first case tests that ReplayToSlot is equivalent to ReplayBlocks
+	hist := newMockHistory(t, specs, five+1)
+	bld := NewCanonicalBuilder(hist, hist, hist)
+
+	st, err := bld.ForSlot(five).ReplayToSlot(ctx, five)
+	require.NoError(t, err)
+	expected := hist.hiddenStates[hist.slotMap[five]]
+	expectedHTR, err := expected.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualHTR, err := st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	expectedLBH := expected.LatestBlockHeader()
+	actualLBH := st.LatestBlockHeader()
+	require.Equal(t, expectedLBH.Slot, actualLBH.Slot)
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.ParentRoot), bytesutil.ToBytes32(actualLBH.ParentRoot))
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.StateRoot), bytesutil.ToBytes32(actualLBH.StateRoot))
+	require.Equal(t, expectedLBH.ProposerIndex, actualLBH.ProposerIndex)
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.BodyRoot), bytesutil.ToBytes32(actualLBH.BodyRoot))
+	require.Equal(t, expectedHTR, actualHTR)
+
+	st, err = bld.ForSlot(five).ReplayToSlot(ctx, five+100)
+	require.NoError(t, err)
+	require.Equal(t, five+100, st.Slot())
+	expectedLBH, err = headerFromBlock(hist.blocks[hist.slotMap[five]])
+	require.NoError(t, err)
+	actualLBH = st.LatestBlockHeader()
+	require.Equal(t, expectedLBH.Slot, actualLBH.Slot)
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.ParentRoot), bytesutil.ToBytes32(actualLBH.ParentRoot))
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.StateRoot), bytesutil.ToBytes32(actualLBH.StateRoot))
+	require.Equal(t, expectedLBH.ProposerIndex, actualLBH.ProposerIndex)
+	require.Equal(t, bytesutil.ToBytes32(expectedLBH.BodyRoot), bytesutil.ToBytes32(actualLBH.BodyRoot))
+}
+
+// happy path tests
+func TestCanonicalBlockForSlotHappy(t *testing.T) {
+	ctx := context.Background()
+	var begin, middle, end types.Slot = 100, 150, 155
+	specs := []mockHistorySpec{
+		{slot: begin},
+		{slot: middle, savedState: true},
+		{slot: end, canonicalBlock: true},
+	}
+	hist := newMockHistory(t, specs, end+1)
+	cc := canonicalChainer{h: hist, c: hist, cs: hist}
+
+	// since only the end block and genesis are canonical, once the slot drops below
+	// end, we should always get genesis
+	cases := []struct {
+		slot    types.Slot
+		highest types.Slot
+		canon   types.Slot
+		name    string
+	}{
+		{slot: hist.current, highest: end, canon: end, name: "slot > end"},
+		{slot: end, highest: end, canon: end, name: "slot == end"},
+		{slot: end - 1, highest: middle, canon: 0, name: "middle < slot < end"},
+		{slot: middle, highest: middle, canon: 0, name: "slot == middle"},
+		{slot: middle - 1, highest: begin, canon: 0, name: "begin < slot < middle"},
+		{slot: begin, highest: begin, canon: 0, name: "slot == begin"},
+		{slot: begin - 1, highest: 0, canon: 0, name: "genesis < slot < begin"},
+		{slot: 0, highest: 0, canon: 0, name: "slot == genesis"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			bs, err := hist.HighestSlotBlocksBelow(ctx, c.slot+1)
+			require.NoError(t, err)
+			require.Equal(t, len(bs), 1)
+			r, err := bs[0].Block().HashTreeRoot()
+			require.NoError(t, err)
+			require.Equal(t, hist.slotMap[c.highest], r)
+			cr, _, err := cc.canonicalBlockForSlot(ctx, c.slot)
+			require.NoError(t, err)
+			require.Equal(t, hist.slotMap[c.canon], cr)
+		})
+	}
+}
+
+func TestCanonicalBlockForSlotNonHappy(t *testing.T) {
+	ctx := context.Background()
+	var begin, middle, end types.Slot = 100, 150, 155
+	specs := []mockHistorySpec{
+		{slot: begin},
+		{slot: middle, savedState: true},
+		{slot: end, canonicalBlock: true},
+	}
+	hist := newMockHistory(t, specs, end+1)
+
+	slotOrderObserved := make([]types.Slot, 0)
+	derp := errors.New("HighestSlotBlocksBelow don't work")
+	// since only the end block and genesis are canonical, once the slot drops below
+	// end, we should always get genesis
+	cases := []struct {
+		name              string
+		slot              types.Slot
+		canon             CanonicalChecker
+		overrideHighest   func(context.Context, types.Slot) ([]block.SignedBeaconBlock, error)
+		slotOrderExpected []types.Slot
+		err               error
+		root              [32]byte
+	}{
+		{
+			name: "HigestSlotBlocksBelow not called for genesis",
+			overrideHighest: func(_ context.Context, _ types.Slot) ([]block.SignedBeaconBlock, error) {
+				return nil, derp
+			},
+			root: hist.slotMap[0],
+		},
+		{
+			name: "wrapped error from HigestSlotBlocksBelow returned",
+			err:  derp,
+			overrideHighest: func(_ context.Context, _ types.Slot) ([]block.SignedBeaconBlock, error) {
+				return nil, derp
+			},
+			slot: end,
+		},
+		{
+			name: "HigestSlotBlocksBelow empty list",
+			err:  ErrNoBlocksBelowSlot,
+			overrideHighest: func(_ context.Context, _ types.Slot) ([]block.SignedBeaconBlock, error) {
+				return []block.SignedBeaconBlock{}, nil
+			},
+			slot: end,
+		},
+		{
+			name:  "HigestSlotBlocksBelow no canonical",
+			err:   ErrNoCanonicalBlockForSlot,
+			canon: &mockCanonicalChecker{is: false},
+			slot:  end,
+		},
+		{
+			name: "slot ordering correct - only genesis canonical",
+			canon: &mockCanonicalChecker{isCanon: func(root [32]byte) (bool, error) {
+				if root == hist.slotMap[0] {
+					return true, nil
+				}
+				return false, nil
+			}},
+			overrideHighest: func(_ context.Context, s types.Slot) ([]block.SignedBeaconBlock, error) {
+				slotOrderObserved = append(slotOrderObserved, s)
+				// this allows the mock HighestSlotBlocksBelow to continue to execute now that we've recorded
+				// the slot in our channel
+				return nil, errFallThroughOverride
+			},
+			slotOrderExpected: []types.Slot{156, 155, 150, 100},
+			slot:              end,
+			root:              hist.slotMap[0],
+		},
+		{
+			name: "slot ordering correct - slot 100 canonical",
+			canon: &mockCanonicalChecker{isCanon: func(root [32]byte) (bool, error) {
+				if root == hist.slotMap[100] {
+					return true, nil
+				}
+				return false, nil
+			}},
+			overrideHighest: func(_ context.Context, s types.Slot) ([]block.SignedBeaconBlock, error) {
+				slotOrderObserved = append(slotOrderObserved, s)
+				// this allows the mock HighestSlotBlocksBelow to continue to execute now that we've recorded
+				// the slot in our channel
+				return nil, errFallThroughOverride
+			},
+			slotOrderExpected: []types.Slot{156, 155, 150},
+			slot:              end,
+			root:              hist.slotMap[100],
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			var canon CanonicalChecker = hist
+			if c.canon != nil {
+				canon = c.canon
+			}
+			cc := canonicalChainer{h: hist, c: canon, cs: hist}
+			hist.overrideHighestSlotBlocksBelow = c.overrideHighest
+			r, _, err := cc.canonicalBlockForSlot(ctx, c.slot)
+			if c.err == nil {
+				require.NoError(t, err)
+			} else {
+				require.ErrorIs(t, err, c.err)
+			}
+			if len(c.slotOrderExpected) > 0 {
+				require.Equal(t, len(c.slotOrderExpected), len(slotOrderObserved), "HighestSlotBlocksBelow not called the expected number of times")
+				for i := range c.slotOrderExpected {
+					require.Equal(t, c.slotOrderExpected[i], slotOrderObserved[i])
+				}
+			}
+			if c.root != [32]byte{} {
+				require.Equal(t, c.root, r)
+			}
+			slotOrderObserved = make([]types.Slot, 0)
+		})
+	}
+}
+
+type mockCurrentSlotter struct {
+	Slot types.Slot
+}
+
+func (c *mockCurrentSlotter) CurrentSlot() types.Slot {
+	return c.Slot
+}
+
+var _ CurrentSlotter = &mockCurrentSlotter{}
+
+func TestCanonicalChainerFuture(t *testing.T) {
+	r := &canonicalChainer{
+		cs: &mockCurrentSlotter{Slot: 0},
+	}
+	_, _, err := r.chainForSlot(context.Background(), 1)
+	require.ErrorIs(t, err, ErrFutureSlotRequested)
+}
+
+func TestAncestorChainCache(t *testing.T) {
+	ctx := context.Background()
+	var begin, middle, end types.Slot = 100, 150, 155
+	specs := []mockHistorySpec{
+		{slot: begin, canonicalBlock: true},
+		{slot: middle, canonicalBlock: true},
+		{slot: end, canonicalBlock: true},
+	}
+	hist := newMockHistory(t, specs, end+1)
+	cc := &canonicalChainer{h: hist, c: hist, cs: hist}
+
+	// should only contain the genesis block
+	require.Equal(t, 1, len(hist.states))
+
+	endBlock := hist.blocks[hist.slotMap[end]]
+	st, bs, err := cc.ancestorChain(ctx, endBlock)
+	require.NoError(t, err)
+	require.Equal(t, 3, len(bs))
+	expectedHTR, err := hist.states[hist.slotMap[0]].HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualHTR, err := st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, expectedHTR, actualHTR)
+
+	// now populate the cache, we should get the cached state instead of genesis
+	cc.cache = &mockCachedGetter{
+		cache: map[[32]byte]state.BeaconState{
+			hist.slotMap[end]: hist.hiddenStates[hist.slotMap[end]],
+		},
+	}
+	st, bs, err = cc.ancestorChain(ctx, endBlock)
+	require.NoError(t, err)
+	require.Equal(t, 0, len(bs))
+	expectedHTR, err = hist.hiddenStates[hist.slotMap[end]].HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualHTR, err = st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, expectedHTR, actualHTR)
+
+	// populate cache with a different state for good measure
+	cc.cache = &mockCachedGetter{
+		cache: map[[32]byte]state.BeaconState{
+			hist.slotMap[begin]: hist.hiddenStates[hist.slotMap[begin]],
+		},
+	}
+	st, bs, err = cc.ancestorChain(ctx, endBlock)
+	require.NoError(t, err)
+	require.Equal(t, 2, len(bs))
+	expectedHTR, err = hist.hiddenStates[hist.slotMap[begin]].HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualHTR, err = st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, expectedHTR, actualHTR)
+
+	// rebuild history w/ last state saved, make sure we get that instead of cache
+	specs[2].savedState = true
+	hist = newMockHistory(t, specs, end+1)
+	cc = &canonicalChainer{h: hist, c: hist, cs: hist}
+	cc.cache = &mockCachedGetter{
+		cache: map[[32]byte]state.BeaconState{
+			hist.slotMap[begin]: hist.hiddenStates[hist.slotMap[begin]],
+		},
+	}
+	st, bs, err = cc.ancestorChain(ctx, endBlock)
+	require.NoError(t, err)
+	require.Equal(t, 0, len(bs))
+	expectedHTR, err = hist.states[hist.slotMap[end]].HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualHTR, err = st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, expectedHTR, actualHTR)
+}
+
+func TestAncestorChainOK(t *testing.T) {
+	ctx := context.Background()
+	var begin, middle, end types.Slot = 100, 150, 155
+	specs := []mockHistorySpec{
+		{slot: begin},
+		{slot: middle, savedState: true},
+		{slot: end, canonicalBlock: true},
+	}
+	hist := newMockHistory(t, specs, end+1)
+	cc := &canonicalChainer{h: hist, c: hist, cs: hist}
+
+	endBlock := hist.blocks[hist.slotMap[end]]
+	st, bs, err := cc.ancestorChain(ctx, endBlock)
+	require.NoError(t, err)
+
+	// middle is the most recent slot where savedState == true
+	require.Equal(t, 1, len(bs))
+	require.DeepEqual(t, endBlock, bs[0])
+	expectedHTR, err := hist.states[hist.slotMap[middle]].HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualHTR, err := st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, expectedHTR, actualHTR)
+
+	middleBlock := hist.blocks[hist.slotMap[middle]]
+	st, bs, err = cc.ancestorChain(ctx, middleBlock)
+	require.NoError(t, err)
+	actualHTR, err = st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, 0, len(bs))
+	require.Equal(t, expectedHTR, actualHTR)
+}
+
+func TestChainForSlot(t *testing.T) {
+	ctx := context.Background()
+	var zero, one, two, three types.Slot = 50, 51, 150, 151
+	specs := []mockHistorySpec{
+		{slot: zero, canonicalBlock: true, savedState: true},
+		{slot: one, canonicalBlock: true},
+		{slot: two},
+		{slot: three, canonicalBlock: true},
+	}
+	hist := newMockHistory(t, specs, three+10)
+	cc := &canonicalChainer{h: hist, c: hist, cs: hist}
+	firstNonGenesisRoot := hist.slotMap[zero]
+	nonGenesisStateRoot, err := hist.states[firstNonGenesisRoot].HashTreeRoot(ctx)
+	require.NoError(t, err)
+
+	cases := []struct {
+		name       string
+		slot       types.Slot
+		stateRoot  [32]byte
+		blockRoots [][32]byte
+	}{
+		{
+			name:       "above latest slot (but before current slot)",
+			slot:       three + 1,
+			stateRoot:  nonGenesisStateRoot,
+			blockRoots: [][32]byte{hist.slotMap[one], hist.slotMap[two], hist.slotMap[three]},
+		},
+		{
+			name:       "last canonical slot - two treated as canonical because it is parent of three",
+			slot:       three,
+			stateRoot:  nonGenesisStateRoot,
+			blockRoots: [][32]byte{hist.slotMap[one], hist.slotMap[two], hist.slotMap[three]},
+		},
+		{
+			name:       "non-canonical slot skipped",
+			slot:       two,
+			stateRoot:  nonGenesisStateRoot,
+			blockRoots: [][32]byte{hist.slotMap[one]},
+		},
+		{
+			name:       "first canonical slot",
+			slot:       one,
+			stateRoot:  nonGenesisStateRoot,
+			blockRoots: [][32]byte{hist.slotMap[one]},
+		},
+		{
+			name:       "slot at saved state",
+			slot:       zero,
+			stateRoot:  nonGenesisStateRoot,
+			blockRoots: [][32]byte{},
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			st, blocks, err := cc.chainForSlot(ctx, c.slot)
+			require.NoError(t, err)
+			actualStRoot, err := st.HashTreeRoot(ctx)
+			require.NoError(t, err)
+			require.Equal(t, c.stateRoot, actualStRoot)
+			require.Equal(t, len(c.blockRoots), len(blocks))
+			for i, b := range blocks {
+				root, err := b.Block().HashTreeRoot()
+				require.NoError(t, err)
+				require.Equal(t, c.blockRoots[i], root)
+			}
+		})
+	}
+}
+
+func TestAncestorChainOrdering(t *testing.T) {
+	ctx := context.Background()
+	var zero, one, two, three, four, five types.Slot = 50, 51, 150, 151, 152, 200
+	specs := []mockHistorySpec{
+		{slot: zero},
+		{slot: one, savedState: true},
+		{slot: two},
+		{slot: three},
+		{slot: four},
+		{slot: five},
+	}
+
+	hist := newMockHistory(t, specs, five+1)
+	endRoot := hist.slotMap[specs[len(specs)-1].slot]
+	endBlock := hist.blocks[endRoot]
+
+	cc := &canonicalChainer{h: hist, c: hist, cs: hist}
+	st, bs, err := cc.ancestorChain(ctx, endBlock)
+	require.NoError(t, err)
+	expectedRoot, err := hist.states[hist.slotMap[one]].HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualRoot, err := st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, expectedRoot, actualRoot)
+	// we asked for the chain leading up to five
+	// one has the savedState. one is applied to the savedState, so it should be omitted
+	// that means we should get two, three, four, five (length of 4)
+	require.Equal(t, 4, len(bs))
+	for i, slot := range []types.Slot{two, three, four, five} {
+		require.Equal(t, slot, bs[i].Block().Slot(), fmt.Sprintf("wrong value at index %d", i))
+	}
+
+	// do the same query, but with the final state saved
+	// we should just get the final state w/o block to apply
+	specs[5].savedState = true
+	hist = newMockHistory(t, specs, five+1)
+	endRoot = hist.slotMap[specs[len(specs)-1].slot]
+	endBlock = hist.blocks[endRoot]
+
+	cc = &canonicalChainer{h: hist, c: hist, cs: hist}
+	st, bs, err = cc.ancestorChain(ctx, endBlock)
+	require.NoError(t, err)
+	expectedRoot, err = hist.states[endRoot].HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualRoot, err = st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, expectedRoot, actualRoot)
+	require.Equal(t, 0, len(bs))
+
+	// slice off the last element for an odd size list (to cover odd/even in the reverseChain func)
+	specs = specs[:len(specs)-1]
+	require.Equal(t, 5, len(specs))
+	hist = newMockHistory(t, specs, five+1)
+
+	cc = &canonicalChainer{h: hist, c: hist, cs: hist}
+	endRoot = hist.slotMap[specs[len(specs)-1].slot]
+	endBlock = hist.blocks[endRoot]
+	st, bs, err = cc.ancestorChain(ctx, endBlock)
+	require.NoError(t, err)
+	expectedRoot, err = hist.states[hist.slotMap[one]].HashTreeRoot(ctx)
+	require.NoError(t, err)
+	actualRoot, err = st.HashTreeRoot(ctx)
+	require.NoError(t, err)
+	require.Equal(t, expectedRoot, actualRoot)
+	require.Equal(t, 3, len(bs))
+	for i, slot := range []types.Slot{two, three, four} {
+		require.Equal(t, slot, bs[i].Block().Slot(), fmt.Sprintf("wrong value at index %d", i))
+	}
+}
+
+func TestBestForSlot(t *testing.T) {
+	nilBlock, err := wrapper.WrappedSignedBeaconBlock(&ethpb.SignedBeaconBlock{})
+	require.NoError(t, err)
+	nilBody, err := wrapper.WrappedSignedBeaconBlock(&ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{}})
+	require.NoError(t, err)
+	derp := errors.New("fake hash tree root method no hash good")
+	badHTR := &mock.SignedBeaconBlock{BeaconBlock: &mock.BeaconBlock{HtrErr: derp, BeaconBlockBody: &mock.BeaconBlockBody{}}}
+	var goodHTR [32]byte
+	copy(goodHTR[:], []byte{23})
+	var betterHTR [32]byte
+	copy(betterHTR[:], []byte{42})
+	good := &mock.SignedBeaconBlock{BeaconBlock: &mock.BeaconBlock{BeaconBlockBody: &mock.BeaconBlockBody{}, Htr: goodHTR}}
+	better := &mock.SignedBeaconBlock{BeaconBlock: &mock.BeaconBlock{BeaconBlockBody: &mock.BeaconBlockBody{}, Htr: betterHTR}}
+
+	cases := []struct {
+		name   string
+		err    error
+		blocks []block.SignedBeaconBlock
+		best   block.SignedBeaconBlock
+		root   [32]byte
+		cc     CanonicalChecker
+	}{
+		{
+			name:   "empty list",
+			err:    ErrNoCanonicalBlockForSlot,
+			blocks: []block.SignedBeaconBlock{},
+		},
+		{
+			name:   "empty SignedBeaconBlock",
+			err:    ErrNoCanonicalBlockForSlot,
+			blocks: []block.SignedBeaconBlock{nil},
+		},
+		{
+			name:   "empty BeaconBlock",
+			err:    ErrNoCanonicalBlockForSlot,
+			blocks: []block.SignedBeaconBlock{nilBlock},
+		},
+		{
+			name:   "empty BeaconBlockBody",
+			err:    ErrNoCanonicalBlockForSlot,
+			blocks: []block.SignedBeaconBlock{nilBody},
+		},
+		{
+			name:   "bad HTR",
+			err:    ErrInvalidDBBlock,
+			blocks: []block.SignedBeaconBlock{badHTR},
+		},
+		{
+			name:   "IsCanonical fail",
+			blocks: []block.SignedBeaconBlock{good, better},
+			cc:     &mockCanonicalChecker{is: true, err: derp},
+			err:    derp,
+		},
+		{
+			name:   "all non-canonical",
+			err:    ErrNoCanonicalBlockForSlot,
+			blocks: []block.SignedBeaconBlock{good, better},
+			cc:     &mockCanonicalChecker{is: false},
+		},
+		{
+			name:   "one canonical",
+			blocks: []block.SignedBeaconBlock{good},
+			cc:     &mockCanonicalChecker{is: true},
+			root:   goodHTR,
+			best:   good,
+		},
+		{
+			name:   "all canonical",
+			blocks: []block.SignedBeaconBlock{better, good},
+			cc:     &mockCanonicalChecker{is: true},
+			root:   betterHTR,
+			best:   better,
+		},
+		{
+			name:   "first wins",
+			blocks: []block.SignedBeaconBlock{good, better},
+			cc:     &mockCanonicalChecker{is: true},
+			root:   goodHTR,
+			best:   good,
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			chk := CanonicalChecker(&mockCanonicalChecker{is: true})
+			if c.cc != nil {
+				chk = c.cc
+			}
+			cc := &canonicalChainer{c: chk}
+			r, b, err := cc.bestForSlot(context.Background(), c.blocks)
+			if c.err == nil {
+				require.NoError(t, err)
+				require.DeepEqual(t, c.best, b)
+				require.Equal(t, c.root, r)
+			} else {
+				require.ErrorIs(t, err, c.err)
+			}
+		})
+	}
+}
+
+type mockCanonicalChecker struct {
+	isCanon func([32]byte) (bool, error)
+	is      bool
+	err     error
+}
+
+func (m *mockCanonicalChecker) IsCanonical(_ context.Context, root [32]byte) (bool, error) {
+	if m.isCanon != nil {
+		return m.isCanon(root)
+	}
+	return m.is, m.err
+}
+
+func TestReverseChain(t *testing.T) {
+	// test 0,1,2,3 elements to handle: zero case; single element; even number; odd number
+	for i := 0; i < 4; i++ {
+		t.Run(fmt.Sprintf("reverseChain with %d elements", i), func(t *testing.T) {
+			actual := mockBlocks(i, incrFwd)
+			expected := mockBlocks(i, incrBwd)
+			reverseChain(actual)
+			if len(actual) != len(expected) {
+				t.Errorf("different list lengths")
+			}
+			for i := 0; i < len(actual); i++ {
+				sblockA, ok := actual[i].(*mock.SignedBeaconBlock)
+				require.Equal(t, true, ok)
+				blockA, ok := sblockA.BeaconBlock.(*mock.BeaconBlock)
+				require.Equal(t, true, ok)
+				sblockE, ok := expected[i].(*mock.SignedBeaconBlock)
+				require.Equal(t, true, ok)
+				blockE, ok := sblockE.BeaconBlock.(*mock.BeaconBlock)
+				require.Equal(t, true, ok)
+				require.Equal(t, blockA.Htr, blockE.Htr)
+			}
+		})
+	}
+}
+
+func incrBwd(n int, c chan uint32) {
+	for i := n - 1; i >= 0; i-- {
+		c <- uint32(i)
+	}
+	close(c)
+}
+
+func incrFwd(n int, c chan uint32) {
+	for i := 0; i < n; i++ {
+		c <- uint32(i)
+	}
+	close(c)
+}
+
+func mockBlocks(n int, iter func(int, chan uint32)) []block.SignedBeaconBlock {
+	bchan := make(chan uint32)
+	go iter(n, bchan)
+	mb := make([]block.SignedBeaconBlock, 0)
+	for i := range bchan {
+		h := [32]byte{}
+		binary.LittleEndian.PutUint32(h[:], i)
+		b := &mock.SignedBeaconBlock{BeaconBlock: &mock.BeaconBlock{BeaconBlockBody: &mock.BeaconBlockBody{}, Htr: h}}
+		mb = append(mb, b)
+	}
+	return mb
+}

beacon-chain/state/stateutil/BUILD.bazel

@@ -37,11 +37,11 @@ go_library(
         "//config/params:go_default_library",
         "//container/trie:go_default_library",
         "//crypto/hash:go_default_library",
+        "//crypto/hash/htr:go_default_library",
         "//encoding/bytesutil:go_default_library",
         "//encoding/ssz:go_default_library",
         "//math:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
-        "@com_github_dgraph_io_ristretto//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prysmaticlabs_eth2_types//:go_default_library",
         "@io_opencensus_go//trace:go_default_library",
@@ -53,20 +53,22 @@ go_test(
     srcs = [
         "benchmark_test.go",
         "field_root_test.go",
+        "field_root_validator_test.go",
         "reference_bench_test.go",
         "state_root_test.go",
-        "stateutil_test.go",
         "trie_helpers_test.go",
         "validator_root_test.go",
     ],
     embed = [":go_default_library"],
     deps = [
+        "//beacon-chain/state:go_default_library",
         "//config/features:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
         "//crypto/hash:go_default_library",
         "//encoding/bytesutil:go_default_library",
         "//encoding/ssz:go_default_library",
+        "//math:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//runtime/interop:go_default_library",
         "//testing/assert:go_default_library",

beacon-chain/state/stateutil/eth1_root.go

@@ -12,27 +12,6 @@ import (
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 )
 
-// eth1DataEncKey returns the encoded key in bytes of input `eth1Data`,
-// the returned key bytes can be used for caching purposes.
-func eth1DataEncKey(eth1Data *ethpb.Eth1Data) []byte {
-	enc := make([]byte, 0, 96)
-	if eth1Data != nil {
-		if len(eth1Data.DepositRoot) > 0 {
-			depRoot := bytesutil.ToBytes32(eth1Data.DepositRoot)
-			enc = append(enc, depRoot[:]...)
-		}
-		eth1DataCountBuf := make([]byte, 8)
-		binary.LittleEndian.PutUint64(eth1DataCountBuf, eth1Data.DepositCount)
-		eth1CountRoot := bytesutil.ToBytes32(eth1DataCountBuf)
-		enc = append(enc, eth1CountRoot[:]...)
-		if len(eth1Data.BlockHash) > 0 {
-			blockHash := bytesutil.ToBytes32(eth1Data.BlockHash)
-			enc = append(enc, blockHash[:]...)
-		}
-	}
-	return enc
-}
-
 // Eth1DataRootWithHasher returns the hash tree root of input `eth1Data`.
 func Eth1DataRootWithHasher(hasher ssz.HashFn, eth1Data *ethpb.Eth1Data) ([32]byte, error) {
 	if eth1Data == nil {
@@ -62,22 +41,6 @@ func Eth1DataRootWithHasher(hasher ssz.HashFn, eth1Data *ethpb.Eth1Data) ([32]by
 	return root, nil
 }
 
-// Eth1DatasEncKey returns the encoded key in bytes of input `eth1Data`s,
-// the returned key bytes can be used for caching purposes.
-func Eth1DatasEncKey(eth1Datas []*ethpb.Eth1Data) ([32]byte, error) {
-	hasher := hash.CustomSHA256Hasher()
-	enc := make([]byte, len(eth1Datas)*32)
-	for i := 0; i < len(eth1Datas); i++ {
-		eth1, err := Eth1DataRootWithHasher(hasher, eth1Datas[i])
-		if err != nil {
-			return [32]byte{}, errors.Wrap(err, "could not compute eth1data merkleization")
-		}
-		copy(enc[(i*32):(i+1)*32], eth1[:])
-	}
-	hashKey := hash.FastSum256(enc)
-	return hashKey, nil
-}
-
 // Eth1DatasRoot returns the hash tree root of input `eth1Datas`.
 func Eth1DatasRoot(eth1Datas []*ethpb.Eth1Data) ([32]byte, error) {
 	hasher := hash.CustomSHA256Hasher()

beacon-chain/state/stateutil/field_root_attestation.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/config/features"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/crypto/hash"
 	"github.com/prysmaticlabs/prysm/encoding/ssz"
@@ -15,14 +14,11 @@ import (
 
 // RootsArrayHashTreeRoot computes the Merkle root of arrays of 32-byte hashes, such as [64][32]byte
 // according to the Simple Serialize specification of Ethereum.
-func RootsArrayHashTreeRoot(vals [][]byte, length uint64, fieldName string) ([32]byte, error) {
-	if features.Get().EnableSSZCache {
-		return CachedHasher.arraysRoot(vals, length, fieldName)
-	}
-	return NocachedHasher.arraysRoot(vals, length, fieldName)
+func RootsArrayHashTreeRoot(vals [][]byte, length uint64) ([32]byte, error) {
+	return arraysRoot(vals, length)
 }
 
-func (h *stateRootHasher) epochAttestationsRoot(atts []*ethpb.PendingAttestation) ([32]byte, error) {
+func epochAttestationsRoot(atts []*ethpb.PendingAttestation) ([32]byte, error) {
 	max := uint64(fieldparams.CurrentEpochAttestationsLength)
 	if uint64(len(atts)) > max {
 		return [32]byte{}, fmt.Errorf("epoch attestation exceeds max length %d", max)
@@ -31,7 +27,7 @@ func (h *stateRootHasher) epochAttestationsRoot(atts []*ethpb.PendingAttestation
 	hasher := hash.CustomSHA256Hasher()
 	roots := make([][]byte, len(atts))
 	for i := 0; i < len(atts); i++ {
-		pendingRoot, err := h.pendingAttestationRoot(hasher, atts[i])
+		pendingRoot, err := pendingAttestationRoot(hasher, atts[i])
 		if err != nil {
 			return [32]byte{}, errors.Wrap(err, "could not attestation merkleization")
 		}
@@ -58,26 +54,9 @@ func (h *stateRootHasher) epochAttestationsRoot(atts []*ethpb.PendingAttestation
 	return res, nil
 }
 
-func (h *stateRootHasher) pendingAttestationRoot(hasher ssz.HashFn, att *ethpb.PendingAttestation) ([32]byte, error) {
+func pendingAttestationRoot(hasher ssz.HashFn, att *ethpb.PendingAttestation) ([32]byte, error) {
 	if att == nil {
 		return [32]byte{}, errors.New("nil pending attestation")
 	}
-	// Marshal attestation to determine if it exists in the cache.
-	enc := pendingAttEncKey(att)
-
-	// Check if it exists in cache:
-	if h.rootsCache != nil {
-		if found, ok := h.rootsCache.Get(string(enc)); found != nil && ok {
-			return found.([32]byte), nil
-		}
-	}
-
-	res, err := PendingAttRootWithHasher(hasher, att)
-	if err != nil {
-		return [32]byte{}, err
-	}
-	if h.rootsCache != nil {
-		h.rootsCache.Set(string(enc), res, 32)
-	}
-	return res, nil
+	return PendingAttRootWithHasher(hasher, att)
 }

beacon-chain/state/stateutil/field_root_eth1.go

@@ -2,7 +2,6 @@ package stateutil
 
 import (
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/config/features"
 	"github.com/prysmaticlabs/prysm/encoding/ssz"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 )
@@ -14,45 +13,12 @@ func Eth1Root(hasher ssz.HashFn, eth1Data *ethpb.Eth1Data) ([32]byte, error) {
 	if eth1Data == nil {
 		return [32]byte{}, errors.New("nil eth1 data")
 	}
-
-	enc := eth1DataEncKey(eth1Data)
-	if features.Get().EnableSSZCache {
-		if found, ok := CachedHasher.rootsCache.Get(string(enc)); ok && found != nil {
-			return found.([32]byte), nil
-		}
-	}
-
-	root, err := Eth1DataRootWithHasher(hasher, eth1Data)
-	if err != nil {
-		return [32]byte{}, err
-	}
-
-	if features.Get().EnableSSZCache {
-		CachedHasher.rootsCache.Set(string(enc), root, 32)
-	}
-	return root, nil
+	return Eth1DataRootWithHasher(hasher, eth1Data)
 }
 
 // eth1DataVotesRoot computes the HashTreeRoot Merkleization of
 // a list of Eth1Data structs according to the eth2
 // Simple Serialize specification.
 func eth1DataVotesRoot(eth1DataVotes []*ethpb.Eth1Data) ([32]byte, error) {
-	hashKey, err := Eth1DatasEncKey(eth1DataVotes)
-	if err != nil {
-		return [32]byte{}, err
-	}
-
-	if features.Get().EnableSSZCache {
-		if found, ok := CachedHasher.rootsCache.Get(string(hashKey[:])); ok && found != nil {
-			return found.([32]byte), nil
-		}
-	}
-	root, err := Eth1DatasRoot(eth1DataVotes)
-	if err != nil {
-		return [32]byte{}, err
-	}
-	if features.Get().EnableSSZCache {
-		CachedHasher.rootsCache.Set(string(hashKey[:]), root, 32)
-	}
-	return root, nil
+	return Eth1DatasRoot(eth1DataVotes)
 }

beacon-chain/state/stateutil/field_root_test.go

@@ -7,17 +7,17 @@ import (
 )
 
 func TestArraysTreeRoot_OnlyPowerOf2(t *testing.T) {
-	_, err := NocachedHasher.arraysRoot([][]byte{}, 1, "testing")
+	_, err := arraysRoot([][]byte{}, 1)
 	assert.NoError(t, err)
-	_, err = NocachedHasher.arraysRoot([][]byte{}, 4, "testing")
+	_, err = arraysRoot([][]byte{}, 4)
 	assert.NoError(t, err)
-	_, err = NocachedHasher.arraysRoot([][]byte{}, 8, "testing")
+	_, err = arraysRoot([][]byte{}, 8)
 	assert.NoError(t, err)
-	_, err = NocachedHasher.arraysRoot([][]byte{}, 10, "testing")
+	_, err = arraysRoot([][]byte{}, 10)
 	assert.ErrorContains(t, "hash layer is a non power of 2", err)
 }
 
 func TestArraysTreeRoot_ZeroLength(t *testing.T) {
-	_, err := NocachedHasher.arraysRoot([][]byte{}, 0, "testing")
+	_, err := arraysRoot([][]byte{}, 0)
 	assert.ErrorContains(t, "zero leaves provided", err)
 }

beacon-chain/state/stateutil/field_root_validator.go

@@ -8,40 +8,43 @@ import (
 	"github.com/prysmaticlabs/prysm/config/features"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/crypto/hash"
+	"github.com/prysmaticlabs/prysm/crypto/hash/htr"
 	"github.com/prysmaticlabs/prysm/encoding/ssz"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 )
 
+const (
+	// number of field roots for the validator object.
+	validatorFieldRoots = 8
+
+	// Depth of tree representation of an individual
+	// validator.
+	// NumOfRoots = 2 ^ (TreeDepth)
+	// 8 = 2 ^ 3
+	validatorTreeDepth = 3
+)
+
 // ValidatorRegistryRoot computes the HashTreeRoot Merkleization of
 // a list of validator structs according to the Ethereum
 // Simple Serialize specification.
 func ValidatorRegistryRoot(vals []*ethpb.Validator) ([32]byte, error) {
-	if features.Get().EnableSSZCache {
-		return CachedHasher.validatorRegistryRoot(vals)
-	}
-	return NocachedHasher.validatorRegistryRoot(vals)
+	return validatorRegistryRoot(vals)
 }
 
-func (h *stateRootHasher) validatorRegistryRoot(validators []*ethpb.Validator) ([32]byte, error) {
-	hashKeyElements := make([]byte, len(validators)*32)
-	roots := make([][32]byte, len(validators))
-	emptyKey := hash.FastSum256(hashKeyElements)
+func validatorRegistryRoot(validators []*ethpb.Validator) ([32]byte, error) {
 	hasher := hash.CustomSHA256Hasher()
-	bytesProcessed := 0
-	for i := 0; i < len(validators); i++ {
-		val, err := h.validatorRoot(hasher, validators[i])
-		if err != nil {
-			return [32]byte{}, errors.Wrap(err, "could not compute validators merkleization")
-		}
-		copy(hashKeyElements[bytesProcessed:bytesProcessed+32], val[:])
-		roots[i] = val
-		bytesProcessed += 32
-	}
 
-	hashKey := hash.FastSum256(hashKeyElements)
-	if hashKey != emptyKey && h.rootsCache != nil {
-		if found, ok := h.rootsCache.Get(string(hashKey[:])); found != nil && ok {
-			return found.([32]byte), nil
+	var err error
+	var roots [][32]byte
+	if features.Get().EnableVectorizedHTR {
+		roots, err = optimizedValidatorRoots(validators)
+		if err != nil {
+			return [32]byte{}, err
+		}
+	} else {
+		roots, err = validatorRoots(hasher, validators)
+		if err != nil {
+			return [32]byte{}, err
 		}
 	}
 
@@ -57,32 +60,49 @@ func (h *stateRootHasher) validatorRegistryRoot(validators []*ethpb.Validator) (
 	var validatorsRootsBufRoot [32]byte
 	copy(validatorsRootsBufRoot[:], validatorsRootsBuf.Bytes())
 	res := ssz.MixInLength(validatorsRootsRoot, validatorsRootsBufRoot[:])
-	if hashKey != emptyKey && h.rootsCache != nil {
-		h.rootsCache.Set(string(hashKey[:]), res, 32)
-	}
+
 	return res, nil
 }
 
-func (h *stateRootHasher) validatorRoot(hasher ssz.HashFn, validator *ethpb.Validator) ([32]byte, error) {
+func validatorRoots(hasher func([]byte) [32]byte, validators []*ethpb.Validator) ([][32]byte, error) {
+	roots := make([][32]byte, len(validators))
+	for i := 0; i < len(validators); i++ {
+		val, err := validatorRoot(hasher, validators[i])
+		if err != nil {
+			return [][32]byte{}, errors.Wrap(err, "could not compute validators merkleization")
+		}
+		roots[i] = val
+	}
+	return roots, nil
+}
+
+func optimizedValidatorRoots(validators []*ethpb.Validator) ([][32]byte, error) {
+	roots := make([][32]byte, 0, len(validators)*validatorFieldRoots)
+	hasher := hash.CustomSHA256Hasher()
+	for i := 0; i < len(validators); i++ {
+		fRoots, err := ValidatorFieldRoots(hasher, validators[i])
+		if err != nil {
+			return [][32]byte{}, errors.Wrap(err, "could not compute validators merkleization")
+		}
+		roots = append(roots, fRoots...)
+	}
+
+	// A validator's tree can represented with a depth of 3. As log2(8) = 3
+	// Using this property we can lay out all the individual fields of a
+	// validator and hash them in single level using our vectorized routine.
+	for i := 0; i < validatorTreeDepth; i++ {
+		// Overwrite input lists as we are hashing by level
+		// and only need the highest level to proceed.
+		outputLen := len(roots) / 2
+		htr.VectorizedSha256(roots, roots)
+		roots = roots[:outputLen]
+	}
+	return roots, nil
+}
+
+func validatorRoot(hasher ssz.HashFn, validator *ethpb.Validator) ([32]byte, error) {
 	if validator == nil {
 		return [32]byte{}, errors.New("nil validator")
 	}
-
-	enc := validatorEncKey(validator)
-	// Check if it exists in cache:
-	if h.rootsCache != nil {
-		if found, ok := h.rootsCache.Get(string(enc)); found != nil && ok {
-			return found.([32]byte), nil
-		}
-	}
-
-	valRoot, err := ValidatorRootWithHasher(hasher, validator)
-	if err != nil {
-		return [32]byte{}, err
-	}
-
-	if h.rootsCache != nil {
-		h.rootsCache.Set(string(enc), valRoot, 32)
-	}
-	return valRoot, nil
+	return ValidatorRootWithHasher(hasher, validator)
 }

beacon-chain/state/stateutil/field_root_validator_test.go

@@ -0,0 +1,32 @@
+package stateutil
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	mathutil "github.com/prysmaticlabs/prysm/math"
+	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/testing/assert"
+)
+
+func TestValidatorConstants(t *testing.T) {
+	v := &ethpb.Validator{}
+	refV := reflect.ValueOf(v).Elem()
+	numFields := refV.NumField()
+	numOfValFields := 0
+
+	for i := 0; i < numFields; i++ {
+		if strings.Contains(refV.Type().Field(i).Name, "state") ||
+			strings.Contains(refV.Type().Field(i).Name, "sizeCache") ||
+			strings.Contains(refV.Type().Field(i).Name, "unknownFields") {
+			continue
+		}
+		numOfValFields++
+	}
+	assert.Equal(t, validatorFieldRoots, numOfValFields)
+	assert.Equal(t, uint64(validatorFieldRoots), mathutil.PowerOf2(validatorTreeDepth))
+
+	_, err := ValidatorRegistryRoot([]*ethpb.Validator{v})
+	assert.NoError(t, err)
+}

beacon-chain/state/stateutil/field_root_vector.go

@@ -6,129 +6,23 @@ import (
 	"github.com/prysmaticlabs/prysm/encoding/ssz"
 )
 
-func (h *stateRootHasher) arraysRoot(input [][]byte, length uint64, fieldName string) ([32]byte, error) {
-	lock.Lock()
-	defer lock.Unlock()
+func arraysRoot(input [][]byte, length uint64) ([32]byte, error) {
 	hashFunc := hash.CustomSHA256Hasher()
-	if _, ok := layersCache[fieldName]; !ok && h.rootsCache != nil {
-		depth := ssz.Depth(length)
-		layersCache[fieldName] = make([][][32]byte, depth+1)
-	}
 
 	leaves := make([][32]byte, length)
 	for i, chunk := range input {
 		copy(leaves[i][:], chunk)
 	}
-	bytesProcessed := 0
-	changedIndices := make([]int, 0)
-	prevLeaves, ok := leavesCache[fieldName]
-	if len(prevLeaves) == 0 || h.rootsCache == nil {
-		prevLeaves = leaves
-	}
-	// Exit early if our previous leaves length don't match with the current set.
-	// This should never happen but better to be defensive here.
-	if len(prevLeaves) != len(leaves) {
-		res, err := h.merkleizeWithCache(leaves, length, fieldName, hashFunc)
-		if err != nil {
-			return [32]byte{}, err
-		}
-		if h.rootsCache != nil {
-			leavesCache[fieldName] = leaves
-		}
-		return res, nil
-	}
-	for i := 0; i < len(leaves); i++ {
-		// We check if any items changed since the roots were last recomputed.
-		notEqual := leaves[i] != prevLeaves[i]
-		if ok && h.rootsCache != nil && notEqual {
-			changedIndices = append(changedIndices, i)
-		}
-		bytesProcessed += 32
-	}
-	if len(changedIndices) > 0 && h.rootsCache != nil {
-		var rt [32]byte
-		var err error
-		// If indices did change since last computation, we only recompute
-		// the modified branches in the cached Merkle tree for this state field.
-		chunks := leaves
-
-		// We need to ensure we recompute indices of the Merkle tree which
-		// changed in-between calls to this function. This check adds an offset
-		// to the recomputed indices to ensure we do so evenly.
-		maxChangedIndex := changedIndices[len(changedIndices)-1]
-		if maxChangedIndex+2 == len(chunks) && maxChangedIndex%2 != 0 {
-			changedIndices = append(changedIndices, maxChangedIndex+1)
-		}
-		for i := 0; i < len(changedIndices); i++ {
-			rt, err = recomputeRoot(changedIndices[i], chunks, fieldName, hashFunc)
-			if err != nil {
-				return [32]byte{}, err
-			}
-		}
-		leavesCache[fieldName] = chunks
-		return rt, nil
-	}
-
-	res, err := h.merkleizeWithCache(leaves, length, fieldName, hashFunc)
+	res, err := merkleize(leaves, length, hashFunc)
 	if err != nil {
 		return [32]byte{}, err
 	}
-	if h.rootsCache != nil {
-		leavesCache[fieldName] = leaves
-	}
+
 	return res, nil
 }
 
-func recomputeRoot(idx int, chunks [][32]byte, fieldName string, hasher func([]byte) [32]byte) ([32]byte, error) {
-	items, ok := layersCache[fieldName]
-	if !ok {
-		return [32]byte{}, errors.New("could not recompute root as there was no cache found")
-	}
-	if items == nil {
-		return [32]byte{}, errors.New("could not recompute root as there were no items found in the layers cache")
-	}
-	layers := items
-	root := chunks[idx]
-	layers[0] = chunks
-	// The merkle tree structure looks as follows:
-	// [[r1, r2, r3, r4], [parent1, parent2], [root]]
-	// Using information about the index which changed, idx, we recompute
-	// only its branch up the tree.
-	currentIndex := idx
-	for i := 0; i < len(layers)-1; i++ {
-		isLeft := currentIndex%2 == 0
-		neighborIdx := currentIndex ^ 1
-
-		neighbor := [32]byte{}
-		if layers[i] != nil && len(layers[i]) != 0 && neighborIdx < len(layers[i]) {
-			neighbor = layers[i][neighborIdx]
-		}
-		if isLeft {
-			parentHash := hasher(append(root[:], neighbor[:]...))
-			root = parentHash
-		} else {
-			parentHash := hasher(append(neighbor[:], root[:]...))
-			root = parentHash
-		}
-		parentIdx := currentIndex / 2
-		// Update the cached layers at the parent index.
-		if len(layers[i+1]) == 0 {
-			layers[i+1] = append(layers[i+1], root)
-		} else {
-			layers[i+1][parentIdx] = root
-		}
-		currentIndex = parentIdx
-	}
-	layersCache[fieldName] = layers
-	// If there is only a single leaf, we return it (the identity element).
-	if len(layers[0]) == 1 {
-		return layers[0][0], nil
-	}
-	return root, nil
-}
-
-func (h *stateRootHasher) merkleizeWithCache(leaves [][32]byte, length uint64,
-	fieldName string, hasher func([]byte) [32]byte) ([32]byte, error) {
+func merkleize(leaves [][32]byte, length uint64,
+	hasher func([]byte) [32]byte) ([32]byte, error) {
 	if len(leaves) == 0 {
 		return [32]byte{}, errors.New("zero leaves provided")
 	}
@@ -137,20 +31,12 @@ func (h *stateRootHasher) merkleizeWithCache(leaves [][32]byte, length uint64,
 	}
 	hashLayer := leaves
 	layers := make([][][32]byte, ssz.Depth(length)+1)
-	if items, ok := layersCache[fieldName]; ok && h.rootsCache != nil {
-		if len(items[0]) == len(leaves) {
-			layers = items
-		}
-	}
 	layers[0] = hashLayer
 	var err error
-	layers, hashLayer, err = MerkleizeTrieLeaves(layers, hashLayer, hasher)
+	_, hashLayer, err = MerkleizeTrieLeaves(layers, hashLayer, hasher)
 	if err != nil {
 		return [32]byte{}, err
 	}
 	root := hashLayer[0]
-	if h.rootsCache != nil {
-		layersCache[fieldName] = layers
-	}
 	return root, nil
 }

beacon-chain/state/stateutil/pending_attestation_root.go

@@ -40,29 +40,6 @@ func PendingAttRootWithHasher(hasher ssz.HashFn, att *ethpb.PendingAttestation)
 	return ssz.BitwiseMerkleizeArrays(hasher, fieldRoots, uint64(len(fieldRoots)), uint64(len(fieldRoots)))
 }
 
-// pendingAttEncKey returns the encoded key in bytes of input `pendingAttestation`,
-// the returned key bytes can be used for caching purposes.
-func pendingAttEncKey(att *ethpb.PendingAttestation) []byte {
-	enc := make([]byte, 2192)
-
-	if att != nil {
-		copy(enc[0:2048], att.AggregationBits)
-
-		inclusionBuf := make([]byte, 8)
-		binary.LittleEndian.PutUint64(inclusionBuf, uint64(att.InclusionDelay))
-		copy(enc[2048:2056], inclusionBuf)
-
-		attDataBuf := marshalAttData(att.Data)
-		copy(enc[2056:2184], attDataBuf)
-
-		proposerBuf := make([]byte, 8)
-		binary.LittleEndian.PutUint64(proposerBuf, uint64(att.ProposerIndex))
-		copy(enc[2184:2192], proposerBuf)
-	}
-
-	return enc
-}
-
 func attDataRootWithHasher(hasher ssz.HashFn, data *ethpb.AttestationData) ([32]byte, error) {
 	fieldRoots := make([][]byte, 5)
 
@@ -100,39 +77,3 @@ func attDataRootWithHasher(hasher ssz.HashFn, data *ethpb.AttestationData) ([32]
 
 	return ssz.BitwiseMerkleize(hasher, fieldRoots, uint64(len(fieldRoots)), uint64(len(fieldRoots)))
 }
-
-func marshalAttData(data *ethpb.AttestationData) []byte {
-	enc := make([]byte, 128)
-
-	if data != nil {
-		// Slot.
-		slotBuf := make([]byte, 8)
-		binary.LittleEndian.PutUint64(slotBuf, uint64(data.Slot))
-		copy(enc[0:8], slotBuf)
-
-		// Committee index.
-		indexBuf := make([]byte, 8)
-		binary.LittleEndian.PutUint64(indexBuf, uint64(data.CommitteeIndex))
-		copy(enc[8:16], indexBuf)
-
-		copy(enc[16:48], data.BeaconBlockRoot)
-
-		// Source epoch and root.
-		if data.Source != nil {
-			sourceEpochBuf := make([]byte, 8)
-			binary.LittleEndian.PutUint64(sourceEpochBuf, uint64(data.Source.Epoch))
-			copy(enc[48:56], sourceEpochBuf)
-			copy(enc[56:88], data.Source.Root)
-		}
-
-		// Target.
-		if data.Target != nil {
-			targetEpochBuf := make([]byte, 8)
-			binary.LittleEndian.PutUint64(targetEpochBuf, uint64(data.Target.Epoch))
-			copy(enc[88:96], targetEpochBuf)
-			copy(enc[96:128], data.Target.Root)
-		}
-	}
-
-	return enc
-}