save progress

* process the optimistic blocks whose parent are optimistic too

* adding unit test

* fix test

Co-authored-by: terence tsao <terence@prysmaticlabs.com>
Co-authored-by: Potuz <potuz@prysmaticlabs.com>

.github/actions/gomodtidy/entrypoint.sh

@@ -1,8 +1,8 @@
 #!/bin/sh -l
 set -e
-export PATH=$PATH:/usr/local/go/bin
+export PATH="$PATH:/usr/local/go/bin"
 
-cd $GITHUB_WORKSPACE
+cd "$GITHUB_WORKSPACE"
 
 cp go.mod go.mod.orig
 cp go.sum go.sum.orig

.github/workflows/dappnode-release-trigger.yml

@@ -1,41 +0,0 @@
-name: Update DAppNodePackages
-
-on:
-  push:
-    tags:
-    - '*'
-
-jobs:
-  dappnode-update-beacon-chain:
-    name: Trigger a beacon-chain release
-    runs-on: ubuntu-latest
-    steps:
-    - name: Get latest tag
-      id: get_tag
-      run: echo ::set-output name=TAG::${GITHUB_REF/refs\/tags\//}
-    - name: Send dispatch event to DAppNodePackage-prysm-beacon-chain
-      env: 
-        DISPATCH_REPO: dappnode/DAppNodePackage-prysm-beacon-chain
-      run: |
-        curl -v -X POST -u "${{ secrets.PAT_GITHUB }}" \
-        -H "Accept: application/vnd.github.everest-preview+json" \
-        -H "Content-Type: application/json" \
-        --data '{"event_type":"new_release", "client_payload": { "tag":"${{ steps.get_tag.outputs.TAG }}"}}' \
-        https://api.github.com/repos/$DISPATCH_REPO/dispatches
-        
-  dappnode-update-validator:
-    name: Trigger a validator release
-    runs-on: ubuntu-latest
-    steps:
-    - name: Get latest tag
-      id: get_tag
-      run: echo ::set-output name=TAG::${GITHUB_REF/refs\/tags\//}
-    - name: Send dispatch event to DAppNodePackage validator repository
-      env: 
-        DISPATCH_REPO: dappnode/DAppNodePackage-prysm-validator
-      run: |
-        curl -v -X POST -u "${{ secrets.PAT_GITHUB }}" \
-        -H "Accept: application/vnd.github.everest-preview+json" \
-        -H "Content-Type: application/json" \
-        --data '{"event_type":"new_release", "client_payload": { "tag":"${{ steps.get_tag.outputs.TAG }}"}}' \
-        https://api.github.com/repos/$DISPATCH_REPO/dispatches

.github/workflows/go.yml

@@ -7,13 +7,12 @@ on:
     branches: [ '*' ]
 
 jobs:
-
-  check:
-    name: Check
+  formatting:
+    name: Formatting
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
 
       - name: Go mod tidy checker
         id: gomodtidy
@@ -31,15 +30,43 @@ jobs:
         with:
           goimports-path: ./
 
-      - name: Gosec security scanner
-        uses: securego/gosec@master
+  gosec:
+    name: Gosec scan
+    runs-on: ubuntu-latest
+    env:
+      GO111MODULE: on
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v3
         with:
-          args: '-exclude=G307 -exclude-dir=crypto/bls/herumi ./...'
+          go-version: 1.17
+      - name: Run Gosec Security Scanner
+        run: | # https://github.com/securego/gosec/issues/469
+          export PATH=$PATH:$(go env GOPATH)/bin
+          go install github.com/securego/gosec/v2/cmd/gosec@latest
+          gosec -exclude=G307 -exclude-dir=crypto/bls/herumi ./...
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.17
+          id: go
 
       - name: Golangci-lint
         uses: golangci/golangci-lint-action@v2
         with:
-          args: --print-issued-lines --sort-results --no-config --timeout=10m --disable-all -E deadcode  -E errcheck -E gosimple --skip-files=validator/web/site_data.go --skip-dirs=proto
+          args: --print-issued-lines --sort-results --no-config --timeout=10m --disable-all -E deadcode  -E errcheck -E gosimple --skip-files=validator/web/site_data.go --skip-dirs=proto --go=1.17
+          version: v1.45.2
+          skip-go-installation: true
 
   build:
     name: Build
@@ -48,7 +75,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v2
         with:
-          go-version: ^1.14
+          go-version: 1.17
         id: go
 
       - name: Check out code into the Go module directory

api/client/beacon/client.go

@@ -132,6 +132,15 @@ func validHostname(h string) (string, error) {
 	return fmt.Sprintf("%s:%s", host, port), nil
 }
 
+// NodeURL returns a human-readable string representation of the beacon node base url.
+func (c *Client) NodeURL() string {
+	u := &url.URL{
+		Scheme: c.scheme,
+		Host:   c.host,
+	}
+	return u.String()
+}
+
 func (c *Client) urlForPath(methodPath string) *url.URL {
 	u := &url.URL{
 		Scheme: c.scheme,

beacon-chain/blockchain/BUILD.bazel

@@ -57,7 +57,6 @@ go_library(
         "//beacon-chain/operations/voluntaryexits:go_default_library",
         "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
-        "//beacon-chain/powchain/engine-api-client/v1:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/stategen:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",
@@ -107,7 +106,6 @@ go_test(
         "init_test.go",
         "log_test.go",
         "metrics_test.go",
-        "mock_engine_test.go",
         "mock_test.go",
         "optimistic_sync_test.go",
         "pow_block_test.go",
@@ -131,6 +129,7 @@ go_test(
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
+        "//beacon-chain/powchain/testing:go_default_library",
         "//beacon-chain/state/stateutil:go_default_library",
         "//beacon-chain/state/v1:go_default_library",
         "//config/fieldparams:go_default_library",
@@ -184,6 +183,7 @@ go_test(
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
+        "//beacon-chain/powchain/testing:go_default_library",
         "//config/params:go_default_library",
         "//encoding/bytesutil:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",

beacon-chain/blockchain/chain_info.go

@@ -362,7 +362,18 @@ func (s *Service) IsOptimisticForRoot(ctx context.Context, root [32]byte) (bool,
 	if err != nil {
 		return false, err
 	}
-	return ss.Slot > lastValidated.Slot, nil
+
+	if ss.Slot > lastValidated.Slot {
+		return true, nil
+	}
+
+	isCanonical, err := s.IsCanonical(ctx, root)
+	if err != nil {
+		return false, err
+	}
+
+	// historical non-canonical blocks here are returned as optimistic for safety.
+	return !isCanonical, nil
 }
 
 // SetGenesisTime sets the genesis time of beacon chain.

beacon-chain/blockchain/chain_info_test.go

@@ -459,7 +459,7 @@ func TestService_IsOptimisticForRoot_DB_ProtoArray(t *testing.T) {
 	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: br[:], Slot: 10}))
 
 	optimisticBlock := util.NewBeaconBlock()
-	optimisticBlock.Block.Slot = 11
+	optimisticBlock.Block.Slot = 97
 	optimisticRoot, err := optimisticBlock.Block.HashTreeRoot()
 	require.NoError(t, err)
 	wsb, err = wrapper.WrappedSignedBeaconBlock(optimisticBlock)
@@ -483,12 +483,19 @@ func TestService_IsOptimisticForRoot_DB_ProtoArray(t *testing.T) {
 	require.Equal(t, true, optimistic)
 
 	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: validatedRoot[:], Slot: 9}))
+	cp := &ethpb.Checkpoint{
+		Epoch: 1,
+		Root:  validatedRoot[:],
+	}
+	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, validatedRoot))
+	require.NoError(t, beaconDB.SaveFinalizedCheckpoint(ctx, cp))
+
 	validated, err := c.IsOptimisticForRoot(ctx, validatedRoot)
 	require.NoError(t, err)
 	require.Equal(t, false, validated)
 }
 
-func TestService_IsOptimisticForRoot__DB_DoublyLinkedTree(t *testing.T) {
+func TestService_IsOptimisticForRoot_DB_DoublyLinkedTree(t *testing.T) {
 	beaconDB := testDB.SetupDB(t)
 	ctx := context.Background()
 	c := &Service{cfg: &config{BeaconDB: beaconDB, ForkChoiceStore: doublylinkedtree.New(0, 0)}, head: &head{slot: 101, root: [32]byte{'b'}}}
@@ -503,7 +510,57 @@ func TestService_IsOptimisticForRoot__DB_DoublyLinkedTree(t *testing.T) {
 	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: br[:], Slot: 10}))
 
 	optimisticBlock := util.NewBeaconBlock()
-	optimisticBlock.Block.Slot = 11
+	optimisticBlock.Block.Slot = 97
+	optimisticRoot, err := optimisticBlock.Block.HashTreeRoot()
+	require.NoError(t, err)
+	wsb, err = wrapper.WrappedSignedBeaconBlock(optimisticBlock)
+	require.NoError(t, err)
+	require.NoError(t, beaconDB.SaveBlock(context.Background(), wsb))
+
+	validatedBlock := util.NewBeaconBlock()
+	validatedBlock.Block.Slot = 9
+	validatedRoot, err := validatedBlock.Block.HashTreeRoot()
+	require.NoError(t, err)
+	wsb, err = wrapper.WrappedSignedBeaconBlock(validatedBlock)
+	require.NoError(t, err)
+	require.NoError(t, beaconDB.SaveBlock(context.Background(), wsb))
+
+	validatedCheckpoint := &ethpb.Checkpoint{Root: br[:]}
+	require.NoError(t, beaconDB.SaveLastValidatedCheckpoint(ctx, validatedCheckpoint))
+
+	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: optimisticRoot[:], Slot: 11}))
+	optimistic, err := c.IsOptimisticForRoot(ctx, optimisticRoot)
+	require.NoError(t, err)
+	require.Equal(t, true, optimistic)
+
+	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: validatedRoot[:], Slot: 9}))
+	cp := &ethpb.Checkpoint{
+		Epoch: 1,
+		Root:  validatedRoot[:],
+	}
+	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, validatedRoot))
+	require.NoError(t, beaconDB.SaveFinalizedCheckpoint(ctx, cp))
+	validated, err := c.IsOptimisticForRoot(ctx, validatedRoot)
+	require.NoError(t, err)
+	require.Equal(t, false, validated)
+}
+
+func TestService_IsOptimisticForRoot_DB_non_canonical(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+	ctx := context.Background()
+	c := &Service{cfg: &config{BeaconDB: beaconDB, ForkChoiceStore: doublylinkedtree.New(0, 0)}, head: &head{slot: 101, root: [32]byte{'b'}}}
+	c.head = &head{root: params.BeaconConfig().ZeroHash}
+	b := util.NewBeaconBlock()
+	b.Block.Slot = 10
+	br, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	wsb, err := wrapper.WrappedSignedBeaconBlock(b)
+	require.NoError(t, err)
+	require.NoError(t, beaconDB.SaveBlock(context.Background(), wsb))
+	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: br[:], Slot: 10}))
+
+	optimisticBlock := util.NewBeaconBlock()
+	optimisticBlock.Block.Slot = 97
 	optimisticRoot, err := optimisticBlock.Block.HashTreeRoot()
 	require.NoError(t, err)
 	wsb, err = wrapper.WrappedSignedBeaconBlock(optimisticBlock)
@@ -529,5 +586,6 @@ func TestService_IsOptimisticForRoot__DB_DoublyLinkedTree(t *testing.T) {
 	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: validatedRoot[:], Slot: 9}))
 	validated, err := c.IsOptimisticForRoot(ctx, validatedRoot)
 	require.NoError(t, err)
-	require.Equal(t, false, validated)
+	require.Equal(t, true, validated)
+
 }

beacon-chain/blockchain/error.go

@@ -16,4 +16,6 @@ var (
 	// errWrongBlockCount is returned when the wrong number of blocks or
 	// block roots is used
 	errWrongBlockCount = errors.New("wrong number of blocks or block roots")
+	// block is not a valid optimistic candidate block
+	errNotOptimisticCandidate = errors.New("block is not suitable for optimistic sync")
 )

beacon-chain/blockchain/head.go

@@ -24,8 +24,9 @@ import (
 	"go.opencensus.io/trace"
 )
 
-// UpdateHeadWithBalances updates the beacon state head after getting justified balanced from cache.
-func (s *Service) UpdateHeadWithBalances(ctx context.Context) error {
+// UpdateAndSaveHeadWithBalances updates the beacon state head after getting justified balanced from cache.
+// This function is only used in spec-tests, it does save the head after updating it.
+func (s *Service) UpdateAndSaveHeadWithBalances(ctx context.Context) error {
 	cp := s.store.JustifiedCheckpt()
 	if cp == nil {
 		return errors.New("no justified checkpoint")
@@ -35,8 +36,11 @@ func (s *Service) UpdateHeadWithBalances(ctx context.Context) error {
 		msg := fmt.Sprintf("could not read balances for state w/ justified checkpoint %#x", cp.Root)
 		return errors.Wrap(err, msg)
 	}
-
-	return s.updateHead(ctx, balances)
+	headRoot, err := s.updateHead(ctx, balances)
+	if err != nil {
+		return errors.Wrap(err, "could not update head")
+	}
+	return s.saveHead(ctx, headRoot)
 }
 
 // This defines the current chain service's view of head.
@@ -49,18 +53,18 @@ type head struct {
 
 // Determined the head from the fork choice service and saves its new data
 // (head root, head block, and head state) to the local service cache.
-func (s *Service) updateHead(ctx context.Context, balances []uint64) error {
+func (s *Service) updateHead(ctx context.Context, balances []uint64) ([32]byte, error) {
 	ctx, span := trace.StartSpan(ctx, "blockChain.updateHead")
 	defer span.End()
 
 	// Get head from the fork choice service.
 	f := s.store.FinalizedCheckpt()
 	if f == nil {
-		return errNilFinalizedInStore
+		return [32]byte{}, errNilFinalizedInStore
 	}
 	j := s.store.JustifiedCheckpt()
 	if j == nil {
-		return errNilJustifiedInStore
+		return [32]byte{}, errNilJustifiedInStore
 	}
 	// To get head before the first justified epoch, the fork choice will start with origin root
 	// instead of zero hashes.
@@ -76,7 +80,7 @@ func (s *Service) updateHead(ctx context.Context, balances []uint64) error {
 	if !s.cfg.ForkChoiceStore.HasNode(headStartRoot) {
 		jb, err := s.cfg.BeaconDB.Block(ctx, headStartRoot)
 		if err != nil {
-			return err
+			return [32]byte{}, err
 		}
 		if features.Get().EnableForkChoiceDoublyLinkedTree {
 			s.cfg.ForkChoiceStore = doublylinkedtree.New(j.Epoch, f.Epoch)
@@ -84,17 +88,11 @@ func (s *Service) updateHead(ctx context.Context, balances []uint64) error {
 			s.cfg.ForkChoiceStore = protoarray.New(j.Epoch, f.Epoch, bytesutil.ToBytes32(f.Root))
 		}
 		if err := s.insertBlockToForkChoiceStore(ctx, jb.Block(), headStartRoot, f, j); err != nil {
-			return err
+			return [32]byte{}, err
 		}
 	}
 
-	headRoot, err := s.cfg.ForkChoiceStore.Head(ctx, j.Epoch, headStartRoot, balances, f.Epoch)
-	if err != nil {
-		return err
-	}
-
-	// Save head to the local service cache.
-	return s.saveHead(ctx, headRoot)
+	return s.cfg.ForkChoiceStore.Head(ctx, j.Epoch, headStartRoot, balances, f.Epoch)
 }
 
 // This saves head info to the local service cache, it also saves the

beacon-chain/blockchain/head_test.go

@@ -9,6 +9,8 @@ import (
 	types "github.com/prysmaticlabs/eth2-types"
 	mock "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	doublylinkedtree "github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/doubly-linked-tree"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	"github.com/prysmaticlabs/prysm/config/features"
 	"github.com/prysmaticlabs/prysm/config/params"
 	ethpbv1 "github.com/prysmaticlabs/prysm/proto/eth/v1"
@@ -154,8 +156,9 @@ func TestUpdateHead_MissingJustifiedRoot(t *testing.T) {
 	service.store.SetJustifiedCheckpt(&ethpb.Checkpoint{Root: r[:]})
 	service.store.SetFinalizedCheckpt(&ethpb.Checkpoint{})
 	service.store.SetBestJustifiedCheckpt(&ethpb.Checkpoint{})
-
-	require.NoError(t, service.updateHead(context.Background(), []uint64{}))
+	headRoot, err := service.updateHead(context.Background(), []uint64{})
+	require.NoError(t, err)
+	require.NoError(t, service.saveHead(context.Background(), headRoot))
 }
 
 func Test_notifyNewHeadEvent(t *testing.T) {
@@ -279,3 +282,43 @@ func TestSaveOrphanedAtts_CanFilter(t *testing.T) {
 	atts := b.Block.Body.Attestations
 	require.DeepNotSSZEqual(t, atts, savedAtts)
 }
+
+func TestUpdateHead_noSavedChanges(t *testing.T) {
+	ctx := context.Background()
+
+	beaconDB := testDB.SetupDB(t)
+	fcs := doublylinkedtree.New(0, 0)
+	opts := []Option{
+		WithDatabase(beaconDB),
+		WithStateGen(stategen.New(beaconDB)),
+		WithForkChoiceStore(fcs),
+	}
+
+	service, err := NewService(ctx, opts...)
+	require.NoError(t, err)
+
+	bellatrixBlk, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlockBellatrix())
+	require.NoError(t, err)
+	bellatrixBlkRoot, err := bellatrixBlk.Block().HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, beaconDB.SaveBlock(ctx, bellatrixBlk))
+	fcp := &ethpb.Checkpoint{
+		Root:  bellatrixBlkRoot[:],
+		Epoch: 1,
+	}
+	service.store.SetFinalizedCheckpt(fcp)
+	service.store.SetJustifiedCheckpt(fcp)
+	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, bellatrixBlkRoot))
+
+	bellatrixState, _ := util.DeterministicGenesisStateBellatrix(t, 2)
+	require.NoError(t, beaconDB.SaveState(ctx, bellatrixState, bellatrixBlkRoot))
+	service.cfg.StateGen.SaveFinalizedState(0, bellatrixBlkRoot, bellatrixState)
+
+	headRoot := service.headRoot()
+	require.Equal(t, [32]byte{}, headRoot)
+
+	newRoot, err := service.updateHead(ctx, []uint64{1, 2})
+	require.NoError(t, err)
+	require.NotEqual(t, headRoot, newRoot)
+	require.Equal(t, headRoot, service.headRoot())
+}

beacon-chain/blockchain/mock_engine_test.go

@@ -1,49 +0,0 @@
-package blockchain
-
-import (
-	"context"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/pkg/errors"
-	enginev1 "github.com/prysmaticlabs/prysm/proto/engine/v1"
-)
-
-type mockEngineService struct {
-	newPayloadError error
-	forkchoiceError error
-	blks            map[[32]byte]*enginev1.ExecutionBlock
-}
-
-func (m *mockEngineService) NewPayload(context.Context, *enginev1.ExecutionPayload) ([]byte, error) {
-	return nil, m.newPayloadError
-}
-
-func (m *mockEngineService) ForkchoiceUpdated(context.Context, *enginev1.ForkchoiceState, *enginev1.PayloadAttributes) (*enginev1.PayloadIDBytes, []byte, error) {
-	return nil, nil, m.forkchoiceError
-}
-
-func (*mockEngineService) GetPayloadV1(
-	_ context.Context, _ enginev1.PayloadIDBytes,
-) *enginev1.ExecutionPayload {
-	return nil
-}
-
-func (*mockEngineService) GetPayload(context.Context, [8]byte) (*enginev1.ExecutionPayload, error) {
-	return nil, nil
-}
-
-func (*mockEngineService) ExchangeTransitionConfiguration(context.Context, *enginev1.TransitionConfiguration) error {
-	return nil
-}
-
-func (*mockEngineService) LatestExecutionBlock(context.Context) (*enginev1.ExecutionBlock, error) {
-	return nil, nil
-}
-
-func (m *mockEngineService) ExecutionBlockByHash(_ context.Context, hash common.Hash) (*enginev1.ExecutionBlock, error) {
-	blk, ok := m.blks[common.BytesToHash(hash.Bytes())]
-	if !ok {
-		return nil, errors.New("block not found")
-	}
-	return blk, nil
-}

beacon-chain/blockchain/optimistic_sync.go

@@ -7,7 +7,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	v1 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
 	"github.com/prysmaticlabs/prysm/config/params"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 	enginev1 "github.com/prysmaticlabs/prysm/proto/engine/v1"
@@ -64,7 +64,7 @@ func (s *Service) notifyForkchoiceUpdate(ctx context.Context, headBlk block.Beac
 	payloadID, _, err := s.cfg.ExecutionEngineCaller.ForkchoiceUpdated(ctx, fcs, nil /*payload attribute*/)
 	if err != nil {
 		switch err {
-		case v1.ErrAcceptedSyncingPayloadStatus:
+		case powchain.ErrAcceptedSyncingPayloadStatus:
 			log.WithFields(logrus.Fields{
 				"headSlot":      headBlk.Slot(),
 				"headHash":      fmt.Sprintf("%#x", bytesutil.Trunc(headPayload.BlockHash)),
@@ -81,64 +81,61 @@ func (s *Service) notifyForkchoiceUpdate(ctx context.Context, headBlk block.Beac
 	return payloadID, nil
 }
 
-// notifyForkchoiceUpdate signals execution engine on a new payload
+// notifyForkchoiceUpdate signals execution engine on a new payload.
+// It returns true if the EL has returned VALID for the block
 func (s *Service) notifyNewPayload(ctx context.Context, preStateVersion, postStateVersion int,
-	preStateHeader, postStateHeader *ethpb.ExecutionPayloadHeader, blk block.SignedBeaconBlock, root [32]byte) error {
+	preStateHeader, postStateHeader *ethpb.ExecutionPayloadHeader, blk block.SignedBeaconBlock) (bool, error) {
 	ctx, span := trace.StartSpan(ctx, "blockChain.notifyNewPayload")
 	defer span.End()
 
 	// Execution payload is only supported in Bellatrix and beyond. Pre
 	// merge blocks are never optimistic
 	if blocks.IsPreBellatrixVersion(postStateVersion) {
-		return s.cfg.ForkChoiceStore.SetOptimisticToValid(ctx, root)
+		return true, nil
 	}
 	if err := helpers.BeaconBlockIsNil(blk); err != nil {
-		return err
+		return false, err
 	}
 	body := blk.Block().Body()
 	enabled, err := blocks.IsExecutionEnabledUsingHeader(postStateHeader, body)
 	if err != nil {
-		return errors.Wrap(err, "could not determine if execution is enabled")
+		return false, errors.Wrap(err, "could not determine if execution is enabled")
 	}
 	if !enabled {
-		return s.cfg.ForkChoiceStore.SetOptimisticToValid(ctx, root)
+		return true, nil
 	}
 	payload, err := body.ExecutionPayload()
 	if err != nil {
-		return errors.Wrap(err, "could not get execution payload")
+		return false, errors.Wrap(err, "could not get execution payload")
 	}
 	_, err = s.cfg.ExecutionEngineCaller.NewPayload(ctx, payload)
 	if err != nil {
 		switch err {
-		case v1.ErrAcceptedSyncingPayloadStatus:
+		case powchain.ErrAcceptedSyncingPayloadStatus:
 			log.WithFields(logrus.Fields{
 				"slot":      blk.Block().Slot(),
 				"blockHash": fmt.Sprintf("%#x", bytesutil.Trunc(payload.BlockHash)),
 			}).Info("Called new payload with optimistic block")
-			return nil
+			return false, nil
 		default:
-			return errors.Wrap(err, "could not validate execution payload from execution engine")
+			return false, errors.Wrap(err, "could not validate execution payload from execution engine")
 		}
 	}
 
-	if err := s.cfg.ForkChoiceStore.SetOptimisticToValid(ctx, root); err != nil {
-		return errors.Wrap(err, "could not set optimistic status")
-	}
-
 	// During the transition event, the transition block should be verified for sanity.
 	if blocks.IsPreBellatrixVersion(preStateVersion) {
 		// Handle case where pre-state is Altair but block contains payload.
 		// To reach here, the block must have contained a valid payload.
-		return s.validateMergeBlock(ctx, blk)
+		return true, s.validateMergeBlock(ctx, blk)
 	}
 	atTransition, err := blocks.IsMergeTransitionBlockUsingPreStatePayloadHeader(preStateHeader, body)
 	if err != nil {
-		return errors.Wrap(err, "could not check if merge block is terminal")
+		return true, errors.Wrap(err, "could not check if merge block is terminal")
 	}
 	if !atTransition {
-		return nil
+		return true, nil
 	}
-	return s.validateMergeBlock(ctx, blk)
+	return true, s.validateMergeBlock(ctx, blk)
 }
 
 // optimisticCandidateBlock returns true if this block can be optimistically synced.
@@ -148,10 +145,6 @@ func (s *Service) notifyNewPayload(ctx context.Context, preStateVersion, postSta
 //    if is_execution_block(opt_store.blocks[block.parent_root]):
 //        return True
 //
-//    justified_root = opt_store.block_states[opt_store.head_block_root].current_justified_checkpoint.root
-//    if is_execution_block(opt_store.blocks[justified_root]):
-//        return True
-//
 //    if block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot:
 //        return True
 //
@@ -173,17 +166,5 @@ func (s *Service) optimisticCandidateBlock(ctx context.Context, blk block.Beacon
 	if err != nil {
 		return false, err
 	}
-	if parentIsExecutionBlock {
-		return true, nil
-	}
-
-	j := s.store.JustifiedCheckpt()
-	if j == nil {
-		return false, errNilJustifiedInStore
-	}
-	jBlock, err := s.cfg.BeaconDB.Block(ctx, bytesutil.ToBytes32(j.Root))
-	if err != nil {
-		return false, err
-	}
-	return blocks.IsExecutionBlock(jBlock.Block().Body())
+	return parentIsExecutionBlock, nil
 }

beacon-chain/blockchain/optimistic_sync_test.go

@@ -8,7 +8,8 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
-	engine "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
+	mockPOW "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
@@ -133,7 +134,7 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 				require.NoError(t, err)
 				return b
 			}(),
-			newForkchoiceErr: engine.ErrAcceptedSyncingPayloadStatus,
+			newForkchoiceErr: powchain.ErrAcceptedSyncingPayloadStatus,
 			finalizedRoot:    bellatrixBlkRoot,
 		},
 		{
@@ -147,7 +148,7 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 				require.NoError(t, err)
 				return b
 			}(),
-			newForkchoiceErr: engine.ErrInvalidPayloadStatus,
+			newForkchoiceErr: powchain.ErrInvalidPayloadStatus,
 			finalizedRoot:    bellatrixBlkRoot,
 			errString:        "could not notify forkchoice update from execution engine: payload status is INVALID",
 		},
@@ -155,7 +156,7 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			service.cfg.ExecutionEngineCaller = &mockEngineService{forkchoiceError: tt.newForkchoiceErr}
+			service.cfg.ExecutionEngineCaller = &mockPOW.EngineClient{ErrForkchoiceUpdated: tt.newForkchoiceErr}
 			_, err := service.notifyForkchoiceUpdate(ctx, tt.blk, service.headRoot(), tt.finalizedRoot)
 			if tt.errString != "" {
 				require.ErrorContains(t, tt.errString, err)
@@ -202,49 +203,56 @@ func Test_NotifyNewPayload(t *testing.T) {
 	require.NoError(t, err)
 
 	tests := []struct {
-		name          string
-		preState      state.BeaconState
-		postState     state.BeaconState
-		blk           block.SignedBeaconBlock
-		newPayloadErr error
-		errString     string
+		name           string
+		preState       state.BeaconState
+		postState      state.BeaconState
+		isValidPayload bool
+		blk            block.SignedBeaconBlock
+		newPayloadErr  error
+		errString      string
 	}{
 		{
-			name:      "phase 0 post state",
-			postState: phase0State,
-			preState:  phase0State,
+			name:           "phase 0 post state",
+			postState:      phase0State,
+			preState:       phase0State,
+			isValidPayload: true,
 		},
 		{
-			name:      "altair post state",
-			postState: altairState,
-			preState:  altairState,
+			name:           "altair post state",
+			postState:      altairState,
+			preState:       altairState,
+			isValidPayload: true,
 		},
 		{
-			name:      "nil beacon block",
-			postState: bellatrixState,
-			preState:  bellatrixState,
-			errString: "signed beacon block can't be nil",
+			name:           "nil beacon block",
+			postState:      bellatrixState,
+			preState:       bellatrixState,
+			errString:      "signed beacon block can't be nil",
+			isValidPayload: false,
 		},
 		{
-			name:          "new payload with optimistic block",
-			postState:     bellatrixState,
-			preState:      bellatrixState,
-			blk:           bellatrixBlk,
-			newPayloadErr: engine.ErrAcceptedSyncingPayloadStatus,
+			name:           "new payload with optimistic block",
+			postState:      bellatrixState,
+			preState:       bellatrixState,
+			blk:            bellatrixBlk,
+			newPayloadErr:  powchain.ErrAcceptedSyncingPayloadStatus,
+			isValidPayload: false,
 		},
 		{
-			name:          "new payload with invalid block",
-			postState:     bellatrixState,
-			preState:      bellatrixState,
-			blk:           bellatrixBlk,
-			newPayloadErr: engine.ErrInvalidPayloadStatus,
-			errString:     "could not validate execution payload from execution engine: payload status is INVALID",
+			name:           "new payload with invalid block",
+			postState:      bellatrixState,
+			preState:       bellatrixState,
+			blk:            bellatrixBlk,
+			newPayloadErr:  powchain.ErrInvalidPayloadStatus,
+			errString:      "could not validate execution payload from execution engine: payload status is INVALID",
+			isValidPayload: false,
 		},
 		{
-			name:      "altair pre state, altair block",
-			postState: bellatrixState,
-			preState:  altairState,
-			blk:       altairBlk,
+			name:           "altair pre state, altair block",
+			postState:      bellatrixState,
+			preState:       altairState,
+			blk:            altairBlk,
+			isValidPayload: true,
 		},
 		{
 			name:      "altair pre state, happy case",
@@ -264,13 +272,15 @@ func Test_NotifyNewPayload(t *testing.T) {
 				require.NoError(t, err)
 				return b
 			}(),
+			isValidPayload: true,
 		},
 		{
-			name:      "could not get merge block",
-			postState: bellatrixState,
-			preState:  bellatrixState,
-			blk:       bellatrixBlk,
-			errString: "could not get merge block parent hash and total difficulty",
+			name:           "could not get merge block",
+			postState:      bellatrixState,
+			preState:       bellatrixState,
+			blk:            bellatrixBlk,
+			errString:      "could not get merge block parent hash and total difficulty",
+			isValidPayload: false,
 		},
 		{
 			name:      "not at merge transition",
@@ -297,13 +307,15 @@ func Test_NotifyNewPayload(t *testing.T) {
 				require.NoError(t, err)
 				return b
 			}(),
+			isValidPayload: true,
 		},
 		{
-			name:      "could not get merge block",
-			postState: bellatrixState,
-			preState:  bellatrixState,
-			blk:       bellatrixBlk,
-			errString: "could not get merge block parent hash and total difficulty",
+			name:           "could not get merge block",
+			postState:      bellatrixState,
+			preState:       bellatrixState,
+			blk:            bellatrixBlk,
+			errString:      "could not get merge block parent hash and total difficulty",
+			isValidPayload: false,
 		},
 		{
 			name:      "happy case",
@@ -323,16 +335,17 @@ func Test_NotifyNewPayload(t *testing.T) {
 				require.NoError(t, err)
 				return b
 			}(),
+			isValidPayload: true,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			e := &mockEngineService{newPayloadError: tt.newPayloadErr, blks: map[[32]byte]*v1.ExecutionBlock{}}
-			e.blks[[32]byte{'a'}] = &v1.ExecutionBlock{
+			e := &mockPOW.EngineClient{ErrNewPayload: tt.newPayloadErr, BlockByHashMap: map[[32]byte]*v1.ExecutionBlock{}}
+			e.BlockByHashMap[[32]byte{'a'}] = &v1.ExecutionBlock{
 				ParentHash:      bytesutil.PadTo([]byte{'b'}, fieldparams.RootLength),
 				TotalDifficulty: "0x2",
 			}
-			e.blks[[32]byte{'b'}] = &v1.ExecutionBlock{
+			e.BlockByHashMap[[32]byte{'b'}] = &v1.ExecutionBlock{
 				ParentHash:      bytesutil.PadTo([]byte{'3'}, fieldparams.RootLength),
 				TotalDifficulty: "0x1",
 			}
@@ -346,11 +359,12 @@ func Test_NotifyNewPayload(t *testing.T) {
 			require.NoError(t, service.cfg.ForkChoiceStore.InsertOptimisticBlock(ctx, 0, root, root, params.BeaconConfig().ZeroHash, 0, 0))
 			postVersion, postHeader, err := getStateVersionAndPayload(tt.postState)
 			require.NoError(t, err)
-			err = service.notifyNewPayload(ctx, tt.preState.Version(), postVersion, payload, postHeader, tt.blk, root)
+			isValidPayload, err := service.notifyNewPayload(ctx, tt.preState.Version(), postVersion, payload, postHeader, tt.blk)
 			if tt.errString != "" {
 				require.ErrorContains(t, tt.errString, err)
 			} else {
 				require.NoError(t, err)
+				require.Equal(t, tt.isValidPayload, isValidPayload)
 			}
 		})
 	}
@@ -382,27 +396,23 @@ func Test_NotifyNewPayload_SetOptimisticToValid(t *testing.T) {
 	require.NoError(t, err)
 	service, err := NewService(ctx, opts...)
 	require.NoError(t, err)
-	e := &mockEngineService{blks: map[[32]byte]*v1.ExecutionBlock{}}
-	e.blks[[32]byte{'a'}] = &v1.ExecutionBlock{
+	e := &mockPOW.EngineClient{BlockByHashMap: map[[32]byte]*v1.ExecutionBlock{}}
+	e.BlockByHashMap[[32]byte{'a'}] = &v1.ExecutionBlock{
 		ParentHash:      bytesutil.PadTo([]byte{'b'}, fieldparams.RootLength),
 		TotalDifficulty: "0x2",
 	}
-	e.blks[[32]byte{'b'}] = &v1.ExecutionBlock{
+	e.BlockByHashMap[[32]byte{'b'}] = &v1.ExecutionBlock{
 		ParentHash:      bytesutil.PadTo([]byte{'3'}, fieldparams.RootLength),
 		TotalDifficulty: "0x1",
 	}
 	service.cfg.ExecutionEngineCaller = e
 	payload, err := bellatrixState.LatestExecutionPayloadHeader()
 	require.NoError(t, err)
-	root := [32]byte{'c'}
-	require.NoError(t, service.cfg.ForkChoiceStore.InsertOptimisticBlock(ctx, 1, root, [32]byte{'a'}, params.BeaconConfig().ZeroHash, 0, 0))
 	postVersion, postHeader, err := getStateVersionAndPayload(bellatrixState)
 	require.NoError(t, err)
-	err = service.notifyNewPayload(ctx, bellatrixState.Version(), postVersion, payload, postHeader, bellatrixBlk, root)
+	validated, err := service.notifyNewPayload(ctx, bellatrixState.Version(), postVersion, payload, postHeader, bellatrixBlk)
 	require.NoError(t, err)
-	optimistic, err := service.IsOptimisticForRoot(ctx, root)
-	require.NoError(t, err)
-	require.Equal(t, false, optimistic)
+	require.Equal(t, true, validated)
 }
 
 func Test_IsOptimisticCandidateBlock(t *testing.T) {
@@ -496,34 +506,6 @@ func Test_IsOptimisticCandidateBlock(t *testing.T) {
 			}(t),
 			want: false,
 		},
-		{
-			name: "shallow block, execution enabled justified chkpt",
-			blk: func(tt *testing.T) block.BeaconBlock {
-				blk := util.NewBeaconBlockBellatrix()
-				blk.Block.Slot = 200
-				blk.Block.ParentRoot = parentRoot[:]
-				wr, err := wrapper.WrappedBeaconBlock(blk.Block)
-				require.NoError(tt, err)
-				return wr
-			}(t),
-			justified: func(tt *testing.T) block.SignedBeaconBlock {
-				blk := util.NewBeaconBlockBellatrix()
-				blk.Block.Slot = 32
-				blk.Block.ParentRoot = parentRoot[:]
-				blk.Block.Body.ExecutionPayload.ParentHash = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
-				blk.Block.Body.ExecutionPayload.FeeRecipient = bytesutil.PadTo([]byte{'a'}, fieldparams.FeeRecipientLength)
-				blk.Block.Body.ExecutionPayload.StateRoot = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
-				blk.Block.Body.ExecutionPayload.ReceiptsRoot = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
-				blk.Block.Body.ExecutionPayload.LogsBloom = bytesutil.PadTo([]byte{'a'}, fieldparams.LogsBloomLength)
-				blk.Block.Body.ExecutionPayload.PrevRandao = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
-				blk.Block.Body.ExecutionPayload.BaseFeePerGas = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
-				blk.Block.Body.ExecutionPayload.BlockHash = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
-				wr, err := wrapper.WrappedSignedBeaconBlock(blk)
-				require.NoError(tt, err)
-				return wr
-			}(t),
-			want: true,
-		},
 	}
 	for _, tt := range tests {
 		jRoot, err := tt.justified.Block().HashTreeRoot()

beacon-chain/blockchain/options.go

@@ -11,7 +11,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
-	enginev1 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
@@ -52,7 +51,7 @@ func WithChainStartFetcher(f powchain.ChainStartFetcher) Option {
 }
 
 // WithExecutionEngineCaller to call execution engine.
-func WithExecutionEngineCaller(c enginev1.Caller) Option {
+func WithExecutionEngineCaller(c powchain.EngineCaller) Option {
 	return func(s *Service) error {
 		s.cfg.ExecutionEngineCaller = c
 		return nil

beacon-chain/blockchain/pow_block_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/holiman/uint256"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	mocks "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
@@ -117,13 +118,13 @@ func Test_validateMergeBlock(t *testing.T) {
 	service, err := NewService(ctx, opts...)
 	require.NoError(t, err)
 
-	engine := &mockEngineService{blks: map[[32]byte]*enginev1.ExecutionBlock{}}
+	engine := &mocks.EngineClient{BlockByHashMap: map[[32]byte]*enginev1.ExecutionBlock{}}
 	service.cfg.ExecutionEngineCaller = engine
-	engine.blks[[32]byte{'a'}] = &enginev1.ExecutionBlock{
+	engine.BlockByHashMap[[32]byte{'a'}] = &enginev1.ExecutionBlock{
 		ParentHash:      bytesutil.PadTo([]byte{'b'}, fieldparams.RootLength),
 		TotalDifficulty: "0x2",
 	}
-	engine.blks[[32]byte{'b'}] = &enginev1.ExecutionBlock{
+	engine.BlockByHashMap[[32]byte{'b'}] = &enginev1.ExecutionBlock{
 		ParentHash:      bytesutil.PadTo([]byte{'3'}, fieldparams.RootLength),
 		TotalDifficulty: "0x1",
 	}
@@ -158,12 +159,12 @@ func Test_getBlkParentHashAndTD(t *testing.T) {
 	service, err := NewService(ctx, opts...)
 	require.NoError(t, err)
 
-	engine := &mockEngineService{blks: map[[32]byte]*enginev1.ExecutionBlock{}}
+	engine := &mocks.EngineClient{BlockByHashMap: map[[32]byte]*enginev1.ExecutionBlock{}}
 	service.cfg.ExecutionEngineCaller = engine
 	h := [32]byte{'a'}
 	p := [32]byte{'b'}
 	td := "0x1"
-	engine.blks[h] = &enginev1.ExecutionBlock{
+	engine.BlockByHashMap[h] = &enginev1.ExecutionBlock{
 		ParentHash:      p[:],
 		TotalDifficulty: td,
 	}
@@ -175,18 +176,18 @@ func Test_getBlkParentHashAndTD(t *testing.T) {
 	_, _, err = service.getBlkParentHashAndTD(ctx, []byte{'c'})
 	require.ErrorContains(t, "could not get pow block: block not found", err)
 
-	engine.blks[h] = nil
+	engine.BlockByHashMap[h] = nil
 	_, _, err = service.getBlkParentHashAndTD(ctx, h[:])
 	require.ErrorContains(t, "pow block is nil", err)
 
-	engine.blks[h] = &enginev1.ExecutionBlock{
+	engine.BlockByHashMap[h] = &enginev1.ExecutionBlock{
 		ParentHash:      p[:],
 		TotalDifficulty: "1",
 	}
 	_, _, err = service.getBlkParentHashAndTD(ctx, h[:])
 	require.ErrorContains(t, "could not decode merge block total difficulty: hex string without 0x prefix", err)
 
-	engine.blks[h] = &enginev1.ExecutionBlock{
+	engine.BlockByHashMap[h] = &enginev1.ExecutionBlock{
 		ParentHash:      p[:],
 		TotalDifficulty: "0XFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
 	}

beacon-chain/blockchain/process_block.go

@@ -109,16 +109,32 @@ func (s *Service) onBlock(ctx context.Context, signed block.SignedBeaconBlock, b
 	if err != nil {
 		return err
 	}
-	if err := s.savePostStateInfo(ctx, blockRoot, signed, postState, false /* reg sync */); err != nil {
-		return err
-	}
 	postStateVersion, postStateHeader, err := getStateVersionAndPayload(postState)
 	if err != nil {
 		return err
 	}
-	if err := s.notifyNewPayload(ctx, preStateVersion, postStateVersion, preStateHeader, postStateHeader, signed, blockRoot); err != nil {
+	isValidPayload, err := s.notifyNewPayload(ctx, preStateVersion, postStateVersion, preStateHeader, postStateHeader, signed)
+	if err != nil {
 		return errors.Wrap(err, "could not verify new payload")
 	}
+	if !isValidPayload {
+		candidate, err := s.optimisticCandidateBlock(ctx, b)
+		if err != nil {
+			return errors.Wrap(err, "could not check if block is optimistic candidate")
+		}
+		if !candidate {
+			return errNotOptimisticCandidate
+		}
+	}
+
+	if err := s.savePostStateInfo(ctx, blockRoot, signed, postState, false /* reg sync */); err != nil {
+		return err
+	}
+	if isValidPayload {
+		if err := s.cfg.ForkChoiceStore.SetOptimisticToValid(ctx, blockRoot); err != nil {
+			return errors.Wrap(err, "could not set optimistic block to valid")
+		}
+	}
 
 	// We add a proposer score boost to fork choice for the block root if applicable, right after
 	// running a successful state transition for the block.
@@ -188,12 +204,16 @@ func (s *Service) onBlock(ctx context.Context, signed block.SignedBeaconBlock, b
 		msg := fmt.Sprintf("could not read balances for state w/ justified checkpoint %#x", justified.Root)
 		return errors.Wrap(err, msg)
 	}
-	if err := s.updateHead(ctx, balances); err != nil {
+	headRoot, err := s.updateHead(ctx, balances)
+	if err != nil {
 		log.WithError(err).Warn("Could not update head")
 	}
 	if _, err := s.notifyForkchoiceUpdate(ctx, s.headBlock().Block(), s.headRoot(), bytesutil.ToBytes32(finalized.Root)); err != nil {
 		return err
 	}
+	if err := s.saveHead(ctx, headRoot); err != nil {
+		return errors.Wrap(err, "could not save head")
+	}
 
 	if err := s.pruneCanonicalAttsFromPool(ctx, blockRoot, signed); err != nil {
 		return err
@@ -376,16 +396,33 @@ func (s *Service) onBlockBatch(ctx context.Context, blks []block.SignedBeaconBlo
 	// blocks have been verified, add them to forkchoice and call the engine
 	for i, b := range blks {
 		s.saveInitSyncBlock(blockRoots[i], b)
-		if err := s.insertBlockToForkChoiceStore(ctx, b.Block(), blockRoots[i], fCheckpoints[i], jCheckpoints[i]); err != nil {
-			return nil, nil, err
-		}
-		if err := s.notifyNewPayload(ctx,
+		isValidPayload, err := s.notifyNewPayload(ctx,
 			preVersionAndHeaders[i].version,
 			postVersionAndHeaders[i].version,
 			preVersionAndHeaders[i].header,
-			postVersionAndHeaders[i].header, b, blockRoots[i]); err != nil {
+			postVersionAndHeaders[i].header, b)
+
+		if err != nil {
 			return nil, nil, err
 		}
+		if !isValidPayload {
+			candidate, err := s.optimisticCandidateBlock(ctx, b.Block())
+			if err != nil {
+				return nil, nil, errors.Wrap(err, "could not check if block is optimistic candidate")
+			}
+			if !candidate {
+				return nil, nil, errNotOptimisticCandidate
+			}
+		}
+
+		if err := s.insertBlockToForkChoiceStore(ctx, b.Block(), blockRoots[i], fCheckpoints[i], jCheckpoints[i]); err != nil {
+			return nil, nil, err
+		}
+		if isValidPayload {
+			if err := s.cfg.ForkChoiceStore.SetOptimisticToValid(ctx, blockRoots[i]); err != nil {
+				return nil, nil, errors.Wrap(err, "could not set optimistic block to valid")
+			}
+		}
 
 		if _, err := s.notifyForkchoiceUpdate(ctx, b.Block(), blockRoots[i], bytesutil.ToBytes32(fCheckpoints[i].Root)); err != nil {
 			return nil, nil, err

beacon-chain/blockchain/receive_attestation.go

@@ -161,19 +161,19 @@ func (s *Service) spawnProcessAttestationsRoutine(stateFeed *event.Feed) {
 					log.WithError(err).Errorf("Unable to get justified balances for root %v", justified.Root)
 					continue
 				}
-				prevHead := s.headRoot()
-				if err := s.updateHead(s.ctx, balances); err != nil {
+				newHeadRoot, err := s.updateHead(s.ctx, balances)
+				if err != nil {
 					log.WithError(err).Warn("Resolving fork due to new attestation")
 				}
-				s.notifyEngineIfChangedHead(prevHead)
+				s.notifyEngineIfChangedHead(s.ctx, newHeadRoot)
 			}
 		}
 	}()
 }
 
 // This calls notify Forkchoice Update in the event that the head has changed
-func (s *Service) notifyEngineIfChangedHead(prevHead [32]byte) {
-	if s.headRoot() == prevHead {
+func (s *Service) notifyEngineIfChangedHead(ctx context.Context, newHeadRoot [32]byte) {
+	if s.headRoot() == newHeadRoot {
 		return
 	}
 	finalized := s.store.FinalizedCheckpt()
@@ -189,6 +189,9 @@ func (s *Service) notifyEngineIfChangedHead(prevHead [32]byte) {
 	if err != nil {
 		log.WithError(err).Error("could not notify forkchoice update")
 	}
+	if err := s.saveHead(ctx, newHeadRoot); err != nil {
+		log.WithError(err).Error("could not save head")
+	}
 }
 
 // This processes fork choice attestations from the pool to account for validator votes and fork choice.

beacon-chain/blockchain/receive_attestation_test.go

@@ -134,12 +134,12 @@ func TestNotifyEngineIfChangedHead(t *testing.T) {
 	service, err := NewService(ctx, opts...)
 	require.NoError(t, err)
 
-	service.notifyEngineIfChangedHead(service.headRoot())
+	service.notifyEngineIfChangedHead(ctx, service.headRoot())
 	hookErr := "could not notify forkchoice update"
 	finalizedErr := "could not get finalized checkpoint"
 	require.LogsDoNotContain(t, hook, finalizedErr)
 	require.LogsDoNotContain(t, hook, hookErr)
-	service.notifyEngineIfChangedHead([32]byte{'a'})
+	service.notifyEngineIfChangedHead(ctx, [32]byte{'a'})
 	require.LogsContain(t, hook, finalizedErr)
 
 	hook.Reset()
@@ -157,7 +157,7 @@ func TestNotifyEngineIfChangedHead(t *testing.T) {
 		block: wsb,
 	}
 	service.store.SetFinalizedCheckpt(finalized)
-	service.notifyEngineIfChangedHead([32]byte{'b'})
+	service.notifyEngineIfChangedHead(ctx, [32]byte{'b'})
 	require.LogsDoNotContain(t, hook, finalizedErr)
 	require.LogsDoNotContain(t, hook, hookErr)
 }

beacon-chain/blockchain/service.go

@@ -29,7 +29,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
-	enginev1 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/flags"
@@ -88,7 +87,7 @@ type config struct {
 	WeakSubjectivityCheckpt *ethpb.Checkpoint
 	BlockFetcher            powchain.POWBlockFetcher
 	FinalizedStateAtStartUp state.BeaconState
-	ExecutionEngineCaller   enginev1.Caller
+	ExecutionEngineCaller   powchain.EngineCaller
 }
 
 // NewService instantiates a new block service instance that will

beacon-chain/blockchain/service_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
+	mockPOW "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	v1 "github.com/prysmaticlabs/prysm/beacon-chain/state/v1"
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/flags"
@@ -74,10 +75,14 @@ func (mb *mockBroadcaster) BroadcastSyncCommitteeMessage(_ context.Context, _ ui
 var _ p2p.Broadcaster = (*mockBroadcaster)(nil)
 
 func setupBeaconChain(t *testing.T, beaconDB db.Database) *Service {
-	endpoint := "http://127.0.0.1"
 	ctx := context.Background()
 	var web3Service *powchain.Service
 	var err error
+	srv, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		srv.Stop()
+	})
 	bState, _ := util.DeterministicGenesisState(t, 10)
 	pbState, err := v1.ProtobufBeaconState(bState.InnerStateUnsafe())
 	require.NoError(t, err)

beacon-chain/blockchain/testing/mock.go

@@ -61,6 +61,7 @@ type ChainService struct {
 	SyncCommitteePubkeys        [][]byte
 	Genesis                     time.Time
 	ForkChoiceStore             forkchoice.ForkChoicer
+	ReceiveBlockMockErr         error
 }
 
 // ForkChoicer mocks the same method in the chain service
@@ -221,6 +222,9 @@ func (s *ChainService) ReceiveBlockBatch(ctx context.Context, blks []block.Signe
 
 // ReceiveBlock mocks ReceiveBlock method in chain service.
 func (s *ChainService) ReceiveBlock(ctx context.Context, block block.SignedBeaconBlock, _ [32]byte) error {
+	if s.ReceiveBlockMockErr != nil {
+		return s.ReceiveBlockMockErr
+	}
 	if s.State == nil {
 		return ErrNilState
 	}

beacon-chain/db/iface/interface.go

@@ -99,7 +99,7 @@ type HeadAccessDatabase interface {
 	SaveHeadBlockRoot(ctx context.Context, blockRoot [32]byte) error
 
 	// Genesis operations.
-	LoadGenesis(ctx context.Context, r io.Reader) error
+	LoadGenesis(ctx context.Context, stateBytes []byte) error
 	SaveGenesisData(ctx context.Context, state state.BeaconState) error
 	EnsureEmbeddedGenesis(ctx context.Context) error
 

beacon-chain/db/kv/genesis.go

@@ -4,8 +4,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io"
-	"io/ioutil"
 
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
@@ -54,14 +52,10 @@ func (s *Store) SaveGenesisData(ctx context.Context, genesisState state.BeaconSt
 	return nil
 }
 
-// LoadGenesis loads a genesis state from a given file path, if no genesis exists already.
-func (s *Store) LoadGenesis(ctx context.Context, r io.Reader) error {
-	b, err := ioutil.ReadAll(r)
-	if err != nil {
-		return err
-	}
+// LoadGenesis loads a genesis state from a ssz-serialized byte slice, if no genesis exists already.
+func (s *Store) LoadGenesis(ctx context.Context, sb []byte) error {
 	st := &ethpb.BeaconState{}
-	if err := st.UnmarshalSSZ(b); err != nil {
+	if err := st.UnmarshalSSZ(sb); err != nil {
 		return err
 	}
 	gs, err := statev1.InitializeFromProtoUnsafe(st)

beacon-chain/db/kv/genesis_test.go

@@ -53,21 +53,16 @@ func TestLoadGenesisFromFile(t *testing.T) {
 	if err == nil {
 		fp = rfp
 	}
-
-	r, err := os.Open(fp)
+	sb, err := os.ReadFile(fp)
 	assert.NoError(t, err)
-	defer func() {
-		assert.NoError(t, r.Close())
-	}()
 
 	db := setupDB(t)
-	assert.NoError(t, db.LoadGenesis(context.Background(), r))
+	assert.NoError(t, db.LoadGenesis(context.Background(), sb))
 	testGenesisDataSaved(t, db)
 
 	// Loading the same genesis again should not throw an error
-	_, err = r.Seek(0, 0)
 	assert.NoError(t, err)
-	assert.NoError(t, db.LoadGenesis(context.Background(), r))
+	assert.NoError(t, db.LoadGenesis(context.Background(), sb))
 }
 
 func TestLoadGenesisFromFile_mismatchedForkVersion(t *testing.T) {
@@ -76,15 +71,12 @@ func TestLoadGenesisFromFile_mismatchedForkVersion(t *testing.T) {
 	if err == nil {
 		fp = rfp
 	}
-	r, err := os.Open(fp)
+	sb, err := os.ReadFile(fp)
 	assert.NoError(t, err)
-	defer func() {
-		assert.NoError(t, r.Close())
-	}()
 
 	// Loading a genesis with the wrong fork version as beacon config should throw an error.
 	db := setupDB(t)
-	assert.ErrorContains(t, "does not match config genesis fork version", db.LoadGenesis(context.Background(), r))
+	assert.ErrorContains(t, "does not match config genesis fork version", db.LoadGenesis(context.Background(), sb))
 }
 
 func TestEnsureEmbeddedGenesis(t *testing.T) {

beacon-chain/forkchoice/protoarray/store.go

@@ -443,6 +443,7 @@ func (s *Store) applyWeightChanges(
 			}
 			iProposerScore, err := pmath.Int(proposerScore)
 			if err != nil {
+				s.proposerBoostLock.Unlock()
 				return err
 			}
 			nodeDelta = nodeDelta + iProposerScore

beacon-chain/node/BUILD.bazel

@@ -43,6 +43,7 @@ go_library(
         "//beacon-chain/sync:go_default_library",
         "//beacon-chain/sync/backfill:go_default_library",
         "//beacon-chain/sync/checkpoint:go_default_library",
+        "//beacon-chain/sync/genesis:go_default_library",
         "//beacon-chain/sync/initial-sync:go_default_library",
         "//cmd:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",
@@ -76,6 +77,8 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//beacon-chain/core/feed/state:go_default_library",
+        "//beacon-chain/powchain:go_default_library",
+        "//beacon-chain/powchain/testing:go_default_library",
         "//cmd:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",
         "//config/params:go_default_library",

beacon-chain/node/node.go

@@ -46,6 +46,7 @@ import (
 	regularsync "github.com/prysmaticlabs/prysm/beacon-chain/sync"
 	"github.com/prysmaticlabs/prysm/beacon-chain/sync/backfill"
 	"github.com/prysmaticlabs/prysm/beacon-chain/sync/checkpoint"
+	"github.com/prysmaticlabs/prysm/beacon-chain/sync/genesis"
 	initialsync "github.com/prysmaticlabs/prysm/beacon-chain/sync/initial-sync"
 	"github.com/prysmaticlabs/prysm/cmd"
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/flags"
@@ -104,6 +105,7 @@ type BeaconNode struct {
 	finalizedStateAtStartUp state.BeaconState
 	serviceFlagOpts         *serviceFlagOpts
 	blockchainFlagOpts      []blockchain.Option
+	GenesisInitializer      genesis.Initializer
 	CheckpointInitializer   checkpoint.Initializer
 }
 
@@ -381,20 +383,10 @@ func (b *BeaconNode) startDB(cliCtx *cli.Context, depositAddress string) error {
 	if err != nil {
 		return errors.Wrap(err, "could not create deposit cache")
 	}
-
 	b.depositCache = depositCache
 
-	if cliCtx.IsSet(flags.GenesisStatePath.Name) {
-		r, err := os.Open(cliCtx.String(flags.GenesisStatePath.Name))
-		if err != nil {
-			return err
-		}
-		defer func() {
-			if err := r.Close(); err != nil {
-				log.WithError(err).Error("Failed to close genesis file")
-			}
-		}()
-		if err := b.db.LoadGenesis(b.ctx, r); err != nil {
+	if b.GenesisInitializer != nil {
+		if err := b.GenesisInitializer.Initialize(b.ctx, d); err != nil {
 			if err == db.ErrExistingGenesisState {
 				return errors.New("Genesis state flag specified but a genesis state " +
 					"exists already. Run again with --clear-db and/or ensure you are using the " +
@@ -403,6 +395,7 @@ func (b *BeaconNode) startDB(cliCtx *cli.Context, depositAddress string) error {
 			return errors.Wrap(err, "could not load genesis from file")
 		}
 	}
+
 	if err := b.db.EnsureEmbeddedGenesis(b.ctx); err != nil {
 		return err
 	}
@@ -581,7 +574,7 @@ func (b *BeaconNode) registerBlockchainService() error {
 		blockchain.WithDatabase(b.db),
 		blockchain.WithDepositCache(b.depositCache),
 		blockchain.WithChainStartFetcher(web3Service),
-		blockchain.WithExecutionEngineCaller(web3Service.EngineAPIClient()),
+		blockchain.WithExecutionEngineCaller(web3Service),
 		blockchain.WithAttestationPool(b.attestationPool),
 		blockchain.WithExitPool(b.exitPool),
 		blockchain.WithSlashingPool(b.slashingsPool),
@@ -808,7 +801,7 @@ func (b *BeaconNode) registerRPCService() error {
 		StateGen:                b.stateGen,
 		EnableDebugRPCEndpoints: enableDebugRPCEndpoints,
 		MaxMsgSize:              maxMsgSize,
-		ExecutionEngineCaller:   web3Service.EngineAPIClient(),
+		ExecutionEngineCaller:   web3Service,
 	})
 
 	return b.services.RegisterService(rpcService)

beacon-chain/node/node_test.go

@@ -8,6 +8,8 @@ import (
 	"testing"
 
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
+	mockPOW "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing"
 	"github.com/prysmaticlabs/prysm/cmd"
 	"github.com/prysmaticlabs/prysm/testing/require"
 	logTest "github.com/sirupsen/logrus/hooks/test"
@@ -45,6 +47,11 @@ func TestNodeClose_OK(t *testing.T) {
 // TestClearDB tests clearing the database
 func TestClearDB(t *testing.T) {
 	hook := logTest.NewGlobal()
+	srv, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		srv.Stop()
+	})
 
 	tmp := filepath.Join(t.TempDir(), "datadirtest")
 
@@ -54,7 +61,9 @@ func TestClearDB(t *testing.T) {
 	set.Bool(cmd.ForceClearDB.Name, true, "force clear db")
 
 	context := cli.NewContext(&app, set, nil)
-	_, err := New(context)
+	_, err = New(context, WithPowchainFlagOptions([]powchain.Option{
+		powchain.WithHttpEndpoints([]string{endpoint}),
+	}))
 	require.NoError(t, err)
 
 	require.LogsContain(t, hook, "Removing database")

beacon-chain/powchain/BUILD.bazel

@@ -3,12 +3,16 @@ load("@prysm//tools/go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
+        "auth.go",
         "block_cache.go",
         "block_reader.go",
         "check_transition_config.go",
         "deposit.go",
+        "engine_client.go",
+        "errors.go",
         "log.go",
         "log_processing.go",
+        "metrics.go",
         "options.go",
         "prometheus.go",
         "provider.go",
@@ -28,7 +32,6 @@ go_library(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db:go_default_library",
-        "//beacon-chain/powchain/engine-api-client/v1:go_default_library",
         "//beacon-chain/powchain/types:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/state-native/v1:go_default_library",
@@ -56,6 +59,7 @@ go_library(
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
         "@com_github_ethereum_go_ethereum//ethclient:go_default_library",
         "@com_github_ethereum_go_ethereum//rpc:go_default_library",
+        "@com_github_golang_jwt_jwt_v4//:go_default_library",
         "@com_github_holiman_uint256//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
@@ -70,10 +74,12 @@ go_test(
     name = "go_default_test",
     size = "medium",
     srcs = [
+        "auth_test.go",
         "block_cache_test.go",
         "block_reader_test.go",
         "check_transition_config_test.go",
         "deposit_test.go",
+        "engine_client_test.go",
         "init_test.go",
         "log_processing_test.go",
         "powchain_test.go",
@@ -92,8 +98,6 @@ go_test(
         "//beacon-chain/core/signing:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
-        "//beacon-chain/powchain/engine-api-client/v1:go_default_library",
-        "//beacon-chain/powchain/engine-api-client/v1/mocks:go_default_library",
         "//beacon-chain/powchain/testing:go_default_library",
         "//beacon-chain/powchain/types:go_default_library",
         "//beacon-chain/state/stategen:go_default_library",
@@ -119,10 +123,15 @@ go_test(
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
+        "@com_github_ethereum_go_ethereum//rpc:go_default_library",
         "@com_github_ethereum_go_ethereum//trie:go_default_library",
+        "@com_github_golang_jwt_jwt_v4//:go_default_library",
+        "@com_github_holiman_uint256//:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prysmaticlabs_eth2_types//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
+        "@org_golang_google_protobuf//proto:go_default_library",
     ],
 )

beacon-chain/powchain/auth.go

@@ -1,4 +1,4 @@
-package v1
+package powchain
 
 import (
 	"net/http"

beacon-chain/powchain/auth_test.go

@@ -1,4 +1,4 @@
-package v1
+package powchain
 
 import (
 	"net/http"
@@ -19,7 +19,7 @@ func TestJWTAuthTransport(t *testing.T) {
 		jwtSecret:           secret,
 	}
 	client := &http.Client{
-		Timeout:   DefaultTimeout,
+		Timeout:   DefaultRPCHTTPTimeout,
 		Transport: authTransport,
 	}
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

beacon-chain/powchain/block_reader_test.go

@@ -18,8 +18,6 @@ import (
 	"github.com/prysmaticlabs/prysm/testing/require"
 )
 
-var endpoint = "http://127.0.0.1"
-
 func setDefaultMocks(service *Service) *Service {
 	service.eth1DataFetcher = &goodFetcher{}
 	service.httpLogger = &goodLogger{}
@@ -32,6 +30,11 @@ func TestLatestMainchainInfo_OK(t *testing.T) {
 	require.NoError(t, err, "Unable to set up simulated backend")
 
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -70,6 +73,11 @@ func TestLatestMainchainInfo_OK(t *testing.T) {
 
 func TestBlockHashByHeight_ReturnsHash(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -97,6 +105,11 @@ func TestBlockHashByHeight_ReturnsHash(t *testing.T) {
 
 func TestBlockHashByHeight_ReturnsError_WhenNoEth1Client(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -113,6 +126,11 @@ func TestBlockHashByHeight_ReturnsError_WhenNoEth1Client(t *testing.T) {
 
 func TestBlockExists_ValidHash(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -144,6 +162,11 @@ func TestBlockExists_ValidHash(t *testing.T) {
 
 func TestBlockExists_InvalidHash(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -158,6 +181,11 @@ func TestBlockExists_InvalidHash(t *testing.T) {
 
 func TestBlockExists_UsesCachedBlockInfo(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -181,6 +209,11 @@ func TestBlockExists_UsesCachedBlockInfo(t *testing.T) {
 
 func TestBlockExistsWithCache_UsesCachedHeaderInfo(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -202,6 +235,11 @@ func TestBlockExistsWithCache_UsesCachedHeaderInfo(t *testing.T) {
 
 func TestBlockExistsWithCache_HeaderNotCached(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -218,6 +256,11 @@ func TestService_BlockNumberByTimestamp(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -245,6 +288,11 @@ func TestService_BlockNumberByTimestampLessTargetTime(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -278,6 +326,11 @@ func TestService_BlockNumberByTimestampMoreTargetTime(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -309,6 +362,11 @@ func TestService_BlockNumberByTimestampMoreTargetTime(t *testing.T) {
 
 func TestService_BlockTimeByHeight_ReturnsError_WhenNoEth1Client(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),

beacon-chain/powchain/check_transition_config.go

@@ -7,12 +7,14 @@ import (
 	"math/big"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/holiman/uint256"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
-	v1 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
 	"github.com/prysmaticlabs/prysm/config/params"
 	pb "github.com/prysmaticlabs/prysm/proto/engine/v1"
+	"github.com/sirupsen/logrus"
 )
 
 var (
@@ -27,16 +29,12 @@ var (
 // If there are any discrepancies, we must log errors to ensure users can resolve
 //the problem and be ready for the merge transition.
 func (s *Service) checkTransitionConfiguration(
-	ctx context.Context, blockNotifications chan *statefeed.BlockProcessedData,
+	ctx context.Context, blockNotifications chan *feed.Event,
 ) {
 	// If Bellatrix fork epoch is not set, we do not run this check.
 	if params.BeaconConfig().BellatrixForkEpoch == math.MaxUint64 {
 		return
 	}
-	// If no engine API, then also avoid running this check.
-	if s.engineAPIClient == nil {
-		return
-	}
 	i := new(big.Int)
 	i.SetString(params.BeaconConfig().TerminalTotalDifficulty, 10)
 	ttd := new(uint256.Int)
@@ -46,9 +44,9 @@ func (s *Service) checkTransitionConfiguration(
 		TerminalBlockHash:       params.BeaconConfig().TerminalBlockHash[:],
 		TerminalBlockNumber:     big.NewInt(0).Bytes(), // A value of 0 is recommended in the request.
 	}
-	err := s.engineAPIClient.ExchangeTransitionConfiguration(ctx, cfg)
+	err := s.ExchangeTransitionConfiguration(ctx, cfg)
 	if err != nil {
-		if errors.Is(err, v1.ErrConfigMismatch) {
+		if errors.Is(err, ErrConfigMismatch) {
 			log.WithError(err).Fatal(configMismatchLog)
 		}
 		log.WithError(err).Error("Could not check configuration values between execution and consensus client")
@@ -58,6 +56,7 @@ func (s *Service) checkTransitionConfiguration(
 	// This serves as a heartbeat to ensure the execution client and Prysm are ready for the
 	// Bellatrix hard-fork transition.
 	ticker := time.NewTicker(checkTransitionPollingInterval)
+	hasTtdReached := false
 	defer ticker.Stop()
 	sub := s.cfg.stateNotifier.StateFeed().Subscribe(blockNotifications)
 	defer sub.Unsubscribe()
@@ -68,7 +67,11 @@ func (s *Service) checkTransitionConfiguration(
 		case <-sub.Err():
 			return
 		case ev := <-blockNotifications:
-			isExecutionBlock, err := blocks.IsExecutionBlock(ev.SignedBlock.Block().Body())
+			data, ok := ev.Data.(*statefeed.BlockProcessedData)
+			if !ok {
+				continue
+			}
+			isExecutionBlock, err := blocks.IsExecutionBlock(data.SignedBlock.Block().Body())
 			if err != nil {
 				log.WithError(err).Debug("Could not check whether signed block is execution block")
 				continue
@@ -77,9 +80,17 @@ func (s *Service) checkTransitionConfiguration(
 				log.Debug("PoS transition is complete, no longer checking for configuration changes")
 				return
 			}
-		case <-ticker.C:
-			err = s.engineAPIClient.ExchangeTransitionConfiguration(ctx, cfg)
+		case tm := <-ticker.C:
+			ctx, cancel := context.WithDeadline(ctx, tm.Add(DefaultRPCHTTPTimeout))
+			err = s.ExchangeTransitionConfiguration(ctx, cfg)
 			s.handleExchangeConfigurationError(err)
+			if !hasTtdReached {
+				hasTtdReached, err = s.logTtdStatus(ctx, ttd)
+				if err != nil {
+					log.WithError(err).Error("Could not log ttd status")
+				}
+			}
+			cancel()
 		}
 	}
 }
@@ -91,16 +102,39 @@ func (s *Service) handleExchangeConfigurationError(err error) {
 	if err == nil {
 		// If there is no error in checking the exchange configuration error, we clear
 		// the run error of the service if we had previously set it to ErrConfigMismatch.
-		if errors.Is(s.runError, v1.ErrConfigMismatch) {
+		if errors.Is(s.runError, ErrConfigMismatch) {
 			s.runError = nil
 		}
 		return
 	}
 	// If the error is a configuration mismatch, we set a runtime error in the service.
-	if errors.Is(err, v1.ErrConfigMismatch) {
+	if errors.Is(err, ErrConfigMismatch) {
 		s.runError = err
 		log.WithError(err).Error(configMismatchLog)
 		return
 	}
 	log.WithError(err).Error("Could not check configuration values between execution and consensus client")
 }
+
+// Logs the terminal total difficulty status.
+func (s *Service) logTtdStatus(ctx context.Context, ttd *uint256.Int) (bool, error) {
+	latest, err := s.LatestExecutionBlock(ctx)
+	if err != nil {
+		return false, err
+	}
+	if latest == nil {
+		return false, errors.New("latest block is nil")
+	}
+	latestTtd, err := hexutil.DecodeBig(latest.TotalDifficulty)
+	if err != nil {
+		return false, err
+	}
+	if latestTtd.Cmp(ttd.ToBig()) >= 0 {
+		return true, nil
+	}
+	log.WithFields(logrus.Fields{
+		"latestDifficulty":   latestTtd.String(),
+		"terminalDifficulty": ttd.ToBig().String(),
+	}).Info("terminal difficulty has not been reached yet")
+	return false, nil
+}

beacon-chain/powchain/check_transition_config_test.go

@@ -2,21 +2,28 @@ package powchain
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 	"time"
 
-	mockChain2 "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/holiman/uint256"
+	mockChain "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
-	v1 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
-	"github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1/mocks"
+	mocks "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
-	enginev1 "github.com/prysmaticlabs/prysm/proto/engine/v1"
+	pb "github.com/prysmaticlabs/prysm/proto/engine/v1"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
 	"github.com/prysmaticlabs/prysm/testing/require"
 	logTest "github.com/sirupsen/logrus/hooks/test"
+	"google.golang.org/protobuf/proto"
 )
 
 func Test_checkTransitionConfiguration(t *testing.T) {
@@ -31,14 +38,11 @@ func Test_checkTransitionConfiguration(t *testing.T) {
 		m := &mocks.EngineClient{}
 		m.Err = errors.New("something went wrong")
 
-		mockChain := &mockChain2.MockStateNotifier{}
-		srv := &Service{
-			cfg: &config{stateNotifier: mockChain},
-		}
-		srv.engineAPIClient = m
+		srv := setupTransitionConfigTest(t)
+		srv.cfg.stateNotifier = &mockChain.MockStateNotifier{}
 		checkTransitionPollingInterval = time.Millisecond
 		ctx, cancel := context.WithCancel(ctx)
-		go srv.checkTransitionConfiguration(ctx, make(chan *statefeed.BlockProcessedData, 1))
+		go srv.checkTransitionConfiguration(ctx, make(chan *feed.Event, 1))
 		<-time.After(100 * time.Millisecond)
 		cancel()
 		require.LogsContain(t, hook, "Could not check configuration values")
@@ -48,16 +52,13 @@ func Test_checkTransitionConfiguration(t *testing.T) {
 		ctx := context.Background()
 		m := &mocks.EngineClient{}
 		m.Err = errors.New("something went wrong")
+		srv := setupTransitionConfigTest(t)
+		srv.cfg.stateNotifier = &mockChain.MockStateNotifier{}
 
-		mockChain := &mockChain2.MockStateNotifier{}
-		srv := &Service{
-			cfg: &config{stateNotifier: mockChain},
-		}
-		srv.engineAPIClient = m
 		checkTransitionPollingInterval = time.Millisecond
 		ctx, cancel := context.WithCancel(ctx)
 		exit := make(chan bool)
-		notification := make(chan *statefeed.BlockProcessedData)
+		notification := make(chan *feed.Event)
 		go func() {
 			srv.checkTransitionConfiguration(ctx, notification)
 			exit <- true
@@ -72,8 +73,11 @@ func Test_checkTransitionConfiguration(t *testing.T) {
 			}},
 		)
 		require.NoError(t, err)
-		notification <- &statefeed.BlockProcessedData{
-			SignedBlock: wrappedBlock,
+		notification <- &feed.Event{
+			Data: &statefeed.BlockProcessedData{
+				SignedBlock: wrappedBlock,
+			},
+			Type: statefeed.BlockProcessed,
 		}
 		<-exit
 		cancel()
@@ -84,33 +88,110 @@ func Test_checkTransitionConfiguration(t *testing.T) {
 func TestService_handleExchangeConfigurationError(t *testing.T) {
 	hook := logTest.NewGlobal()
 	t.Run("clears existing service error", func(t *testing.T) {
-		srv := &Service{isRunning: true}
-		srv.runError = v1.ErrConfigMismatch
+		srv := setupTransitionConfigTest(t)
+		srv.isRunning = true
+		srv.runError = ErrConfigMismatch
 		srv.handleExchangeConfigurationError(nil)
 		require.Equal(t, true, srv.Status() == nil)
 	})
 	t.Run("does not clear existing service error if wrong kind", func(t *testing.T) {
-		srv := &Service{isRunning: true}
+		srv := setupTransitionConfigTest(t)
+		srv.isRunning = true
 		err := errors.New("something else went wrong")
 		srv.runError = err
 		srv.handleExchangeConfigurationError(nil)
 		require.ErrorIs(t, err, srv.Status())
 	})
 	t.Run("sets service error on config mismatch", func(t *testing.T) {
-		srv := &Service{isRunning: true}
-		srv.handleExchangeConfigurationError(v1.ErrConfigMismatch)
-		require.Equal(t, v1.ErrConfigMismatch, srv.Status())
+		srv := setupTransitionConfigTest(t)
+		srv.isRunning = true
+		srv.handleExchangeConfigurationError(ErrConfigMismatch)
+		require.Equal(t, ErrConfigMismatch, srv.Status())
 		require.LogsContain(t, hook, configMismatchLog)
 	})
 	t.Run("does not set service error if unrelated problem", func(t *testing.T) {
-		srv := &Service{isRunning: true}
+		srv := setupTransitionConfigTest(t)
+		srv.isRunning = true
 		srv.handleExchangeConfigurationError(errors.New("foo"))
 		require.Equal(t, true, srv.Status() == nil)
 		require.LogsContain(t, hook, "Could not check configuration values")
 	})
 }
-func emptyPayload() *enginev1.ExecutionPayload {
-	return &enginev1.ExecutionPayload{
+
+func setupTransitionConfigTest(t testing.TB) *Service {
+	fix := fixtures()
+	request, ok := fix["TransitionConfiguration"].(*pb.TransitionConfiguration)
+	require.Equal(t, true, ok)
+	resp, ok := proto.Clone(request).(*pb.TransitionConfiguration)
+	require.Equal(t, true, ok)
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		defer func() {
+			require.NoError(t, r.Body.Close())
+		}()
+
+		// Change the terminal block hash.
+		h := common.BytesToHash([]byte("foo"))
+		resp.TerminalBlockHash = h[:]
+		respJSON := map[string]interface{}{
+			"jsonrpc": "2.0",
+			"id":      1,
+			"result":  resp,
+		}
+		require.NoError(t, json.NewEncoder(w).Encode(respJSON))
+	}))
+	defer srv.Close()
+
+	rpcClient, err := rpc.DialHTTP(srv.URL)
+	require.NoError(t, err)
+	defer rpcClient.Close()
+
+	service := &Service{
+		cfg: &config{},
+	}
+	service.rpcClient = rpcClient
+	return service
+}
+
+func TestService_logTtdStatus(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		defer func() {
+			require.NoError(t, r.Body.Close())
+		}()
+
+		resp := &pb.ExecutionBlock{TotalDifficulty: "0x12345678"}
+		respJSON := map[string]interface{}{
+			"jsonrpc": "2.0",
+			"id":      1,
+			"result":  resp,
+		}
+		require.NoError(t, json.NewEncoder(w).Encode(respJSON))
+	}))
+	defer srv.Close()
+
+	rpcClient, err := rpc.DialHTTP(srv.URL)
+	require.NoError(t, err)
+	defer rpcClient.Close()
+
+	service := &Service{
+		cfg: &config{},
+	}
+	service.rpcClient = rpcClient
+
+	ttd := new(uint256.Int)
+	reached, err := service.logTtdStatus(context.Background(), ttd.SetUint64(24343))
+	require.NoError(t, err)
+	require.Equal(t, true, reached)
+
+	reached, err = service.logTtdStatus(context.Background(), ttd.SetUint64(323423484))
+	require.NoError(t, err)
+	require.Equal(t, false, reached)
+}
+
+func emptyPayload() *pb.ExecutionPayload {
+	return &pb.ExecutionPayload{
 		ParentHash:    make([]byte, fieldparams.RootLength),
 		FeeRecipient:  make([]byte, fieldparams.FeeRecipientLength),
 		StateRoot:     make([]byte, fieldparams.RootLength),

beacon-chain/powchain/deposit_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/signing"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	testing2 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
 	"github.com/prysmaticlabs/prysm/container/trie"
@@ -52,6 +53,11 @@ func TestDepositContractAddress_OK(t *testing.T) {
 
 func TestProcessDeposit_OK(t *testing.T) {
 	beaconDB := testDB.SetupDB(t)
+	server, endpoint, err := testing2.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -76,6 +82,11 @@ func TestProcessDeposit_OK(t *testing.T) {
 
 func TestProcessDeposit_InvalidMerkleBranch(t *testing.T) {
 	beaconDB := testDB.SetupDB(t)
+	server, endpoint, err := testing2.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -102,6 +113,11 @@ func TestProcessDeposit_InvalidMerkleBranch(t *testing.T) {
 func TestProcessDeposit_InvalidPublicKey(t *testing.T) {
 	hook := logTest.NewGlobal()
 	beaconDB := testDB.SetupDB(t)
+	server, endpoint, err := testing2.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -138,6 +154,11 @@ func TestProcessDeposit_InvalidPublicKey(t *testing.T) {
 func TestProcessDeposit_InvalidSignature(t *testing.T) {
 	hook := logTest.NewGlobal()
 	beaconDB := testDB.SetupDB(t)
+	server, endpoint, err := testing2.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -173,6 +194,11 @@ func TestProcessDeposit_InvalidSignature(t *testing.T) {
 func TestProcessDeposit_UnableToVerify(t *testing.T) {
 	hook := logTest.NewGlobal()
 	beaconDB := testDB.SetupDB(t)
+	server, endpoint, err := testing2.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -205,6 +231,11 @@ func TestProcessDeposit_UnableToVerify(t *testing.T) {
 
 func TestProcessDeposit_IncompleteDeposit(t *testing.T) {
 	beaconDB := testDB.SetupDB(t)
+	server, endpoint, err := testing2.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -267,6 +298,11 @@ func TestProcessDeposit_IncompleteDeposit(t *testing.T) {
 
 func TestProcessDeposit_AllDepositedSuccessfully(t *testing.T) {
 	beaconDB := testDB.SetupDB(t)
+	server, endpoint, err := testing2.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),

beacon-chain/powchain/engine-api-client/v1/BUILD.bazel

@@ -1,49 +0,0 @@
-load("@prysm//tools/go:def.bzl", "go_library", "go_test")
-
-go_library(
-    name = "go_default_library",
-    srcs = [
-        "auth.go",
-        "client.go",
-        "errors.go",
-        "metrics.go",
-        "options.go",
-    ],
-    importpath = "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1",
-    visibility = ["//beacon-chain:__subpackages__"],
-    deps = [
-        "//config/params:go_default_library",
-        "//proto/engine/v1:go_default_library",
-        "@com_github_ethereum_go_ethereum//common:go_default_library",
-        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
-        "@com_github_ethereum_go_ethereum//rpc:go_default_library",
-        "@com_github_golang_jwt_jwt_v4//:go_default_library",
-        "@com_github_pkg_errors//:go_default_library",
-        "@com_github_prometheus_client_golang//prometheus:go_default_library",
-        "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
-        "@io_opencensus_go//trace:go_default_library",
-    ],
-)
-
-go_test(
-    name = "go_default_test",
-    srcs = [
-        "auth_test.go",
-        "client_test.go",
-    ],
-    embed = [":go_default_library"],
-    deps = [
-        "//beacon-chain/powchain/engine-api-client/v1/mocks:go_default_library",
-        "//config/fieldparams:go_default_library",
-        "//config/params:go_default_library",
-        "//encoding/bytesutil:go_default_library",
-        "//proto/engine/v1:go_default_library",
-        "//testing/require:go_default_library",
-        "@com_github_ethereum_go_ethereum//common:go_default_library",
-        "@com_github_ethereum_go_ethereum//rpc:go_default_library",
-        "@com_github_golang_jwt_jwt_v4//:go_default_library",
-        "@com_github_holiman_uint256//:go_default_library",
-        "@com_github_pkg_errors//:go_default_library",
-        "@org_golang_google_protobuf//proto:go_default_library",
-    ],
-)

beacon-chain/powchain/engine-api-client/v1/mocks/BUILD.bazel

@@ -1,13 +0,0 @@
-load("@prysm//tools/go:def.bzl", "go_library")
-
-go_library(
-    name = "go_default_library",
-    srcs = ["client.go"],
-    importpath = "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1/mocks",
-    visibility = ["//visibility:public"],
-    deps = [
-        "//proto/engine/v1:go_default_library",
-        "@com_github_ethereum_go_ethereum//common:go_default_library",
-        "@com_github_pkg_errors//:go_default_library",
-    ],
-)

beacon-chain/powchain/engine-api-client/v1/options.go

@@ -1,39 +0,0 @@
-package v1
-
-import (
-	"net/http"
-)
-
-// Option for configuring the engine API client.
-type Option func(c *Client) error
-
-type config struct {
-	httpClient *http.Client
-}
-
-func defaultConfig() *config {
-	return &config{
-		httpClient: &http.Client{
-			Timeout: DefaultTimeout,
-		},
-	}
-}
-
-// WithJWTSecret allows setting a JWT secret for authenticating
-// the client via HTTP connections.
-func WithJWTSecret(secret []byte) Option {
-	return func(c *Client) error {
-		if len(secret) == 0 {
-			return nil
-		}
-		authTransport := &jwtTransport{
-			underlyingTransport: http.DefaultTransport,
-			jwtSecret:           secret,
-		}
-		c.cfg.httpClient = &http.Client{
-			Timeout:   DefaultTimeout,
-			Transport: authTransport,
-		}
-		return nil
-	}
-}

beacon-chain/powchain/engine_client.go

@@ -1,14 +1,10 @@
-// Package v1 defines an API client for the engine API defined in https://github.com/ethereum/execution-apis.
-// This client is used for the Prysm consensus node to connect to execution node as part of
-// the Ethereum proof-of-stake machinery.
-package v1
+package powchain
 
 import (
 	"bytes"
 	"context"
 	"fmt"
 	"math/big"
-	"net/url"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -33,8 +29,6 @@ const (
 	ExecutionBlockByHashMethod = "eth_getBlockByHash"
 	// ExecutionBlockByNumberMethod request string for JSON-RPC.
 	ExecutionBlockByNumberMethod = "eth_getBlockByNumber"
-	// DefaultTimeout for HTTP.
-	DefaultTimeout = time.Second * 5
 )
 
 // ForkchoiceUpdatedResponse is the response kind received by the
@@ -44,9 +38,9 @@ type ForkchoiceUpdatedResponse struct {
 	PayloadId *pb.PayloadIDBytes `json:"payloadId"`
 }
 
-// Caller defines a client that can interact with an Ethereum
+// EngineCaller defines a client that can interact with an Ethereum
 // execution node's engine service via JSON-RPC.
-type Caller interface {
+type EngineCaller interface {
 	NewPayload(ctx context.Context, payload *pb.ExecutionPayload) ([]byte, error)
 	ForkchoiceUpdated(
 		ctx context.Context, state *pb.ForkchoiceState, attrs *pb.PayloadAttributes,
@@ -59,44 +53,8 @@ type Caller interface {
 	ExecutionBlockByHash(ctx context.Context, hash common.Hash) (*pb.ExecutionBlock, error)
 }
 
-// Client defines a new engine API client for the Prysm consensus node
-// to interact with an Ethereum execution node.
-type Client struct {
-	cfg *config
-	rpc *rpc.Client
-}
-
-// New returns a ready, engine API client from an endpoint and configuration options.
-// Only http(s) and ipc (inter-process communication) URL schemes are supported.
-func New(ctx context.Context, endpoint string, opts ...Option) (*Client, error) {
-	u, err := url.Parse(endpoint)
-	if err != nil {
-		return nil, err
-	}
-	c := &Client{
-		cfg: defaultConfig(),
-	}
-	for _, opt := range opts {
-		if err := opt(c); err != nil {
-			return nil, err
-		}
-	}
-	switch u.Scheme {
-	case "http", "https":
-		c.rpc, err = rpc.DialHTTPWithClient(endpoint, c.cfg.httpClient)
-	case "":
-		c.rpc, err = rpc.DialIPC(ctx, endpoint)
-	default:
-		return nil, errors.Wrapf(ErrUnsupportedScheme, "%q", u.Scheme)
-	}
-	if err != nil {
-		return nil, err
-	}
-	return c, nil
-}
-
 // NewPayload calls the engine_newPayloadV1 method via JSON-RPC.
-func (c *Client) NewPayload(ctx context.Context, payload *pb.ExecutionPayload) ([]byte, error) {
+func (s *Service) NewPayload(ctx context.Context, payload *pb.ExecutionPayload) ([]byte, error) {
 	ctx, span := trace.StartSpan(ctx, "powchain.engine-api-client.NewPayload")
 	defer span.End()
 	start := time.Now()
@@ -105,7 +63,7 @@ func (c *Client) NewPayload(ctx context.Context, payload *pb.ExecutionPayload) (
 	}()
 
 	result := &pb.PayloadStatus{}
-	err := c.rpc.CallContext(ctx, result, NewPayloadMethod, payload)
+	err := s.rpcClient.CallContext(ctx, result, NewPayloadMethod, payload)
 	if err != nil {
 		return nil, handleRPCError(err)
 	}
@@ -127,7 +85,7 @@ func (c *Client) NewPayload(ctx context.Context, payload *pb.ExecutionPayload) (
 }
 
 // ForkchoiceUpdated calls the engine_forkchoiceUpdatedV1 method via JSON-RPC.
-func (c *Client) ForkchoiceUpdated(
+func (s *Service) ForkchoiceUpdated(
 	ctx context.Context, state *pb.ForkchoiceState, attrs *pb.PayloadAttributes,
 ) (*pb.PayloadIDBytes, []byte, error) {
 	ctx, span := trace.StartSpan(ctx, "powchain.engine-api-client.ForkchoiceUpdated")
@@ -138,7 +96,7 @@ func (c *Client) ForkchoiceUpdated(
 	}()
 
 	result := &ForkchoiceUpdatedResponse{}
-	err := c.rpc.CallContext(ctx, result, ForkchoiceUpdatedMethod, state, attrs)
+	err := s.rpcClient.CallContext(ctx, result, ForkchoiceUpdatedMethod, state, attrs)
 	if err != nil {
 		return nil, nil, handleRPCError(err)
 	}
@@ -162,7 +120,7 @@ func (c *Client) ForkchoiceUpdated(
 }
 
 // GetPayload calls the engine_getPayloadV1 method via JSON-RPC.
-func (c *Client) GetPayload(ctx context.Context, payloadId [8]byte) (*pb.ExecutionPayload, error) {
+func (s *Service) GetPayload(ctx context.Context, payloadId [8]byte) (*pb.ExecutionPayload, error) {
 	ctx, span := trace.StartSpan(ctx, "powchain.engine-api-client.GetPayload")
 	defer span.End()
 	start := time.Now()
@@ -171,12 +129,12 @@ func (c *Client) GetPayload(ctx context.Context, payloadId [8]byte) (*pb.Executi
 	}()
 
 	result := &pb.ExecutionPayload{}
-	err := c.rpc.CallContext(ctx, result, GetPayloadMethod, pb.PayloadIDBytes(payloadId))
+	err := s.rpcClient.CallContext(ctx, result, GetPayloadMethod, pb.PayloadIDBytes(payloadId))
 	return result, handleRPCError(err)
 }
 
 // ExchangeTransitionConfiguration calls the engine_exchangeTransitionConfigurationV1 method via JSON-RPC.
-func (c *Client) ExchangeTransitionConfiguration(
+func (s *Service) ExchangeTransitionConfiguration(
 	ctx context.Context, cfg *pb.TransitionConfiguration,
 ) error {
 	ctx, span := trace.StartSpan(ctx, "powchain.engine-api-client.ExchangeTransitionConfiguration")
@@ -186,7 +144,7 @@ func (c *Client) ExchangeTransitionConfiguration(
 	zeroBigNum := big.NewInt(0)
 	cfg.TerminalBlockNumber = zeroBigNum.Bytes()
 	result := &pb.TransitionConfiguration{}
-	if err := c.rpc.CallContext(ctx, result, ExchangeTransitionConfigurationMethod, cfg); err != nil {
+	if err := s.rpcClient.CallContext(ctx, result, ExchangeTransitionConfigurationMethod, cfg); err != nil {
 		return handleRPCError(err)
 	}
 	// We surface an error to the user if local configuration settings mismatch
@@ -218,12 +176,12 @@ func (c *Client) ExchangeTransitionConfiguration(
 
 // LatestExecutionBlock fetches the latest execution engine block by calling
 // eth_blockByNumber via JSON-RPC.
-func (c *Client) LatestExecutionBlock(ctx context.Context) (*pb.ExecutionBlock, error) {
+func (s *Service) LatestExecutionBlock(ctx context.Context) (*pb.ExecutionBlock, error) {
 	ctx, span := trace.StartSpan(ctx, "powchain.engine-api-client.LatestExecutionBlock")
 	defer span.End()
 
 	result := &pb.ExecutionBlock{}
-	err := c.rpc.CallContext(
+	err := s.rpcClient.CallContext(
 		ctx,
 		result,
 		ExecutionBlockByNumberMethod,
@@ -235,12 +193,12 @@ func (c *Client) LatestExecutionBlock(ctx context.Context) (*pb.ExecutionBlock,
 
 // ExecutionBlockByHash fetches an execution engine block by hash by calling
 // eth_blockByHash via JSON-RPC.
-func (c *Client) ExecutionBlockByHash(ctx context.Context, hash common.Hash) (*pb.ExecutionBlock, error) {
+func (s *Service) ExecutionBlockByHash(ctx context.Context, hash common.Hash) (*pb.ExecutionBlock, error) {
 	ctx, span := trace.StartSpan(ctx, "powchain.engine-api-client.ExecutionBlockByHash")
 	defer span.End()
 
 	result := &pb.ExecutionBlock{}
-	err := c.rpc.CallContext(ctx, result, ExecutionBlockByHashMethod, hash, false /* no full transaction objects */)
+	err := s.rpcClient.CallContext(ctx, result, ExecutionBlockByHashMethod, hash, false /* no full transaction objects */)
 	return result, handleRPCError(err)
 }
 
@@ -249,6 +207,9 @@ func handleRPCError(err error) error {
 	if err == nil {
 		return nil
 	}
+	if isTimeout(err) {
+		return errors.Wrapf(ErrHTTPTimeout, "%s", err)
+	}
 	e, ok := err.(rpc.Error)
 	if !ok {
 		return errors.Wrap(err, "got an unexpected error")
@@ -277,3 +238,16 @@ func handleRPCError(err error) error {
 		return err
 	}
 }
+
+// ErrHTTPTimeout returns true if the error is a http.Client timeout error.
+var ErrHTTPTimeout = errors.New("timeout from http.Client")
+
+type httpTimeoutError interface {
+	Error() string
+	Timeout() bool
+}
+
+func isTimeout(e error) bool {
+	t, ok := e.(httpTimeoutError)
+	return ok && t.Timeout()
+}

beacon-chain/powchain/engine_client_test.go

@@ -1,4 +1,4 @@
-package v1
+package powchain
 
 import (
 	"context"
@@ -15,7 +15,7 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/holiman/uint256"
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1/mocks"
+	mocks "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
@@ -25,8 +25,8 @@ import (
 )
 
 var (
-	_ = Caller(&Client{})
-	_ = Caller(&mocks.EngineClient{})
+	_ = EngineCaller(&Service{})
+	_ = EngineCaller(&mocks.EngineClient{})
 )
 
 func TestClient_IPC(t *testing.T) {
@@ -34,8 +34,8 @@ func TestClient_IPC(t *testing.T) {
 	defer server.Stop()
 	rpcClient := rpc.DialInProc(server)
 	defer rpcClient.Close()
-	client := &Client{}
-	client.rpc = rpcClient
+	srv := &Service{}
+	srv.rpcClient = rpcClient
 	ctx := context.Background()
 	fix := fixtures()
 
@@ -43,14 +43,14 @@ func TestClient_IPC(t *testing.T) {
 		want, ok := fix["ExecutionPayload"].(*pb.ExecutionPayload)
 		require.Equal(t, true, ok)
 		payloadId := [8]byte{1}
-		resp, err := client.GetPayload(ctx, payloadId)
+		resp, err := srv.GetPayload(ctx, payloadId)
 		require.NoError(t, err)
 		require.DeepEqual(t, want, resp)
 	})
 	t.Run(ForkchoiceUpdatedMethod, func(t *testing.T) {
 		want, ok := fix["ForkchoiceUpdatedResponse"].(*ForkchoiceUpdatedResponse)
 		require.Equal(t, true, ok)
-		payloadID, validHash, err := client.ForkchoiceUpdated(ctx, &pb.ForkchoiceState{}, &pb.PayloadAttributes{})
+		payloadID, validHash, err := srv.ForkchoiceUpdated(ctx, &pb.ForkchoiceState{}, &pb.PayloadAttributes{})
 		require.NoError(t, err)
 		require.DeepEqual(t, want.Status.LatestValidHash, validHash)
 		require.DeepEqual(t, want.PayloadId, payloadID)
@@ -60,20 +60,20 @@ func TestClient_IPC(t *testing.T) {
 		require.Equal(t, true, ok)
 		req, ok := fix["ExecutionPayload"].(*pb.ExecutionPayload)
 		require.Equal(t, true, ok)
-		latestValidHash, err := client.NewPayload(ctx, req)
+		latestValidHash, err := srv.NewPayload(ctx, req)
 		require.NoError(t, err)
 		require.DeepEqual(t, bytesutil.ToBytes32(want.LatestValidHash), bytesutil.ToBytes32(latestValidHash))
 	})
 	t.Run(ExchangeTransitionConfigurationMethod, func(t *testing.T) {
 		want, ok := fix["TransitionConfiguration"].(*pb.TransitionConfiguration)
 		require.Equal(t, true, ok)
-		err := client.ExchangeTransitionConfiguration(ctx, want)
+		err := srv.ExchangeTransitionConfiguration(ctx, want)
 		require.NoError(t, err)
 	})
 	t.Run(ExecutionBlockByNumberMethod, func(t *testing.T) {
 		want, ok := fix["ExecutionBlock"].(*pb.ExecutionBlock)
 		require.Equal(t, true, ok)
-		resp, err := client.LatestExecutionBlock(ctx)
+		resp, err := srv.LatestExecutionBlock(ctx)
 		require.NoError(t, err)
 		require.DeepEqual(t, want, resp)
 	})
@@ -81,7 +81,7 @@ func TestClient_IPC(t *testing.T) {
 		want, ok := fix["ExecutionBlock"].(*pb.ExecutionBlock)
 		require.Equal(t, true, ok)
 		arg := common.BytesToHash([]byte("foo"))
-		resp, err := client.ExecutionBlockByHash(ctx, arg)
+		resp, err := srv.ExecutionBlockByHash(ctx, arg)
 		require.NoError(t, err)
 		require.DeepEqual(t, want, resp)
 	})
@@ -125,8 +125,8 @@ func TestClient_HTTP(t *testing.T) {
 		require.NoError(t, err)
 		defer rpcClient.Close()
 
-		client := &Client{}
-		client.rpc = rpcClient
+		client := &Service{}
+		client.rpcClient = rpcClient
 
 		// We call the RPC method via HTTP and expect a proper result.
 		resp, err := client.GetPayload(ctx, payloadId)
@@ -146,10 +146,10 @@ func TestClient_HTTP(t *testing.T) {
 		}
 		want, ok := fix["ForkchoiceUpdatedResponse"].(*ForkchoiceUpdatedResponse)
 		require.Equal(t, true, ok)
-		client := forkchoiceUpdateSetup(t, forkChoiceState, payloadAttributes, want)
+		srv := forkchoiceUpdateSetup(t, forkChoiceState, payloadAttributes, want)
 
 		// We call the RPC method via HTTP and expect a proper result.
-		payloadID, validHash, err := client.ForkchoiceUpdated(ctx, forkChoiceState, payloadAttributes)
+		payloadID, validHash, err := srv.ForkchoiceUpdated(ctx, forkChoiceState, payloadAttributes)
 		require.NoError(t, err)
 		require.DeepEqual(t, want.Status.LatestValidHash, validHash)
 		require.DeepEqual(t, want.PayloadId, payloadID)
@@ -332,11 +332,11 @@ func TestClient_HTTP(t *testing.T) {
 		require.NoError(t, err)
 		defer rpcClient.Close()
 
-		client := &Client{}
-		client.rpc = rpcClient
+		service := &Service{}
+		service.rpcClient = rpcClient
 
 		// We call the RPC method via HTTP and expect a proper result.
-		resp, err := client.LatestExecutionBlock(ctx)
+		resp, err := service.LatestExecutionBlock(ctx)
 		require.NoError(t, err)
 		require.DeepEqual(t, want, resp)
 	})
@@ -371,8 +371,8 @@ func TestClient_HTTP(t *testing.T) {
 		require.NoError(t, err)
 		defer rpcClient.Close()
 
-		client := &Client{}
-		client.rpc = rpcClient
+		client := &Service{}
+		client.rpcClient = rpcClient
 
 		// We call the RPC method via HTTP and expect a proper result.
 		err = client.ExchangeTransitionConfiguration(ctx, want)
@@ -408,11 +408,11 @@ func TestClient_HTTP(t *testing.T) {
 		require.NoError(t, err)
 		defer rpcClient.Close()
 
-		client := &Client{}
-		client.rpc = rpcClient
+		service := &Service{}
+		service.rpcClient = rpcClient
 
 		// We call the RPC method via HTTP and expect a proper result.
-		resp, err := client.ExecutionBlockByHash(ctx, arg)
+		resp, err := service.ExecutionBlockByHash(ctx, arg)
 		require.NoError(t, err)
 		require.DeepEqual(t, want, resp)
 	})
@@ -449,10 +449,10 @@ func TestExchangeTransitionConfiguration(t *testing.T) {
 		require.NoError(t, err)
 		defer rpcClient.Close()
 
-		client := &Client{}
-		client.rpc = rpcClient
+		service := &Service{}
+		service.rpcClient = rpcClient
 
-		err = client.ExchangeTransitionConfiguration(ctx, request)
+		err = service.ExchangeTransitionConfiguration(ctx, request)
 		require.Equal(t, true, errors.Is(err, ErrConfigMismatch))
 	})
 	t.Run("wrong terminal total difficulty", func(t *testing.T) {
@@ -482,16 +482,17 @@ func TestExchangeTransitionConfiguration(t *testing.T) {
 		require.NoError(t, err)
 		defer rpcClient.Close()
 
-		client := &Client{}
-		client.rpc = rpcClient
+		service := &Service{}
+		service.rpcClient = rpcClient
 
-		err = client.ExchangeTransitionConfiguration(ctx, request)
+		err = service.ExchangeTransitionConfiguration(ctx, request)
 		require.Equal(t, true, errors.Is(err, ErrConfigMismatch))
 	})
 }
 
 type customError struct {
-	code int
+	code    int
+	timeout bool
 }
 
 func (c *customError) ErrorCode() int {
@@ -502,6 +503,10 @@ func (*customError) Error() string {
 	return "something went wrong"
 }
 
+func (c *customError) Timeout() bool {
+	return c.timeout
+}
+
 type dataError struct {
 	code int
 	data interface{}
@@ -534,6 +539,11 @@ func Test_handleRPCError(t *testing.T) {
 			expectedContains: "got an unexpected error",
 			given:            errors.New("foo"),
 		},
+		{
+			name:             "HTTP times out",
+			expectedContains: ErrHTTPTimeout.Error(),
+			given:            &customError{timeout: true},
+		},
 		{
 			name:             "ErrParse",
 			expectedContains: ErrParse.Error(),
@@ -807,7 +817,7 @@ func (*testEngineService) NewPayloadV1(
 	return item
 }
 
-func forkchoiceUpdateSetup(t *testing.T, fcs *pb.ForkchoiceState, att *pb.PayloadAttributes, res *ForkchoiceUpdatedResponse) *Client {
+func forkchoiceUpdateSetup(t *testing.T, fcs *pb.ForkchoiceState, att *pb.PayloadAttributes, res *ForkchoiceUpdatedResponse) *Service {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")
 		defer func() {
@@ -841,13 +851,12 @@ func forkchoiceUpdateSetup(t *testing.T, fcs *pb.ForkchoiceState, att *pb.Payloa
 	rpcClient, err := rpc.DialHTTP(srv.URL)
 	require.NoError(t, err)
 
-	client := &Client{}
-	client.rpc = rpcClient
-
-	return client
+	service := &Service{}
+	service.rpcClient = rpcClient
+	return service
 }
 
-func newPayloadSetup(t *testing.T, status *pb.PayloadStatus, payload *pb.ExecutionPayload) *Client {
+func newPayloadSetup(t *testing.T, status *pb.PayloadStatus, payload *pb.ExecutionPayload) *Service {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")
 		defer func() {
@@ -876,7 +885,7 @@ func newPayloadSetup(t *testing.T, status *pb.PayloadStatus, payload *pb.Executi
 	rpcClient, err := rpc.DialHTTP(srv.URL)
 	require.NoError(t, err)
 
-	client := &Client{}
-	client.rpc = rpcClient
-	return client
+	service := &Service{}
+	service.rpcClient = rpcClient
+	return service
 }

beacon-chain/powchain/errors.go

@@ -1,4 +1,4 @@
-package v1
+package powchain
 
 import "github.com/pkg/errors"
 
@@ -24,12 +24,6 @@ var (
 	// ErrConfigMismatch when the execution node's terminal total difficulty or
 	// terminal block hash received via the API mismatches Prysm's configuration value.
 	ErrConfigMismatch = errors.New("execution client configuration mismatch")
-	// ErrMismatchTerminalBlockHash when the terminal block hash value received via
-	// the API mismatches Prysm's configuration value.
-	ErrMismatchTerminalBlockHash = errors.New("terminal block hash mismatch")
-	// ErrMismatchTerminalTotalDiff when the terminal total difficulty value received via
-	// the API mismatches Prysm's configuration value.
-	ErrMismatchTerminalTotalDiff = errors.New("terminal total difficulty mismatch")
 	// ErrAcceptedSyncingPayloadStatus when the status of the payload is syncing or accepted.
 	ErrAcceptedSyncingPayloadStatus = errors.New("payload status is SYNCING or ACCEPTED")
 	// ErrInvalidPayloadStatus when the status of the payload is invalid.

beacon-chain/powchain/log_processing_test.go

@@ -34,6 +34,11 @@ func TestProcessDepositLog_OK(t *testing.T) {
 	depositCache, err := depositcache.New()
 	require.NoError(t, err)
 
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -97,6 +102,11 @@ func TestProcessDepositLog_InsertsPendingDeposit(t *testing.T) {
 	beaconDB := testDB.SetupDB(t)
 	depositCache, err := depositcache.New()
 	require.NoError(t, err)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
@@ -154,6 +164,11 @@ func TestUnpackDepositLogData_OK(t *testing.T) {
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
 	beaconDB := testDB.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -203,6 +218,11 @@ func TestProcessETH2GenesisLog_8DuplicatePubkeys(t *testing.T) {
 	beaconDB := testDB.SetupDB(t)
 	depositCache, err := depositcache.New()
 	require.NoError(t, err)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
@@ -274,6 +294,11 @@ func TestProcessETH2GenesisLog(t *testing.T) {
 	depositCache, err := depositcache.New()
 	require.NoError(t, err)
 
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -359,6 +384,11 @@ func TestProcessETH2GenesisLog_CorrectNumOfDeposits(t *testing.T) {
 	kvStore := testDB.SetupDB(t)
 	depositCache, err := depositcache.New()
 	require.NoError(t, err)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
@@ -452,6 +482,11 @@ func TestProcessETH2GenesisLog_LargePeriodOfNoLogs(t *testing.T) {
 	kvStore := testDB.SetupDB(t)
 	depositCache, err := depositcache.New()
 	require.NoError(t, err)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
@@ -561,6 +596,11 @@ func TestCheckForChainstart_NoValidator(t *testing.T) {
 func newPowchainService(t *testing.T, eth1Backend *mock.TestAccount, beaconDB db.Database) *Service {
 	depositCache, err := depositcache.New()
 	require.NoError(t, err)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(eth1Backend.ContractAddr),

beacon-chain/powchain/metrics.go

@@ -1,4 +1,4 @@
-package v1
+package powchain
 
 import (
 	"github.com/prometheus/client_golang/prometheus"

beacon-chain/powchain/options.go

@@ -1,6 +1,9 @@
 package powchain
 
 import (
+	"net/http"
+	"time"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
@@ -10,6 +13,9 @@ import (
 	"github.com/prysmaticlabs/prysm/network"
 )
 
+// DefaultRPCHTTPTimeout for HTTP requests via an RPC connection to an execution node.
+const DefaultRPCHTTPTimeout = time.Second * 6
+
 type Option func(s *Service) error
 
 // WithHttpEndpoints deduplicates and parses http endpoints for the powchain service to use,
@@ -32,10 +38,20 @@ func WithHttpEndpoints(endpointStrings []string) Option {
 	}
 }
 
-// WithExecutionClientJWTSecret for authenticating the execution node JSON-RPC endpoint.
-func WithExecutionClientJWTSecret(jwtSecret []byte) Option {
-	return func(s *Service) error {
-		s.cfg.executionEndpointJWTSecret = jwtSecret
+// WithJWTSecret for authenticating the execution node JSON-RPC endpoint.
+func WithJWTSecret(secret []byte) Option {
+	return func(c *Service) error {
+		if len(secret) == 0 {
+			return nil
+		}
+		authTransport := &jwtTransport{
+			underlyingTransport: http.DefaultTransport,
+			jwtSecret:           secret,
+		}
+		c.cfg.httpRPCClient = &http.Client{
+			Timeout:   DefaultRPCHTTPTimeout,
+			Transport: authTransport,
+		}
 		return nil
 	}
 }

beacon-chain/powchain/service.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"math/big"
+	"net/http"
 	"reflect"
 	"runtime/debug"
 	"sort"
@@ -24,10 +25,10 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/transition"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
-	engine "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain/types"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	nativev1 "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/v1"
@@ -72,8 +73,6 @@ var (
 	logPeriod = 1 * time.Minute
 	// threshold of how old we will accept an eth1 node's head to be.
 	eth1Threshold = 20 * time.Minute
-	// error when eth1 node is not synced.
-	errNotSynced = errors.New("eth1 node is still syncing")
 	// error when eth1 node is too far behind.
 	errFarBehind = errors.Errorf("eth1 head is more than %s behind from current wall clock time", eth1Threshold.String())
 )
@@ -123,21 +122,22 @@ type RPCDataFetcher interface {
 // RPCClient defines the rpc methods required to interact with the eth1 node.
 type RPCClient interface {
 	BatchCall(b []gethRPC.BatchElem) error
+	CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error
 }
 
 // config defines a config struct for dependencies into the service.
 type config struct {
-	depositContractAddr        common.Address
-	beaconDB                   db.HeadAccessDatabase
-	depositCache               *depositcache.DepositCache
-	stateNotifier              statefeed.Notifier
-	stateGen                   *stategen.State
-	eth1HeaderReqLimit         uint64
-	beaconNodeStatsUpdater     BeaconNodeStatsUpdater
-	httpEndpoints              []network.Endpoint
-	executionEndpointJWTSecret []byte
-	currHttpEndpoint           network.Endpoint
-	finalizedStateAtStartup    state.BeaconState
+	depositContractAddr     common.Address
+	beaconDB                db.HeadAccessDatabase
+	depositCache            *depositcache.DepositCache
+	stateNotifier           statefeed.Notifier
+	stateGen                *stategen.State
+	eth1HeaderReqLimit      uint64
+	beaconNodeStatsUpdater  BeaconNodeStatsUpdater
+	httpEndpoints           []network.Endpoint
+	httpRPCClient           *http.Client
+	currHttpEndpoint        network.Endpoint
+	finalizedStateAtStartup state.BeaconState
 }
 
 // Service fetches important information about the canonical
@@ -156,7 +156,6 @@ type Service struct {
 	headTicker              *time.Ticker
 	httpLogger              bind.ContractFilterer
 	eth1DataFetcher         RPCDataFetcher
-	engineAPIClient         engine.Caller
 	rpcClient               RPCClient
 	headerCache             *headerCache // cache to store block hash/block height.
 	latestEth1Data          *ethpb.LatestETH1Data
@@ -220,6 +219,7 @@ func NewService(ctx context.Context, opts ...Option) (*Service, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to retrieve eth1 data")
 	}
+
 	if err := s.initializeEth1Data(ctx, eth1Data); err != nil {
 		return nil, err
 	}
@@ -228,6 +228,14 @@ func NewService(ctx context.Context, opts ...Option) (*Service, error) {
 
 // Start a web3 service's main event loop.
 func (s *Service) Start() {
+
+	if err := s.connectToPowChain(); err != nil {
+		log.WithError(err).Fatal("Could not connect to execution endpoint")
+	}
+
+	log.WithFields(logrus.Fields{
+		"endpoint": logs.MaskCredentialsLogging(s.cfg.currHttpEndpoint.Url),
+	}).Info("Connected to Ethereum execution client RPC")
 	// If the chain has not started already and we don't have access to eth1 nodes, we will not be
 	// able to generate the genesis state.
 	if !s.chainStartData.Chainstarted && s.cfg.currHttpEndpoint.Url == "" {
@@ -242,27 +250,15 @@ func (s *Service) Start() {
 		}
 	}
 
-	// Exit early if eth1 endpoint is not set.
-	if s.cfg.currHttpEndpoint.Url == "" {
-		return
-	}
+	s.isRunning = true
 
-	if err := s.initializeEngineAPIClient(s.ctx); err != nil {
-		log.WithError(err).Fatal("unable to initialize engine API client")
-	}
+	// Poll the execution client connection and fallback if errors occur.
+	go s.pollConnectionStatus()
 
 	// Check transition configuration for the engine API client in the background.
-	go s.checkTransitionConfiguration(s.ctx, make(chan *statefeed.BlockProcessedData, 1))
+	go s.checkTransitionConfiguration(s.ctx, make(chan *feed.Event, 1))
 
-	go func() {
-		s.isRunning = true
-		s.waitForConnection()
-		if s.ctx.Err() != nil {
-			log.Info("Context closed, exiting pow goroutine")
-			return
-		}
-		s.run(s.ctx.Done())
-	}()
+	go s.run(s.ctx.Done())
 }
 
 // Stop the web3 service's main event loop and associated goroutines.
@@ -305,12 +301,6 @@ func (s *Service) Status() error {
 	return s.runError
 }
 
-// EngineAPIClient returns the associated engine API client to interact
-// with an execution node via JSON-RPC.
-func (s *Service) EngineAPIClient() engine.Caller {
-	return s.engineAPIClient
-}
-
 func (s *Service) updateBeaconNodeStats() {
 	bs := clientstats.BeaconNodeStats{}
 	if len(s.cfg.httpEndpoints) > 1 {
@@ -389,19 +379,24 @@ func (s *Service) followBlockHeight(_ context.Context) (uint64, error) {
 func (s *Service) connectToPowChain() error {
 	httpClient, rpcClient, err := s.dialETH1Nodes(s.cfg.currHttpEndpoint)
 	if err != nil {
-		return errors.Wrap(err, "could not dial eth1 nodes")
+		return errors.Wrap(err, "could not dial execution node")
 	}
 
 	depositContractCaller, err := contracts.NewDepositContractCaller(s.cfg.depositContractAddr, httpClient)
 	if err != nil {
-		return errors.Wrap(err, "could not create deposit contract caller")
+		return errors.Wrap(err, "could not initialize deposit contract caller")
 	}
 
 	if httpClient == nil || rpcClient == nil || depositContractCaller == nil {
-		return errors.New("eth1 client is nil")
+		return errors.New("execution client RPC is nil")
 	}
+	s.httpLogger = httpClient
+	s.eth1DataFetcher = httpClient
+	s.depositContractCaller = depositContractCaller
+	s.rpcClient = rpcClient
 
-	s.initializeConnection(httpClient, rpcClient, depositContractCaller)
+	s.updateConnectedETH1(true)
+	s.runError = nil
 	return nil
 }
 
@@ -424,15 +419,6 @@ func (s *Service) dialETH1Nodes(endpoint network.Endpoint) (*ethclient.Client, *
 		httpRPCClient.Close()
 		httpClient.Close()
 	}
-	syncProg, err := httpClient.SyncProgress(s.ctx)
-	if err != nil {
-		closeClients()
-		return nil, nil, err
-	}
-	if syncProg != nil {
-		closeClients()
-		return nil, nil, errors.New("eth1 node has not finished syncing yet")
-	}
 	// Make a simple call to ensure we are actually connected to a working node.
 	cID, err := httpClient.ChainID(s.ctx)
 	if err != nil {
@@ -456,17 +442,6 @@ func (s *Service) dialETH1Nodes(endpoint network.Endpoint) (*ethclient.Client, *
 	return httpClient, httpRPCClient, nil
 }
 
-func (s *Service) initializeConnection(
-	httpClient *ethclient.Client,
-	rpcClient *gethRPC.Client,
-	contractCaller *contracts.DepositContractCaller,
-) {
-	s.httpLogger = httpClient
-	s.eth1DataFetcher = httpClient
-	s.depositContractCaller = contractCaller
-	s.rpcClient = rpcClient
-}
-
 // closes down our active eth1 clients.
 func (s *Service) closeClients() {
 	gethClient, ok := s.rpcClient.(*gethRPC.Client)
@@ -479,30 +454,8 @@ func (s *Service) closeClients() {
 	}
 }
 
-func (s *Service) waitForConnection() {
-	errConnect := s.connectToPowChain()
-	if errConnect == nil {
-		synced, errSynced := s.isEth1NodeSynced()
-		// Resume if eth1 node is synced.
-		if synced {
-			s.updateConnectedETH1(true)
-			s.runError = nil
-			log.WithFields(logrus.Fields{
-				"endpoint": logs.MaskCredentialsLogging(s.cfg.currHttpEndpoint.Url),
-			}).Info("Connected to eth1 proof-of-work chain")
-			return
-		}
-		if errSynced != nil {
-			s.runError = errSynced
-			log.WithError(errSynced).Error("Could not check sync status of eth1 chain")
-		}
-	}
-	if errConnect != nil {
-		s.runError = errConnect
-		log.WithError(errConnect).Error("Could not connect to powchain endpoint")
-	}
+func (s *Service) pollConnectionStatus() {
 	// Use a custom logger to only log errors
-	// once in  a while.
 	logCounter := 0
 	errorLogger := func(err error, msg string) {
 		if logCounter > logThreshold {
@@ -511,7 +464,6 @@ func (s *Service) waitForConnection() {
 		}
 		logCounter++
 	}
-
 	ticker := time.NewTicker(backOffPeriod)
 	defer ticker.Stop()
 	for {
@@ -525,23 +477,6 @@ func (s *Service) waitForConnection() {
 				s.fallbackToNextEndpoint()
 				continue
 			}
-			synced, errSynced := s.isEth1NodeSynced()
-			if errSynced != nil {
-				errorLogger(errSynced, "Could not check sync status of eth1 chain")
-				s.runError = errSynced
-				s.fallbackToNextEndpoint()
-				continue
-			}
-			if synced {
-				s.updateConnectedETH1(true)
-				s.runError = nil
-				log.WithFields(logrus.Fields{
-					"endpoint": logs.MaskCredentialsLogging(s.cfg.currHttpEndpoint.Url),
-				}).Info("Connected to eth1 proof-of-work chain")
-				return
-			}
-			s.runError = errNotSynced
-			log.Debug("Eth1 node is currently syncing")
 		case <-s.ctx.Done():
 			log.Debug("Received cancelled context,closing existing powchain service")
 			return
@@ -573,7 +508,10 @@ func (s *Service) retryETH1Node(err error) {
 	// Back off for a while before
 	// resuming dialing the eth1 node.
 	time.Sleep(backOffPeriod)
-	s.waitForConnection()
+	if err := s.connectToPowChain(); err != nil {
+		s.runError = err
+		return
+	}
 	// Reset run error in the event of a successful connection.
 	s.runError = nil
 }
@@ -1056,19 +994,6 @@ func (s *Service) ensureValidPowchainData(ctx context.Context) error {
 	return nil
 }
 
-// Initializes a connection to the engine API if an execution provider endpoint is set.
-func (s *Service) initializeEngineAPIClient(ctx context.Context) error {
-	opts := []engine.Option{
-		engine.WithJWTSecret(s.cfg.executionEndpointJWTSecret),
-	}
-	client, err := engine.New(ctx, s.cfg.currHttpEndpoint.Url, opts...)
-	if err != nil {
-		return err
-	}
-	s.engineAPIClient = client
-	return nil
-}
-
 func dedupEndpoints(endpoints []string) []string {
 	selectionMap := make(map[string]bool)
 	newEndpoints := make([]string, 0, len(endpoints))

beacon-chain/powchain/service_test.go

@@ -3,11 +3,9 @@ package powchain
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"math/big"
 	"strings"
-	"sync"
 	"testing"
 	"time"
 
@@ -15,6 +13,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
 	"github.com/ethereum/go-ethereum/common"
 	gethTypes "github.com/ethereum/go-ethereum/core/types"
+	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/async/event"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
 	dbutil "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
@@ -128,6 +127,11 @@ func TestStart_OK(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -157,90 +161,13 @@ func TestStart_NoHttpEndpointDefinedFails_WithoutChainStarted(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
-	s, err := NewService(context.Background(),
+	_, err = NewService(context.Background(),
 		WithHttpEndpoints([]string{""}),
 		WithDepositContractAddress(testAcc.ContractAddr),
 		WithDatabase(beaconDB),
 	)
 	require.NoError(t, err)
-	// Set custom exit func so test can proceed
-	log.Logger.ExitFunc = func(i int) {
-		panic(i)
-	}
-	defer func() {
-		log.Logger.ExitFunc = nil
-	}()
-	wg := new(sync.WaitGroup)
-	wg.Add(1)
-	// Expect Start function to fail from a fatal call due
-	// to no state existing.
-	go func() {
-		defer func() {
-			if r := recover(); r != nil {
-				wg.Done()
-			}
-		}()
-		s.Start()
-	}()
-	util.WaitTimeout(wg, time.Second)
-	require.LogsContain(t, hook, "cannot create genesis state: no eth1 http endpoint defined")
-	hook.Reset()
-}
-
-func TestStart_NoHttpEndpointDefinedSucceeds_WithGenesisState(t *testing.T) {
-	hook := logTest.NewGlobal()
-	beaconDB := dbutil.SetupDB(t)
-	testAcc, err := mock.Setup()
-	require.NoError(t, err, "Unable to set up simulated backend")
-	st, _ := util.DeterministicGenesisState(t, 10)
-	b := util.NewBeaconBlock()
-	genRoot, err := b.HashTreeRoot()
-	require.NoError(t, err)
-	require.NoError(t, beaconDB.SaveState(context.Background(), st, genRoot))
-	require.NoError(t, beaconDB.SaveGenesisBlockRoot(context.Background(), genRoot))
-	depositCache, err := depositcache.New()
-	require.NoError(t, err)
-	s, err := NewService(context.Background(),
-		WithHttpEndpoints([]string{""}),
-		WithDepositContractAddress(testAcc.ContractAddr),
-		WithDatabase(beaconDB),
-		WithDepositCache(depositCache),
-	)
-	require.NoError(t, err)
-
-	wg := new(sync.WaitGroup)
-	wg.Add(1)
-
-	go func() {
-		s.Start()
-		wg.Done()
-	}()
-	s.cancel()
-	util.WaitTimeout(wg, time.Second)
-	require.LogsDoNotContain(t, hook, "cannot create genesis state: no eth1 http endpoint defined")
-	hook.Reset()
-}
-
-func TestStart_NoHttpEndpointDefinedSucceeds_WithChainStarted(t *testing.T) {
-	hook := logTest.NewGlobal()
-	beaconDB := dbutil.SetupDB(t)
-	testAcc, err := mock.Setup()
-	require.NoError(t, err, "Unable to set up simulated backend")
-
-	require.NoError(t, beaconDB.SavePowchainData(context.Background(), &ethpb.ETH1ChainData{
-		ChainstartData: &ethpb.ChainStartData{Chainstarted: true},
-		Trie:           &ethpb.SparseMerkleTrie{},
-	}))
-	s, err := NewService(context.Background(),
-		WithHttpEndpoints([]string{""}),
-		WithDepositContractAddress(testAcc.ContractAddr),
-		WithDatabase(beaconDB),
-	)
-	require.NoError(t, err)
-
-	s.Start()
-	require.LogsDoNotContain(t, hook, "cannot create genesis state: no eth1 http endpoint defined")
-	hook.Reset()
+	require.LogsDoNotContain(t, hook, "missing address")
 }
 
 func TestStop_OK(t *testing.T) {
@@ -248,6 +175,11 @@ func TestStop_OK(t *testing.T) {
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -273,6 +205,11 @@ func TestService_Eth1Synced(t *testing.T) {
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -298,6 +235,11 @@ func TestFollowBlock_OK(t *testing.T) {
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -371,6 +313,11 @@ func TestStatus(t *testing.T) {
 func TestHandlePanic_OK(t *testing.T) {
 	hook := logTest.NewGlobal()
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -403,6 +350,11 @@ func TestLogTillGenesis_OK(t *testing.T) {
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -537,6 +489,11 @@ func TestNewService_EarliestVotingBlock(t *testing.T) {
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -588,6 +545,11 @@ func TestNewService_Eth1HeaderRequLimit(t *testing.T) {
 	require.NoError(t, err, "Unable to set up simulated backend")
 	beaconDB := dbutil.SetupDB(t)
 
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	s1, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -616,13 +578,26 @@ func (mbs *mockBSUpdater) Update(bs clientstats.BeaconNodeStats) {
 var _ BeaconNodeStatsUpdater = &mockBSUpdater{}
 
 func TestServiceFallbackCorrectly(t *testing.T) {
-	firstEndpoint := "A"
-	secondEndpoint := "B"
-
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
 	beaconDB := dbutil.SetupDB(t)
 
+	server, firstEndpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
+	server2, secondEndpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server2.Stop()
+	})
+	server3, thirdEndpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server3.Stop()
+	})
+
 	mbs := &mockBSUpdater{}
 	s1, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{firstEndpoint}),
@@ -630,10 +605,11 @@ func TestServiceFallbackCorrectly(t *testing.T) {
 		WithDatabase(beaconDB),
 		WithBeaconNodeStatsUpdater(mbs),
 	)
-	s1.cfg.beaconNodeStatsUpdater = mbs
 	require.NoError(t, err)
+	s1.cfg.beaconNodeStatsUpdater = mbs
 
 	assert.Equal(t, firstEndpoint, s1.cfg.currHttpEndpoint.Url, "Unexpected http endpoint")
+
 	// Stay at the first endpoint.
 	s1.fallbackToNextEndpoint()
 	assert.Equal(t, firstEndpoint, s1.cfg.currHttpEndpoint.Url, "Unexpected http endpoint")
@@ -645,17 +621,11 @@ func TestServiceFallbackCorrectly(t *testing.T) {
 	assert.Equal(t, secondEndpoint, s1.cfg.currHttpEndpoint.Url, "Unexpected http endpoint")
 	assert.Equal(t, true, mbs.lastBS.SyncEth1FallbackConfigured, "SyncEth1FallbackConfigured in clientstats update should be true when > 1 endpoint is configured")
 
-	thirdEndpoint := "C"
-	fourthEndpoint := "D"
-
-	s1.cfg.httpEndpoints = append(s1.cfg.httpEndpoints, network.Endpoint{Url: thirdEndpoint}, network.Endpoint{Url: fourthEndpoint})
+	s1.cfg.httpEndpoints = append(s1.cfg.httpEndpoints, network.Endpoint{Url: thirdEndpoint})
 
 	s1.fallbackToNextEndpoint()
 	assert.Equal(t, thirdEndpoint, s1.cfg.currHttpEndpoint.Url, "Unexpected http endpoint")
 
-	s1.fallbackToNextEndpoint()
-	assert.Equal(t, fourthEndpoint, s1.cfg.currHttpEndpoint.Url, "Unexpected http endpoint")
-
 	// Rollover correctly back to the first endpoint
 	s1.fallbackToNextEndpoint()
 	assert.Equal(t, firstEndpoint, s1.cfg.currHttpEndpoint.Url, "Unexpected http endpoint")
@@ -685,8 +655,13 @@ func TestService_EnsureConsistentPowchainData(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
 	cache, err := depositcache.New()
 	require.NoError(t, err)
-
+	srv, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		srv.Stop()
+	})
 	s1, err := NewService(context.Background(),
+		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
 		WithDepositCache(cache),
 	)
@@ -710,7 +685,13 @@ func TestService_InitializeCorrectly(t *testing.T) {
 	cache, err := depositcache.New()
 	require.NoError(t, err)
 
+	srv, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		srv.Stop()
+	})
 	s1, err := NewService(context.Background(),
+		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
 		WithDepositCache(cache),
 	)
@@ -733,8 +714,13 @@ func TestService_EnsureValidPowchainData(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
 	cache, err := depositcache.New()
 	require.NoError(t, err)
-
+	srv, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		srv.Stop()
+	})
 	s1, err := NewService(context.Background(),
+		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
 		WithDepositCache(cache),
 	)
@@ -825,8 +811,16 @@ func TestTimestampIsChecked(t *testing.T) {
 }
 
 func TestETH1Endpoints(t *testing.T) {
-	firstEndpoint := "A"
-	secondEndpoint := "B"
+	server, firstEndpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
+	server, secondEndpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	endpoints := []string{firstEndpoint, secondEndpoint}
 
 	testAcc, err := mock.Setup()

beacon-chain/powchain/testing/BUILD.bazel

@@ -4,6 +4,7 @@ go_library(
     name = "go_default_library",
     testonly = True,
     srcs = [
+        "mock_engine_client.go",
         "mock_faulty_powchain.go",
         "mock_powchain.go",
     ],
@@ -16,7 +17,9 @@ go_library(
         "//beacon-chain/powchain/types:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/v1:go_default_library",
+        "//config/params:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//proto/engine/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "@com_github_ethereum_go_ethereum//accounts/abi/bind/backends:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",

beacon-chain/powchain/testing/mock_engine_client.go

@@ -1,4 +1,4 @@
-package mocks
+package testing
 
 import (
 	"context"
@@ -19,12 +19,13 @@ type EngineClient struct {
 	ErrLatestExecBlock    error
 	ErrExecBlockByHash    error
 	ErrForkchoiceUpdated  error
+	ErrNewPayload         error
 	BlockByHashMap        map[[32]byte]*pb.ExecutionBlock
 }
 
 // NewPayload --
 func (e *EngineClient) NewPayload(_ context.Context, _ *pb.ExecutionPayload) ([]byte, error) {
-	return e.NewPayloadResp, nil
+	return e.NewPayloadResp, e.ErrNewPayload
 }
 
 // ForkchoiceUpdated --

beacon-chain/powchain/testing/mock_powchain.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"fmt"
 	"math/big"
+	"net/http/httptest"
 	"time"
 
 	"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
@@ -16,6 +17,7 @@ import (
 	"github.com/prysmaticlabs/prysm/async/event"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain/types"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/config/params"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 )
@@ -142,6 +144,10 @@ type RPCClient struct {
 	Backend *backends.SimulatedBackend
 }
 
+func (*RPCClient) CallContext(_ context.Context, _ interface{}, _ string, _ ...interface{}) error {
+	return nil
+}
+
 // BatchCall --
 func (r *RPCClient) BatchCall(b []rpc.BatchElem) error {
 	if r.Backend == nil {
@@ -175,3 +181,28 @@ func (m *POWChain) InsertBlock(height int, time uint64, hash []byte) *POWChain {
 func (m *POWChain) BlockExistsWithCache(ctx context.Context, hash common.Hash) (bool, *big.Int, error) {
 	return m.BlockExists(ctx, hash)
 }
+
+func SetupRPCServer() (*rpc.Server, string, error) {
+	srv := rpc.NewServer()
+	if err := srv.RegisterName("eth", &testETHRPC{}); err != nil {
+		return nil, "", err
+	}
+	if err := srv.RegisterName("net", &testETHRPC{}); err != nil {
+		return nil, "", err
+	}
+	hs := httptest.NewUnstartedServer(srv)
+	hs.Start()
+	return srv, hs.URL, nil
+}
+
+type testETHRPC struct{}
+
+func (*testETHRPC) NoArgsRets() {}
+
+func (*testETHRPC) ChainId(_ context.Context) *hexutil.Big {
+	return (*hexutil.Big)(big.NewInt(int64(params.BeaconConfig().DepositChainID)))
+}
+
+func (*testETHRPC) Version(_ context.Context) string {
+	return fmt.Sprintf("%d", params.BeaconConfig().DepositNetworkID)
+}

beacon-chain/rpc/BUILD.bazel

@@ -22,7 +22,6 @@ go_library(
         "//beacon-chain/operations/voluntaryexits:go_default_library",
         "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
-        "//beacon-chain/powchain/engine-api-client/v1:go_default_library",
         "//beacon-chain/rpc/eth/beacon:go_default_library",
         "//beacon-chain/rpc/eth/debug:go_default_library",
         "//beacon-chain/rpc/eth/events:go_default_library",

beacon-chain/rpc/eth/validator/BUILD.bazel

@@ -63,7 +63,6 @@ go_test(
         "//beacon-chain/operations/voluntaryexits:go_default_library",
         "//beacon-chain/p2p/testing:go_default_library",
         "//beacon-chain/p2p/types:go_default_library",
-        "//beacon-chain/powchain/engine-api-client/v1/mocks:go_default_library",
         "//beacon-chain/powchain/testing:go_default_library",
         "//beacon-chain/rpc/prysm/v1alpha1/validator:go_default_library",
         "//beacon-chain/rpc/testutil:go_default_library",

beacon-chain/rpc/eth/validator/validator_test.go

@@ -25,7 +25,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits"
 	p2pmock "github.com/prysmaticlabs/prysm/beacon-chain/p2p/testing"
 	p2pType "github.com/prysmaticlabs/prysm/beacon-chain/p2p/types"
-	"github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1/mocks"
 	mockPOW "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing"
 	v1alpha1validator "github.com/prysmaticlabs/prysm/beacon-chain/rpc/prysm/v1alpha1/validator"
 	"github.com/prysmaticlabs/prysm/beacon-chain/rpc/testutil"
@@ -1020,7 +1019,7 @@ func TestProduceBlockV2(t *testing.T) {
 		require.NoError(t, db.SaveHeadBlockRoot(ctx, parentRoot), "Could not save genesis state")
 
 		v1Alpha1Server := &v1alpha1validator.Server{
-			ExecutionEngineCaller: &mocks.EngineClient{
+			ExecutionEngineCaller: &mockPOW.EngineClient{
 				ExecutionBlock: &enginev1.ExecutionBlock{
 					TotalDifficulty: "0x1",
 				},

beacon-chain/rpc/prysm/v1alpha1/validator/BUILD.bazel

@@ -49,7 +49,6 @@ go_library(
         "//beacon-chain/operations/voluntaryexits:go_default_library",
         "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
-        "//beacon-chain/powchain/engine-api-client/v1:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/stategen:go_default_library",
         "//beacon-chain/sync:go_default_library",
@@ -134,7 +133,6 @@ go_test(
         "//beacon-chain/operations/synccommittee:go_default_library",
         "//beacon-chain/operations/voluntaryexits:go_default_library",
         "//beacon-chain/p2p/testing:go_default_library",
-        "//beacon-chain/powchain/engine-api-client/v1/mocks:go_default_library",
         "//beacon-chain/powchain/testing:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/stategen:go_default_library",

beacon-chain/rpc/prysm/v1alpha1/validator/proposer_execution_payload_test.go

@@ -10,7 +10,6 @@ import (
 	types "github.com/prysmaticlabs/eth2-types"
 	chainMock "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
 	dbTest "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	"github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1/mocks"
 	powtesting "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/config/params"
@@ -133,7 +132,7 @@ func TestServer_getExecutionPayload(t *testing.T) {
 			params.OverrideBeaconConfig(cfg)
 
 			vs := &Server{
-				ExecutionEngineCaller: &mocks.EngineClient{PayloadIDBytes: tt.payloadID, ErrForkchoiceUpdated: tt.forkchoiceErr},
+				ExecutionEngineCaller: &powtesting.EngineClient{PayloadIDBytes: tt.payloadID, ErrForkchoiceUpdated: tt.forkchoiceErr},
 				HeadFetcher:           &chainMock.ChainService{State: tt.st},
 				BeaconDB:              beaconDB,
 			}
@@ -260,7 +259,7 @@ func TestServer_getPowBlockHashAtTerminalTotalDifficulty(t *testing.T) {
 				}
 			}
 			vs := &Server{
-				ExecutionEngineCaller: &mocks.EngineClient{
+				ExecutionEngineCaller: &powtesting.EngineClient{
 					ErrLatestExecBlock: tt.errLatestExecutionBlk,
 					ExecutionBlock:     tt.currentPowBlock,
 					BlockByHashMap:     m,
@@ -335,7 +334,7 @@ func TestServer_getTerminalBlockHashIfExists(t *testing.T) {
 			c.HashesByHeight[0] = tt.wantTerminalBlockHash
 			vs := &Server{
 				Eth1BlockFetcher: c,
-				ExecutionEngineCaller: &mocks.EngineClient{
+				ExecutionEngineCaller: &powtesting.EngineClient{
 					ExecutionBlock: tt.currentPowBlock,
 					BlockByHashMap: m,
 				},

beacon-chain/rpc/prysm/v1alpha1/validator/proposer_test.go

@@ -21,7 +21,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/synccommittee"
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits"
 	mockp2p "github.com/prysmaticlabs/prysm/beacon-chain/p2p/testing"
-	"github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1/mocks"
 	mockPOW "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
@@ -2341,7 +2340,7 @@ func TestProposer_GetBeaconBlock_BellatrixEpoch(t *testing.T) {
 		ExitPool:          voluntaryexits.NewPool(),
 		StateGen:          stategen.New(db),
 		SyncCommitteePool: synccommittee.NewStore(),
-		ExecutionEngineCaller: &mocks.EngineClient{
+		ExecutionEngineCaller: &mockPOW.EngineClient{
 			PayloadIDBytes:   &enginev1.PayloadIDBytes{1},
 			ExecutionPayload: payload,
 		},

beacon-chain/rpc/prysm/v1alpha1/validator/server.go

@@ -23,7 +23,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
-	enginev1 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	"github.com/prysmaticlabs/prysm/beacon-chain/sync"
 	"github.com/prysmaticlabs/prysm/config/params"
@@ -66,7 +65,7 @@ type Server struct {
 	StateGen               stategen.StateManager
 	ReplayerBuilder        stategen.ReplayerBuilder
 	BeaconDB               db.HeadAccessDatabase
-	ExecutionEngineCaller  enginev1.Caller
+	ExecutionEngineCaller  powchain.EngineCaller
 }
 
 // WaitForActivation checks if a validator public key exists in the active validator registry of the current

beacon-chain/rpc/service.go

@@ -26,7 +26,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
-	enginev1 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/rpc/eth/beacon"
 	"github.com/prysmaticlabs/prysm/beacon-chain/rpc/eth/debug"
 	"github.com/prysmaticlabs/prysm/beacon-chain/rpc/eth/events"
@@ -109,7 +108,7 @@ type Config struct {
 	OperationNotifier       opfeed.Notifier
 	StateGen                *stategen.State
 	MaxMsgSize              int
-	ExecutionEngineCaller   enginev1.Caller
+	ExecutionEngineCaller   powchain.EngineCaller
 }
 
 // NewService instantiates a new RPC service instance that will

beacon-chain/sync/BUILD.bazel

@@ -76,6 +76,7 @@ go_library(
         "//beacon-chain/p2p/encoder:go_default_library",
         "//beacon-chain/p2p/peers:go_default_library",
         "//beacon-chain/p2p/types:go_default_library",
+        "//beacon-chain/powchain:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/stategen:go_default_library",
         "//cache/lru:go_default_library",
@@ -184,6 +185,7 @@ go_test(
         "//beacon-chain/p2p/peers:go_default_library",
         "//beacon-chain/p2p/testing:go_default_library",
         "//beacon-chain/p2p/types:go_default_library",
+        "//beacon-chain/powchain:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/stategen:go_default_library",
         "//beacon-chain/state/v1:go_default_library",

beacon-chain/sync/genesis/BUILD.bazel

@@ -0,0 +1,17 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "api.go",
+        "file.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/beacon-chain/sync/genesis",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//api/client/beacon:go_default_library",
+        "//beacon-chain/db:go_default_library",
+        "//io/file:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+    ],
+)

beacon-chain/sync/genesis/api.go

@@ -0,0 +1,35 @@
+package genesis
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/api/client/beacon"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+)
+
+// APIInitializer manages initializing the genesis state and block to prepare the beacon node for syncing.
+// The genesis state is retrieved from the remote beacon node api, using the debug state retrieval endpoint.
+type APIInitializer struct {
+	c *beacon.Client
+}
+
+// NewAPIInitializer creates an APIInitializer, handling the set up of a beacon node api client
+// using the provided host string.
+func NewAPIInitializer(beaconNodeHost string) (*APIInitializer, error) {
+	c, err := beacon.NewClient(beaconNodeHost)
+	if err != nil {
+		return nil, errors.Wrapf(err, "unable to parse beacon node url or hostname - %s", beaconNodeHost)
+	}
+	return &APIInitializer{c: c}, nil
+}
+
+// Initialize downloads origin state and block for checkpoint sync and initializes database records to
+// prepare the node to begin syncing from that point.
+func (dl *APIInitializer) Initialize(ctx context.Context, d db.Database) error {
+	sb, err := dl.c.GetState(ctx, beacon.IdGenesis)
+	if err != nil {
+		return errors.Wrapf(err, "Error retrieving genesis state from %s", dl.c.NodeURL())
+	}
+	return d.LoadGenesis(ctx, sb)
+}

beacon-chain/sync/genesis/file.go

@@ -0,0 +1,59 @@
+package genesis
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/prysmaticlabs/prysm/io/file"
+
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+)
+
+// Initializer describes a type that is able to obtain the checkpoint sync data (BeaconState and SignedBeaconBlock)
+// in some way and perform database setup to prepare the beacon node for syncing from the given checkpoint.
+// See FileInitializer and APIInitializer.
+type Initializer interface {
+	Initialize(ctx context.Context, d db.Database) error
+}
+
+// NewFileInitializer validates the given path information and creates an Initializer which will
+// use the provided state and block files to prepare the node for checkpoint sync.
+func NewFileInitializer(statePath string) (*FileInitializer, error) {
+	var err error
+	if err = existsAndIsFile(statePath); err != nil {
+		return nil, err
+	}
+	// stat just to make sure it actually exists and is a file
+	return &FileInitializer{statePath: statePath}, nil
+}
+
+// FileInitializer initializes a beacon-node database genesis state and block
+// using ssz-encoded state data stored in files on the local filesystem.
+type FileInitializer struct {
+	statePath string
+}
+
+// Initialize is called in the BeaconNode db startup code if an Initializer is present.
+// Initialize prepares the beacondb using the provided genesis state.
+func (fi *FileInitializer) Initialize(ctx context.Context, d db.Database) error {
+	serState, err := file.ReadFileAsBytes(fi.statePath)
+	if err != nil {
+		return errors.Wrapf(err, "error reading state file %s for checkpoint sync init", fi.statePath)
+	}
+	return d.LoadGenesis(ctx, serState)
+}
+
+var _ Initializer = &FileInitializer{}
+
+func existsAndIsFile(path string) error {
+	info, err := os.Stat(path)
+	if err != nil {
+		return errors.Wrapf(err, "error checking existence of ssz-encoded file %s for genesis state init", path)
+	}
+	if info.IsDir() {
+		return fmt.Errorf("%s is a directory, please specify full path to file", path)
+	}
+	return nil
+}

beacon-chain/sync/pending_blocks_queue.go

@@ -12,6 +12,7 @@ import (
 	"github.com/prysmaticlabs/prysm/async"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	p2ptypes "github.com/prysmaticlabs/prysm/beacon-chain/p2p/types"
+	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
 	"github.com/prysmaticlabs/prysm/config/params"
 	"github.com/prysmaticlabs/prysm/crypto/rand"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
@@ -162,9 +163,11 @@ func (s *Service) processPendingBlocks(ctx context.Context) error {
 			}
 
 			if err := s.cfg.chain.ReceiveBlock(ctx, b, blkRoot); err != nil {
-				log.Debugf("Could not process block from slot %d: %v", b.Block().Slot(), err)
-				s.setBadBlock(ctx, blkRoot)
-				tracing.AnnotateError(span, err)
+				if !errors.Is(err, powchain.ErrHTTPTimeout) {
+					log.Debugf("Could not process block from slot %d: %v", b.Block().Slot(), err)
+					tracing.AnnotateError(span, err)
+					s.setBadBlock(ctx, blkRoot)
+				}
 				// In the next iteration of the queue, this block will be removed from
 				// the pending queue as it has been marked as a 'bad' block.
 				span.End()

beacon-chain/sync/pending_blocks_queue_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p/peers"
 	p2ptest "github.com/prysmaticlabs/prysm/beacon-chain/p2p/testing"
 	p2ptypes "github.com/prysmaticlabs/prysm/beacon-chain/p2p/types"
+	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	"github.com/prysmaticlabs/prysm/config/params"
 	"github.com/prysmaticlabs/prysm/crypto/rand"
@@ -110,6 +111,86 @@ func TestRegularSyncBeaconBlockSubscriber_ProcessPendingBlocks1(t *testing.T) {
 	assert.Equal(t, 2, len(r.seenPendingBlocks), "Incorrect size for seen pending block")
 }
 
+func TestRegularSyncBeaconBlockSubscriber_ExecutionEngineTimesOut(t *testing.T) {
+	db := dbtest.SetupDB(t)
+
+	p1 := p2ptest.NewTestP2P(t)
+	r := &Service{
+		cfg: &config{
+			p2p:      p1,
+			beaconDB: db,
+			chain: &mock.ChainService{
+				FinalizedCheckPoint: &ethpb.Checkpoint{
+					Epoch: 0,
+				},
+				ReceiveBlockMockErr: powchain.ErrHTTPTimeout,
+			},
+			stateGen: stategen.New(db),
+		},
+		slotToPendingBlocks: gcache.New(time.Second, 2*time.Second),
+		seenPendingBlocks:   make(map[[32]byte]bool),
+	}
+	r.initCaches()
+
+	b0 := util.NewBeaconBlock()
+	wsb, err := wrapper.WrappedSignedBeaconBlock(b0)
+	require.NoError(t, err)
+	require.NoError(t, r.cfg.beaconDB.SaveBlock(context.Background(), wsb))
+	b0Root, err := b0.Block.HashTreeRoot()
+	require.NoError(t, err)
+	b3 := util.NewBeaconBlock()
+	b3.Block.Slot = 3
+	b3.Block.ParentRoot = b0Root[:]
+	wsb, err = wrapper.WrappedSignedBeaconBlock(b3)
+	require.NoError(t, err)
+	require.NoError(t, r.cfg.beaconDB.SaveBlock(context.Background(), wsb))
+	// Incomplete block link
+	b1 := util.NewBeaconBlock()
+	b1.Block.Slot = 1
+	b1.Block.ParentRoot = b0Root[:]
+	b1Root, err := b1.Block.HashTreeRoot()
+	require.NoError(t, err)
+	b2 := util.NewBeaconBlock()
+	b2.Block.Slot = 2
+	b2.Block.ParentRoot = b1Root[:]
+	b2Root, err := b1.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	// Add b2 to the cache
+	wsb, err = wrapper.WrappedSignedBeaconBlock(b2)
+	require.NoError(t, err)
+	require.NoError(t, r.insertBlockToPendingQueue(b2.Block.Slot, wsb, b2Root))
+
+	require.NoError(t, r.processPendingBlocks(context.Background()))
+	assert.Equal(t, 1, len(r.slotToPendingBlocks.Items()), "Incorrect size for slot to pending blocks cache")
+	assert.Equal(t, 1, len(r.seenPendingBlocks), "Incorrect size for seen pending block")
+
+	// Add b1 to the cache
+	wsb, err = wrapper.WrappedSignedBeaconBlock(b1)
+	require.NoError(t, err)
+	require.NoError(t, r.insertBlockToPendingQueue(b1.Block.Slot, wsb, b1Root))
+	wsb, err = wrapper.WrappedSignedBeaconBlock(b1)
+	require.NoError(t, err)
+	require.NoError(t, r.cfg.beaconDB.SaveBlock(context.Background(), wsb))
+
+	nBlock := util.NewBeaconBlock()
+	nBlock.Block.Slot = b1.Block.Slot
+	nRoot, err := nBlock.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	// Insert bad b1 in the cache to verify the good one doesn't get replaced.
+	wsb, err = wrapper.WrappedSignedBeaconBlock(nBlock)
+	require.NoError(t, err)
+	require.NoError(t, r.insertBlockToPendingQueue(nBlock.Block.Slot, wsb, nRoot))
+	require.NoError(t, r.processPendingBlocks(context.Background())) // Marks a block as bad
+	require.NoError(t, r.processPendingBlocks(context.Background())) // Bad block removed on second run
+
+	assert.Equal(t, 1, len(r.slotToPendingBlocks.Items()), "Incorrect size for slot to pending blocks cache")
+	assert.Equal(t, 2, len(r.seenPendingBlocks), "Incorrect size for seen pending block")
+	require.Equal(t, 1, len(r.badBlockCache.Keys())) // Account for the bad block above
+	require.Equal(t, 0, len(r.seenBlockCache.Keys()))
+}
+
 func TestRegularSync_InsertDuplicateBlocks(t *testing.T) {
 	db := dbtest.SetupDB(t)
 

beacon-chain/sync/subscriber_beacon_blocks.go

@@ -3,8 +3,10 @@ package sync
 import (
 	"context"
 
+	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/transition/interop"
+	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
 	"github.com/prysmaticlabs/prysm/config/features"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
@@ -30,8 +32,10 @@ func (s *Service) beaconBlockSubscriber(ctx context.Context, msg proto.Message)
 	}
 
 	if err := s.cfg.chain.ReceiveBlock(ctx, signed, root); err != nil {
-		interop.WriteBlockToDisk(signed, true /*failed*/)
-		s.setBadBlock(ctx, root)
+		if !errors.Is(err, powchain.ErrHTTPTimeout) {
+			interop.WriteBlockToDisk(signed, true /*failed*/)
+			s.setBadBlock(ctx, root)
+		}
 		return err
 	}
 

beacon-chain/sync/subscriber_beacon_blocks_test.go

@@ -9,6 +9,8 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	dbtest "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations"
+	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
+	lruwrpr "github.com/prysmaticlabs/prysm/cache/lru"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/testing/assert"
 	"github.com/prysmaticlabs/prysm/testing/require"
@@ -108,3 +110,18 @@ func TestService_beaconBlockSubscriber(t *testing.T) {
 		})
 	}
 }
+
+func TestService_BeaconBlockSubscribe_ExecutionEngineTimesOut(t *testing.T) {
+	s := &Service{
+		cfg: &config{
+			chain: &chainMock.ChainService{
+				ReceiveBlockMockErr: powchain.ErrHTTPTimeout,
+			},
+		},
+		seenBlockCache: lruwrpr.New(10),
+		badBlockCache:  lruwrpr.New(10),
+	}
+	require.ErrorIs(t, powchain.ErrHTTPTimeout, s.beaconBlockSubscriber(context.Background(), util.NewBeaconBlock()))
+	require.Equal(t, 0, len(s.badBlockCache.Keys()))
+	require.Equal(t, 1, len(s.seenBlockCache.Keys()))
+}

beacon-chain/sync/validate_beacon_blocks.go

@@ -166,11 +166,11 @@ func (s *Service) validateBeaconBlockPubSub(ctx context.Context, pid peer.ID, ms
 
 	err = s.validateBeaconBlock(ctx, blk, blockRoot)
 	if err != nil {
-		// If the parent is optimistic, be gracious and don't penalize the peer.
-		if errors.Is(ErrOptimisticParent, err) {
-			return pubsub.ValidationIgnore, err
+		// If the parent is optimistic, process the block as usual
+		// This also does not penalize a peer which sends optimistic blocks
+		if !errors.Is(ErrOptimisticParent, err) {
+			return pubsub.ValidationReject, err
 		}
-		return pubsub.ValidationReject, err
 	}
 
 	// Record attribute of valid block.

beacon-chain/sync/validate_beacon_blocks_test.go

@@ -1305,3 +1305,77 @@ func Test_validateBellatrixBeaconBlockParentValidation(t *testing.T) {
 	require.NoError(t, err)
 	require.ErrorContains(t, "parent of the block is optimistic", r.validateBellatrixBeaconBlock(ctx, beaconState, blk.Block()))
 }
+
+func Test_validateBeaconBlockProcessingWhenParentIsOptimistic(t *testing.T) {
+	db := dbtest.SetupDB(t)
+	p := p2ptest.NewTestP2P(t)
+	ctx := context.Background()
+	stateGen := stategen.New(db)
+
+	beaconState, privKeys := util.DeterministicGenesisStateBellatrix(t, 100)
+	parentBlock := util.NewBeaconBlockBellatrix()
+	signedParentBlock, err := wrapper.WrappedSignedBeaconBlock(parentBlock)
+	require.NoError(t, err)
+	require.NoError(t, db.SaveBlock(ctx, signedParentBlock))
+	bRoot, err := parentBlock.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, db.SaveState(ctx, beaconState, bRoot))
+	require.NoError(t, db.SaveStateSummary(ctx, &ethpb.StateSummary{Root: bRoot[:]}))
+	copied := beaconState.Copy()
+	require.NoError(t, copied.SetSlot(1))
+	proposerIdx, err := helpers.BeaconProposerIndex(ctx, copied)
+	require.NoError(t, err)
+
+	msg := util.NewBeaconBlockBellatrix()
+	msg.Block.ParentRoot = bRoot[:]
+	msg.Block.Slot = 1
+	msg.Block.ProposerIndex = proposerIdx
+	msg.Block.Body.ExecutionPayload.Timestamp = beaconState.GenesisTime() + params.BeaconConfig().SecondsPerSlot
+	msg.Block.Body.ExecutionPayload.GasUsed = 10
+	msg.Block.Body.ExecutionPayload.GasLimit = 11
+	msg.Block.Body.ExecutionPayload.BlockHash = bytesutil.PadTo([]byte("blockHash"), 32)
+	msg.Block.Body.ExecutionPayload.ParentHash = bytesutil.PadTo([]byte("parentHash"), 32)
+	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, []byte("transaction 1"))
+	msg.Block.Body.ExecutionPayload.Transactions = append(msg.Block.Body.ExecutionPayload.Transactions, []byte("transaction 2"))
+	msg.Signature, err = signing.ComputeDomainAndSign(beaconState, 0, msg.Block, params.BeaconConfig().DomainBeaconProposer, privKeys[proposerIdx])
+	require.NoError(t, err)
+
+	chainService := &mock.ChainService{Genesis: time.Unix(int64(beaconState.GenesisTime()), 0),
+		Optimistic: true,
+		FinalizedCheckPoint: &ethpb.Checkpoint{
+			Epoch: 0,
+			Root:  make([]byte, 32),
+		}}
+	r := &Service{
+		cfg: &config{
+			beaconDB:      db,
+			p2p:           p,
+			initialSync:   &mockSync.Sync{IsSyncing: false},
+			chain:         chainService,
+			blockNotifier: chainService.BlockNotifier(),
+			stateGen:      stateGen,
+		},
+		seenBlockCache: lruwrpr.New(10),
+		badBlockCache:  lruwrpr.New(10),
+	}
+
+	buf := new(bytes.Buffer)
+	_, err = p.Encoding().EncodeGossip(buf, msg)
+	require.NoError(t, err)
+	topic := p2p.GossipTypeMapping[reflect.TypeOf(msg)]
+	genesisValidatorsRoot := r.cfg.chain.GenesisValidatorsRoot()
+	BellatrixDigest, err := signing.ComputeForkDigest(params.BeaconConfig().BellatrixForkVersion, genesisValidatorsRoot[:])
+	require.NoError(t, err)
+	topic = r.addDigestToTopic(topic, BellatrixDigest)
+	m := &pubsub.Message{
+		Message: &pubsubpb.Message{
+			Data:  buf.Bytes(),
+			Topic: &topic,
+		},
+	}
+
+	res, err := r.validateBeaconBlockPubSub(ctx, "", m)
+	require.NoError(t, err)
+	result := res == pubsub.ValidationAccept
+	assert.Equal(t, true, result)
+}

cmd/beacon-chain/BUILD.bazel

@@ -22,6 +22,7 @@ go_library(
         "//cmd/beacon-chain/flags:go_default_library",
         "//cmd/beacon-chain/powchain:go_default_library",
         "//cmd/beacon-chain/sync/checkpoint:go_default_library",
+        "//cmd/beacon-chain/sync/genesis:go_default_library",
         "//config/features:go_default_library",
         "//io/file:go_default_library",
         "//io/logs:go_default_library",

cmd/beacon-chain/flags/base.go

@@ -191,12 +191,6 @@ var (
 		Usage: "Sets the maximum number of headers that a deposit log query can fetch.",
 		Value: uint64(1000),
 	}
-	// GenesisStatePath defines a flag to start the beacon chain from a give genesis state file.
-	GenesisStatePath = &cli.StringFlag{
-		Name: "genesis-state",
-		Usage: "Load a genesis state from ssz file. Testnet genesis files can be found in the " +
-			"eth2-clients/eth2-testnets repository on github.",
-	}
 	// WeakSubjectivityCheckpoint defines the weak subjectivity checkpoint the node must sync through to defend against long range attacks.
 	WeakSubjectivityCheckpoint = &cli.StringFlag{
 		Name: "weak-subjectivity-checkpoint",

cmd/beacon-chain/main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/flags"
 	powchaincmd "github.com/prysmaticlabs/prysm/cmd/beacon-chain/powchain"
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/sync/checkpoint"
+	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/sync/genesis"
 	"github.com/prysmaticlabs/prysm/config/features"
 	"github.com/prysmaticlabs/prysm/io/file"
 	"github.com/prysmaticlabs/prysm/io/logs"
@@ -65,7 +66,6 @@ var appFlags = []cli.Flag{
 	flags.NetworkID,
 	flags.WeakSubjectivityCheckpoint,
 	flags.Eth1HeaderReqLimit,
-	flags.GenesisStatePath,
 	flags.MinPeersPerSubnet,
 	flags.SuggestedFeeRecipient,
 	cmd.EnableBackupWebhookFlag,
@@ -122,6 +122,8 @@ var appFlags = []cli.Flag{
 	checkpoint.BlockPath,
 	checkpoint.StatePath,
 	checkpoint.RemoteURL,
+	genesis.StatePath,
+	genesis.BeaconAPIURL,
 }
 
 func init() {
@@ -246,13 +248,21 @@ func startNode(ctx *cli.Context) error {
 		node.WithBlockchainFlagOptions(blockchainFlagOpts),
 		node.WithPowchainFlagOptions(powchainFlagOpts),
 	}
-	cptOpts, err := checkpoint.BeaconNodeOptions(ctx)
-	if err != nil {
-		return err
+
+	optFuncs := []func(*cli.Context) (node.Option, error){
+		genesis.BeaconNodeOptions,
+		checkpoint.BeaconNodeOptions,
 	}
-	if cptOpts != nil {
-		opts = append(opts, cptOpts)
+	for _, of := range optFuncs {
+		ofo, err := of(ctx)
+		if err != nil {
+			return err
+		}
+		if ofo != nil {
+			opts = append(opts, ofo)
+		}
 	}
+
 	beacon, err := node.New(ctx, opts...)
 	if err != nil {
 		return err

cmd/beacon-chain/powchain/options.go

@@ -27,7 +27,7 @@ func FlagOptions(c *cli.Context) ([]powchain.Option, error) {
 		powchain.WithEth1HeaderRequestLimit(c.Uint64(flags.Eth1HeaderReqLimit.Name)),
 	}
 	if len(jwtSecret) > 0 {
-		opts = append(opts, powchain.WithExecutionClientJWTSecret(jwtSecret))
+		opts = append(opts, powchain.WithJWTSecret(jwtSecret))
 	}
 	return opts, nil
 }

cmd/beacon-chain/sync/genesis/BUILD.bazel

@@ -0,0 +1,14 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["options.go"],
+    importpath = "github.com/prysmaticlabs/prysm/cmd/beacon-chain/sync/genesis",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//beacon-chain/node:go_default_library",
+        "//beacon-chain/sync/genesis:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
+    ],
+)

cmd/beacon-chain/sync/genesis/options.go

@@ -0,0 +1,53 @@
+package genesis
+
+import (
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/beacon-chain/node"
+	"github.com/prysmaticlabs/prysm/beacon-chain/sync/genesis"
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	// StatePath defines a flag to start the beacon chain from a give genesis state file.
+	StatePath = &cli.PathFlag{
+		Name: "genesis-state",
+		Usage: "Load a genesis state from ssz file. Testnet genesis files can be found in the " +
+			"eth2-clients/eth2-testnets repository on github.",
+	}
+	BeaconAPIURL = &cli.StringFlag{
+		Name: "genesis-beacon-api-url",
+		Usage: "URL of a synced beacon node to trust for obtaining genesis state. " +
+			"As an additional safety measure, it is strongly recommended to only use this option in conjunction with " +
+			"--weak-subjectivity-checkpoint flag",
+	}
+)
+
+// BeaconNodeOptions is responsible for determining if the checkpoint sync options have been used, and if so,
+// reading the block and state ssz-serialized values from the filesystem locations specified and preparing a
+// checkpoint.Initializer, which uses the provided io.ReadClosers to initialize the beacon node database.
+func BeaconNodeOptions(c *cli.Context) (node.Option, error) {
+	statePath := c.Path(StatePath.Name)
+	remoteURL := c.String(BeaconAPIURL.Name)
+	if remoteURL != "" {
+		return func(node *node.BeaconNode) error {
+			var err error
+			node.GenesisInitializer, err = genesis.NewAPIInitializer(remoteURL)
+			if err != nil {
+				return errors.Wrap(err, "error constructing beacon node api client for genesis state init")
+			}
+			return nil
+		}, nil
+	}
+
+	if statePath == "" {
+		return nil, nil
+	}
+
+	return func(node *node.BeaconNode) (err error) {
+		node.GenesisInitializer, err = genesis.NewFileInitializer(statePath)
+		if err != nil {
+			return errors.Wrap(err, "error preparing to initialize genesis db state from local ssz files")
+		}
+		return nil
+	}, nil
+}

cmd/beacon-chain/usage.go

@@ -8,6 +8,7 @@ import (
 	"github.com/prysmaticlabs/prysm/cmd"
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/flags"
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/sync/checkpoint"
+	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/sync/genesis"
 	"github.com/prysmaticlabs/prysm/config/features"
 	"github.com/prysmaticlabs/prysm/runtime/debug"
 	"github.com/urfave/cli/v2"
@@ -76,9 +77,6 @@ var appHelpFlagGroups = []flagGroup{
 			cmd.BoltMMapInitialSizeFlag,
 			cmd.ValidatorMonitorIndicesFlag,
 			cmd.ApiTimeoutFlag,
-			checkpoint.BlockPath,
-			checkpoint.StatePath,
-			checkpoint.RemoteURL,
 		},
 	},
 	{
@@ -127,8 +125,12 @@ var appHelpFlagGroups = []flagGroup{
 			flags.NetworkID,
 			flags.WeakSubjectivityCheckpoint,
 			flags.Eth1HeaderReqLimit,
-			flags.GenesisStatePath,
 			flags.MinPeersPerSubnet,
+			checkpoint.BlockPath,
+			checkpoint.StatePath,
+			checkpoint.RemoteURL,
+			genesis.StatePath,
+			genesis.BeaconAPIURL,
 		},
 	},
 	{

testing/spectest/shared/common/forkchoice/runner.go

@@ -141,7 +141,7 @@ func Run(t *testing.T, config string, fork int) {
 						tdBigint.SetBytes(tdInBigEndian)
 					}
 					if step.Check != nil {
-						require.NoError(t, service.UpdateHeadWithBalances(ctx))
+						require.NoError(t, service.UpdateAndSaveHeadWithBalances(ctx))
 						c := step.Check
 						if c.Head != nil {
 							r, err := service.HeadRoot(ctx)

tools/analyzers/recursivelock/analyzer.go

@@ -25,6 +25,45 @@ var Analyzer = &analysis.Analyzer{
 }
 
 var errNestedRLock = errors.New("found recursive read lock call")
+var errNestedLock = errors.New("found recursive lock call")
+var errNestedMixedLock = errors.New("found recursive mixed lock call")
+
+type mode int
+
+const (
+	LockMode = mode(iota)
+	RLockMode
+)
+
+func (m mode) LockName() string {
+	switch m {
+	case LockMode:
+		return "Lock"
+	case RLockMode:
+		return "RLock"
+	}
+	return ""
+}
+
+func (m mode) UnLockName() string {
+	switch m {
+	case LockMode:
+		return "Unlock"
+	case RLockMode:
+		return "RUnlock"
+	}
+	return ""
+}
+
+func (m mode) ErrorFound() error {
+	switch m {
+	case LockMode:
+		return errNestedLock
+	case RLockMode:
+		return errNestedRLock
+	}
+	return nil
+}
 
 func run(pass *analysis.Pass) (interface{}, error) {
 	inspect, ok := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector)
@@ -33,6 +72,7 @@ func run(pass *analysis.Pass) (interface{}, error) {
 	}
 
 	nodeFilter := []ast.Node{
+		(*ast.GoStmt)(nil),
 		(*ast.CallExpr)(nil),
 		(*ast.DeferStmt)(nil),
 		(*ast.FuncDecl)(nil),
@@ -42,20 +82,36 @@ func run(pass *analysis.Pass) (interface{}, error) {
 		(*ast.ReturnStmt)(nil),
 	}
 
-	keepTrackOf := &tracker{}
+	keepTrackOf := &tracker{
+		rLockTrack: &lockTracker{},
+		lockTrack:  &lockTracker{},
+	}
 	inspect.Preorder(nodeFilter, func(node ast.Node) {
-		if keepTrackOf.funcLitEnd.IsValid() && node.Pos() <= keepTrackOf.funcLitEnd {
+		if keepTrackOf.rLockTrack.funcLitEnd.IsValid() && node.Pos() <= keepTrackOf.rLockTrack.funcLitEnd &&
+			keepTrackOf.lockTrack.funcLitEnd.IsValid() && node.Pos() <= keepTrackOf.lockTrack.funcLitEnd {
 			return
 		}
-		keepTrackOf.funcLitEnd = token.NoPos
-		if keepTrackOf.deferEnd.IsValid() && node.Pos() > keepTrackOf.deferEnd {
-			keepTrackOf.deferEnd = token.NoPos
-		} else if keepTrackOf.deferEnd.IsValid() {
+		keepTrackOf.rLockTrack.funcLitEnd = token.NoPos
+		keepTrackOf.lockTrack.funcLitEnd = token.NoPos
+
+		if keepTrackOf.rLockTrack.deferEnd.IsValid() && node.Pos() > keepTrackOf.rLockTrack.deferEnd {
+			keepTrackOf.rLockTrack.deferEnd = token.NoPos
+		} else if keepTrackOf.rLockTrack.deferEnd.IsValid() {
 			return
 		}
-		if keepTrackOf.retEnd.IsValid() && node.Pos() > keepTrackOf.retEnd {
-			keepTrackOf.retEnd = token.NoPos
-			keepTrackOf.incFRU()
+		if keepTrackOf.lockTrack.deferEnd.IsValid() && node.Pos() > keepTrackOf.lockTrack.deferEnd {
+			keepTrackOf.lockTrack.deferEnd = token.NoPos
+		} else if keepTrackOf.lockTrack.deferEnd.IsValid() {
+			return
+		}
+
+		if keepTrackOf.rLockTrack.retEnd.IsValid() && node.Pos() > keepTrackOf.rLockTrack.retEnd {
+			keepTrackOf.rLockTrack.retEnd = token.NoPos
+			keepTrackOf.rLockTrack.incFRU()
+		}
+		if keepTrackOf.lockTrack.retEnd.IsValid() && node.Pos() > keepTrackOf.lockTrack.retEnd {
+			keepTrackOf.lockTrack.retEnd = token.NoPos
+			keepTrackOf.lockTrack.incFRU()
 		}
 		keepTrackOf = stmtSelector(node, pass, keepTrackOf, inspect)
 	})
@@ -64,57 +120,46 @@ func run(pass *analysis.Pass) (interface{}, error) {
 
 func stmtSelector(node ast.Node, pass *analysis.Pass, keepTrackOf *tracker, inspect *inspector.Inspector) *tracker {
 	switch stmt := node.(type) {
+	case *ast.GoStmt:
+		keepTrackOf.rLockTrack.goroutinePos = stmt.Call.End()
+		keepTrackOf.lockTrack.goroutinePos = stmt.Call.End()
 	case *ast.CallExpr:
+		if stmt.End() == keepTrackOf.rLockTrack.goroutinePos ||
+			stmt.End() == keepTrackOf.lockTrack.goroutinePos {
+			keepTrackOf.rLockTrack.goroutinePos = 0
+			keepTrackOf.lockTrack.goroutinePos = 0
+			break
+		}
 		call := getCallInfo(pass.TypesInfo, stmt)
 		if call == nil {
 			break
 		}
-		name := call.name
 		selMap := mapSelTypes(stmt, pass)
 		if selMap == nil {
 			break
 		}
-		if keepTrackOf.rLockSelector != nil {
-			if keepTrackOf.foundRLock > 0 {
-				if keepTrackOf.rLockSelector.isEqual(selMap, 0) {
-					pass.Reportf(
-						node.Pos(),
-						fmt.Sprintf(
-							"%v",
-							errNestedRLock,
-						),
-					)
-				} else {
-					if stack := hasNestedRLock(keepTrackOf.rLockSelector, selMap, call, inspect, pass, make(map[string]bool)); stack != "" {
-						pass.Reportf(
-							node.Pos(),
-							fmt.Sprintf(
-								"%v\n%v",
-								errNestedRLock,
-								stack,
-							),
-						)
-					}
-				}
-			}
-			if name == "RUnlock" && keepTrackOf.rLockSelector.isEqual(selMap, 1) {
-				keepTrackOf.deincFRU()
-			}
-		} else if name == "RLock" && keepTrackOf.foundRLock == 0 {
-			keepTrackOf.rLockSelector = selMap
-			keepTrackOf.incFRU()
-		}
+		checkForRecLocks(node, pass, inspect, RLockMode, call, keepTrackOf.rLockTrack, selMap)
+		checkForRecLocks(node, pass, inspect, LockMode, call, keepTrackOf.lockTrack, selMap)
 
 	case *ast.File:
-		keepTrackOf = &tracker{}
+		keepTrackOf = &tracker{
+			rLockTrack: &lockTracker{},
+			lockTrack:  &lockTracker{},
+		}
 
 	case *ast.FuncDecl:
-		keepTrackOf = &tracker{}
-		keepTrackOf.funcEnd = stmt.End()
+		keepTrackOf = &tracker{
+			rLockTrack: &lockTracker{},
+			lockTrack:  &lockTracker{},
+		}
+		keepTrackOf.rLockTrack.funcEnd = stmt.End()
 
 	case *ast.FuncLit:
-		if keepTrackOf.funcLitEnd == token.NoPos {
-			keepTrackOf.funcLitEnd = stmt.End()
+		if keepTrackOf.rLockTrack.funcLitEnd == token.NoPos {
+			keepTrackOf.rLockTrack.funcLitEnd = stmt.End()
+		}
+		if keepTrackOf.lockTrack.funcLitEnd == token.NoPos {
+			keepTrackOf.lockTrack.funcLitEnd = stmt.End()
 		}
 	case *ast.IfStmt:
 		stmts := stmt.Body.List
@@ -124,48 +169,113 @@ func stmtSelector(node ast.Node, pass *analysis.Pass, keepTrackOf *tracker, insp
 		keepTrackOf = stmtSelector(stmt.Else, pass, keepTrackOf, inspect)
 	case *ast.DeferStmt:
 		call := getCallInfo(pass.TypesInfo, stmt.Call)
-		if keepTrackOf.deferEnd == token.NoPos {
-			keepTrackOf.deferEnd = stmt.End()
+		if keepTrackOf.rLockTrack.deferEnd == token.NoPos {
+			keepTrackOf.rLockTrack.deferEnd = stmt.End()
 		}
-		if call != nil && call.name == "RUnlock" {
-			keepTrackOf.deferredRUnlock = true
+		if keepTrackOf.lockTrack.deferEnd == token.NoPos {
+			keepTrackOf.lockTrack.deferEnd = stmt.End()
+		}
+
+		if call != nil && call.name == RLockMode.UnLockName() {
+			keepTrackOf.rLockTrack.deferredRUnlock = true
+		}
+		if call != nil && call.name == LockMode.UnLockName() {
+			keepTrackOf.lockTrack.deferredRUnlock = true
 		}
 
 	case *ast.ReturnStmt:
 		for i := 0; i < len(stmt.Results); i++ {
 			keepTrackOf = stmtSelector(stmt.Results[i], pass, keepTrackOf, inspect)
 		}
-		if keepTrackOf.deferredRUnlock && keepTrackOf.retEnd == token.NoPos {
-			keepTrackOf.deincFRU()
-			keepTrackOf.retEnd = stmt.End()
+		if keepTrackOf.rLockTrack.deferredRUnlock && keepTrackOf.rLockTrack.retEnd == token.NoPos {
+			keepTrackOf.rLockTrack.deincFRU()
+			keepTrackOf.rLockTrack.retEnd = stmt.End()
+		}
+		if keepTrackOf.lockTrack.deferredRUnlock && keepTrackOf.lockTrack.retEnd == token.NoPos {
+			keepTrackOf.lockTrack.deincFRU()
+			keepTrackOf.lockTrack.retEnd = stmt.End()
 		}
 	}
 	return keepTrackOf
 }
 
 type tracker struct {
+	rLockTrack *lockTracker
+	lockTrack  *lockTracker
+}
+
+type lockTracker struct {
 	funcEnd         token.Pos
 	retEnd          token.Pos
 	deferEnd        token.Pos
 	funcLitEnd      token.Pos
+	goroutinePos    token.Pos
 	deferredRUnlock bool
 	foundRLock      int
 	rLockSelector   *selIdentList
 }
 
-func (t tracker) String() string {
+func (t lockTracker) String() string {
 	return fmt.Sprintf("funcEnd:%v\nretEnd:%v\ndeferEnd:%v\ndeferredRU:%v\nfoundRLock:%v\n", t.funcEnd, t.retEnd, t.deferEnd, t.deferredRUnlock, t.foundRLock)
 }
 
-func (t *tracker) deincFRU() {
+func (t *lockTracker) deincFRU() {
 	if t.foundRLock > 0 {
 		t.foundRLock -= 1
 	}
 }
-func (t *tracker) incFRU() {
+func (t *lockTracker) incFRU() {
 	t.foundRLock += 1
 }
 
+func checkForRecLocks(node ast.Node, pass *analysis.Pass, inspect *inspector.Inspector, lockmode mode, call *callInfo,
+	lockTracker *lockTracker, selMap *selIdentList) {
+	name := call.name
+	if lockTracker.rLockSelector != nil {
+		if lockTracker.foundRLock > 0 {
+			if lockTracker.rLockSelector.isRelated(selMap, 0) {
+				pass.Reportf(
+					node.Pos(),
+					fmt.Sprintf(
+						"%v",
+						errNestedMixedLock,
+					),
+				)
+			}
+			if lockTracker.rLockSelector.isEqual(selMap, 0) {
+				pass.Reportf(
+					node.Pos(),
+					fmt.Sprintf(
+						"%v",
+						lockmode.ErrorFound(),
+					),
+				)
+			} else {
+				if stack := hasNestedlock(lockTracker.rLockSelector, lockTracker.goroutinePos, selMap, call, inspect, pass, make(map[string]bool),
+					lockmode.UnLockName()); stack != "" {
+					pass.Reportf(
+						node.Pos(),
+						fmt.Sprintf(
+							"%v\n%v",
+							lockmode.ErrorFound(),
+							stack,
+						),
+					)
+				}
+			}
+		}
+		if name == lockmode.UnLockName() && lockTracker.rLockSelector.isEqual(selMap, 1) {
+			lockTracker.deincFRU()
+		}
+		if name == lockmode.LockName() && lockTracker.foundRLock == 0 && lockTracker.rLockSelector.isEqual(selMap, 0) {
+			lockTracker.incFRU()
+		}
+	} else if name == lockmode.LockName() && lockTracker.foundRLock == 0 {
+		lockTracker.rLockSelector = selMap
+		lockTracker.incFRU()
+	}
+}
+
 // Stores the AST and type information of a single item in a selector expression
 // For example, "a.b.c()", a selIdentNode might store the information for "a"
 type selIdentNode struct {
@@ -221,6 +331,45 @@ func (s *selIdentList) isEqual(s2 *selIdentList, offset int) bool {
 	return true
 }
 
+// isRelated checks if our selectors are of the same type and
+// reference the same underlying object. If they do we check
+// if the provided list is referencing a non-equal but related
+// lock. Ex: Lock - RLock, RLock - Lock
+// TODO: Use a generalizable method here instead of hardcoding
+// the lock definitions here.
+func (s *selIdentList) isRelated(s2 *selIdentList, offset int) bool {
+	if s2 == nil || (s.length != s2.length) {
+		return false
+	}
+	s.reset()
+	s2.reset()
+	for i := true; i; {
+		if !s.current.isEqual(s2.current) {
+			return false
+		}
+		if s.currentIndex < s.length-offset-1 && s.next() != nil {
+			s2.next()
+		} else {
+			i = false
+		}
+		// Only check if we are at the last index for
+		// related method calls.
+		if s.currentIndex == s.length-1 {
+			switch s.current.this.String() {
+			case LockMode.LockName():
+				if s2.current.this.String() == RLockMode.LockName() {
+					return true
+				}
+			case RLockMode.LockName():
+				if s2.current.this.String() == LockMode.LockName() {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
 // getSub returns the shared beginning selIdentList of s and s2,
 // if s contains all elements (except the last) of s2,
 // and returns nil otherwise.
@@ -356,11 +505,12 @@ func interfaceMethod(s *types.Signature) bool {
 	return recv != nil && types.IsInterface(recv.Type())
 }
 
-// hasNestedRLock returns a stack trace of the nested or recursive RLock within the declaration of a function/method call (given by call).
-// If the call expression does not contain a nested or recursive RLock, hasNestedRLock returns an empty string.
-// hasNestedRLock finds a nested or recursive RLock by recursively calling itself on any functions called by the function/method represented
+// hasNestedlock returns a stack trace of the nested or recursive lock within the declaration of a function/method call (given by call).
+// If the call expression does not contain a nested or recursive lock, hasNestedlock returns an empty string.
+// hasNestedlock finds a nested or recursive lock by recursively calling itself on any functions called by the function/method represented
 // by callInfo.
-func hasNestedRLock(fullRLockSelector *selIdentList, compareMap *selIdentList, call *callInfo, inspect *inspector.Inspector, pass *analysis.Pass, hist map[string]bool) (retStack string) {
+func hasNestedlock(fullRLockSelector *selIdentList, goPos token.Pos, compareMap *selIdentList, call *callInfo, inspect *inspector.Inspector,
+	pass *analysis.Pass, hist map[string]bool, lmode mode, lCount int) (retStack string) {
 	var rLockSelector *selIdentList
 	f := pass.Fset
 	tInfo := pass.TypesInfo
@@ -390,20 +540,36 @@ func hasNestedRLock(fullRLockSelector *selIdentList, compareMap *selIdentList, c
 	addition := fmt.Sprintf("\t%q at %v\n", call.name, f.Position(call.call.Pos()))
 	ast.Inspect(node, func(iNode ast.Node) bool {
 		switch stmt := iNode.(type) {
+		case *ast.GoStmt:
+			goPos = stmt.End()
 		case *ast.CallExpr:
+			if stmt.End() == goPos {
+				goPos = 0
+				return false
+			}
 			c := getCallInfo(tInfo, stmt)
 			if c == nil {
 				return false
 			}
 			name := c.name
 			selMap := mapSelTypes(stmt, pass)
-			if rLockSelector.isEqual(selMap, 0) { // if the method found is an RLock method
-				retStack += addition + fmt.Sprintf("\t%q at %v\n", name, f.Position(iNode.Pos()))
-			} else if name != "RUnlock" { // name should not equal the previousName to prevent infinite recursive loop
+			switch {
+			case name == lmode.LockName() && rLockSelector.isEqual(selMap, 0):
+				if lCount > 0 {
+					retStack += addition + fmt.Sprintf("\t%q at %v \n", name, f.Position(iNode.Pos()))
+				}
+				lCount += 1
+			case name == lmode.UnLockName() && rLockSelector.isEqual(selMap, 1):
+				if lCount > 0 {
+					lCount -= 1
+				}
+			case lCount > 0 && rLockSelector.isRelated(selMap, 0):
+				retStack += addition + fmt.Sprintf("\t%q at %v \n", name, f.Position(iNode.Pos()))
+			case name != lmode.UnLockName():
 				nt := c.id
 				if !hist[nt] { // make sure we are not in an infinite recursive loop
 					hist[nt] = true
-					stack := hasNestedRLock(rLockSelector, selMap, c, inspect, pass, hist)
+					stack := hasNestedlock(rLockSelector, goPos, selMap, c, inspect, pass, hist, lmode, lCount)
 					delete(hist, nt)
 					if stack != "" {
 						retStack += addition + stack

tools/analyzers/recursivelock/testdata/badlockswithmethods.go

@@ -13,6 +13,31 @@ func (p *ProtectResource) NestedMethod2() {
 	p.RUnlock()
 }
 
+func (p *ProtectResource) NestedMethodMixedLock() {
+	p.Lock()
+	p.GetResource() // want `found recursive lock call`
+	p.Unlock()
+}
+
+func (p *ProtectResource) MixedLock() {
+	p.RLock()
+	p.Lock() // want `found recursive mixed lock call`
+	p.Unlock()
+	p.RUnlock()
+}
+
+func (p *ProtectResource) NestedMethodGoroutine() {
+	p.RLock()
+	defer p.RUnlock()
+	go p.GetResource()
+}
+
+func (p *ProtectResource) NestedResourceGoroutine() {
+	p.RLock()
+	defer p.RUnlock()
+	p.GetResourceNestedGoroutine()
+}
+
 func (p *NestedProtectResource) MultiLevelStruct() {
 	p.nestedPR.RLock()
 	p.nestedPR.GetResource() // want `found recursive read lock call`

tools/analyzers/recursivelock/testdata/complexlocks.go

@@ -15,9 +15,30 @@ func (p *ProtectResource) FuncLitInStructLit() {
 	p.RUnlock()
 }
 
+func (p *ProtectResource) FuncLitInStructLitLocked() {
+	p.Lock()
+	type funcLitContainer struct {
+		funcLit func()
+	}
+	var fl *funcLitContainer = &funcLitContainer{
+		funcLit: func() {
+			p.Lock()
+		},
+	}
+	fl.funcLit() // this is a nested Lock but won't be caught
+	p.Unlock()
+}
+
 func (e *ExposedMutex) FuncReturnsMutex() {
 	e.GetLock().RLock()
 	e.lock.RLock() // this is an obvious nested lock, but won't be caught since the first RLock was called through a getter function
 	e.lock.RUnlock()
 	e.GetLock().RUnlock()
 }
+
+func (e *ExposedMutex) FuncReturnsMutexLocked() {
+	e.GetLock().Lock()
+	e.lock.Lock() // this is an obvious nested lock, but won't be caught since the first RLock was called through a getter function
+	e.lock.Unlock()
+	e.GetLock().Unlock()
+}

tools/analyzers/recursivelock/testdata/nonrlocks.go

@@ -12,3 +12,15 @@ func (resource *NestedProtectResource) NonNestedRLockDifferentRLocks() {
 	resource.GetNestedPResource() // get nested resource uses RLock, but at a deeper level in the struct
 	resource.RUnlock()
 }
+
+func (resource *ProtectResource) NestedLockWithDefer() string {
+	resource.Lock()
+	defer resource.Unlock()
+	return resource.GetResourceLocked() // want `found recursive lock call`
+}
+
+func (resource *NestedProtectResource) NonNestedLockDifferentLocks() {
+	resource.Lock()
+	resource.GetNestedPResourceLocked() // get nested resource uses RLock, but at a deeper level in the struct
+	resource.Unlock()
+}

tools/analyzers/recursivelock/testdata/types.go

@@ -15,10 +15,24 @@ func (r *ProtectResource) GetResource() string {
 	return r.resource
 }
 
+func (r *ProtectResource) GetResourceLocked() string {
+	defer r.Unlock()
+	r.Lock()
+	return r.resource
+}
+
 func (r *ProtectResource) GetResourceNested() string {
 	return r.GetResource()
 }
 
+func (r *ProtectResource) GetResourceNestedGoroutine() {
+	go r.GetResource()
+}
+
+func (r *ProtectResource) GetResourceNestedLock() string {
+	return r.GetResourceLocked()
+}
+
 type NestedProtectResource struct {
 	*sync.RWMutex
 	nestedPR ProtectResource
@@ -30,6 +44,12 @@ func (r *NestedProtectResource) GetNestedPResource() string {
 	return r.nestedPR.resource
 }
 
+func (r *NestedProtectResource) GetNestedPResourceLocked() string {
+	defer r.nestedPR.Unlock()
+	r.nestedPR.Lock()
+	return r.nestedPR.resource
+}
+
 type NotProtected struct {
 	resource string
 }

validator/accounts/accounts.go

@@ -1,7 +1,6 @@
 package accounts
 
 import (
-	"github.com/prysmaticlabs/prysm/validator/accounts/wallet"
 	"github.com/prysmaticlabs/prysm/validator/keymanager"
 )
 
@@ -11,9 +10,8 @@ var (
 	ErrCouldNotInitializeKeymanager = "could not initialize keymanager"
 )
 
-// Config specifies parameters for accounts commands.
-type Config struct {
-	Wallet           *wallet.Wallet
+// DeleteConfig specifies parameters for the accounts delete command.
+type DeleteConfig struct {
 	Keymanager       keymanager.IKeymanager
 	DeletePublicKeys [][]byte
 }

validator/accounts/accounts_delete.go

@@ -91,8 +91,7 @@ func DeleteAccountCli(cliCtx *cli.Context) error {
 			}
 		}
 	}
-	if err := DeleteAccount(cliCtx.Context, &Config{
-		Wallet:           w,
+	if err := DeleteAccount(cliCtx.Context, &DeleteConfig{
 		Keymanager:       kManager,
 		DeletePublicKeys: rawPublicKeys,
 	}); err != nil {
@@ -106,17 +105,13 @@ func DeleteAccountCli(cliCtx *cli.Context) error {
 }
 
 // DeleteAccount deletes the accounts that the user requests to be deleted from the wallet.
-func DeleteAccount(ctx context.Context, cfg *Config) error {
-	deleter, ok := cfg.Keymanager.(keymanager.Deleter)
-	if !ok {
-		return errors.New("keymanager does not implement Deleter interface")
-	}
+func DeleteAccount(ctx context.Context, cfg *DeleteConfig) error {
 	if len(cfg.DeletePublicKeys) == 1 {
 		log.Info("Deleting account...")
 	} else {
 		log.Info("Deleting accounts...")
 	}
-	statuses, err := deleter.DeleteKeystores(ctx, cfg.DeletePublicKeys)
+	statuses, err := cfg.Keymanager.DeleteKeystores(ctx, cfg.DeletePublicKeys)
 	if err != nil {
 		return errors.Wrap(err, "could not delete accounts")
 	}

validator/accounts/accounts_list_test.go

@@ -17,6 +17,7 @@ import (
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/crypto/bls"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	ethpbservice "github.com/prysmaticlabs/prysm/proto/eth/service"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	validatorpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/validator-client"
 	"github.com/prysmaticlabs/prysm/testing/assert"
@@ -59,6 +60,10 @@ func (km *mockRemoteKeymanager) ListKeymanagerAccounts(ctx context.Context, cfg
 	return remote.ListKeymanagerAccountsImpl(ctx, cfg, km, km.opts)
 }
 
+func (*mockRemoteKeymanager) DeleteKeystores(context.Context, [][]byte) ([]*ethpbservice.DeletedKeystoreStatus, error) {
+	return nil, nil
+}
+
 func createRandomKeystore(t testing.TB, password string) *keymanager.Keystore {
 	encryptor := keystorev4.New()
 	id, err := uuid.NewRandom()

validator/client/BUILD.bazel

@@ -116,6 +116,7 @@ go_test(
         "//crypto/bls:go_default_library",
         "//encoding/bytesutil:go_default_library",
         "//io/file:go_default_library",
+        "//proto/eth/service:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//proto/prysm/v1alpha1/block:go_default_library",
         "//proto/prysm/v1alpha1/validator-client:go_default_library",

validator/client/validator_test.go

@@ -21,6 +21,7 @@ import (
 	validator_service_config "github.com/prysmaticlabs/prysm/config/validator/service"
 	"github.com/prysmaticlabs/prysm/crypto/bls"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	ethpbservice "github.com/prysmaticlabs/prysm/proto/eth/service"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	validatorpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/validator-client"
 	"github.com/prysmaticlabs/prysm/testing/assert"
@@ -115,6 +116,11 @@ func (*mockKeymanager) ListKeymanagerAccounts(
 	return nil
 }
 
+func (*mockKeymanager) DeleteKeystores(context.Context, [][]byte,
+) ([]*ethpbservice.DeletedKeystoreStatus, error) {
+	return nil, nil
+}
+
 func generateMockStatusResponse(pubkeys [][]byte) *ethpb.ValidatorActivationResponse {
 	multipleStatus := make([]*ethpb.ValidatorActivationResponse_Status, len(pubkeys))
 	for i, key := range pubkeys {

validator/keymanager/remote-web3signer/BUILD.bazel

@@ -16,6 +16,7 @@ go_library(
         "//config/fieldparams:go_default_library",
         "//crypto/bls:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//proto/eth/service:go_default_library",
         "//proto/prysm/v1alpha1/validator-client:go_default_library",
         "//validator/keymanager:go_default_library",
         "//validator/keymanager/remote-utils:go_default_library",

validator/keymanager/remote-web3signer/keymanager.go

@@ -14,6 +14,7 @@ import (
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/crypto/bls"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	ethpbservice "github.com/prysmaticlabs/prysm/proto/eth/service"
 	validatorpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/validator-client"
 	"github.com/prysmaticlabs/prysm/validator/keymanager"
 	remote_utils "github.com/prysmaticlabs/prysm/validator/keymanager/remote-utils"
@@ -239,13 +240,18 @@ func (*Keymanager) SubscribeAccountChanges(_ chan [][48]byte) event.Subscription
 	})
 }
 
-// ExtractKeystores is not supported for the remote keymanager type.
+// ExtractKeystores is not supported for the remote-web3signer keymanager type.
 func (*Keymanager) ExtractKeystores(
 	ctx context.Context, publicKeys []bls.PublicKey, password string,
 ) ([]*keymanager.Keystore, error) {
 	return nil, errors.New("extracting keys is not supported for a web3signer keymanager")
 }
 
+// DeleteKeystores is not supported for the remote-web3signer keymanager type.
+func (km *Keymanager) DeleteKeystores(context.Context, [][]byte) ([]*ethpbservice.DeletedKeystoreStatus, error) {
+	return nil, errors.New("Wrong wallet type: web3-signer. Only Imported or Derived wallets can delete accounts")
+}
+
 func (km *Keymanager) ListKeymanagerAccounts(ctx context.Context, cfg keymanager.ListKeymanagerAccountConfig) error {
 	au := aurora.NewAurora(true)
 	fmt.Printf("(keymanager kind) %s\n", au.BrightGreen("web3signer").Bold())

validator/keymanager/remote/BUILD.bazel

@@ -17,6 +17,7 @@ go_library(
         "//config/fieldparams:go_default_library",
         "//crypto/bls:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//proto/eth/service:go_default_library",
         "//proto/prysm/v1alpha1/validator-client:go_default_library",
         "//validator/keymanager:go_default_library",
         "//validator/keymanager/remote-utils:go_default_library",

validator/keymanager/remote/keymanager.go

@@ -20,6 +20,7 @@ import (
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/crypto/bls"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	ethpbservice "github.com/prysmaticlabs/prysm/proto/eth/service"
 	validatorpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/validator-client"
 	"github.com/prysmaticlabs/prysm/validator/keymanager"
 	remote_utils "github.com/prysmaticlabs/prysm/validator/keymanager/remote-utils"
@@ -277,6 +278,11 @@ func (*Keymanager) ExtractKeystores(
 	return nil, errors.New("extracting keys not supported for a remote keymanager")
 }
 
+// DeleteKeystores is not supported for the remote keymanager type.
+func (*Keymanager) DeleteKeystores(context.Context, [][]byte) ([]*ethpbservice.DeletedKeystoreStatus, error) {
+	return nil, errors.New("Wrong wallet type: web3-signer. Only Imported or Derived wallets can delete accounts")
+}
+
 func (km *Keymanager) ListKeymanagerAccounts(ctx context.Context, cfg keymanager.ListKeymanagerAccountConfig) error {
 	return ListKeymanagerAccountsImpl(ctx, cfg, km, km.KeymanagerOpts())
 }

validator/keymanager/remote/mock/BUILD.bazel

@@ -10,6 +10,7 @@ go_library(
         "//async/event:go_default_library",
         "//config/fieldparams:go_default_library",
         "//crypto/bls:go_default_library",
+        "//proto/eth/service:go_default_library",
         "//proto/prysm/v1alpha1/validator-client:go_default_library",
         "//validator/keymanager:go_default_library",
     ],

validator/keymanager/remote/mock/mock_keymanager.go

@@ -7,6 +7,7 @@ import (
 	"github.com/prysmaticlabs/prysm/async/event"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/crypto/bls"
+	ethpbservice "github.com/prysmaticlabs/prysm/proto/eth/service"
 	validatorpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/validator-client"
 	"github.com/prysmaticlabs/prysm/validator/keymanager"
 )
@@ -60,3 +61,8 @@ func (*MockKeymanager) ListKeymanagerAccounts(
 	context.Context, keymanager.ListKeymanagerAccountConfig) error {
 	return nil
 }
+
+func (*MockKeymanager) DeleteKeystores(context.Context, [][]byte,
+) ([]*ethpbservice.DeletedKeystoreStatus, error) {
+	return nil, nil
+}

validator/keymanager/types.go

@@ -19,6 +19,7 @@ type IKeymanager interface {
 	KeyChangeSubscriber
 	KeyStoreExtractor
 	AccountLister
+	Deleter
 }
 
 // KeysFetcher for validating private and public keys.

validator/rpc/accounts.go

@@ -173,11 +173,7 @@ func (s *Server) DeleteAccounts(
 	if err != nil {
 		return nil, err
 	}
-	if s.wallet.KeymanagerKind() != keymanager.Local && s.wallet.KeymanagerKind() != keymanager.Derived {
-		return nil, status.Error(codes.FailedPrecondition, "Only Imported or Derived wallets can delete accounts")
-	}
-	if err := accounts.DeleteAccount(ctx, &accounts.Config{
-		Wallet:           s.wallet,
+	if err := accounts.DeleteAccount(ctx, &accounts.DeleteConfig{
 		Keymanager:       km,
 		DeletePublicKeys: req.PublicKeysToDelete,
 	}); err != nil {

validator/rpc/standard_api.go

@@ -152,15 +152,10 @@ func (s *Server) DeleteKeystores(
 	if err != nil {
 		return nil, status.Errorf(codes.Internal, "Could not get keymanager: %v", err)
 	}
-	deleter, ok := km.(keymanager.Deleter)
-	if !ok {
-		statuses := groupExportErrors(req, "Keymanager kind cannot delete keys")
-		return &ethpbservice.DeleteKeystoresResponse{Data: statuses}, nil
-	}
 	if len(req.Pubkeys) == 0 {
 		return &ethpbservice.DeleteKeystoresResponse{Data: make([]*ethpbservice.DeletedKeystoreStatus, 0)}, nil
 	}
-	statuses, err := deleter.DeleteKeystores(ctx, req.Pubkeys)
+	statuses, err := km.DeleteKeystores(ctx, req.Pubkeys)
 	if err != nil {
 		return nil, status.Errorf(codes.Internal, "Could not delete keys: %v", err)
 	}

validator/rpc/standard_api_test.go

@@ -472,11 +472,9 @@ func TestServer_DeleteKeystores_WrongKeymanagerKind(t *testing.T) {
 		wallet:            w,
 		validatorService:  vs,
 	}
-	response, err := s.DeleteKeystores(ctx, &ethpbservice.DeleteKeystoresRequest{Pubkeys: [][]byte{[]byte("a")}})
-	require.NoError(t, err)
-	require.Equal(t, 1, len(response.Data))
-	require.Equal(t, ethpbservice.DeletedKeystoreStatus_ERROR, response.Data[0].Status)
-	require.Equal(t, "Keymanager kind cannot delete keys", response.Data[0].Message)
+	_, err = s.DeleteKeystores(ctx, &ethpbservice.DeleteKeystoresRequest{Pubkeys: [][]byte{[]byte("a")}})
+	require.ErrorContains(t, "Wrong wallet type", err)
+	require.ErrorContains(t, "Only Imported or Derived wallets can delete accounts", err)
 }
 
 func setupServerWithWallet(t testing.TB) *Server {