fix: test errors

* e2e: Provide e2e config yaml to web3signer

* fix build for //testing/endtoend:go_default_test

* Update with web3signer with teku fixes

* buildifier

* Add slasher case for web3signer

* Update testing/endtoend/minimal_e2e_test.go

* Update web3signer to 21.10.6

* Revert "Update web3signer to 21.10.6"

This reverts commit bdf3c408f2.

* Remove slasher part of web3signer e2e tests

* Revert "Remove slasher part of web3signer e2e tests"

This reverts commit 24249802ae.

* fix slasher web3signer test

* fixing build

* updating yaml to match testnet_e2e_config.go

* trying a different order to the e2e test and adding a log

* trying different way to kill process

* handling unhandled error

* testing changes to config WIP

* fixing bazel WIP

* fixing build

* ignoring test for now to test

* fixing bazel

* Test only web3signer e2e

* rolling back some commits to test

* fixing bazel

* trying an updated web3signer version

* changing flag to match version

* trying current version of develop for web3signer

* testing not using the --network property

* addressing build error

* testing config change

* reverting to go bakc to using the network file

* testing adding epochs per sync committee period

* rolling back configs

* removing check to test

* adding log to get sync committee duties and changing bellatrix fork epoch to something large and altair to epoch 1

* fixing bazel

* updating epoch in config file

* fixing more descrepencies between the configurations

* removing un used yaml

* removing goland added duplicates

* reverting using network minimal

* fixing bug

* rolling back some changes

* rolling back some changes

* rolling back changes

* making sure web3signer test doesn't make it to bellatrix fork yet

* reverting changes I did not touch

* undo comment

* Update testing/endtoend/deps.bzl

* Apply suggestions from code review

* rm nl

* fix //testing/endtoend:go_mainnet_test

* Remove unnecessary dep

* fix //testing/endtoend:go_mainnet_test

* addressing review comments

* fixing build and internal conflict

* removing web3signer slasher test as it's unneeded due to the interface nature of key signing, the regular slashing test is enough

* fix: The validator we fetch from the binary can only run before altair, if you add that and the web3signer then these things can never run together as the web3signer sets it to before bellatrix

Co-authored-by: james-prysm <90280386+james-prysm@users.noreply.github.com>
Co-authored-by: James He <james@prysmaticlabs.com>
Co-authored-by: Nishant Das <nishdas93@gmail.com>
Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>

.github/actions/gomodtidy/entrypoint.sh

@@ -1,8 +1,8 @@
 #!/bin/sh -l
 set -e
-export PATH=$PATH:/usr/local/go/bin
+export PATH="$PATH:/usr/local/go/bin"
 
-cd $GITHUB_WORKSPACE
+cd "$GITHUB_WORKSPACE"
 
 cp go.mod go.mod.orig
 cp go.sum go.sum.orig

.github/workflows/dappnode-release-trigger.yml

@@ -1,41 +0,0 @@
-name: Update DAppNodePackages
-
-on:
-  push:
-    tags:
-    - '*'
-
-jobs:
-  dappnode-update-beacon-chain:
-    name: Trigger a beacon-chain release
-    runs-on: ubuntu-latest
-    steps:
-    - name: Get latest tag
-      id: get_tag
-      run: echo ::set-output name=TAG::${GITHUB_REF/refs\/tags\//}
-    - name: Send dispatch event to DAppNodePackage-prysm-beacon-chain
-      env: 
-        DISPATCH_REPO: dappnode/DAppNodePackage-prysm-beacon-chain
-      run: |
-        curl -v -X POST -u "${{ secrets.PAT_GITHUB }}" \
-        -H "Accept: application/vnd.github.everest-preview+json" \
-        -H "Content-Type: application/json" \
-        --data '{"event_type":"new_release", "client_payload": { "tag":"${{ steps.get_tag.outputs.TAG }}"}}' \
-        https://api.github.com/repos/$DISPATCH_REPO/dispatches
-        
-  dappnode-update-validator:
-    name: Trigger a validator release
-    runs-on: ubuntu-latest
-    steps:
-    - name: Get latest tag
-      id: get_tag
-      run: echo ::set-output name=TAG::${GITHUB_REF/refs\/tags\//}
-    - name: Send dispatch event to DAppNodePackage validator repository
-      env: 
-        DISPATCH_REPO: dappnode/DAppNodePackage-prysm-validator
-      run: |
-        curl -v -X POST -u "${{ secrets.PAT_GITHUB }}" \
-        -H "Accept: application/vnd.github.everest-preview+json" \
-        -H "Content-Type: application/json" \
-        --data '{"event_type":"new_release", "client_payload": { "tag":"${{ steps.get_tag.outputs.TAG }}"}}' \
-        https://api.github.com/repos/$DISPATCH_REPO/dispatches

.github/workflows/go.yml

@@ -7,13 +7,12 @@ on:
     branches: [ '*' ]
 
 jobs:
-
-  check:
-    name: Check
+  formatting:
+    name: Formatting
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
 
       - name: Go mod tidy checker
         id: gomodtidy
@@ -31,15 +30,43 @@ jobs:
         with:
           goimports-path: ./
 
-      - name: Gosec security scanner
-        uses: securego/gosec@master
+  gosec:
+    name: Gosec scan
+    runs-on: ubuntu-latest
+    env:
+      GO111MODULE: on
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v3
         with:
-          args: '-exclude=G307 -exclude-dir=crypto/bls/herumi ./...'
+          go-version: 1.17
+      - name: Run Gosec Security Scanner
+        run: | # https://github.com/securego/gosec/issues/469
+          export PATH=$PATH:$(go env GOPATH)/bin
+          go install github.com/securego/gosec/v2/cmd/gosec@latest
+          gosec -exclude=G307 -exclude-dir=crypto/bls/herumi ./...
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.17
+          id: go
 
       - name: Golangci-lint
         uses: golangci/golangci-lint-action@v2
         with:
-          args: --print-issued-lines --sort-results --no-config --timeout=10m --disable-all -E deadcode  -E errcheck -E gosimple --skip-files=validator/web/site_data.go --skip-dirs=proto
+          args: --print-issued-lines --sort-results --no-config --timeout=10m --disable-all -E deadcode  -E errcheck -E gosimple --skip-files=validator/web/site_data.go --skip-dirs=proto --go=1.17
+          version: v1.45.2
+          skip-go-installation: true
 
   build:
     name: Build
@@ -48,7 +75,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v2
         with:
-          go-version: ^1.14
+          go-version: 1.17
         id: go
 
       - name: Check out code into the Go module directory

.github/workflows/horusec.yaml

@@ -0,0 +1,22 @@
+name: Horusec Security Scan
+
+on:
+  schedule:
+    # Runs cron at 16.00 UTC on
+    - cron: '0 0 * * SUN'
+
+jobs:
+  Horusec_Scan:
+    name: horusec-Scan
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/develop'
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+      with: # Required when commit authors is enabled
+        fetch-depth: 0
+        
+    - name: Running Security Scan
+      run: |
+        curl -fsSL https://raw.githubusercontent.com/ZupIT/horusec/main/deployments/scripts/install.sh | bash -s latest
+        horusec start -t="10000" -p="./" -e="true" -i="**/crypto/bls/herumi/**, **/**/*_test.go, **/third_party/afl/**, **/crypto/keystore/key.go"

api/client/beacon/client.go

@@ -132,6 +132,15 @@ func validHostname(h string) (string, error) {
 	return fmt.Sprintf("%s:%s", host, port), nil
 }
 
+// NodeURL returns a human-readable string representation of the beacon node base url.
+func (c *Client) NodeURL() string {
+	u := &url.URL{
+		Scheme: c.scheme,
+		Host:   c.host,
+	}
+	return u.String()
+}
+
 func (c *Client) urlForPath(methodPath string) *url.URL {
 	u := &url.URL{
 		Scheme: c.scheme,

beacon-chain/blockchain/BUILD.bazel

@@ -48,6 +48,7 @@ go_library(
         "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//beacon-chain/db/filters:go_default_library",
+        "//beacon-chain/db/kv:go_default_library",
         "//beacon-chain/forkchoice:go_default_library",
         "//beacon-chain/forkchoice/doubly-linked-tree:go_default_library",
         "//beacon-chain/forkchoice/protoarray:go_default_library",
@@ -57,7 +58,6 @@ go_library(
         "//beacon-chain/operations/voluntaryexits:go_default_library",
         "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
-        "//beacon-chain/powchain/engine-api-client/v1:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/stategen:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",
@@ -66,6 +66,7 @@ go_library(
         "//config/params:go_default_library",
         "//crypto/bls:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//math:go_default_library",
         "//monitoring/tracing:go_default_library",
         "//proto/engine/v1:go_default_library",
         "//proto/eth/v1:go_default_library",
@@ -107,7 +108,6 @@ go_test(
         "init_test.go",
         "log_test.go",
         "metrics_test.go",
-        "mock_engine_test.go",
         "mock_test.go",
         "optimistic_sync_test.go",
         "pow_block_test.go",
@@ -131,6 +131,7 @@ go_test(
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
+        "//beacon-chain/powchain/testing:go_default_library",
         "//beacon-chain/state/stateutil:go_default_library",
         "//beacon-chain/state/v1:go_default_library",
         "//config/fieldparams:go_default_library",
@@ -184,6 +185,7 @@ go_test(
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
+        "//beacon-chain/powchain/testing:go_default_library",
         "//config/params:go_default_library",
         "//encoding/bytesutil:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",

beacon-chain/blockchain/chain_info.go

@@ -328,10 +328,10 @@ func (s *Service) IsOptimistic(ctx context.Context) (bool, error) {
 	return s.IsOptimisticForRoot(ctx, s.head.root)
 }
 
-// IsOptimisticForRoot takes the root and slot as aguments instead of the current head
+// IsOptimisticForRoot takes the root and slot as arguments instead of the current head
 // and returns true if it is optimistic.
 func (s *Service) IsOptimisticForRoot(ctx context.Context, root [32]byte) (bool, error) {
-	optimistic, err := s.cfg.ForkChoiceStore.IsOptimistic(ctx, root)
+	optimistic, err := s.cfg.ForkChoiceStore.IsOptimistic(root)
 	if err == nil {
 		return optimistic, nil
 	}
@@ -358,11 +358,26 @@ func (s *Service) IsOptimisticForRoot(ctx context.Context, root [32]byte) (bool,
 		return false, nil
 	}
 
-	lastValidated, err := s.cfg.BeaconDB.StateSummary(ctx, bytesutil.ToBytes32(validatedCheckpoint.Root))
+	// checkpoint root could be zeros before the first finalized epoch. Use genesis root if the case.
+	lastValidated, err := s.cfg.BeaconDB.StateSummary(ctx, s.ensureRootNotZeros(bytesutil.ToBytes32(validatedCheckpoint.Root)))
 	if err != nil {
 		return false, err
 	}
-	return ss.Slot > lastValidated.Slot, nil
+	if lastValidated == nil {
+		return false, errInvalidNilSummary
+	}
+
+	if ss.Slot > lastValidated.Slot {
+		return true, nil
+	}
+
+	isCanonical, err := s.IsCanonical(ctx, root)
+	if err != nil {
+		return false, err
+	}
+
+	// historical non-canonical blocks here are returned as optimistic for safety.
+	return !isCanonical, nil
 }
 
 // SetGenesisTime sets the genesis time of beacon chain.

beacon-chain/blockchain/chain_info_norace_test.go

@@ -9,6 +9,7 @@ import (
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
 	"github.com/prysmaticlabs/prysm/testing/require"
+	"github.com/prysmaticlabs/prysm/testing/util"
 )
 
 func TestHeadSlot_DataRace(t *testing.T) {
@@ -16,10 +17,13 @@ func TestHeadSlot_DataRace(t *testing.T) {
 	s := &Service{
 		cfg: &config{BeaconDB: beaconDB},
 	}
+	b, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlock())
+	require.NoError(t, err)
+	st, _ := util.DeterministicGenesisState(t, 1)
 	wait := make(chan struct{})
 	go func() {
 		defer close(wait)
-		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}, b, st))
 	}()
 	s.HeadSlot()
 	<-wait
@@ -31,12 +35,16 @@ func TestHeadRoot_DataRace(t *testing.T) {
 		cfg:  &config{BeaconDB: beaconDB, StateGen: stategen.New(beaconDB)},
 		head: &head{root: [32]byte{'A'}},
 	}
+	b, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlock())
+	require.NoError(t, err)
 	wait := make(chan struct{})
+	st, _ := util.DeterministicGenesisState(t, 1)
 	go func() {
 		defer close(wait)
-		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}, b, st))
+
 	}()
-	_, err := s.HeadRoot(context.Background())
+	_, err = s.HeadRoot(context.Background())
 	require.NoError(t, err)
 	<-wait
 }
@@ -49,10 +57,14 @@ func TestHeadBlock_DataRace(t *testing.T) {
 		cfg:  &config{BeaconDB: beaconDB, StateGen: stategen.New(beaconDB)},
 		head: &head{block: wsb},
 	}
+	b, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlock())
+	require.NoError(t, err)
 	wait := make(chan struct{})
+	st, _ := util.DeterministicGenesisState(t, 1)
 	go func() {
 		defer close(wait)
-		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}, b, st))
+
 	}()
 	_, err = s.HeadBlock(context.Background())
 	require.NoError(t, err)
@@ -64,12 +76,16 @@ func TestHeadState_DataRace(t *testing.T) {
 	s := &Service{
 		cfg: &config{BeaconDB: beaconDB, StateGen: stategen.New(beaconDB)},
 	}
+	b, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlock())
+	require.NoError(t, err)
 	wait := make(chan struct{})
+	st, _ := util.DeterministicGenesisState(t, 1)
 	go func() {
 		defer close(wait)
-		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}, b, st))
+
 	}()
-	_, err := s.HeadState(context.Background())
+	_, err = s.HeadState(context.Background())
 	require.NoError(t, err)
 	<-wait
 }

beacon-chain/blockchain/chain_info_test.go

@@ -459,7 +459,7 @@ func TestService_IsOptimisticForRoot_DB_ProtoArray(t *testing.T) {
 	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: br[:], Slot: 10}))
 
 	optimisticBlock := util.NewBeaconBlock()
-	optimisticBlock.Block.Slot = 11
+	optimisticBlock.Block.Slot = 97
 	optimisticRoot, err := optimisticBlock.Block.HashTreeRoot()
 	require.NoError(t, err)
 	wsb, err = wrapper.WrappedSignedBeaconBlock(optimisticBlock)
@@ -477,18 +477,39 @@ func TestService_IsOptimisticForRoot_DB_ProtoArray(t *testing.T) {
 	validatedCheckpoint := &ethpb.Checkpoint{Root: br[:]}
 	require.NoError(t, beaconDB.SaveLastValidatedCheckpoint(ctx, validatedCheckpoint))
 
+	_, err = c.IsOptimisticForRoot(ctx, optimisticRoot)
+	require.ErrorContains(t, "nil summary returned from the DB", err)
+
 	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: optimisticRoot[:], Slot: 11}))
 	optimistic, err := c.IsOptimisticForRoot(ctx, optimisticRoot)
 	require.NoError(t, err)
 	require.Equal(t, true, optimistic)
 
 	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: validatedRoot[:], Slot: 9}))
+	cp := &ethpb.Checkpoint{
+		Epoch: 1,
+		Root:  validatedRoot[:],
+	}
+	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, validatedRoot))
+	require.NoError(t, beaconDB.SaveFinalizedCheckpoint(ctx, cp))
+
 	validated, err := c.IsOptimisticForRoot(ctx, validatedRoot)
 	require.NoError(t, err)
 	require.Equal(t, false, validated)
+
+	// Before the first finalized epoch, finalized root could be zeros.
+	validatedCheckpoint = &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, br))
+	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: params.BeaconConfig().ZeroHash[:], Slot: 10}))
+	require.NoError(t, beaconDB.SaveLastValidatedCheckpoint(ctx, validatedCheckpoint))
+
+	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: optimisticRoot[:], Slot: 11}))
+	optimistic, err = c.IsOptimisticForRoot(ctx, optimisticRoot)
+	require.NoError(t, err)
+	require.Equal(t, true, optimistic)
 }
 
-func TestService_IsOptimisticForRoot__DB_DoublyLinkedTree(t *testing.T) {
+func TestService_IsOptimisticForRoot_DB_DoublyLinkedTree(t *testing.T) {
 	beaconDB := testDB.SetupDB(t)
 	ctx := context.Background()
 	c := &Service{cfg: &config{BeaconDB: beaconDB, ForkChoiceStore: doublylinkedtree.New(0, 0)}, head: &head{slot: 101, root: [32]byte{'b'}}}
@@ -503,7 +524,71 @@ func TestService_IsOptimisticForRoot__DB_DoublyLinkedTree(t *testing.T) {
 	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: br[:], Slot: 10}))
 
 	optimisticBlock := util.NewBeaconBlock()
-	optimisticBlock.Block.Slot = 11
+	optimisticBlock.Block.Slot = 97
+	optimisticRoot, err := optimisticBlock.Block.HashTreeRoot()
+	require.NoError(t, err)
+	wsb, err = wrapper.WrappedSignedBeaconBlock(optimisticBlock)
+	require.NoError(t, err)
+	require.NoError(t, beaconDB.SaveBlock(context.Background(), wsb))
+
+	validatedBlock := util.NewBeaconBlock()
+	validatedBlock.Block.Slot = 9
+	validatedRoot, err := validatedBlock.Block.HashTreeRoot()
+	require.NoError(t, err)
+	wsb, err = wrapper.WrappedSignedBeaconBlock(validatedBlock)
+	require.NoError(t, err)
+	require.NoError(t, beaconDB.SaveBlock(context.Background(), wsb))
+
+	validatedCheckpoint := &ethpb.Checkpoint{Root: br[:]}
+	require.NoError(t, beaconDB.SaveLastValidatedCheckpoint(ctx, validatedCheckpoint))
+
+	_, err = c.IsOptimisticForRoot(ctx, optimisticRoot)
+	require.ErrorContains(t, "nil summary returned from the DB", err)
+
+	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: optimisticRoot[:], Slot: 11}))
+	optimistic, err := c.IsOptimisticForRoot(ctx, optimisticRoot)
+	require.NoError(t, err)
+	require.Equal(t, true, optimistic)
+
+	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: validatedRoot[:], Slot: 9}))
+	cp := &ethpb.Checkpoint{
+		Epoch: 1,
+		Root:  validatedRoot[:],
+	}
+	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, validatedRoot))
+	require.NoError(t, beaconDB.SaveFinalizedCheckpoint(ctx, cp))
+	validated, err := c.IsOptimisticForRoot(ctx, validatedRoot)
+	require.NoError(t, err)
+	require.Equal(t, false, validated)
+
+	// Before the first finalized epoch, finalized root could be zeros.
+	validatedCheckpoint = &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, br))
+	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: params.BeaconConfig().ZeroHash[:], Slot: 10}))
+	require.NoError(t, beaconDB.SaveLastValidatedCheckpoint(ctx, validatedCheckpoint))
+
+	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: optimisticRoot[:], Slot: 11}))
+	optimistic, err = c.IsOptimisticForRoot(ctx, optimisticRoot)
+	require.NoError(t, err)
+	require.Equal(t, true, optimistic)
+}
+
+func TestService_IsOptimisticForRoot_DB_non_canonical(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+	ctx := context.Background()
+	c := &Service{cfg: &config{BeaconDB: beaconDB, ForkChoiceStore: doublylinkedtree.New(0, 0)}, head: &head{slot: 101, root: [32]byte{'b'}}}
+	c.head = &head{root: params.BeaconConfig().ZeroHash}
+	b := util.NewBeaconBlock()
+	b.Block.Slot = 10
+	br, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	wsb, err := wrapper.WrappedSignedBeaconBlock(b)
+	require.NoError(t, err)
+	require.NoError(t, beaconDB.SaveBlock(context.Background(), wsb))
+	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: br[:], Slot: 10}))
+
+	optimisticBlock := util.NewBeaconBlock()
+	optimisticBlock.Block.Slot = 97
 	optimisticRoot, err := optimisticBlock.Block.HashTreeRoot()
 	require.NoError(t, err)
 	wsb, err = wrapper.WrappedSignedBeaconBlock(optimisticBlock)
@@ -529,5 +614,6 @@ func TestService_IsOptimisticForRoot__DB_DoublyLinkedTree(t *testing.T) {
 	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: validatedRoot[:], Slot: 9}))
 	validated, err := c.IsOptimisticForRoot(ctx, validatedRoot)
 	require.NoError(t, err)
-	require.Equal(t, false, validated)
+	require.Equal(t, true, validated)
+
 }

beacon-chain/blockchain/error.go

@@ -13,7 +13,9 @@ var (
 	errInvalidNilSummary = errors.New("nil summary returned from the DB")
 	// errNilParentInDB is returned when a nil parent block is returned from the DB.
 	errNilParentInDB = errors.New("nil parent block in DB")
-	// errWrongBlockCound is returned when the wrong number of blocks or
+	// errWrongBlockCount is returned when the wrong number of blocks or
 	// block roots is used
 	errWrongBlockCount = errors.New("wrong number of blocks or block roots")
+	// block is not a valid optimistic candidate block
+	errNotOptimisticCandidate = errors.New("block is not suitable for optimistic sync")
 )

beacon-chain/blockchain/head.go

@@ -24,8 +24,9 @@ import (
 	"go.opencensus.io/trace"
 )
 
-// UpdateHeadWithBalances updates the beacon state head after getting justified balanced from cache.
-func (s *Service) UpdateHeadWithBalances(ctx context.Context) error {
+// UpdateAndSaveHeadWithBalances updates the beacon state head after getting justified balanced from cache.
+// This function is only used in spec-tests, it does save the head after updating it.
+func (s *Service) UpdateAndSaveHeadWithBalances(ctx context.Context) error {
 	cp := s.store.JustifiedCheckpt()
 	if cp == nil {
 		return errors.New("no justified checkpoint")
@@ -35,8 +36,19 @@ func (s *Service) UpdateHeadWithBalances(ctx context.Context) error {
 		msg := fmt.Sprintf("could not read balances for state w/ justified checkpoint %#x", cp.Root)
 		return errors.Wrap(err, msg)
 	}
-
-	return s.updateHead(ctx, balances)
+	headRoot, err := s.updateHead(ctx, balances)
+	if err != nil {
+		return errors.Wrap(err, "could not update head")
+	}
+	headBlock, err := s.cfg.BeaconDB.Block(ctx, headRoot)
+	if err != nil {
+		return err
+	}
+	headState, err := s.cfg.StateGen.StateByRoot(ctx, headRoot)
+	if err != nil {
+		return errors.Wrap(err, "could not retrieve head state in DB")
+	}
+	return s.saveHead(ctx, headRoot, headBlock, headState)
 }
 
 // This defines the current chain service's view of head.
@@ -49,18 +61,18 @@ type head struct {
 
 // Determined the head from the fork choice service and saves its new data
 // (head root, head block, and head state) to the local service cache.
-func (s *Service) updateHead(ctx context.Context, balances []uint64) error {
+func (s *Service) updateHead(ctx context.Context, balances []uint64) ([32]byte, error) {
 	ctx, span := trace.StartSpan(ctx, "blockChain.updateHead")
 	defer span.End()
 
 	// Get head from the fork choice service.
 	f := s.store.FinalizedCheckpt()
 	if f == nil {
-		return errNilFinalizedInStore
+		return [32]byte{}, errNilFinalizedInStore
 	}
 	j := s.store.JustifiedCheckpt()
 	if j == nil {
-		return errNilJustifiedInStore
+		return [32]byte{}, errNilJustifiedInStore
 	}
 	// To get head before the first justified epoch, the fork choice will start with origin root
 	// instead of zero hashes.
@@ -76,7 +88,7 @@ func (s *Service) updateHead(ctx context.Context, balances []uint64) error {
 	if !s.cfg.ForkChoiceStore.HasNode(headStartRoot) {
 		jb, err := s.cfg.BeaconDB.Block(ctx, headStartRoot)
 		if err != nil {
-			return err
+			return [32]byte{}, err
 		}
 		if features.Get().EnableForkChoiceDoublyLinkedTree {
 			s.cfg.ForkChoiceStore = doublylinkedtree.New(j.Epoch, f.Epoch)
@@ -84,22 +96,16 @@ func (s *Service) updateHead(ctx context.Context, balances []uint64) error {
 			s.cfg.ForkChoiceStore = protoarray.New(j.Epoch, f.Epoch, bytesutil.ToBytes32(f.Root))
 		}
 		if err := s.insertBlockToForkChoiceStore(ctx, jb.Block(), headStartRoot, f, j); err != nil {
-			return err
+			return [32]byte{}, err
 		}
 	}
 
-	headRoot, err := s.cfg.ForkChoiceStore.Head(ctx, j.Epoch, headStartRoot, balances, f.Epoch)
-	if err != nil {
-		return err
-	}
-
-	// Save head to the local service cache.
-	return s.saveHead(ctx, headRoot)
+	return s.cfg.ForkChoiceStore.Head(ctx, j.Epoch, headStartRoot, balances, f.Epoch)
 }
 
 // This saves head info to the local service cache, it also saves the
 // new head root to the DB.
-func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
+func (s *Service) saveHead(ctx context.Context, headRoot [32]byte, headBlock block.SignedBeaconBlock, headState state.BeaconState) error {
 	ctx, span := trace.StartSpan(ctx, "blockChain.saveHead")
 	defer span.End()
 
@@ -111,6 +117,12 @@ func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
 	if headRoot == bytesutil.ToBytes32(r) {
 		return nil
 	}
+	if err := helpers.BeaconBlockIsNil(headBlock); err != nil {
+		return err
+	}
+	if headState == nil || headState.IsNil() {
+		return errors.New("cannot save nil head state")
+	}
 
 	// If the head state is not available, just return nil.
 	// There's nothing to cache
@@ -118,31 +130,13 @@ func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
 		return nil
 	}
 
-	// Get the new head block from DB.
-	newHeadBlock, err := s.cfg.BeaconDB.Block(ctx, headRoot)
-	if err != nil {
-		return err
-	}
-	if err := helpers.BeaconBlockIsNil(newHeadBlock); err != nil {
-		return err
-	}
-
-	// Get the new head state from cached state or DB.
-	newHeadState, err := s.cfg.StateGen.StateByRoot(ctx, headRoot)
-	if err != nil {
-		return errors.Wrap(err, "could not retrieve head state in DB")
-	}
-	if newHeadState == nil || newHeadState.IsNil() {
-		return errors.New("cannot save nil head state")
-	}
-
 	// A chain re-org occurred, so we fire an event notifying the rest of the services.
 	headSlot := s.HeadSlot()
-	newHeadSlot := newHeadBlock.Block().Slot()
+	newHeadSlot := headBlock.Block().Slot()
 	oldHeadRoot := s.headRoot()
 	oldStateRoot := s.headBlock().Block().StateRoot()
-	newStateRoot := newHeadBlock.Block().StateRoot()
-	if bytesutil.ToBytes32(newHeadBlock.Block().ParentRoot()) != bytesutil.ToBytes32(r) {
+	newStateRoot := headBlock.Block().StateRoot()
+	if bytesutil.ToBytes32(headBlock.Block().ParentRoot()) != bytesutil.ToBytes32(r) {
 		log.WithFields(logrus.Fields{
 			"newSlot": fmt.Sprintf("%d", newHeadSlot),
 			"oldSlot": fmt.Sprintf("%d", headSlot),
@@ -174,7 +168,7 @@ func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
 	}
 
 	// Cache the new head info.
-	s.setHead(headRoot, newHeadBlock, newHeadState)
+	s.setHead(headRoot, headBlock, headState)
 
 	// Save the new head root to DB.
 	if err := s.cfg.BeaconDB.SaveHeadBlockRoot(ctx, headRoot); err != nil {
@@ -184,7 +178,7 @@ func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
 	// Forward an event capturing a new chain head over a common event feed
 	// done in a goroutine to avoid blocking the critical runtime main routine.
 	go func() {
-		if err := s.notifyNewHeadEvent(ctx, newHeadSlot, newHeadState, newStateRoot, headRoot[:]); err != nil {
+		if err := s.notifyNewHeadEvent(ctx, newHeadSlot, headState, newStateRoot, headRoot[:]); err != nil {
 			log.WithError(err).Error("Could not notify event feed of new chain head")
 		}
 	}()

beacon-chain/blockchain/head_test.go

@@ -9,6 +9,8 @@ import (
 	types "github.com/prysmaticlabs/eth2-types"
 	mock "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	doublylinkedtree "github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/doubly-linked-tree"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	"github.com/prysmaticlabs/prysm/config/features"
 	"github.com/prysmaticlabs/prysm/config/params"
 	ethpbv1 "github.com/prysmaticlabs/prysm/proto/eth/v1"
@@ -27,8 +29,10 @@ func TestSaveHead_Same(t *testing.T) {
 
 	r := [32]byte{'A'}
 	service.head = &head{slot: 0, root: r}
-
-	require.NoError(t, service.saveHead(context.Background(), r))
+	b, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlock())
+	require.NoError(t, err)
+	st, _ := util.DeterministicGenesisState(t, 1)
+	require.NoError(t, service.saveHead(context.Background(), r, b, st))
 	assert.Equal(t, types.Slot(0), service.headSlot(), "Head did not stay the same")
 	assert.Equal(t, r, service.headRoot(), "Head did not stay the same")
 }
@@ -66,7 +70,7 @@ func TestSaveHead_Different(t *testing.T) {
 	require.NoError(t, headState.SetSlot(1))
 	require.NoError(t, service.cfg.BeaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Slot: 1, Root: newRoot[:]}))
 	require.NoError(t, service.cfg.BeaconDB.SaveState(context.Background(), headState, newRoot))
-	require.NoError(t, service.saveHead(context.Background(), newRoot))
+	require.NoError(t, service.saveHead(context.Background(), newRoot, wsb, headState))
 
 	assert.Equal(t, types.Slot(1), service.HeadSlot(), "Head did not change")
 
@@ -112,7 +116,7 @@ func TestSaveHead_Different_Reorg(t *testing.T) {
 	require.NoError(t, headState.SetSlot(1))
 	require.NoError(t, service.cfg.BeaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Slot: 1, Root: newRoot[:]}))
 	require.NoError(t, service.cfg.BeaconDB.SaveState(context.Background(), headState, newRoot))
-	require.NoError(t, service.saveHead(context.Background(), newRoot))
+	require.NoError(t, service.saveHead(context.Background(), newRoot, wsb, headState))
 
 	assert.Equal(t, types.Slot(1), service.HeadSlot(), "Head did not change")
 
@@ -154,8 +158,10 @@ func TestUpdateHead_MissingJustifiedRoot(t *testing.T) {
 	service.store.SetJustifiedCheckpt(&ethpb.Checkpoint{Root: r[:]})
 	service.store.SetFinalizedCheckpt(&ethpb.Checkpoint{})
 	service.store.SetBestJustifiedCheckpt(&ethpb.Checkpoint{})
-
-	require.NoError(t, service.updateHead(context.Background(), []uint64{}))
+	headRoot, err := service.updateHead(context.Background(), []uint64{})
+	require.NoError(t, err)
+	st, _ := util.DeterministicGenesisState(t, 1)
+	require.NoError(t, service.saveHead(context.Background(), headRoot, wsb, st))
 }
 
 func Test_notifyNewHeadEvent(t *testing.T) {
@@ -279,3 +285,43 @@ func TestSaveOrphanedAtts_CanFilter(t *testing.T) {
 	atts := b.Block.Body.Attestations
 	require.DeepNotSSZEqual(t, atts, savedAtts)
 }
+
+func TestUpdateHead_noSavedChanges(t *testing.T) {
+	ctx := context.Background()
+
+	beaconDB := testDB.SetupDB(t)
+	fcs := doublylinkedtree.New(0, 0)
+	opts := []Option{
+		WithDatabase(beaconDB),
+		WithStateGen(stategen.New(beaconDB)),
+		WithForkChoiceStore(fcs),
+	}
+
+	service, err := NewService(ctx, opts...)
+	require.NoError(t, err)
+
+	bellatrixBlk, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlockBellatrix())
+	require.NoError(t, err)
+	bellatrixBlkRoot, err := bellatrixBlk.Block().HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, beaconDB.SaveBlock(ctx, bellatrixBlk))
+	fcp := &ethpb.Checkpoint{
+		Root:  bellatrixBlkRoot[:],
+		Epoch: 1,
+	}
+	service.store.SetFinalizedCheckpt(fcp)
+	service.store.SetJustifiedCheckpt(fcp)
+	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, bellatrixBlkRoot))
+
+	bellatrixState, _ := util.DeterministicGenesisStateBellatrix(t, 2)
+	require.NoError(t, beaconDB.SaveState(ctx, bellatrixState, bellatrixBlkRoot))
+	service.cfg.StateGen.SaveFinalizedState(0, bellatrixBlkRoot, bellatrixState)
+
+	headRoot := service.headRoot()
+	require.Equal(t, [32]byte{}, headRoot)
+
+	newRoot, err := service.updateHead(ctx, []uint64{1, 2})
+	require.NoError(t, err)
+	require.NotEqual(t, headRoot, newRoot)
+	require.Equal(t, headRoot, service.headRoot())
+}

beacon-chain/blockchain/metrics.go

@@ -138,6 +138,26 @@ var (
 		Name: "state_balance_cache_miss",
 		Help: "Count the number of state balance cache hits.",
 	})
+	newPayloadValidNodeCount = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "new_payload_valid_node_count",
+		Help: "Count the number of valid nodes after newPayload EE call",
+	})
+	newPayloadOptimisticNodeCount = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "new_payload_optimistic_node_count",
+		Help: "Count the number of optimistic nodes after newPayload EE call",
+	})
+	newPayloadInvalidNodeCount = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "new_payload_invalid_node_count",
+		Help: "Count the number of invalid nodes after newPayload EE call",
+	})
+	forkchoiceUpdatedValidNodeCount = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "forkchoice_updated_valid_node_count",
+		Help: "Count the number of valid nodes after forkchoiceUpdated EE call",
+	})
+	forkchoiceUpdatedOptimisticNodeCount = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "forkchoice_updated_optimistic_node_count",
+		Help: "Count the number of optimistic nodes after forkchoiceUpdated EE call",
+	})
 )
 
 // reportSlotMetrics reports slot related metrics.

beacon-chain/blockchain/mock_engine_test.go

@@ -1,49 +0,0 @@
-package blockchain
-
-import (
-	"context"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/pkg/errors"
-	enginev1 "github.com/prysmaticlabs/prysm/proto/engine/v1"
-)
-
-type mockEngineService struct {
-	newPayloadError error
-	forkchoiceError error
-	blks            map[[32]byte]*enginev1.ExecutionBlock
-}
-
-func (m *mockEngineService) NewPayload(context.Context, *enginev1.ExecutionPayload) ([]byte, error) {
-	return nil, m.newPayloadError
-}
-
-func (m *mockEngineService) ForkchoiceUpdated(context.Context, *enginev1.ForkchoiceState, *enginev1.PayloadAttributes) (*enginev1.PayloadIDBytes, []byte, error) {
-	return nil, nil, m.forkchoiceError
-}
-
-func (*mockEngineService) GetPayloadV1(
-	_ context.Context, _ enginev1.PayloadIDBytes,
-) *enginev1.ExecutionPayload {
-	return nil
-}
-
-func (*mockEngineService) GetPayload(context.Context, [8]byte) (*enginev1.ExecutionPayload, error) {
-	return nil, nil
-}
-
-func (*mockEngineService) ExchangeTransitionConfiguration(context.Context, *enginev1.TransitionConfiguration) error {
-	return nil
-}
-
-func (*mockEngineService) LatestExecutionBlock(context.Context) (*enginev1.ExecutionBlock, error) {
-	return nil, nil
-}
-
-func (m *mockEngineService) ExecutionBlockByHash(_ context.Context, hash common.Hash) (*enginev1.ExecutionBlock, error) {
-	blk, ok := m.blks[common.BytesToHash(hash.Bytes())]
-	if !ok {
-		return nil, errors.New("block not found")
-	}
-	return blk, nil
-}

beacon-chain/blockchain/new_slot_test.go

@@ -86,9 +86,9 @@ func TestService_newSlot(t *testing.T) {
 	for _, test := range tests {
 		service, err := NewService(ctx, opts...)
 		require.NoError(t, err)
-		store := store.New(test.args.justified, test.args.finalized)
-		store.SetBestJustifiedCheckpt(test.args.bestJustified)
-		service.store = store
+		s := store.New(test.args.justified, test.args.finalized)
+		s.SetBestJustifiedCheckpt(test.args.bestJustified)
+		service.store = s
 
 		require.NoError(t, service.NewSlot(ctx, test.args.slot))
 		if test.args.shouldEqual {

beacon-chain/blockchain/optimistic_sync.go

@@ -5,14 +5,21 @@ import (
 	"fmt"
 
 	"github.com/pkg/errors"
+	types "github.com/prysmaticlabs/eth2-types"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	v1 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/time"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/transition"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db/kv"
+	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 	enginev1 "github.com/prysmaticlabs/prysm/proto/engine/v1"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
+	"github.com/prysmaticlabs/prysm/time/slots"
 	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 )
@@ -20,7 +27,7 @@ import (
 // notifyForkchoiceUpdate signals execution engine the fork choice updates. Execution engine should:
 // 1. Re-organizes the execution payload chain and corresponding state to make head_block_hash the head.
 // 2. Applies finality to the execution state: it irreversibly persists the chain of all execution payloads and corresponding state, up to and including finalized_block_hash.
-func (s *Service) notifyForkchoiceUpdate(ctx context.Context, headBlk block.BeaconBlock, headRoot [32]byte, finalizedRoot [32]byte) (*enginev1.PayloadIDBytes, error) {
+func (s *Service) notifyForkchoiceUpdate(ctx context.Context, headState state.BeaconState, headBlk block.BeaconBlock, headRoot [32]byte, finalizedRoot [32]byte) (*enginev1.PayloadIDBytes, error) {
 	ctx, span := trace.StartSpan(ctx, "blockChain.notifyForkchoiceUpdate")
 	defer span.End()
 
@@ -39,106 +46,169 @@ func (s *Service) notifyForkchoiceUpdate(ctx context.Context, headBlk block.Beac
 	if err != nil {
 		return nil, errors.Wrap(err, "could not get execution payload")
 	}
-	finalizedBlock, err := s.cfg.BeaconDB.Block(ctx, s.ensureRootNotZeros(finalizedRoot))
+	finalizedHash, err := s.getFinalizedPayloadHash(ctx, finalizedRoot)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not get finalized block")
-	}
-	var finalizedHash []byte
-	if blocks.IsPreBellatrixVersion(finalizedBlock.Block().Version()) {
-		finalizedHash = params.BeaconConfig().ZeroHash[:]
-	} else {
-		payload, err := finalizedBlock.Block().Body().ExecutionPayload()
-		if err != nil {
-			return nil, errors.Wrap(err, "could not get finalized block execution payload")
-		}
-		finalizedHash = payload.BlockHash
+		return nil, errors.Wrap(err, "could not get finalized payload hash")
 	}
 
 	fcs := &enginev1.ForkchoiceState{
 		HeadBlockHash:      headPayload.BlockHash,
 		SafeBlockHash:      headPayload.BlockHash,
-		FinalizedBlockHash: finalizedHash,
+		FinalizedBlockHash: finalizedHash[:],
 	}
 
-	// payload attribute is only required when requesting payload, here we are just updating fork choice, so it is nil.
-	payloadID, _, err := s.cfg.ExecutionEngineCaller.ForkchoiceUpdated(ctx, fcs, nil /*payload attribute*/)
+	nextSlot := s.CurrentSlot() + 1 // Cache payload ID for next slot proposer.
+	hasAttr, attr, proposerId, err := s.getPayloadAttribute(ctx, headState, nextSlot)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get payload attribute")
+	}
+
+	payloadID, _, err := s.cfg.ExecutionEngineCaller.ForkchoiceUpdated(ctx, fcs, attr)
 	if err != nil {
 		switch err {
-		case v1.ErrAcceptedSyncingPayloadStatus:
+		case powchain.ErrAcceptedSyncingPayloadStatus:
+			forkchoiceUpdatedOptimisticNodeCount.Inc()
 			log.WithFields(logrus.Fields{
-				"headSlot":      headBlk.Slot(),
-				"headHash":      fmt.Sprintf("%#x", bytesutil.Trunc(headPayload.BlockHash)),
-				"finalizedHash": fmt.Sprintf("%#x", bytesutil.Trunc(finalizedHash)),
+				"headSlot":                  headBlk.Slot(),
+				"headPayloadBlockHash":      fmt.Sprintf("%#x", bytesutil.Trunc(headPayload.BlockHash)),
+				"finalizedPayloadBlockHash": fmt.Sprintf("%#x", bytesutil.Trunc(finalizedHash[:])),
 			}).Info("Called fork choice updated with optimistic block")
 			return payloadID, nil
 		default:
 			return nil, errors.Wrap(err, "could not notify forkchoice update from execution engine")
 		}
 	}
+	forkchoiceUpdatedValidNodeCount.Inc()
 	if err := s.cfg.ForkChoiceStore.SetOptimisticToValid(ctx, headRoot); err != nil {
 		return nil, errors.Wrap(err, "could not set block to valid")
 	}
+	if hasAttr { // If the forkchoice update call has an attribute, update the proposer payload ID cache.
+		var pId [8]byte
+		copy(pId[:], payloadID[:])
+		s.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(nextSlot, proposerId, pId)
+	}
 	return payloadID, nil
 }
 
-// notifyForkchoiceUpdate signals execution engine on a new payload
+// getFinalizedPayloadHash returns the finalized payload hash for the given finalized block root.
+// It checks the following in order:
+//   1. The finalized block exists in db
+//   2. The finalized block exists in initial sync block cache
+//   3. The finalized block is the weak subjectivity block and exists in db
+// Error is returned if the finalized block is not found from above.
+func (s *Service) getFinalizedPayloadHash(ctx context.Context, finalizedRoot [32]byte) ([32]byte, error) {
+	b, err := s.cfg.BeaconDB.Block(ctx, s.ensureRootNotZeros(finalizedRoot))
+	if err != nil {
+		return [32]byte{}, errors.Wrap(err, "could not get finalized block")
+	}
+	if b != nil {
+		return getPayloadHash(b.Block())
+	}
+
+	b = s.getInitSyncBlock(finalizedRoot)
+	if b != nil {
+		return getPayloadHash(b.Block())
+	}
+
+	r, err := s.cfg.BeaconDB.OriginCheckpointBlockRoot(ctx)
+	if err != nil {
+		return [32]byte{}, errors.Wrap(err, "could not get finalized block")
+	}
+	b, err = s.cfg.BeaconDB.Block(ctx, r)
+	if err != nil {
+		return [32]byte{}, errors.Wrap(err, "could not get finalized block")
+	}
+	if b != nil {
+		return getPayloadHash(b.Block())
+	}
+
+	return [32]byte{}, errors.Errorf("finalized block with root %#x does not exist in the db or our cache", s.ensureRootNotZeros(finalizedRoot))
+}
+
+// getPayloadHash returns the payload hash for the input given block.
+// zeros are returned if the block is older than bellatrix.
+func getPayloadHash(b block.BeaconBlock) ([32]byte, error) {
+	if blocks.IsPreBellatrixVersion(b.Version()) {
+		return params.BeaconConfig().ZeroHash, nil
+	}
+
+	payload, err := b.Body().ExecutionPayload()
+	if err != nil {
+		return [32]byte{}, errors.Wrap(err, "could not get finalized block execution payload")
+	}
+	return bytesutil.ToBytes32(payload.BlockHash), nil
+}
+
+// notifyForkchoiceUpdate signals execution engine on a new payload.
+// It returns true if the EL has returned VALID for the block
 func (s *Service) notifyNewPayload(ctx context.Context, preStateVersion, postStateVersion int,
-	preStateHeader, postStateHeader *ethpb.ExecutionPayloadHeader, blk block.SignedBeaconBlock, root [32]byte) error {
+	preStateHeader, postStateHeader *ethpb.ExecutionPayloadHeader, blk block.SignedBeaconBlock) (bool, error) {
 	ctx, span := trace.StartSpan(ctx, "blockChain.notifyNewPayload")
 	defer span.End()
 
 	// Execution payload is only supported in Bellatrix and beyond. Pre
 	// merge blocks are never optimistic
 	if blocks.IsPreBellatrixVersion(postStateVersion) {
-		return s.cfg.ForkChoiceStore.SetOptimisticToValid(ctx, root)
+		return true, nil
 	}
 	if err := helpers.BeaconBlockIsNil(blk); err != nil {
-		return err
+		return false, err
 	}
 	body := blk.Block().Body()
 	enabled, err := blocks.IsExecutionEnabledUsingHeader(postStateHeader, body)
 	if err != nil {
-		return errors.Wrap(err, "could not determine if execution is enabled")
+		return false, errors.Wrap(err, "could not determine if execution is enabled")
 	}
 	if !enabled {
-		return s.cfg.ForkChoiceStore.SetOptimisticToValid(ctx, root)
+		return true, nil
 	}
 	payload, err := body.ExecutionPayload()
 	if err != nil {
-		return errors.Wrap(err, "could not get execution payload")
+		return false, errors.Wrap(err, "could not get execution payload")
 	}
-	_, err = s.cfg.ExecutionEngineCaller.NewPayload(ctx, payload)
+	lastValidHash, err := s.cfg.ExecutionEngineCaller.NewPayload(ctx, payload)
 	if err != nil {
 		switch err {
-		case v1.ErrAcceptedSyncingPayloadStatus:
+		case powchain.ErrAcceptedSyncingPayloadStatus:
+			newPayloadOptimisticNodeCount.Inc()
 			log.WithFields(logrus.Fields{
-				"slot":      blk.Block().Slot(),
-				"blockHash": fmt.Sprintf("%#x", bytesutil.Trunc(payload.BlockHash)),
+				"slot":             blk.Block().Slot(),
+				"payloadBlockHash": fmt.Sprintf("%#x", bytesutil.Trunc(payload.BlockHash)),
 			}).Info("Called new payload with optimistic block")
-			return nil
+			return false, nil
+		case powchain.ErrInvalidPayloadStatus:
+			newPayloadInvalidNodeCount.Inc()
+			root, err := blk.Block().HashTreeRoot()
+			if err != nil {
+				return false, err
+			}
+			invalidRoots, err := s.ForkChoicer().SetOptimisticToInvalid(ctx, root, bytesutil.ToBytes32(lastValidHash))
+			if err != nil {
+				return false, err
+			}
+			if err := s.removeInvalidBlockAndState(ctx, invalidRoots); err != nil {
+				return false, err
+			}
+			return false, errors.New("could not validate an INVALID payload from execution engine")
 		default:
-			return errors.Wrap(err, "could not validate execution payload from execution engine")
+			return false, errors.Wrap(err, "could not validate execution payload from execution engine")
 		}
 	}
-
-	if err := s.cfg.ForkChoiceStore.SetOptimisticToValid(ctx, root); err != nil {
-		return errors.Wrap(err, "could not set optimistic status")
-	}
-
+	newPayloadValidNodeCount.Inc()
 	// During the transition event, the transition block should be verified for sanity.
 	if blocks.IsPreBellatrixVersion(preStateVersion) {
 		// Handle case where pre-state is Altair but block contains payload.
 		// To reach here, the block must have contained a valid payload.
-		return s.validateMergeBlock(ctx, blk)
+		return true, s.validateMergeBlock(ctx, blk)
 	}
 	atTransition, err := blocks.IsMergeTransitionBlockUsingPreStatePayloadHeader(preStateHeader, body)
 	if err != nil {
-		return errors.Wrap(err, "could not check if merge block is terminal")
+		return true, errors.Wrap(err, "could not check if merge block is terminal")
 	}
 	if !atTransition {
-		return nil
+		return true, nil
 	}
-	return s.validateMergeBlock(ctx, blk)
+	return true, s.validateMergeBlock(ctx, blk)
 }
 
 // optimisticCandidateBlock returns true if this block can be optimistically synced.
@@ -148,10 +218,6 @@ func (s *Service) notifyNewPayload(ctx context.Context, preStateVersion, postSta
 //    if is_execution_block(opt_store.blocks[block.parent_root]):
 //        return True
 //
-//    justified_root = opt_store.block_states[opt_store.head_block_root].current_justified_checkpoint.root
-//    if is_execution_block(opt_store.blocks[justified_root]):
-//        return True
-//
 //    if block.slot + SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY <= current_slot:
 //        return True
 //
@@ -173,17 +239,71 @@ func (s *Service) optimisticCandidateBlock(ctx context.Context, blk block.Beacon
 	if err != nil {
 		return false, err
 	}
-	if parentIsExecutionBlock {
-		return true, nil
+	return parentIsExecutionBlock, nil
+}
+
+// getPayloadAttributes returns the payload attributes for the given state and slot.
+// The attribute is required to initiate a payload build process in the context of an `engine_forkchoiceUpdated` call.
+func (s *Service) getPayloadAttribute(ctx context.Context, st state.BeaconState, slot types.Slot) (bool, *enginev1.PayloadAttributes, types.ValidatorIndex, error) {
+	proposerID, _, ok := s.cfg.ProposerSlotIndexCache.GetProposerPayloadIDs(slot)
+	if !ok { // There's no need to build attribute if there is no proposer for slot.
+		return false, nil, 0, nil
 	}
 
-	j := s.store.JustifiedCheckpt()
-	if j == nil {
-		return false, errNilJustifiedInStore
-	}
-	jBlock, err := s.cfg.BeaconDB.Block(ctx, bytesutil.ToBytes32(j.Root))
+	// Get previous randao.
+	st = st.Copy()
+	st, err := transition.ProcessSlotsIfPossible(ctx, st, slot)
 	if err != nil {
-		return false, err
+		return false, nil, 0, err
 	}
-	return blocks.IsExecutionBlock(jBlock.Block().Body())
+	prevRando, err := helpers.RandaoMix(st, time.CurrentEpoch(st))
+	if err != nil {
+		return false, nil, 0, nil
+	}
+
+	// Get fee recipient.
+	feeRecipient := params.BeaconConfig().DefaultFeeRecipient
+	recipient, err := s.cfg.BeaconDB.FeeRecipientByValidatorID(ctx, proposerID)
+	switch {
+	case errors.Is(err, kv.ErrNotFoundFeeRecipient):
+		if feeRecipient.String() == fieldparams.EthBurnAddressHex {
+			logrus.WithFields(logrus.Fields{
+				"validatorIndex": proposerID,
+				"burnAddress":    fieldparams.EthBurnAddressHex,
+			}).Error("Fee recipient not set. Using burn address")
+		}
+	case err != nil:
+		return false, nil, 0, errors.Wrap(err, "could not get fee recipient in db")
+	default:
+		feeRecipient = recipient
+	}
+
+	// Get timestamp.
+	t, err := slots.ToTime(uint64(s.genesisTime.Unix()), slot)
+	if err != nil {
+		return false, nil, 0, err
+	}
+	attr := &enginev1.PayloadAttributes{
+		Timestamp:             uint64(t.Unix()),
+		PrevRandao:            prevRando,
+		SuggestedFeeRecipient: feeRecipient.Bytes(),
+	}
+	return true, attr, proposerID, nil
+}
+
+// removeInvalidBlockAndState removes the invalid block and its corresponding state from the cache and DB.
+func (s *Service) removeInvalidBlockAndState(ctx context.Context, blkRoots [][32]byte) error {
+	for _, root := range blkRoots {
+		if err := s.cfg.StateGen.DeleteStateFromCaches(ctx, root); err != nil {
+			return err
+		}
+
+		// Delete block also deletes the state as well.
+		if err := s.cfg.BeaconDB.DeleteBlock(ctx, root); err != nil {
+			// TODO(10487): If a caller requests to delete a root that's justified and finalized. We should gracefully shutdown.
+			// This is an irreparable condition, it would me a justified or finalized block has become invalid.
+			return err
+		}
+	}
+	return nil
 }

beacon-chain/blockchain/optimistic_sync_test.go

@@ -5,10 +5,14 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
+	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
-	engine "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
+	mockPOW "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
@@ -23,6 +27,7 @@ import (
 	"github.com/prysmaticlabs/prysm/testing/require"
 	"github.com/prysmaticlabs/prysm/testing/util"
 	"github.com/prysmaticlabs/prysm/time/slots"
+	logTest "github.com/sirupsen/logrus/hooks/test"
 )
 
 func Test_NotifyForkchoiceUpdate(t *testing.T) {
@@ -43,8 +48,13 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 		WithDatabase(beaconDB),
 		WithStateGen(stategen.New(beaconDB)),
 		WithForkChoiceStore(fcs),
+		WithProposerIdsCache(cache.NewProposerPayloadIDsCache()),
 	}
 	service, err := NewService(ctx, opts...)
+	st, _ := util.DeterministicGenesisState(t, 1)
+	service.head = &head{
+		state: st,
+	}
 	require.NoError(t, err)
 	require.NoError(t, fcs.InsertOptimisticBlock(ctx, 0, [32]byte{}, [32]byte{}, params.BeaconConfig().ZeroHash, 0, 0))
 
@@ -133,7 +143,7 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 				require.NoError(t, err)
 				return b
 			}(),
-			newForkchoiceErr: engine.ErrAcceptedSyncingPayloadStatus,
+			newForkchoiceErr: powchain.ErrAcceptedSyncingPayloadStatus,
 			finalizedRoot:    bellatrixBlkRoot,
 		},
 		{
@@ -147,7 +157,7 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 				require.NoError(t, err)
 				return b
 			}(),
-			newForkchoiceErr: engine.ErrInvalidPayloadStatus,
+			newForkchoiceErr: powchain.ErrInvalidPayloadStatus,
 			finalizedRoot:    bellatrixBlkRoot,
 			errString:        "could not notify forkchoice update from execution engine: payload status is INVALID",
 		},
@@ -155,9 +165,9 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			engine := &mockEngineService{forkchoiceError: tt.newForkchoiceErr}
-			service.cfg.ExecutionEngineCaller = engine
-			_, err := service.notifyForkchoiceUpdate(ctx, tt.blk, service.headRoot(), tt.finalizedRoot)
+			service.cfg.ExecutionEngineCaller = &mockPOW.EngineClient{ErrForkchoiceUpdated: tt.newForkchoiceErr}
+			st, _ := util.DeterministicGenesisState(t, 1)
+			_, err := service.notifyForkchoiceUpdate(ctx, st, tt.blk, service.headRoot(), tt.finalizedRoot)
 			if tt.errString != "" {
 				require.ErrorContains(t, tt.errString, err)
 			} else {
@@ -183,13 +193,6 @@ func Test_NotifyNewPayload(t *testing.T) {
 	phase0State, _ := util.DeterministicGenesisState(t, 1)
 	altairState, _ := util.DeterministicGenesisStateAltair(t, 1)
 	bellatrixState, _ := util.DeterministicGenesisStateBellatrix(t, 2)
-	blk := &ethpb.SignedBeaconBlockBellatrix{
-		Block: &ethpb.BeaconBlockBellatrix{
-			Body: &ethpb.BeaconBlockBodyBellatrix{
-				ExecutionPayload: &v1.ExecutionPayload{},
-			},
-		},
-	}
 	a := &ethpb.SignedBeaconBlockAltair{
 		Block: &ethpb.BeaconBlockAltair{
 			Body: &ethpb.BeaconBlockBodyAltair{},
@@ -197,55 +200,84 @@ func Test_NotifyNewPayload(t *testing.T) {
 	}
 	altairBlk, err := wrapper.WrappedSignedBeaconBlock(a)
 	require.NoError(t, err)
-	bellatrixBlk, err := wrapper.WrappedSignedBeaconBlock(blk)
+	blk := &ethpb.SignedBeaconBlockBellatrix{
+		Block: &ethpb.BeaconBlockBellatrix{
+			Slot: 1,
+			Body: &ethpb.BeaconBlockBodyBellatrix{
+				ExecutionPayload: &v1.ExecutionPayload{
+					BlockNumber:   1,
+					ParentHash:    make([]byte, fieldparams.RootLength),
+					FeeRecipient:  make([]byte, fieldparams.FeeRecipientLength),
+					StateRoot:     make([]byte, fieldparams.RootLength),
+					ReceiptsRoot:  make([]byte, fieldparams.RootLength),
+					LogsBloom:     make([]byte, fieldparams.LogsBloomLength),
+					PrevRandao:    make([]byte, fieldparams.RootLength),
+					BaseFeePerGas: make([]byte, fieldparams.RootLength),
+					BlockHash:     make([]byte, fieldparams.RootLength),
+				},
+			},
+		},
+	}
+	bellatrixBlk, err := wrapper.WrappedSignedBeaconBlock(util.HydrateSignedBeaconBlockBellatrix(blk))
 	require.NoError(t, err)
 	service, err := NewService(ctx, opts...)
 	require.NoError(t, err)
+	r, err := bellatrixBlk.Block().HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, fcs.InsertOptimisticBlock(ctx, 0, [32]byte{}, [32]byte{}, params.BeaconConfig().ZeroHash, 0, 0))
+	require.NoError(t, fcs.InsertOptimisticBlock(ctx, 1, r, [32]byte{}, params.BeaconConfig().ZeroHash, 0, 0))
 
 	tests := []struct {
-		name          string
-		preState      state.BeaconState
-		postState     state.BeaconState
-		blk           block.SignedBeaconBlock
-		newPayloadErr error
-		errString     string
+		name           string
+		preState       state.BeaconState
+		postState      state.BeaconState
+		isValidPayload bool
+		blk            block.SignedBeaconBlock
+		newPayloadErr  error
+		errString      string
 	}{
 		{
-			name:      "phase 0 post state",
-			postState: phase0State,
-			preState:  phase0State,
+			name:           "phase 0 post state",
+			postState:      phase0State,
+			preState:       phase0State,
+			isValidPayload: true,
 		},
 		{
-			name:      "altair post state",
-			postState: altairState,
-			preState:  altairState,
+			name:           "altair post state",
+			postState:      altairState,
+			preState:       altairState,
+			isValidPayload: true,
 		},
 		{
-			name:      "nil beacon block",
-			postState: bellatrixState,
-			preState:  bellatrixState,
-			errString: "signed beacon block can't be nil",
+			name:           "nil beacon block",
+			postState:      bellatrixState,
+			preState:       bellatrixState,
+			errString:      "signed beacon block can't be nil",
+			isValidPayload: false,
 		},
 		{
-			name:          "new payload with optimistic block",
-			postState:     bellatrixState,
-			preState:      bellatrixState,
-			blk:           bellatrixBlk,
-			newPayloadErr: engine.ErrAcceptedSyncingPayloadStatus,
+			name:           "new payload with optimistic block",
+			postState:      bellatrixState,
+			preState:       bellatrixState,
+			blk:            bellatrixBlk,
+			newPayloadErr:  powchain.ErrAcceptedSyncingPayloadStatus,
+			isValidPayload: false,
 		},
 		{
-			name:          "new payload with invalid block",
-			postState:     bellatrixState,
-			preState:      bellatrixState,
-			blk:           bellatrixBlk,
-			newPayloadErr: engine.ErrInvalidPayloadStatus,
-			errString:     "could not validate execution payload from execution engine: payload status is INVALID",
+			name:           "new payload with invalid block",
+			postState:      bellatrixState,
+			preState:       bellatrixState,
+			blk:            bellatrixBlk,
+			newPayloadErr:  powchain.ErrInvalidPayloadStatus,
+			errString:      "could not validate an INVALID payload from execution engine",
+			isValidPayload: false,
 		},
 		{
-			name:      "altair pre state, altair block",
-			postState: bellatrixState,
-			preState:  altairState,
-			blk:       altairBlk,
+			name:           "altair pre state, altair block",
+			postState:      bellatrixState,
+			preState:       altairState,
+			blk:            altairBlk,
+			isValidPayload: true,
 		},
 		{
 			name:      "altair pre state, happy case",
@@ -265,13 +297,15 @@ func Test_NotifyNewPayload(t *testing.T) {
 				require.NoError(t, err)
 				return b
 			}(),
+			isValidPayload: true,
 		},
 		{
-			name:      "could not get merge block",
-			postState: bellatrixState,
-			preState:  bellatrixState,
-			blk:       bellatrixBlk,
-			errString: "could not get merge block parent hash and total difficulty",
+			name:           "could not get merge block",
+			postState:      bellatrixState,
+			preState:       bellatrixState,
+			blk:            bellatrixBlk,
+			errString:      "could not get merge block parent hash and total difficulty",
+			isValidPayload: false,
 		},
 		{
 			name:      "not at merge transition",
@@ -298,13 +332,15 @@ func Test_NotifyNewPayload(t *testing.T) {
 				require.NoError(t, err)
 				return b
 			}(),
+			isValidPayload: true,
 		},
 		{
-			name:      "could not get merge block",
-			postState: bellatrixState,
-			preState:  bellatrixState,
-			blk:       bellatrixBlk,
-			errString: "could not get merge block parent hash and total difficulty",
+			name:           "could not get merge block",
+			postState:      bellatrixState,
+			preState:       bellatrixState,
+			blk:            bellatrixBlk,
+			errString:      "could not get merge block parent hash and total difficulty",
+			isValidPayload: false,
 		},
 		{
 			name:      "happy case",
@@ -324,20 +360,21 @@ func Test_NotifyNewPayload(t *testing.T) {
 				require.NoError(t, err)
 				return b
 			}(),
+			isValidPayload: true,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			engine := &mockEngineService{newPayloadError: tt.newPayloadErr, blks: map[[32]byte]*v1.ExecutionBlock{}}
-			engine.blks[[32]byte{'a'}] = &v1.ExecutionBlock{
+			e := &mockPOW.EngineClient{ErrNewPayload: tt.newPayloadErr, BlockByHashMap: map[[32]byte]*v1.ExecutionBlock{}}
+			e.BlockByHashMap[[32]byte{'a'}] = &v1.ExecutionBlock{
 				ParentHash:      bytesutil.PadTo([]byte{'b'}, fieldparams.RootLength),
 				TotalDifficulty: "0x2",
 			}
-			engine.blks[[32]byte{'b'}] = &v1.ExecutionBlock{
+			e.BlockByHashMap[[32]byte{'b'}] = &v1.ExecutionBlock{
 				ParentHash:      bytesutil.PadTo([]byte{'3'}, fieldparams.RootLength),
 				TotalDifficulty: "0x1",
 			}
-			service.cfg.ExecutionEngineCaller = engine
+			service.cfg.ExecutionEngineCaller = e
 			var payload *ethpb.ExecutionPayloadHeader
 			if tt.preState.Version() == version.Bellatrix {
 				payload, err = tt.preState.LatestExecutionPayloadHeader()
@@ -347,11 +384,12 @@ func Test_NotifyNewPayload(t *testing.T) {
 			require.NoError(t, service.cfg.ForkChoiceStore.InsertOptimisticBlock(ctx, 0, root, root, params.BeaconConfig().ZeroHash, 0, 0))
 			postVersion, postHeader, err := getStateVersionAndPayload(tt.postState)
 			require.NoError(t, err)
-			err = service.notifyNewPayload(ctx, tt.preState.Version(), postVersion, payload, postHeader, tt.blk, root)
+			isValidPayload, err := service.notifyNewPayload(ctx, tt.preState.Version(), postVersion, payload, postHeader, tt.blk)
 			if tt.errString != "" {
 				require.ErrorContains(t, tt.errString, err)
 			} else {
 				require.NoError(t, err)
+				require.Equal(t, tt.isValidPayload, isValidPayload)
 			}
 		})
 	}
@@ -383,27 +421,23 @@ func Test_NotifyNewPayload_SetOptimisticToValid(t *testing.T) {
 	require.NoError(t, err)
 	service, err := NewService(ctx, opts...)
 	require.NoError(t, err)
-	engine := &mockEngineService{blks: map[[32]byte]*v1.ExecutionBlock{}}
-	engine.blks[[32]byte{'a'}] = &v1.ExecutionBlock{
+	e := &mockPOW.EngineClient{BlockByHashMap: map[[32]byte]*v1.ExecutionBlock{}}
+	e.BlockByHashMap[[32]byte{'a'}] = &v1.ExecutionBlock{
 		ParentHash:      bytesutil.PadTo([]byte{'b'}, fieldparams.RootLength),
 		TotalDifficulty: "0x2",
 	}
-	engine.blks[[32]byte{'b'}] = &v1.ExecutionBlock{
+	e.BlockByHashMap[[32]byte{'b'}] = &v1.ExecutionBlock{
 		ParentHash:      bytesutil.PadTo([]byte{'3'}, fieldparams.RootLength),
 		TotalDifficulty: "0x1",
 	}
-	service.cfg.ExecutionEngineCaller = engine
+	service.cfg.ExecutionEngineCaller = e
 	payload, err := bellatrixState.LatestExecutionPayloadHeader()
 	require.NoError(t, err)
-	root := [32]byte{'c'}
-	require.NoError(t, service.cfg.ForkChoiceStore.InsertOptimisticBlock(ctx, 1, root, [32]byte{'a'}, params.BeaconConfig().ZeroHash, 0, 0))
 	postVersion, postHeader, err := getStateVersionAndPayload(bellatrixState)
 	require.NoError(t, err)
-	err = service.notifyNewPayload(ctx, bellatrixState.Version(), postVersion, payload, postHeader, bellatrixBlk, root)
+	validated, err := service.notifyNewPayload(ctx, bellatrixState.Version(), postVersion, payload, postHeader, bellatrixBlk)
 	require.NoError(t, err)
-	optimistic, err := service.IsOptimisticForRoot(ctx, root)
-	require.NoError(t, err)
-	require.Equal(t, false, optimistic)
+	require.Equal(t, true, validated)
 }
 
 func Test_IsOptimisticCandidateBlock(t *testing.T) {
@@ -443,7 +477,7 @@ func Test_IsOptimisticCandidateBlock(t *testing.T) {
 				blk := util.NewBeaconBlockBellatrix()
 				blk.Block.Slot = 1
 				blk.Block.ParentRoot = parentRoot[:]
-				wr, err := wrapper.WrappedBellatrixBeaconBlock(blk.Block)
+				wr, err := wrapper.WrappedBeaconBlock(blk.Block)
 				require.NoError(tt, err)
 				return wr
 			}(t),
@@ -463,7 +497,7 @@ func Test_IsOptimisticCandidateBlock(t *testing.T) {
 				blk := util.NewBeaconBlockAltair()
 				blk.Block.Slot = 200
 				blk.Block.ParentRoot = parentRoot[:]
-				wr, err := wrapper.WrappedAltairBeaconBlock(blk.Block)
+				wr, err := wrapper.WrappedBeaconBlock(blk.Block)
 				require.NoError(tt, err)
 				return wr
 			}(t),
@@ -483,7 +517,7 @@ func Test_IsOptimisticCandidateBlock(t *testing.T) {
 				blk := util.NewBeaconBlockBellatrix()
 				blk.Block.Slot = 200
 				blk.Block.ParentRoot = parentRoot[:]
-				wr, err := wrapper.WrappedBellatrixBeaconBlock(blk.Block)
+				wr, err := wrapper.WrappedBeaconBlock(blk.Block)
 				require.NoError(tt, err)
 				return wr
 			}(t),
@@ -497,42 +531,14 @@ func Test_IsOptimisticCandidateBlock(t *testing.T) {
 			}(t),
 			want: false,
 		},
-		{
-			name: "shallow block, execution enabled justified chkpt",
-			blk: func(tt *testing.T) block.BeaconBlock {
-				blk := util.NewBeaconBlockBellatrix()
-				blk.Block.Slot = 200
-				blk.Block.ParentRoot = parentRoot[:]
-				wr, err := wrapper.WrappedBellatrixBeaconBlock(blk.Block)
-				require.NoError(tt, err)
-				return wr
-			}(t),
-			justified: func(tt *testing.T) block.SignedBeaconBlock {
-				blk := util.NewBeaconBlockBellatrix()
-				blk.Block.Slot = 32
-				blk.Block.ParentRoot = parentRoot[:]
-				blk.Block.Body.ExecutionPayload.ParentHash = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
-				blk.Block.Body.ExecutionPayload.FeeRecipient = bytesutil.PadTo([]byte{'a'}, fieldparams.FeeRecipientLength)
-				blk.Block.Body.ExecutionPayload.StateRoot = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
-				blk.Block.Body.ExecutionPayload.ReceiptsRoot = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
-				blk.Block.Body.ExecutionPayload.LogsBloom = bytesutil.PadTo([]byte{'a'}, fieldparams.LogsBloomLength)
-				blk.Block.Body.ExecutionPayload.PrevRandao = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
-				blk.Block.Body.ExecutionPayload.BaseFeePerGas = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
-				blk.Block.Body.ExecutionPayload.BlockHash = bytesutil.PadTo([]byte{'a'}, fieldparams.RootLength)
-				wr, err := wrapper.WrappedSignedBeaconBlock(blk)
-				require.NoError(tt, err)
-				return wr
-			}(t),
-			want: true,
-		},
 	}
 	for _, tt := range tests {
-		jroot, err := tt.justified.Block().HashTreeRoot()
+		jRoot, err := tt.justified.Block().HashTreeRoot()
 		require.NoError(t, err)
 		require.NoError(t, service.cfg.BeaconDB.SaveBlock(ctx, tt.justified))
 		service.store.SetJustifiedCheckpt(
 			&ethpb.Checkpoint{
-				Root:  jroot[:],
+				Root:  jRoot[:],
 				Epoch: slots.ToEpoch(tt.justified.Block().Slot()),
 			})
 		require.NoError(t, service.cfg.BeaconDB.SaveBlock(ctx, wrappedParentBlock))
@@ -571,8 +577,8 @@ func Test_IsOptimisticShallowExecutionParent(t *testing.T) {
 		BlockNumber:   100,
 	}
 	body := &ethpb.BeaconBlockBodyBellatrix{ExecutionPayload: payload}
-	block := &ethpb.BeaconBlockBellatrix{Body: body, Slot: 200}
-	rawSigned := &ethpb.SignedBeaconBlockBellatrix{Block: block}
+	b := &ethpb.BeaconBlockBellatrix{Body: body, Slot: 200}
+	rawSigned := &ethpb.SignedBeaconBlockBellatrix{Block: b}
 	blk := util.HydrateSignedBeaconBlockBellatrix(rawSigned)
 	wr, err := wrapper.WrappedSignedBeaconBlock(blk)
 	require.NoError(t, err)
@@ -592,6 +598,47 @@ func Test_IsOptimisticShallowExecutionParent(t *testing.T) {
 	require.Equal(t, true, candidate)
 }
 
+func Test_GetPayloadAttribute(t *testing.T) {
+	ctx := context.Background()
+	beaconDB := testDB.SetupDB(t)
+	opts := []Option{
+		WithDatabase(beaconDB),
+		WithStateGen(stategen.New(beaconDB)),
+		WithProposerIdsCache(cache.NewProposerPayloadIDsCache()),
+	}
+
+	// Cache miss
+	service, err := NewService(ctx, opts...)
+	require.NoError(t, err)
+	hasPayload, _, vId, err := service.getPayloadAttribute(ctx, nil, 0)
+	require.NoError(t, err)
+	require.Equal(t, false, hasPayload)
+	require.Equal(t, types.ValidatorIndex(0), vId)
+
+	// Cache hit, advance state, no fee recipient
+	suggestedVid := types.ValidatorIndex(1)
+	slot := types.Slot(1)
+	service.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(slot, suggestedVid, [8]byte{})
+	st, _ := util.DeterministicGenesisState(t, 1)
+	hook := logTest.NewGlobal()
+	hasPayload, attr, vId, err := service.getPayloadAttribute(ctx, st, slot)
+	require.NoError(t, err)
+	require.Equal(t, true, hasPayload)
+	require.Equal(t, suggestedVid, vId)
+	require.Equal(t, fieldparams.EthBurnAddressHex, common.BytesToAddress(attr.SuggestedFeeRecipient).String())
+	require.LogsContain(t, hook, "Fee recipient not set. Using burn address")
+
+	// Cache hit, advance state, has fee recipient
+	suggestedAddr := common.HexToAddress("123")
+	require.NoError(t, service.cfg.BeaconDB.SaveFeeRecipientsByValidatorIDs(ctx, []types.ValidatorIndex{suggestedVid}, []common.Address{suggestedAddr}))
+	service.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(slot, suggestedVid, [8]byte{})
+	hasPayload, attr, vId, err = service.getPayloadAttribute(ctx, st, slot)
+	require.NoError(t, err)
+	require.Equal(t, true, hasPayload)
+	require.Equal(t, suggestedVid, vId)
+	require.Equal(t, suggestedAddr, common.BytesToAddress(attr.SuggestedFeeRecipient))
+}
+
 func Test_UpdateLastValidatedCheckpoint(t *testing.T) {
 	params.SetupTestConfigCleanup(t)
 	params.OverrideBeaconConfig(params.MainnetConfig())
@@ -607,7 +654,6 @@ func Test_UpdateLastValidatedCheckpoint(t *testing.T) {
 	}
 	service, err := NewService(ctx, opts...)
 	require.NoError(t, err)
-
 	genesisStateRoot := [32]byte{}
 	genesisBlk := blocks.NewGenesisBlock(genesisStateRoot[:])
 	wr, err := wrapper.WrappedSignedBeaconBlock(genesisBlk)
@@ -690,3 +736,130 @@ func Test_UpdateLastValidatedCheckpoint(t *testing.T) {
 	require.DeepEqual(t, validCheckpoint.Root, cp.Root)
 	require.Equal(t, validCheckpoint.Epoch, cp.Epoch)
 }
+
+func TestService_removeInvalidBlockAndState(t *testing.T) {
+	ctx := context.Background()
+	beaconDB := testDB.SetupDB(t)
+	opts := []Option{
+		WithDatabase(beaconDB),
+		WithStateGen(stategen.New(beaconDB)),
+		WithForkChoiceStore(protoarray.New(0, 0, [32]byte{})),
+	}
+	service, err := NewService(ctx, opts...)
+	require.NoError(t, err)
+
+	// Deleting unknown block should not error.
+	require.NoError(t, service.removeInvalidBlockAndState(ctx, [][32]byte{{'a'}, {'b'}, {'c'}}))
+
+	// Happy case
+	b1 := util.NewBeaconBlock()
+	b1.Block.Slot = 1
+	blk1, err := wrapper.WrappedSignedBeaconBlock(b1)
+	require.NoError(t, err)
+	r1, err := blk1.Block().HashTreeRoot()
+	require.NoError(t, err)
+	st, _ := util.DeterministicGenesisStateBellatrix(t, 1)
+	require.NoError(t, service.cfg.BeaconDB.SaveBlock(ctx, blk1))
+	require.NoError(t, service.cfg.BeaconDB.SaveStateSummary(ctx, &ethpb.StateSummary{
+		Slot: 1,
+		Root: r1[:],
+	}))
+	require.NoError(t, service.cfg.BeaconDB.SaveState(ctx, st, r1))
+
+	b2 := util.NewBeaconBlock()
+	b2.Block.Slot = 2
+	blk2, err := wrapper.WrappedSignedBeaconBlock(b2)
+	require.NoError(t, err)
+	r2, err := blk2.Block().HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, service.cfg.BeaconDB.SaveBlock(ctx, blk2))
+	require.NoError(t, service.cfg.BeaconDB.SaveStateSummary(ctx, &ethpb.StateSummary{
+		Slot: 2,
+		Root: r2[:],
+	}))
+	require.NoError(t, service.cfg.BeaconDB.SaveState(ctx, st, r2))
+
+	require.NoError(t, service.removeInvalidBlockAndState(ctx, [][32]byte{r1, r2}))
+
+	require.Equal(t, false, service.hasBlock(ctx, r1))
+	require.Equal(t, false, service.hasBlock(ctx, r2))
+	require.Equal(t, false, service.cfg.BeaconDB.HasStateSummary(ctx, r1))
+	require.Equal(t, false, service.cfg.BeaconDB.HasStateSummary(ctx, r2))
+	has, err := service.cfg.StateGen.HasState(ctx, r1)
+	require.NoError(t, err)
+	require.Equal(t, false, has)
+	has, err = service.cfg.StateGen.HasState(ctx, r2)
+	require.NoError(t, err)
+	require.Equal(t, false, has)
+}
+
+func TestService_getFinalizedPayloadHash(t *testing.T) {
+	ctx := context.Background()
+	beaconDB := testDB.SetupDB(t)
+	opts := []Option{
+		WithDatabase(beaconDB),
+		WithStateGen(stategen.New(beaconDB)),
+	}
+	service, err := NewService(ctx, opts...)
+	require.NoError(t, err)
+
+	// Use the block in DB
+	b := util.NewBeaconBlockBellatrix()
+	b.Block.Body.ExecutionPayload.BlockHash = bytesutil.PadTo([]byte("hi"), 32)
+	blk, err := wrapper.WrappedSignedBeaconBlock(b)
+	require.NoError(t, err)
+	r, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, service.cfg.BeaconDB.SaveBlock(ctx, blk))
+	h, err := service.getFinalizedPayloadHash(ctx, r)
+	require.NoError(t, err)
+	require.Equal(t, bytesutil.ToBytes32(b.Block.Body.ExecutionPayload.BlockHash), h)
+
+	// Use the block in init sync cache
+	b = util.NewBeaconBlockBellatrix()
+	b.Block.Body.ExecutionPayload.BlockHash = bytesutil.PadTo([]byte("hello"), 32)
+	blk, err = wrapper.WrappedSignedBeaconBlock(b)
+	require.NoError(t, err)
+	r, err = b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	service.initSyncBlocks[r] = blk
+	h, err = service.getFinalizedPayloadHash(ctx, r)
+	require.NoError(t, err)
+	require.Equal(t, bytesutil.ToBytes32(b.Block.Body.ExecutionPayload.BlockHash), h)
+
+	// Use the weak subjectivity sync block
+	b = util.NewBeaconBlockBellatrix()
+	b.Block.Body.ExecutionPayload.BlockHash = bytesutil.PadTo([]byte("howdy"), 32)
+	blk, err = wrapper.WrappedSignedBeaconBlock(b)
+	require.NoError(t, err)
+	r, err = b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, service.cfg.BeaconDB.SaveBlock(ctx, blk))
+	require.NoError(t, service.cfg.BeaconDB.SaveOriginCheckpointBlockRoot(ctx, r))
+	h, err = service.getFinalizedPayloadHash(ctx, r)
+	require.NoError(t, err)
+	require.Equal(t, bytesutil.ToBytes32(b.Block.Body.ExecutionPayload.BlockHash), h)
+
+	// None of the above should error
+	require.NoError(t, service.cfg.BeaconDB.SaveOriginCheckpointBlockRoot(ctx, [32]byte{'a'}))
+	_, err = service.getFinalizedPayloadHash(ctx, [32]byte{'a'})
+	require.ErrorContains(t, "does not exist in the db or our cache", err)
+}
+
+func TestService_getPayloadHash(t *testing.T) {
+	// Pre-bellatrix
+	blk, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlock())
+	require.NoError(t, err)
+	h, err := getPayloadHash(blk.Block())
+	require.NoError(t, err)
+	require.Equal(t, [32]byte{}, h)
+
+	// Post bellatrix
+	b := util.NewBeaconBlockBellatrix()
+	b.Block.Body.ExecutionPayload.BlockHash = bytesutil.PadTo([]byte("hi"), 32)
+	blk, err = wrapper.WrappedSignedBeaconBlock(b)
+	require.NoError(t, err)
+	h, err = getPayloadHash(blk.Block())
+	require.NoError(t, err)
+	require.Equal(t, bytesutil.ToBytes32(bytesutil.PadTo([]byte("hi"), 32)), h)
+}

beacon-chain/blockchain/options.go

@@ -2,6 +2,7 @@ package blockchain
 
 import (
 	"github.com/prysmaticlabs/prysm/async/event"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
@@ -11,7 +12,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
-	enginev1 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
@@ -52,7 +52,7 @@ func WithChainStartFetcher(f powchain.ChainStartFetcher) Option {
 }
 
 // WithExecutionEngineCaller to call execution engine.
-func WithExecutionEngineCaller(c enginev1.Caller) Option {
+func WithExecutionEngineCaller(c powchain.EngineCaller) Option {
 	return func(s *Service) error {
 		s.cfg.ExecutionEngineCaller = c
 		return nil
@@ -67,6 +67,14 @@ func WithDepositCache(c *depositcache.DepositCache) Option {
 	}
 }
 
+// WithProposerIdsCache for proposer id cache.
+func WithProposerIdsCache(c *cache.ProposerPayloadIDsCache) Option {
+	return func(s *Service) error {
+		s.cfg.ProposerSlotIndexCache = c
+		return nil
+	}
+}
+
 // WithAttestationPool for attestation lifecycle after chain inclusion.
 func WithAttestationPool(p attestations.Pool) Option {
 	return func(s *Service) error {

beacon-chain/blockchain/pow_block_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/holiman/uint256"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	mocks "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
@@ -117,13 +118,13 @@ func Test_validateMergeBlock(t *testing.T) {
 	service, err := NewService(ctx, opts...)
 	require.NoError(t, err)
 
-	engine := &mockEngineService{blks: map[[32]byte]*enginev1.ExecutionBlock{}}
+	engine := &mocks.EngineClient{BlockByHashMap: map[[32]byte]*enginev1.ExecutionBlock{}}
 	service.cfg.ExecutionEngineCaller = engine
-	engine.blks[[32]byte{'a'}] = &enginev1.ExecutionBlock{
+	engine.BlockByHashMap[[32]byte{'a'}] = &enginev1.ExecutionBlock{
 		ParentHash:      bytesutil.PadTo([]byte{'b'}, fieldparams.RootLength),
 		TotalDifficulty: "0x2",
 	}
-	engine.blks[[32]byte{'b'}] = &enginev1.ExecutionBlock{
+	engine.BlockByHashMap[[32]byte{'b'}] = &enginev1.ExecutionBlock{
 		ParentHash:      bytesutil.PadTo([]byte{'3'}, fieldparams.RootLength),
 		TotalDifficulty: "0x1",
 	}
@@ -158,12 +159,12 @@ func Test_getBlkParentHashAndTD(t *testing.T) {
 	service, err := NewService(ctx, opts...)
 	require.NoError(t, err)
 
-	engine := &mockEngineService{blks: map[[32]byte]*enginev1.ExecutionBlock{}}
+	engine := &mocks.EngineClient{BlockByHashMap: map[[32]byte]*enginev1.ExecutionBlock{}}
 	service.cfg.ExecutionEngineCaller = engine
 	h := [32]byte{'a'}
 	p := [32]byte{'b'}
 	td := "0x1"
-	engine.blks[h] = &enginev1.ExecutionBlock{
+	engine.BlockByHashMap[h] = &enginev1.ExecutionBlock{
 		ParentHash:      p[:],
 		TotalDifficulty: td,
 	}
@@ -175,18 +176,18 @@ func Test_getBlkParentHashAndTD(t *testing.T) {
 	_, _, err = service.getBlkParentHashAndTD(ctx, []byte{'c'})
 	require.ErrorContains(t, "could not get pow block: block not found", err)
 
-	engine.blks[h] = nil
+	engine.BlockByHashMap[h] = nil
 	_, _, err = service.getBlkParentHashAndTD(ctx, h[:])
 	require.ErrorContains(t, "pow block is nil", err)
 
-	engine.blks[h] = &enginev1.ExecutionBlock{
+	engine.BlockByHashMap[h] = &enginev1.ExecutionBlock{
 		ParentHash:      p[:],
 		TotalDifficulty: "1",
 	}
 	_, _, err = service.getBlkParentHashAndTD(ctx, h[:])
 	require.ErrorContains(t, "could not decode merge block total difficulty: hex string without 0x prefix", err)
 
-	engine.blks[h] = &enginev1.ExecutionBlock{
+	engine.BlockByHashMap[h] = &enginev1.ExecutionBlock{
 		ParentHash:      p[:],
 		TotalDifficulty: "0XFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
 	}

beacon-chain/blockchain/process_block.go

@@ -109,16 +109,32 @@ func (s *Service) onBlock(ctx context.Context, signed block.SignedBeaconBlock, b
 	if err != nil {
 		return err
 	}
-	if err := s.savePostStateInfo(ctx, blockRoot, signed, postState, false /* reg sync */); err != nil {
-		return err
-	}
 	postStateVersion, postStateHeader, err := getStateVersionAndPayload(postState)
 	if err != nil {
 		return err
 	}
-	if err := s.notifyNewPayload(ctx, preStateVersion, postStateVersion, preStateHeader, postStateHeader, signed, blockRoot); err != nil {
+	isValidPayload, err := s.notifyNewPayload(ctx, preStateVersion, postStateVersion, preStateHeader, postStateHeader, signed)
+	if err != nil {
 		return errors.Wrap(err, "could not verify new payload")
 	}
+	if !isValidPayload {
+		candidate, err := s.optimisticCandidateBlock(ctx, b)
+		if err != nil {
+			return errors.Wrap(err, "could not check if block is optimistic candidate")
+		}
+		if !candidate {
+			return errNotOptimisticCandidate
+		}
+	}
+
+	if err := s.insertBlockAndAttestationsToForkChoiceStore(ctx, signed.Block(), blockRoot, postState); err != nil {
+		return errors.Wrapf(err, "could not insert block %d to fork choice store", signed.Block().Slot())
+	}
+	if isValidPayload {
+		if err := s.cfg.ForkChoiceStore.SetOptimisticToValid(ctx, blockRoot); err != nil {
+			return errors.Wrap(err, "could not set optimistic block to valid")
+		}
+	}
 
 	// We add a proposer score boost to fork choice for the block root if applicable, right after
 	// running a successful state transition for the block.
@@ -132,6 +148,9 @@ func (s *Service) onBlock(ctx context.Context, signed block.SignedBeaconBlock, b
 		return err
 	}
 
+	if err := s.savePostStateInfo(ctx, blockRoot, signed, postState, false /* reg sync */); err != nil {
+		return err
+	}
 	// If slasher is configured, forward the attestations in the block via
 	// an event feed for processing.
 	if features.Get().EnableSlasher {
@@ -188,12 +207,24 @@ func (s *Service) onBlock(ctx context.Context, signed block.SignedBeaconBlock, b
 		msg := fmt.Sprintf("could not read balances for state w/ justified checkpoint %#x", justified.Root)
 		return errors.Wrap(err, msg)
 	}
-	if err := s.updateHead(ctx, balances); err != nil {
+	headRoot, err := s.updateHead(ctx, balances)
+	if err != nil {
 		log.WithError(err).Warn("Could not update head")
 	}
-	if _, err := s.notifyForkchoiceUpdate(ctx, s.headBlock().Block(), s.headRoot(), bytesutil.ToBytes32(finalized.Root)); err != nil {
+	headBlock, err := s.cfg.BeaconDB.Block(ctx, headRoot)
+	if err != nil {
 		return err
 	}
+	headState, err := s.cfg.StateGen.StateByRoot(ctx, headRoot)
+	if err != nil {
+		return err
+	}
+	if _, err := s.notifyForkchoiceUpdate(ctx, headState, headBlock.Block(), headRoot, bytesutil.ToBytes32(finalized.Root)); err != nil {
+		return err
+	}
+	if err := s.saveHead(ctx, headRoot, headBlock, headState); err != nil {
+		return errors.Wrap(err, "could not save head")
+	}
 
 	if err := s.pruneCanonicalAttsFromPool(ctx, blockRoot, signed); err != nil {
 		return err
@@ -238,7 +269,7 @@ func (s *Service) onBlock(ctx context.Context, signed block.SignedBeaconBlock, b
 		if err := s.cfg.ForkChoiceStore.Prune(ctx, fRoot); err != nil {
 			return errors.Wrap(err, "could not prune proto array fork choice nodes")
 		}
-		isOptimistic, err := s.cfg.ForkChoiceStore.IsOptimistic(ctx, fRoot)
+		isOptimistic, err := s.cfg.ForkChoiceStore.IsOptimistic(fRoot)
 		if err != nil {
 			return errors.Wrap(err, "could not check if node is optimistically synced")
 		}
@@ -376,18 +407,35 @@ func (s *Service) onBlockBatch(ctx context.Context, blks []block.SignedBeaconBlo
 	// blocks have been verified, add them to forkchoice and call the engine
 	for i, b := range blks {
 		s.saveInitSyncBlock(blockRoots[i], b)
-		if err := s.insertBlockToForkChoiceStore(ctx, b.Block(), blockRoots[i], fCheckpoints[i], jCheckpoints[i]); err != nil {
-			return nil, nil, err
-		}
-		if err := s.notifyNewPayload(ctx,
+		isValidPayload, err := s.notifyNewPayload(ctx,
 			preVersionAndHeaders[i].version,
 			postVersionAndHeaders[i].version,
 			preVersionAndHeaders[i].header,
-			postVersionAndHeaders[i].header, b, blockRoots[i]); err != nil {
+			postVersionAndHeaders[i].header, b)
+
+		if err != nil {
 			return nil, nil, err
 		}
+		if !isValidPayload {
+			candidate, err := s.optimisticCandidateBlock(ctx, b.Block())
+			if err != nil {
+				return nil, nil, errors.Wrap(err, "could not check if block is optimistic candidate")
+			}
+			if !candidate {
+				return nil, nil, errNotOptimisticCandidate
+			}
+		}
 
-		if _, err := s.notifyForkchoiceUpdate(ctx, b.Block(), blockRoots[i], bytesutil.ToBytes32(fCheckpoints[i].Root)); err != nil {
+		if err := s.insertBlockToForkChoiceStore(ctx, b.Block(), blockRoots[i], fCheckpoints[i], jCheckpoints[i]); err != nil {
+			return nil, nil, err
+		}
+		if isValidPayload {
+			if err := s.cfg.ForkChoiceStore.SetOptimisticToValid(ctx, blockRoots[i]); err != nil {
+				return nil, nil, errors.Wrap(err, "could not set optimistic block to valid")
+			}
+		}
+
+		if _, err := s.notifyForkchoiceUpdate(ctx, preState, b.Block(), blockRoots[i], bytesutil.ToBytes32(fCheckpoints[i].Root)); err != nil {
 			return nil, nil, err
 		}
 	}
@@ -564,9 +612,6 @@ func (s *Service) savePostStateInfo(ctx context.Context, r [32]byte, b block.Sig
 	if err := s.cfg.StateGen.SaveState(ctx, r, st); err != nil {
 		return errors.Wrap(err, "could not save state")
 	}
-	if err := s.insertBlockAndAttestationsToForkChoiceStore(ctx, b.Block(), r, st); err != nil {
-		return errors.Wrapf(err, "could not insert block %d to fork choice store", b.Block().Slot())
-	}
 	return nil
 }
 

beacon-chain/blockchain/process_block_helpers.go

@@ -11,6 +11,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/config/params"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	mathutil "github.com/prysmaticlabs/prysm/math"
 	"github.com/prysmaticlabs/prysm/monitoring/tracing"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
@@ -247,7 +248,7 @@ func (s *Service) updateFinalized(ctx context.Context, cp *ethpb.Checkpoint) err
 	}
 
 	fRoot := bytesutil.ToBytes32(cp.Root)
-	optimistic, err := s.cfg.ForkChoiceStore.IsOptimistic(ctx, fRoot)
+	optimistic, err := s.cfg.ForkChoiceStore.IsOptimistic(fRoot)
 	if err != nil {
 		return err
 	}
@@ -402,10 +403,13 @@ func (s *Service) insertFinalizedDeposits(ctx context.Context, fRoot [32]byte) e
 	// We update the cache up to the last deposit index in the finalized block's state.
 	// We can be confident that these deposits will be included in some block
 	// because the Eth1 follow distance makes such long-range reorgs extremely unlikely.
-	eth1DepositIndex := int64(finalizedState.Eth1Data().DepositCount - 1)
-	s.cfg.DepositCache.InsertFinalizedDeposits(ctx, eth1DepositIndex)
+	eth1DepositIndex, err := mathutil.Int(finalizedState.Eth1DepositIndex())
+	if err != nil {
+		return errors.Wrap(err, "could not cast eth1 deposit index")
+	}
+	s.cfg.DepositCache.InsertFinalizedDeposits(ctx, int64(eth1DepositIndex))
 	// Deposit proofs are only used during state transition and can be safely removed to save space.
-	if err = s.cfg.DepositCache.PruneProofs(ctx, eth1DepositIndex); err != nil {
+	if err = s.cfg.DepositCache.PruneProofs(ctx, int64(eth1DepositIndex)); err != nil {
 		return errors.Wrap(err, "could not prune deposit proofs")
 	}
 	return nil

beacon-chain/blockchain/process_block_test.go

@@ -1516,6 +1516,7 @@ func TestInsertFinalizedDeposits(t *testing.T) {
 	service.store.SetFinalizedCheckpt(&ethpb.Checkpoint{Root: gRoot[:]})
 	gs = gs.Copy()
 	assert.NoError(t, gs.SetEth1Data(&ethpb.Eth1Data{DepositCount: 10}))
+	assert.NoError(t, gs.SetEth1DepositIndex(7))
 	assert.NoError(t, service.cfg.StateGen.SaveState(ctx, [32]byte{'m', 'o', 'c', 'k'}, gs))
 	zeroSig := [96]byte{}
 	for i := uint64(0); i < uint64(4*params.BeaconConfig().SlotsPerEpoch); i++ {
@@ -1529,8 +1530,64 @@ func TestInsertFinalizedDeposits(t *testing.T) {
 	}
 	assert.NoError(t, service.insertFinalizedDeposits(ctx, [32]byte{'m', 'o', 'c', 'k'}))
 	fDeposits := depositCache.FinalizedDeposits(ctx)
-	assert.Equal(t, 9, int(fDeposits.MerkleTrieIndex), "Finalized deposits not inserted correctly")
-	deps := depositCache.AllDeposits(ctx, big.NewInt(109))
+	assert.Equal(t, 7, int(fDeposits.MerkleTrieIndex), "Finalized deposits not inserted correctly")
+	deps := depositCache.AllDeposits(ctx, big.NewInt(107))
+	for _, d := range deps {
+		assert.DeepEqual(t, [][]byte(nil), d.Proof, "Proofs are not empty")
+	}
+}
+
+func TestInsertFinalizedDeposits_MultipleFinalizedRoutines(t *testing.T) {
+	ctx := context.Background()
+	opts := testServiceOptsWithDB(t)
+	depositCache, err := depositcache.New()
+	require.NoError(t, err)
+	opts = append(opts, WithDepositCache(depositCache))
+	service, err := NewService(ctx, opts...)
+	require.NoError(t, err)
+
+	gs, _ := util.DeterministicGenesisState(t, 32)
+	require.NoError(t, service.saveGenesisData(ctx, gs))
+	gBlk, err := service.cfg.BeaconDB.GenesisBlock(ctx)
+	require.NoError(t, err)
+	gRoot, err := gBlk.Block().HashTreeRoot()
+	require.NoError(t, err)
+	service.store.SetFinalizedCheckpt(&ethpb.Checkpoint{Root: gRoot[:]})
+	gs = gs.Copy()
+	assert.NoError(t, gs.SetEth1Data(&ethpb.Eth1Data{DepositCount: 7}))
+	assert.NoError(t, gs.SetEth1DepositIndex(5))
+	assert.NoError(t, service.cfg.StateGen.SaveState(ctx, [32]byte{'m', 'o', 'c', 'k'}, gs))
+	gs2 := gs.Copy()
+	assert.NoError(t, gs2.SetEth1Data(&ethpb.Eth1Data{DepositCount: 15}))
+	assert.NoError(t, gs2.SetEth1DepositIndex(12))
+	assert.NoError(t, service.cfg.StateGen.SaveState(ctx, [32]byte{'m', 'o', 'c', 'k', '2'}, gs2))
+	zeroSig := [96]byte{}
+	for i := uint64(0); i < uint64(4*params.BeaconConfig().SlotsPerEpoch); i++ {
+		root := []byte(strconv.Itoa(int(i)))
+		assert.NoError(t, depositCache.InsertDeposit(ctx, &ethpb.Deposit{Data: &ethpb.Deposit_Data{
+			PublicKey:             bytesutil.FromBytes48([fieldparams.BLSPubkeyLength]byte{}),
+			WithdrawalCredentials: params.BeaconConfig().ZeroHash[:],
+			Amount:                0,
+			Signature:             zeroSig[:],
+		}, Proof: [][]byte{root}}, 100+i, int64(i), bytesutil.ToBytes32(root)))
+	}
+	// Insert 3 deposits before hand.
+	depositCache.InsertFinalizedDeposits(ctx, 2)
+
+	assert.NoError(t, service.insertFinalizedDeposits(ctx, [32]byte{'m', 'o', 'c', 'k'}))
+	fDeposits := depositCache.FinalizedDeposits(ctx)
+	assert.Equal(t, 5, int(fDeposits.MerkleTrieIndex), "Finalized deposits not inserted correctly")
+
+	deps := depositCache.AllDeposits(ctx, big.NewInt(105))
+	for _, d := range deps {
+		assert.DeepEqual(t, [][]byte(nil), d.Proof, "Proofs are not empty")
+	}
+
+	// Insert New Finalized State with higher deposit count.
+	assert.NoError(t, service.insertFinalizedDeposits(ctx, [32]byte{'m', 'o', 'c', 'k', '2'}))
+	fDeposits = depositCache.FinalizedDeposits(ctx)
+	assert.Equal(t, 12, int(fDeposits.MerkleTrieIndex), "Finalized deposits not inserted correctly")
+	deps = depositCache.AllDeposits(ctx, big.NewInt(112))
 	for _, d := range deps {
 		assert.DeepEqual(t, [][]byte(nil), d.Proof, "Proofs are not empty")
 	}
@@ -1628,9 +1685,9 @@ func Test_getStateVersionAndPayload(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			version, header, err := getStateVersionAndPayload(tt.st)
+			ver, header, err := getStateVersionAndPayload(tt.st)
 			require.NoError(t, err)
-			require.Equal(t, tt.version, version)
+			require.Equal(t, tt.version, ver)
 			require.DeepEqual(t, tt.header, header)
 		})
 	}

beacon-chain/blockchain/receive_attestation.go

@@ -161,19 +161,19 @@ func (s *Service) spawnProcessAttestationsRoutine(stateFeed *event.Feed) {
 					log.WithError(err).Errorf("Unable to get justified balances for root %v", justified.Root)
 					continue
 				}
-				prevHead := s.headRoot()
-				if err := s.updateHead(s.ctx, balances); err != nil {
+				newHeadRoot, err := s.updateHead(s.ctx, balances)
+				if err != nil {
 					log.WithError(err).Warn("Resolving fork due to new attestation")
 				}
-				s.notifyEngineIfChangedHead(prevHead)
+				s.notifyEngineIfChangedHead(s.ctx, newHeadRoot)
 			}
 		}
 	}()
 }
 
 // This calls notify Forkchoice Update in the event that the head has changed
-func (s *Service) notifyEngineIfChangedHead(prevHead [32]byte) {
-	if s.headRoot() == prevHead {
+func (s *Service) notifyEngineIfChangedHead(ctx context.Context, newHeadRoot [32]byte) {
+	if s.headRoot() == newHeadRoot {
 		return
 	}
 	finalized := s.store.FinalizedCheckpt()
@@ -181,14 +181,28 @@ func (s *Service) notifyEngineIfChangedHead(prevHead [32]byte) {
 		log.WithError(errNilFinalizedInStore).Error("could not get finalized checkpoint")
 		return
 	}
-	_, err := s.notifyForkchoiceUpdate(s.ctx,
-		s.headBlock().Block(),
-		s.headRoot(),
+	newHeadBlock, err := s.cfg.BeaconDB.Block(ctx, newHeadRoot)
+	if err != nil {
+		log.WithError(err).Error("Could not get block from db")
+		return
+	}
+	headState, err := s.cfg.StateGen.StateByRoot(ctx, newHeadRoot)
+	if err != nil {
+		log.WithError(err).Error("Could not get state from db")
+		return
+	}
+	_, err = s.notifyForkchoiceUpdate(s.ctx,
+		headState,
+		newHeadBlock.Block(),
+		newHeadRoot,
 		bytesutil.ToBytes32(finalized.Root),
 	)
 	if err != nil {
 		log.WithError(err).Error("could not notify forkchoice update")
 	}
+	if err := s.saveHead(ctx, newHeadRoot, newHeadBlock, headState); err != nil {
+		log.WithError(err).Error("could not save head")
+	}
 }
 
 // This processes fork choice attestations from the pool to account for validator votes and fork choice.

beacon-chain/blockchain/receive_attestation_test.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/transition"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
@@ -133,31 +134,43 @@ func TestNotifyEngineIfChangedHead(t *testing.T) {
 
 	service, err := NewService(ctx, opts...)
 	require.NoError(t, err)
-
-	service.notifyEngineIfChangedHead(service.headRoot())
+	service.cfg.ProposerSlotIndexCache = cache.NewProposerPayloadIDsCache()
+	service.notifyEngineIfChangedHead(ctx, service.headRoot())
 	hookErr := "could not notify forkchoice update"
 	finalizedErr := "could not get finalized checkpoint"
 	require.LogsDoNotContain(t, hook, finalizedErr)
 	require.LogsDoNotContain(t, hook, hookErr)
-	service.notifyEngineIfChangedHead([32]byte{'a'})
+	service.notifyEngineIfChangedHead(ctx, [32]byte{'a'})
 	require.LogsContain(t, hook, finalizedErr)
 
 	hook.Reset()
+	service.head = &head{
+		root:  [32]byte{'a'},
+		block: nil, /* should not panic if notify head uses correct head */
+	}
+
 	b := util.NewBeaconBlock()
-	b.Block.Slot = 1
+	b.Block.Slot = 2
 	wsb, err := wrapper.WrappedSignedBeaconBlock(b)
 	require.NoError(t, err)
 	require.NoError(t, service.cfg.BeaconDB.SaveBlock(ctx, wsb))
-	r, err := b.Block.HashTreeRoot()
+	r1, err := b.Block.HashTreeRoot()
 	require.NoError(t, err)
-	finalized := &ethpb.Checkpoint{Root: r[:], Epoch: 0}
+	finalized := &ethpb.Checkpoint{Root: r1[:], Epoch: 0}
+	st, _ := util.DeterministicGenesisState(t, 1)
 	service.head = &head{
 		slot:  1,
-		root:  r,
+		root:  r1,
 		block: wsb,
+		state: st,
 	}
+	service.cfg.ProposerSlotIndexCache.SetProposerAndPayloadIDs(2, 1, [8]byte{1})
 	service.store.SetFinalizedCheckpt(finalized)
-	service.notifyEngineIfChangedHead([32]byte{'b'})
+	service.notifyEngineIfChangedHead(ctx, r1)
 	require.LogsDoNotContain(t, hook, finalizedErr)
 	require.LogsDoNotContain(t, hook, hookErr)
+	vId, payloadID, has := service.cfg.ProposerSlotIndexCache.GetProposerPayloadIDs(2)
+	require.Equal(t, true, has)
+	require.Equal(t, types.ValidatorIndex(1), vId)
+	require.Equal(t, [8]byte{1}, payloadID)
 }

beacon-chain/blockchain/receive_block.go

@@ -88,7 +88,6 @@ func (s *Service) ReceiveBlockBatch(ctx context.Context, blocks []block.SignedBe
 
 	for i, b := range blocks {
 		blockCopy := b.Copy()
-		// TODO(10261) check optimistic status
 		if err = s.handleBlockAfterBatchVerify(ctx, blockCopy, blkRoots[i], fCheckpoints[i], jCheckpoints[i]); err != nil {
 			tracing.AnnotateError(span, err)
 			return err

beacon-chain/blockchain/service.go

@@ -29,7 +29,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
-	enginev1 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/flags"
@@ -75,6 +74,7 @@ type config struct {
 	ChainStartFetcher       powchain.ChainStartFetcher
 	BeaconDB                db.HeadAccessDatabase
 	DepositCache            *depositcache.DepositCache
+	ProposerSlotIndexCache  *cache.ProposerPayloadIDsCache
 	AttPool                 attestations.Pool
 	ExitPool                voluntaryexits.PoolManager
 	SlashingPool            slashings.PoolManager
@@ -88,7 +88,7 @@ type config struct {
 	WeakSubjectivityCheckpt *ethpb.Checkpoint
 	BlockFetcher            powchain.POWBlockFetcher
 	FinalizedStateAtStartUp state.BeaconState
-	ExecutionEngineCaller   enginev1.Caller
+	ExecutionEngineCaller   powchain.EngineCaller
 }
 
 // NewService instantiates a new block service instance that will
@@ -191,14 +191,14 @@ func (s *Service) StartFromSavedState(saved state.BeaconState) error {
 	}
 	s.store = store.New(justified, finalized)
 
-	var store f.ForkChoicer
+	var f f.ForkChoicer
 	fRoot := bytesutil.ToBytes32(finalized.Root)
 	if features.Get().EnableForkChoiceDoublyLinkedTree {
-		store = doublylinkedtree.New(justified.Epoch, finalized.Epoch)
+		f = doublylinkedtree.New(justified.Epoch, finalized.Epoch)
 	} else {
-		store = protoarray.New(justified.Epoch, finalized.Epoch, fRoot)
+		f = protoarray.New(justified.Epoch, finalized.Epoch, fRoot)
 	}
-	s.cfg.ForkChoiceStore = store
+	s.cfg.ForkChoiceStore = f
 	fb, err := s.cfg.BeaconDB.Block(s.ctx, s.ensureRootNotZeros(fRoot))
 	if err != nil {
 		return errors.Wrap(err, "could not get finalized checkpoint block")
@@ -211,7 +211,7 @@ func (s *Service) StartFromSavedState(saved state.BeaconState) error {
 		return errors.Wrap(err, "could not get execution payload hash")
 	}
 	fSlot := fb.Block().Slot()
-	if err := store.InsertOptimisticBlock(s.ctx, fSlot, fRoot, params.BeaconConfig().ZeroHash,
+	if err := f.InsertOptimisticBlock(s.ctx, fSlot, fRoot, params.BeaconConfig().ZeroHash,
 		payloadHash, justified.Epoch, finalized.Epoch); err != nil {
 		return errors.Wrap(err, "could not insert finalized block to forkchoice")
 	}
@@ -221,7 +221,7 @@ func (s *Service) StartFromSavedState(saved state.BeaconState) error {
 		return errors.Wrap(err, "could not get last validated checkpoint")
 	}
 	if bytes.Equal(finalized.Root, lastValidatedCheckpoint.Root) {
-		if err := store.SetOptimisticToValid(s.ctx, fRoot); err != nil {
+		if err := f.SetOptimisticToValid(s.ctx, fRoot); err != nil {
 			return errors.Wrap(err, "could not set finalized block as validated")
 		}
 	}
@@ -362,9 +362,9 @@ func (s *Service) startFromPOWChain() error {
 		defer stateSub.Unsubscribe()
 		for {
 			select {
-			case event := <-stateChannel:
-				if event.Type == statefeed.ChainStarted {
-					data, ok := event.Data.(*statefeed.ChainStartedData)
+			case e := <-stateChannel:
+				if e.Type == statefeed.ChainStarted {
+					data, ok := e.Data.(*statefeed.ChainStartedData)
 					if !ok {
 						log.Error("event data is not type *statefeed.ChainStartedData")
 						return

beacon-chain/blockchain/service_norace_test.go

@@ -6,7 +6,9 @@ import (
 	"testing"
 
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
 	"github.com/prysmaticlabs/prysm/testing/require"
+	"github.com/prysmaticlabs/prysm/testing/util"
 	"github.com/sirupsen/logrus"
 )
 
@@ -20,8 +22,11 @@ func TestChainService_SaveHead_DataRace(t *testing.T) {
 	s := &Service{
 		cfg: &config{BeaconDB: beaconDB},
 	}
+	b, err := wrapper.WrappedSignedBeaconBlock(util.NewBeaconBlock())
+	st, _ := util.DeterministicGenesisState(t, 1)
+	require.NoError(t, err)
 	go func() {
-		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}, b, st))
 	}()
-	require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
+	require.NoError(t, s.saveHead(context.Background(), [32]byte{}, b, st))
 }

beacon-chain/blockchain/service_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
+	mockPOW "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	v1 "github.com/prysmaticlabs/prysm/beacon-chain/state/v1"
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/flags"
@@ -74,10 +75,14 @@ func (mb *mockBroadcaster) BroadcastSyncCommitteeMessage(_ context.Context, _ ui
 var _ p2p.Broadcaster = (*mockBroadcaster)(nil)
 
 func setupBeaconChain(t *testing.T, beaconDB db.Database) *Service {
-	endpoint := "http://127.0.0.1"
 	ctx := context.Background()
 	var web3Service *powchain.Service
 	var err error
+	srv, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		srv.Stop()
+	})
 	bState, _ := util.DeterministicGenesisState(t, 10)
 	pbState, err := v1.ProtobufBeaconState(bState.InnerStateUnsafe())
 	require.NoError(t, err)
@@ -496,12 +501,12 @@ func TestHasBlock_ForkChoiceAndDB_ProtoArray(t *testing.T) {
 		store: &store.Store{},
 	}
 	s.store.SetFinalizedCheckpt(&ethpb.Checkpoint{Epoch: 0, Root: params.BeaconConfig().ZeroHash[:]})
-	block := util.NewBeaconBlock()
-	r, err := block.Block.HashTreeRoot()
+	b := util.NewBeaconBlock()
+	r, err := b.Block.HashTreeRoot()
 	require.NoError(t, err)
 	beaconState, err := util.NewBeaconState()
 	require.NoError(t, err)
-	wsb, err := wrapper.WrappedSignedBeaconBlock(block)
+	wsb, err := wrapper.WrappedSignedBeaconBlock(b)
 	require.NoError(t, err)
 	require.NoError(t, s.insertBlockAndAttestationsToForkChoiceStore(ctx, wsb.Block(), r, beaconState))
 
@@ -517,12 +522,12 @@ func TestHasBlock_ForkChoiceAndDB_DoublyLinkedTree(t *testing.T) {
 		store: &store.Store{},
 	}
 	s.store.SetFinalizedCheckpt(&ethpb.Checkpoint{Epoch: 0, Root: params.BeaconConfig().ZeroHash[:]})
-	block := util.NewBeaconBlock()
-	r, err := block.Block.HashTreeRoot()
+	b := util.NewBeaconBlock()
+	r, err := b.Block.HashTreeRoot()
 	require.NoError(t, err)
 	beaconState, err := util.NewBeaconState()
 	require.NoError(t, err)
-	wsb, err := wrapper.WrappedSignedBeaconBlock(block)
+	wsb, err := wrapper.WrappedSignedBeaconBlock(b)
 	require.NoError(t, err)
 	require.NoError(t, s.insertBlockAndAttestationsToForkChoiceStore(ctx, wsb.Block(), r, beaconState))
 
@@ -539,10 +544,10 @@ func TestServiceStop_SaveCachedBlocks(t *testing.T) {
 		cancel:         cancel,
 		initSyncBlocks: make(map[[32]byte]block.SignedBeaconBlock),
 	}
-	block := util.NewBeaconBlock()
-	r, err := block.Block.HashTreeRoot()
+	b := util.NewBeaconBlock()
+	r, err := b.Block.HashTreeRoot()
 	require.NoError(t, err)
-	wsb, err := wrapper.WrappedSignedBeaconBlock(block)
+	wsb, err := wrapper.WrappedSignedBeaconBlock(b)
 	require.NoError(t, err)
 	s.saveInitSyncBlock(r, wsb)
 	require.NoError(t, s.Stop())
@@ -569,11 +574,11 @@ func BenchmarkHasBlockDB(b *testing.B) {
 	s := &Service{
 		cfg: &config{BeaconDB: beaconDB},
 	}
-	block := util.NewBeaconBlock()
-	wsb, err := wrapper.WrappedSignedBeaconBlock(block)
+	blk := util.NewBeaconBlock()
+	wsb, err := wrapper.WrappedSignedBeaconBlock(blk)
 	require.NoError(b, err)
 	require.NoError(b, s.cfg.BeaconDB.SaveBlock(ctx, wsb))
-	r, err := block.Block.HashTreeRoot()
+	r, err := blk.Block.HashTreeRoot()
 	require.NoError(b, err)
 
 	b.ResetTimer()
@@ -613,13 +618,13 @@ func BenchmarkHasBlockForkChoiceStore_DoublyLinkedTree(b *testing.B) {
 		store: &store.Store{},
 	}
 	s.store.SetFinalizedCheckpt(&ethpb.Checkpoint{Epoch: 0, Root: params.BeaconConfig().ZeroHash[:]})
-	block := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}}}
-	r, err := block.Block.HashTreeRoot()
+	blk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}}}
+	r, err := blk.Block.HashTreeRoot()
 	require.NoError(b, err)
 	bs := &ethpb.BeaconState{FinalizedCheckpoint: &ethpb.Checkpoint{Root: make([]byte, 32)}, CurrentJustifiedCheckpoint: &ethpb.Checkpoint{Root: make([]byte, 32)}}
 	beaconState, err := v1.InitializeFromProto(bs)
 	require.NoError(b, err)
-	wsb, err := wrapper.WrappedSignedBeaconBlock(block)
+	wsb, err := wrapper.WrappedSignedBeaconBlock(blk)
 	require.NoError(b, err)
 	require.NoError(b, s.insertBlockAndAttestationsToForkChoiceStore(ctx, wsb.Block(), r, beaconState))
 

beacon-chain/blockchain/state_balance_cache.go

@@ -24,7 +24,7 @@ type stateByRooter interface {
 	StateByRoot(context.Context, [32]byte) (state.BeaconState, error)
 }
 
-// newStateBalanceCache exists to remind us that stateBalanceCache needs a stagegen
+// newStateBalanceCache exists to remind us that stateBalanceCache needs a state gen
 // to avoid nil pointer bugs when updating the cache in the read path (get())
 func newStateBalanceCache(sg *stategen.State) (*stateBalanceCache, error) {
 	if sg == nil {

beacon-chain/blockchain/state_balance_cache_test.go

@@ -15,15 +15,6 @@ import (
 	"github.com/prysmaticlabs/prysm/time/slots"
 )
 
-type mockStateByRoot struct {
-	state state.BeaconState
-	err   error
-}
-
-func (m *mockStateByRoot) StateByRoot(context.Context, [32]byte) (state.BeaconState, error) {
-	return m.state, m.err
-}
-
 type testStateOpt func(*ethpb.BeaconStateAltair)
 
 func testStateWithValidators(v []*ethpb.Validator) testStateOpt {

beacon-chain/blockchain/testing/mock.go

@@ -61,6 +61,7 @@ type ChainService struct {
 	SyncCommitteePubkeys        [][]byte
 	Genesis                     time.Time
 	ForkChoiceStore             forkchoice.ForkChoicer
+	ReceiveBlockMockErr         error
 }
 
 // ForkChoicer mocks the same method in the chain service
@@ -195,32 +196,35 @@ func (s *ChainService) ReceiveBlockBatch(ctx context.Context, blks []block.Signe
 	if s.State == nil {
 		return ErrNilState
 	}
-	for _, block := range blks {
-		if !bytes.Equal(s.Root, block.Block().ParentRoot()) {
-			return errors.Errorf("wanted %#x but got %#x", s.Root, block.Block().ParentRoot())
+	for _, b := range blks {
+		if !bytes.Equal(s.Root, b.Block().ParentRoot()) {
+			return errors.Errorf("wanted %#x but got %#x", s.Root, b.Block().ParentRoot())
 		}
-		if err := s.State.SetSlot(block.Block().Slot()); err != nil {
+		if err := s.State.SetSlot(b.Block().Slot()); err != nil {
 			return err
 		}
-		s.BlocksReceived = append(s.BlocksReceived, block)
-		signingRoot, err := block.Block().HashTreeRoot()
+		s.BlocksReceived = append(s.BlocksReceived, b)
+		signingRoot, err := b.Block().HashTreeRoot()
 		if err != nil {
 			return err
 		}
 		if s.DB != nil {
-			if err := s.DB.SaveBlock(ctx, block); err != nil {
+			if err := s.DB.SaveBlock(ctx, b); err != nil {
 				return err
 			}
-			logrus.Infof("Saved block with root: %#x at slot %d", signingRoot, block.Block().Slot())
+			logrus.Infof("Saved block with root: %#x at slot %d", signingRoot, b.Block().Slot())
 		}
 		s.Root = signingRoot[:]
-		s.Block = block
+		s.Block = b
 	}
 	return nil
 }
 
 // ReceiveBlock mocks ReceiveBlock method in chain service.
 func (s *ChainService) ReceiveBlock(ctx context.Context, block block.SignedBeaconBlock, _ [32]byte) error {
+	if s.ReceiveBlockMockErr != nil {
+		return s.ReceiveBlockMockErr
+	}
 	if s.State == nil {
 		return ErrNilState
 	}
@@ -417,7 +421,7 @@ func (s *ChainService) HeadValidatorIndexToPublicKey(_ context.Context, _ types.
 }
 
 // HeadSyncCommitteeIndices mocks HeadSyncCommitteeIndices and always return `HeadNextSyncCommitteeIndices`.
-func (s *ChainService) HeadSyncCommitteeIndices(_ context.Context, index types.ValidatorIndex, slot types.Slot) ([]types.CommitteeIndex, error) {
+func (s *ChainService) HeadSyncCommitteeIndices(_ context.Context, _ types.ValidatorIndex, _ types.Slot) ([]types.CommitteeIndex, error) {
 	return s.SyncCommitteeIndices, nil
 }
 

beacon-chain/cache/BUILD.bazel

@@ -13,6 +13,7 @@ go_library(
         "common.go",
         "doc.go",
         "error.go",
+        "payload_id.go",
         "proposer_indices.go",
         "proposer_indices_disabled.go",  # keep
         "proposer_indices_type.go",
@@ -26,6 +27,7 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/cache",
     visibility = [
         "//beacon-chain:__subpackages__",
+        "//testing/spectest:__subpackages__",
         "//tools:__subpackages__",
     ],
     deps = [
@@ -61,6 +63,7 @@ go_test(
         "checkpoint_state_test.go",
         "committee_fuzz_test.go",
         "committee_test.go",
+        "payload_id_test.go",
         "proposer_indices_test.go",
         "skip_slot_cache_test.go",
         "subnet_ids_test.go",

beacon-chain/cache/depositcache/deposits_cache.go

@@ -36,7 +36,7 @@ type DepositFetcher interface {
 	DepositByPubkey(ctx context.Context, pubKey []byte) (*ethpb.Deposit, *big.Int)
 	DepositsNumberAndRootAtHeight(ctx context.Context, blockHeight *big.Int) (uint64, [32]byte)
 	FinalizedDeposits(ctx context.Context) *FinalizedDeposits
-	NonFinalizedDeposits(ctx context.Context, untilBlk *big.Int) []*ethpb.Deposit
+	NonFinalizedDeposits(ctx context.Context, lastFinalizedIndex int64, untilBlk *big.Int) []*ethpb.Deposit
 }
 
 // FinalizedDeposits stores the trie of deposits that have been included
@@ -246,7 +246,7 @@ func (dc *DepositCache) FinalizedDeposits(ctx context.Context) *FinalizedDeposit
 
 // NonFinalizedDeposits returns the list of non-finalized deposits until the given block number (inclusive).
 // If no block is specified then this method returns all non-finalized deposits.
-func (dc *DepositCache) NonFinalizedDeposits(ctx context.Context, untilBlk *big.Int) []*ethpb.Deposit {
+func (dc *DepositCache) NonFinalizedDeposits(ctx context.Context, lastFinalizedIndex int64, untilBlk *big.Int) []*ethpb.Deposit {
 	ctx, span := trace.StartSpan(ctx, "DepositsCache.NonFinalizedDeposits")
 	defer span.End()
 	dc.depositsLock.RLock()
@@ -256,10 +256,9 @@ func (dc *DepositCache) NonFinalizedDeposits(ctx context.Context, untilBlk *big.
 		return dc.allDeposits(untilBlk)
 	}
 
-	lastFinalizedDepositIndex := dc.finalizedDeposits.MerkleTrieIndex
 	var deposits []*ethpb.Deposit
 	for _, d := range dc.deposits {
-		if (d.Index > lastFinalizedDepositIndex) && (untilBlk == nil || untilBlk.Uint64() >= d.Eth1BlockHeight) {
+		if (d.Index > lastFinalizedIndex) && (untilBlk == nil || untilBlk.Uint64() >= d.Eth1BlockHeight) {
 			deposits = append(deposits, d.Deposit)
 		}
 	}

beacon-chain/cache/depositcache/deposits_cache_test.go

@@ -554,7 +554,7 @@ func TestNonFinalizedDeposits_ReturnsAllNonFinalizedDeposits(t *testing.T) {
 		})
 	dc.InsertFinalizedDeposits(context.Background(), 1)
 
-	deps := dc.NonFinalizedDeposits(context.Background(), nil)
+	deps := dc.NonFinalizedDeposits(context.Background(), 1, nil)
 	assert.Equal(t, 2, len(deps))
 }
 
@@ -611,7 +611,7 @@ func TestNonFinalizedDeposits_ReturnsNonFinalizedDepositsUpToBlockNumber(t *test
 		})
 	dc.InsertFinalizedDeposits(context.Background(), 1)
 
-	deps := dc.NonFinalizedDeposits(context.Background(), big.NewInt(10))
+	deps := dc.NonFinalizedDeposits(context.Background(), 1, big.NewInt(10))
 	assert.Equal(t, 1, len(deps))
 }
 

beacon-chain/cache/payload_id.go

@@ -0,0 +1,73 @@
+package cache
+
+import (
+	"sync"
+
+	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+)
+
+const vIdLength = 8
+const pIdLength = 8
+const vpIdsLength = vIdLength + pIdLength
+
+// ProposerPayloadIDsCache is a cache of proposer payload IDs.
+// The key is the slot. The value is the concatenation of the proposer and payload IDs. 8 bytes each.
+type ProposerPayloadIDsCache struct {
+	slotToProposerAndPayloadIDs map[types.Slot][vpIdsLength]byte
+	sync.RWMutex
+}
+
+// NewProposerPayloadIDsCache creates a new proposer payload IDs cache.
+func NewProposerPayloadIDsCache() *ProposerPayloadIDsCache {
+	return &ProposerPayloadIDsCache{
+		slotToProposerAndPayloadIDs: make(map[types.Slot][vpIdsLength]byte),
+	}
+}
+
+// GetProposerPayloadIDs returns the proposer and  payload IDs for the given slot.
+func (f *ProposerPayloadIDsCache) GetProposerPayloadIDs(slot types.Slot) (types.ValidatorIndex, [8]byte, bool) {
+	f.RLock()
+	defer f.RUnlock()
+	ids, ok := f.slotToProposerAndPayloadIDs[slot]
+	if !ok {
+		return 0, [8]byte{}, false
+	}
+	vId := ids[:vIdLength]
+
+	b := ids[vIdLength:]
+	var pId [pIdLength]byte
+	copy(pId[:], b)
+
+	return types.ValidatorIndex(bytesutil.BytesToUint64BigEndian(vId)), pId, true
+}
+
+// SetProposerAndPayloadIDs sets the proposer and payload IDs for the given slot.
+func (f *ProposerPayloadIDsCache) SetProposerAndPayloadIDs(slot types.Slot, vId types.ValidatorIndex, pId [8]byte) {
+	f.Lock()
+	defer f.Unlock()
+	var vIdBytes [vIdLength]byte
+	copy(vIdBytes[:], bytesutil.Uint64ToBytesBigEndian(uint64(vId)))
+
+	var bytes [vpIdsLength]byte
+	copy(bytes[:], append(vIdBytes[:], pId[:]...))
+
+	_, ok := f.slotToProposerAndPayloadIDs[slot]
+	// Ok to overwrite if the slot is already set but the payload ID is not set.
+	// This combats the re-org case where payload assignment could change the epoch of.
+	if !ok || (ok && pId != [pIdLength]byte{}) {
+		f.slotToProposerAndPayloadIDs[slot] = bytes
+	}
+}
+
+// PrunePayloadIDs removes the payload id entries that's current than input slot.
+func (f *ProposerPayloadIDsCache) PrunePayloadIDs(slot types.Slot) {
+	f.Lock()
+	defer f.Unlock()
+
+	for s := range f.slotToProposerAndPayloadIDs {
+		if slot > s {
+			delete(f.slotToProposerAndPayloadIDs, s)
+		}
+	}
+}

beacon-chain/cache/payload_id_test.go

@@ -0,0 +1,60 @@
+package cache
+
+import (
+	"testing"
+
+	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/testing/require"
+)
+
+func TestValidatorPayloadIDsCache_GetAndSaveValidatorPayloadIDs(t *testing.T) {
+	cache := NewProposerPayloadIDsCache()
+	i, p, ok := cache.GetProposerPayloadIDs(0)
+	require.Equal(t, false, ok)
+	require.Equal(t, types.ValidatorIndex(0), i)
+	require.Equal(t, [pIdLength]byte{}, p)
+
+	slot := types.Slot(1234)
+	vid := types.ValidatorIndex(34234324)
+	pid := [8]byte{1, 2, 3, 3, 7, 8, 7, 8}
+	cache.SetProposerAndPayloadIDs(slot, vid, pid)
+	i, p, ok = cache.GetProposerPayloadIDs(slot)
+	require.Equal(t, true, ok)
+	require.Equal(t, vid, i)
+	require.Equal(t, pid, p)
+
+	slot = types.Slot(9456456)
+	vid = types.ValidatorIndex(6786745)
+	cache.SetProposerAndPayloadIDs(slot, vid, [pIdLength]byte{})
+	i, p, ok = cache.GetProposerPayloadIDs(slot)
+	require.Equal(t, true, ok)
+	require.Equal(t, vid, i)
+	require.Equal(t, [pIdLength]byte{}, p)
+
+	// reset cache without pid
+	slot = types.Slot(9456456)
+	vid = types.ValidatorIndex(11111)
+	pid = [8]byte{3, 2, 3, 33, 72, 8, 7, 8}
+	cache.SetProposerAndPayloadIDs(slot, vid, pid)
+	i, p, ok = cache.GetProposerPayloadIDs(slot)
+	require.Equal(t, true, ok)
+	require.Equal(t, vid, i)
+	require.Equal(t, pid, p)
+
+	// reset cache with existing pid
+	slot = types.Slot(9456456)
+	vid = types.ValidatorIndex(11111)
+	newPid := [8]byte{1, 2, 3, 33, 72, 8, 7, 1}
+	cache.SetProposerAndPayloadIDs(slot, vid, newPid)
+	i, p, ok = cache.GetProposerPayloadIDs(slot)
+	require.Equal(t, true, ok)
+	require.Equal(t, vid, i)
+	require.Equal(t, newPid, p)
+
+	// remove cache entry
+	cache.PrunePayloadIDs(slot + 1)
+	i, p, ok = cache.GetProposerPayloadIDs(slot)
+	require.Equal(t, false, ok)
+	require.Equal(t, types.ValidatorIndex(0), i)
+	require.Equal(t, [pIdLength]byte{}, p)
+}

beacon-chain/core/helpers/beacon_committee.go

@@ -175,9 +175,7 @@ func CommitteeAssignments(
 		return nil, nil, err
 	}
 	proposerIndexToSlots := make(map[types.ValidatorIndex][]types.Slot, params.BeaconConfig().SlotsPerEpoch)
-	// Proposal epochs do not have a look ahead, so we skip them over here.
-	validProposalEpoch := epoch < nextEpoch
-	for slot := startSlot; slot < startSlot+params.BeaconConfig().SlotsPerEpoch && validProposalEpoch; slot++ {
+	for slot := startSlot; slot < startSlot+params.BeaconConfig().SlotsPerEpoch; slot++ {
 		// Skip proposer assignment for genesis slot.
 		if slot == 0 {
 			continue
@@ -192,6 +190,15 @@ func CommitteeAssignments(
 		proposerIndexToSlots[i] = append(proposerIndexToSlots[i], slot)
 	}
 
+	// If previous proposer indices computation is outside if current proposal epoch range,
+	// we need to reset state slot back to start slot so that we can compute the correct committees.
+	currentProposalEpoch := epoch < nextEpoch
+	if !currentProposalEpoch {
+		if err := state.SetSlot(state.Slot() - params.BeaconConfig().SlotsPerEpoch); err != nil {
+			return nil, nil, err
+		}
+	}
+
 	activeValidatorIndices, err := ActiveValidatorIndices(ctx, state, epoch)
 	if err != nil {
 		return nil, nil, err

beacon-chain/core/helpers/beacon_committee_test.go

@@ -232,7 +232,7 @@ func TestCommitteeAssignments_CannotRetrieveFuture(t *testing.T) {
 
 	_, proposerIndxs, err = CommitteeAssignments(context.Background(), state, time.CurrentEpoch(state)+1)
 	require.NoError(t, err)
-	require.Equal(t, 0, len(proposerIndxs), "wanted empty proposer index set")
+	require.NotEqual(t, 0, len(proposerIndxs), "wanted non-zero proposer index set")
 }
 
 func TestCommitteeAssignments_EverySlotHasMin1Proposer(t *testing.T) {

beacon-chain/db/iface/interface.go

@@ -99,12 +99,13 @@ type HeadAccessDatabase interface {
 	SaveHeadBlockRoot(ctx context.Context, blockRoot [32]byte) error
 
 	// Genesis operations.
-	LoadGenesis(ctx context.Context, r io.Reader) error
+	LoadGenesis(ctx context.Context, stateBytes []byte) error
 	SaveGenesisData(ctx context.Context, state state.BeaconState) error
 	EnsureEmbeddedGenesis(ctx context.Context) error
 
 	// initialization method needed for origin checkpoint sync
 	SaveOrigin(ctx context.Context, serState, serBlock []byte) error
+	SaveOriginCheckpointBlockRoot(ctx context.Context, blockRoot [32]byte) error
 	SaveBackfillBlockRoot(ctx context.Context, blockRoot [32]byte) error
 }
 

beacon-chain/db/kv/BUILD.bazel

@@ -94,6 +94,7 @@ go_test(
         "state_test.go",
         "utils_test.go",
         "validated_checkpoint_test.go",
+        "wss_test.go",
     ],
     data = glob(["testdata/**"]),
     embed = [":go_default_library"],
@@ -101,6 +102,7 @@ go_test(
         "//beacon-chain/db/filters:go_default_library",
         "//beacon-chain/db/iface:go_default_library",
         "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/genesis:go_default_library",
         "//beacon-chain/state/v1:go_default_library",
         "//beacon-chain/state/v2:go_default_library",
         "//config/features:go_default_library",

beacon-chain/db/kv/blocks.go

@@ -233,13 +233,17 @@ func (s *Store) DeleteBlock(ctx context.Context, root [32]byte) error {
 	defer span.End()
 
 	if err := s.DeleteState(ctx, root); err != nil {
-		return errDeleteFinalized
+		return err
+	}
+
+	if err := s.deleteStateSummary(root); err != nil {
+		return err
 	}
 
 	return s.db.Update(func(tx *bolt.Tx) error {
 		bkt := tx.Bucket(finalizedBlockRootsIndexBucket)
 		if b := bkt.Get(root[:]); b != nil {
-			return errDeleteFinalized
+			return ErrDeleteJustifiedAndFinalized
 		}
 
 		if err := tx.Bucket(blocksBucket).Delete(root[:]); err != nil {

beacon-chain/db/kv/blocks_test.go

@@ -191,6 +191,16 @@ func TestStore_DeleteBlock(t *testing.T) {
 	require.NoError(t, db.SaveGenesisBlockRoot(ctx, genesisBlockRoot))
 	blks := makeBlocks(t, 0, slotsPerEpoch*4, genesisBlockRoot)
 	require.NoError(t, db.SaveBlocks(ctx, blks))
+	ss := make([]*ethpb.StateSummary, len(blks))
+	for i, blk := range blks {
+		r, err := blk.Block().HashTreeRoot()
+		require.NoError(t, err)
+		ss[i] = &ethpb.StateSummary{
+			Slot: blk.Block().Slot(),
+			Root: r[:],
+		}
+	}
+	require.NoError(t, db.SaveStateSummaries(ctx, ss))
 
 	root, err := blks[slotsPerEpoch].Block().HashTreeRoot()
 	require.NoError(t, err)
@@ -216,11 +226,50 @@ func TestStore_DeleteBlock(t *testing.T) {
 	b, err = db.Block(ctx, root2)
 	require.NoError(t, err)
 	require.Equal(t, b, nil)
+	require.Equal(t, false, db.HasStateSummary(ctx, root2))
 
-	require.ErrorIs(t, db.DeleteBlock(ctx, root), errDeleteFinalized)
-
+	require.ErrorIs(t, db.DeleteBlock(ctx, root), ErrDeleteJustifiedAndFinalized)
 }
 
+func TestStore_DeleteJustifiedBlock(t *testing.T) {
+	db := setupDB(t)
+	ctx := context.Background()
+	b := util.NewBeaconBlock()
+	b.Block.Slot = 1
+	root, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	cp := &ethpb.Checkpoint{
+		Root: root[:],
+	}
+	st, err := util.NewBeaconState()
+	require.NoError(t, err)
+	blk, err := wrapper.WrappedSignedBeaconBlock(b)
+	require.NoError(t, err)
+	require.NoError(t, db.SaveBlock(ctx, blk))
+	require.NoError(t, db.SaveState(ctx, st, root))
+	require.NoError(t, db.SaveJustifiedCheckpoint(ctx, cp))
+	require.ErrorIs(t, db.DeleteBlock(ctx, root), ErrDeleteJustifiedAndFinalized)
+}
+
+func TestStore_DeleteFinalizedBlock(t *testing.T) {
+	db := setupDB(t)
+	ctx := context.Background()
+	b := util.NewBeaconBlock()
+	root, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	cp := &ethpb.Checkpoint{
+		Root: root[:],
+	}
+	st, err := util.NewBeaconState()
+	require.NoError(t, err)
+	blk, err := wrapper.WrappedSignedBeaconBlock(b)
+	require.NoError(t, err)
+	require.NoError(t, db.SaveBlock(ctx, blk))
+	require.NoError(t, db.SaveState(ctx, st, root))
+	require.NoError(t, db.SaveGenesisBlockRoot(ctx, root))
+	require.NoError(t, db.SaveFinalizedCheckpoint(ctx, cp))
+	require.ErrorIs(t, db.DeleteBlock(ctx, root), ErrDeleteJustifiedAndFinalized)
+}
 func TestStore_GenesisBlock(t *testing.T) {
 	db := setupDB(t)
 	ctx := context.Background()

beacon-chain/db/kv/error.go

@@ -2,8 +2,8 @@ package kv
 
 import "github.com/pkg/errors"
 
-// errDeleteFinalized is raised when we attempt to delete a finalized block/state
-var errDeleteFinalized = errors.New("cannot delete finalized block or state")
+// ErrDeleteJustifiedAndFinalized is raised when we attempt to delete a finalized block/state
+var ErrDeleteJustifiedAndFinalized = errors.New("cannot delete finalized block or state")
 
 // ErrNotFound can be used directly, or as a wrapped DBError, whenever a db method needs to
 // indicate that a value couldn't be found.

beacon-chain/db/kv/genesis.go

@@ -4,8 +4,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io"
-	"io/ioutil"
 
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
@@ -54,14 +52,10 @@ func (s *Store) SaveGenesisData(ctx context.Context, genesisState state.BeaconSt
 	return nil
 }
 
-// LoadGenesis loads a genesis state from a given file path, if no genesis exists already.
-func (s *Store) LoadGenesis(ctx context.Context, r io.Reader) error {
-	b, err := ioutil.ReadAll(r)
-	if err != nil {
-		return err
-	}
+// LoadGenesis loads a genesis state from a ssz-serialized byte slice, if no genesis exists already.
+func (s *Store) LoadGenesis(ctx context.Context, sb []byte) error {
 	st := &ethpb.BeaconState{}
-	if err := st.UnmarshalSSZ(b); err != nil {
+	if err := st.UnmarshalSSZ(sb); err != nil {
 		return err
 	}
 	gs, err := statev1.InitializeFromProtoUnsafe(st)

beacon-chain/db/kv/genesis_test.go

@@ -53,21 +53,16 @@ func TestLoadGenesisFromFile(t *testing.T) {
 	if err == nil {
 		fp = rfp
 	}
-
-	r, err := os.Open(fp)
+	sb, err := os.ReadFile(fp)
 	assert.NoError(t, err)
-	defer func() {
-		assert.NoError(t, r.Close())
-	}()
 
 	db := setupDB(t)
-	assert.NoError(t, db.LoadGenesis(context.Background(), r))
+	assert.NoError(t, db.LoadGenesis(context.Background(), sb))
 	testGenesisDataSaved(t, db)
 
 	// Loading the same genesis again should not throw an error
-	_, err = r.Seek(0, 0)
 	assert.NoError(t, err)
-	assert.NoError(t, db.LoadGenesis(context.Background(), r))
+	assert.NoError(t, db.LoadGenesis(context.Background(), sb))
 }
 
 func TestLoadGenesisFromFile_mismatchedForkVersion(t *testing.T) {
@@ -76,15 +71,12 @@ func TestLoadGenesisFromFile_mismatchedForkVersion(t *testing.T) {
 	if err == nil {
 		fp = rfp
 	}
-	r, err := os.Open(fp)
+	sb, err := os.ReadFile(fp)
 	assert.NoError(t, err)
-	defer func() {
-		assert.NoError(t, r.Close())
-	}()
 
 	// Loading a genesis with the wrong fork version as beacon config should throw an error.
 	db := setupDB(t)
-	assert.ErrorContains(t, "does not match config genesis fork version", db.LoadGenesis(context.Background(), r))
+	assert.ErrorContains(t, "does not match config genesis fork version", db.LoadGenesis(context.Background(), sb))
 }
 
 func TestEnsureEmbeddedGenesis(t *testing.T) {

beacon-chain/db/kv/state.go

@@ -346,19 +346,31 @@ func (s *Store) DeleteState(ctx context.Context, blockRoot [32]byte) error {
 
 		bkt = tx.Bucket(checkpointBucket)
 		enc := bkt.Get(finalizedCheckpointKey)
-		checkpoint := &ethpb.Checkpoint{}
+		finalized := &ethpb.Checkpoint{}
 		if enc == nil {
-			checkpoint = &ethpb.Checkpoint{Root: genesisBlockRoot}
-		} else if err := decode(ctx, enc, checkpoint); err != nil {
+			finalized = &ethpb.Checkpoint{Root: genesisBlockRoot}
+		} else if err := decode(ctx, enc, finalized); err != nil {
+			return err
+		}
+
+		enc = bkt.Get(justifiedCheckpointKey)
+		justified := &ethpb.Checkpoint{}
+		if enc == nil {
+			justified = &ethpb.Checkpoint{Root: genesisBlockRoot}
+		} else if err := decode(ctx, enc, justified); err != nil {
 			return err
 		}
 
-		blockBkt := tx.Bucket(blocksBucket)
-		headBlkRoot := blockBkt.Get(headBlockRootKey)
 		bkt = tx.Bucket(stateBucket)
-		// Safe guard against deleting genesis, finalized, head state.
-		if bytes.Equal(blockRoot[:], checkpoint.Root) || bytes.Equal(blockRoot[:], genesisBlockRoot) || bytes.Equal(blockRoot[:], headBlkRoot) {
-			return errors.New("cannot delete genesis, finalized, or head state")
+		// Safeguard against deleting genesis, finalized, head state.
+		if bytes.Equal(blockRoot[:], finalized.Root) || bytes.Equal(blockRoot[:], genesisBlockRoot) || bytes.Equal(blockRoot[:], justified.Root) {
+			return ErrDeleteJustifiedAndFinalized
+		}
+
+		// Nothing to delete if state doesn't exist.
+		enc = bkt.Get(blockRoot[:])
+		if enc == nil {
+			return nil
 		}
 
 		slot, err := s.slotByBlockRoot(ctx, tx, blockRoot[:])

beacon-chain/db/kv/state_summary.go

@@ -110,3 +110,12 @@ func (s *Store) saveCachedStateSummariesDB(ctx context.Context) error {
 	s.stateSummaryCache.clear()
 	return nil
 }
+
+// deleteStateSummary deletes a state summary object from the db using input block root.
+func (s *Store) deleteStateSummary(blockRoot [32]byte) error {
+	s.stateSummaryCache.delete(blockRoot)
+	return s.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket(stateSummaryBucket)
+		return bucket.Delete(blockRoot[:])
+	})
+}

beacon-chain/db/kv/state_summary_cache.go

@@ -37,6 +37,13 @@ func (c *stateSummaryCache) has(r [32]byte) bool {
 	return ok
 }
 
+// delete state summary in cache.
+func (c *stateSummaryCache) delete(r [32]byte) {
+	c.initSyncStateSummariesLock.Lock()
+	defer c.initSyncStateSummariesLock.Unlock()
+	delete(c.initSyncStateSummaries, r)
+}
+
 // get retrieves a state summary from the initial sync state summaries cache using the root of
 // the block.
 func (c *stateSummaryCache) get(r [32]byte) *ethpb.StateSummary {

beacon-chain/db/kv/state_summary_test.go

@@ -62,3 +62,17 @@ func TestStateSummary_CacheToDB(t *testing.T) {
 		require.Equal(t, true, db.HasStateSummary(context.Background(), bytesutil.ToBytes32(r)))
 	}
 }
+
+func TestStateSummary_CanDelete(t *testing.T) {
+	db := setupDB(t)
+	ctx := context.Background()
+	r1 := bytesutil.ToBytes32([]byte{'A'})
+	s1 := &ethpb.StateSummary{Slot: 1, Root: r1[:]}
+
+	require.Equal(t, false, db.HasStateSummary(ctx, r1), "State summary should not be saved")
+	require.NoError(t, db.SaveStateSummary(ctx, s1))
+	require.Equal(t, true, db.HasStateSummary(ctx, r1), "State summary should be saved")
+
+	require.NoError(t, db.deleteStateSummary(r1))
+	require.Equal(t, false, db.HasStateSummary(ctx, r1), "State summary should not be saved")
+}

beacon-chain/db/kv/state_test.go

@@ -412,7 +412,7 @@ func TestStore_DeleteGenesisState(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, st.SetSlot(100))
 	require.NoError(t, db.SaveState(ctx, st, genesisBlockRoot))
-	wantedErr := "cannot delete genesis, finalized, or head state"
+	wantedErr := "cannot delete finalized block or state"
 	assert.ErrorContains(t, wantedErr, db.DeleteState(ctx, genesisBlockRoot))
 }
 
@@ -440,7 +440,7 @@ func TestStore_DeleteFinalizedState(t *testing.T) {
 	require.NoError(t, db.SaveState(ctx, finalizedState, finalizedBlockRoot))
 	finalizedCheckpoint := &ethpb.Checkpoint{Root: finalizedBlockRoot[:]}
 	require.NoError(t, db.SaveFinalizedCheckpoint(ctx, finalizedCheckpoint))
-	wantedErr := "cannot delete genesis, finalized, or head state"
+	wantedErr := "cannot delete finalized block or state"
 	assert.ErrorContains(t, wantedErr, db.DeleteState(ctx, finalizedBlockRoot))
 }
 
@@ -465,8 +465,7 @@ func TestStore_DeleteHeadState(t *testing.T) {
 	require.NoError(t, st.SetSlot(100))
 	require.NoError(t, db.SaveState(ctx, st, headBlockRoot))
 	require.NoError(t, db.SaveHeadBlockRoot(ctx, headBlockRoot))
-	wantedErr := "cannot delete genesis, finalized, or head state"
-	assert.ErrorContains(t, wantedErr, db.DeleteState(ctx, headBlockRoot))
+	require.NoError(t, db.DeleteState(ctx, headBlockRoot)) // Ok to delete head state if it's optimistic.
 }
 
 func TestStore_SaveDeleteState_CanGetHighestBelow(t *testing.T) {

beacon-chain/db/kv/wss.go

@@ -77,6 +77,12 @@ func (s *Store) SaveOrigin(ctx context.Context, serState, serBlock []byte) error
 		return errors.Wrap(err, "could not save head block root")
 	}
 
+	// save origin block root in a special key, to be used when the canonical
+	// origin (start of chain, ie alternative to genesis) block or state is needed
+	if err = s.SaveOriginCheckpointBlockRoot(ctx, blockRoot); err != nil {
+		return errors.Wrap(err, "could not save origin block root")
+	}
+
 	// rebuild the checkpoint from the block
 	// use it to mark the block as justified and finalized
 	slotEpoch, err := wblk.Block().Slot().SafeDivSlot(params.BeaconConfig().SlotsPerEpoch)
@@ -94,11 +100,5 @@ func (s *Store) SaveOrigin(ctx context.Context, serState, serBlock []byte) error
 		return errors.Wrap(err, "could not mark checkpoint sync block as finalized")
 	}
 
-	// save origin block root in a special key, to be used when the canonical
-	// origin (start of chain, ie alternative to genesis) block or state is needed
-	if err = s.SaveOriginCheckpointBlockRoot(ctx, blockRoot); err != nil {
-		return errors.Wrap(err, "could not save origin block root")
-	}
-
 	return nil
 }

beacon-chain/db/kv/wss_test.go

@@ -0,0 +1,45 @@
+package kv
+
+import (
+	"context"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/genesis"
+	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
+	"github.com/prysmaticlabs/prysm/testing/require"
+	"github.com/prysmaticlabs/prysm/testing/util"
+)
+
+func TestSaveOrigin(t *testing.T) {
+	// Embedded Genesis works with Mainnet config
+	params.SetupTestConfigCleanup(t)
+	cfg := params.BeaconConfig()
+	cfg.ConfigName = params.ConfigNames[params.Mainnet]
+	params.OverrideBeaconConfig(cfg)
+
+	ctx := context.Background()
+	db := setupDB(t)
+
+	st, err := genesis.State(params.Mainnet.String())
+	require.NoError(t, err)
+
+	sb, err := st.MarshalSSZ()
+	require.NoError(t, err)
+	require.NoError(t, db.LoadGenesis(ctx, sb))
+
+	// this is necessary for mainnet, because LoadGenesis is short-circuited by the embedded state,
+	// so the genesis root key is never written to the db.
+	require.NoError(t, db.EnsureEmbeddedGenesis(ctx))
+
+	cst, err := util.NewBeaconState()
+	require.NoError(t, err)
+	csb, err := cst.MarshalSSZ()
+	require.NoError(t, err)
+	cb := util.NewBeaconBlock()
+	scb, err := wrapper.WrappedSignedBeaconBlock(cb)
+	require.NoError(t, err)
+	cbb, err := scb.MarshalSSZ()
+	require.NoError(t, err)
+	require.NoError(t, db.SaveOrigin(ctx, csb, cbb))
+}

beacon-chain/deterministic-genesis/service.go

@@ -155,7 +155,7 @@ func (_ *Service) FinalizedDeposits(_ context.Context) *depositcache.FinalizedDe
 }
 
 // NonFinalizedDeposits mocks out the deposit cache functionality for interop.
-func (_ *Service) NonFinalizedDeposits(_ context.Context, _ *big.Int) []*ethpb.Deposit {
+func (_ *Service) NonFinalizedDeposits(_ context.Context, _ int64, _ *big.Int) []*ethpb.Deposit {
 	return []*ethpb.Deposit{}
 }
 

beacon-chain/forkchoice/doubly-linked-tree/BUILD.bazel

@@ -22,11 +22,13 @@ go_library(
         "//beacon-chain/forkchoice/types:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
+        "//encoding/bytesutil:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
         "@com_github_prysmaticlabs_eth2_types//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
         "@io_opencensus_go//trace:go_default_library",
     ],
 )

beacon-chain/forkchoice/doubly-linked-tree/errors.go

@@ -3,8 +3,8 @@ package doublylinkedtree
 import "errors"
 
 var ErrNilNode = errors.New("invalid nil or unknown node")
-var errInvalidBalance = errors.New("invalid node balance")
 var errInvalidProposerBoostRoot = errors.New("invalid proposer boost root")
 var errUnknownFinalizedRoot = errors.New("unknown finalized root")
 var errUnknownJustifiedRoot = errors.New("unknown justified root")
 var errInvalidOptimisticStatus = errors.New("invalid optimistic status")
+var errUnknownPayloadHash = errors.New("unknown payload hash")

beacon-chain/forkchoice/doubly-linked-tree/forkchoice.go

@@ -2,12 +2,15 @@ package doublylinkedtree
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/pkg/errors"
 	types "github.com/prysmaticlabs/eth2-types"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 	pbrpc "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 )
 
@@ -18,6 +21,7 @@ func New(justifiedEpoch, finalizedEpoch types.Epoch) *ForkChoice {
 		finalizedEpoch:    finalizedEpoch,
 		proposerBoostRoot: [32]byte{},
 		nodeByRoot:        make(map[[fieldparams.RootLength]byte]*Node),
+		nodeByPayload:     make(map[[fieldparams.RootLength]byte]*Node),
 		pruneThreshold:    defaultPruneThreshold,
 	}
 
@@ -168,7 +172,7 @@ func (f *ForkChoice) IsCanonical(root [32]byte) bool {
 }
 
 // IsOptimistic returns true if the given root has been optimistically synced.
-func (f *ForkChoice) IsOptimistic(_ context.Context, root [32]byte) (bool, error) {
+func (f *ForkChoice) IsOptimistic(root [32]byte) (bool, error) {
 	f.store.nodesLock.RLock()
 	defer f.store.nodesLock.RUnlock()
 
@@ -249,9 +253,21 @@ func (f *ForkChoice) updateBalances(newBalances []uint64) error {
 					return ErrNilNode
 				}
 				if currentNode.balance < oldBalance {
-					return errInvalidBalance
+					f.store.proposerBoostLock.RLock()
+					log.WithFields(logrus.Fields{
+						"nodeRoot":                   fmt.Sprintf("%#x", bytesutil.Trunc(vote.currentRoot[:])),
+						"oldBalance":                 oldBalance,
+						"nodeBalance":                currentNode.balance,
+						"nodeWeight":                 currentNode.weight,
+						"proposerBoostRoot":          fmt.Sprintf("%#x", bytesutil.Trunc(f.store.proposerBoostRoot[:])),
+						"previousProposerBoostRoot":  fmt.Sprintf("%#x", bytesutil.Trunc(f.store.previousProposerBoostRoot[:])),
+						"previousProposerBoostScore": f.store.previousProposerBoostScore,
+					}).Warning("node with invalid balance, setting it to zero")
+					f.store.proposerBoostLock.RUnlock()
+					currentNode.balance = 0
+				} else {
+					currentNode.balance -= oldBalance
 				}
-				currentNode.balance -= oldBalance
 			}
 		}
 
@@ -302,6 +318,6 @@ func (f *ForkChoice) ForkChoiceNodes() []*pbrpc.ForkChoiceNode {
 }
 
 // SetOptimisticToInvalid removes a block with an invalid execution payload from fork choice store
-func (f *ForkChoice) SetOptimisticToInvalid(ctx context.Context, root [fieldparams.RootLength]byte) ([][32]byte, error) {
-	return f.store.removeNode(ctx, root)
+func (f *ForkChoice) SetOptimisticToInvalid(ctx context.Context, root, payloadHash [fieldparams.RootLength]byte) ([][32]byte, error) {
+	return f.store.setOptimisticToInvalid(ctx, root, payloadHash)
 }

beacon-chain/forkchoice/doubly-linked-tree/forkchoice_test.go

@@ -58,6 +58,30 @@ func TestForkChoice_UpdateBalancesNegativeChange(t *testing.T) {
 	assert.Equal(t, uint64(30), s.nodeByRoot[indexToHash(3)].balance)
 }
 
+func TestForkChoice_UpdateBalancesUnderflow(t *testing.T) {
+	f := setup(0, 0)
+	ctx := context.Background()
+	require.NoError(t, f.InsertOptimisticBlock(ctx, 1, indexToHash(1), params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, 0, 0))
+	require.NoError(t, f.InsertOptimisticBlock(ctx, 2, indexToHash(2), indexToHash(1), params.BeaconConfig().ZeroHash, 0, 0))
+	require.NoError(t, f.InsertOptimisticBlock(ctx, 3, indexToHash(3), indexToHash(2), params.BeaconConfig().ZeroHash, 0, 0))
+	s := f.store
+	s.nodeByRoot[indexToHash(1)].balance = 100
+	s.nodeByRoot[indexToHash(2)].balance = 100
+	s.nodeByRoot[indexToHash(3)].balance = 100
+
+	f.balances = []uint64{125, 125, 125}
+	f.votes = []Vote{
+		{indexToHash(1), indexToHash(1), 0},
+		{indexToHash(2), indexToHash(2), 0},
+		{indexToHash(3), indexToHash(3), 0},
+	}
+
+	require.NoError(t, f.updateBalances([]uint64{10, 20, 30}))
+	assert.Equal(t, uint64(0), s.nodeByRoot[indexToHash(1)].balance)
+	assert.Equal(t, uint64(0), s.nodeByRoot[indexToHash(2)].balance)
+	assert.Equal(t, uint64(5), s.nodeByRoot[indexToHash(3)].balance)
+}
+
 func TestForkChoice_IsCanonical(t *testing.T) {
 	f := setup(1, 1)
 	ctx := context.Background()

beacon-chain/forkchoice/doubly-linked-tree/metrics.go

@@ -3,9 +3,12 @@ package doublylinkedtree
 import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/sirupsen/logrus"
 )
 
 var (
+	log = logrus.WithField("prefix", "forkchoice-doublylinkedtree")
+
 	headSlotNumber = promauto.NewGauge(
 		prometheus.GaugeOpts{
 			Name: "doublylinkedtree_head_slot",
@@ -48,10 +51,4 @@ var (
 			Help: "The number of times pruning happened.",
 		},
 	)
-	validatedCount = promauto.NewGauge(
-		prometheus.GaugeOpts{
-			Name: "doublylinkedtree_validated_count",
-			Help: "The number of blocks that have been fully validated.",
-		},
-	)
 )

beacon-chain/forkchoice/doubly-linked-tree/node.go

@@ -109,7 +109,7 @@ func (n *Node) leadsToViableHead(justifiedEpoch, finalizedEpoch types.Epoch) boo
 	return n.bestDescendant.viableForHead(justifiedEpoch, finalizedEpoch)
 }
 
-// setNodeAndParentValidated sets the current node and the parent as validated (i.e. non-optimistic).
+// setNodeAndParentValidated sets the current node and all the ancestors as validated (i.e. non-optimistic).
 func (n *Node) setNodeAndParentValidated(ctx context.Context) error {
 	if ctx.Err() != nil {
 		return ctx.Err()
@@ -120,7 +120,6 @@ func (n *Node) setNodeAndParentValidated(ctx context.Context) error {
 	}
 
 	n.optimistic = false
-	validatedCount.Inc()
 	return n.parent.setNodeAndParentValidated(ctx)
 }
 

beacon-chain/forkchoice/doubly-linked-tree/node_test.go

@@ -180,27 +180,27 @@ func TestNode_SetFullyValidated(t *testing.T) {
 	require.NoError(t, f.InsertOptimisticBlock(ctx, 4, indexToHash(4), indexToHash(3), params.BeaconConfig().ZeroHash, 1, 1))
 	require.NoError(t, f.InsertOptimisticBlock(ctx, 5, indexToHash(5), indexToHash(1), params.BeaconConfig().ZeroHash, 1, 1))
 
-	opt, err := f.IsOptimistic(ctx, indexToHash(5))
+	opt, err := f.IsOptimistic(indexToHash(5))
 	require.NoError(t, err)
 	require.Equal(t, true, opt)
 
-	opt, err = f.IsOptimistic(ctx, indexToHash(4))
+	opt, err = f.IsOptimistic(indexToHash(4))
 	require.NoError(t, err)
 	require.Equal(t, true, opt)
 
 	require.NoError(t, f.store.nodeByRoot[indexToHash(4)].setNodeAndParentValidated(ctx))
 
 	// block 5 should still be optimistic
-	opt, err = f.IsOptimistic(ctx, indexToHash(5))
+	opt, err = f.IsOptimistic(indexToHash(5))
 	require.NoError(t, err)
 	require.Equal(t, true, opt)
 
 	// block 4 and 3 should now be valid
-	opt, err = f.IsOptimistic(ctx, indexToHash(4))
+	opt, err = f.IsOptimistic(indexToHash(4))
 	require.NoError(t, err)
 	require.Equal(t, false, opt)
 
-	opt, err = f.IsOptimistic(ctx, indexToHash(3))
+	opt, err = f.IsOptimistic(indexToHash(3))
 	require.NoError(t, err)
 	require.Equal(t, false, opt)
 }

beacon-chain/forkchoice/doubly-linked-tree/optimistic_sync.go

@@ -2,22 +2,54 @@ package doublylinkedtree
 
 import (
 	"context"
+
+	"github.com/prysmaticlabs/prysm/config/params"
 )
 
+func (s *Store) setOptimisticToInvalid(ctx context.Context, root, payloadHash [32]byte) ([][32]byte, error) {
+	s.nodesLock.Lock()
+	invalidRoots := make([][32]byte, 0)
+	node, ok := s.nodeByRoot[root]
+	if !ok || node == nil {
+		s.nodesLock.Unlock()
+		return invalidRoots, ErrNilNode
+	}
+	// Check if last valid hash is an ancestor of the passed node.
+	lastValid, ok := s.nodeByPayload[payloadHash]
+	if !ok || lastValid == nil {
+		s.nodesLock.Unlock()
+		return invalidRoots, errUnknownPayloadHash
+	}
+	firstInvalid := node
+	for ; firstInvalid.parent != nil && firstInvalid.parent.payloadHash != payloadHash; firstInvalid = firstInvalid.parent {
+		if ctx.Err() != nil {
+			s.nodesLock.Unlock()
+			return invalidRoots, ctx.Err()
+		}
+	}
+	// If the last valid payload is in a different fork, we remove only the
+	// passed node.
+	if firstInvalid.parent == nil {
+		firstInvalid = node
+	}
+	s.nodesLock.Unlock()
+	return s.removeNode(ctx, firstInvalid)
+}
+
 // removeNode removes the node with the given root and all of its children
 // from the Fork Choice Store.
-func (s *Store) removeNode(ctx context.Context, root [32]byte) ([][32]byte, error) {
+func (s *Store) removeNode(ctx context.Context, node *Node) ([][32]byte, error) {
 	s.nodesLock.Lock()
 	defer s.nodesLock.Unlock()
 	invalidRoots := make([][32]byte, 0)
 
-	node, ok := s.nodeByRoot[root]
-	if !ok || node == nil {
+	if node == nil {
 		return invalidRoots, ErrNilNode
 	}
 	if !node.optimistic || node.parent == nil {
 		return invalidRoots, errInvalidOptimisticStatus
 	}
+
 	children := node.parent.children
 	if len(children) == 1 {
 		node.parent.children = []*Node{}
@@ -47,6 +79,16 @@ func (s *Store) removeNodeAndChildren(ctx context.Context, node *Node, invalidRo
 		}
 	}
 	invalidRoots = append(invalidRoots, node.root)
+	s.proposerBoostLock.Lock()
+	if node.root == s.proposerBoostRoot {
+		s.proposerBoostRoot = [32]byte{}
+	}
+	if node.root == s.previousProposerBoostRoot {
+		s.previousProposerBoostRoot = params.BeaconConfig().ZeroHash
+		s.previousProposerBoostScore = 0
+	}
+	s.proposerBoostLock.Unlock()
 	delete(s.nodeByRoot, node.root)
+	delete(s.nodeByPayload, node.payloadHash)
 	return invalidRoots, nil
 }

beacon-chain/forkchoice/doubly-linked-tree/optimistic_sync_test.go

@@ -24,32 +24,71 @@ import (
 func TestPruneInvalid(t *testing.T) {
 	tests := []struct {
 		root             [32]byte // the root of the new INVALID block
+		payload          [32]byte // the last valid hash
 		wantedNodeNumber int
 		wantedRoots      [][32]byte
 	}{
 		{
 			[32]byte{'j'},
+			[32]byte{'B'},
 			12,
 			[][32]byte{[32]byte{'j'}},
 		},
 		{
 			[32]byte{'c'},
+			[32]byte{'B'},
 			4,
 			[][32]byte{[32]byte{'f'}, [32]byte{'e'}, [32]byte{'i'}, [32]byte{'h'}, [32]byte{'l'},
 				[32]byte{'k'}, [32]byte{'g'}, [32]byte{'d'}, [32]byte{'c'}},
 		},
 		{
 			[32]byte{'i'},
+			[32]byte{'H'},
 			12,
 			[][32]byte{[32]byte{'i'}},
 		},
 		{
 			[32]byte{'h'},
+			[32]byte{'G'},
 			11,
 			[][32]byte{[32]byte{'i'}, [32]byte{'h'}},
 		},
 		{
 			[32]byte{'g'},
+			[32]byte{'D'},
+			8,
+			[][32]byte{[32]byte{'i'}, [32]byte{'h'}, [32]byte{'l'}, [32]byte{'k'}, [32]byte{'g'}},
+		},
+		{
+			[32]byte{'i'},
+			[32]byte{'D'},
+			8,
+			[][32]byte{[32]byte{'i'}, [32]byte{'h'}, [32]byte{'l'}, [32]byte{'k'}, [32]byte{'g'}},
+		},
+		{
+			[32]byte{'f'},
+			[32]byte{'D'},
+			11,
+			[][32]byte{[32]byte{'f'}, [32]byte{'e'}},
+		},
+		{
+			[32]byte{'h'},
+			[32]byte{'C'},
+			5,
+			[][32]byte{
+				[32]byte{'f'},
+				[32]byte{'e'},
+				[32]byte{'i'},
+				[32]byte{'h'},
+				[32]byte{'l'},
+				[32]byte{'k'},
+				[32]byte{'g'},
+				[32]byte{'d'},
+			},
+		},
+		{
+			[32]byte{'g'},
+			[32]byte{'E'},
 			8,
 			[][32]byte{[32]byte{'i'}, [32]byte{'h'}, [32]byte{'l'}, [32]byte{'k'}, [32]byte{'g'}},
 		},
@@ -58,22 +97,45 @@ func TestPruneInvalid(t *testing.T) {
 		ctx := context.Background()
 		f := setup(1, 1)
 
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 100, [32]byte{'a'}, params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 101, [32]byte{'b'}, [32]byte{'a'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'c'}, [32]byte{'b'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'j'}, [32]byte{'b'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 103, [32]byte{'d'}, [32]byte{'c'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'e'}, [32]byte{'d'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'g'}, [32]byte{'d'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'f'}, [32]byte{'e'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'h'}, [32]byte{'g'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'k'}, [32]byte{'g'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'i'}, [32]byte{'h'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'l'}, [32]byte{'k'}, params.BeaconConfig().ZeroHash, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 100, [32]byte{'a'}, params.BeaconConfig().ZeroHash, [32]byte{'A'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 101, [32]byte{'b'}, [32]byte{'a'}, [32]byte{'B'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'c'}, [32]byte{'b'}, [32]byte{'C'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'j'}, [32]byte{'b'}, [32]byte{'J'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 103, [32]byte{'d'}, [32]byte{'c'}, [32]byte{'D'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'e'}, [32]byte{'d'}, [32]byte{'E'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'g'}, [32]byte{'d'}, [32]byte{'G'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'f'}, [32]byte{'e'}, [32]byte{'F'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'h'}, [32]byte{'g'}, [32]byte{'H'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'k'}, [32]byte{'g'}, [32]byte{'K'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'i'}, [32]byte{'h'}, [32]byte{'I'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'l'}, [32]byte{'k'}, [32]byte{'L'}, 1, 1))
 
-		roots, err := f.store.removeNode(context.Background(), tc.root)
+		roots, err := f.store.setOptimisticToInvalid(context.Background(), tc.root, tc.payload)
 		require.NoError(t, err)
 		require.DeepEqual(t, tc.wantedRoots, roots)
 		require.Equal(t, tc.wantedNodeNumber, f.NodeCount())
 	}
 }
+
+// This is a regression test (10445)
+func TestSetOptimisticToInvalid_ProposerBoost(t *testing.T) {
+	ctx := context.Background()
+	f := setup(1, 1)
+
+	require.NoError(t, f.InsertOptimisticBlock(ctx, 100, [32]byte{'a'}, params.BeaconConfig().ZeroHash, [32]byte{'A'}, 1, 1))
+	require.NoError(t, f.InsertOptimisticBlock(ctx, 101, [32]byte{'b'}, [32]byte{'a'}, [32]byte{'B'}, 1, 1))
+	require.NoError(t, f.InsertOptimisticBlock(ctx, 101, [32]byte{'c'}, [32]byte{'b'}, [32]byte{'C'}, 1, 1))
+	f.store.proposerBoostLock.Lock()
+	f.store.proposerBoostRoot = [32]byte{'c'}
+	f.store.previousProposerBoostScore = 10
+	f.store.previousProposerBoostRoot = [32]byte{'b'}
+	f.store.proposerBoostLock.Unlock()
+
+	_, err := f.SetOptimisticToInvalid(ctx, [32]byte{'c'}, [32]byte{'A'})
+	require.NoError(t, err)
+	f.store.proposerBoostLock.RLock()
+	require.Equal(t, uint64(0), f.store.previousProposerBoostScore)
+	require.DeepEqual(t, [32]byte{}, f.store.proposerBoostRoot)
+	require.DeepEqual(t, params.BeaconConfig().ZeroHash, f.store.previousProposerBoostRoot)
+	f.store.proposerBoostLock.RUnlock()
+}

beacon-chain/forkchoice/doubly-linked-tree/store.go

@@ -121,6 +121,7 @@ func (s *Store) insert(ctx context.Context,
 		payloadHash:    payloadHash,
 	}
 
+	s.nodeByPayload[payloadHash] = n
 	s.nodeByRoot[root] = n
 	if parent != nil {
 		parent.children = append(parent.children, n)

beacon-chain/forkchoice/doubly-linked-tree/store_test.go

@@ -107,7 +107,8 @@ func TestStore_Insert(t *testing.T) {
 	// The new node does not have a parent.
 	treeRootNode := &Node{slot: 0, root: indexToHash(0)}
 	nodeByRoot := map[[32]byte]*Node{indexToHash(0): treeRootNode}
-	s := &Store{nodeByRoot: nodeByRoot, treeRootNode: treeRootNode}
+	nodeByPayload := map[[32]byte]*Node{indexToHash(0): treeRootNode}
+	s := &Store{nodeByRoot: nodeByRoot, treeRootNode: treeRootNode, nodeByPayload: nodeByPayload}
 	payloadHash := [32]byte{'a'}
 	require.NoError(t, s.insert(context.Background(), 100, indexToHash(100), indexToHash(0), payloadHash, 1, 1))
 	assert.Equal(t, 2, len(s.nodeByRoot), "Did not insert block")

beacon-chain/forkchoice/doubly-linked-tree/types.go

@@ -26,6 +26,7 @@ type Store struct {
 	treeRootNode               *Node                                  // the root node of the store tree.
 	headNode                   *Node                                  // last head Node
 	nodeByRoot                 map[[fieldparams.RootLength]byte]*Node // nodes indexed by roots.
+	nodeByPayload              map[[fieldparams.RootLength]byte]*Node // nodes indexed by payload Hash
 	nodesLock                  sync.RWMutex
 	proposerBoostLock          sync.RWMutex
 }

beacon-chain/forkchoice/interfaces.go

@@ -24,7 +24,7 @@ type ForkChoicer interface {
 type HeadRetriever interface {
 	Head(context.Context, types.Epoch, [32]byte, []uint64, types.Epoch) ([32]byte, error)
 	Tips() ([][32]byte, []types.Slot)
-	IsOptimistic(ctx context.Context, root [32]byte) (bool, error)
+	IsOptimistic(root [32]byte) (bool, error)
 }
 
 // BlockProcessor processes the block that's used for accounting fork choice.
@@ -71,5 +71,5 @@ type Getter interface {
 // Setter allows to set forkchoice information
 type Setter interface {
 	SetOptimisticToValid(context.Context, [fieldparams.RootLength]byte) error
-	SetOptimisticToInvalid(context.Context, [fieldparams.RootLength]byte) ([][32]byte, error)
+	SetOptimisticToInvalid(context.Context, [fieldparams.RootLength]byte, [fieldparams.RootLength]byte) ([][32]byte, error)
 }

beacon-chain/forkchoice/protoarray/BUILD.bazel

@@ -22,12 +22,14 @@ go_library(
         "//beacon-chain/forkchoice/types:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
+        "//encoding/bytesutil:go_default_library",
         "//math:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
         "@com_github_prysmaticlabs_eth2_types//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
         "@io_opencensus_go//trace:go_default_library",
     ],
 )

beacon-chain/forkchoice/protoarray/errors.go

@@ -5,11 +5,11 @@ import "errors"
 var errUnknownFinalizedRoot = errors.New("unknown finalized root")
 var errUnknownJustifiedRoot = errors.New("unknown justified root")
 var errInvalidNodeIndex = errors.New("node index is invalid")
+var errInvalidFinalizedNode = errors.New("invalid finalized block on chain")
 var ErrUnknownNodeRoot = errors.New("unknown block root")
 var errInvalidJustifiedIndex = errors.New("justified index is invalid")
-var errInvalidBestChildIndex = errors.New("best child index is invalid")
 var errInvalidBestDescendantIndex = errors.New("best descendant index is invalid")
 var errInvalidParentDelta = errors.New("parent delta is invalid")
 var errInvalidNodeDelta = errors.New("node delta is invalid")
 var errInvalidDeltaLength = errors.New("delta length is invalid")
-var errInvalidSyncedTips = errors.New("invalid synced tips")
+var errInvalidOptimisticStatus = errors.New("invalid optimistic status")

beacon-chain/forkchoice/protoarray/helpers.go

@@ -85,12 +85,9 @@ func copyNode(node *Node) *Node {
 		return &Node{}
 	}
 
-	copiedRoot := [32]byte{}
-	copy(copiedRoot[:], node.root[:])
-
 	return &Node{
 		slot:           node.slot,
-		root:           copiedRoot,
+		root:           node.root,
 		parent:         node.parent,
 		payloadHash:    node.payloadHash,
 		justifiedEpoch: node.justifiedEpoch,
@@ -98,5 +95,6 @@ func copyNode(node *Node) *Node {
 		weight:         node.weight,
 		bestChild:      node.bestChild,
 		bestDescendant: node.bestDescendant,
+		status:         node.status,
 	}
 }

beacon-chain/forkchoice/protoarray/metrics.go

@@ -3,9 +3,12 @@ package protoarray
 import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/sirupsen/logrus"
 )
 
 var (
+	log = logrus.WithField("prefix", "forkchoice-protoarray")
+
 	headSlotNumber = promauto.NewGauge(
 		prometheus.GaugeOpts{
 			Name: "proto_array_head_slot",
@@ -48,16 +51,4 @@ var (
 			Help: "The number of times pruning happened.",
 		},
 	)
-	lastSyncedTipSlot = promauto.NewGauge(
-		prometheus.GaugeOpts{
-			Name: "proto_array_last_synced_tip_slot",
-			Help: "The slot of the last fully validated block added to the proto array.",
-		},
-	)
-	syncedTipsCount = promauto.NewGauge(
-		prometheus.GaugeOpts{
-			Name: "proto_array_synced_tips_count",
-			Help: "The number of elements in the syncedTips structure.",
-		},
-	)
 )

beacon-chain/forkchoice/protoarray/node.go

@@ -43,8 +43,3 @@ func (n *Node) BestChild() uint64 {
 func (n *Node) BestDescendant() uint64 {
 	return n.bestDescendant
 }
-
-// Graffiti of the fork choice node.
-func (n *Node) Graffiti() [32]byte {
-	return n.graffiti
-}

beacon-chain/forkchoice/protoarray/node_test.go

@@ -16,7 +16,6 @@ func TestNode_Getters(t *testing.T) {
 	weight := uint64(10000)
 	bestChild := uint64(5)
 	bestDescendant := uint64(4)
-	graffiti := [32]byte{'b'}
 	n := &Node{
 		slot:           slot,
 		root:           root,
@@ -26,7 +25,6 @@ func TestNode_Getters(t *testing.T) {
 		weight:         weight,
 		bestChild:      bestChild,
 		bestDescendant: bestDescendant,
-		graffiti:       graffiti,
 	}
 
 	require.Equal(t, slot, n.Slot())
@@ -37,5 +35,4 @@ func TestNode_Getters(t *testing.T) {
 	require.Equal(t, weight, n.Weight())
 	require.Equal(t, bestChild, n.BestChild())
 	require.Equal(t, bestDescendant, n.BestDescendant())
-	require.Equal(t, graffiti, n.Graffiti())
 }

beacon-chain/forkchoice/protoarray/optimistic_sync.go

@@ -3,314 +3,159 @@ package protoarray
 import (
 	"context"
 
-	types "github.com/prysmaticlabs/eth2-types"
 	"github.com/prysmaticlabs/prysm/config/params"
 )
 
-// This returns the minimum and maximum slot of the synced_tips tree
-func (f *ForkChoice) boundarySyncedTips() (types.Slot, types.Slot) {
-	f.syncedTips.RLock()
-	defer f.syncedTips.RUnlock()
-
-	min := params.BeaconConfig().FarFutureSlot
-	max := types.Slot(0)
-	for _, slot := range f.syncedTips.validatedTips {
-		if slot > max {
-			max = slot
-		}
-		if slot < min {
-			min = slot
-		}
-	}
-	return min, max
-}
-
 // IsOptimistic returns true if this node is optimistically synced
 // A optimistically synced block is synced as usual, but its
 // execution payload is not validated, while the EL is still syncing.
 // This function returns an error if the block is not found in the fork choice
 // store
-func (f *ForkChoice) IsOptimistic(ctx context.Context, root [32]byte) (bool, error) {
-	if ctx.Err() != nil {
-		return false, ctx.Err()
-	}
+func (f *ForkChoice) IsOptimistic(root [32]byte) (bool, error) {
 	f.store.nodesLock.RLock()
+	defer f.store.nodesLock.RUnlock()
 	index, ok := f.store.nodesIndices[root]
 	if !ok {
-		f.store.nodesLock.RUnlock()
 		return false, ErrUnknownNodeRoot
 	}
 	node := f.store.nodes[index]
-	slot := node.slot
-
-	// If the node is a synced tip, then it's fully validated
-	f.syncedTips.RLock()
-	_, ok = f.syncedTips.validatedTips[root]
-	if ok {
-		f.syncedTips.RUnlock()
-		f.store.nodesLock.RUnlock()
-		return false, nil
-	}
-	f.syncedTips.RUnlock()
-
-	// If the slot is higher than the max synced tip, it's optimistic
-	min, max := f.boundarySyncedTips()
-	if slot > max {
-		f.store.nodesLock.RUnlock()
-		return true, nil
-	}
-
-	// If the slot is lower than the min synced tip, it's fully validated
-	if slot <= min {
-		f.store.nodesLock.RUnlock()
-		return false, nil
-	}
-
-	// if the node is a leaf of the Fork Choice tree, then it's
-	// optimistic
-	childIndex := node.BestChild()
-	if childIndex == NonExistentNode {
-		f.store.nodesLock.RUnlock()
-		return true, nil
-	}
-
-	// recurse to the child
-	child := f.store.nodes[childIndex]
-	root = child.root
-	f.store.nodesLock.RUnlock()
-	return f.IsOptimistic(ctx, root)
-}
-
-// This function returns the index of sync tip node that's ancestor to the input node.
-// In the event of none, `NonExistentNode` is returned.
-// This internal method assumes the caller holds a lock on syncedTips and s.nodesLock
-func (s *Store) findSyncedTip(ctx context.Context, node *Node, syncedTips *optimisticStore) (uint64, error) {
-	for {
-		if ctx.Err() != nil {
-			return 0, ctx.Err()
-		}
-		if _, ok := syncedTips.validatedTips[node.root]; ok {
-			return s.nodesIndices[node.root], nil
-		}
-		if node.parent == NonExistentNode {
-			return NonExistentNode, nil
-		}
-		node = s.nodes[node.parent]
-	}
+	return node.status == syncing, nil
 }
 
 // SetOptimisticToValid is called with the root of a block that was returned as
-// VALID by the EL. This routine recomputes and updates the synced_tips map to
-// account for this new tip.
-// WARNING: This method returns an error if the root is not found in forkchoice or
-// if the root is not a leaf of the fork choice tree.
+// VALID by the EL.
+// WARNING: This method returns an error if the root is not found in forkchoice
 func (f *ForkChoice) SetOptimisticToValid(ctx context.Context, root [32]byte) error {
-	f.store.nodesLock.RLock()
+	f.store.nodesLock.Lock()
+	defer f.store.nodesLock.Unlock()
 	// We can only update if given root is in Fork Choice
 	index, ok := f.store.nodesIndices[root]
 	if !ok {
-		return errInvalidNodeIndex
+		return ErrUnknownNodeRoot
 	}
-	node := f.store.nodes[index]
-	f.store.nodesLock.RUnlock()
 
-	// Stop early if the node is Valid
-	optimistic, err := f.IsOptimistic(ctx, root)
-	if err != nil {
-		return err
-	}
-	if !optimistic {
-		return nil
-	}
-	f.store.nodesLock.RLock()
-	defer f.store.nodesLock.RUnlock()
-
-	// Cache root and slot to validated tips
-	newTips := make(map[[32]byte]types.Slot)
-	newValidSlot := node.slot
-	newTips[root] = newValidSlot
-
-	// Compute the full valid path from the given node to its previous synced tip
-	// This path will now consist of fully validated blocks. Notice that
-	// the previous tip may have been outside the Fork Choice store.
-	// In this case, only one block can be in syncedTips as the whole
-	// Fork Choice would be a descendant of this block.
-	validPath := make(map[uint64]bool)
-	validPath[index] = true
-	for {
+	for node := f.store.nodes[index]; node.status == syncing; node = f.store.nodes[index] {
 		if ctx.Err() != nil {
 			return ctx.Err()
 		}
-
-		parentIndex := node.parent
-		if parentIndex == NonExistentNode {
+		node.status = valid
+		index = node.parent
+		if index == NonExistentNode {
 			break
 		}
-		if parentIndex >= uint64(len(f.store.nodes)) {
-			return errInvalidNodeIndex
-		}
-		node = f.store.nodes[parentIndex]
-		_, ok = f.syncedTips.validatedTips[node.root]
-		if ok {
-			break
-		}
-		validPath[parentIndex] = true
 	}
-
-	// Retrieve the list of leaves in the Fork Choice
-	// These are all the nodes that have NonExistentNode as best child.
-	leaves, err := f.store.leaves()
-	if err != nil {
-		return err
-	}
-
-	// For each leaf, recompute the new tip.
-	for _, i := range leaves {
-		node = f.store.nodes[i]
-		j := i
-		for {
-			if ctx.Err() != nil {
-				return ctx.Err()
-			}
-			// Stop if we reached the previous tip
-			_, ok = f.syncedTips.validatedTips[node.root]
-			if ok {
-				newTips[node.root] = node.slot
-				break
-			}
-
-			// Stop if we reach valid path
-			_, ok = validPath[j]
-			if ok {
-				newTips[node.root] = node.slot
-				break
-			}
-
-			j = node.parent
-			if j == NonExistentNode {
-				break
-			}
-			if j >= uint64(len(f.store.nodes)) {
-				return errInvalidNodeIndex
-			}
-			node = f.store.nodes[j]
-		}
-	}
-
-	f.syncedTips.validatedTips = newTips
-	lastSyncedTipSlot.Set(float64(newValidSlot))
-	syncedTipsCount.Set(float64(len(newTips)))
 	return nil
 }
 
-//  SetOptimisticToInvalid updates the synced_tips map when the block with the given root becomes INVALID.
-func (f *ForkChoice) SetOptimisticToInvalid(ctx context.Context, root [32]byte) ([][32]byte, error) {
+// SetOptimisticToInvalid updates the synced_tips map when the block with the given root becomes INVALID.
+// It takes two parameters: the root of the INVALID block and the payload Hash
+// of the last valid block.s
+func (f *ForkChoice) SetOptimisticToInvalid(ctx context.Context, root, payloadHash [32]byte) ([][32]byte, error) {
 	f.store.nodesLock.Lock()
 	defer f.store.nodesLock.Unlock()
 	invalidRoots := make([][32]byte, 0)
-	idx, ok := f.store.nodesIndices[root]
+	// We only support setting invalid a node existing in Forkchoice
+	invalidIndex, ok := f.store.nodesIndices[root]
 	if !ok {
-		return invalidRoots, errInvalidNodeIndex
+		return invalidRoots, ErrUnknownNodeRoot
 	}
-	node := f.store.nodes[idx]
-	// We only support changing status for the tips in Fork Choice store.
-	if node.bestChild != NonExistentNode {
-		return invalidRoots, errInvalidNodeIndex
+	node := f.store.nodes[invalidIndex]
+
+	lastValidIndex, ok := f.store.payloadIndices[payloadHash]
+	if !ok || lastValidIndex == NonExistentNode {
+		return invalidRoots, errInvalidFinalizedNode
 	}
 
-	parentIndex := node.parent
-	// This should not happen
-	if parentIndex == NonExistentNode {
-		return invalidRoots, errInvalidNodeIndex
+	// Check if last valid hash is an ancestor of the passed node
+	firstInvalidIndex := node.parent
+	for ; firstInvalidIndex != NonExistentNode && firstInvalidIndex != lastValidIndex; firstInvalidIndex = node.parent {
+		node = f.store.nodes[firstInvalidIndex]
 	}
-	// Update the weights of the nodes subtracting the INVALID node's weight
+
+	// if the last valid hash is not an ancestor of the invalid block, we
+	// just remove the invalid block.
+	if node.parent != lastValidIndex {
+		node = f.store.nodes[invalidIndex]
+		firstInvalidIndex = invalidIndex
+		lastValidIndex = node.parent
+		if lastValidIndex == NonExistentNode {
+			return invalidRoots, errInvalidFinalizedNode
+		}
+	} else {
+		firstInvalidIndex = f.store.nodesIndices[node.root]
+	}
+
+	// Update the weights of the nodes subtracting the first INVALID node's weight
 	weight := node.weight
-	node = f.store.nodes[parentIndex]
-	for {
-		if ctx.Err() != nil {
-			return invalidRoots, ctx.Err()
-		}
-		node.weight -= weight
-		if node.parent == NonExistentNode {
-			break
-		}
-		node = f.store.nodes[node.parent]
-	}
-	parent := copyNode(f.store.nodes[parentIndex])
-
-	// delete the invalid node, order is important
-	f.store.nodes = append(f.store.nodes[:idx], f.store.nodes[idx+1:]...)
-	delete(f.store.nodesIndices, root)
-	invalidRoots = append(invalidRoots, root)
-	// Fix parent and best child for each node
-	for _, node := range f.store.nodes {
-		if node.parent == NonExistentNode {
-			node.parent = NonExistentNode
-		} else if node.parent > idx {
-			node.parent -= 1
-		}
-		if node.bestChild == NonExistentNode || node.bestChild == idx {
-			node.bestChild = NonExistentNode
-		} else if node.bestChild > idx {
-			node.bestChild -= 1
-		}
-		if node.bestDescendant == NonExistentNode || node.bestDescendant == idx {
-			node.bestDescendant = NonExistentNode
-		} else if node.bestDescendant > idx {
-			node.bestDescendant -= 1
-		}
+	var validNode *Node
+	for index := lastValidIndex; index != NonExistentNode; index = validNode.parent {
+		validNode = f.store.nodes[index]
+		validNode.weight -= weight
 	}
 
-	// Update the parent's best child and best descendant if necessary.
-	if parent.bestChild == idx || parent.bestDescendant == idx {
-		for childIndex, child := range f.store.nodes {
-			if child.parent == parentIndex {
-				err := f.store.updateBestChildAndDescendant(
-					parentIndex, uint64(childIndex))
-				if err != nil {
-					return invalidRoots, err
-				}
-				break
+	// Find the current proposer boost (it should be set to zero if an
+	// INVALID block was boosted)
+	f.store.proposerBoostLock.RLock()
+	boostRoot := f.store.proposerBoostRoot
+	previousBoostRoot := f.store.previousProposerBoostRoot
+	f.store.proposerBoostLock.RUnlock()
+
+	// Remove the invalid roots from our store maps and adjust their weight
+	// to zero
+	boosted := node.root == boostRoot
+	previouslyBoosted := node.root == previousBoostRoot
+
+	invalidIndices := map[uint64]bool{firstInvalidIndex: true}
+	node.status = invalid
+	node.weight = 0
+	delete(f.store.nodesIndices, node.root)
+	delete(f.store.canonicalNodes, node.root)
+	delete(f.store.payloadIndices, node.payloadHash)
+	for index := firstInvalidIndex + 1; index < uint64(len(f.store.nodes)); index++ {
+		invalidNode := f.store.nodes[index]
+		if _, ok := invalidIndices[invalidNode.parent]; !ok {
+			continue
+		}
+		if invalidNode.status == valid {
+			return invalidRoots, errInvalidOptimisticStatus
+		}
+		if !boosted && invalidNode.root == boostRoot {
+			boosted = true
+		}
+		if !previouslyBoosted && invalidNode.root == previousBoostRoot {
+			previouslyBoosted = true
+		}
+		invalidNode.status = invalid
+		invalidIndices[index] = true
+		invalidNode.weight = 0
+		delete(f.store.nodesIndices, invalidNode.root)
+		delete(f.store.canonicalNodes, invalidNode.root)
+		delete(f.store.payloadIndices, invalidNode.payloadHash)
+	}
+	if boosted {
+		if err := f.ResetBoostedProposerRoot(ctx); err != nil {
+			return invalidRoots, err
+		}
+	}
+	if previouslyBoosted {
+		f.store.proposerBoostLock.Lock()
+		f.store.previousProposerBoostRoot = params.BeaconConfig().ZeroHash
+		f.store.previousProposerBoostScore = 0
+		f.store.proposerBoostLock.Unlock()
+	}
+
+	for index := range invalidIndices {
+		invalidRoots = append(invalidRoots, f.store.nodes[index].root)
+	}
+
+	// Update the best child and descendant
+	for i := len(f.store.nodes) - 1; i >= 0; i-- {
+		n := f.store.nodes[i]
+		if n.parent != NonExistentNode {
+			if err := f.store.updateBestChildAndDescendant(n.parent, uint64(i)); err != nil {
+				return invalidRoots, err
 			}
 		}
 	}
-
-	// Return early if the parent is not a synced_tip.
-	f.syncedTips.Lock()
-	defer f.syncedTips.Unlock()
-	parentRoot := parent.root
-	_, ok = f.syncedTips.validatedTips[parentRoot]
-	if !ok {
-		return invalidRoots, nil
-	}
-
-	leaves, err := f.store.leaves()
-	if err != nil {
-		return invalidRoots, err
-	}
-
-	for _, i := range leaves {
-		node = f.store.nodes[i]
-		for {
-			if ctx.Err() != nil {
-				return invalidRoots, ctx.Err()
-			}
-
-			// Return early if the parent is still a synced tip
-			if node.root == parentRoot {
-				return invalidRoots, nil
-			}
-			_, ok = f.syncedTips.validatedTips[node.root]
-			if ok {
-				break
-			}
-			if node.parent == NonExistentNode {
-				break
-			}
-			node = f.store.nodes[node.parent]
-		}
-	}
-	delete(f.syncedTips.validatedTips, parentRoot)
-	syncedTipsCount.Set(float64(len(f.syncedTips.validatedTips)))
 	return invalidRoots, nil
 }

beacon-chain/forkchoice/protoarray/optimistic_sync_test.go

@@ -10,116 +10,38 @@ import (
 	"github.com/prysmaticlabs/prysm/testing/require"
 )
 
-// We test the algorithm to check the optimistic status of a node. The
-// status for this test is the following branching diagram
-//
-//                       -- E -- F
-//                      /
-//                  -- C -- D
-//                 /
-// 0 -- 1 -- A -- B      -- J -- K
-//                 \    /
-//                  -- G -- H -- I
-//
-// Here nodes 0, 1, A, B, C, D are fully validated and nodes
-// E, F, G, H, J, K are optimistic.
-// Synced Tips are nodes B, C, D
-// nodes 0 and 1 are outside the Fork Choice Store.
+func slicesEqual(a, b [][32]byte) bool {
+	if len(a) != len(b) {
+		return false
+	}
 
-func TestOptimistic(t *testing.T) {
+	mapA := make(map[[32]byte]bool, len(a))
+	for _, root := range a {
+		mapA[root] = true
+	}
+	for _, root := range b {
+		_, ok := mapA[root]
+		if !ok {
+			return false
+		}
+	}
+	return true
+}
+
+func TestOptimistic_Outside_ForkChoice(t *testing.T) {
 	root0 := bytesutil.ToBytes32([]byte("hello0"))
-	root1 := bytesutil.ToBytes32([]byte("hello1"))
 
 	nodeA := &Node{
 		slot:      types.Slot(100),
 		root:      bytesutil.ToBytes32([]byte("helloA")),
 		bestChild: 1,
-	}
-	nodeB := &Node{
-		slot:      types.Slot(101),
-		root:      bytesutil.ToBytes32([]byte("helloB")),
-		bestChild: 2,
-		parent:    0,
-	}
-	nodeC := &Node{
-		slot:      types.Slot(102),
-		root:      bytesutil.ToBytes32([]byte("helloC")),
-		bestChild: 3,
-		parent:    1,
-	}
-	nodeD := &Node{
-		slot:      types.Slot(103),
-		root:      bytesutil.ToBytes32([]byte("helloD")),
-		bestChild: NonExistentNode,
-		parent:    2,
-	}
-	nodeE := &Node{
-		slot:      types.Slot(103),
-		root:      bytesutil.ToBytes32([]byte("helloE")),
-		bestChild: 5,
-		parent:    2,
-	}
-	nodeF := &Node{
-		slot:      types.Slot(104),
-		root:      bytesutil.ToBytes32([]byte("helloF")),
-		bestChild: NonExistentNode,
-		parent:    4,
-	}
-	nodeG := &Node{
-		slot:      types.Slot(102),
-		root:      bytesutil.ToBytes32([]byte("helloG")),
-		bestChild: 7,
-		parent:    1,
-	}
-	nodeH := &Node{
-		slot:      types.Slot(103),
-		root:      bytesutil.ToBytes32([]byte("helloH")),
-		bestChild: 8,
-		parent:    6,
-	}
-	nodeI := &Node{
-		slot:      types.Slot(104),
-		root:      bytesutil.ToBytes32([]byte("helloI")),
-		bestChild: NonExistentNode,
-		parent:    7,
-	}
-	nodeJ := &Node{
-		slot:      types.Slot(103),
-		root:      bytesutil.ToBytes32([]byte("helloJ")),
-		bestChild: 10,
-		parent:    6,
-	}
-	nodeK := &Node{
-		slot:      types.Slot(104),
-		root:      bytesutil.ToBytes32([]byte("helloK")),
-		bestChild: NonExistentNode,
-		parent:    9,
+		status:    valid,
 	}
 	nodes := []*Node{
 		nodeA,
-		nodeB,
-		nodeC,
-		nodeD,
-		nodeE,
-		nodeF,
-		nodeG,
-		nodeH,
-		nodeI,
-		nodeJ,
-		nodeK,
 	}
 	ni := map[[32]byte]uint64{
 		nodeA.root: 0,
-		nodeB.root: 1,
-		nodeC.root: 2,
-		nodeD.root: 3,
-		nodeE.root: 4,
-		nodeF.root: 5,
-		nodeG.root: 6,
-		nodeH.root: 7,
-		nodeI.root: 8,
-		nodeJ.root: 9,
-		nodeK.root: 10,
 	}
 
 	s := &Store{
@@ -127,82 +49,14 @@ func TestOptimistic(t *testing.T) {
 		nodesIndices: ni,
 	}
 
-	tips := map[[32]byte]types.Slot{
-		nodeB.root: nodeB.slot,
-		nodeC.root: nodeC.slot,
-		nodeD.root: nodeD.slot,
-	}
-	st := &optimisticStore{
-		validatedTips: tips,
-	}
 	f := &ForkChoice{
-		store:      s,
-		syncedTips: st,
+		store: s,
 	}
-	ctx := context.Background()
-	// We test the implementation of boundarySyncedTips
-	min, max := f.boundarySyncedTips()
-	require.Equal(t, min, types.Slot(101), "minimum tip slot is different")
-	require.Equal(t, max, types.Slot(103), "maximum tip slot is different")
-
-	// We test first nodes outside the Fork Choice store
-	_, err := f.IsOptimistic(ctx, root0)
+	_, err := f.IsOptimistic(root0)
 	require.ErrorIs(t, ErrUnknownNodeRoot, err)
-
-	_, err = f.IsOptimistic(ctx, root1)
-	require.ErrorIs(t, ErrUnknownNodeRoot, err)
-
-	// We check all nodes in the Fork Choice store.
-	op, err := f.IsOptimistic(ctx, nodeA.root)
-	require.NoError(t, err)
-	require.Equal(t, op, false)
-
-	op, err = f.IsOptimistic(ctx, nodeB.root)
-	require.NoError(t, err)
-	require.Equal(t, op, false)
-
-	op, err = f.IsOptimistic(ctx, nodeC.root)
-	require.NoError(t, err)
-	require.Equal(t, op, false)
-
-	op, err = f.IsOptimistic(ctx, nodeD.root)
-	require.NoError(t, err)
-	require.Equal(t, op, false)
-
-	op, err = f.IsOptimistic(ctx, nodeE.root)
-	require.NoError(t, err)
-	require.Equal(t, op, true)
-
-	op, err = f.IsOptimistic(ctx, nodeF.root)
-	require.NoError(t, err)
-	require.Equal(t, op, true)
-
-	op, err = f.IsOptimistic(ctx, nodeG.root)
-	require.NoError(t, err)
-	require.Equal(t, op, true)
-
-	op, err = f.IsOptimistic(ctx, nodeH.root)
-	require.NoError(t, err)
-	require.Equal(t, op, true)
-
-	op, err = f.IsOptimistic(ctx, nodeI.root)
-	require.NoError(t, err)
-	require.Equal(t, op, true)
-
-	op, err = f.IsOptimistic(ctx, nodeJ.root)
-	require.NoError(t, err)
-	require.Equal(t, op, true)
-
-	op, err = f.IsOptimistic(ctx, nodeK.root)
-	require.NoError(t, err)
-	require.Equal(t, op, true)
-
-	// request a write Lock to synced Tips regression #10289
-	f.syncedTips.Lock()
-	defer f.syncedTips.Unlock()
 }
 
-// This tests the algorithm to update syncedTips
+// This tests the algorithm to update optimistic Status
 // We start with the following diagram
 //
 //                E -- F
@@ -213,165 +67,105 @@ func TestOptimistic(t *testing.T) {
 //        \        \
 //         J        -- K -- L
 //
-// And every block in the Fork choice is optimistic. Synced_Tips contains a
-// single block that is outside of Fork choice
+// The Chain A -- B -- C -- D -- E is VALID.
 //
 func TestSetOptimisticToValid(t *testing.T) {
 	ctx := context.Background()
-	f := setup(1, 1)
-
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 100, [32]byte{'a'}, params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 101, [32]byte{'b'}, [32]byte{'a'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'c'}, [32]byte{'b'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'j'}, [32]byte{'b'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 103, [32]byte{'d'}, [32]byte{'c'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'e'}, [32]byte{'d'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'g'}, [32]byte{'d'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'f'}, [32]byte{'e'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'h'}, [32]byte{'g'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'k'}, [32]byte{'g'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'i'}, [32]byte{'h'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'l'}, [32]byte{'k'}, params.BeaconConfig().ZeroHash, 1, 1))
 	tests := []struct {
-		root      [32]byte                // the root of the new VALID block
-		tips      map[[32]byte]types.Slot // the old synced tips
-		newTips   map[[32]byte]types.Slot // the updated synced tips
-		wantedErr error
+		root             [32]byte // the root of the new VALID block
+		testRoot         [32]byte // root of the node we will test optimistic status
+		wantedOptimistic bool     // wanted optimistic status for tested node
+		wantedErr        error    // wanted error message
 	}{
 		{
 			[32]byte{'i'},
-			map[[32]byte]types.Slot{[32]byte{'z'}: 90},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'d'}: 103,
-				[32]byte{'g'}: 104,
-				[32]byte{'i'}: 106,
-			},
+			[32]byte{'i'},
+			false,
 			nil,
 		},
 		{
 			[32]byte{'i'},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'d'}: 103,
-			},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'d'}: 103,
-				[32]byte{'g'}: 104,
-				[32]byte{'i'}: 106,
-			},
+			[32]byte{'f'},
+			true,
 			nil,
 		},
 		{
 			[32]byte{'i'},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'d'}: 103,
-				[32]byte{'e'}: 103,
-			},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'e'}: 104,
-				[32]byte{'g'}: 104,
-				[32]byte{'i'}: 106,
-			},
-			nil,
-		},
-		{
-			[32]byte{'j'},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'f'}: 105,
-				[32]byte{'g'}: 104,
-				[32]byte{'i'}: 106,
-			},
-			map[[32]byte]types.Slot{
-				[32]byte{'f'}: 105,
-				[32]byte{'g'}: 104,
-				[32]byte{'i'}: 106,
-				[32]byte{'j'}: 102,
-			},
-			nil,
-		},
-		{
-			[32]byte{'g'},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'f'}: 105,
-				[32]byte{'g'}: 104,
-				[32]byte{'i'}: 106,
-			},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'f'}: 105,
-				[32]byte{'g'}: 104,
-				[32]byte{'i'}: 106,
-			},
+			[32]byte{'b'},
+			false,
 			nil,
 		},
 		{
+			[32]byte{'i'},
 			[32]byte{'h'},
-			map[[32]byte]types.Slot{
-				[32]byte{'z'}: 90,
-			},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'d'}: 103,
-				[32]byte{'g'}: 104,
-				[32]byte{'h'}: 105,
-			},
+			false,
 			nil,
 		},
 		{
+			[32]byte{'b'},
+			[32]byte{'b'},
+			false,
+			nil,
+		},
+		{
+			[32]byte{'b'},
 			[32]byte{'h'},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'d'}: 103,
-				[32]byte{'g'}: 104,
-				[32]byte{'i'}: 106,
-			},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'d'}: 103,
-				[32]byte{'g'}: 104,
-				[32]byte{'i'}: 106,
-			},
+			true,
 			nil,
 		},
 		{
-			[32]byte{'g'},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'d'}: 103,
-				[32]byte{'e'}: 104,
-			},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'e'}: 104,
-				[32]byte{'g'}: 104,
-			},
+			[32]byte{'b'},
+			[32]byte{'a'},
+			false,
+			nil,
+		},
+		{
+			[32]byte{'k'},
+			[32]byte{'k'},
+			false,
+			nil,
+		},
+		{
+			[32]byte{'k'},
+			[32]byte{'l'},
+			true,
 			nil,
 		},
 		{
 			[32]byte{'p'},
-			map[[32]byte]types.Slot{},
-			map[[32]byte]types.Slot{},
-			errInvalidNodeIndex,
+			[32]byte{},
+			false,
+			ErrUnknownNodeRoot,
 		},
 	}
 	for _, tc := range tests {
-		f.syncedTips.Lock()
-		f.syncedTips.validatedTips = tc.tips
-		f.syncedTips.Unlock()
-		err := f.SetOptimisticToValid(context.Background(), tc.root)
+		f := setup(1, 1)
+
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 100, [32]byte{'a'}, params.BeaconConfig().ZeroHash, [32]byte{'A'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 101, [32]byte{'b'}, [32]byte{'a'}, [32]byte{'B'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'c'}, [32]byte{'b'}, [32]byte{'C'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'j'}, [32]byte{'b'}, [32]byte{'J'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 103, [32]byte{'d'}, [32]byte{'c'}, [32]byte{'D'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'e'}, [32]byte{'d'}, [32]byte{'E'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'g'}, [32]byte{'d'}, [32]byte{'G'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'f'}, [32]byte{'e'}, [32]byte{'F'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'h'}, [32]byte{'g'}, [32]byte{'H'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'k'}, [32]byte{'g'}, [32]byte{'K'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'i'}, [32]byte{'h'}, [32]byte{'I'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'l'}, [32]byte{'k'}, [32]byte{'L'}, 1, 1))
+		require.NoError(t, f.SetOptimisticToValid(context.Background(), [32]byte{'e'}))
+		optimistic, err := f.IsOptimistic([32]byte{'b'})
+		require.NoError(t, err)
+		require.Equal(t, false, optimistic)
+
+		err = f.SetOptimisticToValid(context.Background(), tc.root)
 		if tc.wantedErr != nil {
 			require.ErrorIs(t, err, tc.wantedErr)
 		} else {
 			require.NoError(t, err)
-			f.syncedTips.RLock()
-			require.DeepEqual(t, f.syncedTips.validatedTips, tc.newTips)
-			f.syncedTips.RUnlock()
+			optimistic, err := f.IsOptimistic(tc.testRoot)
+			require.NoError(t, err)
+			require.Equal(t, tc.wantedOptimistic, optimistic)
 		}
 	}
 }
@@ -387,70 +181,81 @@ func TestSetOptimisticToValid(t *testing.T) {
 //            \               \
 //             J(1)             -- K(1) -- L(0)
 //
-// And every block in the Fork choice is optimistic. Synced_Tips contains a
-// single block that is outside of Fork choice. The numbers in parentheses are
-// the weights of the nodes before removal
+// And the chain A -- B -- C -- D -- E has been fully validated. The numbers in parentheses are
+// the weights of the nodes.
 //
 func TestSetOptimisticToInvalid(t *testing.T) {
 	tests := []struct {
-		root              [32]byte                // the root of the new INVALID block
-		tips              map[[32]byte]types.Slot // the old synced tips
-		wantedParentTip   bool
+		name              string   // test description
+		root              [32]byte // the root of the new INVALID block
+		payload           [32]byte // the payload of the last valid hash
 		newBestChild      uint64
 		newBestDescendant uint64
 		newParentWeight   uint64
 		returnedRoots     [][32]byte
 	}{
 		{
+			"Remove tip, parent was valid",
 			[32]byte{'j'},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'d'}: 103,
-				[32]byte{'g'}: 104,
-			},
-			false,
+			[32]byte{'B'},
 			3,
-			4,
-			8,
-			[][32]byte{[32]byte{'j'}},
-		},
-		{
-			[32]byte{'j'},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-			},
-			true,
-			3,
-			4,
+			12,
 			8,
 			[][32]byte{[32]byte{'j'}},
 		},
 		{
+			"Remove tip, parent was optimistic",
 			[32]byte{'i'},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'d'}: 103,
-				[32]byte{'g'}: 104,
-				[32]byte{'h'}: 105,
-			},
-			true,
+			[32]byte{'H'},
 			NonExistentNode,
 			NonExistentNode,
 			1,
 			[][32]byte{[32]byte{'i'}},
 		},
 		{
+			"Remove tip, lvh is inner and valid",
 			[32]byte{'i'},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'d'}: 103,
-				[32]byte{'g'}: 104,
-			},
-			false,
+			[32]byte{'D'},
+			6,
+			8,
+			3,
+			[][32]byte{[32]byte{'g'}, [32]byte{'h'}, [32]byte{'k'}, [32]byte{'i'}, [32]byte{'l'}},
+		},
+		{
+			"Remove inner, lvh is inner and optimistic",
+			[32]byte{'h'},
+			[32]byte{'G'},
+			10,
+			12,
+			2,
+			[][32]byte{[32]byte{'h'}, [32]byte{'i'}},
+		},
+		{
+			"Remove tip, lvh is inner and optimistic",
+			[32]byte{'l'},
+			[32]byte{'G'},
+			9,
+			11,
+			2,
+			[][32]byte{[32]byte{'k'}, [32]byte{'l'}},
+		},
+		{
+			"Remove tip, lvh is not an ancestor",
+			[32]byte{'j'},
+			[32]byte{'C'},
+			5,
+			12,
+			7,
+			[][32]byte{[32]byte{'j'}},
+		},
+		{
+			"Remove inner, lvh is not an ancestor",
+			[32]byte{'g'},
+			[32]byte{'J'},
 			NonExistentNode,
 			NonExistentNode,
 			1,
-			[][32]byte{[32]byte{'i'}},
+			[][32]byte{[32]byte{'g'}, [32]byte{'h'}, [32]byte{'k'}, [32]byte{'i'}, [32]byte{'l'}},
 		},
 	}
 	for _, tc := range tests {
@@ -458,184 +263,71 @@ func TestSetOptimisticToInvalid(t *testing.T) {
 		f := setup(1, 1)
 
 		require.NoError(t, f.InsertOptimisticBlock(ctx, 100, [32]byte{'a'}, params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 101, [32]byte{'b'}, [32]byte{'a'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'c'}, [32]byte{'b'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'j'}, [32]byte{'b'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 103, [32]byte{'d'}, [32]byte{'c'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'e'}, [32]byte{'d'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'g'}, [32]byte{'d'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'f'}, [32]byte{'e'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'h'}, [32]byte{'g'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'k'}, [32]byte{'g'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'i'}, [32]byte{'h'}, params.BeaconConfig().ZeroHash, 1, 1))
-		require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'l'}, [32]byte{'k'}, params.BeaconConfig().ZeroHash, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 101, [32]byte{'b'}, [32]byte{'a'}, [32]byte{'B'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'c'}, [32]byte{'b'}, [32]byte{'C'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'j'}, [32]byte{'b'}, [32]byte{'J'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 103, [32]byte{'d'}, [32]byte{'c'}, [32]byte{'D'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'e'}, [32]byte{'d'}, [32]byte{'E'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'g'}, [32]byte{'d'}, [32]byte{'G'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'f'}, [32]byte{'e'}, [32]byte{'F'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'h'}, [32]byte{'g'}, [32]byte{'H'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'k'}, [32]byte{'g'}, [32]byte{'K'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'i'}, [32]byte{'h'}, [32]byte{'I'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'l'}, [32]byte{'k'}, [32]byte{'L'}, 1, 1))
 		weights := []uint64{10, 10, 9, 7, 1, 6, 2, 3, 1, 1, 1, 0, 0}
-		f.syncedTips.Lock()
-		f.syncedTips.validatedTips = tc.tips
-		f.syncedTips.Unlock()
 		f.store.nodesLock.Lock()
 		for i, node := range f.store.nodes {
 			node.weight = weights[i]
 		}
-		// Make j be the best child and descendant of b
-		nodeB := f.store.nodes[2]
-		nodeB.bestChild = 4
-		nodeB.bestDescendant = 4
-		idx := f.store.nodesIndices[tc.root]
-		node := f.store.nodes[idx]
-		parentIndex := node.parent
-		require.NotEqual(t, NonExistentNode, parentIndex)
-		parent := f.store.nodes[parentIndex]
 		f.store.nodesLock.Unlock()
-		roots, err := f.SetOptimisticToInvalid(context.Background(), tc.root)
+		require.NoError(t, f.SetOptimisticToValid(ctx, [32]byte{'e'}))
+		roots, err := f.SetOptimisticToInvalid(ctx, tc.root, tc.payload)
 		require.NoError(t, err)
-		require.DeepEqual(t, tc.returnedRoots, roots)
-		f.syncedTips.RLock()
-		_, parentSyncedTip := f.syncedTips.validatedTips[parent.root]
-		f.syncedTips.RUnlock()
-		require.Equal(t, tc.wantedParentTip, parentSyncedTip)
-		require.Equal(t, tc.newBestChild, parent.bestChild)
-		require.Equal(t, tc.newBestDescendant, parent.bestDescendant)
-		require.Equal(t, tc.newParentWeight, parent.weight)
-	}
-}
-
-// This tests the algorithm to find the tip of a given node
-// We start with the following diagram
-//
-//                E -- F
-//               /
-//         C -- D
-//        /      \
-//  A -- B        G -- H -- I
-//        \        \
-//         J        -- K -- L
-//
-//
-func TestFindSyncedTip(t *testing.T) {
-	ctx := context.Background()
-	f := setup(1, 1)
-
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 100, [32]byte{'a'}, params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 101, [32]byte{'b'}, [32]byte{'a'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'c'}, [32]byte{'b'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'j'}, [32]byte{'b'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 103, [32]byte{'d'}, [32]byte{'c'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'e'}, [32]byte{'d'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'g'}, [32]byte{'d'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'f'}, [32]byte{'e'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'h'}, [32]byte{'g'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'k'}, [32]byte{'g'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'i'}, [32]byte{'h'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'l'}, [32]byte{'k'}, params.BeaconConfig().ZeroHash, 1, 1))
-	tests := []struct {
-		root   [32]byte                // the root of the block
-		tips   map[[32]byte]types.Slot // the synced tips
-		wanted [32]byte                // the root of expected tip
-	}{
-		{
-			[32]byte{'i'},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'d'}: 103,
-				[32]byte{'g'}: 104,
-			},
-			[32]byte{'g'},
-		},
-		{
-			[32]byte{'g'},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'d'}: 103,
-				[32]byte{'h'}: 104,
-				[32]byte{'k'}: 106,
-			},
-			[32]byte{'d'},
-		},
-		{
-			[32]byte{'e'},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'d'}: 103,
-				[32]byte{'g'}: 103,
-			},
-			[32]byte{'d'},
-		},
-		{
-			[32]byte{'j'},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'f'}: 105,
-				[32]byte{'g'}: 104,
-				[32]byte{'i'}: 106,
-			},
-			[32]byte{'b'},
-		},
-		{
-			[32]byte{'g'},
-			map[[32]byte]types.Slot{
-				[32]byte{'b'}: 101,
-				[32]byte{'f'}: 105,
-				[32]byte{'g'}: 104,
-				[32]byte{'i'}: 106,
-			},
-			[32]byte{'g'},
-		},
-	}
-	for _, tc := range tests {
 		f.store.nodesLock.RLock()
-		node := f.store.nodes[f.store.nodesIndices[tc.root]]
-		syncedTips := &optimisticStore{
-			validatedTips: tc.tips,
-		}
-		syncedTips.RLock()
-		idx, err := f.store.findSyncedTip(ctx, node, syncedTips)
-		require.NoError(t, err)
-		require.Equal(t, tc.wanted, f.store.nodes[idx].root)
-
+		_, ok := f.store.nodesIndices[tc.root]
+		require.Equal(t, false, ok)
+		lvh := f.store.nodes[f.store.payloadIndices[tc.payload]]
+		require.Equal(t, true, slicesEqual(tc.returnedRoots, roots))
+		require.Equal(t, tc.newBestChild, lvh.bestChild)
+		require.Equal(t, tc.newBestDescendant, lvh.bestDescendant)
+		require.Equal(t, tc.newParentWeight, lvh.weight)
+		require.Equal(t, syncing, f.store.nodes[8].status /* F */)
+		require.Equal(t, valid, f.store.nodes[5].status /* E */)
 		f.store.nodesLock.RUnlock()
-		syncedTips.RUnlock()
 	}
 }
 
-// This is a regression test (10341)
-func TestIsOptimistic_DeadLock(t *testing.T) {
+func TestSetOptimisticToInvalid_InvalidRoots(t *testing.T) {
 	ctx := context.Background()
 	f := setup(1, 1)
+
 	require.NoError(t, f.InsertOptimisticBlock(ctx, 100, [32]byte{'a'}, params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 90, [32]byte{'b'}, params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 101, [32]byte{'c'}, params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'d'}, params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 103, [32]byte{'e'}, params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, 1, 1))
-	tips := map[[32]byte]types.Slot{
-		[32]byte{'a'}: 100,
-		[32]byte{'d'}: 102,
-	}
-	f.syncedTips.validatedTips = tips
-	_, err := f.IsOptimistic(ctx, [32]byte{'a'})
-	require.NoError(t, err)
-
-	// Acquire a write lock, this should not hang
-	f.store.nodesLock.Lock()
-	f.store.nodesLock.Unlock()
-	_, err = f.IsOptimistic(ctx, [32]byte{'e'})
-	require.NoError(t, err)
-
-	// Acquire a write lock, this should not hang
-	f.store.nodesLock.Lock()
-	f.store.nodesLock.Unlock()
-	_, err = f.IsOptimistic(ctx, [32]byte{'b'})
-	require.NoError(t, err)
-
-	// Acquire a write lock, this should not hang
-	f.store.nodesLock.Lock()
-	f.store.nodesLock.Unlock()
-
-	_, err = f.IsOptimistic(ctx, [32]byte{'c'})
-	require.NoError(t, err)
-
-	// Acquire a write lock, this should not hang
-	f.store.nodesLock.Lock()
-	f.store.nodesLock.Unlock()
-
+	require.NoError(t, f.InsertOptimisticBlock(ctx, 101, [32]byte{'b'}, [32]byte{'a'}, [32]byte{'B'}, 1, 1))
+	_, err := f.SetOptimisticToInvalid(ctx, [32]byte{'p'}, [32]byte{'B'})
+	require.ErrorIs(t, ErrUnknownNodeRoot, err)
+	_, err = f.SetOptimisticToInvalid(ctx, [32]byte{'a'}, [32]byte{'p'})
+	require.ErrorIs(t, errInvalidFinalizedNode, err)
+}
+
+// This is a regression test (10445)
+func TestSetOptimisticToInvalid_ProposerBoost(t *testing.T) {
+	ctx := context.Background()
+	f := setup(1, 1)
+
+	require.NoError(t, f.InsertOptimisticBlock(ctx, 100, [32]byte{'a'}, params.BeaconConfig().ZeroHash, [32]byte{'A'}, 1, 1))
+	require.NoError(t, f.InsertOptimisticBlock(ctx, 101, [32]byte{'b'}, [32]byte{'a'}, [32]byte{'B'}, 1, 1))
+	require.NoError(t, f.InsertOptimisticBlock(ctx, 101, [32]byte{'c'}, [32]byte{'b'}, [32]byte{'C'}, 1, 1))
+	f.store.proposerBoostLock.Lock()
+	f.store.proposerBoostRoot = [32]byte{'c'}
+	f.store.previousProposerBoostScore = 10
+	f.store.previousProposerBoostRoot = [32]byte{'b'}
+	f.store.proposerBoostLock.Unlock()
+
+	_, err := f.SetOptimisticToInvalid(ctx, [32]byte{'c'}, [32]byte{'A'})
+	require.NoError(t, err)
+	f.store.proposerBoostLock.RLock()
+	require.Equal(t, uint64(0), f.store.previousProposerBoostScore)
+	require.DeepEqual(t, [32]byte{}, f.store.proposerBoostRoot)
+	require.DeepEqual(t, params.BeaconConfig().ZeroHash, f.store.previousProposerBoostRoot)
+	f.store.proposerBoostLock.RUnlock()
 }

beacon-chain/forkchoice/protoarray/store.go

@@ -9,8 +9,10 @@ import (
 	types "github.com/prysmaticlabs/eth2-types"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 	pmath "github.com/prysmaticlabs/prysm/math"
 	pbrpc "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 )
 
@@ -30,43 +32,14 @@ func New(justifiedEpoch, finalizedEpoch types.Epoch, finalizedRoot [32]byte) *Fo
 		proposerBoostRoot: [32]byte{},
 		nodes:             make([]*Node, 0),
 		nodesIndices:      make(map[[32]byte]uint64),
+		payloadIndices:    make(map[[32]byte]uint64),
 		canonicalNodes:    make(map[[32]byte]bool),
 		pruneThreshold:    defaultPruneThreshold,
 	}
 
 	b := make([]uint64, 0)
 	v := make([]Vote, 0)
-	st := &optimisticStore{
-		validatedTips: make(map[[32]byte]types.Slot),
-	}
-	return &ForkChoice{store: s, balances: b, votes: v, syncedTips: st}
-}
-
-// SetSyncedTips sets the synced and validated tips from the passed map
-func (f *ForkChoice) SetSyncedTips(tips map[[32]byte]types.Slot) error {
-	if len(tips) == 0 {
-		return errInvalidSyncedTips
-	}
-	newTips := make(map[[32]byte]types.Slot, len(tips))
-	for k, v := range tips {
-		newTips[k] = v
-	}
-	f.syncedTips.Lock()
-	defer f.syncedTips.Unlock()
-	f.syncedTips.validatedTips = newTips
-	return nil
-}
-
-// SyncedTips returns the synced and validated tips from the fork choice store.
-func (f *ForkChoice) SyncedTips() map[[32]byte]types.Slot {
-	f.syncedTips.RLock()
-	defer f.syncedTips.RUnlock()
-
-	m := make(map[[32]byte]types.Slot)
-	for k, v := range f.syncedTips.validatedTips {
-		m[k] = v
-	}
-	return m
+	return &ForkChoice{store: s, balances: b, votes: v}
 }
 
 // Head returns the head root from fork choice store.
@@ -159,7 +132,7 @@ func (f *ForkChoice) InsertOptimisticBlock(
 // Prune prunes the fork choice store with the new finalized root. The store is only pruned if the input
 // root is different than the current store finalized root, and the number of the store has met prune threshold.
 func (f *ForkChoice) Prune(ctx context.Context, finalizedRoot [32]byte) error {
-	return f.store.prune(ctx, finalizedRoot, f.syncedTips)
+	return f.store.prune(ctx, finalizedRoot)
 }
 
 // HasNode returns true if the node exists in fork choice store,
@@ -375,6 +348,7 @@ func (s *Store) insert(ctx context.Context,
 	}
 
 	s.nodesIndices[root] = index
+	s.payloadIndices[payloadHash] = index
 	s.nodes = append(s.nodes, n)
 
 	// Update parent with the best child and descendant only if it's available.
@@ -443,6 +417,7 @@ func (s *Store) applyWeightChanges(
 			}
 			iProposerScore, err := pmath.Int(proposerScore)
 			if err != nil {
+				s.proposerBoostLock.Unlock()
 				return err
 			}
 			nodeDelta = nodeDelta + iProposerScore
@@ -453,6 +428,16 @@ func (s *Store) applyWeightChanges(
 		if nodeDelta < 0 {
 			d := uint64(-nodeDelta)
 			if n.weight < d {
+				s.proposerBoostLock.RLock()
+				log.WithFields(logrus.Fields{
+					"nodeDelta":                  d,
+					"nodeRoot":                   fmt.Sprintf("%#x", bytesutil.Trunc(n.root[:])),
+					"nodeWeight":                 n.weight,
+					"proposerBoostRoot":          fmt.Sprintf("%#x", bytesutil.Trunc(s.proposerBoostRoot[:])),
+					"previousProposerBoostRoot":  fmt.Sprintf("%#x", bytesutil.Trunc(s.previousProposerBoostRoot[:])),
+					"previousProposerBoostScore": s.previousProposerBoostScore,
+				}).Warning("node with invalid weight, setting it to zero")
+				s.proposerBoostLock.RUnlock()
 				n.weight = 0
 			} else {
 				n.weight -= d
@@ -598,7 +583,7 @@ func (s *Store) updateBestChildAndDescendant(parentIndex, childIndex uint64) err
 // prune prunes the store with the new finalized root. The tree is only
 // pruned if the input finalized root are different than the one in stored and
 // the number of the nodes in store has met prune threshold.
-func (s *Store) prune(ctx context.Context, finalizedRoot [32]byte, syncedTips *optimisticStore) error {
+func (s *Store) prune(ctx context.Context, finalizedRoot [32]byte) error {
 	_, span := trace.StartSpan(ctx, "protoArrayForkChoice.prune")
 	defer span.End()
 
@@ -618,18 +603,9 @@ func (s *Store) prune(ctx context.Context, finalizedRoot [32]byte, syncedTips *o
 		return nil
 	}
 
-	// Traverse through the node list starting from the finalized node at index 0.
-	// Nodes that are not branching off from the finalized node will be removed.
-	syncedTips.Lock()
-	defer syncedTips.Unlock()
-
 	canonicalNodesMap := make(map[uint64]uint64, uint64(len(s.nodes))-finalizedIndex)
 	canonicalNodes := make([]*Node, 1, uint64(len(s.nodes))-finalizedIndex)
 	finalizedNode := s.nodes[finalizedIndex]
-	finalizedTipIndex, err := s.findSyncedTip(ctx, finalizedNode, syncedTips)
-	if err != nil {
-		return err
-	}
 	finalizedNode.parent = NonExistentNode
 	canonicalNodes[0] = finalizedNode
 	canonicalNodesMap[finalizedIndex] = uint64(0)
@@ -638,20 +614,22 @@ func (s *Store) prune(ctx context.Context, finalizedRoot [32]byte, syncedTips *o
 		node := copyNode(s.nodes[idx])
 		parentIdx, ok := canonicalNodesMap[node.parent]
 		if ok {
-			s.nodesIndices[node.root] = uint64(len(canonicalNodes))
-			canonicalNodesMap[idx] = uint64(len(canonicalNodes))
+			currentIndex := uint64(len(canonicalNodes))
+			s.nodesIndices[node.root] = currentIndex
+			s.payloadIndices[node.payloadHash] = currentIndex
+			canonicalNodesMap[idx] = currentIndex
 			node.parent = parentIdx
 			canonicalNodes = append(canonicalNodes, node)
 		} else {
-			// Remove node and synced tip that is not part of finalized branch.
+			// Remove node that is not part of finalized branch.
 			delete(s.nodesIndices, node.root)
-			_, ok := syncedTips.validatedTips[node.root]
-			if ok && idx != finalizedTipIndex {
-				delete(syncedTips.validatedTips, node.root)
-			}
+			delete(s.canonicalNodes, node.root)
+			delete(s.payloadIndices, node.payloadHash)
 		}
 	}
 	s.nodesIndices[finalizedRoot] = uint64(0)
+	s.canonicalNodes[finalizedRoot] = true
+	s.payloadIndices[finalizedNode.payloadHash] = uint64(0)
 
 	// Recompute the best child and descendant for each canonical nodes.
 	for _, node := range canonicalNodes {
@@ -665,7 +643,6 @@ func (s *Store) prune(ctx context.Context, finalizedRoot [32]byte, syncedTips *o
 
 	s.nodes = canonicalNodes
 	prunedCount.Inc()
-	syncedTipsCount.Set(float64(len(syncedTips.validatedTips)))
 	return nil
 }
 
@@ -673,6 +650,10 @@ func (s *Store) prune(ctx context.Context, finalizedRoot [32]byte, syncedTips *o
 // Any node with diff finalized or justified epoch than the ones in fork choice store
 // should not be viable to head.
 func (s *Store) leadsToViableHead(node *Node) (bool, error) {
+	if node.status == invalid {
+		return false, nil
+	}
+
 	var bestDescendantViable bool
 	bestDescendantIndex := node.bestDescendant
 
@@ -704,20 +685,6 @@ func (s *Store) viableForHead(node *Node) bool {
 	return justified && finalized
 }
 
-// Returns the list of leaves in the Fork Choice store.
-// These are all the nodes that have NonExistentNode as best child.
-// This internal method assumes that the caller holds a lock in s.nodesLock.
-func (s *Store) leaves() ([]uint64, error) {
-	var leaves []uint64
-	for i := uint64(0); i < uint64(len(s.nodes)); i++ {
-		node := s.nodes[i]
-		if node.bestChild == NonExistentNode {
-			leaves = append(leaves, i)
-		}
-	}
-	return leaves, nil
-}
-
 // Tips returns all possible chain heads (leaves of fork choice tree).
 // Heads roots and heads slots are returned.
 func (f *ForkChoice) Tips() ([][32]byte, []types.Slot) {

beacon-chain/forkchoice/protoarray/store_test.go

@@ -100,7 +100,7 @@ func TestStore_Head_ContextCancelled(t *testing.T) {
 
 func TestStore_Insert_UnknownParent(t *testing.T) {
 	// The new node does not have a parent.
-	s := &Store{nodesIndices: make(map[[32]byte]uint64)}
+	s := &Store{nodesIndices: make(map[[32]byte]uint64), payloadIndices: make(map[[32]byte]uint64)}
 	require.NoError(t, s.insert(context.Background(), 100, [32]byte{'A'}, [32]byte{'B'}, params.BeaconConfig().ZeroHash, 1, 1))
 	assert.Equal(t, 1, len(s.nodes), "Did not insert block")
 	assert.Equal(t, 1, len(s.nodesIndices), "Did not insert block")
@@ -113,7 +113,7 @@ func TestStore_Insert_UnknownParent(t *testing.T) {
 func TestStore_Insert_KnownParent(t *testing.T) {
 	// Similar to UnknownParent test, but this time the new node has a valid parent already in store.
 	// The new node builds on top of the parent.
-	s := &Store{nodesIndices: make(map[[32]byte]uint64)}
+	s := &Store{nodesIndices: make(map[[32]byte]uint64), payloadIndices: make(map[[32]byte]uint64)}
 	s.nodes = []*Node{{}}
 	p := [32]byte{'B'}
 	s.nodesIndices[p] = 0
@@ -336,11 +336,10 @@ func TestStore_Prune_LessThanThreshold(t *testing.T) {
 	})
 
 	s := &Store{nodes: nodes, nodesIndices: indices, pruneThreshold: 100}
-	syncedTips := &optimisticStore{}
 
 	// Finalized root is at index 99 so everything before 99 should be pruned,
 	// but PruneThreshold is at 100 so nothing will be pruned.
-	require.NoError(t, s.prune(context.Background(), indexToHash(99), syncedTips))
+	require.NoError(t, s.prune(context.Background(), indexToHash(99)))
 	assert.Equal(t, 100, len(s.nodes), "Incorrect nodes count")
 	assert.Equal(t, 100, len(s.nodesIndices), "Incorrect node indices count")
 }
@@ -376,11 +375,10 @@ func TestStore_Prune_MoreThanThreshold(t *testing.T) {
 		parent:         uint64(numOfNodes - 2),
 	})
 	indices[indexToHash(uint64(numOfNodes-1))] = uint64(numOfNodes - 1)
-	s := &Store{nodes: nodes, nodesIndices: indices}
-	syncedTips := &optimisticStore{}
+	s := &Store{nodes: nodes, nodesIndices: indices, canonicalNodes: map[[32]byte]bool{}, payloadIndices: map[[32]byte]uint64{}}
 
 	// Finalized root is at index 99 so everything before 99 should be pruned.
-	require.NoError(t, s.prune(context.Background(), indexToHash(99), syncedTips))
+	require.NoError(t, s.prune(context.Background(), indexToHash(99)))
 	assert.Equal(t, 1, len(s.nodes), "Incorrect nodes count")
 	assert.Equal(t, 1, len(s.nodesIndices), "Incorrect node indices count")
 }
@@ -415,16 +413,15 @@ func TestStore_Prune_MoreThanOnce(t *testing.T) {
 		parent:         uint64(numOfNodes - 2),
 	})
 
-	s := &Store{nodes: nodes, nodesIndices: indices}
-	syncedTips := &optimisticStore{}
+	s := &Store{nodes: nodes, nodesIndices: indices, canonicalNodes: map[[32]byte]bool{}, payloadIndices: map[[32]byte]uint64{}}
 
 	// Finalized root is at index 11 so everything before 11 should be pruned.
-	require.NoError(t, s.prune(context.Background(), indexToHash(10), syncedTips))
+	require.NoError(t, s.prune(context.Background(), indexToHash(10)))
 	assert.Equal(t, 90, len(s.nodes), "Incorrect nodes count")
 	assert.Equal(t, 90, len(s.nodesIndices), "Incorrect node indices count")
 
 	// One more time.
-	require.NoError(t, s.prune(context.Background(), indexToHash(20), syncedTips))
+	require.NoError(t, s.prune(context.Background(), indexToHash(20)))
 	assert.Equal(t, 80, len(s.nodes), "Incorrect nodes count")
 	assert.Equal(t, 80, len(s.nodesIndices), "Incorrect node indices count")
 }
@@ -444,6 +441,7 @@ func TestStore_Prune_NoDanglingBranch(t *testing.T) {
 			bestDescendant: 1,
 			root:           indexToHash(uint64(0)),
 			parent:         NonExistentNode,
+			payloadHash:    [32]byte{'A'},
 		},
 		{
 			slot:           101,
@@ -451,6 +449,7 @@ func TestStore_Prune_NoDanglingBranch(t *testing.T) {
 			bestChild:      NonExistentNode,
 			bestDescendant: NonExistentNode,
 			parent:         0,
+			payloadHash:    [32]byte{'B'},
 		},
 		{
 			slot:           101,
@@ -458,9 +457,9 @@ func TestStore_Prune_NoDanglingBranch(t *testing.T) {
 			parent:         0,
 			bestChild:      NonExistentNode,
 			bestDescendant: NonExistentNode,
+			payloadHash:    [32]byte{'C'},
 		},
 	}
-	syncedTips := &optimisticStore{}
 	s := &Store{
 		pruneThreshold: 0,
 		nodes:          nodes,
@@ -469,9 +468,22 @@ func TestStore_Prune_NoDanglingBranch(t *testing.T) {
 			indexToHash(uint64(1)): 1,
 			indexToHash(uint64(2)): 2,
 		},
+		canonicalNodes: map[[32]byte]bool{
+			indexToHash(uint64(0)): true,
+			indexToHash(uint64(1)): true,
+			indexToHash(uint64(2)): true,
+		},
+		payloadIndices: map[[32]byte]uint64{
+			[32]byte{'A'}: 0,
+			[32]byte{'B'}: 1,
+			[32]byte{'C'}: 2,
+		},
 	}
-	require.NoError(t, s.prune(context.Background(), indexToHash(uint64(1)), syncedTips))
-	require.Equal(t, len(s.nodes), 1)
+	require.NoError(t, s.prune(context.Background(), indexToHash(uint64(1))))
+	require.Equal(t, 1, len(s.nodes))
+	require.Equal(t, 1, len(s.nodesIndices))
+	require.Equal(t, 1, len(s.canonicalNodes))
+	require.Equal(t, 1, len(s.payloadIndices))
 }
 
 // This test starts with the following branching diagram
@@ -486,38 +498,74 @@ func TestStore_Prune_NoDanglingBranch(t *testing.T) {
 //         J        -- K -- L
 //
 //
-// Synced tips are B, D and E. And we finalize F. All that is left in fork
-// choice is F, and the only synced tip left is E which is now away from Fork
-// Choice.
-func TestStore_PruneSyncedTips(t *testing.T) {
+func TestStore_PruneBranched(t *testing.T) {
 	ctx := context.Background()
-	f := setup(1, 1)
 
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 100, [32]byte{'a'}, params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 101, [32]byte{'b'}, [32]byte{'a'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'c'}, [32]byte{'b'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'j'}, [32]byte{'b'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 103, [32]byte{'d'}, [32]byte{'c'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'e'}, [32]byte{'d'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'g'}, [32]byte{'d'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'f'}, [32]byte{'e'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'h'}, [32]byte{'g'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'k'}, [32]byte{'g'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'i'}, [32]byte{'h'}, params.BeaconConfig().ZeroHash, 1, 1))
-	require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'l'}, [32]byte{'k'}, params.BeaconConfig().ZeroHash, 1, 1))
-	syncedTips := &optimisticStore{
-		validatedTips: map[[32]byte]types.Slot{
-			[32]byte{'b'}: 101,
-			[32]byte{'d'}: 103,
-			[32]byte{'e'}: 104,
+	tests := []struct {
+		finalizedRoot      [32]byte
+		wantedCanonical    [32]byte
+		wantedNonCanonical [32]byte
+		canonicalCount     int
+		payloadHash        [32]byte
+		payloadIndex       uint64
+		nonExistentPayload [32]byte
+	}{
+		{
+			[32]byte{'f'},
+			[32]byte{'f'},
+			[32]byte{'a'},
+			1,
+			[32]byte{'F'},
+			0,
+			[32]byte{'H'},
+		},
+		{
+			[32]byte{'d'},
+			[32]byte{'e'},
+			[32]byte{'i'},
+			3,
+			[32]byte{'E'},
+			1,
+			[32]byte{'C'},
+		},
+		{
+			[32]byte{'b'},
+			[32]byte{'f'},
+			[32]byte{'h'},
+			5,
+			[32]byte{'D'},
+			3,
+			[32]byte{'A'},
 		},
 	}
-	f.syncedTips = syncedTips
-	f.store.pruneThreshold = 0
-	require.NoError(t, f.Prune(ctx, [32]byte{'f'}))
-	require.Equal(t, 1, len(f.syncedTips.validatedTips))
-	_, ok := f.syncedTips.validatedTips[[32]byte{'e'}]
-	require.Equal(t, true, ok)
+
+	for _, tc := range tests {
+		f := setup(1, 1)
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 100, [32]byte{'a'}, params.BeaconConfig().ZeroHash, [32]byte{'A'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 101, [32]byte{'b'}, [32]byte{'a'}, [32]byte{'B'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'c'}, [32]byte{'b'}, [32]byte{'C'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 102, [32]byte{'j'}, [32]byte{'b'}, [32]byte{'J'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 103, [32]byte{'d'}, [32]byte{'c'}, [32]byte{'D'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'e'}, [32]byte{'d'}, [32]byte{'E'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 104, [32]byte{'g'}, [32]byte{'d'}, [32]byte{'G'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'f'}, [32]byte{'e'}, [32]byte{'F'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'h'}, [32]byte{'g'}, [32]byte{'H'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 105, [32]byte{'k'}, [32]byte{'g'}, [32]byte{'K'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'i'}, [32]byte{'h'}, [32]byte{'I'}, 1, 1))
+		require.NoError(t, f.InsertOptimisticBlock(ctx, 106, [32]byte{'l'}, [32]byte{'k'}, [32]byte{'L'}, 1, 1))
+		f.store.pruneThreshold = 0
+		require.NoError(t, f.store.updateCanonicalNodes(ctx, [32]byte{'f'}))
+		require.Equal(t, true, f.IsCanonical([32]byte{'a'}))
+		require.Equal(t, true, f.IsCanonical([32]byte{'f'}))
+
+		require.NoError(t, f.Prune(ctx, tc.finalizedRoot))
+		require.Equal(t, tc.canonicalCount, len(f.store.canonicalNodes))
+		require.Equal(t, true, f.IsCanonical(tc.wantedCanonical))
+		require.Equal(t, false, f.IsCanonical(tc.wantedNonCanonical))
+		require.Equal(t, tc.payloadIndex, f.store.payloadIndices[tc.payloadHash])
+		_, ok := f.store.payloadIndices[tc.nonExistentPayload]
+		require.Equal(t, false, ok)
+	}
 }
 
 func TestStore_LeadsToViableHead(t *testing.T) {
@@ -546,20 +594,6 @@ func TestStore_LeadsToViableHead(t *testing.T) {
 	}
 }
 
-func TestStore_SetSyncedTips(t *testing.T) {
-	f := setup(1, 1)
-	tips := make(map[[32]byte]types.Slot)
-	require.ErrorIs(t, errInvalidSyncedTips, f.SetSyncedTips(tips))
-	tips[bytesutil.ToBytes32([]byte{'a'})] = 1
-	require.NoError(t, f.SetSyncedTips(tips))
-	f.syncedTips.RLock()
-	defer f.syncedTips.RUnlock()
-	require.Equal(t, 1, len(f.syncedTips.validatedTips))
-	slot, ok := f.syncedTips.validatedTips[bytesutil.ToBytes32([]byte{'a'})]
-	require.Equal(t, true, ok)
-	require.Equal(t, types.Slot(1), slot)
-}
-
 func TestStore_ViableForHead(t *testing.T) {
 	tests := []struct {
 		n              *Node

beacon-chain/forkchoice/protoarray/types.go

@@ -9,11 +9,10 @@ import (
 
 // ForkChoice defines the overall fork choice store which includes all block nodes, validator's latest votes and balances.
 type ForkChoice struct {
-	store      *Store
-	votes      []Vote // tracks individual validator's last vote.
-	votesLock  sync.RWMutex
-	balances   []uint64 // tracks individual validator's last justified balances.
-	syncedTips *optimisticStore
+	store     *Store
+	votes     []Vote // tracks individual validator's last vote.
+	votesLock sync.RWMutex
+	balances  []uint64 // tracks individual validator's last justified balances.
 }
 
 // Store defines the fork choice store which includes block nodes and the last view of checkpoint information.
@@ -28,6 +27,7 @@ type Store struct {
 	nodes                      []*Node                                 // list of block nodes, each node is a representation of one block.
 	nodesIndices               map[[fieldparams.RootLength]byte]uint64 // the root of block node and the nodes index in the list.
 	canonicalNodes             map[[fieldparams.RootLength]byte]bool   // the canonical block nodes.
+	payloadIndices             map[[fieldparams.RootLength]byte]uint64 // the payload hash of block node and the index in the list
 	nodesLock                  sync.RWMutex
 	proposerBoostLock          sync.RWMutex
 }
@@ -44,15 +44,17 @@ type Node struct {
 	weight         uint64                       // weight of this node.
 	bestChild      uint64                       // bestChild index of this node.
 	bestDescendant uint64                       // bestDescendant of this node.
-	graffiti       [fieldparams.RootLength]byte // graffiti of the block node.
+	status         status                       // optimistic status of this node
 }
 
-// optimisticStore defines a structure that tracks the tips of the fully
-// validated blocks tree.
-type optimisticStore struct {
-	validatedTips map[[32]byte]types.Slot
-	sync.RWMutex
-}
+// enum used as optimistic status of a node
+type status uint8
+
+const (
+	syncing status = iota // the node is optimistic
+	valid                 //fully validated node
+	invalid               // invalid execution payload
+)
 
 // Vote defines an individual validator's vote.
 type Vote struct {

beacon-chain/forkchoice/protoarray/vote_test.go

@@ -12,7 +12,6 @@ import (
 func TestVotes_CanFindHead(t *testing.T) {
 	balances := []uint64{1, 1}
 	f := setup(1, 1)
-	syncedTips := &optimisticStore{}
 
 	// The head should always start at the finalized block.
 	r, err := f.Head(context.Background(), 1, params.BeaconConfig().ZeroHash, balances, 1)
@@ -249,7 +248,7 @@ func TestVotes_CanFindHead(t *testing.T) {
 
 	// Verify pruning below the prune threshold does not affect head.
 	f.store.pruneThreshold = 1000
-	require.NoError(t, f.store.prune(context.Background(), indexToHash(5), syncedTips))
+	require.NoError(t, f.store.prune(context.Background(), indexToHash(5)))
 	assert.Equal(t, 11, len(f.store.nodes), "Incorrect nodes length after prune")
 
 	r, err = f.Head(context.Background(), 2, indexToHash(5), balances, 2)
@@ -273,7 +272,7 @@ func TestVotes_CanFindHead(t *testing.T) {
 	//         / \
 	//        9  10
 	f.store.pruneThreshold = 1
-	require.NoError(t, f.store.prune(context.Background(), indexToHash(5), syncedTips))
+	require.NoError(t, f.store.prune(context.Background(), indexToHash(5)))
 	assert.Equal(t, 5, len(f.store.nodes), "Incorrect nodes length after prune")
 
 	r, err = f.Head(context.Background(), 2, indexToHash(5), balances, 2)

beacon-chain/node/BUILD.bazel

@@ -18,6 +18,7 @@ go_library(
         "//api/gateway:go_default_library",
         "//async/event:go_default_library",
         "//beacon-chain/blockchain:go_default_library",
+        "//beacon-chain/cache:go_default_library",
         "//beacon-chain/cache/depositcache:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//beacon-chain/db/kv:go_default_library",
@@ -43,6 +44,7 @@ go_library(
         "//beacon-chain/sync:go_default_library",
         "//beacon-chain/sync/backfill:go_default_library",
         "//beacon-chain/sync/checkpoint:go_default_library",
+        "//beacon-chain/sync/genesis:go_default_library",
         "//beacon-chain/sync/initial-sync:go_default_library",
         "//cmd:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",
@@ -76,6 +78,8 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//beacon-chain/core/feed/state:go_default_library",
+        "//beacon-chain/powchain:go_default_library",
+        "//beacon-chain/powchain/testing:go_default_library",
         "//cmd:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",
         "//config/params:go_default_library",

beacon-chain/node/node.go

@@ -21,6 +21,7 @@ import (
 	apigateway "github.com/prysmaticlabs/prysm/api/gateway"
 	"github.com/prysmaticlabs/prysm/async/event"
 	"github.com/prysmaticlabs/prysm/beacon-chain/blockchain"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db/kv"
@@ -46,6 +47,7 @@ import (
 	regularsync "github.com/prysmaticlabs/prysm/beacon-chain/sync"
 	"github.com/prysmaticlabs/prysm/beacon-chain/sync/backfill"
 	"github.com/prysmaticlabs/prysm/beacon-chain/sync/checkpoint"
+	"github.com/prysmaticlabs/prysm/beacon-chain/sync/genesis"
 	initialsync "github.com/prysmaticlabs/prysm/beacon-chain/sync/initial-sync"
 	"github.com/prysmaticlabs/prysm/cmd"
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/flags"
@@ -93,6 +95,7 @@ type BeaconNode struct {
 	slashingsPool           slashings.PoolManager
 	syncCommitteePool       synccommittee.Pool
 	depositCache            *depositcache.DepositCache
+	proposerIdsCache        *cache.ProposerPayloadIDsCache
 	stateFeed               *event.Feed
 	blockFeed               *event.Feed
 	opFeed                  *event.Feed
@@ -104,6 +107,7 @@ type BeaconNode struct {
 	finalizedStateAtStartUp state.BeaconState
 	serviceFlagOpts         *serviceFlagOpts
 	blockchainFlagOpts      []blockchain.Option
+	GenesisInitializer      genesis.Initializer
 	CheckpointInitializer   checkpoint.Initializer
 }
 
@@ -150,6 +154,7 @@ func New(cliCtx *cli.Context, opts ...Option) (*BeaconNode, error) {
 		slasherBlockHeadersFeed: new(event.Feed),
 		slasherAttestationsFeed: new(event.Feed),
 		serviceFlagOpts:         &serviceFlagOpts{},
+		proposerIdsCache:        cache.NewProposerPayloadIDsCache(),
 	}
 
 	for _, opt := range opts {
@@ -381,20 +386,10 @@ func (b *BeaconNode) startDB(cliCtx *cli.Context, depositAddress string) error {
 	if err != nil {
 		return errors.Wrap(err, "could not create deposit cache")
 	}
-
 	b.depositCache = depositCache
 
-	if cliCtx.IsSet(flags.GenesisStatePath.Name) {
-		r, err := os.Open(cliCtx.String(flags.GenesisStatePath.Name))
-		if err != nil {
-			return err
-		}
-		defer func() {
-			if err := r.Close(); err != nil {
-				log.WithError(err).Error("Failed to close genesis file")
-			}
-		}()
-		if err := b.db.LoadGenesis(b.ctx, r); err != nil {
+	if b.GenesisInitializer != nil {
+		if err := b.GenesisInitializer.Initialize(b.ctx, d); err != nil {
 			if err == db.ErrExistingGenesisState {
 				return errors.New("Genesis state flag specified but a genesis state " +
 					"exists already. Run again with --clear-db and/or ensure you are using the " +
@@ -403,6 +398,7 @@ func (b *BeaconNode) startDB(cliCtx *cli.Context, depositAddress string) error {
 			return errors.Wrap(err, "could not load genesis from file")
 		}
 	}
+
 	if err := b.db.EnsureEmbeddedGenesis(b.ctx); err != nil {
 		return err
 	}
@@ -581,7 +577,7 @@ func (b *BeaconNode) registerBlockchainService() error {
 		blockchain.WithDatabase(b.db),
 		blockchain.WithDepositCache(b.depositCache),
 		blockchain.WithChainStartFetcher(web3Service),
-		blockchain.WithExecutionEngineCaller(web3Service.EngineAPIClient()),
+		blockchain.WithExecutionEngineCaller(web3Service),
 		blockchain.WithAttestationPool(b.attestationPool),
 		blockchain.WithExitPool(b.exitPool),
 		blockchain.WithSlashingPool(b.slashingsPool),
@@ -592,6 +588,7 @@ func (b *BeaconNode) registerBlockchainService() error {
 		blockchain.WithStateGen(b.stateGen),
 		blockchain.WithSlasherAttestationsFeed(b.slasherAttestationsFeed),
 		blockchain.WithFinalizedStateAtStartUp(b.finalizedStateAtStartUp),
+		blockchain.WithProposerIdsCache(b.proposerIdsCache),
 	)
 	blockchainService, err := blockchain.NewService(b.ctx, opts...)
 	if err != nil {
@@ -808,7 +805,8 @@ func (b *BeaconNode) registerRPCService() error {
 		StateGen:                b.stateGen,
 		EnableDebugRPCEndpoints: enableDebugRPCEndpoints,
 		MaxMsgSize:              maxMsgSize,
-		ExecutionEngineCaller:   web3Service.EngineAPIClient(),
+		ProposerIdsCache:        b.proposerIdsCache,
+		ExecutionEngineCaller:   web3Service,
 	})
 
 	return b.services.RegisterService(rpcService)

beacon-chain/node/node_test.go

@@ -8,6 +8,8 @@ import (
 	"testing"
 
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
+	mockPOW "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing"
 	"github.com/prysmaticlabs/prysm/cmd"
 	"github.com/prysmaticlabs/prysm/testing/require"
 	logTest "github.com/sirupsen/logrus/hooks/test"
@@ -45,6 +47,11 @@ func TestNodeClose_OK(t *testing.T) {
 // TestClearDB tests clearing the database
 func TestClearDB(t *testing.T) {
 	hook := logTest.NewGlobal()
+	srv, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		srv.Stop()
+	})
 
 	tmp := filepath.Join(t.TempDir(), "datadirtest")
 
@@ -54,7 +61,9 @@ func TestClearDB(t *testing.T) {
 	set.Bool(cmd.ForceClearDB.Name, true, "force clear db")
 
 	context := cli.NewContext(&app, set, nil)
-	_, err := New(context)
+	_, err = New(context, WithPowchainFlagOptions([]powchain.Option{
+		powchain.WithHttpEndpoints([]string{endpoint}),
+	}))
 	require.NoError(t, err)
 
 	require.LogsContain(t, hook, "Removing database")

beacon-chain/powchain/BUILD.bazel

@@ -7,11 +7,15 @@ go_library(
         "block_reader.go",
         "check_transition_config.go",
         "deposit.go",
+        "engine_client.go",
+        "errors.go",
         "log.go",
         "log_processing.go",
+        "metrics.go",
         "options.go",
         "prometheus.go",
         "provider.go",
+        "rpc_connection.go",
         "service.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/powchain",
@@ -28,7 +32,6 @@ go_library(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db:go_default_library",
-        "//beacon-chain/powchain/engine-api-client/v1:go_default_library",
         "//beacon-chain/powchain/types:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/state-native/v1:go_default_library",
@@ -74,6 +77,7 @@ go_test(
         "block_reader_test.go",
         "check_transition_config_test.go",
         "deposit_test.go",
+        "engine_client_test.go",
         "init_test.go",
         "log_processing_test.go",
         "powchain_test.go",
@@ -92,8 +96,6 @@ go_test(
         "//beacon-chain/core/signing:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
-        "//beacon-chain/powchain/engine-api-client/v1:go_default_library",
-        "//beacon-chain/powchain/engine-api-client/v1/mocks:go_default_library",
         "//beacon-chain/powchain/testing:go_default_library",
         "//beacon-chain/powchain/types:go_default_library",
         "//beacon-chain/state/stategen:go_default_library",
@@ -119,10 +121,14 @@ go_test(
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
+        "@com_github_ethereum_go_ethereum//rpc:go_default_library",
         "@com_github_ethereum_go_ethereum//trie:go_default_library",
+        "@com_github_holiman_uint256//:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prysmaticlabs_eth2_types//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
+        "@org_golang_google_protobuf//proto:go_default_library",
     ],
 )

beacon-chain/powchain/block_reader_test.go

@@ -18,8 +18,6 @@ import (
 	"github.com/prysmaticlabs/prysm/testing/require"
 )
 
-var endpoint = "http://127.0.0.1"
-
 func setDefaultMocks(service *Service) *Service {
 	service.eth1DataFetcher = &goodFetcher{}
 	service.httpLogger = &goodLogger{}
@@ -32,6 +30,11 @@ func TestLatestMainchainInfo_OK(t *testing.T) {
 	require.NoError(t, err, "Unable to set up simulated backend")
 
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -70,6 +73,11 @@ func TestLatestMainchainInfo_OK(t *testing.T) {
 
 func TestBlockHashByHeight_ReturnsHash(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -97,6 +105,11 @@ func TestBlockHashByHeight_ReturnsHash(t *testing.T) {
 
 func TestBlockHashByHeight_ReturnsError_WhenNoEth1Client(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -113,6 +126,11 @@ func TestBlockHashByHeight_ReturnsError_WhenNoEth1Client(t *testing.T) {
 
 func TestBlockExists_ValidHash(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -144,6 +162,11 @@ func TestBlockExists_ValidHash(t *testing.T) {
 
 func TestBlockExists_InvalidHash(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -158,6 +181,11 @@ func TestBlockExists_InvalidHash(t *testing.T) {
 
 func TestBlockExists_UsesCachedBlockInfo(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -181,6 +209,11 @@ func TestBlockExists_UsesCachedBlockInfo(t *testing.T) {
 
 func TestBlockExistsWithCache_UsesCachedHeaderInfo(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -202,6 +235,11 @@ func TestBlockExistsWithCache_UsesCachedHeaderInfo(t *testing.T) {
 
 func TestBlockExistsWithCache_HeaderNotCached(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -218,6 +256,11 @@ func TestService_BlockNumberByTimestamp(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -245,6 +288,11 @@ func TestService_BlockNumberByTimestampLessTargetTime(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -278,6 +326,11 @@ func TestService_BlockNumberByTimestampMoreTargetTime(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -309,6 +362,11 @@ func TestService_BlockNumberByTimestampMoreTargetTime(t *testing.T) {
 
 func TestService_BlockTimeByHeight_ReturnsError_WhenNoEth1Client(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),

beacon-chain/powchain/check_transition_config.go

@@ -7,12 +7,15 @@ import (
 	"math/big"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/holiman/uint256"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
-	v1 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
 	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/network"
 	pb "github.com/prysmaticlabs/prysm/proto/engine/v1"
+	"github.com/sirupsen/logrus"
 )
 
 var (
@@ -27,16 +30,12 @@ var (
 // If there are any discrepancies, we must log errors to ensure users can resolve
 //the problem and be ready for the merge transition.
 func (s *Service) checkTransitionConfiguration(
-	ctx context.Context, blockNotifications chan *statefeed.BlockProcessedData,
+	ctx context.Context, blockNotifications chan *feed.Event,
 ) {
 	// If Bellatrix fork epoch is not set, we do not run this check.
 	if params.BeaconConfig().BellatrixForkEpoch == math.MaxUint64 {
 		return
 	}
-	// If no engine API, then also avoid running this check.
-	if s.engineAPIClient == nil {
-		return
-	}
 	i := new(big.Int)
 	i.SetString(params.BeaconConfig().TerminalTotalDifficulty, 10)
 	ttd := new(uint256.Int)
@@ -46,9 +45,9 @@ func (s *Service) checkTransitionConfiguration(
 		TerminalBlockHash:       params.BeaconConfig().TerminalBlockHash[:],
 		TerminalBlockNumber:     big.NewInt(0).Bytes(), // A value of 0 is recommended in the request.
 	}
-	err := s.engineAPIClient.ExchangeTransitionConfiguration(ctx, cfg)
+	err := s.ExchangeTransitionConfiguration(ctx, cfg)
 	if err != nil {
-		if errors.Is(err, v1.ErrConfigMismatch) {
+		if errors.Is(err, ErrConfigMismatch) {
 			log.WithError(err).Fatal(configMismatchLog)
 		}
 		log.WithError(err).Error("Could not check configuration values between execution and consensus client")
@@ -58,6 +57,7 @@ func (s *Service) checkTransitionConfiguration(
 	// This serves as a heartbeat to ensure the execution client and Prysm are ready for the
 	// Bellatrix hard-fork transition.
 	ticker := time.NewTicker(checkTransitionPollingInterval)
+	hasTtdReached := false
 	defer ticker.Stop()
 	sub := s.cfg.stateNotifier.StateFeed().Subscribe(blockNotifications)
 	defer sub.Unsubscribe()
@@ -68,7 +68,11 @@ func (s *Service) checkTransitionConfiguration(
 		case <-sub.Err():
 			return
 		case ev := <-blockNotifications:
-			isExecutionBlock, err := blocks.IsExecutionBlock(ev.SignedBlock.Block().Body())
+			data, ok := ev.Data.(*statefeed.BlockProcessedData)
+			if !ok {
+				continue
+			}
+			isExecutionBlock, err := blocks.IsExecutionBlock(data.SignedBlock.Block().Body())
 			if err != nil {
 				log.WithError(err).Debug("Could not check whether signed block is execution block")
 				continue
@@ -77,9 +81,17 @@ func (s *Service) checkTransitionConfiguration(
 				log.Debug("PoS transition is complete, no longer checking for configuration changes")
 				return
 			}
-		case <-ticker.C:
-			err = s.engineAPIClient.ExchangeTransitionConfiguration(ctx, cfg)
+		case tm := <-ticker.C:
+			ctx, cancel := context.WithDeadline(ctx, tm.Add(network.DefaultRPCHTTPTimeout))
+			err = s.ExchangeTransitionConfiguration(ctx, cfg)
 			s.handleExchangeConfigurationError(err)
+			if !hasTtdReached {
+				hasTtdReached, err = s.logTtdStatus(ctx, ttd)
+				if err != nil {
+					log.WithError(err).Error("Could not log ttd status")
+				}
+			}
+			cancel()
 		}
 	}
 }
@@ -91,16 +103,39 @@ func (s *Service) handleExchangeConfigurationError(err error) {
 	if err == nil {
 		// If there is no error in checking the exchange configuration error, we clear
 		// the run error of the service if we had previously set it to ErrConfigMismatch.
-		if errors.Is(s.runError, v1.ErrConfigMismatch) {
+		if errors.Is(s.runError, ErrConfigMismatch) {
 			s.runError = nil
 		}
 		return
 	}
 	// If the error is a configuration mismatch, we set a runtime error in the service.
-	if errors.Is(err, v1.ErrConfigMismatch) {
+	if errors.Is(err, ErrConfigMismatch) {
 		s.runError = err
 		log.WithError(err).Error(configMismatchLog)
 		return
 	}
 	log.WithError(err).Error("Could not check configuration values between execution and consensus client")
 }
+
+// Logs the terminal total difficulty status.
+func (s *Service) logTtdStatus(ctx context.Context, ttd *uint256.Int) (bool, error) {
+	latest, err := s.LatestExecutionBlock(ctx)
+	if err != nil {
+		return false, err
+	}
+	if latest == nil {
+		return false, errors.New("latest block is nil")
+	}
+	latestTtd, err := hexutil.DecodeBig(latest.TotalDifficulty)
+	if err != nil {
+		return false, err
+	}
+	if latestTtd.Cmp(ttd.ToBig()) >= 0 {
+		return true, nil
+	}
+	log.WithFields(logrus.Fields{
+		"latestDifficulty":   latestTtd.String(),
+		"terminalDifficulty": ttd.ToBig().String(),
+	}).Info("terminal difficulty has not been reached yet")
+	return false, nil
+}

beacon-chain/powchain/check_transition_config_test.go

@@ -2,21 +2,28 @@ package powchain
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 	"time"
 
-	mockChain2 "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/holiman/uint256"
+	mockChain "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
-	v1 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
-	"github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1/mocks"
+	mocks "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
-	enginev1 "github.com/prysmaticlabs/prysm/proto/engine/v1"
+	pb "github.com/prysmaticlabs/prysm/proto/engine/v1"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
 	"github.com/prysmaticlabs/prysm/testing/require"
 	logTest "github.com/sirupsen/logrus/hooks/test"
+	"google.golang.org/protobuf/proto"
 )
 
 func Test_checkTransitionConfiguration(t *testing.T) {
@@ -31,14 +38,11 @@ func Test_checkTransitionConfiguration(t *testing.T) {
 		m := &mocks.EngineClient{}
 		m.Err = errors.New("something went wrong")
 
-		mockChain := &mockChain2.MockStateNotifier{}
-		srv := &Service{
-			cfg: &config{stateNotifier: mockChain},
-		}
-		srv.engineAPIClient = m
+		srv := setupTransitionConfigTest(t)
+		srv.cfg.stateNotifier = &mockChain.MockStateNotifier{}
 		checkTransitionPollingInterval = time.Millisecond
 		ctx, cancel := context.WithCancel(ctx)
-		go srv.checkTransitionConfiguration(ctx, make(chan *statefeed.BlockProcessedData, 1))
+		go srv.checkTransitionConfiguration(ctx, make(chan *feed.Event, 1))
 		<-time.After(100 * time.Millisecond)
 		cancel()
 		require.LogsContain(t, hook, "Could not check configuration values")
@@ -48,16 +52,13 @@ func Test_checkTransitionConfiguration(t *testing.T) {
 		ctx := context.Background()
 		m := &mocks.EngineClient{}
 		m.Err = errors.New("something went wrong")
+		srv := setupTransitionConfigTest(t)
+		srv.cfg.stateNotifier = &mockChain.MockStateNotifier{}
 
-		mockChain := &mockChain2.MockStateNotifier{}
-		srv := &Service{
-			cfg: &config{stateNotifier: mockChain},
-		}
-		srv.engineAPIClient = m
 		checkTransitionPollingInterval = time.Millisecond
 		ctx, cancel := context.WithCancel(ctx)
 		exit := make(chan bool)
-		notification := make(chan *statefeed.BlockProcessedData)
+		notification := make(chan *feed.Event)
 		go func() {
 			srv.checkTransitionConfiguration(ctx, notification)
 			exit <- true
@@ -72,8 +73,11 @@ func Test_checkTransitionConfiguration(t *testing.T) {
 			}},
 		)
 		require.NoError(t, err)
-		notification <- &statefeed.BlockProcessedData{
-			SignedBlock: wrappedBlock,
+		notification <- &feed.Event{
+			Data: &statefeed.BlockProcessedData{
+				SignedBlock: wrappedBlock,
+			},
+			Type: statefeed.BlockProcessed,
 		}
 		<-exit
 		cancel()
@@ -84,33 +88,110 @@ func Test_checkTransitionConfiguration(t *testing.T) {
 func TestService_handleExchangeConfigurationError(t *testing.T) {
 	hook := logTest.NewGlobal()
 	t.Run("clears existing service error", func(t *testing.T) {
-		srv := &Service{isRunning: true}
-		srv.runError = v1.ErrConfigMismatch
+		srv := setupTransitionConfigTest(t)
+		srv.isRunning = true
+		srv.runError = ErrConfigMismatch
 		srv.handleExchangeConfigurationError(nil)
 		require.Equal(t, true, srv.Status() == nil)
 	})
 	t.Run("does not clear existing service error if wrong kind", func(t *testing.T) {
-		srv := &Service{isRunning: true}
+		srv := setupTransitionConfigTest(t)
+		srv.isRunning = true
 		err := errors.New("something else went wrong")
 		srv.runError = err
 		srv.handleExchangeConfigurationError(nil)
 		require.ErrorIs(t, err, srv.Status())
 	})
 	t.Run("sets service error on config mismatch", func(t *testing.T) {
-		srv := &Service{isRunning: true}
-		srv.handleExchangeConfigurationError(v1.ErrConfigMismatch)
-		require.Equal(t, v1.ErrConfigMismatch, srv.Status())
+		srv := setupTransitionConfigTest(t)
+		srv.isRunning = true
+		srv.handleExchangeConfigurationError(ErrConfigMismatch)
+		require.Equal(t, ErrConfigMismatch, srv.Status())
 		require.LogsContain(t, hook, configMismatchLog)
 	})
 	t.Run("does not set service error if unrelated problem", func(t *testing.T) {
-		srv := &Service{isRunning: true}
+		srv := setupTransitionConfigTest(t)
+		srv.isRunning = true
 		srv.handleExchangeConfigurationError(errors.New("foo"))
 		require.Equal(t, true, srv.Status() == nil)
 		require.LogsContain(t, hook, "Could not check configuration values")
 	})
 }
-func emptyPayload() *enginev1.ExecutionPayload {
-	return &enginev1.ExecutionPayload{
+
+func setupTransitionConfigTest(t testing.TB) *Service {
+	fix := fixtures()
+	request, ok := fix["TransitionConfiguration"].(*pb.TransitionConfiguration)
+	require.Equal(t, true, ok)
+	resp, ok := proto.Clone(request).(*pb.TransitionConfiguration)
+	require.Equal(t, true, ok)
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		defer func() {
+			require.NoError(t, r.Body.Close())
+		}()
+
+		// Change the terminal block hash.
+		h := common.BytesToHash([]byte("foo"))
+		resp.TerminalBlockHash = h[:]
+		respJSON := map[string]interface{}{
+			"jsonrpc": "2.0",
+			"id":      1,
+			"result":  resp,
+		}
+		require.NoError(t, json.NewEncoder(w).Encode(respJSON))
+	}))
+	defer srv.Close()
+
+	rpcClient, err := rpc.DialHTTP(srv.URL)
+	require.NoError(t, err)
+	defer rpcClient.Close()
+
+	service := &Service{
+		cfg: &config{},
+	}
+	service.rpcClient = rpcClient
+	return service
+}
+
+func TestService_logTtdStatus(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		defer func() {
+			require.NoError(t, r.Body.Close())
+		}()
+
+		resp := &pb.ExecutionBlock{TotalDifficulty: "0x12345678"}
+		respJSON := map[string]interface{}{
+			"jsonrpc": "2.0",
+			"id":      1,
+			"result":  resp,
+		}
+		require.NoError(t, json.NewEncoder(w).Encode(respJSON))
+	}))
+	defer srv.Close()
+
+	rpcClient, err := rpc.DialHTTP(srv.URL)
+	require.NoError(t, err)
+	defer rpcClient.Close()
+
+	service := &Service{
+		cfg: &config{},
+	}
+	service.rpcClient = rpcClient
+
+	ttd := new(uint256.Int)
+	reached, err := service.logTtdStatus(context.Background(), ttd.SetUint64(24343))
+	require.NoError(t, err)
+	require.Equal(t, true, reached)
+
+	reached, err = service.logTtdStatus(context.Background(), ttd.SetUint64(323423484))
+	require.NoError(t, err)
+	require.Equal(t, false, reached)
+}
+
+func emptyPayload() *pb.ExecutionPayload {
+	return &pb.ExecutionPayload{
 		ParentHash:    make([]byte, fieldparams.RootLength),
 		FeeRecipient:  make([]byte, fieldparams.FeeRecipientLength),
 		StateRoot:     make([]byte, fieldparams.RootLength),

beacon-chain/powchain/deposit_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/signing"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	testing2 "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
 	"github.com/prysmaticlabs/prysm/container/trie"
@@ -52,6 +53,11 @@ func TestDepositContractAddress_OK(t *testing.T) {
 
 func TestProcessDeposit_OK(t *testing.T) {
 	beaconDB := testDB.SetupDB(t)
+	server, endpoint, err := testing2.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -76,6 +82,11 @@ func TestProcessDeposit_OK(t *testing.T) {
 
 func TestProcessDeposit_InvalidMerkleBranch(t *testing.T) {
 	beaconDB := testDB.SetupDB(t)
+	server, endpoint, err := testing2.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -102,6 +113,11 @@ func TestProcessDeposit_InvalidMerkleBranch(t *testing.T) {
 func TestProcessDeposit_InvalidPublicKey(t *testing.T) {
 	hook := logTest.NewGlobal()
 	beaconDB := testDB.SetupDB(t)
+	server, endpoint, err := testing2.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -138,6 +154,11 @@ func TestProcessDeposit_InvalidPublicKey(t *testing.T) {
 func TestProcessDeposit_InvalidSignature(t *testing.T) {
 	hook := logTest.NewGlobal()
 	beaconDB := testDB.SetupDB(t)
+	server, endpoint, err := testing2.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -173,6 +194,11 @@ func TestProcessDeposit_InvalidSignature(t *testing.T) {
 func TestProcessDeposit_UnableToVerify(t *testing.T) {
 	hook := logTest.NewGlobal()
 	beaconDB := testDB.SetupDB(t)
+	server, endpoint, err := testing2.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -205,6 +231,11 @@ func TestProcessDeposit_UnableToVerify(t *testing.T) {
 
 func TestProcessDeposit_IncompleteDeposit(t *testing.T) {
 	beaconDB := testDB.SetupDB(t)
+	server, endpoint, err := testing2.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -267,6 +298,11 @@ func TestProcessDeposit_IncompleteDeposit(t *testing.T) {
 
 func TestProcessDeposit_AllDepositedSuccessfully(t *testing.T) {
 	beaconDB := testDB.SetupDB(t)
+	server, endpoint, err := testing2.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),

beacon-chain/powchain/engine-api-client/v1/BUILD.bazel

@@ -1,49 +0,0 @@
-load("@prysm//tools/go:def.bzl", "go_library", "go_test")
-
-go_library(
-    name = "go_default_library",
-    srcs = [
-        "auth.go",
-        "client.go",
-        "errors.go",
-        "metrics.go",
-        "options.go",
-    ],
-    importpath = "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1",
-    visibility = ["//beacon-chain:__subpackages__"],
-    deps = [
-        "//config/params:go_default_library",
-        "//proto/engine/v1:go_default_library",
-        "@com_github_ethereum_go_ethereum//common:go_default_library",
-        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
-        "@com_github_ethereum_go_ethereum//rpc:go_default_library",
-        "@com_github_golang_jwt_jwt_v4//:go_default_library",
-        "@com_github_pkg_errors//:go_default_library",
-        "@com_github_prometheus_client_golang//prometheus:go_default_library",
-        "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
-        "@io_opencensus_go//trace:go_default_library",
-    ],
-)
-
-go_test(
-    name = "go_default_test",
-    srcs = [
-        "auth_test.go",
-        "client_test.go",
-    ],
-    embed = [":go_default_library"],
-    deps = [
-        "//beacon-chain/powchain/engine-api-client/v1/mocks:go_default_library",
-        "//config/fieldparams:go_default_library",
-        "//config/params:go_default_library",
-        "//encoding/bytesutil:go_default_library",
-        "//proto/engine/v1:go_default_library",
-        "//testing/require:go_default_library",
-        "@com_github_ethereum_go_ethereum//common:go_default_library",
-        "@com_github_ethereum_go_ethereum//rpc:go_default_library",
-        "@com_github_golang_jwt_jwt_v4//:go_default_library",
-        "@com_github_holiman_uint256//:go_default_library",
-        "@com_github_pkg_errors//:go_default_library",
-        "@org_golang_google_protobuf//proto:go_default_library",
-    ],
-)

beacon-chain/powchain/engine-api-client/v1/mocks/BUILD.bazel

@@ -1,13 +0,0 @@
-load("@prysm//tools/go:def.bzl", "go_library")
-
-go_library(
-    name = "go_default_library",
-    srcs = ["client.go"],
-    importpath = "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1/mocks",
-    visibility = ["//visibility:public"],
-    deps = [
-        "//proto/engine/v1:go_default_library",
-        "@com_github_ethereum_go_ethereum//common:go_default_library",
-        "@com_github_pkg_errors//:go_default_library",
-    ],
-)

beacon-chain/powchain/engine-api-client/v1/mocks/client.go

@@ -1,59 +0,0 @@
-package mocks
-
-import (
-	"context"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/pkg/errors"
-	pb "github.com/prysmaticlabs/prysm/proto/engine/v1"
-)
-
-// EngineClient --
-type EngineClient struct {
-	NewPayloadResp        []byte
-	PayloadIDBytes        *pb.PayloadIDBytes
-	ForkChoiceUpdatedResp []byte
-	ExecutionPayload      *pb.ExecutionPayload
-	ExecutionBlock        *pb.ExecutionBlock
-	Err                   error
-	ErrLatestExecBlock    error
-	ErrExecBlockByHash    error
-	ErrForkchoiceUpdated  error
-	BlockByHashMap        map[[32]byte]*pb.ExecutionBlock
-}
-
-// NewPayload --
-func (e *EngineClient) NewPayload(_ context.Context, _ *pb.ExecutionPayload) ([]byte, error) {
-	return e.NewPayloadResp, nil
-}
-
-// ForkchoiceUpdated --
-func (e *EngineClient) ForkchoiceUpdated(
-	_ context.Context, _ *pb.ForkchoiceState, _ *pb.PayloadAttributes,
-) (*pb.PayloadIDBytes, []byte, error) {
-	return e.PayloadIDBytes, e.ForkChoiceUpdatedResp, e.ErrForkchoiceUpdated
-}
-
-// GetPayload --
-func (e *EngineClient) GetPayload(_ context.Context, _ [8]byte) (*pb.ExecutionPayload, error) {
-	return e.ExecutionPayload, nil
-}
-
-// ExchangeTransitionConfiguration --
-func (e *EngineClient) ExchangeTransitionConfiguration(_ context.Context, _ *pb.TransitionConfiguration) error {
-	return e.Err
-}
-
-// LatestExecutionBlock --
-func (e *EngineClient) LatestExecutionBlock(_ context.Context) (*pb.ExecutionBlock, error) {
-	return e.ExecutionBlock, e.ErrLatestExecBlock
-}
-
-// ExecutionBlockByHash --
-func (e *EngineClient) ExecutionBlockByHash(_ context.Context, h common.Hash) (*pb.ExecutionBlock, error) {
-	b, ok := e.BlockByHashMap[h]
-	if !ok {
-		return nil, errors.New("block not found")
-	}
-	return b, e.ErrExecBlockByHash
-}

beacon-chain/powchain/engine-api-client/v1/options.go

@@ -1,39 +0,0 @@
-package v1
-
-import (
-	"net/http"
-)
-
-// Option for configuring the engine API client.
-type Option func(c *Client) error
-
-type config struct {
-	httpClient *http.Client
-}
-
-func defaultConfig() *config {
-	return &config{
-		httpClient: &http.Client{
-			Timeout: DefaultTimeout,
-		},
-	}
-}
-
-// WithJWTSecret allows setting a JWT secret for authenticating
-// the client via HTTP connections.
-func WithJWTSecret(secret []byte) Option {
-	return func(c *Client) error {
-		if len(secret) == 0 {
-			return nil
-		}
-		authTransport := &jwtTransport{
-			underlyingTransport: http.DefaultTransport,
-			jwtSecret:           secret,
-		}
-		c.cfg.httpClient = &http.Client{
-			Timeout:   DefaultTimeout,
-			Transport: authTransport,
-		}
-		return nil
-	}
-}

beacon-chain/powchain/engine_client.go

@@ -1,22 +1,21 @@
-// Package v1 defines an API client for the engine API defined in https://github.com/ethereum/execution-apis.
-// This client is used for the Prysm consensus node to connect to execution node as part of
-// the Ethereum proof-of-stake machinery.
-package v1
+package powchain
 
 import (
 	"bytes"
 	"context"
 	"fmt"
 	"math/big"
-	"net/url"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/holiman/uint256"
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
 	pb "github.com/prysmaticlabs/prysm/proto/engine/v1"
+	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 )
 
@@ -33,8 +32,6 @@ const (
 	ExecutionBlockByHashMethod = "eth_getBlockByHash"
 	// ExecutionBlockByNumberMethod request string for JSON-RPC.
 	ExecutionBlockByNumberMethod = "eth_getBlockByNumber"
-	// DefaultTimeout for HTTP.
-	DefaultTimeout = time.Second * 5
 )
 
 // ForkchoiceUpdatedResponse is the response kind received by the
@@ -44,9 +41,9 @@ type ForkchoiceUpdatedResponse struct {
 	PayloadId *pb.PayloadIDBytes `json:"payloadId"`
 }
 
-// Caller defines a client that can interact with an Ethereum
+// EngineCaller defines a client that can interact with an Ethereum
 // execution node's engine service via JSON-RPC.
-type Caller interface {
+type EngineCaller interface {
 	NewPayload(ctx context.Context, payload *pb.ExecutionPayload) ([]byte, error)
 	ForkchoiceUpdated(
 		ctx context.Context, state *pb.ForkchoiceState, attrs *pb.PayloadAttributes,
@@ -55,48 +52,12 @@ type Caller interface {
 	ExchangeTransitionConfiguration(
 		ctx context.Context, cfg *pb.TransitionConfiguration,
 	) error
-	LatestExecutionBlock(ctx context.Context) (*pb.ExecutionBlock, error)
 	ExecutionBlockByHash(ctx context.Context, hash common.Hash) (*pb.ExecutionBlock, error)
-}
-
-// Client defines a new engine API client for the Prysm consensus node
-// to interact with an Ethereum execution node.
-type Client struct {
-	cfg *config
-	rpc *rpc.Client
-}
-
-// New returns a ready, engine API client from an endpoint and configuration options.
-// Only http(s) and ipc (inter-process communication) URL schemes are supported.
-func New(ctx context.Context, endpoint string, opts ...Option) (*Client, error) {
-	u, err := url.Parse(endpoint)
-	if err != nil {
-		return nil, err
-	}
-	c := &Client{
-		cfg: defaultConfig(),
-	}
-	for _, opt := range opts {
-		if err := opt(c); err != nil {
-			return nil, err
-		}
-	}
-	switch u.Scheme {
-	case "http", "https":
-		c.rpc, err = rpc.DialHTTPWithClient(endpoint, c.cfg.httpClient)
-	case "":
-		c.rpc, err = rpc.DialIPC(ctx, endpoint)
-	default:
-		return nil, errors.Wrapf(ErrUnsupportedScheme, "%q", u.Scheme)
-	}
-	if err != nil {
-		return nil, err
-	}
-	return c, nil
+	GetTerminalBlockHash(ctx context.Context) ([]byte, bool, error)
 }
 
 // NewPayload calls the engine_newPayloadV1 method via JSON-RPC.
-func (c *Client) NewPayload(ctx context.Context, payload *pb.ExecutionPayload) ([]byte, error) {
+func (s *Service) NewPayload(ctx context.Context, payload *pb.ExecutionPayload) ([]byte, error) {
 	ctx, span := trace.StartSpan(ctx, "powchain.engine-api-client.NewPayload")
 	defer span.End()
 	start := time.Now()
@@ -105,7 +66,7 @@ func (c *Client) NewPayload(ctx context.Context, payload *pb.ExecutionPayload) (
 	}()
 
 	result := &pb.PayloadStatus{}
-	err := c.rpc.CallContext(ctx, result, NewPayloadMethod, payload)
+	err := s.rpcClient.CallContext(ctx, result, NewPayloadMethod, payload)
 	if err != nil {
 		return nil, handleRPCError(err)
 	}
@@ -127,7 +88,7 @@ func (c *Client) NewPayload(ctx context.Context, payload *pb.ExecutionPayload) (
 }
 
 // ForkchoiceUpdated calls the engine_forkchoiceUpdatedV1 method via JSON-RPC.
-func (c *Client) ForkchoiceUpdated(
+func (s *Service) ForkchoiceUpdated(
 	ctx context.Context, state *pb.ForkchoiceState, attrs *pb.PayloadAttributes,
 ) (*pb.PayloadIDBytes, []byte, error) {
 	ctx, span := trace.StartSpan(ctx, "powchain.engine-api-client.ForkchoiceUpdated")
@@ -138,7 +99,7 @@ func (c *Client) ForkchoiceUpdated(
 	}()
 
 	result := &ForkchoiceUpdatedResponse{}
-	err := c.rpc.CallContext(ctx, result, ForkchoiceUpdatedMethod, state, attrs)
+	err := s.rpcClient.CallContext(ctx, result, ForkchoiceUpdatedMethod, state, attrs)
 	if err != nil {
 		return nil, nil, handleRPCError(err)
 	}
@@ -162,7 +123,7 @@ func (c *Client) ForkchoiceUpdated(
 }
 
 // GetPayload calls the engine_getPayloadV1 method via JSON-RPC.
-func (c *Client) GetPayload(ctx context.Context, payloadId [8]byte) (*pb.ExecutionPayload, error) {
+func (s *Service) GetPayload(ctx context.Context, payloadId [8]byte) (*pb.ExecutionPayload, error) {
 	ctx, span := trace.StartSpan(ctx, "powchain.engine-api-client.GetPayload")
 	defer span.End()
 	start := time.Now()
@@ -171,12 +132,12 @@ func (c *Client) GetPayload(ctx context.Context, payloadId [8]byte) (*pb.Executi
 	}()
 
 	result := &pb.ExecutionPayload{}
-	err := c.rpc.CallContext(ctx, result, GetPayloadMethod, pb.PayloadIDBytes(payloadId))
+	err := s.rpcClient.CallContext(ctx, result, GetPayloadMethod, pb.PayloadIDBytes(payloadId))
 	return result, handleRPCError(err)
 }
 
 // ExchangeTransitionConfiguration calls the engine_exchangeTransitionConfigurationV1 method via JSON-RPC.
-func (c *Client) ExchangeTransitionConfiguration(
+func (s *Service) ExchangeTransitionConfiguration(
 	ctx context.Context, cfg *pb.TransitionConfiguration,
 ) error {
 	ctx, span := trace.StartSpan(ctx, "powchain.engine-api-client.ExchangeTransitionConfiguration")
@@ -186,7 +147,7 @@ func (c *Client) ExchangeTransitionConfiguration(
 	zeroBigNum := big.NewInt(0)
 	cfg.TerminalBlockNumber = zeroBigNum.Bytes()
 	result := &pb.TransitionConfiguration{}
-	if err := c.rpc.CallContext(ctx, result, ExchangeTransitionConfigurationMethod, cfg); err != nil {
+	if err := s.rpcClient.CallContext(ctx, result, ExchangeTransitionConfigurationMethod, cfg); err != nil {
 		return handleRPCError(err)
 	}
 	// We surface an error to the user if local configuration settings mismatch
@@ -216,14 +177,86 @@ func (c *Client) ExchangeTransitionConfiguration(
 	return nil
 }
 
+// GetTerminalBlockHash returns the valid terminal block hash based on total difficulty.
+//
+// Spec code:
+// def get_pow_block_at_terminal_total_difficulty(pow_chain: Dict[Hash32, PowBlock]) -> Optional[PowBlock]:
+//    # `pow_chain` abstractly represents all blocks in the PoW chain
+//    for block in pow_chain:
+//        parent = pow_chain[block.parent_hash]
+//        block_reached_ttd = block.total_difficulty >= TERMINAL_TOTAL_DIFFICULTY
+//        parent_reached_ttd = parent.total_difficulty >= TERMINAL_TOTAL_DIFFICULTY
+//        if block_reached_ttd and not parent_reached_ttd:
+//            return block
+//
+//    return None
+func (s *Service) GetTerminalBlockHash(ctx context.Context) ([]byte, bool, error) {
+	ttd := new(big.Int)
+	ttd.SetString(params.BeaconConfig().TerminalTotalDifficulty, 10)
+	terminalTotalDifficulty, overflows := uint256.FromBig(ttd)
+	if overflows {
+		return nil, false, errors.New("could not convert terminal total difficulty to uint256")
+	}
+	blk, err := s.LatestExecutionBlock(ctx)
+	if err != nil {
+		return nil, false, errors.Wrap(err, "could not get latest execution block")
+	}
+	if blk == nil {
+		return nil, false, errors.New("latest execution block is nil")
+	}
+
+	for {
+		if ctx.Err() != nil {
+			return nil, false, ctx.Err()
+		}
+		currentTotalDifficulty, err := tDStringToUint256(blk.TotalDifficulty)
+		if err != nil {
+			return nil, false, errors.Wrap(err, "could not convert total difficulty to uint256")
+		}
+		blockReachedTTD := currentTotalDifficulty.Cmp(terminalTotalDifficulty) >= 0
+
+		parentHash := bytesutil.ToBytes32(blk.ParentHash)
+		if len(blk.ParentHash) == 0 || parentHash == params.BeaconConfig().ZeroHash {
+			return nil, false, nil
+		}
+		parentBlk, err := s.ExecutionBlockByHash(ctx, parentHash)
+		if err != nil {
+			return nil, false, errors.Wrap(err, "could not get parent execution block")
+		}
+		if parentBlk == nil {
+			return nil, false, errors.New("parent execution block is nil")
+		}
+		if blockReachedTTD {
+			parentTotalDifficulty, err := tDStringToUint256(parentBlk.TotalDifficulty)
+			if err != nil {
+				return nil, false, errors.Wrap(err, "could not convert total difficulty to uint256")
+			}
+			parentReachedTTD := parentTotalDifficulty.Cmp(terminalTotalDifficulty) >= 0
+			if !parentReachedTTD {
+				log.WithFields(logrus.Fields{
+					"number":   blk.Number,
+					"hash":     fmt.Sprintf("%#x", bytesutil.Trunc(blk.Hash)),
+					"td":       blk.TotalDifficulty,
+					"parentTd": parentBlk.TotalDifficulty,
+					"ttd":      terminalTotalDifficulty,
+				}).Info("Retrieved terminal block hash")
+				return blk.Hash, true, nil
+			}
+		} else {
+			return nil, false, nil
+		}
+		blk = parentBlk
+	}
+}
+
 // LatestExecutionBlock fetches the latest execution engine block by calling
 // eth_blockByNumber via JSON-RPC.
-func (c *Client) LatestExecutionBlock(ctx context.Context) (*pb.ExecutionBlock, error) {
+func (s *Service) LatestExecutionBlock(ctx context.Context) (*pb.ExecutionBlock, error) {
 	ctx, span := trace.StartSpan(ctx, "powchain.engine-api-client.LatestExecutionBlock")
 	defer span.End()
 
 	result := &pb.ExecutionBlock{}
-	err := c.rpc.CallContext(
+	err := s.rpcClient.CallContext(
 		ctx,
 		result,
 		ExecutionBlockByNumberMethod,
@@ -235,12 +268,12 @@ func (c *Client) LatestExecutionBlock(ctx context.Context) (*pb.ExecutionBlock,
 
 // ExecutionBlockByHash fetches an execution engine block by hash by calling
 // eth_blockByHash via JSON-RPC.
-func (c *Client) ExecutionBlockByHash(ctx context.Context, hash common.Hash) (*pb.ExecutionBlock, error) {
+func (s *Service) ExecutionBlockByHash(ctx context.Context, hash common.Hash) (*pb.ExecutionBlock, error) {
 	ctx, span := trace.StartSpan(ctx, "powchain.engine-api-client.ExecutionBlockByHash")
 	defer span.End()
 
 	result := &pb.ExecutionBlock{}
-	err := c.rpc.CallContext(ctx, result, ExecutionBlockByHashMethod, hash, false /* no full transaction objects */)
+	err := s.rpcClient.CallContext(ctx, result, ExecutionBlockByHashMethod, hash, false /* no full transaction objects */)
 	return result, handleRPCError(err)
 }
 
@@ -249,6 +282,9 @@ func handleRPCError(err error) error {
 	if err == nil {
 		return nil
 	}
+	if isTimeout(err) {
+		return errors.Wrapf(ErrHTTPTimeout, "%s", err)
+	}
 	e, ok := err.(rpc.Error)
 	if !ok {
 		return errors.Wrap(err, "got an unexpected error")
@@ -277,3 +313,28 @@ func handleRPCError(err error) error {
 		return err
 	}
 }
+
+// ErrHTTPTimeout returns true if the error is a http.Client timeout error.
+var ErrHTTPTimeout = errors.New("timeout from http.Client")
+
+type httpTimeoutError interface {
+	Error() string
+	Timeout() bool
+}
+
+func isTimeout(e error) bool {
+	t, ok := e.(httpTimeoutError)
+	return ok && t.Timeout()
+}
+
+func tDStringToUint256(td string) (*uint256.Int, error) {
+	b, err := hexutil.DecodeBig(td)
+	if err != nil {
+		return nil, err
+	}
+	i, overflows := uint256.FromBig(b)
+	if overflows {
+		return nil, errors.New("total difficulty overflowed")
+	}
+	return i, nil
+}

beacon-chain/powchain/engine_client_test.go

@@ -1,4 +1,4 @@
-package v1
+package powchain
 
 import (
 	"context"
@@ -15,7 +15,7 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/holiman/uint256"
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1/mocks"
+	mocks "github.com/prysmaticlabs/prysm/beacon-chain/powchain/testing"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
@@ -25,8 +25,8 @@ import (
 )
 
 var (
-	_ = Caller(&Client{})
-	_ = Caller(&mocks.EngineClient{})
+	_ = EngineCaller(&Service{})
+	_ = EngineCaller(&mocks.EngineClient{})
 )
 
 func TestClient_IPC(t *testing.T) {
@@ -34,8 +34,8 @@ func TestClient_IPC(t *testing.T) {
 	defer server.Stop()
 	rpcClient := rpc.DialInProc(server)
 	defer rpcClient.Close()
-	client := &Client{}
-	client.rpc = rpcClient
+	srv := &Service{}
+	srv.rpcClient = rpcClient
 	ctx := context.Background()
 	fix := fixtures()
 
@@ -43,14 +43,14 @@ func TestClient_IPC(t *testing.T) {
 		want, ok := fix["ExecutionPayload"].(*pb.ExecutionPayload)
 		require.Equal(t, true, ok)
 		payloadId := [8]byte{1}
-		resp, err := client.GetPayload(ctx, payloadId)
+		resp, err := srv.GetPayload(ctx, payloadId)
 		require.NoError(t, err)
 		require.DeepEqual(t, want, resp)
 	})
 	t.Run(ForkchoiceUpdatedMethod, func(t *testing.T) {
 		want, ok := fix["ForkchoiceUpdatedResponse"].(*ForkchoiceUpdatedResponse)
 		require.Equal(t, true, ok)
-		payloadID, validHash, err := client.ForkchoiceUpdated(ctx, &pb.ForkchoiceState{}, &pb.PayloadAttributes{})
+		payloadID, validHash, err := srv.ForkchoiceUpdated(ctx, &pb.ForkchoiceState{}, &pb.PayloadAttributes{})
 		require.NoError(t, err)
 		require.DeepEqual(t, want.Status.LatestValidHash, validHash)
 		require.DeepEqual(t, want.PayloadId, payloadID)
@@ -60,20 +60,20 @@ func TestClient_IPC(t *testing.T) {
 		require.Equal(t, true, ok)
 		req, ok := fix["ExecutionPayload"].(*pb.ExecutionPayload)
 		require.Equal(t, true, ok)
-		latestValidHash, err := client.NewPayload(ctx, req)
+		latestValidHash, err := srv.NewPayload(ctx, req)
 		require.NoError(t, err)
 		require.DeepEqual(t, bytesutil.ToBytes32(want.LatestValidHash), bytesutil.ToBytes32(latestValidHash))
 	})
 	t.Run(ExchangeTransitionConfigurationMethod, func(t *testing.T) {
 		want, ok := fix["TransitionConfiguration"].(*pb.TransitionConfiguration)
 		require.Equal(t, true, ok)
-		err := client.ExchangeTransitionConfiguration(ctx, want)
+		err := srv.ExchangeTransitionConfiguration(ctx, want)
 		require.NoError(t, err)
 	})
 	t.Run(ExecutionBlockByNumberMethod, func(t *testing.T) {
 		want, ok := fix["ExecutionBlock"].(*pb.ExecutionBlock)
 		require.Equal(t, true, ok)
-		resp, err := client.LatestExecutionBlock(ctx)
+		resp, err := srv.LatestExecutionBlock(ctx)
 		require.NoError(t, err)
 		require.DeepEqual(t, want, resp)
 	})
@@ -81,7 +81,7 @@ func TestClient_IPC(t *testing.T) {
 		want, ok := fix["ExecutionBlock"].(*pb.ExecutionBlock)
 		require.Equal(t, true, ok)
 		arg := common.BytesToHash([]byte("foo"))
-		resp, err := client.ExecutionBlockByHash(ctx, arg)
+		resp, err := srv.ExecutionBlockByHash(ctx, arg)
 		require.NoError(t, err)
 		require.DeepEqual(t, want, resp)
 	})
@@ -125,8 +125,8 @@ func TestClient_HTTP(t *testing.T) {
 		require.NoError(t, err)
 		defer rpcClient.Close()
 
-		client := &Client{}
-		client.rpc = rpcClient
+		client := &Service{}
+		client.rpcClient = rpcClient
 
 		// We call the RPC method via HTTP and expect a proper result.
 		resp, err := client.GetPayload(ctx, payloadId)
@@ -146,10 +146,10 @@ func TestClient_HTTP(t *testing.T) {
 		}
 		want, ok := fix["ForkchoiceUpdatedResponse"].(*ForkchoiceUpdatedResponse)
 		require.Equal(t, true, ok)
-		client := forkchoiceUpdateSetup(t, forkChoiceState, payloadAttributes, want)
+		srv := forkchoiceUpdateSetup(t, forkChoiceState, payloadAttributes, want)
 
 		// We call the RPC method via HTTP and expect a proper result.
-		payloadID, validHash, err := client.ForkchoiceUpdated(ctx, forkChoiceState, payloadAttributes)
+		payloadID, validHash, err := srv.ForkchoiceUpdated(ctx, forkChoiceState, payloadAttributes)
 		require.NoError(t, err)
 		require.DeepEqual(t, want.Status.LatestValidHash, validHash)
 		require.DeepEqual(t, want.PayloadId, payloadID)
@@ -332,11 +332,11 @@ func TestClient_HTTP(t *testing.T) {
 		require.NoError(t, err)
 		defer rpcClient.Close()
 
-		client := &Client{}
-		client.rpc = rpcClient
+		service := &Service{}
+		service.rpcClient = rpcClient
 
 		// We call the RPC method via HTTP and expect a proper result.
-		resp, err := client.LatestExecutionBlock(ctx)
+		resp, err := service.LatestExecutionBlock(ctx)
 		require.NoError(t, err)
 		require.DeepEqual(t, want, resp)
 	})
@@ -371,8 +371,8 @@ func TestClient_HTTP(t *testing.T) {
 		require.NoError(t, err)
 		defer rpcClient.Close()
 
-		client := &Client{}
-		client.rpc = rpcClient
+		client := &Service{}
+		client.rpcClient = rpcClient
 
 		// We call the RPC method via HTTP and expect a proper result.
 		err = client.ExchangeTransitionConfiguration(ctx, want)
@@ -408,16 +408,165 @@ func TestClient_HTTP(t *testing.T) {
 		require.NoError(t, err)
 		defer rpcClient.Close()
 
-		client := &Client{}
-		client.rpc = rpcClient
+		service := &Service{}
+		service.rpcClient = rpcClient
 
 		// We call the RPC method via HTTP and expect a proper result.
-		resp, err := client.ExecutionBlockByHash(ctx, arg)
+		resp, err := service.ExecutionBlockByHash(ctx, arg)
 		require.NoError(t, err)
 		require.DeepEqual(t, want, resp)
 	})
 }
 
+func TestServer_getPowBlockHashAtTerminalTotalDifficulty(t *testing.T) {
+	tests := []struct {
+		name                  string
+		paramsTd              string
+		currentPowBlock       *pb.ExecutionBlock
+		parentPowBlock        *pb.ExecutionBlock
+		errLatestExecutionBlk error
+		wantTerminalBlockHash []byte
+		wantExists            bool
+		errString             string
+	}{
+		{
+			name:      "config td overflows",
+			paramsTd:  "1115792089237316195423570985008687907853269984665640564039457584007913129638912",
+			errString: "could not convert terminal total difficulty to uint256",
+		},
+		{
+			name:                  "could not get latest execution block",
+			paramsTd:              "1",
+			errLatestExecutionBlk: errors.New("blah"),
+			errString:             "could not get latest execution block",
+		},
+		{
+			name:      "nil latest execution block",
+			paramsTd:  "1",
+			errString: "latest execution block is nil",
+		},
+		{
+			name:     "current execution block invalid TD",
+			paramsTd: "1",
+			currentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'a'},
+				TotalDifficulty: "1115792089237316195423570985008687907853269984665640564039457584007913129638912",
+			},
+			errString: "could not convert total difficulty to uint256",
+		},
+		{
+			name:     "current execution block has zero hash parent",
+			paramsTd: "2",
+			currentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'a'},
+				ParentHash:      params.BeaconConfig().ZeroHash[:],
+				TotalDifficulty: "0x3",
+			},
+		},
+		{
+			name:     "could not get parent block",
+			paramsTd: "2",
+			currentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'a'},
+				ParentHash:      []byte{'b'},
+				TotalDifficulty: "0x3",
+			},
+			errString: "could not get parent execution block",
+		},
+		{
+			name:     "parent execution block invalid TD",
+			paramsTd: "2",
+			currentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'a'},
+				ParentHash:      []byte{'b'},
+				TotalDifficulty: "0x3",
+			},
+			parentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'b'},
+				ParentHash:      []byte{'c'},
+				TotalDifficulty: "1",
+			},
+			errString: "could not convert total difficulty to uint256",
+		},
+		{
+			name:     "happy case",
+			paramsTd: "2",
+			currentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'a'},
+				ParentHash:      []byte{'b'},
+				TotalDifficulty: "0x3",
+			},
+			parentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'b'},
+				ParentHash:      []byte{'c'},
+				TotalDifficulty: "0x1",
+			},
+			wantExists:            true,
+			wantTerminalBlockHash: []byte{'a'},
+		},
+		{
+			name:     "ttd not reached",
+			paramsTd: "3",
+			currentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'a'},
+				ParentHash:      []byte{'b'},
+				TotalDifficulty: "0x2",
+			},
+			parentPowBlock: &pb.ExecutionBlock{
+				Hash:            []byte{'b'},
+				ParentHash:      []byte{'c'},
+				TotalDifficulty: "0x1",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := params.BeaconConfig()
+			cfg.TerminalTotalDifficulty = tt.paramsTd
+			params.OverrideBeaconConfig(cfg)
+			var m map[[32]byte]*pb.ExecutionBlock
+			if tt.parentPowBlock != nil {
+				m = map[[32]byte]*pb.ExecutionBlock{
+					bytesutil.ToBytes32(tt.parentPowBlock.Hash): tt.parentPowBlock,
+				}
+			}
+			client := mocks.EngineClient{
+				ErrLatestExecBlock: tt.errLatestExecutionBlk,
+				ExecutionBlock:     tt.currentPowBlock,
+				BlockByHashMap:     m,
+			}
+			b, e, err := client.GetTerminalBlockHash(context.Background())
+			if tt.errString != "" {
+				require.ErrorContains(t, tt.errString, err)
+			} else {
+				require.NoError(t, err)
+				require.DeepEqual(t, tt.wantExists, e)
+				require.DeepEqual(t, tt.wantTerminalBlockHash, b)
+			}
+		})
+	}
+}
+
+func Test_tDStringToUint256(t *testing.T) {
+	i, err := tDStringToUint256("0x0")
+	require.NoError(t, err)
+	require.DeepEqual(t, uint256.NewInt(0), i)
+
+	i, err = tDStringToUint256("0x10000")
+	require.NoError(t, err)
+	require.DeepEqual(t, uint256.NewInt(65536), i)
+
+	_, err = tDStringToUint256("100")
+	require.ErrorContains(t, "hex string without 0x prefix", err)
+
+	_, err = tDStringToUint256("0xzzzzzz")
+	require.ErrorContains(t, "invalid hex string", err)
+
+	_, err = tDStringToUint256("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" +
+		"FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF")
+	require.ErrorContains(t, "hex number > 256 bits", err)
+}
+
 func TestExchangeTransitionConfiguration(t *testing.T) {
 	fix := fixtures()
 	ctx := context.Background()
@@ -449,10 +598,10 @@ func TestExchangeTransitionConfiguration(t *testing.T) {
 		require.NoError(t, err)
 		defer rpcClient.Close()
 
-		client := &Client{}
-		client.rpc = rpcClient
+		service := &Service{}
+		service.rpcClient = rpcClient
 
-		err = client.ExchangeTransitionConfiguration(ctx, request)
+		err = service.ExchangeTransitionConfiguration(ctx, request)
 		require.Equal(t, true, errors.Is(err, ErrConfigMismatch))
 	})
 	t.Run("wrong terminal total difficulty", func(t *testing.T) {
@@ -482,16 +631,17 @@ func TestExchangeTransitionConfiguration(t *testing.T) {
 		require.NoError(t, err)
 		defer rpcClient.Close()
 
-		client := &Client{}
-		client.rpc = rpcClient
+		service := &Service{}
+		service.rpcClient = rpcClient
 
-		err = client.ExchangeTransitionConfiguration(ctx, request)
+		err = service.ExchangeTransitionConfiguration(ctx, request)
 		require.Equal(t, true, errors.Is(err, ErrConfigMismatch))
 	})
 }
 
 type customError struct {
-	code int
+	code    int
+	timeout bool
 }
 
 func (c *customError) ErrorCode() int {
@@ -502,6 +652,10 @@ func (*customError) Error() string {
 	return "something went wrong"
 }
 
+func (c *customError) Timeout() bool {
+	return c.timeout
+}
+
 type dataError struct {
 	code int
 	data interface{}
@@ -534,6 +688,11 @@ func Test_handleRPCError(t *testing.T) {
 			expectedContains: "got an unexpected error",
 			given:            errors.New("foo"),
 		},
+		{
+			name:             "HTTP times out",
+			expectedContains: ErrHTTPTimeout.Error(),
+			given:            &customError{timeout: true},
+		},
 		{
 			name:             "ErrParse",
 			expectedContains: ErrParse.Error(),
@@ -807,7 +966,7 @@ func (*testEngineService) NewPayloadV1(
 	return item
 }
 
-func forkchoiceUpdateSetup(t *testing.T, fcs *pb.ForkchoiceState, att *pb.PayloadAttributes, res *ForkchoiceUpdatedResponse) *Client {
+func forkchoiceUpdateSetup(t *testing.T, fcs *pb.ForkchoiceState, att *pb.PayloadAttributes, res *ForkchoiceUpdatedResponse) *Service {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")
 		defer func() {
@@ -841,13 +1000,12 @@ func forkchoiceUpdateSetup(t *testing.T, fcs *pb.ForkchoiceState, att *pb.Payloa
 	rpcClient, err := rpc.DialHTTP(srv.URL)
 	require.NoError(t, err)
 
-	client := &Client{}
-	client.rpc = rpcClient
-
-	return client
+	service := &Service{}
+	service.rpcClient = rpcClient
+	return service
 }
 
-func newPayloadSetup(t *testing.T, status *pb.PayloadStatus, payload *pb.ExecutionPayload) *Client {
+func newPayloadSetup(t *testing.T, status *pb.PayloadStatus, payload *pb.ExecutionPayload) *Service {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")
 		defer func() {
@@ -876,7 +1034,7 @@ func newPayloadSetup(t *testing.T, status *pb.PayloadStatus, payload *pb.Executi
 	rpcClient, err := rpc.DialHTTP(srv.URL)
 	require.NoError(t, err)
 
-	client := &Client{}
-	client.rpc = rpcClient
-	return client
+	service := &Service{}
+	service.rpcClient = rpcClient
+	return service
 }

beacon-chain/powchain/errors.go

@@ -1,4 +1,4 @@
-package v1
+package powchain
 
 import "github.com/pkg/errors"
 
@@ -24,12 +24,6 @@ var (
 	// ErrConfigMismatch when the execution node's terminal total difficulty or
 	// terminal block hash received via the API mismatches Prysm's configuration value.
 	ErrConfigMismatch = errors.New("execution client configuration mismatch")
-	// ErrMismatchTerminalBlockHash when the terminal block hash value received via
-	// the API mismatches Prysm's configuration value.
-	ErrMismatchTerminalBlockHash = errors.New("terminal block hash mismatch")
-	// ErrMismatchTerminalTotalDiff when the terminal total difficulty value received via
-	// the API mismatches Prysm's configuration value.
-	ErrMismatchTerminalTotalDiff = errors.New("terminal total difficulty mismatch")
 	// ErrAcceptedSyncingPayloadStatus when the status of the payload is syncing or accepted.
 	ErrAcceptedSyncingPayloadStatus = errors.New("payload status is SYNCING or ACCEPTED")
 	// ErrInvalidPayloadStatus when the status of the payload is invalid.

beacon-chain/powchain/log_processing_test.go

@@ -34,6 +34,11 @@ func TestProcessDepositLog_OK(t *testing.T) {
 	depositCache, err := depositcache.New()
 	require.NoError(t, err)
 
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -97,6 +102,11 @@ func TestProcessDepositLog_InsertsPendingDeposit(t *testing.T) {
 	beaconDB := testDB.SetupDB(t)
 	depositCache, err := depositcache.New()
 	require.NoError(t, err)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
@@ -154,6 +164,11 @@ func TestUnpackDepositLogData_OK(t *testing.T) {
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
 	beaconDB := testDB.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -203,6 +218,11 @@ func TestProcessETH2GenesisLog_8DuplicatePubkeys(t *testing.T) {
 	beaconDB := testDB.SetupDB(t)
 	depositCache, err := depositcache.New()
 	require.NoError(t, err)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
@@ -274,6 +294,11 @@ func TestProcessETH2GenesisLog(t *testing.T) {
 	depositCache, err := depositcache.New()
 	require.NoError(t, err)
 
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -359,6 +384,11 @@ func TestProcessETH2GenesisLog_CorrectNumOfDeposits(t *testing.T) {
 	kvStore := testDB.SetupDB(t)
 	depositCache, err := depositcache.New()
 	require.NoError(t, err)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
@@ -452,6 +482,11 @@ func TestProcessETH2GenesisLog_LargePeriodOfNoLogs(t *testing.T) {
 	kvStore := testDB.SetupDB(t)
 	depositCache, err := depositcache.New()
 	require.NoError(t, err)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
@@ -561,6 +596,11 @@ func TestCheckForChainstart_NoValidator(t *testing.T) {
 func newPowchainService(t *testing.T, eth1Backend *mock.TestAccount, beaconDB db.Database) *Service {
 	depositCache, err := depositcache.New()
 	require.NoError(t, err)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(eth1Backend.ContractAddr),

beacon-chain/powchain/metrics.go

@@ -1,4 +1,4 @@
-package v1
+package powchain
 
 import (
 	"github.com/prometheus/client_golang/prometheus"

beacon-chain/powchain/options.go

@@ -8,6 +8,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	"github.com/prysmaticlabs/prysm/network"
+	"github.com/prysmaticlabs/prysm/network/authorization"
 )
 
 type Option func(s *Service) error
@@ -32,10 +33,29 @@ func WithHttpEndpoints(endpointStrings []string) Option {
 	}
 }
 
-// WithExecutionClientJWTSecret for authenticating the execution node JSON-RPC endpoint.
-func WithExecutionClientJWTSecret(jwtSecret []byte) Option {
+// WithHttpEndpointsAndJWTSecret for authenticating the execution node JSON-RPC endpoint.
+func WithHttpEndpointsAndJWTSecret(endpointStrings []string, secret []byte) Option {
 	return func(s *Service) error {
-		s.cfg.executionEndpointJWTSecret = jwtSecret
+		if len(secret) == 0 {
+			return nil
+		}
+		stringEndpoints := dedupEndpoints(endpointStrings)
+		endpoints := make([]network.Endpoint, len(stringEndpoints))
+		// Overwrite authorization type for all endpoints to be of a bearer
+		// type.
+		for i, e := range stringEndpoints {
+			hEndpoint := HttpEndpoint(e)
+			hEndpoint.Auth.Method = authorization.Bearer
+			hEndpoint.Auth.Value = string(secret)
+			endpoints[i] = hEndpoint
+		}
+		// Select first http endpoint in the provided list.
+		var currEndpoint network.Endpoint
+		if len(endpointStrings) > 0 {
+			currEndpoint = endpoints[0]
+		}
+		s.cfg.httpEndpoints = endpoints
+		s.cfg.currHttpEndpoint = currEndpoint
 		return nil
 	}
 }

beacon-chain/powchain/rpc_connection.go

@@ -0,0 +1,176 @@
+package powchain
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"time"
+
+	"github.com/ethereum/go-ethereum/ethclient"
+	gethRPC "github.com/ethereum/go-ethereum/rpc"
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/config/params"
+	contracts "github.com/prysmaticlabs/prysm/contracts/deposit"
+	"github.com/prysmaticlabs/prysm/io/logs"
+	"github.com/prysmaticlabs/prysm/network"
+	"github.com/prysmaticlabs/prysm/network/authorization"
+)
+
+func (s *Service) setupExecutionClientConnections(ctx context.Context, currEndpoint network.Endpoint) error {
+	client, err := s.newRPCClientWithAuth(ctx, currEndpoint)
+	if err != nil {
+		return errors.Wrap(err, "could not dial execution node")
+	}
+	// Attach the clients to the service struct.
+	fetcher := ethclient.NewClient(client)
+	s.rpcClient = client
+	s.httpLogger = fetcher
+	s.eth1DataFetcher = fetcher
+
+	depositContractCaller, err := contracts.NewDepositContractCaller(s.cfg.depositContractAddr, fetcher)
+	if err != nil {
+		client.Close()
+		return errors.Wrap(err, "could not initialize deposit contract caller")
+	}
+	s.depositContractCaller = depositContractCaller
+
+	// Ensure we have the correct chain and deposit IDs.
+	if err := ensureCorrectExecutionChain(ctx, fetcher); err != nil {
+		client.Close()
+		return errors.Wrap(err, "could not make initial request to verify execution chain ID")
+	}
+	s.updateConnectedETH1(true)
+	s.runError = nil
+	return nil
+}
+
+// Every N seconds, defined as a backoffPeriod, attempts to re-establish an execution client
+// connection and if this does not work, we fallback to the next endpoint if defined.
+func (s *Service) pollConnectionStatus(ctx context.Context) {
+	// Use a custom logger to only log errors
+	logCounter := 0
+	errorLogger := func(err error, msg string) {
+		if logCounter > logThreshold {
+			log.Errorf("%s: %v", msg, err)
+			logCounter = 0
+		}
+		logCounter++
+	}
+	ticker := time.NewTicker(backOffPeriod)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ticker.C:
+			log.Debugf("Trying to dial endpoint: %s", logs.MaskCredentialsLogging(s.cfg.currHttpEndpoint.Url))
+			if err := s.setupExecutionClientConnections(ctx, s.cfg.currHttpEndpoint); err != nil {
+				errorLogger(err, "Could not connect to execution client endpoint")
+				s.runError = err
+				s.fallbackToNextEndpoint()
+			}
+		case <-s.ctx.Done():
+			log.Debug("Received cancelled context,closing existing powchain service")
+			return
+		}
+	}
+}
+
+// Forces to retry an execution client connection.
+func (s *Service) retryExecutionClientConnection(ctx context.Context, err error) {
+	s.runError = err
+	s.updateConnectedETH1(false)
+	// Back off for a while before redialing.
+	time.Sleep(backOffPeriod)
+	if err := s.setupExecutionClientConnections(ctx, s.cfg.currHttpEndpoint); err != nil {
+		s.runError = err
+		return
+	}
+	// Reset run error in the event of a successful connection.
+	s.runError = nil
+}
+
+// This performs a health check on our primary endpoint, and if it
+// is ready to serve we connect to it again. This method is only
+// relevant if we are on our backup endpoint.
+func (s *Service) checkDefaultEndpoint(ctx context.Context) {
+	primaryEndpoint := s.cfg.httpEndpoints[0]
+	// Return early if we are running on our primary
+	// endpoint.
+	if s.cfg.currHttpEndpoint.Equals(primaryEndpoint) {
+		return
+	}
+
+	if err := s.setupExecutionClientConnections(ctx, primaryEndpoint); err != nil {
+		log.Debugf("Primary endpoint not ready: %v", err)
+		return
+	}
+	s.updateCurrHttpEndpoint(primaryEndpoint)
+}
+
+// This is an inefficient way to search for the next endpoint, but given N is
+// expected to be small, it is fine to search this way.
+func (s *Service) fallbackToNextEndpoint() {
+	currEndpoint := s.cfg.currHttpEndpoint
+	currIndex := 0
+	totalEndpoints := len(s.cfg.httpEndpoints)
+
+	for i, endpoint := range s.cfg.httpEndpoints {
+		if endpoint.Equals(currEndpoint) {
+			currIndex = i
+			break
+		}
+	}
+	nextIndex := currIndex + 1
+	if nextIndex >= totalEndpoints {
+		nextIndex = 0
+	}
+	s.updateCurrHttpEndpoint(s.cfg.httpEndpoints[nextIndex])
+	if nextIndex != currIndex {
+		log.Infof("Falling back to alternative endpoint: %s", logs.MaskCredentialsLogging(s.cfg.currHttpEndpoint.Url))
+	}
+}
+
+// Initializes an RPC connection with authentication headers.
+func (s *Service) newRPCClientWithAuth(ctx context.Context, endpoint network.Endpoint) (*gethRPC.Client, error) {
+	// Need to handle ipc and http
+	var client *gethRPC.Client
+	u, err := url.Parse(endpoint.Url)
+	if err != nil {
+		return nil, err
+	}
+	switch u.Scheme {
+	case "http", "https":
+		client, err = gethRPC.DialHTTPWithClient(endpoint.Url, endpoint.HttpClient())
+		if err != nil {
+			return nil, err
+		}
+	case "":
+		client, err = gethRPC.DialIPC(ctx, endpoint.Url)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("no known transport for URL scheme %q", u.Scheme)
+	}
+	if endpoint.Auth.Method != authorization.None {
+		header, err := endpoint.Auth.ToHeaderValue()
+		if err != nil {
+			return nil, err
+		}
+		client.SetHeader("Authorization", header)
+	}
+	return client, nil
+}
+
+// Checks the chain ID of the execution client to ensure
+// it matches local parameters of what Prysm expects.
+func ensureCorrectExecutionChain(ctx context.Context, client *ethclient.Client) error {
+	cID, err := client.ChainID(ctx)
+	if err != nil {
+		return err
+	}
+	wantChainID := params.BeaconConfig().DepositChainID
+	if cID.Uint64() != wantChainID {
+		return fmt.Errorf("wanted chain ID %d, got %d", wantChainID, cID.Uint64())
+	}
+	return nil
+}

beacon-chain/powchain/service.go

@@ -24,10 +24,10 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/transition"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
-	engine "github.com/prysmaticlabs/prysm/beacon-chain/powchain/engine-api-client/v1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain/types"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	nativev1 "github.com/prysmaticlabs/prysm/beacon-chain/state/state-native/v1"
@@ -38,10 +38,8 @@ import (
 	"github.com/prysmaticlabs/prysm/container/trie"
 	contracts "github.com/prysmaticlabs/prysm/contracts/deposit"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
-	"github.com/prysmaticlabs/prysm/io/logs"
 	"github.com/prysmaticlabs/prysm/monitoring/clientstats"
 	"github.com/prysmaticlabs/prysm/network"
-	"github.com/prysmaticlabs/prysm/network/authorization"
 	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	prysmTime "github.com/prysmaticlabs/prysm/time"
 	"github.com/prysmaticlabs/prysm/time/slots"
@@ -72,8 +70,6 @@ var (
 	logPeriod = 1 * time.Minute
 	// threshold of how old we will accept an eth1 node's head to be.
 	eth1Threshold = 20 * time.Minute
-	// error when eth1 node is not synced.
-	errNotSynced = errors.New("eth1 node is still syncing")
 	// error when eth1 node is too far behind.
 	errFarBehind = errors.Errorf("eth1 head is more than %s behind from current wall clock time", eth1Threshold.String())
 )
@@ -115,6 +111,7 @@ type Chain interface {
 // RPCDataFetcher defines a subset of methods conformed to by ETH1.0 RPC clients for
 // fetching eth1 data from the clients.
 type RPCDataFetcher interface {
+	Close()
 	HeaderByNumber(ctx context.Context, number *big.Int) (*gethTypes.Header, error)
 	HeaderByHash(ctx context.Context, hash common.Hash) (*gethTypes.Header, error)
 	SyncProgress(ctx context.Context) (*ethereum.SyncProgress, error)
@@ -122,22 +119,23 @@ type RPCDataFetcher interface {
 
 // RPCClient defines the rpc methods required to interact with the eth1 node.
 type RPCClient interface {
+	Close()
 	BatchCall(b []gethRPC.BatchElem) error
+	CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error
 }
 
 // config defines a config struct for dependencies into the service.
 type config struct {
-	depositContractAddr        common.Address
-	beaconDB                   db.HeadAccessDatabase
-	depositCache               *depositcache.DepositCache
-	stateNotifier              statefeed.Notifier
-	stateGen                   *stategen.State
-	eth1HeaderReqLimit         uint64
-	beaconNodeStatsUpdater     BeaconNodeStatsUpdater
-	httpEndpoints              []network.Endpoint
-	executionEndpointJWTSecret []byte
-	currHttpEndpoint           network.Endpoint
-	finalizedStateAtStartup    state.BeaconState
+	depositContractAddr     common.Address
+	beaconDB                db.HeadAccessDatabase
+	depositCache            *depositcache.DepositCache
+	stateNotifier           statefeed.Notifier
+	stateGen                *stategen.State
+	eth1HeaderReqLimit      uint64
+	beaconNodeStatsUpdater  BeaconNodeStatsUpdater
+	httpEndpoints           []network.Endpoint
+	currHttpEndpoint        network.Endpoint
+	finalizedStateAtStartup state.BeaconState
 }
 
 // Service fetches important information about the canonical
@@ -156,7 +154,6 @@ type Service struct {
 	headTicker              *time.Ticker
 	httpLogger              bind.ContractFilterer
 	eth1DataFetcher         RPCDataFetcher
-	engineAPIClient         engine.Caller
 	rpcClient               RPCClient
 	headerCache             *headerCache // cache to store block hash/block height.
 	latestEth1Data          *ethpb.LatestETH1Data
@@ -220,6 +217,7 @@ func NewService(ctx context.Context, opts ...Option) (*Service, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to retrieve eth1 data")
 	}
+
 	if err := s.initializeEth1Data(ctx, eth1Data); err != nil {
 		return nil, err
 	}
@@ -228,6 +226,9 @@ func NewService(ctx context.Context, opts ...Option) (*Service, error) {
 
 // Start a web3 service's main event loop.
 func (s *Service) Start() {
+	if err := s.setupExecutionClientConnections(s.ctx, s.cfg.currHttpEndpoint); err != nil {
+		log.WithError(err).Error("Could not connect to execution endpoint")
+	}
 	// If the chain has not started already and we don't have access to eth1 nodes, we will not be
 	// able to generate the genesis state.
 	if !s.chainStartData.Chainstarted && s.cfg.currHttpEndpoint.Url == "" {
@@ -242,27 +243,15 @@ func (s *Service) Start() {
 		}
 	}
 
-	// Exit early if eth1 endpoint is not set.
-	if s.cfg.currHttpEndpoint.Url == "" {
-		return
-	}
+	s.isRunning = true
 
-	if err := s.initializeEngineAPIClient(s.ctx); err != nil {
-		log.WithError(err).Fatal("unable to initialize engine API client")
-	}
+	// Poll the execution client connection and fallback if errors occur.
+	go s.pollConnectionStatus(s.ctx)
 
 	// Check transition configuration for the engine API client in the background.
-	go s.checkTransitionConfiguration(s.ctx, make(chan *statefeed.BlockProcessedData, 1))
+	go s.checkTransitionConfiguration(s.ctx, make(chan *feed.Event, 1))
 
-	go func() {
-		s.isRunning = true
-		s.waitForConnection()
-		if s.ctx.Err() != nil {
-			log.Info("Context closed, exiting pow goroutine")
-			return
-		}
-		s.run(s.ctx.Done())
-	}()
+	go s.run(s.ctx.Done())
 }
 
 // Stop the web3 service's main event loop and associated goroutines.
@@ -270,7 +259,12 @@ func (s *Service) Stop() error {
 	if s.cancel != nil {
 		defer s.cancel()
 	}
-	s.closeClients()
+	if s.rpcClient != nil {
+		s.rpcClient.Close()
+	}
+	if s.eth1DataFetcher != nil {
+		s.eth1DataFetcher.Close()
+	}
 	return nil
 }
 
@@ -305,12 +299,6 @@ func (s *Service) Status() error {
 	return s.runError
 }
 
-// EngineAPIClient returns the associated engine API client to interact
-// with an execution node via JSON-RPC.
-func (s *Service) EngineAPIClient() engine.Caller {
-	return s.engineAPIClient
-}
-
 func (s *Service) updateBeaconNodeStats() {
 	bs := clientstats.BeaconNodeStats{}
 	if len(s.cfg.httpEndpoints) > 1 {
@@ -348,10 +336,7 @@ func (s *Service) CurrentETH1Endpoint() string {
 
 // CurrentETH1ConnectionError returns the error (if any) of the current connection.
 func (s *Service) CurrentETH1ConnectionError() error {
-	httpClient, rpcClient, err := s.dialETH1Nodes(s.cfg.currHttpEndpoint)
-	httpClient.Close()
-	rpcClient.Close()
-	return err
+	return s.runError
 }
 
 // ETH1Endpoints returns the slice of HTTP endpoint URLs (default is 0th element).
@@ -368,10 +353,17 @@ func (s *Service) ETH1Endpoints() []string {
 func (s *Service) ETH1ConnectionErrors() []error {
 	var errs []error
 	for _, ep := range s.cfg.httpEndpoints {
-		httpClient, rpcClient, err := s.dialETH1Nodes(ep)
-		httpClient.Close()
-		rpcClient.Close()
-		errs = append(errs, err)
+		client, err := s.newRPCClientWithAuth(s.ctx, ep)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		if err := ensureCorrectExecutionChain(s.ctx, ethclient.NewClient(client)); err != nil {
+			client.Close()
+			errs = append(errs, err)
+			continue
+		}
+		client.Close()
 	}
 	return errs
 }
@@ -386,198 +378,6 @@ func (s *Service) followBlockHeight(_ context.Context) (uint64, error) {
 	return latestValidBlock, nil
 }
 
-func (s *Service) connectToPowChain() error {
-	httpClient, rpcClient, err := s.dialETH1Nodes(s.cfg.currHttpEndpoint)
-	if err != nil {
-		return errors.Wrap(err, "could not dial eth1 nodes")
-	}
-
-	depositContractCaller, err := contracts.NewDepositContractCaller(s.cfg.depositContractAddr, httpClient)
-	if err != nil {
-		return errors.Wrap(err, "could not create deposit contract caller")
-	}
-
-	if httpClient == nil || rpcClient == nil || depositContractCaller == nil {
-		return errors.New("eth1 client is nil")
-	}
-
-	s.initializeConnection(httpClient, rpcClient, depositContractCaller)
-	return nil
-}
-
-func (s *Service) dialETH1Nodes(endpoint network.Endpoint) (*ethclient.Client, *gethRPC.Client, error) {
-	httpRPCClient, err := gethRPC.Dial(endpoint.Url)
-	if err != nil {
-		return nil, nil, err
-	}
-	if endpoint.Auth.Method != authorization.None {
-		header, err := endpoint.Auth.ToHeaderValue()
-		if err != nil {
-			return nil, nil, err
-		}
-		httpRPCClient.SetHeader("Authorization", header)
-	}
-	httpClient := ethclient.NewClient(httpRPCClient)
-	// Add a method to clean-up and close clients in the event
-	// of any connection failure.
-	closeClients := func() {
-		httpRPCClient.Close()
-		httpClient.Close()
-	}
-	syncProg, err := httpClient.SyncProgress(s.ctx)
-	if err != nil {
-		closeClients()
-		return nil, nil, err
-	}
-	if syncProg != nil {
-		closeClients()
-		return nil, nil, errors.New("eth1 node has not finished syncing yet")
-	}
-	// Make a simple call to ensure we are actually connected to a working node.
-	cID, err := httpClient.ChainID(s.ctx)
-	if err != nil {
-		closeClients()
-		return nil, nil, err
-	}
-	nID, err := httpClient.NetworkID(s.ctx)
-	if err != nil {
-		closeClients()
-		return nil, nil, err
-	}
-	if cID.Uint64() != params.BeaconConfig().DepositChainID {
-		closeClients()
-		return nil, nil, fmt.Errorf("eth1 node using incorrect chain id, %d != %d", cID.Uint64(), params.BeaconConfig().DepositChainID)
-	}
-	if nID.Uint64() != params.BeaconConfig().DepositNetworkID {
-		closeClients()
-		return nil, nil, fmt.Errorf("eth1 node using incorrect network id, %d != %d", nID.Uint64(), params.BeaconConfig().DepositNetworkID)
-	}
-
-	return httpClient, httpRPCClient, nil
-}
-
-func (s *Service) initializeConnection(
-	httpClient *ethclient.Client,
-	rpcClient *gethRPC.Client,
-	contractCaller *contracts.DepositContractCaller,
-) {
-	s.httpLogger = httpClient
-	s.eth1DataFetcher = httpClient
-	s.depositContractCaller = contractCaller
-	s.rpcClient = rpcClient
-}
-
-// closes down our active eth1 clients.
-func (s *Service) closeClients() {
-	gethClient, ok := s.rpcClient.(*gethRPC.Client)
-	if ok {
-		gethClient.Close()
-	}
-	httpClient, ok := s.eth1DataFetcher.(*ethclient.Client)
-	if ok {
-		httpClient.Close()
-	}
-}
-
-func (s *Service) waitForConnection() {
-	errConnect := s.connectToPowChain()
-	if errConnect == nil {
-		synced, errSynced := s.isEth1NodeSynced()
-		// Resume if eth1 node is synced.
-		if synced {
-			s.updateConnectedETH1(true)
-			s.runError = nil
-			log.WithFields(logrus.Fields{
-				"endpoint": logs.MaskCredentialsLogging(s.cfg.currHttpEndpoint.Url),
-			}).Info("Connected to eth1 proof-of-work chain")
-			return
-		}
-		if errSynced != nil {
-			s.runError = errSynced
-			log.WithError(errSynced).Error("Could not check sync status of eth1 chain")
-		}
-	}
-	if errConnect != nil {
-		s.runError = errConnect
-		log.WithError(errConnect).Error("Could not connect to powchain endpoint")
-	}
-	// Use a custom logger to only log errors
-	// once in  a while.
-	logCounter := 0
-	errorLogger := func(err error, msg string) {
-		if logCounter > logThreshold {
-			log.Errorf("%s: %v", msg, err)
-			logCounter = 0
-		}
-		logCounter++
-	}
-
-	ticker := time.NewTicker(backOffPeriod)
-	defer ticker.Stop()
-	for {
-		select {
-		case <-ticker.C:
-			log.Debugf("Trying to dial endpoint: %s", logs.MaskCredentialsLogging(s.cfg.currHttpEndpoint.Url))
-			errConnect := s.connectToPowChain()
-			if errConnect != nil {
-				errorLogger(errConnect, "Could not connect to powchain endpoint")
-				s.runError = errConnect
-				s.fallbackToNextEndpoint()
-				continue
-			}
-			synced, errSynced := s.isEth1NodeSynced()
-			if errSynced != nil {
-				errorLogger(errSynced, "Could not check sync status of eth1 chain")
-				s.runError = errSynced
-				s.fallbackToNextEndpoint()
-				continue
-			}
-			if synced {
-				s.updateConnectedETH1(true)
-				s.runError = nil
-				log.WithFields(logrus.Fields{
-					"endpoint": logs.MaskCredentialsLogging(s.cfg.currHttpEndpoint.Url),
-				}).Info("Connected to eth1 proof-of-work chain")
-				return
-			}
-			s.runError = errNotSynced
-			log.Debug("Eth1 node is currently syncing")
-		case <-s.ctx.Done():
-			log.Debug("Received cancelled context,closing existing powchain service")
-			return
-		}
-	}
-}
-
-// checks if the eth1 node is healthy and ready to serve before
-// fetching data from  it.
-func (s *Service) isEth1NodeSynced() (bool, error) {
-	syncProg, err := s.eth1DataFetcher.SyncProgress(s.ctx)
-	if err != nil {
-		return false, err
-	}
-	if syncProg != nil {
-		return false, nil
-	}
-	head, err := s.eth1DataFetcher.HeaderByNumber(s.ctx, nil)
-	if err != nil {
-		return false, err
-	}
-	return !eth1HeadIsBehind(head.Time), nil
-}
-
-// Reconnect to eth1 node in case of any failure.
-func (s *Service) retryETH1Node(err error) {
-	s.runError = err
-	s.updateConnectedETH1(false)
-	// Back off for a while before
-	// resuming dialing the eth1 node.
-	time.Sleep(backOffPeriod)
-	s.waitForConnection()
-	// Reset run error in the event of a successful connection.
-	s.runError = nil
-}
-
 func (s *Service) initDepositCaches(ctx context.Context, ctrs []*ethpb.DepositContainer) error {
 	if len(ctrs) == 0 {
 		return nil
@@ -643,6 +443,7 @@ func (s *Service) processBlockHeader(header *gethTypes.Header) {
 	log.WithFields(logrus.Fields{
 		"blockNumber": s.latestEth1Data.BlockHeight,
 		"blockHash":   hexutil.Encode(s.latestEth1Data.BlockHash),
+		"difficulty":  header.Difficulty.String(),
 	}).Debug("Latest eth1 chain event")
 }
 
@@ -711,7 +512,7 @@ func (s *Service) handleETH1FollowDistance() {
 	fiveMinutesTimeout := prysmTime.Now().Add(-5 * time.Minute)
 	// check that web3 client is syncing
 	if time.Unix(int64(s.latestEth1Data.BlockTime), 0).Before(fiveMinutesTimeout) {
-		log.Warn("eth1 client is not syncing")
+		log.Warn("Execution client is not syncing")
 	}
 	if !s.chainStartData.Chainstarted {
 		if err := s.checkBlockNumberForChainStart(ctx, big.NewInt(int64(s.latestEth1Data.LastRequestedBlock))); err != nil {
@@ -741,6 +542,15 @@ func (s *Service) handleETH1FollowDistance() {
 }
 
 func (s *Service) initPOWService() {
+	// Use a custom logger to only log errors
+	logCounter := 0
+	errorLogger := func(err error, msg string) {
+		if logCounter > logThreshold {
+			log.Errorf("%s: %v", msg, err)
+			logCounter = 0
+		}
+		logCounter++
+	}
 
 	// Run in a select loop to retry in the event of any failures.
 	for {
@@ -751,8 +561,8 @@ func (s *Service) initPOWService() {
 			ctx := s.ctx
 			header, err := s.eth1DataFetcher.HeaderByNumber(ctx, nil)
 			if err != nil {
-				log.Errorf("Unable to retrieve latest ETH1.0 chain header: %v", err)
-				s.retryETH1Node(err)
+				s.retryExecutionClientConnection(ctx, err)
+				errorLogger(err, "Unable to retrieve latest execution client header")
 				continue
 			}
 
@@ -761,14 +571,14 @@ func (s *Service) initPOWService() {
 			s.latestEth1Data.BlockTime = header.Time
 
 			if err := s.processPastLogs(ctx); err != nil {
-				log.Errorf("Unable to process past logs %v", err)
-				s.retryETH1Node(err)
+				s.retryExecutionClientConnection(ctx, err)
+				errorLogger(err, "Unable to process past deposit contract logs")
 				continue
 			}
 			// Cache eth1 headers from our voting period.
 			if err := s.cacheHeadersForEth1DataVote(ctx); err != nil {
-				log.Errorf("Unable to process past headers %v", err)
-				s.retryETH1Node(err)
+				s.retryExecutionClientConnection(ctx, err)
+				errorLogger(err, "Unable to cache headers for execution client votes")
 				continue
 			}
 			// Handle edge case with embedded genesis state by fetching genesis header to determine
@@ -781,15 +591,15 @@ func (s *Service) initPOWService() {
 				if genHash != [32]byte{} {
 					genHeader, err := s.eth1DataFetcher.HeaderByHash(ctx, genHash)
 					if err != nil {
-						log.Errorf("Unable to retrieve genesis ETH1.0 chain header: %v", err)
-						s.retryETH1Node(err)
+						s.retryExecutionClientConnection(ctx, err)
+						errorLogger(err, "Unable to retrieve proof-of-stake genesis block data")
 						continue
 					}
 					genBlock = genHeader.Number.Uint64()
 				}
 				s.chainStartData.GenesisBlock = genBlock
 				if err := s.savePowchainData(ctx); err != nil {
-					log.Errorf("Unable to save powchain data: %v", err)
+					errorLogger(err, "Unable to save execution client data")
 				}
 			}
 			return
@@ -818,17 +628,16 @@ func (s *Service) run(done <-chan struct{}) {
 			head, err := s.eth1DataFetcher.HeaderByNumber(s.ctx, nil)
 			if err != nil {
 				log.WithError(err).Debug("Could not fetch latest eth1 header")
-				s.retryETH1Node(err)
 				continue
 			}
 			if eth1HeadIsBehind(head.Time) {
+				s.retryExecutionClientConnection(s.ctx, err)
 				log.WithError(errFarBehind).Debug("Could not get an up to date eth1 header")
-				s.retryETH1Node(errFarBehind)
 				continue
 			}
 			s.processBlockHeader(head)
 			s.handleETH1FollowDistance()
-			s.checkDefaultEndpoint()
+			s.checkDefaultEndpoint(s.ctx)
 		case <-chainstartTicker.C:
 			if s.chainStartData.Chainstarted {
 				chainstartTicker.Stop()
@@ -915,59 +724,6 @@ func (s *Service) determineEarliestVotingBlock(ctx context.Context, followBlock
 	return hdr.Number.Uint64(), nil
 }
 
-// This performs a health check on our primary endpoint, and if it
-// is ready to serve we connect to it again. This method is only
-// relevant if we are on our backup endpoint.
-func (s *Service) checkDefaultEndpoint() {
-	primaryEndpoint := s.cfg.httpEndpoints[0]
-	// Return early if we are running on our primary
-	// endpoint.
-	if s.cfg.currHttpEndpoint.Equals(primaryEndpoint) {
-		return
-	}
-
-	httpClient, rpcClient, err := s.dialETH1Nodes(primaryEndpoint)
-	if err != nil {
-		log.Debugf("Primary endpoint not ready: %v", err)
-		return
-	}
-	log.Info("Primary endpoint ready again, switching back to it")
-	// Close the clients and let our main connection routine
-	// properly connect with it.
-	httpClient.Close()
-	rpcClient.Close()
-	// Close current active clients.
-	s.closeClients()
-
-	// Switch back to primary endpoint and try connecting
-	// to it again.
-	s.updateCurrHttpEndpoint(primaryEndpoint)
-	s.retryETH1Node(nil)
-}
-
-// This is an inefficient way to search for the next endpoint, but given N is expected to be
-// small ( < 25), it is fine to search this way.
-func (s *Service) fallbackToNextEndpoint() {
-	currEndpoint := s.cfg.currHttpEndpoint
-	currIndex := 0
-	totalEndpoints := len(s.cfg.httpEndpoints)
-
-	for i, endpoint := range s.cfg.httpEndpoints {
-		if endpoint.Equals(currEndpoint) {
-			currIndex = i
-			break
-		}
-	}
-	nextIndex := currIndex + 1
-	if nextIndex >= totalEndpoints {
-		nextIndex = 0
-	}
-	s.updateCurrHttpEndpoint(s.cfg.httpEndpoints[nextIndex])
-	if nextIndex != currIndex {
-		log.Infof("Falling back to alternative endpoint: %s", logs.MaskCredentialsLogging(s.cfg.currHttpEndpoint.Url))
-	}
-}
-
 // initializes our service from the provided eth1data object by initializing all the relevant
 // fields and data.
 func (s *Service) initializeEth1Data(ctx context.Context, eth1DataInDB *ethpb.ETH1ChainData) error {
@@ -1056,19 +812,6 @@ func (s *Service) ensureValidPowchainData(ctx context.Context) error {
 	return nil
 }
 
-// Initializes a connection to the engine API if an execution provider endpoint is set.
-func (s *Service) initializeEngineAPIClient(ctx context.Context) error {
-	opts := []engine.Option{
-		engine.WithJWTSecret(s.cfg.executionEndpointJWTSecret),
-	}
-	client, err := engine.New(ctx, s.cfg.currHttpEndpoint.Url, opts...)
-	if err != nil {
-		return err
-	}
-	s.engineAPIClient = client
-	return nil
-}
-
 func dedupEndpoints(endpoints []string) []string {
 	selectionMap := make(map[string]bool)
 	newEndpoints := make([]string, 0, len(endpoints))

beacon-chain/powchain/service_test.go

@@ -3,11 +3,9 @@ package powchain
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"math/big"
 	"strings"
-	"sync"
 	"testing"
 	"time"
 
@@ -15,6 +13,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
 	"github.com/ethereum/go-ethereum/common"
 	gethTypes "github.com/ethereum/go-ethereum/core/types"
+	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/async/event"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
 	dbutil "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
@@ -43,6 +42,8 @@ type goodLogger struct {
 	backend *backends.SimulatedBackend
 }
 
+func (_ *goodLogger) Close() {}
+
 func (g *goodLogger) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- gethTypes.Log) (ethereum.Subscription, error) {
 	if g.backend == nil {
 		return new(event.Feed).Subscribe(ch), nil
@@ -81,6 +82,8 @@ type goodFetcher struct {
 	backend *backends.SimulatedBackend
 }
 
+func (_ *goodFetcher) Close() {}
+
 func (g *goodFetcher) HeaderByHash(_ context.Context, hash common.Hash) (*gethTypes.Header, error) {
 	if bytes.Equal(hash.Bytes(), common.BytesToHash([]byte{0}).Bytes()) {
 		return nil, fmt.Errorf("expected block hash to be nonzero %v", hash)
@@ -128,6 +131,11 @@ func TestStart_OK(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -157,90 +165,13 @@ func TestStart_NoHttpEndpointDefinedFails_WithoutChainStarted(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
-	s, err := NewService(context.Background(),
+	_, err = NewService(context.Background(),
 		WithHttpEndpoints([]string{""}),
 		WithDepositContractAddress(testAcc.ContractAddr),
 		WithDatabase(beaconDB),
 	)
 	require.NoError(t, err)
-	// Set custom exit func so test can proceed
-	log.Logger.ExitFunc = func(i int) {
-		panic(i)
-	}
-	defer func() {
-		log.Logger.ExitFunc = nil
-	}()
-	wg := new(sync.WaitGroup)
-	wg.Add(1)
-	// Expect Start function to fail from a fatal call due
-	// to no state existing.
-	go func() {
-		defer func() {
-			if r := recover(); r != nil {
-				wg.Done()
-			}
-		}()
-		s.Start()
-	}()
-	util.WaitTimeout(wg, time.Second)
-	require.LogsContain(t, hook, "cannot create genesis state: no eth1 http endpoint defined")
-	hook.Reset()
-}
-
-func TestStart_NoHttpEndpointDefinedSucceeds_WithGenesisState(t *testing.T) {
-	hook := logTest.NewGlobal()
-	beaconDB := dbutil.SetupDB(t)
-	testAcc, err := mock.Setup()
-	require.NoError(t, err, "Unable to set up simulated backend")
-	st, _ := util.DeterministicGenesisState(t, 10)
-	b := util.NewBeaconBlock()
-	genRoot, err := b.HashTreeRoot()
-	require.NoError(t, err)
-	require.NoError(t, beaconDB.SaveState(context.Background(), st, genRoot))
-	require.NoError(t, beaconDB.SaveGenesisBlockRoot(context.Background(), genRoot))
-	depositCache, err := depositcache.New()
-	require.NoError(t, err)
-	s, err := NewService(context.Background(),
-		WithHttpEndpoints([]string{""}),
-		WithDepositContractAddress(testAcc.ContractAddr),
-		WithDatabase(beaconDB),
-		WithDepositCache(depositCache),
-	)
-	require.NoError(t, err)
-
-	wg := new(sync.WaitGroup)
-	wg.Add(1)
-
-	go func() {
-		s.Start()
-		wg.Done()
-	}()
-	s.cancel()
-	util.WaitTimeout(wg, time.Second)
-	require.LogsDoNotContain(t, hook, "cannot create genesis state: no eth1 http endpoint defined")
-	hook.Reset()
-}
-
-func TestStart_NoHttpEndpointDefinedSucceeds_WithChainStarted(t *testing.T) {
-	hook := logTest.NewGlobal()
-	beaconDB := dbutil.SetupDB(t)
-	testAcc, err := mock.Setup()
-	require.NoError(t, err, "Unable to set up simulated backend")
-
-	require.NoError(t, beaconDB.SavePowchainData(context.Background(), &ethpb.ETH1ChainData{
-		ChainstartData: &ethpb.ChainStartData{Chainstarted: true},
-		Trie:           &ethpb.SparseMerkleTrie{},
-	}))
-	s, err := NewService(context.Background(),
-		WithHttpEndpoints([]string{""}),
-		WithDepositContractAddress(testAcc.ContractAddr),
-		WithDatabase(beaconDB),
-	)
-	require.NoError(t, err)
-
-	s.Start()
-	require.LogsDoNotContain(t, hook, "cannot create genesis state: no eth1 http endpoint defined")
-	hook.Reset()
+	require.LogsDoNotContain(t, hook, "missing address")
 }
 
 func TestStop_OK(t *testing.T) {
@@ -248,6 +179,11 @@ func TestStop_OK(t *testing.T) {
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -273,6 +209,11 @@ func TestService_Eth1Synced(t *testing.T) {
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -288,16 +229,17 @@ func TestService_Eth1Synced(t *testing.T) {
 	now := time.Now()
 	assert.NoError(t, testAcc.Backend.AdjustTime(now.Sub(time.Unix(int64(currTime), 0))))
 	testAcc.Backend.Commit()
-
-	synced, err := web3Service.isEth1NodeSynced()
-	require.NoError(t, err)
-	assert.Equal(t, true, synced, "Expected eth1 nodes to be synced")
 }
 
 func TestFollowBlock_OK(t *testing.T) {
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -371,6 +313,11 @@ func TestStatus(t *testing.T) {
 func TestHandlePanic_OK(t *testing.T) {
 	hook := logTest.NewGlobal()
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
@@ -403,6 +350,11 @@ func TestLogTillGenesis_OK(t *testing.T) {
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -528,8 +480,8 @@ func TestInitDepositCacheWithFinalization_OK(t *testing.T) {
 
 	s.chainStartData.Chainstarted = true
 	require.NoError(t, s.initDepositCaches(context.Background(), ctrs))
-
-	deps := s.cfg.depositCache.NonFinalizedDeposits(context.Background(), nil)
+	fDeposits := s.cfg.depositCache.FinalizedDeposits(ctx)
+	deps := s.cfg.depositCache.NonFinalizedDeposits(context.Background(), fDeposits.MerkleTrieIndex, nil)
 	assert.Equal(t, 0, len(deps))
 }
 
@@ -537,6 +489,11 @@ func TestNewService_EarliestVotingBlock(t *testing.T) {
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
 	beaconDB := dbutil.SetupDB(t)
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	web3Service, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -588,6 +545,11 @@ func TestNewService_Eth1HeaderRequLimit(t *testing.T) {
 	require.NoError(t, err, "Unable to set up simulated backend")
 	beaconDB := dbutil.SetupDB(t)
 
+	server, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	s1, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{endpoint}),
 		WithDepositContractAddress(testAcc.ContractAddr),
@@ -616,13 +578,26 @@ func (mbs *mockBSUpdater) Update(bs clientstats.BeaconNodeStats) {
 var _ BeaconNodeStatsUpdater = &mockBSUpdater{}
 
 func TestServiceFallbackCorrectly(t *testing.T) {
-	firstEndpoint := "A"
-	secondEndpoint := "B"
-
 	testAcc, err := mock.Setup()
 	require.NoError(t, err, "Unable to set up simulated backend")
 	beaconDB := dbutil.SetupDB(t)
 
+	server, firstEndpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
+	server2, secondEndpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server2.Stop()
+	})
+	server3, thirdEndpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server3.Stop()
+	})
+
 	mbs := &mockBSUpdater{}
 	s1, err := NewService(context.Background(),
 		WithHttpEndpoints([]string{firstEndpoint}),
@@ -630,10 +605,11 @@ func TestServiceFallbackCorrectly(t *testing.T) {
 		WithDatabase(beaconDB),
 		WithBeaconNodeStatsUpdater(mbs),
 	)
-	s1.cfg.beaconNodeStatsUpdater = mbs
 	require.NoError(t, err)
+	s1.cfg.beaconNodeStatsUpdater = mbs
 
 	assert.Equal(t, firstEndpoint, s1.cfg.currHttpEndpoint.Url, "Unexpected http endpoint")
+
 	// Stay at the first endpoint.
 	s1.fallbackToNextEndpoint()
 	assert.Equal(t, firstEndpoint, s1.cfg.currHttpEndpoint.Url, "Unexpected http endpoint")
@@ -645,17 +621,11 @@ func TestServiceFallbackCorrectly(t *testing.T) {
 	assert.Equal(t, secondEndpoint, s1.cfg.currHttpEndpoint.Url, "Unexpected http endpoint")
 	assert.Equal(t, true, mbs.lastBS.SyncEth1FallbackConfigured, "SyncEth1FallbackConfigured in clientstats update should be true when > 1 endpoint is configured")
 
-	thirdEndpoint := "C"
-	fourthEndpoint := "D"
-
-	s1.cfg.httpEndpoints = append(s1.cfg.httpEndpoints, network.Endpoint{Url: thirdEndpoint}, network.Endpoint{Url: fourthEndpoint})
+	s1.cfg.httpEndpoints = append(s1.cfg.httpEndpoints, network.Endpoint{Url: thirdEndpoint})
 
 	s1.fallbackToNextEndpoint()
 	assert.Equal(t, thirdEndpoint, s1.cfg.currHttpEndpoint.Url, "Unexpected http endpoint")
 
-	s1.fallbackToNextEndpoint()
-	assert.Equal(t, fourthEndpoint, s1.cfg.currHttpEndpoint.Url, "Unexpected http endpoint")
-
 	// Rollover correctly back to the first endpoint
 	s1.fallbackToNextEndpoint()
 	assert.Equal(t, firstEndpoint, s1.cfg.currHttpEndpoint.Url, "Unexpected http endpoint")
@@ -685,8 +655,13 @@ func TestService_EnsureConsistentPowchainData(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
 	cache, err := depositcache.New()
 	require.NoError(t, err)
-
+	srv, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		srv.Stop()
+	})
 	s1, err := NewService(context.Background(),
+		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
 		WithDepositCache(cache),
 	)
@@ -710,7 +685,13 @@ func TestService_InitializeCorrectly(t *testing.T) {
 	cache, err := depositcache.New()
 	require.NoError(t, err)
 
+	srv, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		srv.Stop()
+	})
 	s1, err := NewService(context.Background(),
+		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
 		WithDepositCache(cache),
 	)
@@ -733,8 +714,13 @@ func TestService_EnsureValidPowchainData(t *testing.T) {
 	beaconDB := dbutil.SetupDB(t)
 	cache, err := depositcache.New()
 	require.NoError(t, err)
-
+	srv, endpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		srv.Stop()
+	})
 	s1, err := NewService(context.Background(),
+		WithHttpEndpoints([]string{endpoint}),
 		WithDatabase(beaconDB),
 		WithDepositCache(cache),
 	)
@@ -825,8 +811,16 @@ func TestTimestampIsChecked(t *testing.T) {
 }
 
 func TestETH1Endpoints(t *testing.T) {
-	firstEndpoint := "A"
-	secondEndpoint := "B"
+	server, firstEndpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
+	server, secondEndpoint, err := mockPOW.SetupRPCServer()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		server.Stop()
+	})
 	endpoints := []string{firstEndpoint, secondEndpoint}
 
 	testAcc, err := mock.Setup()

beacon-chain/powchain/testing/BUILD.bazel

@@ -4,6 +4,7 @@ go_library(
     name = "go_default_library",
     testonly = True,
     srcs = [
+        "mock_engine_client.go",
         "mock_faulty_powchain.go",
         "mock_powchain.go",
     ],
@@ -16,13 +17,16 @@ go_library(
         "//beacon-chain/powchain/types:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/v1:go_default_library",
+        "//config/params:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//proto/engine/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "@com_github_ethereum_go_ethereum//accounts/abi/bind/backends:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
         "@com_github_ethereum_go_ethereum//rpc:go_default_library",
+        "@com_github_holiman_uint256//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
     ],
 )