Merge branch 'develop' into beacon-proposer-settings

* `Test_processAttestations`: Remove duplicated tests.

* Sort indexed attestations by data root.

* `processAttestations`: Don't return duplicate slashings anymore.

Fix https://github.com/prysmaticlabs/prysm/issues/13592.

* `AttesterDoubleVote`: Rename fields.

* Detect double votes in different batches.

In order to do that:
1. Each attestation of the batch is tested against the other attestations of the batch.
2. Each attestation of the batch is tested against the content of the database.
2. Attestations are saved into the database.

Fixes https://github.com/prysmaticlabs/prysm/issues/13590.

.bazelversion

@@ -1 +1 @@
-6.4.0
+7.0.0

.buildkite-bazelrc

@@ -10,7 +10,7 @@
 
 # Prysm specific remote-cache properties.
 build:remote-cache --remote_download_minimal
-build:remote-cache --experimental_remote_build_event_upload=minimal
+build:remote-cache --remote_build_event_upload=minimal
 build:remote-cache --remote_cache=grpc://bazel-remote-cache:9092
 # Does not work with rules_oci. See https://github.com/bazel-contrib/rules_oci/issues/292
 #build:remote-cache --experimental_remote_downloader=grpc://bazel-remote-cache:9092

.golangci.yml

@@ -80,7 +80,6 @@ linters:
     - thelper
     - unparam
     - varnamelen
-    - wastedassign
     - wrapcheck
     - wsl
 

BUILD.bazel

@@ -194,33 +194,6 @@ nogo(
     config = ":nogo_config_with_excludes",
     visibility = ["//visibility:public"],
     deps = [
-        "@org_golang_x_tools//go/analysis/passes/unsafeptr:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/unreachable:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/unmarshal:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/tests:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/structtag:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/stdmethods:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/shift:go_default_library",
-        # "@org_golang_x_tools//go/analysis/passes/shadow:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/printf:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/pkgfact:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/nilness:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/nilfunc:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/loopclosure:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/httpresponse:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/findcall:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/deepequalerrors:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/ctrlflow:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/copylock:go_default_library",
-        #  "@org_golang_x_tools//go/analysis/passes/cgocall:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/buildtag:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/buildssa:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/bools:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/atomicalign:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/atomic:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/assign:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/inspect:go_default_library",
-        "@org_golang_x_tools//go/analysis/passes/asmdecl:go_default_library",
         "//tools/analyzers/comparesame:go_default_library",
         "//tools/analyzers/cryptorand:go_default_library",
         "//tools/analyzers/errcheck:go_default_library",
@@ -236,6 +209,53 @@ nogo(
         "//tools/analyzers/shadowpredecl:go_default_library",
         "//tools/analyzers/slicedirect:go_default_library",
         "//tools/analyzers/uintcast:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/appends:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/asmdecl:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/assign:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/atomic:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/atomicalign:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/bools:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/buildssa:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/buildtag:go_default_library",
+        # cgocall disabled
+        #"@org_golang_x_tools//go/analysis/passes/cgocall:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/copylock:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/ctrlflow:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/deepequalerrors:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/defers:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/directive:go_default_library",
+        # fieldalignment disabled
+        #"@org_golang_x_tools//go/analysis/passes/fieldalignment:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/findcall:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/framepointer:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/httpmux:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/httpresponse:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/ifaceassert:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/inspect:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/loopclosure:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/nilfunc:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/nilness:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/pkgfact:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/printf:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/reflectvaluecompare:go_default_library",
+        # shadow disabled
+        #"@org_golang_x_tools//go/analysis/passes/shadow:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/shift:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/sigchanyzer:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/slog:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/sortslice:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/stdmethods:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/stringintconv:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/structtag:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/testinggoroutine:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/tests:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/timeformat:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/unmarshal:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/unreachable:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/unsafeptr:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/unusedresult:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/unusedwrite:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/usesgenerics:go_default_library",
     ] + select({
         # nogo checks that fail with coverage enabled.
         ":coverage_enabled": [],

DEPENDENCIES.md

@@ -55,7 +55,7 @@ bazel build //beacon-chain --config=release
 ## Adding / updating dependencies
 
 1. Add your dependency as you would with go modules. I.e. `go get ...`
-1. Run `gazelle update-repos -from_file=go.mod` to update the bazel managed dependencies.
+1. Run `bazel run //:gazelle -- update-repos -from_file=go.mod` to update the bazel managed dependencies.
 
 Example:
 

MODULE.bazel

@@ -0,0 +1,6 @@
+###############################################################################
+# Bazel now uses Bzlmod by default to manage external dependencies.
+# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel.
+#
+# For more details, please check https://github.com/bazelbuild/bazel/issues/18958
+###############################################################################

WORKSPACE

@@ -106,6 +106,13 @@ load("@rules_distroless//distroless:dependencies.bzl", "rules_distroless_depende
 
 rules_distroless_dependencies()
 
+http_archive(
+    name = "distroless",
+    integrity = "sha256-Cf00kUp1NyXA3LzbdyYy4Kda27wbkB8+A9MliTxq4jE=",
+    strip_prefix = "distroless-9dc924b9fe812eec2fa0061824dcad39eb09d0d6",
+    url = "https://github.com/GoogleContainerTools/distroless/archive/9dc924b9fe812eec2fa0061824dcad39eb09d0d6.tar.gz",  # 2024-01-24
+)
+
 load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies", "aspect_bazel_lib_register_toolchains")
 
 aspect_bazel_lib_dependencies()
@@ -144,6 +151,10 @@ http_archive(
     ],
 )
 
+load("//:distroless_deps.bzl", "distroless_deps")
+
+distroless_deps()
+
 # Override default import in rules_go with special patch until
 # https://github.com/gogo/protobuf/pull/582 is merged.
 git_repository(
@@ -182,7 +193,7 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe
 go_rules_dependencies()
 
 go_register_toolchains(
-    go_version = "1.21.5",
+    go_version = "1.21.6",
     nogo = "@//:nogo",
 )
 
@@ -223,7 +234,7 @@ filegroup(
     url = "https://github.com/ethereum/EIPs/archive/5480440fe51742ed23342b68cf106cefd427e39d.tar.gz",
 )
 
-consensus_spec_version = "v1.4.0-beta.5"
+consensus_spec_version = "v1.4.0-beta.6"
 
 bls_test_version = "v0.1.1"
 
@@ -239,7 +250,7 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "9017ffff84d64a7c4c9e6ff9f421f9479f71d3b463b738f54e02158dbb4f50f0",
+    sha256 = "7dc467d7be97525c88a1d3683665c1354cc86297fd62009e7cf5000905b25652",
     url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/general.tar.gz" % consensus_spec_version,
 )
 
@@ -255,7 +266,7 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "f08711682553fe7c9362f1400ed8c56b2fa9576df08581fcad4c508ba8ad4788",
+    sha256 = "e163011254b6ce100205fb779ba660faedc9bc9f7bb4408c25746a7aa5e8d8bc",
     url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/minimal.tar.gz" % consensus_spec_version,
 )
 
@@ -271,7 +282,7 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "7ea3189e3879f2ac62467cbf2945c00b6c94d30cdefb2d645c630b1018c50e10",
+    sha256 = "b73c81b6386053a2141f6f43b457489668621c7013f740ed93edf9ac0e34f091",
     url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/mainnet.tar.gz" % consensus_spec_version,
 )
 
@@ -286,7 +297,7 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "4119992a2efc79e5cb2bdc07ed08c0b1fa32332cbd0d88e6467f34938df97026",
+    sha256 = "47726c527512d03ef3e706a8e7f8d5db6a5f2153351db0470dab780f6a87c4dd",
     strip_prefix = "consensus-specs-" + consensus_spec_version[1:],
     url = "https://github.com/ethereum/consensus-specs/archive/refs/tags/%s.tar.gz" % consensus_spec_version,
 )
@@ -322,6 +333,22 @@ filegroup(
     url = "https://github.com/eth-clients/eth2-networks/archive/7b4897888cebef23801540236f73123e21774954.tar.gz",
 )
 
+http_archive(
+    name = "goerli_testnet",
+    build_file_content = """
+filegroup(
+    name = "configs",
+    srcs = [
+        "prater/config.yaml",
+    ],
+    visibility = ["//visibility:public"],
+)
+    """,
+    sha256 = "43fc0f55ddff7b511713e2de07aa22846a67432df997296fb4fc09cd8ed1dcdb",
+    strip_prefix = "goerli-6522ac6684693740cd4ddcc2a0662e03702aa4a1",
+    url = "https://github.com/eth-clients/goerli/archive/6522ac6684693740cd4ddcc2a0662e03702aa4a1.tar.gz",
+)
+
 http_archive(
     name = "holesky_testnet",
     build_file_content = """
@@ -333,17 +360,17 @@ filegroup(
     visibility = ["//visibility:public"],
 )
 """,
-    sha256 = "9f66d8d5644982d3d0d2e3d2b9ebe77a5f96638a5d7fcd715599c32818195cb3",
-    strip_prefix = "holesky-ea39b9006210848e13f28d92e12a30548cecd41d",
-    url = "https://github.com/eth-clients/holesky/archive/ea39b9006210848e13f28d92e12a30548cecd41d.tar.gz",  # 2023-09-21
+    sha256 = "5f4be6fd088683ea9db45c863b9c5a1884422449e5b59fd2d561d3ba0f73ffd9",
+    strip_prefix = "holesky-9d9aabf2d4de51334ee5fed6c79a4d55097d1a43",
+    url = "https://github.com/eth-clients/holesky/archive/9d9aabf2d4de51334ee5fed6c79a4d55097d1a43.tar.gz",  # 2024-01-22
 )
 
 http_archive(
     name = "com_google_protobuf",
-    sha256 = "4e176116949be52b0408dfd24f8925d1eb674a781ae242a75296b17a1c721395",
-    strip_prefix = "protobuf-23.3",
+    sha256 = "9bd87b8280ef720d3240514f884e56a712f2218f0d693b48050c836028940a42",
+    strip_prefix = "protobuf-25.1",
     urls = [
-        "https://github.com/protocolbuffers/protobuf/archive/v23.3.tar.gz",
+        "https://github.com/protocolbuffers/protobuf/archive/v25.1.tar.gz",
     ],
 )
 

api/client/beacon/BUILD.bazel

@@ -12,11 +12,8 @@ go_library(
     deps = [
         "//api/client:go_default_library",
         "//api/server:go_default_library",
+        "//api/server/structs:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
-        "//beacon-chain/rpc/eth/beacon:go_default_library",
-        "//beacon-chain/rpc/eth/config:go_default_library",
-        "//beacon-chain/rpc/eth/shared:go_default_library",
-        "//beacon-chain/rpc/prysm/beacon:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//consensus-types/interfaces:go_default_library",
         "//consensus-types/primitives:go_default_library",

api/client/beacon/client.go

@@ -17,10 +17,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v4/api/client"
 	"github.com/prysmaticlabs/prysm/v4/api/server"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/beacon"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/config"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
-	apibeacon "github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/prysm/beacon"
+	"github.com/prysmaticlabs/prysm/v4/api/server/structs"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
 	"github.com/prysmaticlabs/prysm/v4/network/forks"
@@ -150,8 +147,8 @@ func (c *Client) GetFork(ctx context.Context, stateId StateOrBlockId) (*ethpb.Fo
 	if err != nil {
 		return nil, errors.Wrapf(err, "error requesting fork by state id = %s", stateId)
 	}
-	fr := &shared.Fork{}
-	dataWrapper := &struct{ Data *shared.Fork }{Data: fr}
+	fr := &structs.Fork{}
+	dataWrapper := &struct{ Data *structs.Fork }{Data: fr}
 	err = json.Unmarshal(body, dataWrapper)
 	if err != nil {
 		return nil, errors.Wrap(err, "error decoding json response in GetFork")
@@ -179,12 +176,12 @@ func (c *Client) GetForkSchedule(ctx context.Context) (forks.OrderedSchedule, er
 }
 
 // GetConfigSpec retrieve the current configs of the network used by the beacon node.
-func (c *Client) GetConfigSpec(ctx context.Context) (*config.GetSpecResponse, error) {
+func (c *Client) GetConfigSpec(ctx context.Context) (*structs.GetSpecResponse, error) {
 	body, err := c.Get(ctx, getConfigSpecPath)
 	if err != nil {
 		return nil, errors.Wrap(err, "error requesting configSpecPath")
 	}
-	fsr := &config.GetSpecResponse{}
+	fsr := &structs.GetSpecResponse{}
 	err = json.Unmarshal(body, fsr)
 	if err != nil {
 		return nil, err
@@ -259,7 +256,7 @@ func (c *Client) GetWeakSubjectivity(ctx context.Context) (*WeakSubjectivityData
 	if err != nil {
 		return nil, err
 	}
-	v := &apibeacon.GetWeakSubjectivityResponse{}
+	v := &structs.GetWeakSubjectivityResponse{}
 	err = json.Unmarshal(body, v)
 	if err != nil {
 		return nil, err
@@ -285,7 +282,7 @@ func (c *Client) GetWeakSubjectivity(ctx context.Context) (*WeakSubjectivityData
 
 // SubmitChangeBLStoExecution calls a beacon API endpoint to set the withdrawal addresses based on the given signed messages.
 // If the API responds with something other than OK there will be failure messages associated to the corresponding request message.
-func (c *Client) SubmitChangeBLStoExecution(ctx context.Context, request []*shared.SignedBLSToExecutionChange) error {
+func (c *Client) SubmitChangeBLStoExecution(ctx context.Context, request []*structs.SignedBLSToExecutionChange) error {
 	u := c.BaseURL().ResolveReference(&url.URL{Path: changeBLStoExecutionPath})
 	body, err := json.Marshal(request)
 	if err != nil {
@@ -324,12 +321,12 @@ func (c *Client) SubmitChangeBLStoExecution(ctx context.Context, request []*shar
 
 // GetBLStoExecutionChanges gets all the set withdrawal messages in the node's operation pool.
 // Returns a struct representation of json response.
-func (c *Client) GetBLStoExecutionChanges(ctx context.Context) (*beacon.BLSToExecutionChangesPoolResponse, error) {
+func (c *Client) GetBLStoExecutionChanges(ctx context.Context) (*structs.BLSToExecutionChangesPoolResponse, error) {
 	body, err := c.Get(ctx, changeBLStoExecutionPath)
 	if err != nil {
 		return nil, err
 	}
-	poolResponse := &beacon.BLSToExecutionChangesPoolResponse{}
+	poolResponse := &structs.BLSToExecutionChangesPoolResponse{}
 	err = json.Unmarshal(body, poolResponse)
 	if err != nil {
 		return nil, err
@@ -338,7 +335,7 @@ func (c *Client) GetBLStoExecutionChanges(ctx context.Context) (*beacon.BLSToExe
 }
 
 type forkScheduleResponse struct {
-	Data []shared.Fork
+	Data []structs.Fork
 }
 
 func (fsr *forkScheduleResponse) OrderedForkSchedule() (forks.OrderedSchedule, error) {

api/client/builder/BUILD.bazel

@@ -11,7 +11,7 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/v4/api/client/builder",
     visibility = ["//visibility:public"],
     deps = [
-        "//beacon-chain/rpc/eth/shared:go_default_library",
+        "//api/server/structs:go_default_library",
         "//config/fieldparams:go_default_library",
         "//consensus-types:go_default_library",
         "//consensus-types/blocks:go_default_library",
@@ -40,7 +40,7 @@ go_test(
     data = glob(["testdata/**"]),
     embed = [":go_default_library"],
     deps = [
-        "//beacon-chain/rpc/eth/shared:go_default_library",
+        "//api/server/structs:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
         "//consensus-types/blocks:go_default_library",

api/client/builder/bid.go

@@ -165,7 +165,7 @@ func WrappedBuilderBidCapella(p *ethpb.BuilderBidCapella) (Bid, error) {
 // Header returns the execution data interface.
 func (b builderBidCapella) Header() (interfaces.ExecutionData, error) {
 	// We have to convert big endian to little endian because the value is coming from the execution layer.
-	return blocks.WrappedExecutionPayloadHeaderCapella(b.p.Header, blocks.PayloadValueToGwei(b.p.Value))
+	return blocks.WrappedExecutionPayloadHeaderCapella(b.p.Header, blocks.PayloadValueToWei(b.p.Value))
 }
 
 // BlobKzgCommitments --
@@ -249,7 +249,7 @@ func (b builderBidDeneb) HashTreeRootWith(hh *ssz.Hasher) error {
 // Header --
 func (b builderBidDeneb) Header() (interfaces.ExecutionData, error) {
 	// We have to convert big endian to little endian because the value is coming from the execution layer.
-	return blocks.WrappedExecutionPayloadHeaderDeneb(b.p.Header, blocks.PayloadValueToGwei(b.p.Value))
+	return blocks.WrappedExecutionPayloadHeaderDeneb(b.p.Header, blocks.PayloadValueToWei(b.p.Value))
 }
 
 // BlobKzgCommitments --

api/client/builder/client.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"math/big"
 	"net"
 	"net/http"
 	"net/url"
@@ -13,7 +14,7 @@ import (
 	"text/template"
 
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
+	"github.com/prysmaticlabs/prysm/v4/api/server/structs"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
@@ -266,9 +267,9 @@ func (c *Client) RegisterValidator(ctx context.Context, svr []*ethpb.SignedValid
 		tracing.AnnotateError(span, err)
 		return err
 	}
-	vs := make([]*shared.SignedValidatorRegistration, len(svr))
+	vs := make([]*structs.SignedValidatorRegistration, len(svr))
 	for i := 0; i < len(svr); i++ {
-		vs[i] = shared.SignedValidatorRegistrationFromConsensus(svr[i])
+		vs[i] = structs.SignedValidatorRegistrationFromConsensus(svr[i])
 	}
 	body, err := json.Marshal(vs)
 	if err != nil {
@@ -293,7 +294,7 @@ func (c *Client) SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlyS
 		if err != nil {
 			return nil, nil, errors.Wrapf(err, "could not get protobuf block")
 		}
-		b, err := shared.SignedBlindedBeaconBlockBellatrixFromConsensus(&ethpb.SignedBlindedBeaconBlockBellatrix{Block: psb.Block, Signature: bytesutil.SafeCopyBytes(psb.Signature)})
+		b, err := structs.SignedBlindedBeaconBlockBellatrixFromConsensus(&ethpb.SignedBlindedBeaconBlockBellatrix{Block: psb.Block, Signature: bytesutil.SafeCopyBytes(psb.Signature)})
 		if err != nil {
 			return nil, nil, errors.Wrapf(err, "could not convert SignedBlindedBeaconBlockBellatrix to json marshalable type")
 		}
@@ -330,7 +331,7 @@ func (c *Client) SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlyS
 		if err != nil {
 			return nil, nil, errors.Wrapf(err, "could not get protobuf block")
 		}
-		b, err := shared.SignedBlindedBeaconBlockCapellaFromConsensus(&ethpb.SignedBlindedBeaconBlockCapella{Block: psb.Block, Signature: bytesutil.SafeCopyBytes(psb.Signature)})
+		b, err := structs.SignedBlindedBeaconBlockCapellaFromConsensus(&ethpb.SignedBlindedBeaconBlockCapella{Block: psb.Block, Signature: bytesutil.SafeCopyBytes(psb.Signature)})
 		if err != nil {
 			return nil, nil, errors.Wrapf(err, "could not convert SignedBlindedBeaconBlockCapella to json marshalable type")
 		}
@@ -357,7 +358,7 @@ func (c *Client) SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlyS
 		if err != nil {
 			return nil, nil, errors.Wrapf(err, "could not extract proto message from payload")
 		}
-		payload, err := blocks.WrappedExecutionPayloadCapella(p, 0)
+		payload, err := blocks.WrappedExecutionPayloadCapella(p, big.NewInt(0))
 		if err != nil {
 			return nil, nil, errors.Wrapf(err, "could not wrap execution payload in interface")
 		}
@@ -367,7 +368,7 @@ func (c *Client) SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlyS
 		if err != nil {
 			return nil, nil, errors.Wrapf(err, "could not get protobuf block")
 		}
-		b, err := shared.SignedBlindedBeaconBlockDenebFromConsensus(&ethpb.SignedBlindedBeaconBlockDeneb{Message: psb.Message, Signature: bytesutil.SafeCopyBytes(psb.Signature)})
+		b, err := structs.SignedBlindedBeaconBlockDenebFromConsensus(&ethpb.SignedBlindedBeaconBlockDeneb{Message: psb.Message, Signature: bytesutil.SafeCopyBytes(psb.Signature)})
 		if err != nil {
 			return nil, nil, errors.Wrapf(err, "could not convert SignedBlindedBeaconBlockDeneb to json marshalable type")
 		}
@@ -394,7 +395,7 @@ func (c *Client) SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlyS
 		if err != nil {
 			return nil, nil, errors.Wrapf(err, "could not extract proto message from payload")
 		}
-		payload, err := blocks.WrappedExecutionPayloadDeneb(p, 0)
+		payload, err := blocks.WrappedExecutionPayloadDeneb(p, big.NewInt(0))
 		if err != nil {
 			return nil, nil, errors.Wrapf(err, "could not wrap execution payload in interface")
 		}

api/client/builder/client_test.go

@@ -13,7 +13,7 @@ import (
 	"testing"
 
 	"github.com/prysmaticlabs/go-bitfield"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
+	"github.com/prysmaticlabs/prysm/v4/api/server/structs"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	types "github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
@@ -275,7 +275,24 @@ func TestClient_GetHeader(t *testing.T) {
 			require.Equal(t, len(kcgCommitments[i]) == 48, true)
 		}
 	})
-
+	t.Run("deneb, too many kzg commitments", func(t *testing.T) {
+		hc := &http.Client{
+			Transport: roundtrip(func(r *http.Request) (*http.Response, error) {
+				require.Equal(t, expectedPath, r.URL.Path)
+				return &http.Response{
+					StatusCode: http.StatusOK,
+					Body:       io.NopCloser(bytes.NewBufferString(testExampleHeaderResponseDenebTooManyBlobs)),
+					Request:    r.Clone(ctx),
+				}, nil
+			}),
+		}
+		c := &Client{
+			hc:      hc,
+			baseURL: &url.URL{Host: "localhost:3500", Scheme: "http"},
+		}
+		_, err := c.GetHeader(ctx, slot, bytesutil.ToBytes32(parentHash), bytesutil.ToBytes48(pubkey))
+		require.ErrorContains(t, "could not extract proto message from header: too many blob commitments: 7", err)
+	})
 	t.Run("unsupported version", func(t *testing.T) {
 		hc := &http.Client{
 			Transport: roundtrip(func(r *http.Request) (*http.Response, error) {
@@ -359,7 +376,7 @@ func TestSubmitBlindedBlock(t *testing.T) {
 			Transport: roundtrip(func(r *http.Request) (*http.Response, error) {
 				require.Equal(t, postBlindedBeaconBlockPath, r.URL.Path)
 				require.Equal(t, "deneb", r.Header.Get("Eth-Consensus-Version"))
-				var req shared.SignedBlindedBeaconBlockDeneb
+				var req structs.SignedBlindedBeaconBlockDeneb
 				err := json.NewDecoder(r.Body).Decode(&req)
 				require.NoError(t, err)
 				block, err := req.ToConsensus()
@@ -412,7 +429,7 @@ func TestSubmitBlindedBlock(t *testing.T) {
 		require.ErrorContains(t, "not a bellatrix payload", err)
 	})
 	t.Run("not blinded", func(t *testing.T) {
-		sbb, err := blocks.NewSignedBeaconBlock(&eth.SignedBeaconBlockBellatrix{Block: &eth.BeaconBlockBellatrix{Body: &eth.BeaconBlockBodyBellatrix{}}})
+		sbb, err := blocks.NewSignedBeaconBlock(&eth.SignedBeaconBlockBellatrix{Block: &eth.BeaconBlockBellatrix{Body: &eth.BeaconBlockBodyBellatrix{ExecutionPayload: &v1.ExecutionPayload{}}}})
 		require.NoError(t, err)
 		_, _, err = (&Client{}).SubmitBlindedBlock(ctx, sbb)
 		require.ErrorIs(t, err, errNotBlinded)

api/client/builder/types.go

@@ -908,6 +908,9 @@ func (bb *BuilderBidDeneb) ToProto() (*eth.BuilderBidDeneb, error) {
 	if err != nil {
 		return nil, err
 	}
+	if len(bb.BlobKzgCommitments) > fieldparams.MaxBlobsPerBlock {
+		return nil, fmt.Errorf("too many blob commitments: %d", len(bb.BlobKzgCommitments))
+	}
 	kzgCommitments := make([][]byte, len(bb.BlobKzgCommitments))
 	for i, commit := range bb.BlobKzgCommitments {
 		if len(commit) != fieldparams.BLSPubkeyLength {

api/client/builder/types_test.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/prysmaticlabs/go-bitfield"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
+	"github.com/prysmaticlabs/prysm/v4/api/server/structs"
 	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/math"
 	v1 "github.com/prysmaticlabs/prysm/v4/proto/engine/v1"
@@ -38,7 +38,7 @@ func TestSignedValidatorRegistration_MarshalJSON(t *testing.T) {
 		},
 		Signature: make([]byte, 96),
 	}
-	a := shared.SignedValidatorRegistrationFromConsensus(svr)
+	a := structs.SignedValidatorRegistrationFromConsensus(svr)
 	je, err := json.Marshal(a)
 	require.NoError(t, err)
 	// decode with a struct w/ plain strings so we can check the string encoding of the hex fields
@@ -55,7 +55,7 @@ func TestSignedValidatorRegistration_MarshalJSON(t *testing.T) {
 	require.Equal(t, "0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", un.Message.Pubkey)
 
 	t.Run("roundtrip", func(t *testing.T) {
-		b := &shared.SignedValidatorRegistration{}
+		b := &structs.SignedValidatorRegistration{}
 		if err := json.Unmarshal(je, b); err != nil {
 			require.NoError(t, err)
 		}
@@ -209,6 +209,39 @@ var testExampleHeaderResponseUnknownVersion = `{
   }
 }`
 
+var testExampleHeaderResponseDenebTooManyBlobs = `{
+  "version": "deneb",
+  "data": {
+    "message": {
+      "header": {
+        "parent_hash": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+        "fee_recipient": "0xabcf8e0d4e9587369b2301d0790347320302cc09",
+        "state_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+        "receipts_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+        "logs_bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+        "prev_randao": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+        "block_number": "1",
+        "gas_limit": "1",
+        "gas_used": "1",
+        "timestamp": "1",
+        "extra_data": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+        "base_fee_per_gas": "452312848583266388373324160190187140051835877600158453279131187530910662656",
+		"block_hash": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+        "transactions_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+        "withdrawals_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
+		"blob_gas_used": "1",	
+		"excess_blob_gas": "2"	
+      },
+      "blob_kzg_commitments": [
+          "","","","","","",""
+	  ],
+      "value": "652312848583266388373324160190187140051835877600158453279131187530910662656",
+      "pubkey": "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a"
+    },
+    "signature": "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
+  }
+}`
+
 func TestExecutionHeaderResponseUnmarshal(t *testing.T) {
 	hr := &ExecHeaderResponse{}
 	require.NoError(t, json.Unmarshal([]byte(testExampleHeaderResponse), hr))
@@ -1685,7 +1718,7 @@ func TestUint256UnmarshalTooBig(t *testing.T) {
 func TestMarshalBlindedBeaconBlockBodyBellatrix(t *testing.T) {
 	expected, err := os.ReadFile("testdata/blinded-block.json")
 	require.NoError(t, err)
-	b, err := shared.BlindedBeaconBlockBellatrixFromConsensus(&eth.BlindedBeaconBlockBellatrix{
+	b, err := structs.BlindedBeaconBlockBellatrixFromConsensus(&eth.BlindedBeaconBlockBellatrix{
 		Slot:          1,
 		ProposerIndex: 1,
 		ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
@@ -1715,7 +1748,7 @@ func TestMarshalBlindedBeaconBlockBodyBellatrix(t *testing.T) {
 func TestMarshalBlindedBeaconBlockBodyCapella(t *testing.T) {
 	expected, err := os.ReadFile("testdata/blinded-block-capella.json")
 	require.NoError(t, err)
-	b, err := shared.BlindedBeaconBlockCapellaFromConsensus(&eth.BlindedBeaconBlockCapella{
+	b, err := structs.BlindedBeaconBlockCapellaFromConsensus(&eth.BlindedBeaconBlockCapella{
 		Slot:          1,
 		ProposerIndex: 1,
 		ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),

api/constants.go

@@ -1,3 +1,7 @@
 package api
 
-const WebUrlPrefix = "/v2/validator/"
+const (
+	WebUrlPrefix        = "/v2/validator/"
+	WebApiUrlPrefix     = "/api/v2/validator/"
+	KeymanagerApiPrefix = "/eth/v1"
+)

api/headers.go

@@ -7,4 +7,6 @@ const (
 	ConsensusBlockValueHeader     = "Eth-Consensus-Block-Value"
 	JsonMediaType                 = "application/json"
 	OctetStreamMediaType          = "application/octet-stream"
+	EventStreamMediaType          = "text/event-stream"
+	KeepAlive                     = "keep-alive"
 )

api/server/structs/BUILD.bazel

@@ -0,0 +1,50 @@
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "block.go",
+        "conversions.go",
+        "conversions_block.go",
+        "conversions_state.go",
+        "endpoints_beacon.go",
+        "endpoints_blob.go",
+        "endpoints_builder.go",
+        "endpoints_config.go",
+        "endpoints_debug.go",
+        "endpoints_events.go",
+        "endpoints_lightclient.go",
+        "endpoints_node.go",
+        "endpoints_rewards.go",
+        "endpoints_validator.go",
+        "other.go",
+        "state.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/v4/api/server/structs",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//api/server:go_default_library",
+        "//beacon-chain/state:go_default_library",
+        "//config/fieldparams:go_default_library",
+        "//consensus-types/primitives:go_default_library",
+        "//consensus-types/validator:go_default_library",
+        "//container/slice:go_default_library",
+        "//encoding/bytesutil:go_default_library",
+        "//math:go_default_library",
+        "//proto/engine/v1:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
+        "@com_github_ethereum_go_ethereum//common:go_default_library",
+        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["conversions_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//proto/prysm/v1alpha1:go_default_library",
+        "//testing/require:go_default_library",
+    ],
+)

api/server/structs/block.go

@@ -1,4 +1,4 @@
-package shared
+package structs
 
 type SignedBeaconBlock struct {
 	Message   *BeaconBlock `json:"message"`
@@ -325,11 +325,11 @@ type ExecutionPayloadDeneb struct {
 	Timestamp     string        `json:"timestamp"`
 	ExtraData     string        `json:"extra_data"`
 	BaseFeePerGas string        `json:"base_fee_per_gas"`
-	BlobGasUsed   string        `json:"blob_gas_used"`
-	ExcessBlobGas string        `json:"excess_blob_gas"`
 	BlockHash     string        `json:"block_hash"`
 	Transactions  []string      `json:"transactions"`
 	Withdrawals   []*Withdrawal `json:"withdrawals"`
+	BlobGasUsed   string        `json:"blob_gas_used"`
+	ExcessBlobGas string        `json:"excess_blob_gas"`
 }
 
 type ExecutionPayloadHeaderDeneb struct {
@@ -345,9 +345,9 @@ type ExecutionPayloadHeaderDeneb struct {
 	Timestamp        string `json:"timestamp"`
 	ExtraData        string `json:"extra_data"`
 	BaseFeePerGas    string `json:"base_fee_per_gas"`
-	BlobGasUsed      string `json:"blob_gas_used"`
-	ExcessBlobGas    string `json:"excess_blob_gas"`
 	BlockHash        string `json:"block_hash"`
 	TransactionsRoot string `json:"transactions_root"`
 	WithdrawalsRoot  string `json:"withdrawals_root"`
+	BlobGasUsed      string `json:"blob_gas_used"`
+	ExcessBlobGas    string `json:"excess_blob_gas"`
 }

api/server/structs/conversions.go

@@ -1,4 +1,4 @@
-package shared
+package structs
 
 import (
 	"fmt"
@@ -23,7 +23,7 @@ var errNilValue = errors.New("nil value")
 
 func ValidatorFromConsensus(v *eth.Validator) *Validator {
 	return &Validator{
-		PublicKey:                  hexutil.Encode(v.PublicKey),
+		Pubkey:                     hexutil.Encode(v.PublicKey),
 		WithdrawalCredentials:      hexutil.Encode(v.WithdrawalCredentials),
 		EffectiveBalance:           fmt.Sprintf("%d", v.EffectiveBalance),
 		Slashed:                    v.Slashed,
@@ -769,32 +769,36 @@ func ProposerSlashingsToConsensus(src []*ProposerSlashing) ([]*eth.ProposerSlash
 func ProposerSlashingsFromConsensus(src []*eth.ProposerSlashing) []*ProposerSlashing {
 	proposerSlashings := make([]*ProposerSlashing, len(src))
 	for i, s := range src {
-		proposerSlashings[i] = &ProposerSlashing{
-			SignedHeader1: &SignedBeaconBlockHeader{
-				Message: &BeaconBlockHeader{
-					Slot:          fmt.Sprintf("%d", s.Header_1.Header.Slot),
-					ProposerIndex: fmt.Sprintf("%d", s.Header_1.Header.ProposerIndex),
-					ParentRoot:    hexutil.Encode(s.Header_1.Header.ParentRoot),
-					StateRoot:     hexutil.Encode(s.Header_1.Header.StateRoot),
-					BodyRoot:      hexutil.Encode(s.Header_1.Header.BodyRoot),
-				},
-				Signature: hexutil.Encode(s.Header_1.Signature),
-			},
-			SignedHeader2: &SignedBeaconBlockHeader{
-				Message: &BeaconBlockHeader{
-					Slot:          fmt.Sprintf("%d", s.Header_2.Header.Slot),
-					ProposerIndex: fmt.Sprintf("%d", s.Header_2.Header.ProposerIndex),
-					ParentRoot:    hexutil.Encode(s.Header_2.Header.ParentRoot),
-					StateRoot:     hexutil.Encode(s.Header_2.Header.StateRoot),
-					BodyRoot:      hexutil.Encode(s.Header_2.Header.BodyRoot),
-				},
-				Signature: hexutil.Encode(s.Header_2.Signature),
-			},
-		}
+		proposerSlashings[i] = ProposerSlashingFromConsensus(s)
 	}
 	return proposerSlashings
 }
 
+func ProposerSlashingFromConsensus(src *eth.ProposerSlashing) *ProposerSlashing {
+	return &ProposerSlashing{
+		SignedHeader1: &SignedBeaconBlockHeader{
+			Message: &BeaconBlockHeader{
+				Slot:          fmt.Sprintf("%d", src.Header_1.Header.Slot),
+				ProposerIndex: fmt.Sprintf("%d", src.Header_1.Header.ProposerIndex),
+				ParentRoot:    hexutil.Encode(src.Header_1.Header.ParentRoot),
+				StateRoot:     hexutil.Encode(src.Header_1.Header.StateRoot),
+				BodyRoot:      hexutil.Encode(src.Header_1.Header.BodyRoot),
+			},
+			Signature: hexutil.Encode(src.Header_1.Signature),
+		},
+		SignedHeader2: &SignedBeaconBlockHeader{
+			Message: &BeaconBlockHeader{
+				Slot:          fmt.Sprintf("%d", src.Header_2.Header.Slot),
+				ProposerIndex: fmt.Sprintf("%d", src.Header_2.Header.ProposerIndex),
+				ParentRoot:    hexutil.Encode(src.Header_2.Header.ParentRoot),
+				StateRoot:     hexutil.Encode(src.Header_2.Header.StateRoot),
+				BodyRoot:      hexutil.Encode(src.Header_2.Header.BodyRoot),
+			},
+			Signature: hexutil.Encode(src.Header_2.Signature),
+		},
+	}
+}
+
 func AttesterSlashingsToConsensus(src []*AttesterSlashing) ([]*eth.AttesterSlashing, error) {
 	if src == nil {
 		return nil, errNilValue
@@ -875,54 +879,58 @@ func AttesterSlashingsToConsensus(src []*AttesterSlashing) ([]*eth.AttesterSlash
 func AttesterSlashingsFromConsensus(src []*eth.AttesterSlashing) []*AttesterSlashing {
 	attesterSlashings := make([]*AttesterSlashing, len(src))
 	for i, s := range src {
-		a1AttestingIndices := make([]string, len(s.Attestation_1.AttestingIndices))
-		for j, ix := range s.Attestation_1.AttestingIndices {
-			a1AttestingIndices[j] = fmt.Sprintf("%d", ix)
-		}
-		a2AttestingIndices := make([]string, len(s.Attestation_2.AttestingIndices))
-		for j, ix := range s.Attestation_2.AttestingIndices {
-			a2AttestingIndices[j] = fmt.Sprintf("%d", ix)
-		}
-		attesterSlashings[i] = &AttesterSlashing{
-			Attestation1: &IndexedAttestation{
-				AttestingIndices: a1AttestingIndices,
-				Data: &AttestationData{
-					Slot:            fmt.Sprintf("%d", s.Attestation_1.Data.Slot),
-					CommitteeIndex:  fmt.Sprintf("%d", s.Attestation_1.Data.CommitteeIndex),
-					BeaconBlockRoot: hexutil.Encode(s.Attestation_1.Data.BeaconBlockRoot),
-					Source: &Checkpoint{
-						Epoch: fmt.Sprintf("%d", s.Attestation_1.Data.Source.Epoch),
-						Root:  hexutil.Encode(s.Attestation_1.Data.Source.Root),
-					},
-					Target: &Checkpoint{
-						Epoch: fmt.Sprintf("%d", s.Attestation_1.Data.Target.Epoch),
-						Root:  hexutil.Encode(s.Attestation_1.Data.Target.Root),
-					},
-				},
-				Signature: hexutil.Encode(s.Attestation_1.Signature),
-			},
-			Attestation2: &IndexedAttestation{
-				AttestingIndices: a2AttestingIndices,
-				Data: &AttestationData{
-					Slot:            fmt.Sprintf("%d", s.Attestation_2.Data.Slot),
-					CommitteeIndex:  fmt.Sprintf("%d", s.Attestation_2.Data.CommitteeIndex),
-					BeaconBlockRoot: hexutil.Encode(s.Attestation_2.Data.BeaconBlockRoot),
-					Source: &Checkpoint{
-						Epoch: fmt.Sprintf("%d", s.Attestation_2.Data.Source.Epoch),
-						Root:  hexutil.Encode(s.Attestation_2.Data.Source.Root),
-					},
-					Target: &Checkpoint{
-						Epoch: fmt.Sprintf("%d", s.Attestation_2.Data.Target.Epoch),
-						Root:  hexutil.Encode(s.Attestation_2.Data.Target.Root),
-					},
-				},
-				Signature: hexutil.Encode(s.Attestation_2.Signature),
-			},
-		}
+		attesterSlashings[i] = AttesterSlashingFromConsensus(s)
 	}
 	return attesterSlashings
 }
 
+func AttesterSlashingFromConsensus(src *eth.AttesterSlashing) *AttesterSlashing {
+	a1AttestingIndices := make([]string, len(src.Attestation_1.AttestingIndices))
+	for j, ix := range src.Attestation_1.AttestingIndices {
+		a1AttestingIndices[j] = fmt.Sprintf("%d", ix)
+	}
+	a2AttestingIndices := make([]string, len(src.Attestation_2.AttestingIndices))
+	for j, ix := range src.Attestation_2.AttestingIndices {
+		a2AttestingIndices[j] = fmt.Sprintf("%d", ix)
+	}
+	return &AttesterSlashing{
+		Attestation1: &IndexedAttestation{
+			AttestingIndices: a1AttestingIndices,
+			Data: &AttestationData{
+				Slot:            fmt.Sprintf("%d", src.Attestation_1.Data.Slot),
+				CommitteeIndex:  fmt.Sprintf("%d", src.Attestation_1.Data.CommitteeIndex),
+				BeaconBlockRoot: hexutil.Encode(src.Attestation_1.Data.BeaconBlockRoot),
+				Source: &Checkpoint{
+					Epoch: fmt.Sprintf("%d", src.Attestation_1.Data.Source.Epoch),
+					Root:  hexutil.Encode(src.Attestation_1.Data.Source.Root),
+				},
+				Target: &Checkpoint{
+					Epoch: fmt.Sprintf("%d", src.Attestation_1.Data.Target.Epoch),
+					Root:  hexutil.Encode(src.Attestation_1.Data.Target.Root),
+				},
+			},
+			Signature: hexutil.Encode(src.Attestation_1.Signature),
+		},
+		Attestation2: &IndexedAttestation{
+			AttestingIndices: a2AttestingIndices,
+			Data: &AttestationData{
+				Slot:            fmt.Sprintf("%d", src.Attestation_2.Data.Slot),
+				CommitteeIndex:  fmt.Sprintf("%d", src.Attestation_2.Data.CommitteeIndex),
+				BeaconBlockRoot: hexutil.Encode(src.Attestation_2.Data.BeaconBlockRoot),
+				Source: &Checkpoint{
+					Epoch: fmt.Sprintf("%d", src.Attestation_2.Data.Source.Epoch),
+					Root:  hexutil.Encode(src.Attestation_2.Data.Source.Root),
+				},
+				Target: &Checkpoint{
+					Epoch: fmt.Sprintf("%d", src.Attestation_2.Data.Target.Epoch),
+					Root:  hexutil.Encode(src.Attestation_2.Data.Target.Root),
+				},
+			},
+			Signature: hexutil.Encode(src.Attestation_2.Signature),
+		},
+	}
+}
+
 func AttsToConsensus(src []*Attestation) ([]*eth.Attestation, error) {
 	if src == nil {
 		return nil, errNilValue
@@ -1066,3 +1074,17 @@ func sszBytesToUint256String(b []byte) (string, error) {
 	}
 	return bi.String(), nil
 }
+
+func DepositSnapshotFromConsensus(ds *eth.DepositSnapshot) *DepositSnapshot {
+	finalized := make([]string, 0, len(ds.Finalized))
+	for _, f := range ds.Finalized {
+		finalized = append(finalized, hexutil.Encode(f))
+	}
+	return &DepositSnapshot{
+		Finalized:            finalized,
+		DepositRoot:          hexutil.Encode(ds.DepositRoot),
+		DepositCount:         fmt.Sprintf("%d", ds.DepositCount),
+		ExecutionBlockHash:   hexutil.Encode(ds.ExecutionHash),
+		ExecutionBlockHeight: fmt.Sprintf("%d", ds.ExecutionDepth),
+	}
+}

api/server/structs/conversions_block.go

@@ -1,4 +1,4 @@
-package shared
+package structs
 
 import (
 	"fmt"
@@ -559,7 +559,7 @@ func (b *SignedBlindedBeaconBlockBellatrix) ToGeneric() (*eth.GenericSignedBeaco
 		Block:     bl,
 		Signature: sig,
 	}
-	return &eth.GenericSignedBeaconBlock{Block: &eth.GenericSignedBeaconBlock_BlindedBellatrix{BlindedBellatrix: block}, IsBlinded: true, PayloadValue: 0 /* can't get payload value from blinded block */}, nil
+	return &eth.GenericSignedBeaconBlock{Block: &eth.GenericSignedBeaconBlock_BlindedBellatrix{BlindedBellatrix: block}, IsBlinded: true}, nil
 }
 
 func (b *BlindedBeaconBlockBellatrix) ToGeneric() (*eth.GenericBeaconBlock, error) {
@@ -567,7 +567,7 @@ func (b *BlindedBeaconBlockBellatrix) ToGeneric() (*eth.GenericBeaconBlock, erro
 	if err != nil {
 		return nil, err
 	}
-	return &eth.GenericBeaconBlock{Block: &eth.GenericBeaconBlock_BlindedBellatrix{BlindedBellatrix: block}, IsBlinded: true, PayloadValue: 0 /* can't get payload value from blinded block */}, nil
+	return &eth.GenericBeaconBlock{Block: &eth.GenericBeaconBlock_BlindedBellatrix{BlindedBellatrix: block}, IsBlinded: true}, nil
 }
 
 func (b *BlindedBeaconBlockBellatrix) ToConsensus() (*eth.BlindedBeaconBlockBellatrix, error) {
@@ -1016,7 +1016,7 @@ func (b *SignedBlindedBeaconBlockCapella) ToGeneric() (*eth.GenericSignedBeaconB
 		Block:     bl,
 		Signature: sig,
 	}
-	return &eth.GenericSignedBeaconBlock{Block: &eth.GenericSignedBeaconBlock_BlindedCapella{BlindedCapella: block}, IsBlinded: true, PayloadValue: 0 /* can't get payload value from blinded block */}, nil
+	return &eth.GenericSignedBeaconBlock{Block: &eth.GenericSignedBeaconBlock_BlindedCapella{BlindedCapella: block}, IsBlinded: true}, nil
 }
 
 func (b *BlindedBeaconBlockCapella) ToGeneric() (*eth.GenericBeaconBlock, error) {
@@ -1024,7 +1024,7 @@ func (b *BlindedBeaconBlockCapella) ToGeneric() (*eth.GenericBeaconBlock, error)
 	if err != nil {
 		return nil, err
 	}
-	return &eth.GenericBeaconBlock{Block: &eth.GenericBeaconBlock_BlindedCapella{BlindedCapella: block}, IsBlinded: true, PayloadValue: 0 /* can't get payload value from blinded block */}, nil
+	return &eth.GenericBeaconBlock{Block: &eth.GenericBeaconBlock_BlindedCapella{BlindedCapella: block}, IsBlinded: true}, nil
 }
 
 func (b *BlindedBeaconBlockCapella) ToConsensus() (*eth.BlindedBeaconBlockCapella, error) {
@@ -2333,10 +2333,10 @@ func ExecutionPayloadHeaderDenebFromConsensus(payload *enginev1.ExecutionPayload
 		Timestamp:        fmt.Sprintf("%d", payload.Timestamp),
 		ExtraData:        hexutil.Encode(payload.ExtraData),
 		BaseFeePerGas:    baseFeePerGas,
-		BlobGasUsed:      fmt.Sprintf("%d", payload.BlobGasUsed),
-		ExcessBlobGas:    fmt.Sprintf("%d", payload.ExcessBlobGas),
 		BlockHash:        hexutil.Encode(payload.BlockHash),
 		TransactionsRoot: hexutil.Encode(payload.TransactionsRoot),
 		WithdrawalsRoot:  hexutil.Encode(payload.WithdrawalsRoot),
+		BlobGasUsed:      fmt.Sprintf("%d", payload.BlobGasUsed),
+		ExcessBlobGas:    fmt.Sprintf("%d", payload.ExcessBlobGas),
 	}, nil
 }

api/server/structs/conversions_state.go

@@ -1,4 +1,4 @@
-package shared
+package structs
 
 import (
 	"errors"

api/server/structs/conversions_test.go

@@ -0,0 +1,26 @@
+package structs
+
+import (
+	"testing"
+
+	eth "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/v4/testing/require"
+)
+
+func TestDepositSnapshotFromConsensus(t *testing.T) {
+	ds := &eth.DepositSnapshot{
+		Finalized:      [][]byte{{0xde, 0xad, 0xbe, 0xef}, {0xca, 0xfe, 0xba, 0xbe}},
+		DepositRoot:    []byte{0xab, 0xcd},
+		DepositCount:   12345,
+		ExecutionHash:  []byte{0x12, 0x34},
+		ExecutionDepth: 67890,
+	}
+
+	res := DepositSnapshotFromConsensus(ds)
+	require.NotNil(t, res)
+	require.DeepEqual(t, []string{"0xdeadbeef", "0xcafebabe"}, res.Finalized)
+	require.Equal(t, "0xabcd", res.DepositRoot)
+	require.Equal(t, "12345", res.DepositCount)
+	require.Equal(t, "0x1234", res.ExecutionBlockHash)
+	require.Equal(t, "67890", res.ExecutionBlockHeight)
+}

api/server/structs/endpoints_beacon.go

@@ -1,9 +1,7 @@
-package beacon
+package structs
 
 import (
 	"encoding/json"
-
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
 )
 
 type BlockRootResponse struct {
@@ -17,31 +15,31 @@ type BlockRoot struct {
 }
 
 type GetCommitteesResponse struct {
-	Data                []*shared.Committee `json:"data"`
-	ExecutionOptimistic bool                `json:"execution_optimistic"`
-	Finalized           bool                `json:"finalized"`
+	Data                []*Committee `json:"data"`
+	ExecutionOptimistic bool         `json:"execution_optimistic"`
+	Finalized           bool         `json:"finalized"`
 }
 
 type ListAttestationsResponse struct {
-	Data []*shared.Attestation `json:"data"`
+	Data []*Attestation `json:"data"`
 }
 
 type SubmitAttestationsRequest struct {
-	Data []*shared.Attestation `json:"data"`
+	Data []*Attestation `json:"data"`
 }
 
 type ListVoluntaryExitsResponse struct {
-	Data []*shared.SignedVoluntaryExit `json:"data"`
+	Data []*SignedVoluntaryExit `json:"data"`
 }
 
 type SubmitSyncCommitteeSignaturesRequest struct {
-	Data []*shared.SyncCommitteeMessage `json:"data"`
+	Data []*SyncCommitteeMessage `json:"data"`
 }
 
 type GetStateForkResponse struct {
-	Data                *shared.Fork `json:"data"`
-	ExecutionOptimistic bool         `json:"execution_optimistic"`
-	Finalized           bool         `json:"finalized"`
+	Data                *Fork `json:"data"`
+	ExecutionOptimistic bool  `json:"execution_optimistic"`
+	Finalized           bool  `json:"finalized"`
 }
 
 type GetFinalityCheckpointsResponse struct {
@@ -51,9 +49,9 @@ type GetFinalityCheckpointsResponse struct {
 }
 
 type FinalityCheckpoints struct {
-	PreviousJustified *shared.Checkpoint `json:"previous_justified"`
-	CurrentJustified  *shared.Checkpoint `json:"current_justified"`
-	Finalized         *shared.Checkpoint `json:"finalized"`
+	PreviousJustified *Checkpoint `json:"previous_justified"`
+	CurrentJustified  *Checkpoint `json:"current_justified"`
+	Finalized         *Checkpoint `json:"finalized"`
 }
 
 type GetGenesisResponse struct {
@@ -67,15 +65,15 @@ type Genesis struct {
 }
 
 type GetBlockHeadersResponse struct {
-	Data                []*shared.SignedBeaconBlockHeaderContainer `json:"data"`
-	ExecutionOptimistic bool                                       `json:"execution_optimistic"`
-	Finalized           bool                                       `json:"finalized"`
+	Data                []*SignedBeaconBlockHeaderContainer `json:"data"`
+	ExecutionOptimistic bool                                `json:"execution_optimistic"`
+	Finalized           bool                                `json:"finalized"`
 }
 
 type GetBlockHeaderResponse struct {
-	ExecutionOptimistic bool                                     `json:"execution_optimistic"`
-	Finalized           bool                                     `json:"finalized"`
-	Data                *shared.SignedBeaconBlockHeaderContainer `json:"data"`
+	ExecutionOptimistic bool                              `json:"execution_optimistic"`
+	Finalized           bool                              `json:"finalized"`
+	Data                *SignedBeaconBlockHeaderContainer `json:"data"`
 }
 
 type GetValidatorsRequest struct {
@@ -108,17 +106,6 @@ type ValidatorContainer struct {
 	Validator *Validator `json:"validator"`
 }
 
-type Validator struct {
-	Pubkey                     string `json:"pubkey"`
-	WithdrawalCredentials      string `json:"withdrawal_credentials"`
-	EffectiveBalance           string `json:"effective_balance"`
-	Slashed                    bool   `json:"slashed"`
-	ActivationEligibilityEpoch string `json:"activation_eligibility_epoch"`
-	ActivationEpoch            string `json:"activation_epoch"`
-	ExitEpoch                  string `json:"exit_epoch"`
-	WithdrawableEpoch          string `json:"withdrawable_epoch"`
-}
-
 type ValidatorBalance struct {
 	Index   string `json:"index"`
 	Balance string `json:"balance"`
@@ -141,9 +128,9 @@ type SignedBlock struct {
 }
 
 type GetBlockAttestationsResponse struct {
-	ExecutionOptimistic bool                  `json:"execution_optimistic"`
-	Finalized           bool                  `json:"finalized"`
-	Data                []*shared.Attestation `json:"data"`
+	ExecutionOptimistic bool           `json:"execution_optimistic"`
+	Finalized           bool           `json:"finalized"`
+	Data                []*Attestation `json:"data"`
 }
 
 type GetStateRootResponse struct {
@@ -178,13 +165,34 @@ type SyncCommitteeValidators struct {
 }
 
 type BLSToExecutionChangesPoolResponse struct {
-	Data []*shared.SignedBLSToExecutionChange `json:"data"`
+	Data []*SignedBLSToExecutionChange `json:"data"`
 }
 
 type GetAttesterSlashingsResponse struct {
-	Data []*shared.AttesterSlashing `json:"data"`
+	Data []*AttesterSlashing `json:"data"`
 }
 
 type GetProposerSlashingsResponse struct {
-	Data []*shared.ProposerSlashing `json:"data"`
+	Data []*ProposerSlashing `json:"data"`
+}
+
+type GetWeakSubjectivityResponse struct {
+	Data *WeakSubjectivityData `json:"data"`
+}
+
+type WeakSubjectivityData struct {
+	WsCheckpoint *Checkpoint `json:"ws_checkpoint"`
+	StateRoot    string      `json:"state_root"`
+}
+
+type GetDepositSnapshotResponse struct {
+	Data *DepositSnapshot `json:"data"`
+}
+
+type DepositSnapshot struct {
+	Finalized            []string `json:"finalized"`
+	DepositRoot          string   `json:"deposit_root"`
+	DepositCount         string   `json:"deposit_count"`
+	ExecutionBlockHash   string   `json:"execution_block_hash"`
+	ExecutionBlockHeight string   `json:"execution_block_height"`
 }

api/server/structs/endpoints_blob.go

@@ -0,0 +1,14 @@
+package structs
+
+type SidecarsResponse struct {
+	Data []*Sidecar `json:"data"`
+}
+
+type Sidecar struct {
+	Index                    string                   `json:"index"`
+	Blob                     string                   `json:"blob"`
+	SignedBeaconBlockHeader  *SignedBeaconBlockHeader `json:"signed_block_header"`
+	KzgCommitment            string                   `json:"kzg_commitment"`
+	KzgProof                 string                   `json:"kzg_proof"`
+	CommitmentInclusionProof []string                 `json:"kzg_commitment_inclusion_proof"`
+}

api/server/structs/endpoints_builder.go

@@ -1,4 +1,4 @@
-package builder
+package structs
 
 type ExpectedWithdrawalsResponse struct {
 	Data                []*ExpectedWithdrawal `json:"data"`

api/server/structs/endpoints_config.go

@@ -1,6 +1,4 @@
-package config
-
-import "github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
+package structs
 
 type GetDepositContractResponse struct {
 	Data *DepositContractData `json:"data"`
@@ -12,7 +10,7 @@ type DepositContractData struct {
 }
 
 type GetForkScheduleResponse struct {
-	Data []*shared.Fork `json:"data"`
+	Data []*Fork `json:"data"`
 }
 
 type GetSpecResponse struct {

api/server/structs/endpoints_debug.go

@@ -1,9 +1,7 @@
-package debug
+package structs
 
 import (
 	"encoding/json"
-
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
 )
 
 type GetBeaconStateV2Response struct {
@@ -24,18 +22,18 @@ type ForkChoiceHead struct {
 }
 
 type GetForkChoiceDumpResponse struct {
-	JustifiedCheckpoint *shared.Checkpoint       `json:"justified_checkpoint"`
-	FinalizedCheckpoint *shared.Checkpoint       `json:"finalized_checkpoint"`
+	JustifiedCheckpoint *Checkpoint              `json:"justified_checkpoint"`
+	FinalizedCheckpoint *Checkpoint              `json:"finalized_checkpoint"`
 	ForkChoiceNodes     []*ForkChoiceNode        `json:"fork_choice_nodes"`
 	ExtraData           *ForkChoiceDumpExtraData `json:"extra_data"`
 }
 
 type ForkChoiceDumpExtraData struct {
-	UnrealizedJustifiedCheckpoint *shared.Checkpoint `json:"unrealized_justified_checkpoint"`
-	UnrealizedFinalizedCheckpoint *shared.Checkpoint `json:"unrealized_finalized_checkpoint"`
-	ProposerBoostRoot             string             `json:"proposer_boost_root"`
-	PreviousProposerBoostRoot     string             `json:"previous_proposer_boost_root"`
-	HeadRoot                      string             `json:"head_root"`
+	UnrealizedJustifiedCheckpoint *Checkpoint `json:"unrealized_justified_checkpoint"`
+	UnrealizedFinalizedCheckpoint *Checkpoint `json:"unrealized_finalized_checkpoint"`
+	ProposerBoostRoot             string      `json:"proposer_boost_root"`
+	PreviousProposerBoostRoot     string      `json:"previous_proposer_boost_root"`
+	HeadRoot                      string      `json:"head_root"`
 }
 
 type ForkChoiceNode struct {

api/server/structs/endpoints_events.go

@@ -1,9 +1,7 @@
-package events
+package structs
 
 import (
 	"encoding/json"
-
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
 )
 
 type HeadEvent struct {
@@ -23,13 +21,13 @@ type BlockEvent struct {
 }
 
 type AggregatedAttEventSource struct {
-	Aggregate *shared.Attestation `json:"aggregate"`
+	Aggregate *Attestation `json:"aggregate"`
 }
 
 type UnaggregatedAttEventSource struct {
-	AggregationBits string                  `json:"aggregation_bits"`
-	Data            *shared.AttestationData `json:"data"`
-	Signature       string                  `json:"signature"`
+	AggregationBits string           `json:"aggregation_bits"`
+	Data            *AttestationData `json:"data"`
+	Signature       string           `json:"signature"`
 }
 
 type FinalizedCheckpointEvent struct {
@@ -71,18 +69,18 @@ type PayloadAttributesV1 struct {
 }
 
 type PayloadAttributesV2 struct {
-	Timestamp             string               `json:"timestamp"`
-	PrevRandao            string               `json:"prev_randao"`
-	SuggestedFeeRecipient string               `json:"suggested_fee_recipient"`
-	Withdrawals           []*shared.Withdrawal `json:"withdrawals"`
+	Timestamp             string        `json:"timestamp"`
+	PrevRandao            string        `json:"prev_randao"`
+	SuggestedFeeRecipient string        `json:"suggested_fee_recipient"`
+	Withdrawals           []*Withdrawal `json:"withdrawals"`
 }
 
 type PayloadAttributesV3 struct {
-	Timestamp             string               `json:"timestamp"`
-	PrevRandao            string               `json:"prev_randao"`
-	SuggestedFeeRecipient string               `json:"suggested_fee_recipient"`
-	Withdrawals           []*shared.Withdrawal `json:"withdrawals"`
-	ParentBeaconBlockRoot string               `json:"parent_beacon_block_root"`
+	Timestamp             string        `json:"timestamp"`
+	PrevRandao            string        `json:"prev_randao"`
+	SuggestedFeeRecipient string        `json:"suggested_fee_recipient"`
+	Withdrawals           []*Withdrawal `json:"withdrawals"`
+	ParentBeaconBlockRoot string        `json:"parent_beacon_block_root"`
 }
 
 type BlobSidecarEvent struct {
@@ -92,3 +90,27 @@ type BlobSidecarEvent struct {
 	KzgCommitment string `json:"kzg_commitment"`
 	VersionedHash string `json:"versioned_hash"`
 }
+
+type LightClientFinalityUpdateEvent struct {
+	Version string                     `json:"version"`
+	Data    *LightClientFinalityUpdate `json:"data"`
+}
+
+type LightClientFinalityUpdate struct {
+	AttestedHeader  *BeaconBlockHeader `json:"attested_header"`
+	FinalizedHeader *BeaconBlockHeader `json:"finalized_header"`
+	FinalityBranch  []string           `json:"finality_branch"`
+	SyncAggregate   *SyncAggregate     `json:"sync_aggregate"`
+	SignatureSlot   string             `json:"signature_slot"`
+}
+
+type LightClientOptimisticUpdateEvent struct {
+	Version string                       `json:"version"`
+	Data    *LightClientOptimisticUpdate `json:"data"`
+}
+
+type LightClientOptimisticUpdate struct {
+	AttestedHeader *BeaconBlockHeader `json:"attested_header"`
+	SyncAggregate  *SyncAggregate     `json:"sync_aggregate"`
+	SignatureSlot  string             `json:"signature_slot"`
+}

api/server/structs/endpoints_lightclient.go

@@ -0,0 +1,31 @@
+package structs
+
+type LightClientBootstrapResponse struct {
+	Version string                `json:"version"`
+	Data    *LightClientBootstrap `json:"data"`
+}
+
+type LightClientBootstrap struct {
+	Header                     *BeaconBlockHeader `json:"header"`
+	CurrentSyncCommittee       *SyncCommittee     `json:"current_sync_committee"`
+	CurrentSyncCommitteeBranch []string           `json:"current_sync_committee_branch"`
+}
+
+type LightClientUpdate struct {
+	AttestedHeader          *BeaconBlockHeader `json:"attested_header"`
+	NextSyncCommittee       *SyncCommittee     `json:"next_sync_committee,omitempty"`
+	FinalizedHeader         *BeaconBlockHeader `json:"finalized_header,omitempty"`
+	SyncAggregate           *SyncAggregate     `json:"sync_aggregate"`
+	NextSyncCommitteeBranch []string           `json:"next_sync_committee_branch,omitempty"`
+	FinalityBranch          []string           `json:"finality_branch,omitempty"`
+	SignatureSlot           string             `json:"signature_slot"`
+}
+
+type LightClientUpdateWithVersion struct {
+	Version string             `json:"version"`
+	Data    *LightClientUpdate `json:"data"`
+}
+
+type LightClientUpdatesByRangeResponse struct {
+	Updates []*LightClientUpdateWithVersion `json:"updates"`
+}

api/server/structs/endpoints_node.go

@@ -1,4 +1,4 @@
-package node
+package structs
 
 type SyncStatusResponse struct {
 	Data *SyncStatusResponseData `json:"data"`
@@ -63,3 +63,11 @@ type GetVersionResponse struct {
 type Version struct {
 	Version string `json:"version"`
 }
+
+type AddrRequest struct {
+	Addr string `json:"addr"`
+}
+
+type PeersResponse struct {
+	Peers []*Peer `json:"peers"`
+}

api/server/structs/endpoints_rewards.go

@@ -1,4 +1,4 @@
-package rewards
+package structs
 
 type BlockRewardsResponse struct {
 	Data                *BlockRewards `json:"data"`
@@ -31,6 +31,7 @@ type IdealAttestationReward struct {
 	Head             string `json:"head"`
 	Target           string `json:"target"`
 	Source           string `json:"source"`
+	Inactivity       string `json:"inactivity"`
 }
 
 type TotalAttestationReward struct {
@@ -38,7 +39,7 @@ type TotalAttestationReward struct {
 	Head           string `json:"head"`
 	Target         string `json:"target"`
 	Source         string `json:"source"`
-	InclusionDelay string `json:"inclusion_delay"`
+	Inactivity     string `json:"inactivity"`
 }
 
 type SyncCommitteeRewardsResponse struct {

api/server/structs/endpoints_validator.go

@@ -1,37 +1,37 @@
-package validator
+package structs
 
 import (
 	"encoding/json"
 
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 )
 
 type AggregateAttestationResponse struct {
-	Data *shared.Attestation `json:"data"`
+	Data *Attestation `json:"data"`
 }
 
 type SubmitContributionAndProofsRequest struct {
-	Data []*shared.SignedContributionAndProof `json:"data"`
+	Data []*SignedContributionAndProof `json:"data"`
 }
 
 type SubmitAggregateAndProofsRequest struct {
-	Data []*shared.SignedAggregateAttestationAndProof `json:"data"`
+	Data []*SignedAggregateAttestationAndProof `json:"data"`
 }
 
 type SubmitSyncCommitteeSubscriptionsRequest struct {
-	Data []*shared.SyncCommitteeSubscription `json:"data"`
+	Data []*SyncCommitteeSubscription `json:"data"`
 }
 
 type SubmitBeaconCommitteeSubscriptionsRequest struct {
-	Data []*shared.BeaconCommitteeSubscription `json:"data"`
+	Data []*BeaconCommitteeSubscription `json:"data"`
 }
 
 type GetAttestationDataResponse struct {
-	Data *shared.AttestationData `json:"data"`
+	Data *AttestationData `json:"data"`
 }
 
 type ProduceSyncCommitteeContributionResponse struct {
-	Data *shared.SyncCommitteeContribution `json:"data"`
+	Data *SyncCommitteeContribution `json:"data"`
 }
 
 type GetAttesterDutiesResponse struct {
@@ -90,3 +90,31 @@ type Liveness struct {
 	Index  string `json:"index"`
 	IsLive bool   `json:"is_live"`
 }
+
+type GetValidatorCountResponse struct {
+	ExecutionOptimistic string            `json:"execution_optimistic"`
+	Finalized           string            `json:"finalized"`
+	Data                []*ValidatorCount `json:"data"`
+}
+
+type ValidatorCount struct {
+	Status string `json:"status"`
+	Count  string `json:"count"`
+}
+
+type GetValidatorPerformanceRequest struct {
+	PublicKeys [][]byte                    `json:"public_keys,omitempty"`
+	Indices    []primitives.ValidatorIndex `json:"indices,omitempty"`
+}
+
+type GetValidatorPerformanceResponse struct {
+	PublicKeys                    [][]byte `json:"public_keys,omitempty"`
+	CorrectlyVotedSource          []bool   `json:"correctly_voted_source,omitempty"`
+	CorrectlyVotedTarget          []bool   `json:"correctly_voted_target,omitempty"`
+	CorrectlyVotedHead            []bool   `json:"correctly_voted_head,omitempty"`
+	CurrentEffectiveBalances      []uint64 `json:"current_effective_balances,omitempty"`
+	BalancesBeforeEpochTransition []uint64 `json:"balances_before_epoch_transition,omitempty"`
+	BalancesAfterEpochTransition  []uint64 `json:"balances_after_epoch_transition,omitempty"`
+	MissingValidators             [][]byte `json:"missing_validators,omitempty"`
+	InactivityScores              []uint64 `json:"inactivity_scores,omitempty"`
+}

api/server/structs/other.go

@@ -1,7 +1,7 @@
-package shared
+package structs
 
 type Validator struct {
-	PublicKey                  string `json:"pubkey"`
+	Pubkey                     string `json:"pubkey"`
 	WithdrawalCredentials      string `json:"withdrawal_credentials"`
 	EffectiveBalance           string `json:"effective_balance"`
 	Slashed                    bool   `json:"slashed"`

api/server/structs/state.go

@@ -1,4 +1,4 @@
-package shared
+package structs
 
 type BeaconState struct {
 	GenesisTime                 string                `json:"genesis_time"`

beacon-chain/blockchain/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
         "chain_info.go",
         "chain_info_forkchoice.go",
         "currently_syncing_block.go",
+        "defragment.go",
         "error.go",
         "execution_engine.go",
         "forkchoice_update_execution.go",
@@ -50,6 +51,7 @@ go_library(
         "//beacon-chain/core/signing:go_default_library",
         "//beacon-chain/core/time:go_default_library",
         "//beacon-chain/core/transition:go_default_library",
+        "//beacon-chain/das:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//beacon-chain/db/filesystem:go_default_library",
         "//beacon-chain/db/filters:go_default_library",
@@ -141,6 +143,7 @@ go_test(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/signing:go_default_library",
         "//beacon-chain/core/transition:go_default_library",
+        "//beacon-chain/das:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//beacon-chain/db/filesystem:go_default_library",
         "//beacon-chain/db/testing:go_default_library",

beacon-chain/blockchain/chain_info.go

@@ -6,7 +6,10 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"go.opencensus.io/trace"
+
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/helpers"
+	f "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice"
 	doublylinkedtree "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/doubly-linked-tree"
 	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
@@ -18,7 +21,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v4/time/slots"
-	"go.opencensus.io/trace"
 )
 
 // ChainInfoFetcher defines a common interface for methods in blockchain service which
@@ -334,12 +336,21 @@ func (s *Service) HeadValidatorIndexToPublicKey(_ context.Context, index primiti
 	return v.PublicKey(), nil
 }
 
+// ForkChoicer returns the forkchoice interface.
+func (s *Service) ForkChoicer() f.ForkChoicer {
+	return s.cfg.ForkChoiceStore
+}
+
 // IsOptimistic returns true if the current head is optimistic.
 func (s *Service) IsOptimistic(_ context.Context) (bool, error) {
 	if slots.ToEpoch(s.CurrentSlot()) < params.BeaconConfig().BellatrixForkEpoch {
 		return false, nil
 	}
 	s.headLock.RLock()
+	if s.head == nil {
+		s.headLock.RUnlock()
+		return false, ErrNilHead
+	}
 	headRoot := s.head.root
 	headSlot := s.head.slot
 	headOptimistic := s.head.optimistic
@@ -545,3 +556,10 @@ func (s *Service) RecentBlockSlot(root [32]byte) (primitives.Slot, error) {
 	defer s.cfg.ForkChoiceStore.RUnlock()
 	return s.cfg.ForkChoiceStore.Slot(root)
 }
+
+// inRegularSync queries the initial sync service to
+// determine if the node is in regular sync or is still
+// syncing to the head of the chain.
+func (s *Service) inRegularSync() bool {
+	return s.cfg.SyncChecker.Synced()
+}

beacon-chain/blockchain/chain_info_test.go

@@ -429,6 +429,11 @@ func TestService_IsOptimistic(t *testing.T) {
 	opt, err = c.IsOptimistic(ctx)
 	require.NoError(t, err)
 	require.Equal(t, true, opt)
+
+	// If head is nil, for some reason, an error should be returned rather than panic.
+	c = &Service{}
+	_, err = c.IsOptimistic(ctx)
+	require.ErrorIs(t, err, ErrNilHead)
 }
 
 func TestService_IsOptimisticBeforeBellatrix(t *testing.T) {

beacon-chain/blockchain/defragment.go

@@ -0,0 +1,27 @@
+package blockchain
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/v4/config/features"
+	"github.com/prysmaticlabs/prysm/v4/time"
+)
+
+var stateDefragmentationTime = promauto.NewSummary(prometheus.SummaryOpts{
+	Name: "head_state_defragmentation_milliseconds",
+	Help: "Milliseconds it takes to defragment the head state",
+})
+
+// This method defragments our state, so that any specific fields which have
+// a higher number of fragmented indexes are reallocated to a new separate slice for
+// that field.
+func (s *Service) defragmentState(st state.BeaconState) {
+	if !features.Get().EnableExperimentalState {
+		return
+	}
+	startTime := time.Now()
+	st.Defragment()
+	elapsedTime := time.Since(startTime)
+	stateDefragmentationTime.Observe(float64(elapsedTime.Milliseconds()))
+}

beacon-chain/blockchain/error.go

@@ -28,6 +28,8 @@ var (
 	// ErrNotCheckpoint is returned when a given checkpoint is not a
 	// checkpoint in any chain known to forkchoice
 	ErrNotCheckpoint = errors.New("not a checkpoint in forkchoice")
+	// ErrNilHead is returned when no head is present in the blockchain service.
+	ErrNilHead = errors.New("nil head")
 )
 
 var errMaxBlobsExceeded = errors.New("Expected commitments in block exceeds MAX_BLOBS_PER_BLOCK")

beacon-chain/blockchain/execution_engine.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/time"
@@ -31,21 +32,18 @@ const blobCommitmentVersionKZG uint8 = 0x01
 
 var defaultLatestValidHash = bytesutil.PadTo([]byte{0xff}, 32)
 
-// notifyForkchoiceUpdateArg is the argument for the forkchoice update notification `notifyForkchoiceUpdate`.
-type notifyForkchoiceUpdateArg struct {
-	headState state.BeaconState
-	headRoot  [32]byte
-	headBlock interfaces.ReadOnlyBeaconBlock
-}
-
 // notifyForkchoiceUpdate signals execution engine the fork choice updates. Execution engine should:
 // 1. Re-organizes the execution payload chain and corresponding state to make head_block_hash the head.
 // 2. Applies finality to the execution state: it irreversibly persists the chain of all execution payloads and corresponding state, up to and including finalized_block_hash.
-func (s *Service) notifyForkchoiceUpdate(ctx context.Context, arg *notifyForkchoiceUpdateArg) (*enginev1.PayloadIDBytes, error) {
+func (s *Service) notifyForkchoiceUpdate(ctx context.Context, arg *fcuConfig) (*enginev1.PayloadIDBytes, error) {
 	ctx, span := trace.StartSpan(ctx, "blockChain.notifyForkchoiceUpdate")
 	defer span.End()
 
-	headBlk := arg.headBlock
+	if arg.headBlock.IsNil() {
+		log.Error("Head block is nil")
+		return nil, nil
+	}
+	headBlk := arg.headBlock.Block()
 	if headBlk == nil || headBlk.IsNil() || headBlk.Body().IsNil() {
 		log.Error("Head block is nil")
 		return nil, nil
@@ -71,11 +69,10 @@ func (s *Service) notifyForkchoiceUpdate(ctx context.Context, arg *notifyForkcho
 		SafeBlockHash:      justifiedHash[:],
 		FinalizedBlockHash: finalizedHash[:],
 	}
-
-	nextSlot := s.CurrentSlot() + 1 // Cache payload ID for next slot proposer.
-	hasAttr, attr := s.getPayloadAttribute(ctx, arg.headState, nextSlot, arg.headRoot[:])
-
-	payloadID, lastValidHash, err := s.cfg.ExecutionEngineCaller.ForkchoiceUpdated(ctx, fcs, attr)
+	if arg.attributes == nil {
+		arg.attributes = payloadattribute.EmptyWithVersion(headBlk.Version())
+	}
+	payloadID, lastValidHash, err := s.cfg.ExecutionEngineCaller.ForkchoiceUpdated(ctx, fcs, arg.attributes)
 	if err != nil {
 		switch err {
 		case execution.ErrAcceptedSyncingPayloadStatus:
@@ -121,10 +118,11 @@ func (s *Service) notifyForkchoiceUpdate(ctx context.Context, arg *notifyForkcho
 				log.WithError(err).Error("Could not get head state")
 				return nil, nil
 			}
-			pid, err := s.notifyForkchoiceUpdate(ctx, &notifyForkchoiceUpdateArg{
-				headState: st,
-				headRoot:  r,
-				headBlock: b.Block(),
+			pid, err := s.notifyForkchoiceUpdate(ctx, &fcuConfig{
+				headState:  st,
+				headRoot:   r,
+				headBlock:  b,
+				attributes: arg.attributes,
 			})
 			if err != nil {
 				return nil, err // Returning err because it's recursive here.
@@ -153,6 +151,8 @@ func (s *Service) notifyForkchoiceUpdate(ctx context.Context, arg *notifyForkcho
 		return nil, nil
 	}
 	// If the forkchoice update call has an attribute, update the payload ID cache.
+	hasAttr := arg.attributes != nil && !arg.attributes.IsEmpty()
+	nextSlot := s.CurrentSlot() + 1
 	if hasAttr && payloadID != nil {
 		var pId [8]byte
 		copy(pId[:], payloadID[:])
@@ -276,35 +276,50 @@ func (s *Service) pruneInvalidBlock(ctx context.Context, root, parentRoot, lvh [
 
 // getPayloadAttributes returns the payload attributes for the given state and slot.
 // The attribute is required to initiate a payload build process in the context of an `engine_forkchoiceUpdated` call.
-func (s *Service) getPayloadAttribute(ctx context.Context, st state.BeaconState, slot primitives.Slot, headRoot []byte) (bool, payloadattribute.Attributer) {
+func (s *Service) getPayloadAttribute(ctx context.Context, st state.BeaconState, slot primitives.Slot, headRoot []byte) payloadattribute.Attributer {
 	emptyAttri := payloadattribute.EmptyWithVersion(st.Version())
 
-	val, ok := s.trackedProposer(st, slot)
-	if !ok && !features.Get().PrepareAllPayloads {
-		return false, emptyAttri
+	// If it is an epoch boundary then process slots to get the right
+	// shuffling before checking if the proposer is tracked. Otherwise
+	// perform this check before. This is cheap as the NSC has already been updated.
+	var val cache.TrackedValidator
+	var ok bool
+	e := slots.ToEpoch(slot)
+	stateEpoch := slots.ToEpoch(st.Slot())
+	if e == stateEpoch {
+		val, ok = s.trackedProposer(st, slot)
+		if !ok {
+			return emptyAttri
+		}
 	}
-
-	// Get previous randao.
 	st = st.Copy()
 	if slot > st.Slot() {
 		var err error
 		st, err = transition.ProcessSlotsUsingNextSlotCache(ctx, st, headRoot, slot)
 		if err != nil {
 			log.WithError(err).Error("Could not process slots to get payload attribute")
-			return false, emptyAttri
+			return emptyAttri
 		}
 	}
+	if e > stateEpoch {
+		emptyAttri := payloadattribute.EmptyWithVersion(st.Version())
+		val, ok = s.trackedProposer(st, slot)
+		if !ok {
+			return emptyAttri
+		}
+	}
+	// Get previous randao.
 	prevRando, err := helpers.RandaoMix(st, time.CurrentEpoch(st))
 	if err != nil {
 		log.WithError(err).Error("Could not get randao mix to get payload attribute")
-		return false, emptyAttri
+		return emptyAttri
 	}
 
 	// Get timestamp.
 	t, err := slots.ToTime(uint64(s.genesisTime.Unix()), slot)
 	if err != nil {
 		log.WithError(err).Error("Could not get timestamp to get payload attribute")
-		return false, emptyAttri
+		return emptyAttri
 	}
 
 	var attr payloadattribute.Attributer
@@ -313,7 +328,7 @@ func (s *Service) getPayloadAttribute(ctx context.Context, st state.BeaconState,
 		withdrawals, err := st.ExpectedWithdrawals()
 		if err != nil {
 			log.WithError(err).Error("Could not get expected withdrawals to get payload attribute")
-			return false, emptyAttri
+			return emptyAttri
 		}
 		attr, err = payloadattribute.New(&enginev1.PayloadAttributesV3{
 			Timestamp:             uint64(t.Unix()),
@@ -324,13 +339,13 @@ func (s *Service) getPayloadAttribute(ctx context.Context, st state.BeaconState,
 		})
 		if err != nil {
 			log.WithError(err).Error("Could not get payload attribute")
-			return false, emptyAttri
+			return emptyAttri
 		}
 	case version.Capella:
 		withdrawals, err := st.ExpectedWithdrawals()
 		if err != nil {
 			log.WithError(err).Error("Could not get expected withdrawals to get payload attribute")
-			return false, emptyAttri
+			return emptyAttri
 		}
 		attr, err = payloadattribute.New(&enginev1.PayloadAttributesV2{
 			Timestamp:             uint64(t.Unix()),
@@ -340,7 +355,7 @@ func (s *Service) getPayloadAttribute(ctx context.Context, st state.BeaconState,
 		})
 		if err != nil {
 			log.WithError(err).Error("Could not get payload attribute")
-			return false, emptyAttri
+			return emptyAttri
 		}
 	case version.Bellatrix:
 		attr, err = payloadattribute.New(&enginev1.PayloadAttributes{
@@ -350,14 +365,14 @@ func (s *Service) getPayloadAttribute(ctx context.Context, st state.BeaconState,
 		})
 		if err != nil {
 			log.WithError(err).Error("Could not get payload attribute")
-			return false, emptyAttri
+			return emptyAttri
 		}
 	default:
 		log.WithField("version", st.Version()).Error("Could not get payload attribute due to unknown state version")
-		return false, emptyAttri
+		return emptyAttri
 	}
 
-	return true, attr
+	return attr
 }
 
 // removeInvalidBlockAndState removes the invalid block, blob and its corresponding state from the cache and DB.
@@ -372,9 +387,9 @@ func (s *Service) removeInvalidBlockAndState(ctx context.Context, blkRoots [][32
 			// This is an irreparable condition, it would me a justified or finalized block has become invalid.
 			return err
 		}
-		// No op if the sidecar does not exist.
-		if err := s.cfg.BeaconDB.DeleteBlobSidecars(ctx, root); err != nil {
-			return err
+		if err := s.blobStorage.Remove(root); err != nil {
+			// Blobs may not exist for some blocks, leading to deletion failures. Log such errors at debug level.
+			log.WithError(err).Debug("Could not remove blob from blob storage")
 		}
 	}
 	return nil

beacon-chain/blockchain/execution_engine_test.go

@@ -56,11 +56,14 @@ func Test_NotifyForkchoiceUpdate_GetPayloadAttrErrorCanContinue(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, fcs.InsertNode(ctx, state, blkRoot))
 
-	b, err := consensusblocks.NewBeaconBlock(&ethpb.BeaconBlockBellatrix{
-		Body: &ethpb.BeaconBlockBodyBellatrix{
-			ExecutionPayload: &v1.ExecutionPayload{},
+	sb := &ethpb.SignedBeaconBlockBellatrix{
+		Block: &ethpb.BeaconBlockBellatrix{
+			Body: &ethpb.BeaconBlockBodyBellatrix{
+				ExecutionPayload: &v1.ExecutionPayload{},
+			},
 		},
-	})
+	}
+	b, err := consensusblocks.NewSignedBeaconBlock(sb)
 	require.NoError(t, err)
 
 	pid := &v1.PayloadIDBytes{1}
@@ -72,7 +75,7 @@ func Test_NotifyForkchoiceUpdate_GetPayloadAttrErrorCanContinue(t *testing.T) {
 	// Intentionally generate a bad state such that `hash_tree_root` fails during `process_slot`
 	s, err := state_native.InitializeFromProtoPhase0(&ethpb.BeaconState{})
 	require.NoError(t, err)
-	arg := &notifyForkchoiceUpdateArg{
+	arg := &fcuConfig{
 		headState: s,
 		headRoot:  [32]byte{},
 		headBlock: b,
@@ -113,7 +116,7 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 
 	tests := []struct {
 		name             string
-		blk              interfaces.ReadOnlyBeaconBlock
+		blk              interfaces.ReadOnlySignedBeaconBlock
 		headRoot         [32]byte
 		finalizedRoot    [32]byte
 		justifiedRoot    [32]byte
@@ -122,24 +125,24 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 	}{
 		{
 			name: "phase0 block",
-			blk: func() interfaces.ReadOnlyBeaconBlock {
-				b, err := consensusblocks.NewBeaconBlock(&ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}})
+			blk: func() interfaces.ReadOnlySignedBeaconBlock {
+				b, err := consensusblocks.NewSignedBeaconBlock(&ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}}})
 				require.NoError(t, err)
 				return b
 			}(),
 		},
 		{
 			name: "altair block",
-			blk: func() interfaces.ReadOnlyBeaconBlock {
-				b, err := consensusblocks.NewBeaconBlock(&ethpb.BeaconBlockAltair{Body: &ethpb.BeaconBlockBodyAltair{}})
+			blk: func() interfaces.ReadOnlySignedBeaconBlock {
+				b, err := consensusblocks.NewSignedBeaconBlock(&ethpb.SignedBeaconBlockAltair{Block: &ethpb.BeaconBlockAltair{Body: &ethpb.BeaconBlockBodyAltair{}}})
 				require.NoError(t, err)
 				return b
 			}(),
 		},
 		{
 			name: "not execution block",
-			blk: func() interfaces.ReadOnlyBeaconBlock {
-				b, err := consensusblocks.NewBeaconBlock(&ethpb.BeaconBlockBellatrix{
+			blk: func() interfaces.ReadOnlySignedBeaconBlock {
+				b, err := consensusblocks.NewSignedBeaconBlock(&ethpb.SignedBeaconBlockBellatrix{Block: &ethpb.BeaconBlockBellatrix{
 					Body: &ethpb.BeaconBlockBodyBellatrix{
 						ExecutionPayload: &v1.ExecutionPayload{
 							ParentHash:    make([]byte, fieldparams.RootLength),
@@ -148,23 +151,25 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 							ReceiptsRoot:  make([]byte, fieldparams.RootLength),
 							LogsBloom:     make([]byte, fieldparams.LogsBloomLength),
 							PrevRandao:    make([]byte, fieldparams.RootLength),
+							ExtraData:     make([]byte, 0),
 							BaseFeePerGas: make([]byte, fieldparams.RootLength),
 							BlockHash:     make([]byte, fieldparams.RootLength),
+							Transactions:  make([][]byte, 0),
 						},
 					},
-				})
+				}})
 				require.NoError(t, err)
 				return b
 			}(),
 		},
 		{
 			name: "happy case: finalized root is altair block",
-			blk: func() interfaces.ReadOnlyBeaconBlock {
-				b, err := consensusblocks.NewBeaconBlock(&ethpb.BeaconBlockBellatrix{
+			blk: func() interfaces.ReadOnlySignedBeaconBlock {
+				b, err := consensusblocks.NewSignedBeaconBlock(&ethpb.SignedBeaconBlockBellatrix{Block: &ethpb.BeaconBlockBellatrix{
 					Body: &ethpb.BeaconBlockBodyBellatrix{
 						ExecutionPayload: &v1.ExecutionPayload{},
 					},
-				})
+				}})
 				require.NoError(t, err)
 				return b
 			}(),
@@ -173,12 +178,12 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 		},
 		{
 			name: "happy case: finalized root is bellatrix block",
-			blk: func() interfaces.ReadOnlyBeaconBlock {
-				b, err := consensusblocks.NewBeaconBlock(&ethpb.BeaconBlockBellatrix{
+			blk: func() interfaces.ReadOnlySignedBeaconBlock {
+				b, err := consensusblocks.NewSignedBeaconBlock(&ethpb.SignedBeaconBlockBellatrix{Block: &ethpb.BeaconBlockBellatrix{
 					Body: &ethpb.BeaconBlockBodyBellatrix{
 						ExecutionPayload: &v1.ExecutionPayload{},
 					},
-				})
+				}})
 				require.NoError(t, err)
 				return b
 			}(),
@@ -187,12 +192,12 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 		},
 		{
 			name: "forkchoice updated with optimistic block",
-			blk: func() interfaces.ReadOnlyBeaconBlock {
-				b, err := consensusblocks.NewBeaconBlock(&ethpb.BeaconBlockBellatrix{
+			blk: func() interfaces.ReadOnlySignedBeaconBlock {
+				b, err := consensusblocks.NewSignedBeaconBlock(&ethpb.SignedBeaconBlockBellatrix{Block: &ethpb.BeaconBlockBellatrix{
 					Body: &ethpb.BeaconBlockBodyBellatrix{
 						ExecutionPayload: &v1.ExecutionPayload{},
 					},
-				})
+				}})
 				require.NoError(t, err)
 				return b
 			}(),
@@ -202,12 +207,12 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 		},
 		{
 			name: "forkchoice updated with invalid block",
-			blk: func() interfaces.ReadOnlyBeaconBlock {
-				b, err := consensusblocks.NewBeaconBlock(&ethpb.BeaconBlockBellatrix{
+			blk: func() interfaces.ReadOnlySignedBeaconBlock {
+				b, err := consensusblocks.NewSignedBeaconBlock(&ethpb.SignedBeaconBlockBellatrix{Block: &ethpb.BeaconBlockBellatrix{
 					Body: &ethpb.BeaconBlockBodyBellatrix{
 						ExecutionPayload: &v1.ExecutionPayload{},
 					},
-				})
+				}})
 				require.NoError(t, err)
 				return b
 			}(),
@@ -225,7 +230,7 @@ func Test_NotifyForkchoiceUpdate(t *testing.T) {
 			st, _ := util.DeterministicGenesisState(t, 1)
 			require.NoError(t, beaconDB.SaveState(ctx, st, tt.finalizedRoot))
 			require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, tt.finalizedRoot))
-			arg := &notifyForkchoiceUpdateArg{
+			arg := &fcuConfig{
 				headState: st,
 				headRoot:  tt.headRoot,
 				headBlock: tt.blk,
@@ -305,9 +310,9 @@ func Test_NotifyForkchoiceUpdate_NIlLVH(t *testing.T) {
 
 	require.NoError(t, beaconDB.SaveState(ctx, st, bra))
 	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, bra))
-	a := &notifyForkchoiceUpdateArg{
+	a := &fcuConfig{
 		headState: st,
-		headBlock: wbd.Block(),
+		headBlock: wbd,
 		headRoot:  brd,
 	}
 	_, err = service.notifyForkchoiceUpdate(ctx, a)
@@ -442,9 +447,9 @@ func Test_NotifyForkchoiceUpdateRecursive_DoublyLinkedTree(t *testing.T) {
 
 	require.NoError(t, beaconDB.SaveState(ctx, st, bra))
 	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, bra))
-	a := &notifyForkchoiceUpdateArg{
+	a := &fcuConfig{
 		headState: st,
-		headBlock: wbg.Block(),
+		headBlock: wbg,
 		headRoot:  brg,
 	}
 	_, err = service.notifyForkchoiceUpdate(ctx, a)
@@ -491,8 +496,10 @@ func Test_NotifyNewPayload(t *testing.T) {
 					ReceiptsRoot:  make([]byte, fieldparams.RootLength),
 					LogsBloom:     make([]byte, fieldparams.LogsBloomLength),
 					PrevRandao:    make([]byte, fieldparams.RootLength),
+					ExtraData:     make([]byte, 0),
 					BaseFeePerGas: make([]byte, fieldparams.RootLength),
 					BlockHash:     make([]byte, fieldparams.RootLength),
+					Transactions:  make([][]byte, 0),
 				},
 			},
 		},
@@ -594,8 +601,10 @@ func Test_NotifyNewPayload(t *testing.T) {
 								ReceiptsRoot:  make([]byte, fieldparams.RootLength),
 								LogsBloom:     make([]byte, fieldparams.LogsBloomLength),
 								PrevRandao:    make([]byte, fieldparams.RootLength),
+								ExtraData:     make([]byte, 0),
 								BaseFeePerGas: make([]byte, fieldparams.RootLength),
 								BlockHash:     make([]byte, fieldparams.RootLength),
+								Transactions:  make([][]byte, 0),
 							},
 						},
 					},
@@ -780,23 +789,23 @@ func Test_GetPayloadAttribute(t *testing.T) {
 	ctx := tr.ctx
 
 	st, _ := util.DeterministicGenesisStateBellatrix(t, 1)
-	hasPayload, _ := service.getPayloadAttribute(ctx, st, 0, []byte{})
-	require.Equal(t, false, hasPayload)
+	attr := service.getPayloadAttribute(ctx, st, 0, []byte{})
+	require.Equal(t, true, attr.IsEmpty())
 
 	service.cfg.TrackedValidatorsCache.Set(cache.TrackedValidator{Active: true, Index: 0})
 	// Cache hit, advance state, no fee recipient
 	slot := primitives.Slot(1)
 	service.cfg.PayloadIDCache.Set(slot, [32]byte{}, [8]byte{})
-	hasPayload, attr := service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
-	require.Equal(t, true, hasPayload)
+	attr = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
+	require.Equal(t, false, attr.IsEmpty())
 	require.Equal(t, params.BeaconConfig().EthBurnAddressHex, common.BytesToAddress(attr.SuggestedFeeRecipient()).String())
 
 	// Cache hit, advance state, has fee recipient
 	suggestedAddr := common.HexToAddress("123")
 	service.cfg.TrackedValidatorsCache.Set(cache.TrackedValidator{Active: true, FeeRecipient: primitives.ExecutionAddress(suggestedAddr), Index: 0})
 	service.cfg.PayloadIDCache.Set(slot, [32]byte{}, [8]byte{})
-	hasPayload, attr = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
-	require.Equal(t, true, hasPayload)
+	attr = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
+	require.Equal(t, false, attr.IsEmpty())
 	require.Equal(t, suggestedAddr, common.BytesToAddress(attr.SuggestedFeeRecipient()))
 }
 
@@ -810,8 +819,8 @@ func Test_GetPayloadAttribute_PrepareAllPayloads(t *testing.T) {
 	ctx := tr.ctx
 
 	st, _ := util.DeterministicGenesisStateBellatrix(t, 1)
-	hasPayload, attr := service.getPayloadAttribute(ctx, st, 0, []byte{})
-	require.Equal(t, true, hasPayload)
+	attr := service.getPayloadAttribute(ctx, st, 0, []byte{})
+	require.Equal(t, false, attr.IsEmpty())
 	require.Equal(t, params.BeaconConfig().EthBurnAddressHex, common.BytesToAddress(attr.SuggestedFeeRecipient()).String())
 }
 
@@ -820,15 +829,15 @@ func Test_GetPayloadAttributeV2(t *testing.T) {
 	ctx := tr.ctx
 
 	st, _ := util.DeterministicGenesisStateCapella(t, 1)
-	hasPayload, _ := service.getPayloadAttribute(ctx, st, 0, []byte{})
-	require.Equal(t, false, hasPayload)
+	attr := service.getPayloadAttribute(ctx, st, 0, []byte{})
+	require.Equal(t, true, attr.IsEmpty())
 
 	// Cache hit, advance state, no fee recipient
 	service.cfg.TrackedValidatorsCache.Set(cache.TrackedValidator{Active: true, Index: 0})
 	slot := primitives.Slot(1)
 	service.cfg.PayloadIDCache.Set(slot, [32]byte{}, [8]byte{})
-	hasPayload, attr := service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
-	require.Equal(t, true, hasPayload)
+	attr = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
+	require.Equal(t, false, attr.IsEmpty())
 	require.Equal(t, params.BeaconConfig().EthBurnAddressHex, common.BytesToAddress(attr.SuggestedFeeRecipient()).String())
 	a, err := attr.Withdrawals()
 	require.NoError(t, err)
@@ -838,8 +847,8 @@ func Test_GetPayloadAttributeV2(t *testing.T) {
 	suggestedAddr := common.HexToAddress("123")
 	service.cfg.TrackedValidatorsCache.Set(cache.TrackedValidator{Active: true, FeeRecipient: primitives.ExecutionAddress(suggestedAddr), Index: 0})
 	service.cfg.PayloadIDCache.Set(slot, [32]byte{}, [8]byte{})
-	hasPayload, attr = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
-	require.Equal(t, true, hasPayload)
+	attr = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
+	require.Equal(t, false, attr.IsEmpty())
 	require.Equal(t, suggestedAddr, common.BytesToAddress(attr.SuggestedFeeRecipient()))
 	a, err = attr.Withdrawals()
 	require.NoError(t, err)
@@ -851,15 +860,15 @@ func Test_GetPayloadAttributeDeneb(t *testing.T) {
 	ctx := tr.ctx
 
 	st, _ := util.DeterministicGenesisStateDeneb(t, 1)
-	hasPayload, _ := service.getPayloadAttribute(ctx, st, 0, []byte{})
-	require.Equal(t, false, hasPayload)
+	attr := service.getPayloadAttribute(ctx, st, 0, []byte{})
+	require.Equal(t, true, attr.IsEmpty())
 
 	// Cache hit, advance state, no fee recipient
 	slot := primitives.Slot(1)
 	service.cfg.TrackedValidatorsCache.Set(cache.TrackedValidator{Active: true, Index: 0})
 	service.cfg.PayloadIDCache.Set(slot, [32]byte{}, [8]byte{})
-	hasPayload, attr := service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
-	require.Equal(t, true, hasPayload)
+	attr = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
+	require.Equal(t, false, attr.IsEmpty())
 	require.Equal(t, params.BeaconConfig().EthBurnAddressHex, common.BytesToAddress(attr.SuggestedFeeRecipient()).String())
 	a, err := attr.Withdrawals()
 	require.NoError(t, err)
@@ -869,8 +878,8 @@ func Test_GetPayloadAttributeDeneb(t *testing.T) {
 	suggestedAddr := common.HexToAddress("123")
 	service.cfg.TrackedValidatorsCache.Set(cache.TrackedValidator{Active: true, FeeRecipient: primitives.ExecutionAddress(suggestedAddr), Index: 0})
 	service.cfg.PayloadIDCache.Set(slot, [32]byte{}, [8]byte{})
-	hasPayload, attr = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
-	require.Equal(t, true, hasPayload)
+	attr = service.getPayloadAttribute(ctx, st, slot, params.BeaconConfig().ZeroHash[:])
+	require.Equal(t, false, attr.IsEmpty())
 	require.Equal(t, suggestedAddr, common.BytesToAddress(attr.SuggestedFeeRecipient()))
 	a, err = attr.Withdrawals()
 	require.NoError(t, err)
@@ -1093,3 +1102,35 @@ func TestKZGCommitmentToVersionedHashes(t *testing.T) {
 	require.Equal(t, vhs[0].String(), vh0)
 	require.Equal(t, vhs[1].String(), vh1)
 }
+
+func TestComputePayloadAttribute(t *testing.T) {
+	service, tr := minimalTestService(t, WithPayloadIDCache(cache.NewPayloadIDCache()))
+	ctx := tr.ctx
+
+	st, _ := util.DeterministicGenesisStateBellatrix(t, 1)
+
+	service.cfg.TrackedValidatorsCache.Set(cache.TrackedValidator{Active: true, Index: 0})
+	// Cache hit, advance state, no fee recipient
+	slot := primitives.Slot(1)
+	service.cfg.PayloadIDCache.Set(slot, [32]byte{}, [8]byte{})
+	cfg := &postBlockProcessConfig{
+		ctx:       ctx,
+		blockRoot: [32]byte{'a'},
+	}
+	fcu := &fcuConfig{
+		headState:     st,
+		proposingSlot: slot,
+		headRoot:      [32]byte{},
+	}
+	require.NoError(t, service.computePayloadAttributes(cfg, fcu))
+	require.Equal(t, false, fcu.attributes.IsEmpty())
+	require.Equal(t, params.BeaconConfig().EthBurnAddressHex, common.BytesToAddress(fcu.attributes.SuggestedFeeRecipient()).String())
+
+	// Cache hit, advance state, has fee recipient
+	suggestedAddr := common.HexToAddress("123")
+	service.cfg.TrackedValidatorsCache.Set(cache.TrackedValidator{Active: true, FeeRecipient: primitives.ExecutionAddress(suggestedAddr), Index: 0})
+	service.cfg.PayloadIDCache.Set(slot, [32]byte{}, [8]byte{})
+	require.NoError(t, service.computePayloadAttributes(cfg, fcu))
+	require.Equal(t, false, fcu.attributes.IsEmpty())
+	require.Equal(t, suggestedAddr, common.BytesToAddress(fcu.attributes.SuggestedFeeRecipient()))
+}

beacon-chain/blockchain/forkchoice_update_execution.go

@@ -8,9 +8,9 @@ import (
 	"github.com/pkg/errors"
 	doublylinkedtree "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/doubly-linked-tree"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
-	"github.com/prysmaticlabs/prysm/v4/config/features"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
+	payloadattribute "github.com/prysmaticlabs/prysm/v4/consensus-types/payload-attribute"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/time/slots"
 	"github.com/sirupsen/logrus"
@@ -44,50 +44,69 @@ func (s *Service) getStateAndBlock(ctx context.Context, r [32]byte) (state.Beaco
 	return headState, newHeadBlock, nil
 }
 
+type fcuConfig struct {
+	headState     state.BeaconState
+	headBlock     interfaces.ReadOnlySignedBeaconBlock
+	headRoot      [32]byte
+	proposingSlot primitives.Slot
+	attributes    payloadattribute.Attributer
+}
+
+// sendFCU handles the logic to notify the engine of a forckhoice update
+// for the first time when processing an incoming block during regular sync. It
+// always updates the shuffling caches and handles epoch transitions when the
+// incoming block is late, preparing payload attributes in this case while it
+// only sends a message with empty attributes for early blocks.
+func (s *Service) sendFCU(cfg *postBlockProcessConfig, fcuArgs *fcuConfig) error {
+	if !s.isNewHead(cfg.headRoot) {
+		return nil
+	}
+	if fcuArgs.attributes != nil && !fcuArgs.attributes.IsEmpty() && s.shouldOverrideFCU(cfg.headRoot, s.CurrentSlot()+1) {
+		return nil
+	}
+	return s.forkchoiceUpdateWithExecution(cfg.ctx, fcuArgs)
+}
+
+// sendFCUWithAttributes computes the payload attributes and sends an FCU message
+// to the engine if needed
+func (s *Service) sendFCUWithAttributes(cfg *postBlockProcessConfig, fcuArgs *fcuConfig) {
+	slotCtx, cancel := context.WithTimeout(context.Background(), slotDeadline)
+	defer cancel()
+	cfg.ctx = slotCtx
+	if err := s.computePayloadAttributes(cfg, fcuArgs); err != nil {
+		log.WithError(err).Error("could not compute payload attributes")
+		return
+	}
+	if fcuArgs.attributes.IsEmpty() {
+		return
+	}
+	s.cfg.ForkChoiceStore.RLock()
+	defer s.cfg.ForkChoiceStore.RUnlock()
+	if _, err := s.notifyForkchoiceUpdate(cfg.ctx, fcuArgs); err != nil {
+		log.WithError(err).Error("could not update forkchoice with payload attributes for proposal")
+	}
+}
+
 // fockchoiceUpdateWithExecution is a wrapper around notifyForkchoiceUpdate. It decides whether a new call to FCU should be made.
-// it returns true if the new head is updated
-func (s *Service) forkchoiceUpdateWithExecution(ctx context.Context, newHeadRoot [32]byte, proposingSlot primitives.Slot) (bool, error) {
+func (s *Service) forkchoiceUpdateWithExecution(ctx context.Context, args *fcuConfig) error {
 	_, span := trace.StartSpan(ctx, "beacon-chain.blockchain.forkchoiceUpdateWithExecution")
 	defer span.End()
 	// Note: Use the service context here to avoid the parent context being ended during a forkchoice update.
 	ctx = trace.NewContext(s.ctx, span)
-
-	isNewHead := s.isNewHead(newHeadRoot)
-	if !isNewHead {
-		return false, nil
-	}
-
-	headState, headBlock, err := s.getStateAndBlock(ctx, newHeadRoot)
+	_, err := s.notifyForkchoiceUpdate(ctx, args)
 	if err != nil {
-		log.WithError(err).Error("Could not get forkchoice update argument")
-		return false, nil
+		return errors.Wrap(err, "could not notify forkchoice update")
 	}
 
-	_, tracked := s.trackedProposer(headState, proposingSlot)
-	if (tracked || features.Get().PrepareAllPayloads) && !features.Get().DisableReorgLateBlocks {
-		if s.shouldOverrideFCU(newHeadRoot, proposingSlot) {
-			return false, nil
-		}
-	}
-
-	_, err = s.notifyForkchoiceUpdate(ctx, &notifyForkchoiceUpdateArg{
-		headState: headState,
-		headRoot:  newHeadRoot,
-		headBlock: headBlock.Block(),
-	})
-	if err != nil {
-		return false, errors.Wrap(err, "could not notify forkchoice update")
-	}
-
-	if err := s.saveHead(ctx, newHeadRoot, headBlock, headState); err != nil {
+	if err := s.saveHead(ctx, args.headRoot, args.headBlock, args.headState); err != nil {
 		log.WithError(err).Error("could not save head")
 	}
 
 	// Only need to prune attestations from pool if the head has changed.
-	if err := s.pruneAttsFromPool(headBlock); err != nil {
+	if err := s.pruneAttsFromPool(args.headBlock); err != nil {
 		log.WithError(err).Error("could not prune attestations from pool")
 	}
-	return true, nil
+	return nil
 }
 
 // shouldOverrideFCU checks whether the incoming block is still subject to being

beacon-chain/blockchain/forkchoice_update_execution_test.go

@@ -58,33 +58,14 @@ func TestService_getHeadStateAndBlock(t *testing.T) {
 }
 
 func TestService_forkchoiceUpdateWithExecution_exceptionalCases(t *testing.T) {
-	hook := logTest.NewGlobal()
 	ctx := context.Background()
 	opts := testServiceOptsWithDB(t)
 
 	service, err := NewService(ctx, opts...)
 	require.NoError(t, err)
 	service.cfg.PayloadIDCache = cache.NewPayloadIDCache()
-	_, err = service.forkchoiceUpdateWithExecution(ctx, service.headRoot(), service.CurrentSlot()+1)
-	require.NoError(t, err)
-	hookErr := "could not notify forkchoice update"
-	invalidStateErr := "could not get state summary: could not find block in DB"
-	require.LogsDoNotContain(t, hook, invalidStateErr)
-	require.LogsDoNotContain(t, hook, hookErr)
-	gb, err := blocks.NewSignedBeaconBlock(util.NewBeaconBlock())
-	require.NoError(t, err)
-	require.NoError(t, service.saveInitSyncBlock(ctx, [32]byte{'a'}, gb))
-	_, err = service.forkchoiceUpdateWithExecution(ctx, [32]byte{'a'}, service.CurrentSlot()+1)
-	require.NoError(t, err)
-	require.LogsContain(t, hook, invalidStateErr)
+	service.cfg.TrackedValidatorsCache = cache.NewTrackedValidatorsCache()
 
-	hook.Reset()
-	service.head = &head{
-		root:  [32]byte{'a'},
-		block: nil, /* should not panic if notify head uses correct head */
-	}
-
-	// Block in Cache
 	b := util.NewBeaconBlock()
 	b.Block.Slot = 2
 	wsb, err := blocks.NewSignedBeaconBlock(b)
@@ -99,12 +80,6 @@ func TestService_forkchoiceUpdateWithExecution_exceptionalCases(t *testing.T) {
 		state: st,
 	}
 	service.cfg.PayloadIDCache.Set(2, [32]byte{2}, [8]byte{1})
-	_, err = service.forkchoiceUpdateWithExecution(ctx, r1, service.CurrentSlot())
-	require.NoError(t, err)
-	require.LogsDoNotContain(t, hook, invalidStateErr)
-	require.LogsDoNotContain(t, hook, hookErr)
-
-	// Block in DB
 	b = util.NewBeaconBlock()
 	b.Block.Slot = 3
 	util.SaveBlock(t, ctx, service.cfg.BeaconDB, b)
@@ -117,19 +92,17 @@ func TestService_forkchoiceUpdateWithExecution_exceptionalCases(t *testing.T) {
 		state: st,
 	}
 	service.cfg.PayloadIDCache.Set(2, [32]byte{2}, [8]byte{1})
-	_, err = service.forkchoiceUpdateWithExecution(ctx, r1, service.CurrentSlot()+1)
-	require.NoError(t, err)
-	require.LogsDoNotContain(t, hook, invalidStateErr)
-	require.LogsDoNotContain(t, hook, hookErr)
+	args := &fcuConfig{
+		headState:     st,
+		headRoot:      r1,
+		headBlock:     wsb,
+		proposingSlot: service.CurrentSlot() + 1,
+	}
+	require.NoError(t, service.forkchoiceUpdateWithExecution(ctx, args))
+
 	payloadID, has := service.cfg.PayloadIDCache.PayloadID(2, [32]byte{2})
 	require.Equal(t, true, has)
 	require.Equal(t, primitives.PayloadID{1}, payloadID)
-
-	// Test zero headRoot returns immediately.
-	headRoot := service.headRoot()
-	_, err = service.forkchoiceUpdateWithExecution(ctx, [32]byte{}, service.CurrentSlot()+1)
-	require.NoError(t, err)
-	require.Equal(t, service.headRoot(), headRoot)
 }
 
 func TestService_forkchoiceUpdateWithExecution_SameHeadRootNewProposer(t *testing.T) {
@@ -173,9 +146,13 @@ func TestService_forkchoiceUpdateWithExecution_SameHeadRootNewProposer(t *testin
 	service.head.block = sb
 	service.head.state = st
 	service.cfg.PayloadIDCache.Set(service.CurrentSlot()+1, [32]byte{} /* root */, [8]byte{})
-	_, err = service.forkchoiceUpdateWithExecution(ctx, r, service.CurrentSlot()+1)
-	require.NoError(t, err)
-
+	args := &fcuConfig{
+		headState:     st,
+		headBlock:     sb,
+		headRoot:      r,
+		proposingSlot: service.CurrentSlot() + 1,
+	}
+	require.NoError(t, service.forkchoiceUpdateWithExecution(ctx, args))
 }
 
 func TestShouldOverrideFCU(t *testing.T) {

beacon-chain/blockchain/kzg/BUILD.bazel

@@ -11,7 +11,6 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//consensus-types/blocks:go_default_library",
-        "//proto/prysm/v1alpha1:go_default_library",
         "@com_github_crate_crypto_go_kzg_4844//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
     ],
@@ -25,7 +24,7 @@ go_test(
     ],
     embed = [":go_default_library"],
     deps = [
-        "//proto/prysm/v1alpha1:go_default_library",
+        "//consensus-types/blocks:go_default_library",
         "//testing/require:go_default_library",
         "@com_github_consensys_gnark_crypto//ecc/bls12-381/fr:go_default_library",
         "@com_github_crate_crypto_go_kzg_4844//:go_default_library",

beacon-chain/blockchain/kzg/validation.go

@@ -1,42 +1,32 @@
 package kzg
 
 import (
-	"fmt"
-
 	GoKZG "github.com/crate-crypto/go-kzg-4844"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
-	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 )
 
-// IsDataAvailable checks that
-// - all blobs in the block are available
-// - Expected KZG commitments match the number of blobs in the block
-// - That the number of proofs match the number of blobs
-// - That the proofs are verified against the KZG commitments
-func IsDataAvailable(commitments [][]byte, sidecars []*ethpb.DeprecatedBlobSidecar) error {
-	if len(commitments) != len(sidecars) {
-		return fmt.Errorf("could not check data availability, expected %d commitments, obtained %d",
-			len(commitments), len(sidecars))
-	}
-	if len(commitments) == 0 {
+// Verify performs single or batch verification of commitments depending on the number of given BlobSidecars.
+func Verify(sidecars ...blocks.ROBlob) error {
+	if len(sidecars) == 0 {
 		return nil
 	}
-	blobs := make([]GoKZG.Blob, len(commitments))
-	proofs := make([]GoKZG.KZGProof, len(commitments))
-	cmts := make([]GoKZG.KZGCommitment, len(commitments))
+	if len(sidecars) == 1 {
+		return kzgContext.VerifyBlobKZGProof(
+			bytesToBlob(sidecars[0].Blob),
+			bytesToCommitment(sidecars[0].KzgCommitment),
+			bytesToKZGProof(sidecars[0].KzgProof))
+	}
+	blobs := make([]GoKZG.Blob, len(sidecars))
+	cmts := make([]GoKZG.KZGCommitment, len(sidecars))
+	proofs := make([]GoKZG.KZGProof, len(sidecars))
 	for i, sidecar := range sidecars {
 		blobs[i] = bytesToBlob(sidecar.Blob)
+		cmts[i] = bytesToCommitment(sidecar.KzgCommitment)
 		proofs[i] = bytesToKZGProof(sidecar.KzgProof)
-		cmts[i] = bytesToCommitment(commitments[i])
 	}
 	return kzgContext.VerifyBlobKZGProofBatch(blobs, cmts, proofs)
 }
 
-// VerifyROBlobCommitment is a helper that massages the fields of an ROBlob into the types needed to call VerifyBlobKZGProof.
-func VerifyROBlobCommitment(sc blocks.ROBlob) error {
-	return kzgContext.VerifyBlobKZGProof(bytesToBlob(sc.Blob), bytesToCommitment(sc.KzgCommitment), bytesToKZGProof(sc.KzgProof))
-}
-
 func bytesToBlob(blob []byte) (ret GoKZG.Blob) {
 	copy(ret[:], blob)
 	return

beacon-chain/blockchain/kzg/validation_test.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/consensys/gnark-crypto/ecc/bls12-381/fr"
 	GoKZG "github.com/crate-crypto/go-kzg-4844"
-	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	"github.com/prysmaticlabs/prysm/v4/testing/require"
 	"github.com/sirupsen/logrus"
 )
@@ -58,10 +58,9 @@ func GenerateCommitmentAndProof(blob GoKZG.Blob) (GoKZG.KZGCommitment, GoKZG.KZG
 	return commitment, proof, err
 }
 
-func TestIsDataAvailable(t *testing.T) {
-	sidecars := make([]*ethpb.DeprecatedBlobSidecar, 0)
-	commitments := make([][]byte, 0)
-	require.NoError(t, IsDataAvailable(commitments, sidecars))
+func TestVerify(t *testing.T) {
+	sidecars := make([]blocks.ROBlob, 0)
+	require.NoError(t, Verify(sidecars...))
 }
 
 func TestBytesToAny(t *testing.T) {

beacon-chain/blockchain/metrics.go

@@ -182,6 +182,10 @@ var (
 		Name: "chain_service_processing_milliseconds",
 		Help: "Total time to call a chain service in ReceiveBlock()",
 	})
+	dataAvailWaitedTime = promauto.NewSummary(prometheus.SummaryOpts{
+		Name: "da_waited_time_milliseconds",
+		Help: "Total time spent waiting for a data availability check in ReceiveBlock()",
+	})
 	processAttsElapsedTime = promauto.NewHistogram(
 		prometheus.HistogramOpts{
 			Name:    "process_attestations_milliseconds",
@@ -358,6 +362,7 @@ func reportEpochMetrics(ctx context.Context, postState, headState state.BeaconSt
 	for name, val := range refMap {
 		stateTrieReferences.WithLabelValues(name).Set(float64(val))
 	}
+	postState.RecordStateMetrics()
 
 	return nil
 }

beacon-chain/blockchain/options.go

@@ -198,3 +198,10 @@ func WithBlobStorage(b *filesystem.BlobStorage) Option {
 		return nil
 	}
 }
+
+func WithSyncChecker(checker Checker) Option {
+	return func(s *Service) error {
+		s.cfg.SyncChecker = checker
+		return nil
+	}
+}

beacon-chain/blockchain/process_block.go

@@ -6,12 +6,15 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"go.opencensus.io/trace"
+
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed"
 	statefeed "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/helpers"
 	coreTime "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/time"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/transition"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/das"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
@@ -28,8 +31,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1/attestation"
 	"github.com/prysmaticlabs/prysm/v4/runtime/version"
 	"github.com/prysmaticlabs/prysm/v4/time/slots"
-	"github.com/sirupsen/logrus"
-	"go.opencensus.io/trace"
 )
 
 // A custom slot deadline for processing state slots in our cache.
@@ -41,106 +42,71 @@ const depositDeadline = 20 * time.Second
 // This defines size of the upper bound for initial sync block cache.
 var initialSyncBlockCacheSize = uint64(2 * params.BeaconConfig().SlotsPerEpoch)
 
+// postBlockProcessConfig is a structure that contains the data needed to
+// process the beacon block after validating the state transition function
+type postBlockProcessConfig struct {
+	ctx            context.Context
+	signed         interfaces.ReadOnlySignedBeaconBlock
+	blockRoot      [32]byte
+	headRoot       [32]byte
+	postState      state.BeaconState
+	isValidPayload bool
+}
+
 // postBlockProcess is called when a gossip block is received. This function performs
 // several duties most importantly informing the engine if head was updated,
 // saving the new head information to the blockchain package and
 // handling attestations, slashings and similar included in the block.
-func (s *Service) postBlockProcess(ctx context.Context, signed interfaces.ReadOnlySignedBeaconBlock, blockRoot [32]byte, postState state.BeaconState, isValidPayload bool) error {
-	ctx, span := trace.StartSpan(ctx, "blockChain.onBlock")
+func (s *Service) postBlockProcess(cfg *postBlockProcessConfig) error {
+	ctx, span := trace.StartSpan(cfg.ctx, "blockChain.onBlock")
 	defer span.End()
-	if err := consensusblocks.BeaconBlockIsNil(signed); err != nil {
+	cfg.ctx = ctx
+	if err := consensusblocks.BeaconBlockIsNil(cfg.signed); err != nil {
 		return invalidBlock{error: err}
 	}
 	startTime := time.Now()
-	b := signed.Block()
+	fcuArgs := &fcuConfig{}
 
-	if err := s.cfg.ForkChoiceStore.InsertNode(ctx, postState, blockRoot); err != nil {
-		return errors.Wrapf(err, "could not insert block %d to fork choice store", signed.Block().Slot())
+	if s.inRegularSync() {
+		defer s.handleSecondFCUCall(cfg, fcuArgs)
 	}
-	if err := s.handleBlockAttestations(ctx, signed.Block(), postState); err != nil {
+	defer s.sendLightClientFeeds(cfg)
+	defer s.sendStateFeedOnBlock(cfg)
+	defer reportProcessingTime(startTime)
+	defer reportAttestationInclusion(cfg.signed.Block())
+
+	err := s.cfg.ForkChoiceStore.InsertNode(ctx, cfg.postState, cfg.blockRoot)
+	if err != nil {
+		return errors.Wrapf(err, "could not insert block %d to fork choice store", cfg.signed.Block().Slot())
+	}
+	if err := s.handleBlockAttestations(ctx, cfg.signed.Block(), cfg.postState); err != nil {
 		return errors.Wrap(err, "could not handle block's attestations")
 	}
 
-	s.InsertSlashingsToForkChoiceStore(ctx, signed.Block().Body().AttesterSlashings())
-	if isValidPayload {
-		if err := s.cfg.ForkChoiceStore.SetOptimisticToValid(ctx, blockRoot); err != nil {
+	s.InsertSlashingsToForkChoiceStore(ctx, cfg.signed.Block().Body().AttesterSlashings())
+	if cfg.isValidPayload {
+		if err := s.cfg.ForkChoiceStore.SetOptimisticToValid(ctx, cfg.blockRoot); err != nil {
 			return errors.Wrap(err, "could not set optimistic block to valid")
 		}
 	}
-
 	start := time.Now()
-	headRoot, err := s.cfg.ForkChoiceStore.Head(ctx)
+	cfg.headRoot, err = s.cfg.ForkChoiceStore.Head(ctx)
 	if err != nil {
 		log.WithError(err).Warn("Could not update head")
 	}
-	if blockRoot != headRoot {
-		receivedWeight, err := s.cfg.ForkChoiceStore.Weight(blockRoot)
-		if err != nil {
-			log.WithField("root", fmt.Sprintf("%#x", blockRoot)).Warn("could not determine node weight")
-		}
-		headWeight, err := s.cfg.ForkChoiceStore.Weight(headRoot)
-		if err != nil {
-			log.WithField("root", fmt.Sprintf("%#x", headRoot)).Warn("could not determine node weight")
-		}
-		log.WithFields(logrus.Fields{
-			"receivedRoot":   fmt.Sprintf("%#x", blockRoot),
-			"receivedWeight": receivedWeight,
-			"headRoot":       fmt.Sprintf("%#x", headRoot),
-			"headWeight":     headWeight,
-		}).Debug("Head block is not the received block")
-	}
 	newBlockHeadElapsedTime.Observe(float64(time.Since(start).Milliseconds()))
-
-	// verify conditions for FCU, notifies FCU, and saves the new head.
-	// This function also prunes attestations, other similar operations happen in prunePostBlockOperationPools.
-	if _, err := s.forkchoiceUpdateWithExecution(ctx, headRoot, s.CurrentSlot()+1); err != nil {
-		return err
+	if cfg.headRoot != cfg.blockRoot {
+		s.logNonCanonicalBlockReceived(cfg.blockRoot, cfg.headRoot)
+		return nil
+	}
+	if err := s.getFCUArgs(cfg, fcuArgs); err != nil {
+		log.WithError(err).Error("Could not get forkchoice update argument")
+		return nil
+	}
+	if err := s.sendFCU(cfg, fcuArgs); err != nil {
+		return errors.Wrap(err, "could not send FCU to engine")
 	}
 
-	optimistic, err := s.cfg.ForkChoiceStore.IsOptimistic(blockRoot)
-	if err != nil {
-		log.WithError(err).Debug("Could not check if block is optimistic")
-		optimistic = true
-	}
-
-	// Send notification of the processed block to the state feed.
-	s.cfg.StateNotifier.StateFeed().Send(&feed.Event{
-		Type: statefeed.BlockProcessed,
-		Data: &statefeed.BlockProcessedData{
-			Slot:        signed.Block().Slot(),
-			BlockRoot:   blockRoot,
-			SignedBlock: signed,
-			Verified:    true,
-			Optimistic:  optimistic,
-		},
-	})
-
-	defer reportAttestationInclusion(b)
-	if headRoot == blockRoot {
-		// Updating next slot state cache can happen in the background
-		// except in the epoch boundary in which case we lock to handle
-		// the shuffling and proposer caches updates.
-		// We handle these caches only on canonical
-		// blocks, otherwise this will be handled by lateBlockTasks
-		slot := postState.Slot()
-		if slots.IsEpochEnd(slot) {
-			if err := transition.UpdateNextSlotCache(ctx, blockRoot[:], postState); err != nil {
-				return errors.Wrap(err, "could not update next slot state cache")
-			}
-			if err := s.handleEpochBoundary(ctx, slot, postState, blockRoot[:]); err != nil {
-				return errors.Wrap(err, "could not handle epoch boundary")
-			}
-		} else {
-			go func() {
-				slotCtx, cancel := context.WithTimeout(context.Background(), slotDeadline)
-				defer cancel()
-				if err := transition.UpdateNextSlotCache(slotCtx, blockRoot[:], postState); err != nil {
-					log.WithError(err).Error("could not update next slot state cache")
-				}
-			}()
-		}
-	}
-	onBlockProcessingTime.Observe(float64(time.Since(startTime).Milliseconds()))
 	return nil
 }
 
@@ -162,7 +128,7 @@ func getStateVersionAndPayload(st state.BeaconState) (int, interfaces.ExecutionD
 	return preStateVersion, preStateHeader, nil
 }
 
-func (s *Service) onBlockBatch(ctx context.Context, blks []consensusblocks.ROBlock) error {
+func (s *Service) onBlockBatch(ctx context.Context, blks []consensusblocks.ROBlock, avs das.AvailabilityStore) error {
 	ctx, span := trace.StartSpan(ctx, "blockChain.onBlockBatch")
 	defer span.End()
 
@@ -265,8 +231,8 @@ func (s *Service) onBlockBatch(ctx context.Context, blks []consensusblocks.ROBlo
 				return err
 			}
 		}
-		if err := s.databaseDACheck(ctx, b); err != nil {
-			return errors.Wrap(err, "could not validate blob data availability")
+		if err := avs.IsDataAvailable(ctx, s.CurrentSlot(), b); err != nil {
+			return errors.Wrapf(err, "could not validate blob data availability at slot %d", b.Block().Slot())
 		}
 		args := &forkchoicetypes.BlockAndCheckpoints{Block: b.Block(),
 			JustifiedCheckpoint: jCheckpoints[i],
@@ -322,10 +288,10 @@ func (s *Service) onBlockBatch(ctx context.Context, blks []consensusblocks.ROBlo
 			return errors.Wrap(err, "could not set optimistic block to valid")
 		}
 	}
-	arg := &notifyForkchoiceUpdateArg{
+	arg := &fcuConfig{
 		headState: preState,
 		headRoot:  lastBR,
-		headBlock: lastB.Block(),
+		headBlock: lastB,
 	}
 	if _, err := s.notifyForkchoiceUpdate(ctx, arg); err != nil {
 		return err
@@ -333,37 +299,6 @@ func (s *Service) onBlockBatch(ctx context.Context, blks []consensusblocks.ROBlo
 	return s.saveHeadNoDB(ctx, lastB, lastBR, preState, !isValidPayload)
 }
 
-func commitmentsToCheck(b consensusblocks.ROBlock, current primitives.Slot) [][]byte {
-	if b.Version() < version.Deneb {
-		return nil
-	}
-	// We are only required to check within MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS
-	if !params.WithinDAPeriod(slots.ToEpoch(b.Block().Slot()), slots.ToEpoch(current)) {
-		return nil
-	}
-	kzgCommitments, err := b.Block().Body().BlobKzgCommitments()
-	if err != nil {
-		return nil
-	}
-	return kzgCommitments
-}
-
-func (s *Service) databaseDACheck(ctx context.Context, b consensusblocks.ROBlock) error {
-	commitments := commitmentsToCheck(b, s.CurrentSlot())
-	if len(commitments) == 0 {
-		return nil
-	}
-	missing, err := missingIndices(s.blobStorage, b.Root(), commitments)
-	if err != nil {
-		return err
-	}
-	if len(missing) == 0 {
-		return nil
-	}
-	// TODO: don't worry that this error isn't informative, it will be superceded by a detailed sidecar cache error.
-	return errors.New("not all kzg commitments are available")
-}
-
 func (s *Service) updateEpochBoundaryCaches(ctx context.Context, st state.BeaconState) error {
 	e := coreTime.CurrentEpoch(st)
 	if err := helpers.UpdateCommitteeCache(ctx, st, e); err != nil {
@@ -381,9 +316,6 @@ func (s *Service) updateEpochBoundaryCaches(ctx context.Context, st state.Beacon
 		if err := helpers.UpdateCommitteeCache(slotCtx, st, e+1); err != nil {
 			log.WithError(err).Warn("Could not update committee cache")
 		}
-		if err := helpers.UpdateUnsafeProposerIndicesInCache(slotCtx, st, e+1); err != nil {
-			log.WithError(err).Warn("Failed to cache next epoch proposers")
-		}
 	}()
 	// The latest block header is from the previous epoch
 	r, err := st.LatestBlockHeader().HashTreeRoot()
@@ -393,7 +325,10 @@ func (s *Service) updateEpochBoundaryCaches(ctx context.Context, st state.Beacon
 	}
 	// The proposer indices cache takes the target root for the previous
 	// epoch as key
-	target, err := s.cfg.ForkChoiceStore.TargetRootForEpoch(r, e-1)
+	if e > 0 {
+		e = e - 1
+	}
+	target, err := s.cfg.ForkChoiceStore.TargetRootForEpoch(r, e)
 	if err != nil {
 		log.WithError(err).Error("could not update proposer index state-root map")
 		return nil
@@ -650,10 +585,15 @@ func (s *Service) lateBlockTasks(ctx context.Context) {
 	if s.CurrentSlot() == s.HeadSlot() {
 		return
 	}
+	s.cfg.ForkChoiceStore.RLock()
+	defer s.cfg.ForkChoiceStore.RUnlock()
+	// return early if we are in init sync
+	if !s.inRegularSync() {
+		return
+	}
 	s.cfg.StateNotifier.StateFeed().Send(&feed.Event{
 		Type: statefeed.MissedSlot,
 	})
-
 	s.headLock.RLock()
 	headRoot := s.headRoot()
 	headState := s.headState(ctx)
@@ -668,23 +608,22 @@ func (s *Service) lateBlockTasks(ctx context.Context) {
 	if err := transition.UpdateNextSlotCache(ctx, lastRoot, lastState); err != nil {
 		log.WithError(err).Debug("could not update next slot state cache")
 	}
-	// handleEpochBoundary requires a forkchoice lock to obtain the target root.
-	s.cfg.ForkChoiceStore.RLock()
 	if err := s.handleEpochBoundary(ctx, currentSlot, headState, headRoot[:]); err != nil {
 		log.WithError(err).Error("lateBlockTasks: could not update epoch boundary caches")
 	}
-	s.cfg.ForkChoiceStore.RUnlock()
-	_, tracked := s.trackedProposer(headState, s.CurrentSlot()+1)
-	// return early if we are not proposing next slot.
-	if !tracked && !features.Get().PrepareAllPayloads {
-		return
-	}
 	// return early if we already started building a block for the current
 	// head root
 	_, has := s.cfg.PayloadIDCache.PayloadID(s.CurrentSlot()+1, headRoot)
 	if has {
 		return
 	}
+
+	attribute := s.getPayloadAttribute(ctx, headState, s.CurrentSlot()+1, headRoot[:])
+	// return early if we are not proposing next slot
+	if attribute.IsEmpty() {
+		return
+	}
+
 	s.headLock.RLock()
 	headBlock, err := s.headBlock()
 	if err != nil {
@@ -693,13 +632,14 @@ func (s *Service) lateBlockTasks(ctx context.Context) {
 		return
 	}
 	s.headLock.RUnlock()
-	s.cfg.ForkChoiceStore.RLock()
-	_, err = s.notifyForkchoiceUpdate(ctx, &notifyForkchoiceUpdateArg{
-		headState: headState,
-		headRoot:  headRoot,
-		headBlock: headBlock.Block(),
-	})
-	s.cfg.ForkChoiceStore.RUnlock()
+
+	fcuArgs := &fcuConfig{
+		headState:  headState,
+		headRoot:   headRoot,
+		headBlock:  headBlock,
+		attributes: attribute,
+	}
+	_, err = s.notifyForkchoiceUpdate(ctx, fcuArgs)
 	if err != nil {
 		log.WithError(err).Debug("could not perform late block tasks: failed to update forkchoice with engine")
 	}

beacon-chain/blockchain/process_block_helpers.go

@@ -3,21 +3,28 @@ package blockchain
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"go.opencensus.io/trace"
+
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed"
+	statefeed "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed/state"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/transition"
 	doublylinkedtree "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/doubly-linked-tree"
 	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/v4/config/features"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
 	mathutil "github.com/prysmaticlabs/prysm/v4/math"
+	ethpbv2 "github.com/prysmaticlabs/prysm/v4/proto/eth/v2"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
-	"github.com/prysmaticlabs/prysm/v4/time"
 	"github.com/prysmaticlabs/prysm/v4/time/slots"
-	"go.opencensus.io/trace"
 )
 
 // CurrentSlot returns the current slot based on time.
@@ -25,6 +32,252 @@ func (s *Service) CurrentSlot() primitives.Slot {
 	return slots.CurrentSlot(uint64(s.genesisTime.Unix()))
 }
 
+// getFCUArgs returns the arguments to call forkchoice update
+func (s *Service) getFCUArgs(cfg *postBlockProcessConfig, fcuArgs *fcuConfig) error {
+	if err := s.getFCUArgsEarlyBlock(cfg, fcuArgs); err != nil {
+		return err
+	}
+	if !s.inRegularSync() {
+		return nil
+	}
+	slot := cfg.signed.Block().Slot()
+	if slots.WithinVotingWindow(uint64(s.genesisTime.Unix()), slot) {
+		return nil
+	}
+	return s.computePayloadAttributes(cfg, fcuArgs)
+}
+
+func (s *Service) getFCUArgsEarlyBlock(cfg *postBlockProcessConfig, fcuArgs *fcuConfig) error {
+	if cfg.blockRoot == cfg.headRoot {
+		fcuArgs.headState = cfg.postState
+		fcuArgs.headBlock = cfg.signed
+		fcuArgs.headRoot = cfg.headRoot
+		fcuArgs.proposingSlot = s.CurrentSlot() + 1
+		return nil
+	}
+	return s.fcuArgsNonCanonicalBlock(cfg, fcuArgs)
+}
+
+// logNonCanonicalBlockReceived prints a message informing that the received
+// block is not the head of the chain. It requires the caller holds a lock on
+// Foprkchoice.
+func (s *Service) logNonCanonicalBlockReceived(blockRoot [32]byte, headRoot [32]byte) {
+	receivedWeight, err := s.cfg.ForkChoiceStore.Weight(blockRoot)
+	if err != nil {
+		log.WithField("root", fmt.Sprintf("%#x", blockRoot)).Warn("could not determine node weight")
+	}
+	headWeight, err := s.cfg.ForkChoiceStore.Weight(headRoot)
+	if err != nil {
+		log.WithField("root", fmt.Sprintf("%#x", headRoot)).Warn("could not determine node weight")
+	}
+	log.WithFields(logrus.Fields{
+		"receivedRoot":   fmt.Sprintf("%#x", blockRoot),
+		"receivedWeight": receivedWeight,
+		"headRoot":       fmt.Sprintf("%#x", headRoot),
+		"headWeight":     headWeight,
+	}).Debug("Head block is not the received block")
+}
+
+// fcuArgsNonCanonicalBlock returns the arguments to the FCU call when the
+// incoming block is non-canonical, that is, based on the head root.
+func (s *Service) fcuArgsNonCanonicalBlock(cfg *postBlockProcessConfig, fcuArgs *fcuConfig) error {
+	headState, headBlock, err := s.getStateAndBlock(cfg.ctx, cfg.headRoot)
+	if err != nil {
+		return err
+	}
+	fcuArgs.headState = headState
+	fcuArgs.headBlock = headBlock
+	fcuArgs.headRoot = cfg.headRoot
+	fcuArgs.proposingSlot = s.CurrentSlot() + 1
+	return nil
+}
+
+// sendStateFeedOnBlock sends an event that a new block has been synced
+func (s *Service) sendStateFeedOnBlock(cfg *postBlockProcessConfig) {
+	optimistic, err := s.cfg.ForkChoiceStore.IsOptimistic(cfg.blockRoot)
+	if err != nil {
+		log.WithError(err).Debug("Could not check if block is optimistic")
+		optimistic = true
+	}
+	// Send notification of the processed block to the state feed.
+	s.cfg.StateNotifier.StateFeed().Send(&feed.Event{
+		Type: statefeed.BlockProcessed,
+		Data: &statefeed.BlockProcessedData{
+			Slot:        cfg.signed.Block().Slot(),
+			BlockRoot:   cfg.blockRoot,
+			SignedBlock: cfg.signed,
+			Verified:    true,
+			Optimistic:  optimistic,
+		},
+	})
+}
+
+// sendLightClientFeeds sends the light client feeds when feature flag is enabled.
+func (s *Service) sendLightClientFeeds(cfg *postBlockProcessConfig) {
+	if features.Get().EnableLightClient {
+		if _, err := s.sendLightClientOptimisticUpdate(cfg.ctx, cfg.signed, cfg.postState); err != nil {
+			log.WithError(err).Error("Failed to send light client optimistic update")
+		}
+
+		// Get the finalized checkpoint
+		finalized := s.ForkChoicer().FinalizedCheckpoint()
+
+		// LightClientFinalityUpdate needs super majority
+		s.tryPublishLightClientFinalityUpdate(cfg.ctx, cfg.signed, finalized, cfg.postState)
+	}
+}
+
+func (s *Service) tryPublishLightClientFinalityUpdate(ctx context.Context, signed interfaces.ReadOnlySignedBeaconBlock, finalized *forkchoicetypes.Checkpoint, postState state.BeaconState) {
+	if finalized.Epoch <= s.lastPublishedLightClientEpoch {
+		return
+	}
+
+	config := params.BeaconConfig()
+	if finalized.Epoch < config.AltairForkEpoch {
+		return
+	}
+
+	syncAggregate, err := signed.Block().Body().SyncAggregate()
+	if err != nil || syncAggregate == nil {
+		return
+	}
+
+	// LightClientFinalityUpdate needs super majority
+	if syncAggregate.SyncCommitteeBits.Count()*3 < config.SyncCommitteeSize*2 {
+		return
+	}
+
+	_, err = s.sendLightClientFinalityUpdate(ctx, signed, postState)
+	if err != nil {
+		log.WithError(err).Error("Failed to send light client finality update")
+	} else {
+		s.lastPublishedLightClientEpoch = finalized.Epoch
+	}
+}
+
+// sendLightClientFinalityUpdate sends a light client finality update notification to the state feed.
+func (s *Service) sendLightClientFinalityUpdate(ctx context.Context, signed interfaces.ReadOnlySignedBeaconBlock,
+	postState state.BeaconState) (int, error) {
+	// Get attested state
+	attestedRoot := signed.Block().ParentRoot()
+	attestedState, err := s.cfg.StateGen.StateByRoot(ctx, attestedRoot)
+	if err != nil {
+		return 0, errors.Wrap(err, "could not get attested state")
+	}
+
+	// Get finalized block
+	var finalizedBlock interfaces.ReadOnlySignedBeaconBlock
+	finalizedCheckPoint := attestedState.FinalizedCheckpoint()
+	if finalizedCheckPoint != nil {
+		finalizedRoot := bytesutil.ToBytes32(finalizedCheckPoint.Root)
+		finalizedBlock, err = s.cfg.BeaconDB.Block(ctx, finalizedRoot)
+		if err != nil {
+			finalizedBlock = nil
+		}
+	}
+
+	update, err := NewLightClientFinalityUpdateFromBeaconState(
+		ctx,
+		postState,
+		signed,
+		attestedState,
+		finalizedBlock,
+	)
+
+	if err != nil {
+		return 0, errors.Wrap(err, "could not create light client update")
+	}
+
+	// Return the result
+	result := &ethpbv2.LightClientFinalityUpdateWithVersion{
+		Version: ethpbv2.Version(signed.Version()),
+		Data:    CreateLightClientFinalityUpdate(update),
+	}
+
+	// Send event
+	return s.cfg.StateNotifier.StateFeed().Send(&feed.Event{
+		Type: statefeed.LightClientFinalityUpdate,
+		Data: result,
+	}), nil
+}
+
+// sendLightClientOptimisticUpdate sends a light client optimistic update notification to the state feed.
+func (s *Service) sendLightClientOptimisticUpdate(ctx context.Context, signed interfaces.ReadOnlySignedBeaconBlock,
+	postState state.BeaconState) (int, error) {
+	// Get attested state
+	attestedRoot := signed.Block().ParentRoot()
+	attestedState, err := s.cfg.StateGen.StateByRoot(ctx, attestedRoot)
+	if err != nil {
+		return 0, errors.Wrap(err, "could not get attested state")
+	}
+
+	update, err := NewLightClientOptimisticUpdateFromBeaconState(
+		ctx,
+		postState,
+		signed,
+		attestedState,
+	)
+
+	if err != nil {
+		return 0, errors.Wrap(err, "could not create light client update")
+	}
+
+	// Return the result
+	result := &ethpbv2.LightClientOptimisticUpdateWithVersion{
+		Version: ethpbv2.Version(signed.Version()),
+		Data:    CreateLightClientOptimisticUpdate(update),
+	}
+
+	return s.cfg.StateNotifier.StateFeed().Send(&feed.Event{
+		Type: statefeed.LightClientOptimisticUpdate,
+		Data: result,
+	}), nil
+}
+
+// updateCachesPostBlockProcessing updates the next slot cache and handles the epoch
+// boundary in order to compute the right proposer indices after processing
+// state transition. This function is called on late blocks while still locked,
+// before sending FCU to the engine.
+func (s *Service) updateCachesPostBlockProcessing(cfg *postBlockProcessConfig) error {
+	slot := cfg.postState.Slot()
+	if err := transition.UpdateNextSlotCache(cfg.ctx, cfg.blockRoot[:], cfg.postState); err != nil {
+		return errors.Wrap(err, "could not update next slot state cache")
+	}
+	if !slots.IsEpochEnd(slot) {
+		return nil
+	}
+	return s.handleEpochBoundary(cfg.ctx, slot, cfg.postState, cfg.blockRoot[:])
+}
+
+// handleSecondFCUCall handles a second call to FCU when syncing a new block.
+// This is useful when proposing in the next block and we want to defer the
+// computation of the next slot shuffling.
+func (s *Service) handleSecondFCUCall(cfg *postBlockProcessConfig, fcuArgs *fcuConfig) {
+	if (fcuArgs.attributes == nil || fcuArgs.attributes.IsEmpty()) && cfg.headRoot == cfg.blockRoot {
+		go s.sendFCUWithAttributes(cfg, fcuArgs)
+	}
+}
+
+// reportProcessingTime reports the metric of how long it took to process the
+// current block
+func reportProcessingTime(startTime time.Time) {
+	onBlockProcessingTime.Observe(float64(time.Since(startTime).Milliseconds()))
+}
+
+// computePayloadAttributes modifies the passed FCU arguments to
+// contain the right payload attributes with the tracked proposer. It gets
+// called on blocks that arrive after the attestation voting window, or in a
+// background routine after syncing early blocks.
+func (s *Service) computePayloadAttributes(cfg *postBlockProcessConfig, fcuArgs *fcuConfig) error {
+	if cfg.blockRoot == cfg.headRoot {
+		if err := s.updateCachesPostBlockProcessing(cfg); err != nil {
+			return err
+		}
+	}
+	fcuArgs.attributes = s.getPayloadAttribute(cfg.ctx, fcuArgs.headState, fcuArgs.proposingSlot, cfg.headRoot[:])
+	return nil
+}
+
 // getBlockPreState returns the pre state of an incoming block. It uses the parent root of the block
 // to retrieve the state in DB. It verifies the pre state's validity and the incoming block
 // is in the correct time window.

beacon-chain/blockchain/process_block_test.go

@@ -1,7 +1,6 @@
 package blockchain
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"math/big"
@@ -17,6 +16,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/signing"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/transition"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/das"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	testDB "github.com/prysmaticlabs/prysm/v4/beacon-chain/db/testing"
@@ -40,7 +40,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/testing/require"
 	"github.com/prysmaticlabs/prysm/v4/testing/util"
 	prysmTime "github.com/prysmaticlabs/prysm/v4/time"
-	"github.com/prysmaticlabs/prysm/v4/time/slots"
 	logTest "github.com/sirupsen/logrus/hooks/test"
 )
 
@@ -69,7 +68,7 @@ func TestStore_OnBlockBatch(t *testing.T) {
 		require.NoError(t, err)
 		blks = append(blks, rwsb)
 	}
-	err := service.onBlockBatch(ctx, blks)
+	err := service.onBlockBatch(ctx, blks, &das.MockAvailabilityStore{})
 	require.NoError(t, err)
 	jcp := service.CurrentJustifiedCheckpt()
 	jroot := bytesutil.ToBytes32(jcp.Root)
@@ -99,7 +98,7 @@ func TestStore_OnBlockBatch_NotifyNewPayload(t *testing.T) {
 		require.NoError(t, service.saveInitSyncBlock(ctx, rwsb.Root(), wsb))
 		blks = append(blks, rwsb)
 	}
-	require.NoError(t, service.onBlockBatch(ctx, blks))
+	require.NoError(t, service.onBlockBatch(ctx, blks, &das.MockAvailabilityStore{}))
 }
 
 func TestCachedPreState_CanGetFromStateSummary(t *testing.T) {
@@ -567,7 +566,7 @@ func TestOnBlock_CanFinalize_WithOnTick(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, r, wsb, postState))
-		require.NoError(t, service.postBlockProcess(ctx, wsb, r, postState, true))
+		require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, r, [32]byte{}, postState, true}))
 		require.NoError(t, service.updateJustificationOnBlock(ctx, preState, postState, currStoreJustifiedEpoch))
 		_, err = service.updateFinalizationOnBlock(ctx, preState, postState, currStoreFinalizedEpoch)
 		require.NoError(t, err)
@@ -615,7 +614,7 @@ func TestOnBlock_CanFinalize(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, r, wsb, postState))
-		require.NoError(t, service.postBlockProcess(ctx, wsb, r, postState, true))
+		require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, r, [32]byte{}, postState, true}))
 		require.NoError(t, service.updateJustificationOnBlock(ctx, preState, postState, currStoreJustifiedEpoch))
 		_, err = service.updateFinalizationOnBlock(ctx, preState, postState, currStoreFinalizedEpoch)
 		require.NoError(t, err)
@@ -641,7 +640,7 @@ func TestOnBlock_CanFinalize(t *testing.T) {
 
 func TestOnBlock_NilBlock(t *testing.T) {
 	service, tr := minimalTestService(t)
-	err := service.postBlockProcess(tr.ctx, nil, [32]byte{}, nil, true)
+	err := service.postBlockProcess(&postBlockProcessConfig{tr.ctx, nil, [32]byte{}, [32]byte{}, nil, true})
 	require.Equal(t, true, IsInvalidBlock(err))
 }
 
@@ -689,7 +688,7 @@ func TestOnBlock_CallNewPayloadAndForkchoiceUpdated(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, r, wsb, postState))
-		require.NoError(t, service.postBlockProcess(ctx, wsb, r, postState, false))
+		require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, r, [32]byte{}, postState, false}))
 		testState, err = service.cfg.StateGen.StateByRoot(ctx, r)
 		require.NoError(t, err)
 	}
@@ -912,7 +911,6 @@ func Test_validateMergeTransitionBlock(t *testing.T) {
 			name:         "state older than Bellatrix, nil payload",
 			stateVersion: 1,
 			payload:      nil,
-			errString:    "attempted to wrap nil",
 		},
 		{
 			name:         "state older than Bellatrix, empty payload",
@@ -924,8 +922,10 @@ func Test_validateMergeTransitionBlock(t *testing.T) {
 				ReceiptsRoot:  make([]byte, fieldparams.RootLength),
 				LogsBloom:     make([]byte, fieldparams.LogsBloomLength),
 				PrevRandao:    make([]byte, fieldparams.RootLength),
+				ExtraData:     make([]byte, 0),
 				BaseFeePerGas: make([]byte, fieldparams.RootLength),
 				BlockHash:     make([]byte, fieldparams.RootLength),
+				Transactions:  make([][]byte, 0),
 			},
 		},
 		{
@@ -939,7 +939,6 @@ func Test_validateMergeTransitionBlock(t *testing.T) {
 			name:         "state is Bellatrix, nil payload",
 			stateVersion: 2,
 			payload:      nil,
-			errString:    "attempted to wrap nil",
 		},
 		{
 			name:         "state is Bellatrix, empty payload",
@@ -969,6 +968,7 @@ func Test_validateMergeTransitionBlock(t *testing.T) {
 					ReceiptsRoot:     make([]byte, fieldparams.RootLength),
 					LogsBloom:        make([]byte, fieldparams.LogsBloomLength),
 					PrevRandao:       make([]byte, fieldparams.RootLength),
+					ExtraData:        make([]byte, 0),
 					BaseFeePerGas:    make([]byte, fieldparams.RootLength),
 					BlockHash:        make([]byte, fieldparams.RootLength),
 					TransactionsRoot: make([]byte, fieldparams.RootLength),
@@ -1111,7 +1111,7 @@ func TestOnBlock_ProcessBlocksParallel(t *testing.T) {
 			postState, err := service.validateStateTransition(ctx, preState, wsb1)
 			require.NoError(t, err)
 			lock.Lock()
-			require.NoError(t, service.postBlockProcess(ctx, wsb1, r1, postState, true))
+			require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb1, r1, [32]byte{}, postState, true}))
 			lock.Unlock()
 			wg.Done()
 		}()
@@ -1121,7 +1121,7 @@ func TestOnBlock_ProcessBlocksParallel(t *testing.T) {
 			postState, err := service.validateStateTransition(ctx, preState, wsb2)
 			require.NoError(t, err)
 			lock.Lock()
-			require.NoError(t, service.postBlockProcess(ctx, wsb2, r2, postState, true))
+			require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb2, r2, [32]byte{}, postState, true}))
 			lock.Unlock()
 			wg.Done()
 		}()
@@ -1131,7 +1131,7 @@ func TestOnBlock_ProcessBlocksParallel(t *testing.T) {
 			postState, err := service.validateStateTransition(ctx, preState, wsb3)
 			require.NoError(t, err)
 			lock.Lock()
-			require.NoError(t, service.postBlockProcess(ctx, wsb3, r3, postState, true))
+			require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb3, r3, [32]byte{}, postState, true}))
 			lock.Unlock()
 			wg.Done()
 		}()
@@ -1141,7 +1141,7 @@ func TestOnBlock_ProcessBlocksParallel(t *testing.T) {
 			postState, err := service.validateStateTransition(ctx, preState, wsb4)
 			require.NoError(t, err)
 			lock.Lock()
-			require.NoError(t, service.postBlockProcess(ctx, wsb4, r4, postState, true))
+			require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb4, r4, [32]byte{}, postState, true}))
 			lock.Unlock()
 			wg.Done()
 		}()
@@ -1216,7 +1216,7 @@ func TestStore_NoViableHead_FCU(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		require.NoError(t, service.postBlockProcess(ctx, wsb, root, postState, false))
+		require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false}))
 	}
 
 	for i := 6; i < 12; i++ {
@@ -1234,7 +1234,7 @@ func TestStore_NoViableHead_FCU(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		err = service.postBlockProcess(ctx, wsb, root, postState, false)
+		err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false})
 		require.NoError(t, err)
 	}
 
@@ -1253,7 +1253,7 @@ func TestStore_NoViableHead_FCU(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		err = service.postBlockProcess(ctx, wsb, root, postState, false)
+		err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false})
 		require.NoError(t, err)
 	}
 	// Check that we haven't justified the second epoch yet
@@ -1275,7 +1275,7 @@ func TestStore_NoViableHead_FCU(t *testing.T) {
 	postState, err := service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, firstInvalidRoot, wsb, postState))
-	err = service.postBlockProcess(ctx, wsb, firstInvalidRoot, postState, false)
+	err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, firstInvalidRoot, [32]byte{}, postState, false})
 	require.NoError(t, err)
 	jc = service.cfg.ForkChoiceStore.JustifiedCheckpoint()
 	require.Equal(t, primitives.Epoch(2), jc.Epoch)
@@ -1303,7 +1303,7 @@ func TestStore_NoViableHead_FCU(t *testing.T) {
 	postState, err = service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-	err = service.postBlockProcess(ctx, wsb, root, postState, false)
+	err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false})
 	require.ErrorContains(t, "received an INVALID payload from execution engine", err)
 	// Check that forkchoice's head is the last invalid block imported. The
 	// store's headroot is the previous head (since the invalid block did
@@ -1332,7 +1332,7 @@ func TestStore_NoViableHead_FCU(t *testing.T) {
 	postState, err = service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-	err = service.postBlockProcess(ctx, wsb, root, postState, true)
+	err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, true})
 	require.NoError(t, err)
 	// Check the newly imported block is head, it justified the right
 	// checkpoint and the node is no longer optimistic
@@ -1394,7 +1394,7 @@ func TestStore_NoViableHead_NewPayload(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		require.NoError(t, service.postBlockProcess(ctx, wsb, root, postState, false))
+		require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false}))
 	}
 
 	for i := 6; i < 12; i++ {
@@ -1412,7 +1412,7 @@ func TestStore_NoViableHead_NewPayload(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		err = service.postBlockProcess(ctx, wsb, root, postState, false)
+		err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false})
 		require.NoError(t, err)
 	}
 
@@ -1432,7 +1432,7 @@ func TestStore_NoViableHead_NewPayload(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		err = service.postBlockProcess(ctx, wsb, root, postState, false)
+		err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false})
 		require.NoError(t, err)
 	}
 	// Check that we haven't justified the second epoch yet
@@ -1454,7 +1454,7 @@ func TestStore_NoViableHead_NewPayload(t *testing.T) {
 	postState, err := service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, firstInvalidRoot, wsb, postState))
-	err = service.postBlockProcess(ctx, wsb, firstInvalidRoot, postState, false)
+	err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, firstInvalidRoot, [32]byte{}, postState, false})
 	require.NoError(t, err)
 	jc = service.cfg.ForkChoiceStore.JustifiedCheckpoint()
 	require.Equal(t, primitives.Epoch(2), jc.Epoch)
@@ -1510,7 +1510,7 @@ func TestStore_NoViableHead_NewPayload(t *testing.T) {
 	postState, err = service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-	err = service.postBlockProcess(ctx, wsb, root, postState, true)
+	err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, true})
 	require.NoError(t, err)
 	// Check the newly imported block is head, it justified the right
 	// checkpoint and the node is no longer optimistic
@@ -1574,7 +1574,7 @@ func TestStore_NoViableHead_Liveness(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		require.NoError(t, service.postBlockProcess(ctx, wsb, root, postState, false))
+		require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false}))
 	}
 
 	for i := 6; i < 12; i++ {
@@ -1593,7 +1593,7 @@ func TestStore_NoViableHead_Liveness(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		err = service.postBlockProcess(ctx, wsb, root, postState, false)
+		err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false})
 		require.NoError(t, err)
 	}
 
@@ -1612,7 +1612,7 @@ func TestStore_NoViableHead_Liveness(t *testing.T) {
 	postState, err := service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, lastValidRoot, wsb, postState))
-	err = service.postBlockProcess(ctx, wsb, lastValidRoot, postState, false)
+	err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, lastValidRoot, [32]byte{}, postState, false})
 	require.NoError(t, err)
 	// save the post state and the payload Hash of this block since it will
 	// be the LVH
@@ -1639,7 +1639,7 @@ func TestStore_NoViableHead_Liveness(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, invalidRoots[i-13], wsb, postState))
-		err = service.postBlockProcess(ctx, wsb, invalidRoots[i-13], postState, false)
+		err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, invalidRoots[i-13], [32]byte{}, postState, false})
 		require.NoError(t, err)
 	}
 	// Check that we have justified the second epoch
@@ -1704,7 +1704,7 @@ func TestStore_NoViableHead_Liveness(t *testing.T) {
 	postState, err = service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-	require.NoError(t, service.postBlockProcess(ctx, wsb, root, postState, true))
+	require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, true}))
 	// Check that the head is still INVALID and the node is still optimistic
 	require.Equal(t, invalidHeadRoot, service.cfg.ForkChoiceStore.CachedHeadRoot())
 	optimistic, err = service.IsOptimistic(ctx)
@@ -1727,7 +1727,7 @@ func TestStore_NoViableHead_Liveness(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		err = service.postBlockProcess(ctx, wsb, root, postState, true)
+		err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, true})
 		require.NoError(t, err)
 		st, err = service.cfg.StateGen.StateByRoot(ctx, root)
 		require.NoError(t, err)
@@ -1753,7 +1753,7 @@ func TestStore_NoViableHead_Liveness(t *testing.T) {
 	postState, err = service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-	err = service.postBlockProcess(ctx, wsb, root, postState, true)
+	err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, true})
 	require.NoError(t, err)
 	require.Equal(t, root, service.cfg.ForkChoiceStore.CachedHeadRoot())
 	sjc = service.CurrentJustifiedCheckpt()
@@ -1809,7 +1809,7 @@ func TestNoViableHead_Reboot(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		require.NoError(t, service.postBlockProcess(ctx, wsb, root, postState, false))
+		require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false}))
 	}
 
 	for i := 6; i < 12; i++ {
@@ -1827,7 +1827,7 @@ func TestNoViableHead_Reboot(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		err = service.postBlockProcess(ctx, wsb, root, postState, false)
+		err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false})
 		require.NoError(t, err)
 	}
 
@@ -1846,7 +1846,7 @@ func TestNoViableHead_Reboot(t *testing.T) {
 	postState, err := service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, lastValidRoot, wsb, postState))
-	err = service.postBlockProcess(ctx, wsb, lastValidRoot, postState, false)
+	err = service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, lastValidRoot, [32]byte{}, postState, false})
 	require.NoError(t, err)
 	// save the post state and the payload Hash of this block since it will
 	// be the LVH
@@ -1875,7 +1875,7 @@ func TestNoViableHead_Reboot(t *testing.T) {
 		postState, err := service.validateStateTransition(ctx, preState, wsb)
 		require.NoError(t, err)
 		require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-		require.NoError(t, service.postBlockProcess(ctx, wsb, root, postState, false))
+		require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false}))
 		require.NoError(t, service.updateJustificationOnBlock(ctx, preState, postState, currStoreJustifiedEpoch))
 		_, err = service.updateFinalizationOnBlock(ctx, preState, postState, currStoreFinalizedEpoch)
 		require.NoError(t, err)
@@ -1946,7 +1946,7 @@ func TestNoViableHead_Reboot(t *testing.T) {
 	rwsb, err := consensusblocks.NewROBlock(wsb)
 	require.NoError(t, err)
 	// We use onBlockBatch here because the valid chain is missing in forkchoice
-	require.NoError(t, service.onBlockBatch(ctx, []consensusblocks.ROBlock{rwsb}))
+	require.NoError(t, service.onBlockBatch(ctx, []consensusblocks.ROBlock{rwsb}, &das.MockAvailabilityStore{}))
 	// Check that the head is now VALID and the node is not optimistic
 	require.Equal(t, genesisRoot, service.ensureRootNotZeros(service.cfg.ForkChoiceStore.CachedHeadRoot()))
 	headRoot, err = service.HeadRoot(ctx)
@@ -1990,7 +1990,7 @@ func TestOnBlock_HandleBlockAttestations(t *testing.T) {
 	postState, err := service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, root, wsb, postState))
-	require.NoError(t, service.postBlockProcess(ctx, wsb, root, postState, false))
+	require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, root, [32]byte{}, postState, false}))
 
 	st, err = service.HeadState(ctx)
 	require.NoError(t, err)
@@ -2045,76 +2045,10 @@ func TestFillMissingBlockPayloadId_PrepareAllPayloads(t *testing.T) {
 // Helper function to simulate the block being on time or delayed for proposer
 // boost. It alters the genesisTime tracked by the store.
 func driftGenesisTime(s *Service, slot, delay int64) {
-	offset := slot*int64(params.BeaconConfig().SecondsPerSlot) - delay
-	s.SetGenesisTime(time.Unix(time.Now().Unix()-offset, 0))
-}
-
-func Test_commitmentsToCheck(t *testing.T) {
-	windowSlots, err := slots.EpochEnd(params.BeaconConfig().MinEpochsForBlobsSidecarsRequest)
-	require.NoError(t, err)
-	commits := [][]byte{
-		bytesutil.PadTo([]byte("a"), 48),
-		bytesutil.PadTo([]byte("b"), 48),
-		bytesutil.PadTo([]byte("c"), 48),
-		bytesutil.PadTo([]byte("d"), 48),
-	}
-	cases := []struct {
-		name    string
-		commits [][]byte
-		block   func(*testing.T) consensusblocks.ROBlock
-		slot    primitives.Slot
-	}{
-		{
-			name: "pre deneb",
-			block: func(t *testing.T) consensusblocks.ROBlock {
-				bb := util.NewBeaconBlockBellatrix()
-				sb, err := consensusblocks.NewSignedBeaconBlock(bb)
-				require.NoError(t, err)
-				rb, err := consensusblocks.NewROBlock(sb)
-				require.NoError(t, err)
-				return rb
-			},
-		},
-		{
-			name: "commitments within da",
-			block: func(t *testing.T) consensusblocks.ROBlock {
-				d := util.NewBeaconBlockDeneb()
-				d.Block.Body.BlobKzgCommitments = commits
-				d.Block.Slot = 100
-				sb, err := consensusblocks.NewSignedBeaconBlock(d)
-				require.NoError(t, err)
-				rb, err := consensusblocks.NewROBlock(sb)
-				require.NoError(t, err)
-				return rb
-			},
-			commits: commits,
-			slot:    100,
-		},
-		{
-			name: "commitments outside da",
-			block: func(t *testing.T) consensusblocks.ROBlock {
-				d := util.NewBeaconBlockDeneb()
-				// block is from slot 0, "current slot" is window size +1 (so outside the window)
-				d.Block.Body.BlobKzgCommitments = commits
-				sb, err := consensusblocks.NewSignedBeaconBlock(d)
-				require.NoError(t, err)
-				rb, err := consensusblocks.NewROBlock(sb)
-				require.NoError(t, err)
-				return rb
-			},
-			slot: windowSlots + 1,
-		},
-	}
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			b := c.block(t)
-			co := commitmentsToCheck(b, c.slot)
-			require.Equal(t, len(c.commits), len(co))
-			for i := 0; i < len(c.commits); i++ {
-				require.Equal(t, true, bytes.Equal(c.commits[i], co[i]))
-			}
-		})
-	}
+	offset := slot*int64(params.BeaconConfig().SecondsPerSlot) + delay
+	newTime := time.Unix(time.Now().Unix()-offset, 0)
+	s.SetGenesisTime(newTime)
+	s.cfg.ForkChoiceStore.SetGenesisTime(uint64(newTime.Unix()))
 }
 
 func TestMissingIndices(t *testing.T) {
@@ -2197,6 +2131,35 @@ func TestMissingIndices(t *testing.T) {
 	}
 }
 
+func Test_getFCUArgs(t *testing.T) {
+	s, tr := minimalTestService(t)
+	ctx := tr.ctx
+	st, keys := util.DeterministicGenesisState(t, 64)
+	b, err := util.GenerateFullBlock(st, keys, util.DefaultBlockGenConfig(), 1)
+	require.NoError(t, err)
+	wsb, err := consensusblocks.NewSignedBeaconBlock(b)
+	require.NoError(t, err)
+
+	cfg := &postBlockProcessConfig{
+		ctx:            ctx,
+		signed:         wsb,
+		blockRoot:      [32]byte{'a'},
+		postState:      st,
+		isValidPayload: true,
+	}
+	// error branch
+	fcuArgs := &fcuConfig{}
+	err = s.getFCUArgs(cfg, fcuArgs)
+	require.ErrorContains(t, "block does not exist", err)
+
+	// canonical branch
+	cfg.headRoot = cfg.blockRoot
+	fcuArgs = &fcuConfig{}
+	err = s.getFCUArgs(cfg, fcuArgs)
+	require.NoError(t, err)
+	require.Equal(t, cfg.blockRoot, fcuArgs.headRoot)
+}
+
 func fakeCommitments(n int) [][]byte {
 	f := make([][]byte, n)
 	for i := range f {

beacon-chain/blockchain/receive_attestation.go

@@ -9,7 +9,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
-	"github.com/prysmaticlabs/prysm/v4/config/features"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
@@ -122,35 +121,43 @@ func (s *Service) UpdateHead(ctx context.Context, proposingSlot primitives.Slot)
 	defer s.cfg.ForkChoiceStore.Unlock()
 	// This function is only called at 10 seconds or 0 seconds into the slot
 	disparity := params.BeaconConfig().MaximumGossipClockDisparityDuration()
-	if !features.Get().DisableReorgLateBlocks {
-		disparity += reorgLateBlockCountAttestations
-	}
+	disparity += reorgLateBlockCountAttestations
+
 	s.processAttestations(ctx, disparity)
 
 	processAttsElapsedTime.Observe(float64(time.Since(start).Milliseconds()))
 
 	start = time.Now()
+	// return early if we haven't changed head
 	newHeadRoot, err := s.cfg.ForkChoiceStore.Head(ctx)
 	if err != nil {
 		log.WithError(err).Error("Could not compute head from new attestations")
-		// Fallback to our current head root in the event of a failure.
-		s.headLock.RLock()
-		newHeadRoot = s.headRoot()
-		s.headLock.RUnlock()
+		return
+	}
+	if !s.isNewHead(newHeadRoot) {
+		return
+	}
+	log.WithField("newHeadRoot", fmt.Sprintf("%#x", newHeadRoot)).Debug("Head changed due to attestations")
+	headState, headBlock, err := s.getStateAndBlock(ctx, newHeadRoot)
+	if err != nil {
+		log.WithError(err).Error("could not get head block")
+		return
 	}
 	newAttHeadElapsedTime.Observe(float64(time.Since(start).Milliseconds()))
-
-	changed, err := s.forkchoiceUpdateWithExecution(s.ctx, newHeadRoot, proposingSlot)
-	if err != nil {
-		log.WithError(err).Error("could not update forkchoice")
+	fcuArgs := &fcuConfig{
+		headState:     headState,
+		headRoot:      newHeadRoot,
+		headBlock:     headBlock,
+		proposingSlot: proposingSlot,
 	}
-	if changed {
-		s.headLock.RLock()
-		log.WithFields(logrus.Fields{
-			"oldHeadRoot": fmt.Sprintf("%#x", s.headRoot()),
-			"newHeadRoot": fmt.Sprintf("%#x", newHeadRoot),
-		}).Debug("Head changed due to attestations")
-		s.headLock.RUnlock()
+	if s.inRegularSync() {
+		fcuArgs.attributes = s.getPayloadAttribute(ctx, headState, proposingSlot, newHeadRoot[:])
+	}
+	if fcuArgs.attributes != nil && s.shouldOverrideFCU(newHeadRoot, proposingSlot) {
+		return
+	}
+	if err := s.forkchoiceUpdateWithExecution(s.ctx, fcuArgs); err != nil {
+		log.WithError(err).Error("could not update forkchoice")
 	}
 }
 

beacon-chain/blockchain/receive_attestation_test.go

@@ -112,7 +112,7 @@ func TestService_ProcessAttestationsAndUpdateHead(t *testing.T) {
 	postState, err := service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, tRoot, wsb, postState))
-	require.NoError(t, service.postBlockProcess(ctx, wsb, tRoot, postState, false))
+	require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, tRoot, [32]byte{}, postState, false}))
 	copied, err = service.cfg.StateGen.StateByRoot(ctx, tRoot)
 	require.NoError(t, err)
 	require.Equal(t, 2, fcs.NodeCount())
@@ -168,7 +168,7 @@ func TestService_UpdateHead_NoAtts(t *testing.T) {
 	postState, err := service.validateStateTransition(ctx, preState, wsb)
 	require.NoError(t, err)
 	require.NoError(t, service.savePostStateInfo(ctx, tRoot, wsb, postState))
-	require.NoError(t, service.postBlockProcess(ctx, wsb, tRoot, postState, false))
+	require.NoError(t, service.postBlockProcess(&postBlockProcessConfig{ctx, wsb, tRoot, [32]byte{}, postState, false}))
 	require.Equal(t, 2, fcs.NodeCount())
 	require.NoError(t, service.cfg.BeaconDB.SaveBlock(ctx, wsb))
 	require.Equal(t, tRoot, service.head.root)

beacon-chain/blockchain/receive_block.go

@@ -12,6 +12,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/helpers"
 	coreTime "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/time"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/transition"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/das"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/v4/config/features"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
@@ -33,8 +34,8 @@ var epochsSinceFinalitySaveHotStateDB = primitives.Epoch(100)
 
 // BlockReceiver interface defines the methods of chain service for receiving and processing new blocks.
 type BlockReceiver interface {
-	ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock, blockRoot [32]byte) error
-	ReceiveBlockBatch(ctx context.Context, blocks []blocks.ROBlock) error
+	ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock, blockRoot [32]byte, avs das.AvailabilityStore) error
+	ReceiveBlockBatch(ctx context.Context, blocks []blocks.ROBlock, avs das.AvailabilityStore) error
 	HasBlock(ctx context.Context, root [32]byte) bool
 	RecentBlockSlot(root [32]byte) (primitives.Slot, error)
 	BlockBeingSynced([32]byte) bool
@@ -56,7 +57,7 @@ type SlashingReceiver interface {
 //  1. Validate block, apply state transition and update checkpoints
 //  2. Apply fork choice to the processed block
 //  3. Save latest head info
-func (s *Service) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock, blockRoot [32]byte) error {
+func (s *Service) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock, blockRoot [32]byte, avs das.AvailabilityStore) error {
 	ctx, span := trace.StartSpan(ctx, "blockChain.ReceiveBlock")
 	defer span.End()
 	// Return early if the block has been synced
@@ -72,6 +73,10 @@ func (s *Service) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySig
 	if err != nil {
 		return err
 	}
+	rob, err := blocks.NewROBlockWithRoot(block, blockRoot)
+	if err != nil {
+		return err
+	}
 
 	preState, err := s.getBlockPreState(ctx, blockCopy.Block())
 	if err != nil {
@@ -106,12 +111,21 @@ func (s *Service) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySig
 	if err := eg.Wait(); err != nil {
 		return err
 	}
-
 	daStartTime := time.Now()
-	if err := s.isDataAvailable(ctx, blockRoot, blockCopy); err != nil {
-		return errors.Wrap(err, "could not validate blob data availability")
+	if avs != nil {
+		if err := avs.IsDataAvailable(ctx, s.CurrentSlot(), rob); err != nil {
+			return errors.Wrap(err, "could not validate blob data availability (AvailabilityStore.IsDataAvailable)")
+		}
+	} else {
+		if err := s.isDataAvailable(ctx, blockRoot, blockCopy); err != nil {
+			return errors.Wrap(err, "could not validate blob data availability")
+		}
 	}
 	daWaitedTime := time.Since(daStartTime)
+	dataAvailWaitedTime.Observe(float64(daWaitedTime.Milliseconds()))
+
+	// Defragment the state before continuing block processing.
+	s.defragmentState(postState)
 
 	// The rest of block processing takes a lock on forkchoice.
 	s.cfg.ForkChoiceStore.Lock()
@@ -119,7 +133,14 @@ func (s *Service) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySig
 	if err := s.savePostStateInfo(ctx, blockRoot, blockCopy, postState); err != nil {
 		return errors.Wrap(err, "could not save post state info")
 	}
-	if err := s.postBlockProcess(ctx, blockCopy, blockRoot, postState, isValidPayload); err != nil {
+	args := &postBlockProcessConfig{
+		ctx:            ctx,
+		signed:         blockCopy,
+		blockRoot:      blockRoot,
+		postState:      postState,
+		isValidPayload: isValidPayload,
+	}
+	if err := s.postBlockProcess(args); err != nil {
 		err := errors.Wrap(err, "could not process block")
 		tracing.AnnotateError(span, err)
 		return err
@@ -196,7 +217,7 @@ func (s *Service) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySig
 // ReceiveBlockBatch processes the whole block batch at once, assuming the block batch is linear ,transitioning
 // the state, performing batch verification of all collected signatures and then performing the appropriate
 // actions for a block post-transition.
-func (s *Service) ReceiveBlockBatch(ctx context.Context, blocks []blocks.ROBlock) error {
+func (s *Service) ReceiveBlockBatch(ctx context.Context, blocks []blocks.ROBlock, avs das.AvailabilityStore) error {
 	ctx, span := trace.StartSpan(ctx, "blockChain.ReceiveBlockBatch")
 	defer span.End()
 
@@ -204,7 +225,7 @@ func (s *Service) ReceiveBlockBatch(ctx context.Context, blocks []blocks.ROBlock
 	defer s.cfg.ForkChoiceStore.Unlock()
 
 	// Apply state transition on the incoming newly received block batches, one by one.
-	if err := s.onBlockBatch(ctx, blocks); err != nil {
+	if err := s.onBlockBatch(ctx, blocks, avs); err != nil {
 		err := errors.Wrap(err, "could not process block in batch")
 		tracing.AnnotateError(span, err)
 		return err

beacon-chain/blockchain/receive_block_test.go

@@ -8,6 +8,7 @@ import (
 
 	blockchainTesting "github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain/testing"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/das"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/operations/voluntaryexits"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
@@ -146,7 +147,7 @@ func TestService_ReceiveBlock(t *testing.T) {
 			require.NoError(t, err)
 			wsb, err := blocks.NewSignedBeaconBlock(tt.args.block)
 			require.NoError(t, err)
-			err = s.ReceiveBlock(ctx, wsb, root)
+			err = s.ReceiveBlock(ctx, wsb, root, nil)
 			if tt.wantedErr != "" {
 				assert.ErrorContains(t, tt.wantedErr, err)
 			} else {
@@ -179,7 +180,7 @@ func TestService_ReceiveBlockUpdateHead(t *testing.T) {
 	go func() {
 		wsb, err := blocks.NewSignedBeaconBlock(b)
 		require.NoError(t, err)
-		require.NoError(t, s.ReceiveBlock(ctx, wsb, root))
+		require.NoError(t, s.ReceiveBlock(ctx, wsb, root, nil))
 		wg.Done()
 	}()
 	wg.Wait()
@@ -243,7 +244,7 @@ func TestService_ReceiveBlockBatch(t *testing.T) {
 			require.NoError(t, err)
 			rwsb, err := blocks.NewROBlock(wsb)
 			require.NoError(t, err)
-			err = s.ReceiveBlockBatch(ctx, []blocks.ROBlock{rwsb})
+			err = s.ReceiveBlockBatch(ctx, []blocks.ROBlock{rwsb}, &das.MockAvailabilityStore{})
 			if tt.wantedErr != "" {
 				assert.ErrorContains(t, tt.wantedErr, err)
 			} else {

beacon-chain/blockchain/service.go

@@ -11,6 +11,8 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"go.opencensus.io/trace"
+
 	"github.com/prysmaticlabs/prysm/v4/async/event"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain/kzg"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
@@ -37,34 +39,35 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	prysmTime "github.com/prysmaticlabs/prysm/v4/time"
 	"github.com/prysmaticlabs/prysm/v4/time/slots"
-	"go.opencensus.io/trace"
 )
 
 // Service represents a service that handles the internal
 // logic of managing the full PoS beacon chain.
 type Service struct {
-	cfg                  *config
-	ctx                  context.Context
-	cancel               context.CancelFunc
-	genesisTime          time.Time
-	head                 *head
-	headLock             sync.RWMutex
-	originBlockRoot      [32]byte // genesis root, or weak subjectivity checkpoint root, depending on how the node is initialized
-	boundaryRoots        [][32]byte
-	checkpointStateCache *cache.CheckpointStateCache
-	initSyncBlocks       map[[32]byte]interfaces.ReadOnlySignedBeaconBlock
-	initSyncBlocksLock   sync.RWMutex
-	wsVerifier           *WeakSubjectivityVerifier
-	clockSetter          startup.ClockSetter
-	clockWaiter          startup.ClockWaiter
-	syncComplete         chan struct{}
-	blobNotifiers        *blobNotifierMap
-	blockBeingSynced     *currentlySyncingBlock
-	blobStorage          *filesystem.BlobStorage
+	cfg                           *config
+	ctx                           context.Context
+	cancel                        context.CancelFunc
+	genesisTime                   time.Time
+	head                          *head
+	headLock                      sync.RWMutex
+	originBlockRoot               [32]byte // genesis root, or weak subjectivity checkpoint root, depending on how the node is initialized
+	boundaryRoots                 [][32]byte
+	checkpointStateCache          *cache.CheckpointStateCache
+	initSyncBlocks                map[[32]byte]interfaces.ReadOnlySignedBeaconBlock
+	initSyncBlocksLock            sync.RWMutex
+	wsVerifier                    *WeakSubjectivityVerifier
+	clockSetter                   startup.ClockSetter
+	clockWaiter                   startup.ClockWaiter
+	syncComplete                  chan struct{}
+	blobNotifiers                 *blobNotifierMap
+	blockBeingSynced              *currentlySyncingBlock
+	blobStorage                   *filesystem.BlobStorage
+	lastPublishedLightClientEpoch primitives.Epoch
 }
 
 // config options for the service.
@@ -90,6 +93,13 @@ type config struct {
 	BlockFetcher            execution.POWBlockFetcher
 	FinalizedStateAtStartUp state.BeaconState
 	ExecutionEngineCaller   execution.EngineCaller
+	SyncChecker             Checker
+}
+
+// Checker is an interface used to determine if a node is in initial sync
+// or regular sync.
+type Checker interface {
+	Synced() bool
 }
 
 var ErrMissingClockSetter = errors.New("blockchain Service initialized without a startup.ClockSetter")

beacon-chain/blockchain/setup_test.go

@@ -6,10 +6,12 @@ import (
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/v4/async/event"
+	mock "github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain/testing"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache/depositcache"
 	statefeed "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	testDB "github.com/prysmaticlabs/prysm/v4/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice"
 	doublylinkedtree "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/doubly-linked-tree"
@@ -116,6 +118,8 @@ func minimalTestService(t *testing.T, opts ...Option) (*Service, *testServiceReq
 		WithBLSToExecPool(req.blsPool),
 		WithDepositCache(dc),
 		WithTrackedValidatorsCache(cache.NewTrackedValidatorsCache()),
+		WithBlobStorage(filesystem.NewEphemeralBlobStorage(t)),
+		WithSyncChecker(mock.MockChecker{}),
 	}
 	// append the variadic opts so they override the defaults by being processed afterwards
 	opts = append(defOpts, opts...)

beacon-chain/blockchain/testing/BUILD.bazel

@@ -17,6 +17,7 @@ go_library(
         "//beacon-chain/core/feed/operation:go_default_library",
         "//beacon-chain/core/feed/state:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
+        "//beacon-chain/das:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//beacon-chain/forkchoice:go_default_library",
         "//beacon-chain/state:go_default_library",

beacon-chain/blockchain/testing/mock.go

@@ -16,6 +16,7 @@ import (
 	opfeed "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed/operation"
 	statefeed "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/das"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
@@ -179,6 +180,14 @@ func (mon *MockOperationNotifier) OperationFeed() *event.Feed {
 	return mon.feed
 }
 
+// MockChecker is a mock sync checker.
+type MockChecker struct{}
+
+// Synced returns true.
+func (_ MockChecker) Synced() bool {
+	return true
+}
+
 // ReceiveBlockInitialSync mocks ReceiveBlockInitialSync method in chain service.
 func (s *ChainService) ReceiveBlockInitialSync(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock, _ [32]byte) error {
 	if s.State == nil {
@@ -208,7 +217,7 @@ func (s *ChainService) ReceiveBlockInitialSync(ctx context.Context, block interf
 }
 
 // ReceiveBlockBatch processes blocks in batches from initial-sync.
-func (s *ChainService) ReceiveBlockBatch(ctx context.Context, blks []blocks.ROBlock) error {
+func (s *ChainService) ReceiveBlockBatch(ctx context.Context, blks []blocks.ROBlock, _ das.AvailabilityStore) error {
 	if s.State == nil {
 		return ErrNilState
 	}
@@ -238,7 +247,7 @@ func (s *ChainService) ReceiveBlockBatch(ctx context.Context, blks []blocks.ROBl
 }
 
 // ReceiveBlock mocks ReceiveBlock method in chain service.
-func (s *ChainService) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock, _ [32]byte) error {
+func (s *ChainService) ReceiveBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock, _ [32]byte, _ das.AvailabilityStore) error {
 	if s.ReceiveBlockMockErr != nil {
 		return s.ReceiveBlockMockErr
 	}
@@ -320,7 +329,7 @@ func (s *ChainService) PreviousJustifiedCheckpt() *ethpb.Checkpoint {
 }
 
 // ReceiveAttestation mocks ReceiveAttestation method in chain service.
-func (_ *ChainService) ReceiveAttestation(_ context.Context, _ *ethpb.Attestation) error {
+func (*ChainService) ReceiveAttestation(_ context.Context, _ *ethpb.Attestation) error {
 	return nil
 }
 
@@ -400,12 +409,12 @@ func (s *ChainService) RecentBlockSlot([32]byte) (primitives.Slot, error) {
 }
 
 // HeadGenesisValidatorsRoot mocks HeadGenesisValidatorsRoot method in chain service.
-func (_ *ChainService) HeadGenesisValidatorsRoot() [32]byte {
+func (*ChainService) HeadGenesisValidatorsRoot() [32]byte {
 	return [32]byte{}
 }
 
 // VerifyLmdFfgConsistency mocks VerifyLmdFfgConsistency and always returns nil.
-func (_ *ChainService) VerifyLmdFfgConsistency(_ context.Context, a *ethpb.Attestation) error {
+func (*ChainService) VerifyLmdFfgConsistency(_ context.Context, a *ethpb.Attestation) error {
 	if !bytes.Equal(a.Data.BeaconBlockRoot, a.Data.Target.Root) {
 		return errors.New("LMD and FFG miss matched")
 	}
@@ -413,7 +422,7 @@ func (_ *ChainService) VerifyLmdFfgConsistency(_ context.Context, a *ethpb.Attes
 }
 
 // ChainHeads mocks ChainHeads and always return nil.
-func (_ *ChainService) ChainHeads() ([][32]byte, []primitives.Slot) {
+func (*ChainService) ChainHeads() ([][32]byte, []primitives.Slot) {
 	return [][32]byte{
 			bytesutil.ToBytes32(bytesutil.PadTo([]byte("foo"), 32)),
 			bytesutil.ToBytes32(bytesutil.PadTo([]byte("bar"), 32)),
@@ -422,7 +431,7 @@ func (_ *ChainService) ChainHeads() ([][32]byte, []primitives.Slot) {
 }
 
 // HeadPublicKeyToValidatorIndex mocks HeadPublicKeyToValidatorIndex and always return 0 and true.
-func (_ *ChainService) HeadPublicKeyToValidatorIndex(_ [fieldparams.BLSPubkeyLength]byte) (primitives.ValidatorIndex, bool) {
+func (*ChainService) HeadPublicKeyToValidatorIndex(_ [fieldparams.BLSPubkeyLength]byte) (primitives.ValidatorIndex, bool) {
 	return 0, true
 }
 
@@ -486,7 +495,7 @@ func (s *ChainService) UpdateHead(ctx context.Context, slot primitives.Slot) {
 }
 
 // ReceiveAttesterSlashing mocks the same method in the chain service.
-func (s *ChainService) ReceiveAttesterSlashing(context.Context, *ethpb.AttesterSlashing) {}
+func (*ChainService) ReceiveAttesterSlashing(context.Context, *ethpb.AttesterSlashing) {}
 
 // IsFinalized mocks the same method in the chain service.
 func (s *ChainService) IsFinalized(_ context.Context, blockRoot [32]byte) bool {
@@ -599,12 +608,12 @@ func (s *ChainService) ProposerBoost() [32]byte {
 }
 
 // FinalizedBlockHash mocks the same method in the chain service
-func (s *ChainService) FinalizedBlockHash() [32]byte {
+func (*ChainService) FinalizedBlockHash() [32]byte {
 	return [32]byte{}
 }
 
 // UnrealizedJustifiedPayloadBlockHash mocks the same method in the chain service
-func (s *ChainService) UnrealizedJustifiedPayloadBlockHash() [32]byte {
+func (*ChainService) UnrealizedJustifiedPayloadBlockHash() [32]byte {
 	return [32]byte{}
 }
 

beacon-chain/blockchain/tracked_proposer.go

@@ -4,6 +4,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/v4/config/features"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 )
 
@@ -11,6 +12,9 @@ import (
 // validators/prepare_proposer endpoint, of the proposer at the given slot.
 // It only returns true if the tracked proposer is present and active.
 func (s *Service) trackedProposer(st state.ReadOnlyBeaconState, slot primitives.Slot) (cache.TrackedValidator, bool) {
+	if features.Get().PrepareAllPayloads {
+		return cache.TrackedValidator{Active: true}, true
+	}
 	id, err := helpers.BeaconProposerIndexAtSlot(s.ctx, st, slot)
 	if err != nil {
 		return cache.TrackedValidator{}, false

beacon-chain/builder/testing/mock.go

@@ -2,6 +2,7 @@ package testing
 
 import (
 	"context"
+	"math/big"
 
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v4/api/client/builder"
@@ -54,13 +55,13 @@ func (s *MockBuilderService) SubmitBlindedBlock(_ context.Context, b interfaces.
 		}
 		return w, nil, s.ErrSubmitBlindedBlock
 	case version.Capella:
-		w, err := blocks.WrappedExecutionPayloadCapella(s.PayloadCapella, 0)
+		w, err := blocks.WrappedExecutionPayloadCapella(s.PayloadCapella, big.NewInt(0))
 		if err != nil {
 			return nil, nil, errors.Wrap(err, "could not wrap capella payload")
 		}
 		return w, nil, s.ErrSubmitBlindedBlock
 	case version.Deneb:
-		w, err := blocks.WrappedExecutionPayloadDeneb(s.PayloadDeneb, 0)
+		w, err := blocks.WrappedExecutionPayloadDeneb(s.PayloadDeneb, big.NewInt(0))
 		if err != nil {
 			return nil, nil, errors.Wrap(err, "could not wrap deneb payload")
 		}

beacon-chain/cache/depositsnapshot/deposit_cache_test.go

@@ -796,7 +796,7 @@ func TestFinalizedDeposits_ReturnsTrieCorrectly(t *testing.T) {
 	err = dc.InsertFinalizedDeposits(context.Background(), 4, [32]byte{}, 0)
 	require.NoError(t, err)
 
-	// Mimick finalized deposit trie fetch.
+	// Mimic finalized deposit trie fetch.
 	fd, err := dc.FinalizedDeposits(context.Background())
 	require.NoError(t, err)
 	deps := dc.NonFinalizedDeposits(context.Background(), fd.MerkleTrieIndex(), nil)

beacon-chain/cache/proposer_indices.go

@@ -42,17 +42,15 @@ var (
 // root would be for slot 32 if present.
 type ProposerIndicesCache struct {
 	sync.Mutex
-	indices       map[primitives.Epoch]map[[32]byte][fieldparams.SlotsPerEpoch]primitives.ValidatorIndex
-	unsafeIndices map[primitives.Epoch]map[[32]byte][fieldparams.SlotsPerEpoch]primitives.ValidatorIndex
-	rootMap       map[forkchoicetypes.Checkpoint][32]byte // A map from checkpoint root to state root
+	indices map[primitives.Epoch]map[[32]byte][fieldparams.SlotsPerEpoch]primitives.ValidatorIndex
+	rootMap map[forkchoicetypes.Checkpoint][32]byte // A map from checkpoint root to state root
 }
 
 // NewProposerIndicesCache returns a newly created cache
 func NewProposerIndicesCache() *ProposerIndicesCache {
 	return &ProposerIndicesCache{
-		indices:       make(map[primitives.Epoch]map[[32]byte][fieldparams.SlotsPerEpoch]primitives.ValidatorIndex),
-		unsafeIndices: make(map[primitives.Epoch]map[[32]byte][fieldparams.SlotsPerEpoch]primitives.ValidatorIndex),
-		rootMap:       make(map[forkchoicetypes.Checkpoint][32]byte),
+		indices: make(map[primitives.Epoch]map[[32]byte][fieldparams.SlotsPerEpoch]primitives.ValidatorIndex),
+		rootMap: make(map[forkchoicetypes.Checkpoint][32]byte),
 	}
 }
 
@@ -74,18 +72,6 @@ func (p *ProposerIndicesCache) ProposerIndices(epoch primitives.Epoch, root [32]
 	return indices, exists
 }
 
-// UnsafeProposerIndices returns the proposer indices (unsafe) for the given root
-func (p *ProposerIndicesCache) UnsafeProposerIndices(epoch primitives.Epoch, root [32]byte) ([fieldparams.SlotsPerEpoch]primitives.ValidatorIndex, bool) {
-	p.Lock()
-	defer p.Unlock()
-	inner, ok := p.unsafeIndices[epoch]
-	if !ok {
-		return [fieldparams.SlotsPerEpoch]primitives.ValidatorIndex{}, false
-	}
-	indices, exists := inner[root]
-	return indices, exists
-}
-
 // Prune resets the ProposerIndicesCache to its initial state
 func (p *ProposerIndicesCache) Prune(epoch primitives.Epoch) {
 	p.Lock()
@@ -95,11 +81,6 @@ func (p *ProposerIndicesCache) Prune(epoch primitives.Epoch) {
 			delete(p.indices, key)
 		}
 	}
-	for key := range p.unsafeIndices {
-		if key < epoch {
-			delete(p.unsafeIndices, key)
-		}
-	}
 	for key := range p.rootMap {
 		if key.Epoch+1 < epoch {
 			delete(p.rootMap, key)
@@ -120,18 +101,6 @@ func (p *ProposerIndicesCache) Set(epoch primitives.Epoch, root [32]byte, indice
 	inner[root] = indices
 }
 
-// SetUnsafe sets the unsafe proposer indices for the given root as key
-func (p *ProposerIndicesCache) SetUnsafe(epoch primitives.Epoch, root [32]byte, indices [fieldparams.SlotsPerEpoch]primitives.ValidatorIndex) {
-	p.Lock()
-	defer p.Unlock()
-	inner, ok := p.unsafeIndices[epoch]
-	if !ok {
-		inner = make(map[[32]byte][fieldparams.SlotsPerEpoch]primitives.ValidatorIndex)
-		p.unsafeIndices[epoch] = inner
-	}
-	inner[root] = indices
-}
-
 // SetCheckpoint updates the map from checkpoints to state roots
 func (p *ProposerIndicesCache) SetCheckpoint(c forkchoicetypes.Checkpoint, root [32]byte) {
 	p.Lock()
@@ -146,6 +115,7 @@ func (p *ProposerIndicesCache) IndicesFromCheckpoint(c forkchoicetypes.Checkpoin
 	root, ok := p.rootMap[c]
 	p.Unlock()
 	if !ok {
+		ProposerIndicesCacheMiss.Inc()
 		return emptyIndices, ok
 	}
 	return p.ProposerIndices(c.Epoch+1, root)

beacon-chain/cache/proposer_indices_test.go

@@ -34,121 +34,72 @@ func TestProposerCache_Set(t *testing.T) {
 	require.Equal(t, emptyIndices, received)
 }
 
-func TestProposerCache_SetUnsafe(t *testing.T) {
-	cache := NewProposerIndicesCache()
-	bRoot := [32]byte{'A'}
-	indices, ok := cache.UnsafeProposerIndices(0, bRoot)
-	require.Equal(t, false, ok)
-	emptyIndices := [fieldparams.SlotsPerEpoch]primitives.ValidatorIndex{}
-	require.Equal(t, indices, emptyIndices, "Expected committee count not to exist in empty cache")
-	emptyIndices[0] = 1
-	cache.SetUnsafe(0, bRoot, emptyIndices)
-
-	received, ok := cache.UnsafeProposerIndices(0, bRoot)
-	require.Equal(t, true, ok)
-	require.Equal(t, received, emptyIndices)
-
-	newRoot := [32]byte{'B'}
-	copy(emptyIndices[3:], []primitives.ValidatorIndex{1, 2, 3, 4, 5, 6})
-	cache.SetUnsafe(0, newRoot, emptyIndices)
-
-	received, ok = cache.UnsafeProposerIndices(0, newRoot)
-	require.Equal(t, true, ok)
-	require.Equal(t, emptyIndices, received)
-}
-
 func TestProposerCache_CheckpointAndPrune(t *testing.T) {
 	cache := NewProposerIndicesCache()
 	indices := [fieldparams.SlotsPerEpoch]primitives.ValidatorIndex{}
-	root := [32]byte{'a'}
-	cpRoot := [32]byte{'b'}
 	copy(indices[3:], []primitives.ValidatorIndex{1, 2, 3, 4, 5, 6})
 	for i := 1; i < 10; i++ {
+		root := [32]byte{byte(i)}
 		cache.Set(primitives.Epoch(i), root, indices)
-		cache.SetUnsafe(primitives.Epoch(i), root, indices)
+		cpRoot := [32]byte{byte(i - 1)}
 		cache.SetCheckpoint(forkchoicetypes.Checkpoint{Epoch: primitives.Epoch(i - 1), Root: cpRoot}, root)
 	}
-	received, ok := cache.ProposerIndices(1, root)
+	received, ok := cache.ProposerIndices(1, [32]byte{1})
 	require.Equal(t, true, ok)
 	require.Equal(t, indices, received)
 
-	received, ok = cache.ProposerIndices(4, root)
+	received, ok = cache.ProposerIndices(4, [32]byte{4})
 	require.Equal(t, true, ok)
 	require.Equal(t, indices, received)
 
-	received, ok = cache.ProposerIndices(9, root)
+	received, ok = cache.ProposerIndices(9, [32]byte{9})
 	require.Equal(t, true, ok)
 	require.Equal(t, indices, received)
 
-	received, ok = cache.UnsafeProposerIndices(1, root)
+	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{})
 	require.Equal(t, true, ok)
 	require.Equal(t, indices, received)
 
-	received, ok = cache.UnsafeProposerIndices(4, root)
+	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 3, Root: [32]byte{3}})
 	require.Equal(t, true, ok)
 	require.Equal(t, indices, received)
 
-	received, ok = cache.UnsafeProposerIndices(9, root)
+	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 4, Root: [32]byte{4}})
 	require.Equal(t, true, ok)
 	require.Equal(t, indices, received)
 
-	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 0, Root: cpRoot})
-	require.Equal(t, true, ok)
-	require.Equal(t, indices, received)
-
-	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 3, Root: cpRoot})
-	require.Equal(t, true, ok)
-	require.Equal(t, indices, received)
-
-	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 4, Root: cpRoot})
-	require.Equal(t, true, ok)
-	require.Equal(t, indices, received)
-
-	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 8, Root: cpRoot})
+	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 8, Root: [32]byte{8}})
 	require.Equal(t, true, ok)
 	require.Equal(t, indices, received)
 
 	cache.Prune(5)
 
 	emptyIndices := [fieldparams.SlotsPerEpoch]primitives.ValidatorIndex{}
-	received, ok = cache.ProposerIndices(1, root)
+	received, ok = cache.ProposerIndices(1, [32]byte{1})
 	require.Equal(t, false, ok)
 	require.Equal(t, emptyIndices, received)
 
-	received, ok = cache.ProposerIndices(4, root)
+	received, ok = cache.ProposerIndices(4, [32]byte{4})
 	require.Equal(t, false, ok)
 	require.Equal(t, emptyIndices, received)
 
-	received, ok = cache.ProposerIndices(9, root)
+	received, ok = cache.ProposerIndices(9, [32]byte{9})
 	require.Equal(t, true, ok)
 	require.Equal(t, indices, received)
 
-	received, ok = cache.UnsafeProposerIndices(1, root)
+	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 0, Root: [32]byte{0}})
 	require.Equal(t, false, ok)
 	require.Equal(t, emptyIndices, received)
 
-	received, ok = cache.UnsafeProposerIndices(4, root)
+	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 3, Root: [32]byte{3}})
 	require.Equal(t, false, ok)
 	require.Equal(t, emptyIndices, received)
 
-	received, ok = cache.UnsafeProposerIndices(9, root)
+	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 4, Root: [32]byte{4}})
 	require.Equal(t, true, ok)
 	require.Equal(t, indices, received)
 
-	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 0, Root: cpRoot})
-	require.Equal(t, false, ok)
-	require.Equal(t, emptyIndices, received)
-
-	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 3, Root: cpRoot})
-	require.Equal(t, false, ok)
-	require.Equal(t, emptyIndices, received)
-
-	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 4, Root: cpRoot})
+	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 8, Root: [32]byte{8}})
 	require.Equal(t, true, ok)
 	require.Equal(t, indices, received)
-
-	received, ok = cache.IndicesFromCheckpoint(forkchoicetypes.Checkpoint{Epoch: 8, Root: cpRoot})
-	require.Equal(t, true, ok)
-	require.Equal(t, indices, received)
-
 }

beacon-chain/core/altair/epoch_precompute.go

@@ -15,11 +15,12 @@ import (
 
 // AttDelta contains rewards and penalties for a single attestation.
 type AttDelta struct {
-	HeadReward    uint64
-	SourceReward  uint64
-	SourcePenalty uint64
-	TargetReward  uint64
-	TargetPenalty uint64
+	HeadReward        uint64
+	SourceReward      uint64
+	SourcePenalty     uint64
+	TargetReward      uint64
+	TargetPenalty     uint64
+	InactivityPenalty uint64
 }
 
 // InitializePrecomputeValidators precomputes individual validator for its attested balances and the total sum of validators attested balances of the epoch.
@@ -251,7 +252,7 @@ func ProcessRewardsAndPenaltiesPrecompute(
 		if err != nil {
 			return nil, err
 		}
-		balances[i] = helpers.DecreaseBalanceWithVal(balances[i], delta.SourcePenalty+delta.TargetPenalty)
+		balances[i] = helpers.DecreaseBalanceWithVal(balances[i], delta.SourcePenalty+delta.TargetPenalty+delta.InactivityPenalty)
 
 		vals[i].AfterEpochTransitionBalance = balances[i]
 	}
@@ -351,7 +352,7 @@ func attestationDelta(
 		if err != nil {
 			return &AttDelta{}, err
 		}
-		attDelta.TargetPenalty += n / inactivityDenominator
+		attDelta.InactivityPenalty = n / inactivityDenominator
 	}
 
 	return attDelta, nil

beacon-chain/core/altair/epoch_precompute_test.go

@@ -220,7 +220,7 @@ func TestAttestationsDelta(t *testing.T) {
 	penalties := make([]uint64, len(deltas))
 	for i, d := range deltas {
 		rewards[i] = d.HeadReward + d.SourceReward + d.TargetReward
-		penalties[i] = d.SourcePenalty + d.TargetPenalty
+		penalties[i] = d.SourcePenalty + d.TargetPenalty + d.InactivityPenalty
 	}
 
 	// Reward amount should increase as validator index increases due to setup.
@@ -258,7 +258,7 @@ func TestAttestationsDeltaBellatrix(t *testing.T) {
 	penalties := make([]uint64, len(deltas))
 	for i, d := range deltas {
 		rewards[i] = d.HeadReward + d.SourceReward + d.TargetReward
-		penalties[i] = d.SourcePenalty + d.TargetPenalty
+		penalties[i] = d.SourcePenalty + d.TargetPenalty + d.InactivityPenalty
 	}
 
 	// Reward amount should increase as validator index increases due to setup.
@@ -306,7 +306,7 @@ func TestProcessRewardsAndPenaltiesPrecompute_Ok(t *testing.T) {
 	penalties := make([]uint64, len(deltas))
 	for i, d := range deltas {
 		rewards[i] = d.HeadReward + d.SourceReward + d.TargetReward
-		penalties[i] = d.SourcePenalty + d.TargetPenalty
+		penalties[i] = d.SourcePenalty + d.TargetPenalty + d.InactivityPenalty
 	}
 	for i := range rewards {
 		wanted[i] += rewards[i]

beacon-chain/core/blocks/genesis.go

@@ -105,6 +105,7 @@ func NewGenesisBlockForState(ctx context.Context, st state.BeaconState) (interfa
 						ReceiptsRoot:  make([]byte, 32),
 						LogsBloom:     make([]byte, 256),
 						PrevRandao:    make([]byte, 32),
+						ExtraData:     make([]byte, 0),
 						BaseFeePerGas: make([]byte, 32),
 						BlockHash:     make([]byte, 32),
 						Transactions:  make([][]byte, 0),
@@ -136,6 +137,7 @@ func NewGenesisBlockForState(ctx context.Context, st state.BeaconState) (interfa
 						ReceiptsRoot:  make([]byte, 32),
 						LogsBloom:     make([]byte, 256),
 						PrevRandao:    make([]byte, 32),
+						ExtraData:     make([]byte, 0),
 						BaseFeePerGas: make([]byte, 32),
 						BlockHash:     make([]byte, 32),
 						Transactions:  make([][]byte, 0),
@@ -168,6 +170,7 @@ func NewGenesisBlockForState(ctx context.Context, st state.BeaconState) (interfa
 						ReceiptsRoot:  make([]byte, 32),
 						LogsBloom:     make([]byte, 256),
 						PrevRandao:    make([]byte, 32),
+						ExtraData:     make([]byte, 0),
 						BaseFeePerGas: make([]byte, 32),
 						BlockHash:     make([]byte, 32),
 						Transactions:  make([][]byte, 0),

beacon-chain/core/blocks/payload_test.go

@@ -1,6 +1,7 @@
 package blocks_test
 
 import (
+	"math/big"
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/blocks"
@@ -609,7 +610,7 @@ func Test_ProcessPayloadCapella(t *testing.T) {
 	random, err := helpers.RandaoMix(st, time.CurrentEpoch(st))
 	require.NoError(t, err)
 	payload.PrevRandao = random
-	wrapped, err := consensusblocks.WrappedExecutionPayloadCapella(payload, 0)
+	wrapped, err := consensusblocks.WrappedExecutionPayloadCapella(payload, big.NewInt(0))
 	require.NoError(t, err)
 	_, err = blocks.ProcessPayload(st, wrapped)
 	require.NoError(t, err)
@@ -853,10 +854,10 @@ func emptyPayloadHeader() (interfaces.ExecutionData, error) {
 		ReceiptsRoot:     make([]byte, fieldparams.RootLength),
 		LogsBloom:        make([]byte, fieldparams.LogsBloomLength),
 		PrevRandao:       make([]byte, fieldparams.RootLength),
+		ExtraData:        make([]byte, 0),
 		BaseFeePerGas:    make([]byte, fieldparams.RootLength),
 		BlockHash:        make([]byte, fieldparams.RootLength),
 		TransactionsRoot: make([]byte, fieldparams.RootLength),
-		ExtraData:        make([]byte, 0),
 	})
 }
 
@@ -868,12 +869,12 @@ func emptyPayloadHeaderCapella() (interfaces.ExecutionData, error) {
 		ReceiptsRoot:     make([]byte, fieldparams.RootLength),
 		LogsBloom:        make([]byte, fieldparams.LogsBloomLength),
 		PrevRandao:       make([]byte, fieldparams.RootLength),
+		ExtraData:        make([]byte, 0),
 		BaseFeePerGas:    make([]byte, fieldparams.RootLength),
 		BlockHash:        make([]byte, fieldparams.RootLength),
 		TransactionsRoot: make([]byte, fieldparams.RootLength),
 		WithdrawalsRoot:  make([]byte, fieldparams.RootLength),
-		ExtraData:        make([]byte, 0),
-	}, 0)
+	}, big.NewInt(0))
 }
 
 func emptyPayload() *enginev1.ExecutionPayload {
@@ -884,10 +885,10 @@ func emptyPayload() *enginev1.ExecutionPayload {
 		ReceiptsRoot:  make([]byte, fieldparams.RootLength),
 		LogsBloom:     make([]byte, fieldparams.LogsBloomLength),
 		PrevRandao:    make([]byte, fieldparams.RootLength),
+		ExtraData:     make([]byte, 0),
 		BaseFeePerGas: make([]byte, fieldparams.RootLength),
 		BlockHash:     make([]byte, fieldparams.RootLength),
 		Transactions:  make([][]byte, 0),
-		ExtraData:     make([]byte, 0),
 	}
 }
 
@@ -899,10 +900,10 @@ func emptyPayloadCapella() *enginev1.ExecutionPayloadCapella {
 		ReceiptsRoot:  make([]byte, fieldparams.RootLength),
 		LogsBloom:     make([]byte, fieldparams.LogsBloomLength),
 		PrevRandao:    make([]byte, fieldparams.RootLength),
+		ExtraData:     make([]byte, 0),
 		BaseFeePerGas: make([]byte, fieldparams.RootLength),
 		BlockHash:     make([]byte, fieldparams.RootLength),
 		Transactions:  make([][]byte, 0),
 		Withdrawals:   make([]*enginev1.Withdrawal, 0),
-		ExtraData:     make([]byte, 0),
 	}
 }

beacon-chain/core/blocks/withdrawals_test.go

@@ -1,6 +1,7 @@
 package blocks_test
 
 import (
+	"math/big"
 	"math/rand"
 	"testing"
 
@@ -642,7 +643,10 @@ func TestProcessBlindWithdrawals(t *testing.T) {
 			require.NoError(t, err)
 			wdRoot, err := ssz.WithdrawalSliceRoot(test.Args.Withdrawals, fieldparams.MaxWithdrawalsPerPayload)
 			require.NoError(t, err)
-			p, err := consensusblocks.WrappedExecutionPayloadHeaderCapella(&enginev1.ExecutionPayloadHeaderCapella{WithdrawalsRoot: wdRoot[:]}, 0)
+			p, err := consensusblocks.WrappedExecutionPayloadHeaderCapella(
+				&enginev1.ExecutionPayloadHeaderCapella{WithdrawalsRoot: wdRoot[:]},
+				big.NewInt(0),
+			)
 			require.NoError(t, err)
 			post, err := blocks.ProcessWithdrawals(st, p)
 			if test.Control.ExpectedError {
@@ -1060,7 +1064,7 @@ func TestProcessWithdrawals(t *testing.T) {
 			}
 			st, err := prepareValidators(spb, test.Args)
 			require.NoError(t, err)
-			p, err := consensusblocks.WrappedExecutionPayloadCapella(&enginev1.ExecutionPayloadCapella{Withdrawals: test.Args.Withdrawals}, 0)
+			p, err := consensusblocks.WrappedExecutionPayloadCapella(&enginev1.ExecutionPayloadCapella{Withdrawals: test.Args.Withdrawals}, big.NewInt(0))
 			require.NoError(t, err)
 			post, err := blocks.ProcessWithdrawals(st, p)
 			if test.Control.ExpectedError {

beacon-chain/core/capella/upgrade_test.go

@@ -84,6 +84,7 @@ func TestUpgradeToCapella(t *testing.T) {
 		GasLimit:         prevHeader.GasLimit(),
 		GasUsed:          prevHeader.GasUsed(),
 		Timestamp:        prevHeader.Timestamp(),
+		ExtraData:        prevHeader.ExtraData(),
 		BaseFeePerGas:    prevHeader.BaseFeePerGas(),
 		BlockHash:        prevHeader.BlockHash(),
 		TransactionsRoot: txRoot,

beacon-chain/core/deneb/upgrade.go

@@ -57,6 +57,10 @@ func UpgradeToDeneb(state state.BeaconState) (state.BeaconState, error) {
 	if err != nil {
 		return nil, err
 	}
+	historicalRoots, err := state.HistoricalRoots()
+	if err != nil {
+		return nil, err
+	}
 
 	s := &ethpb.BeaconStateDeneb{
 		GenesisTime:           state.GenesisTime(),
@@ -70,7 +74,7 @@ func UpgradeToDeneb(state state.BeaconState) (state.BeaconState, error) {
 		LatestBlockHeader:           state.LatestBlockHeader(),
 		BlockRoots:                  state.BlockRoots(),
 		StateRoots:                  state.StateRoots(),
-		HistoricalRoots:             [][]byte{},
+		HistoricalRoots:             historicalRoots,
 		Eth1Data:                    state.Eth1Data(),
 		Eth1DataVotes:               state.Eth1DataVotes(),
 		Eth1DepositIndex:            state.Eth1DepositIndex(),
@@ -101,10 +105,10 @@ func UpgradeToDeneb(state state.BeaconState) (state.BeaconState, error) {
 			ExtraData:        payloadHeader.ExtraData(),
 			BaseFeePerGas:    payloadHeader.BaseFeePerGas(),
 			BlockHash:        payloadHeader.BlockHash(),
-			ExcessBlobGas:    0,
-			BlobGasUsed:      0,
 			TransactionsRoot: txRoot,
 			WithdrawalsRoot:  wdRoot,
+			ExcessBlobGas:    0,
+			BlobGasUsed:      0,
 		},
 		NextWithdrawalIndex:          wi,
 		NextWithdrawalValidatorIndex: vi,

beacon-chain/core/deneb/upgrade_test.go

@@ -14,6 +14,7 @@ import (
 
 func TestUpgradeToDeneb(t *testing.T) {
 	st, _ := util.DeterministicGenesisStateCapella(t, params.BeaconConfig().MaxValidatorsPerCommittee)
+	require.NoError(t, st.SetHistoricalRoots([][]byte{{1}}))
 	preForkState := st.Copy()
 	mSt, err := deneb.UpgradeToDeneb(st)
 	require.NoError(t, err)
@@ -46,6 +47,12 @@ func TestUpgradeToDeneb(t *testing.T) {
 	require.NoError(t, err)
 	require.DeepSSZEqual(t, make([]uint64, numValidators), s)
 
+	hr1, err := preForkState.HistoricalRoots()
+	require.NoError(t, err)
+	hr2, err := mSt.HistoricalRoots()
+	require.NoError(t, err)
+	require.DeepEqual(t, hr1, hr2)
+
 	f := mSt.Fork()
 	require.DeepSSZEqual(t, &ethpb.Fork{
 		PreviousVersion: st.Fork().CurrentVersion,
@@ -85,6 +92,7 @@ func TestUpgradeToDeneb(t *testing.T) {
 		GasLimit:         prevHeader.GasLimit(),
 		GasUsed:          prevHeader.GasUsed(),
 		Timestamp:        prevHeader.Timestamp(),
+		ExtraData:        prevHeader.ExtraData(),
 		BaseFeePerGas:    prevHeader.BaseFeePerGas(),
 		BlockHash:        prevHeader.BlockHash(),
 		TransactionsRoot: txRoot,

beacon-chain/core/execution/upgrade.go

@@ -79,6 +79,7 @@ func UpgradeToBellatrix(state state.BeaconState) (state.BeaconState, error) {
 			GasLimit:         0,
 			GasUsed:          0,
 			Timestamp:        0,
+			ExtraData:        make([]byte, 0),
 			BaseFeePerGas:    make([]byte, 32),
 			BlockHash:        make([]byte, 32),
 			TransactionsRoot: make([]byte, 32),

beacon-chain/core/execution/upgrade_test.go

@@ -79,6 +79,7 @@ func TestUpgradeToBellatrix(t *testing.T) {
 		GasLimit:         0,
 		GasUsed:          0,
 		Timestamp:        0,
+		ExtraData:        make([]byte, 0),
 		BaseFeePerGas:    make([]byte, 32),
 		BlockHash:        make([]byte, 32),
 		TransactionsRoot: make([]byte, 32),

beacon-chain/core/feed/operation/events.go

@@ -26,6 +26,12 @@ const (
 
 	// BlobSidecarReceived is sent after a blob sidecar is received from gossip or rpc.
 	BlobSidecarReceived = 6
+
+	// ProposerSlashingReceived is sent after a proposer slashing is received from gossip or rpc
+	ProposerSlashingReceived = 7
+
+	// AttesterSlashingReceived is sent after an attester slashing is received from gossip or rpc
+	AttesterSlashingReceived = 8
 )
 
 // UnAggregatedAttReceivedData is the data sent with UnaggregatedAttReceived events.
@@ -61,3 +67,13 @@ type BLSToExecutionChangeReceivedData struct {
 type BlobSidecarReceivedData struct {
 	Blob *blocks.VerifiedROBlob
 }
+
+// ProposerSlashingReceivedData is the data sent with ProposerSlashingReceived events.
+type ProposerSlashingReceivedData struct {
+	ProposerSlashing *ethpb.ProposerSlashing
+}
+
+// AttesterSlashingReceivedData is the data sent with AttesterSlashingReceived events.
+type AttesterSlashingReceivedData struct {
+	AttesterSlashing *ethpb.AttesterSlashing
+}

beacon-chain/core/feed/state/events.go

@@ -27,6 +27,10 @@ const (
 	NewHead
 	// MissedSlot is sent when we need to notify users that a slot was missed.
 	MissedSlot
+	// LightClientFinalityUpdate event
+	LightClientFinalityUpdate
+	// LightClientOptimisticUpdate event
+	LightClientOptimisticUpdate
 )
 
 // BlockProcessedData is the data sent with BlockProcessed events.

beacon-chain/core/helpers/BUILD.bazel

@@ -50,7 +50,6 @@ go_test(
         "attestation_test.go",
         "beacon_committee_test.go",
         "block_test.go",
-        "main_test.go",
         "randao_test.go",
         "rewards_penalties_test.go",
         "shuffle_test.go",
@@ -64,6 +63,7 @@ go_test(
     deps = [
         "//beacon-chain/cache:go_default_library",
         "//beacon-chain/core/time:go_default_library",
+        "//beacon-chain/forkchoice/types:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/state-native:go_default_library",
         "//config/fieldparams:go_default_library",

beacon-chain/core/helpers/attestation_test.go

@@ -20,6 +20,8 @@ import (
 
 func TestAttestation_IsAggregator(t *testing.T) {
 	t.Run("aggregator", func(t *testing.T) {
+		helpers.ClearCache()
+
 		beaconState, privKeys := util.DeterministicGenesisState(t, 100)
 		committee, err := helpers.BeaconCommitteeFromState(context.Background(), beaconState, 0, 0)
 		require.NoError(t, err)
@@ -30,6 +32,8 @@ func TestAttestation_IsAggregator(t *testing.T) {
 	})
 
 	t.Run("not aggregator", func(t *testing.T) {
+		helpers.ClearCache()
+
 		params.SetupTestConfigCleanup(t)
 		params.OverrideBeaconConfig(params.MinimalSpecConfig())
 		beaconState, privKeys := util.DeterministicGenesisState(t, 2048)
@@ -44,6 +48,8 @@ func TestAttestation_IsAggregator(t *testing.T) {
 }
 
 func TestAttestation_ComputeSubnetForAttestation(t *testing.T) {
+	helpers.ClearCache()
+
 	// Create 10 committees
 	committeeCount := uint64(10)
 	validatorCount := committeeCount * params.BeaconConfig().TargetCommitteeSize
@@ -204,6 +210,8 @@ func Test_ValidateAttestationTime(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			helpers.ClearCache()
+
 			err := helpers.ValidateAttestationTime(tt.args.attSlot, tt.args.genesisTime,
 				params.BeaconConfig().MaximumGossipClockDisparityDuration())
 			if tt.wantedErr != "" {
@@ -216,6 +224,8 @@ func Test_ValidateAttestationTime(t *testing.T) {
 }
 
 func TestVerifyCheckpointEpoch_Ok(t *testing.T) {
+	helpers.ClearCache()
+
 	// Genesis was 6 epochs ago exactly.
 	offset := params.BeaconConfig().SlotsPerEpoch.Mul(params.BeaconConfig().SecondsPerSlot * 6)
 	genesis := time.Now().Add(-1 * time.Second * time.Duration(offset))
@@ -285,6 +295,8 @@ func TestValidateNilAttestation(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			helpers.ClearCache()
+
 			if tt.errString != "" {
 				require.ErrorContains(t, tt.errString, helpers.ValidateNilAttestation(tt.attestation))
 			} else {
@@ -326,6 +338,8 @@ func TestValidateSlotTargetEpoch(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			helpers.ClearCache()
+
 			if tt.errString != "" {
 				require.ErrorContains(t, tt.errString, helpers.ValidateSlotTargetEpoch(tt.attestation.Data))
 			} else {

beacon-chain/core/helpers/beacon_committee.go

@@ -379,7 +379,7 @@ func UpdateCachedCheckpointToStateRoot(state state.ReadOnlyBeaconState, cp *fork
 	if cp.Epoch <= params.BeaconConfig().GenesisEpoch+params.BeaconConfig().MinSeedLookahead {
 		return nil
 	}
-	slot, err := slots.EpochEnd(cp.Epoch - 1)
+	slot, err := slots.EpochEnd(cp.Epoch)
 	if err != nil {
 		return err
 	}
@@ -391,49 +391,6 @@ func UpdateCachedCheckpointToStateRoot(state state.ReadOnlyBeaconState, cp *fork
 	return nil
 }
 
-// UpdateUnsafeProposerIndicesInCache updates proposer indices entry of the
-// cache one epoch in advance.
-// Input state is used to retrieve active validator indices.
-// Input root is to use as key in the cache.
-// Input epoch is the epoch to retrieve proposer indices for.
-func UpdateUnsafeProposerIndicesInCache(ctx context.Context, state state.ReadOnlyBeaconState, epoch primitives.Epoch) error {
-	// The cache uses the state root at the end of (current epoch - 2) as key.
-	// (e.g. for epoch 2, the key is root at slot 31)
-	if epoch <= params.BeaconConfig().GenesisEpoch+2*params.BeaconConfig().MinSeedLookahead {
-		return nil
-	}
-	slot, err := slots.EpochEnd(epoch - 2)
-	if err != nil {
-		return err
-	}
-	root, err := state.StateRootAtIndex(uint64(slot % params.BeaconConfig().SlotsPerHistoricalRoot))
-	if err != nil {
-		return err
-	}
-	// Skip cache update if the key already exists
-	_, ok := proposerIndicesCache.UnsafeProposerIndices(epoch, [32]byte(root))
-	if ok {
-		return nil
-	}
-	indices, err := ActiveValidatorIndices(ctx, state, epoch)
-	if err != nil {
-		return err
-	}
-	proposerIndices, err := precomputeProposerIndices(state, indices, epoch)
-	if err != nil {
-		return err
-	}
-	if len(proposerIndices) != int(params.BeaconConfig().SlotsPerEpoch) {
-		return errors.New("invalid proposer length returned from state")
-	}
-	// This is here to deal with tests only
-	var indicesArray [fieldparams.SlotsPerEpoch]primitives.ValidatorIndex
-	copy(indicesArray[:], proposerIndices)
-	proposerIndicesCache.Prune(epoch - 2)
-	proposerIndicesCache.SetUnsafe(epoch, [32]byte(root), indicesArray)
-	return nil
-}
-
 // ClearCache clears the beacon committee cache and sync committee cache.
 func ClearCache() {
 	committeeCache.Clear()

beacon-chain/core/helpers/beacon_committee_test.go

@@ -21,6 +21,8 @@ import (
 )
 
 func TestComputeCommittee_WithoutCache(t *testing.T) {
+	ClearCache()
+
 	// Create 10 committees
 	committeeCount := uint64(10)
 	validatorCount := committeeCount * params.BeaconConfig().TargetCommitteeSize
@@ -71,6 +73,8 @@ func TestComputeCommittee_WithoutCache(t *testing.T) {
 }
 
 func TestComputeCommittee_RegressionTest(t *testing.T) {
+	ClearCache()
+
 	indices := []primitives.ValidatorIndex{1, 3, 8, 16, 18, 19, 20, 23, 30, 35, 43, 46, 47, 54, 56, 58, 69, 70, 71, 83, 84, 85, 91, 96, 100, 103, 105, 106, 112, 121, 127, 128, 129, 140, 142, 144, 146, 147, 149, 152, 153, 154, 157, 160, 173, 175, 180, 182, 188, 189, 191, 194, 201, 204, 217, 221, 226, 228, 230, 231, 239, 241, 249, 250, 255}
 	seed := [32]byte{68, 110, 161, 250, 98, 230, 161, 172, 227, 226, 99, 11, 138, 124, 201, 134, 38, 197, 0, 120, 6, 165, 122, 34, 19, 216, 43, 226, 210, 114, 165, 183}
 	index := uint64(215)
@@ -80,6 +84,8 @@ func TestComputeCommittee_RegressionTest(t *testing.T) {
 }
 
 func TestVerifyBitfieldLength_OK(t *testing.T) {
+	ClearCache()
+
 	bf := bitfield.Bitlist{0xFF, 0x01}
 	committeeSize := uint64(8)
 	assert.NoError(t, VerifyBitfieldLength(bf, committeeSize), "Bitfield is not validated when it was supposed to be")
@@ -91,7 +97,7 @@ func TestVerifyBitfieldLength_OK(t *testing.T) {
 
 func TestCommitteeAssignments_CannotRetrieveFutureEpoch(t *testing.T) {
 	ClearCache()
-	defer ClearCache()
+
 	epoch := primitives.Epoch(1)
 	state, err := state_native.InitializeFromProtoPhase0(&ethpb.BeaconState{
 		Slot: 0, // Epoch 0.
@@ -103,7 +109,7 @@ func TestCommitteeAssignments_CannotRetrieveFutureEpoch(t *testing.T) {
 
 func TestCommitteeAssignments_NoProposerForSlot0(t *testing.T) {
 	ClearCache()
-	defer ClearCache()
+
 	validators := make([]*ethpb.Validator, 4*params.BeaconConfig().SlotsPerEpoch)
 	for i := 0; i < len(validators); i++ {
 		var activationEpoch primitives.Epoch
@@ -190,10 +196,10 @@ func TestCommitteeAssignments_CanRetrieve(t *testing.T) {
 		},
 	}
 
-	defer ClearCache()
 	for i, tt := range tests {
 		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
 			ClearCache()
+
 			validatorIndexToCommittee, proposerIndexToSlots, err := CommitteeAssignments(context.Background(), state, slots.ToEpoch(tt.slot))
 			require.NoError(t, err, "Failed to determine CommitteeAssignments")
 			cac := validatorIndexToCommittee[tt.index]
@@ -209,6 +215,8 @@ func TestCommitteeAssignments_CanRetrieve(t *testing.T) {
 }
 
 func TestCommitteeAssignments_CannotRetrieveFuture(t *testing.T) {
+	ClearCache()
+
 	// Initialize test with 256 validators, each slot and each index gets 4 validators.
 	validators := make([]*ethpb.Validator, 4*params.BeaconConfig().SlotsPerEpoch)
 	for i := 0; i < len(validators); i++ {
@@ -239,6 +247,8 @@ func TestCommitteeAssignments_CannotRetrieveFuture(t *testing.T) {
 }
 
 func TestCommitteeAssignments_CannotRetrieveOlderThanSlotsPerHistoricalRoot(t *testing.T) {
+	ClearCache()
+
 	// Initialize test with 256 validators, each slot and each index gets 4 validators.
 	validators := make([]*ethpb.Validator, 4*params.BeaconConfig().SlotsPerEpoch)
 	for i := 0; i < len(validators); i++ {
@@ -259,7 +269,7 @@ func TestCommitteeAssignments_CannotRetrieveOlderThanSlotsPerHistoricalRoot(t *t
 
 func TestCommitteeAssignments_EverySlotHasMin1Proposer(t *testing.T) {
 	ClearCache()
-	defer ClearCache()
+
 	// Initialize test with 256 validators, each slot and each index gets 4 validators.
 	validators := make([]*ethpb.Validator, 4*params.BeaconConfig().SlotsPerEpoch)
 	for i := 0; i < len(validators); i++ {
@@ -380,9 +390,9 @@ func TestVerifyAttestationBitfieldLengths_OK(t *testing.T) {
 		},
 	}
 
-	defer ClearCache()
 	for i, tt := range tests {
 		ClearCache()
+
 		require.NoError(t, state.SetSlot(tt.stateSlot))
 		err := VerifyAttestationBitfieldLengths(context.Background(), state, tt.attestation)
 		if tt.verificationFailure {
@@ -395,7 +405,7 @@ func TestVerifyAttestationBitfieldLengths_OK(t *testing.T) {
 
 func TestUpdateCommitteeCache_CanUpdate(t *testing.T) {
 	ClearCache()
-	defer ClearCache()
+
 	validatorCount := params.BeaconConfig().MinGenesisActiveValidatorCount
 	validators := make([]*ethpb.Validator, validatorCount)
 	indices := make([]primitives.ValidatorIndex, validatorCount)
@@ -425,7 +435,7 @@ func TestUpdateCommitteeCache_CanUpdate(t *testing.T) {
 
 func TestUpdateCommitteeCache_CanUpdateAcrossEpochs(t *testing.T) {
 	ClearCache()
-	defer ClearCache()
+
 	validatorCount := params.BeaconConfig().MinGenesisActiveValidatorCount
 	validators := make([]*ethpb.Validator, validatorCount)
 	indices := make([]primitives.ValidatorIndex, validatorCount)

beacon-chain/core/helpers/block_test.go

@@ -60,6 +60,8 @@ func TestBlockRootAtSlot_CorrectBlockRoot(t *testing.T) {
 	}
 	for i, tt := range tests {
 		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			helpers.ClearCache()
+
 			s.Slot = tt.stateSlot
 			state, err := state_native.InitializeFromProtoPhase0(s)
 			require.NoError(t, err)
@@ -110,6 +112,8 @@ func TestBlockRootAtSlot_OutOfBounds(t *testing.T) {
 		},
 	}
 	for _, tt := range tests {
+		helpers.ClearCache()
+
 		state.Slot = tt.stateSlot
 		s, err := state_native.InitializeFromProtoPhase0(state)
 		require.NoError(t, err)

beacon-chain/core/helpers/main_test.go

@@ -1,13 +0,0 @@
-package helpers
-
-import (
-	"os"
-	"testing"
-)
-
-// run ClearCache before each test to prevent cross-test side effects
-func TestMain(m *testing.M) {
-	ClearCache()
-	code := m.Run()
-	os.Exit(code)
-}

beacon-chain/core/helpers/randao_test.go

@@ -40,6 +40,8 @@ func TestRandaoMix_OK(t *testing.T) {
 		},
 	}
 	for _, test := range tests {
+		ClearCache()
+
 		require.NoError(t, state.SetSlot(params.BeaconConfig().SlotsPerEpoch.Mul(uint64(test.epoch+1))))
 		mix, err := RandaoMix(state, test.epoch)
 		require.NoError(t, err)
@@ -74,6 +76,8 @@ func TestRandaoMix_CopyOK(t *testing.T) {
 		},
 	}
 	for _, test := range tests {
+		ClearCache()
+
 		require.NoError(t, state.SetSlot(params.BeaconConfig().SlotsPerEpoch.Mul(uint64(test.epoch+1))))
 		mix, err := RandaoMix(state, test.epoch)
 		require.NoError(t, err)
@@ -88,6 +92,8 @@ func TestRandaoMix_CopyOK(t *testing.T) {
 }
 
 func TestGenerateSeed_OK(t *testing.T) {
+	ClearCache()
+
 	randaoMixes := make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector)
 	for i := 0; i < len(randaoMixes); i++ {
 		intInBytes := make([]byte, 32)

beacon-chain/core/helpers/rewards_penalties_test.go

@@ -14,6 +14,8 @@ import (
 )
 
 func TestTotalBalance_OK(t *testing.T) {
+	ClearCache()
+
 	state, err := state_native.InitializeFromProtoPhase0(&ethpb.BeaconState{Validators: []*ethpb.Validator{
 		{EffectiveBalance: 27 * 1e9}, {EffectiveBalance: 28 * 1e9},
 		{EffectiveBalance: 32 * 1e9}, {EffectiveBalance: 40 * 1e9},
@@ -27,6 +29,8 @@ func TestTotalBalance_OK(t *testing.T) {
 }
 
 func TestTotalBalance_ReturnsEffectiveBalanceIncrement(t *testing.T) {
+	ClearCache()
+
 	state, err := state_native.InitializeFromProtoPhase0(&ethpb.BeaconState{Validators: []*ethpb.Validator{}})
 	require.NoError(t, err)
 
@@ -47,6 +51,8 @@ func TestGetBalance_OK(t *testing.T) {
 		{i: 2, b: []uint64{0, 0, 0}},
 	}
 	for _, test := range tests {
+		ClearCache()
+
 		state, err := state_native.InitializeFromProtoPhase0(&ethpb.BeaconState{Balances: test.b})
 		require.NoError(t, err)
 		assert.Equal(t, test.b[test.i], state.Balances()[test.i], "Incorrect Validator balance")
@@ -62,6 +68,8 @@ func TestTotalActiveBalance(t *testing.T) {
 		{10000},
 	}
 	for _, test := range tests {
+		ClearCache()
+
 		validators := make([]*ethpb.Validator, 0)
 		for i := 0; i < test.vCount; i++ {
 			validators = append(validators, &ethpb.Validator{EffectiveBalance: params.BeaconConfig().MaxEffectiveBalance, ExitEpoch: 1})
@@ -75,8 +83,6 @@ func TestTotalActiveBalance(t *testing.T) {
 }
 
 func TestTotalActiveBal_ReturnMin(t *testing.T) {
-	ClearCache()
-	defer ClearCache()
 	tests := []struct {
 		vCount int
 	}{
@@ -85,6 +91,8 @@ func TestTotalActiveBal_ReturnMin(t *testing.T) {
 		{10000},
 	}
 	for _, test := range tests {
+		ClearCache()
+
 		validators := make([]*ethpb.Validator, 0)
 		for i := 0; i < test.vCount; i++ {
 			validators = append(validators, &ethpb.Validator{EffectiveBalance: 1, ExitEpoch: 1})
@@ -98,8 +106,6 @@ func TestTotalActiveBal_ReturnMin(t *testing.T) {
 }
 
 func TestTotalActiveBalance_WithCache(t *testing.T) {
-	ClearCache()
-	defer ClearCache()
 	tests := []struct {
 		vCount    int
 		wantCount int
@@ -109,6 +115,8 @@ func TestTotalActiveBalance_WithCache(t *testing.T) {
 		{vCount: 10000, wantCount: 10000},
 	}
 	for _, test := range tests {
+		ClearCache()
+
 		validators := make([]*ethpb.Validator, 0)
 		for i := 0; i < test.vCount; i++ {
 			validators = append(validators, &ethpb.Validator{EffectiveBalance: params.BeaconConfig().MaxEffectiveBalance, ExitEpoch: 1})
@@ -133,6 +141,8 @@ func TestIncreaseBalance_OK(t *testing.T) {
 		{i: 2, b: []uint64{27 * 1e9, 28 * 1e9, 32 * 1e9}, nb: 33 * 1e9, eb: 65 * 1e9},
 	}
 	for _, test := range tests {
+		ClearCache()
+
 		state, err := state_native.InitializeFromProtoPhase0(&ethpb.BeaconState{
 			Validators: []*ethpb.Validator{
 				{EffectiveBalance: 4}, {EffectiveBalance: 4}, {EffectiveBalance: 4}},
@@ -157,6 +167,8 @@ func TestDecreaseBalance_OK(t *testing.T) {
 		{i: 3, b: []uint64{27 * 1e9, 28 * 1e9, 1, 28 * 1e9}, nb: 28 * 1e9, eb: 0},
 	}
 	for _, test := range tests {
+		ClearCache()
+
 		state, err := state_native.InitializeFromProtoPhase0(&ethpb.BeaconState{
 			Validators: []*ethpb.Validator{
 				{EffectiveBalance: 4}, {EffectiveBalance: 4}, {EffectiveBalance: 4}, {EffectiveBalance: 3}},
@@ -169,6 +181,8 @@ func TestDecreaseBalance_OK(t *testing.T) {
 }
 
 func TestFinalityDelay(t *testing.T) {
+	ClearCache()
+
 	base := buildState(params.BeaconConfig().SlotsPerEpoch*10, 1)
 	base.FinalizedCheckpoint = &ethpb.Checkpoint{Epoch: 3}
 	beaconState, err := state_native.InitializeFromProtoPhase0(base)
@@ -199,6 +213,8 @@ func TestFinalityDelay(t *testing.T) {
 }
 
 func TestIsInInactivityLeak(t *testing.T) {
+	ClearCache()
+
 	base := buildState(params.BeaconConfig().SlotsPerEpoch*10, 1)
 	base.FinalizedCheckpoint = &ethpb.Checkpoint{Epoch: 3}
 	beaconState, err := state_native.InitializeFromProtoPhase0(base)
@@ -269,6 +285,8 @@ func TestIncreaseBadBalance_NotOK(t *testing.T) {
 		{i: 2, b: []uint64{math.MaxUint64, math.MaxUint64, math.MaxUint64}, nb: 33 * 1e9},
 	}
 	for _, test := range tests {
+		ClearCache()
+
 		state, err := state_native.InitializeFromProtoPhase0(&ethpb.BeaconState{
 			Validators: []*ethpb.Validator{
 				{EffectiveBalance: 4}, {EffectiveBalance: 4}, {EffectiveBalance: 4}},

beacon-chain/core/helpers/shuffle_test.go

@@ -13,6 +13,8 @@ import (
 )
 
 func TestShuffleList_InvalidValidatorCount(t *testing.T) {
+	ClearCache()
+
 	maxShuffleListSize = 20
 	list := make([]primitives.ValidatorIndex, 21)
 	if _, err := ShuffleList(list, [32]byte{123, 125}); err == nil {
@@ -23,6 +25,8 @@ func TestShuffleList_InvalidValidatorCount(t *testing.T) {
 }
 
 func TestShuffleList_OK(t *testing.T) {
+	ClearCache()
+
 	var list1 []primitives.ValidatorIndex
 	seed1 := [32]byte{1, 128, 12}
 	seed2 := [32]byte{2, 128, 12}
@@ -47,6 +51,8 @@ func TestShuffleList_OK(t *testing.T) {
 }
 
 func TestSplitIndices_OK(t *testing.T) {
+	ClearCache()
+
 	var l []uint64
 	numValidators := uint64(64000)
 	for i := uint64(0); i < numValidators; i++ {
@@ -61,6 +67,8 @@ func TestSplitIndices_OK(t *testing.T) {
 }
 
 func TestShuffleList_Vs_ShuffleIndex(t *testing.T) {
+	ClearCache()
+
 	var list []primitives.ValidatorIndex
 	listSize := uint64(1000)
 	seed := [32]byte{123, 42}
@@ -125,6 +133,8 @@ func BenchmarkShuffleList(b *testing.B) {
 }
 
 func TestShuffledIndex(t *testing.T) {
+	ClearCache()
+
 	var list []primitives.ValidatorIndex
 	listSize := uint64(399)
 	for i := primitives.ValidatorIndex(0); uint64(i) < listSize; i++ {
@@ -147,6 +157,8 @@ func TestShuffledIndex(t *testing.T) {
 }
 
 func TestSplitIndicesAndOffset_OK(t *testing.T) {
+	ClearCache()
+
 	var l []uint64
 	validators := uint64(64000)
 	for i := uint64(0); i < validators; i++ {

beacon-chain/core/helpers/sync_committee_test.go

@@ -18,7 +18,7 @@ import (
 
 func TestIsCurrentEpochSyncCommittee_UsingCache(t *testing.T) {
 	ClearCache()
-	defer ClearCache()
+
 	validators := make([]*ethpb.Validator, params.BeaconConfig().SyncCommitteeSize)
 	syncCommittee := &ethpb.SyncCommittee{
 		AggregatePubkey: bytesutil.PadTo([]byte{}, params.BeaconConfig().BLSPubkeyLength),
@@ -49,7 +49,7 @@ func TestIsCurrentEpochSyncCommittee_UsingCache(t *testing.T) {
 
 func TestIsCurrentEpochSyncCommittee_UsingCommittee(t *testing.T) {
 	ClearCache()
-	defer ClearCache()
+
 	validators := make([]*ethpb.Validator, params.BeaconConfig().SyncCommitteeSize)
 	syncCommittee := &ethpb.SyncCommittee{
 		AggregatePubkey: bytesutil.PadTo([]byte{}, params.BeaconConfig().BLSPubkeyLength),
@@ -77,7 +77,7 @@ func TestIsCurrentEpochSyncCommittee_UsingCommittee(t *testing.T) {
 
 func TestIsCurrentEpochSyncCommittee_DoesNotExist(t *testing.T) {
 	ClearCache()
-	defer ClearCache()
+
 	validators := make([]*ethpb.Validator, params.BeaconConfig().SyncCommitteeSize)
 	syncCommittee := &ethpb.SyncCommittee{
 		AggregatePubkey: bytesutil.PadTo([]byte{}, params.BeaconConfig().BLSPubkeyLength),
@@ -105,7 +105,7 @@ func TestIsCurrentEpochSyncCommittee_DoesNotExist(t *testing.T) {
 
 func TestIsNextEpochSyncCommittee_UsingCache(t *testing.T) {
 	ClearCache()
-	defer ClearCache()
+
 	validators := make([]*ethpb.Validator, params.BeaconConfig().SyncCommitteeSize)
 	syncCommittee := &ethpb.SyncCommittee{
 		AggregatePubkey: bytesutil.PadTo([]byte{}, params.BeaconConfig().BLSPubkeyLength),
@@ -135,6 +135,8 @@ func TestIsNextEpochSyncCommittee_UsingCache(t *testing.T) {
 }
 
 func TestIsNextEpochSyncCommittee_UsingCommittee(t *testing.T) {
+	ClearCache()
+
 	validators := make([]*ethpb.Validator, params.BeaconConfig().SyncCommitteeSize)
 	syncCommittee := &ethpb.SyncCommittee{
 		AggregatePubkey: bytesutil.PadTo([]byte{}, params.BeaconConfig().BLSPubkeyLength),
@@ -161,6 +163,8 @@ func TestIsNextEpochSyncCommittee_UsingCommittee(t *testing.T) {
 }
 
 func TestIsNextEpochSyncCommittee_DoesNotExist(t *testing.T) {
+	ClearCache()
+
 	validators := make([]*ethpb.Validator, params.BeaconConfig().SyncCommitteeSize)
 	syncCommittee := &ethpb.SyncCommittee{
 		AggregatePubkey: bytesutil.PadTo([]byte{}, params.BeaconConfig().BLSPubkeyLength),
@@ -188,7 +192,7 @@ func TestIsNextEpochSyncCommittee_DoesNotExist(t *testing.T) {
 
 func TestCurrentEpochSyncSubcommitteeIndices_UsingCache(t *testing.T) {
 	ClearCache()
-	defer ClearCache()
+
 	validators := make([]*ethpb.Validator, params.BeaconConfig().SyncCommitteeSize)
 	syncCommittee := &ethpb.SyncCommittee{
 		AggregatePubkey: bytesutil.PadTo([]byte{}, params.BeaconConfig().BLSPubkeyLength),
@@ -219,7 +223,7 @@ func TestCurrentEpochSyncSubcommitteeIndices_UsingCache(t *testing.T) {
 
 func TestCurrentEpochSyncSubcommitteeIndices_UsingCommittee(t *testing.T) {
 	ClearCache()
-	defer ClearCache()
+
 	validators := make([]*ethpb.Validator, params.BeaconConfig().SyncCommitteeSize)
 	syncCommittee := &ethpb.SyncCommittee{
 		AggregatePubkey: bytesutil.PadTo([]byte{}, params.BeaconConfig().BLSPubkeyLength),
@@ -260,7 +264,7 @@ func TestCurrentEpochSyncSubcommitteeIndices_UsingCommittee(t *testing.T) {
 
 func TestCurrentEpochSyncSubcommitteeIndices_DoesNotExist(t *testing.T) {
 	ClearCache()
-	defer ClearCache()
+
 	validators := make([]*ethpb.Validator, params.BeaconConfig().SyncCommitteeSize)
 	syncCommittee := &ethpb.SyncCommittee{
 		AggregatePubkey: bytesutil.PadTo([]byte{}, params.BeaconConfig().BLSPubkeyLength),
@@ -288,7 +292,7 @@ func TestCurrentEpochSyncSubcommitteeIndices_DoesNotExist(t *testing.T) {
 
 func TestNextEpochSyncSubcommitteeIndices_UsingCache(t *testing.T) {
 	ClearCache()
-	defer ClearCache()
+
 	validators := make([]*ethpb.Validator, params.BeaconConfig().SyncCommitteeSize)
 	syncCommittee := &ethpb.SyncCommittee{
 		AggregatePubkey: bytesutil.PadTo([]byte{}, params.BeaconConfig().BLSPubkeyLength),
@@ -318,6 +322,8 @@ func TestNextEpochSyncSubcommitteeIndices_UsingCache(t *testing.T) {
 }
 
 func TestNextEpochSyncSubcommitteeIndices_UsingCommittee(t *testing.T) {
+	ClearCache()
+
 	validators := make([]*ethpb.Validator, params.BeaconConfig().SyncCommitteeSize)
 	syncCommittee := &ethpb.SyncCommittee{
 		AggregatePubkey: bytesutil.PadTo([]byte{}, params.BeaconConfig().BLSPubkeyLength),
@@ -345,7 +351,7 @@ func TestNextEpochSyncSubcommitteeIndices_UsingCommittee(t *testing.T) {
 
 func TestNextEpochSyncSubcommitteeIndices_DoesNotExist(t *testing.T) {
 	ClearCache()
-	defer ClearCache()
+
 	validators := make([]*ethpb.Validator, params.BeaconConfig().SyncCommitteeSize)
 	syncCommittee := &ethpb.SyncCommittee{
 		AggregatePubkey: bytesutil.PadTo([]byte{}, params.BeaconConfig().BLSPubkeyLength),
@@ -372,6 +378,8 @@ func TestNextEpochSyncSubcommitteeIndices_DoesNotExist(t *testing.T) {
 }
 
 func TestUpdateSyncCommitteeCache_BadSlot(t *testing.T) {
+	ClearCache()
+
 	state, err := state_native.InitializeFromProtoPhase0(&ethpb.BeaconState{
 		Slot: 1,
 	})
@@ -388,6 +396,8 @@ func TestUpdateSyncCommitteeCache_BadSlot(t *testing.T) {
 }
 
 func TestUpdateSyncCommitteeCache_BadRoot(t *testing.T) {
+	ClearCache()
+
 	state, err := state_native.InitializeFromProtoPhase0(&ethpb.BeaconState{
 		Slot:              primitives.Slot(params.BeaconConfig().EpochsPerSyncCommitteePeriod)*params.BeaconConfig().SlotsPerEpoch - 1,
 		LatestBlockHeader: &ethpb.BeaconBlockHeader{StateRoot: params.BeaconConfig().ZeroHash[:]},
@@ -399,7 +409,7 @@ func TestUpdateSyncCommitteeCache_BadRoot(t *testing.T) {
 
 func TestIsCurrentEpochSyncCommittee_SameBlockRoot(t *testing.T) {
 	ClearCache()
-	defer ClearCache()
+
 	validators := make([]*ethpb.Validator, params.BeaconConfig().SyncCommitteeSize)
 	syncCommittee := &ethpb.SyncCommittee{
 		AggregatePubkey: bytesutil.PadTo([]byte{}, params.BeaconConfig().BLSPubkeyLength),

beacon-chain/core/helpers/validators.go

@@ -9,6 +9,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/time"
+	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
@@ -271,11 +272,22 @@ func BeaconProposerIndex(ctx context.Context, state state.ReadOnlyBeaconState) (
 func cachedProposerIndexAtSlot(slot primitives.Slot, root [32]byte) (primitives.ValidatorIndex, error) {
 	proposerIndices, has := proposerIndicesCache.ProposerIndices(slots.ToEpoch(slot), root)
 	if !has {
-		cache.ProposerIndicesCacheMiss.Inc()
 		return 0, errProposerIndexMiss
 	}
 	if len(proposerIndices) != int(params.BeaconConfig().SlotsPerEpoch) {
-		cache.ProposerIndicesCacheMiss.Inc()
+		return 0, errProposerIndexMiss
+	}
+	return proposerIndices[slot%params.BeaconConfig().SlotsPerEpoch], nil
+}
+
+// ProposerIndexAtSlotFromCheckpoint returns the proposer index at the given
+// slot from the cache at the given checkpoint
+func ProposerIndexAtSlotFromCheckpoint(c *forkchoicetypes.Checkpoint, slot primitives.Slot) (primitives.ValidatorIndex, error) {
+	proposerIndices, has := proposerIndicesCache.IndicesFromCheckpoint(*c)
+	if !has {
+		return 0, errProposerIndexMiss
+	}
+	if len(proposerIndices) != int(params.BeaconConfig().SlotsPerEpoch) {
 		return 0, errProposerIndexMiss
 	}
 	return proposerIndices[slot%params.BeaconConfig().SlotsPerEpoch], nil
@@ -302,7 +314,7 @@ func BeaconProposerIndexAtSlot(ctx context.Context, state state.ReadOnlyBeaconSt
 				return pid, nil
 			}
 			if err := UpdateProposerIndicesInCache(ctx, state, e); err != nil {
-				return 0, errors.Wrap(err, "could not update committee cache")
+				return 0, errors.Wrap(err, "could not update proposer index cache")
 			}
 			pid, err = cachedProposerIndexAtSlot(slot, [32]byte(r))
 			if err == nil {

beacon-chain/core/helpers/validators_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/time"
+	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
 	state_native "github.com/prysmaticlabs/prysm/v4/beacon-chain/state/state-native"
 	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
@@ -178,8 +179,6 @@ func TestIsSlashableValidator_OK(t *testing.T) {
 
 func TestBeaconProposerIndex_OK(t *testing.T) {
 	params.SetupTestConfigCleanup(t)
-	ClearCache()
-	defer ClearCache()
 	c := params.BeaconConfig()
 	c.MinGenesisActiveValidatorCount = 16384
 	params.OverrideBeaconConfig(c)
@@ -223,9 +222,9 @@ func TestBeaconProposerIndex_OK(t *testing.T) {
 		},
 	}
 
-	defer ClearCache()
 	for _, tt := range tests {
 		ClearCache()
+
 		require.NoError(t, state.SetSlot(tt.slot))
 		result, err := BeaconProposerIndex(context.Background(), state)
 		require.NoError(t, err, "Failed to get shard and committees at slot")
@@ -234,9 +233,9 @@ func TestBeaconProposerIndex_OK(t *testing.T) {
 }
 
 func TestBeaconProposerIndex_BadState(t *testing.T) {
-	params.SetupTestConfigCleanup(t)
 	ClearCache()
-	defer ClearCache()
+
+	params.SetupTestConfigCleanup(t)
 	c := params.BeaconConfig()
 	c.MinGenesisActiveValidatorCount = 16384
 	params.OverrideBeaconConfig(c)
@@ -267,6 +266,8 @@ func TestBeaconProposerIndex_BadState(t *testing.T) {
 }
 
 func TestComputeProposerIndex_Compatibility(t *testing.T) {
+	ClearCache()
+
 	validators := make([]*ethpb.Validator, params.BeaconConfig().MinGenesisActiveValidatorCount)
 	for i := 0; i < len(validators); i++ {
 		validators[i] = &ethpb.Validator{
@@ -308,12 +309,16 @@ func TestComputeProposerIndex_Compatibility(t *testing.T) {
 }
 
 func TestDelayedActivationExitEpoch_OK(t *testing.T) {
+	ClearCache()
+
 	epoch := primitives.Epoch(9999)
 	wanted := epoch + 1 + params.BeaconConfig().MaxSeedLookahead
 	assert.Equal(t, wanted, ActivationExitEpoch(epoch))
 }
 
 func TestActiveValidatorCount_Genesis(t *testing.T) {
+	ClearCache()
+
 	c := 1000
 	validators := make([]*ethpb.Validator, c)
 	for i := 0; i < len(validators); i++ {
@@ -347,7 +352,6 @@ func TestChurnLimit_OK(t *testing.T) {
 		{validatorCount: 1000000, wantedChurn: 15 /* validatorCount/churnLimitQuotient */},
 		{validatorCount: 2000000, wantedChurn: 30 /* validatorCount/churnLimitQuotient */},
 	}
-	defer ClearCache()
 	for _, test := range tests {
 		ClearCache()
 
@@ -381,9 +385,6 @@ func TestChurnLimitDeneb_OK(t *testing.T) {
 		{1000000, params.BeaconConfig().MaxPerEpochActivationChurnLimit},
 		{2000000, params.BeaconConfig().MaxPerEpochActivationChurnLimit},
 	}
-
-	defer ClearCache()
-
 	for _, test := range tests {
 		ClearCache()
 
@@ -416,7 +417,7 @@ func TestChurnLimitDeneb_OK(t *testing.T) {
 // Test basic functionality of ActiveValidatorIndices without caching. This test will need to be
 // rewritten when releasing some cache flag.
 func TestActiveValidatorIndices(t *testing.T) {
-	farFutureEpoch := params.BeaconConfig().FarFutureEpoch
+	//farFutureEpoch := params.BeaconConfig().FarFutureEpoch
 	type args struct {
 		state *ethpb.BeaconState
 		epoch primitives.Epoch
@@ -427,7 +428,7 @@ func TestActiveValidatorIndices(t *testing.T) {
 		want      []primitives.ValidatorIndex
 		wantedErr string
 	}{
-		{
+		/*{
 			name: "all_active_epoch_10",
 			args: args{
 				state: &ethpb.BeaconState{
@@ -558,7 +559,7 @@ func TestActiveValidatorIndices(t *testing.T) {
 				epoch: 10,
 			},
 			want: []primitives.ValidatorIndex{0, 2, 3},
-		},
+		},*/
 		{
 			name: "impossible_zero_validators", // Regression test for issue #13051
 			args: args{
@@ -568,22 +569,21 @@ func TestActiveValidatorIndices(t *testing.T) {
 				},
 				epoch: 10,
 			},
-			wantedErr: "no active validator indices",
+			wantedErr: "state has nil validator slice",
 		},
 	}
-	defer ClearCache()
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			ClearCache()
+
 			s, err := state_native.InitializeFromProtoPhase0(tt.args.state)
 			require.NoError(t, err)
-			require.NoError(t, s.SetValidators(tt.args.state.Validators))
 			got, err := ActiveValidatorIndices(context.Background(), s, tt.args.epoch)
 			if tt.wantedErr != "" {
 				assert.ErrorContains(t, tt.wantedErr, err)
 				return
 			}
 			assert.DeepEqual(t, tt.want, got, "ActiveValidatorIndices()")
-			ClearCache()
 		})
 	}
 }
@@ -684,6 +684,8 @@ func TestComputeProposerIndex(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			ClearCache()
+
 			bState := &ethpb.BeaconState{Validators: tt.args.validators}
 			stTrie, err := state_native.InitializeFromProtoUnsafePhase0(bState)
 			require.NoError(t, err)
@@ -716,6 +718,8 @@ func TestIsEligibleForActivationQueue(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			ClearCache()
+
 			assert.Equal(t, tt.want, IsEligibleForActivationQueue(tt.validator), "IsEligibleForActivationQueue()")
 		})
 	}
@@ -743,6 +747,8 @@ func TestIsIsEligibleForActivation(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			ClearCache()
+
 			s, err := state_native.InitializeFromProtoPhase0(tt.state)
 			require.NoError(t, err)
 			assert.Equal(t, tt.want, IsEligibleForActivation(s, tt.validator), "IsEligibleForActivation()")
@@ -781,6 +787,8 @@ func computeProposerIndexWithValidators(validators []*ethpb.Validator, activeInd
 }
 
 func TestLastActivatedValidatorIndex_OK(t *testing.T) {
+	ClearCache()
+
 	beaconState, err := state_native.InitializeFromProtoPhase0(&ethpb.BeaconState{})
 	require.NoError(t, err)
 
@@ -802,3 +810,20 @@ func TestLastActivatedValidatorIndex_OK(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, index, primitives.ValidatorIndex(3))
 }
+
+func TestProposerIndexFromCheckpoint(t *testing.T) {
+	ClearCache()
+
+	e := primitives.Epoch(2)
+	r := [32]byte{'a'}
+	root := [32]byte{'b'}
+	ids := [32]primitives.ValidatorIndex{}
+	slot := primitives.Slot(69) // slot 5 in the Epoch
+	ids[5] = primitives.ValidatorIndex(19)
+	proposerIndicesCache.Set(e, r, ids)
+	c := &forkchoicetypes.Checkpoint{Root: root, Epoch: e - 1}
+	proposerIndicesCache.SetCheckpoint(*c, r)
+	id, err := ProposerIndexAtSlotFromCheckpoint(c, slot)
+	require.NoError(t, err)
+	require.Equal(t, ids[5], id)
+}

beacon-chain/core/helpers/weak_subjectivity.go

@@ -202,3 +202,14 @@ func ParseWeakSubjectivityInputString(wsCheckpointString string) (*v1alpha1.Chec
 		Root:  bRoot,
 	}, nil
 }
+
+// MinEpochsForBlockRequests computes the number of epochs of block history that we need to maintain,
+// relative to the current epoch, per the p2p specs. This is used to compute the slot where backfill is complete.
+// value defined:
+// https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/p2p-interface.md#configuration
+// MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2 (= 33024, ~5 months)
+// detailed rationale: https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/p2p-interface.md#why-are-blocksbyrange-requests-only-required-to-be-served-for-the-latest-min_epochs_for_block_requests-epochs
+func MinEpochsForBlockRequests() primitives.Epoch {
+	return params.BeaconConfig().MinValidatorWithdrawabilityDelay +
+		primitives.Epoch(params.BeaconConfig().ChurnLimitQuotient/2)
+}

beacon-chain/core/helpers/weak_subjectivity_test.go

@@ -48,6 +48,7 @@ func TestWeakSubjectivity_ComputeWeakSubjectivityPeriod(t *testing.T) {
 		t.Run(fmt.Sprintf("valCount: %d, avgBalance: %d", tt.valCount, tt.avgBalance), func(t *testing.T) {
 			// Reset committee cache - as we need to recalculate active validator set for each test.
 			helpers.ClearCache()
+
 			got, err := helpers.ComputeWeakSubjectivityPeriod(context.Background(), genState(t, tt.valCount, tt.avgBalance), params.BeaconConfig())
 			require.NoError(t, err)
 			assert.Equal(t, tt.want, got, "valCount: %v, avgBalance: %v", tt.valCount, tt.avgBalance)
@@ -177,6 +178,8 @@ func TestWeakSubjectivity_IsWithinWeakSubjectivityPeriod(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			helpers.ClearCache()
+
 			sr, _, e := tt.genWsCheckpoint()
 			got, err := helpers.IsWithinWeakSubjectivityPeriod(context.Background(), tt.epoch, tt.genWsState(), sr, e, params.BeaconConfig())
 			if tt.wantedErr != "" {
@@ -247,6 +250,8 @@ func TestWeakSubjectivity_ParseWeakSubjectivityInputString(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			helpers.ClearCache()
+
 			wsCheckpt, err := helpers.ParseWeakSubjectivityInputString(tt.input)
 			if tt.wantedErr != "" {
 				require.ErrorContains(t, tt.wantedErr, err)
@@ -281,3 +286,21 @@ func genState(t *testing.T, valCount, avgBalance uint64) state.BeaconState {
 
 	return beaconState
 }
+
+func TestMinEpochsForBlockRequests(t *testing.T) {
+	helpers.ClearCache()
+
+	params.SetActiveTestCleanup(t, params.MainnetConfig())
+	var expected primitives.Epoch = 33024
+	// expected value of 33024 via spec commentary:
+	// https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/p2p-interface.md#why-are-blocksbyrange-requests-only-required-to-be-served-for-the-latest-min_epochs_for_block_requests-epochs
+	//	MIN_EPOCHS_FOR_BLOCK_REQUESTS is calculated using the arithmetic from compute_weak_subjectivity_period found in the weak subjectivity guide. Specifically to find this max epoch range, we use the worst case event of a very large validator size (>= MIN_PER_EPOCH_CHURN_LIMIT * CHURN_LIMIT_QUOTIENT).
+	//
+	//	MIN_EPOCHS_FOR_BLOCK_REQUESTS = (
+	//	    MIN_VALIDATOR_WITHDRAWABILITY_DELAY
+	//	    + MAX_SAFETY_DECAY * CHURN_LIMIT_QUOTIENT // (2 * 100)
+	//	)
+	//
+	//	Where MAX_SAFETY_DECAY = 100 and thus MIN_EPOCHS_FOR_BLOCK_REQUESTS = 33024 (~5 months).
+	require.Equal(t, expected, helpers.MinEpochsForBlockRequests())
+}

beacon-chain/core/transition/bellatrix_transition_no_verify_sig_test.go

@@ -245,10 +245,10 @@ func createFullBellatrixBlockWithOperations(t *testing.T) (state.BeaconState,
 					ReceiptsRoot:  make([]byte, fieldparams.RootLength),
 					LogsBloom:     make([]byte, fieldparams.LogsBloomLength),
 					PrevRandao:    make([]byte, fieldparams.RootLength),
+					ExtraData:     make([]byte, 0),
 					BaseFeePerGas: bytesutil.PadTo([]byte{1, 2, 3, 4}, fieldparams.RootLength),
 					BlockHash:     make([]byte, fieldparams.RootLength),
 					Transactions:  make([][]byte, 0),
-					ExtraData:     make([]byte, 0),
 				},
 			},
 		},
@@ -284,11 +284,11 @@ func createFullCapellaBlockWithOperations(t *testing.T) (state.BeaconState,
 					ReceiptsRoot:  make([]byte, fieldparams.RootLength),
 					LogsBloom:     make([]byte, fieldparams.LogsBloomLength),
 					PrevRandao:    make([]byte, fieldparams.RootLength),
+					ExtraData:     make([]byte, 0),
 					BaseFeePerGas: bytesutil.PadTo([]byte{1, 2, 3, 4}, fieldparams.RootLength),
 					BlockHash:     make([]byte, fieldparams.RootLength),
 					Transactions:  make([][]byte, 0),
 					Withdrawals:   make([]*enginev1.Withdrawal, 0),
-					ExtraData:     make([]byte, 0),
 				},
 			},
 		},

beacon-chain/core/transition/state-bellatrix.go

@@ -209,6 +209,7 @@ func OptimizedGenesisBeaconStateBellatrix(genesisTime uint64, preState state.Bea
 			ReceiptsRoot:  make([]byte, 32),
 			LogsBloom:     make([]byte, 256),
 			PrevRandao:    make([]byte, 32),
+			ExtraData:     make([]byte, 0),
 			BaseFeePerGas: make([]byte, 32),
 			BlockHash:     make([]byte, 32),
 			Transactions:  make([][]byte, 0),
@@ -269,6 +270,7 @@ func EmptyGenesisStateBellatrix() (state.BeaconState, error) {
 			ReceiptsRoot:     make([]byte, 32),
 			LogsBloom:        make([]byte, 256),
 			PrevRandao:       make([]byte, 32),
+			ExtraData:        make([]byte, 0),
 			BaseFeePerGas:    make([]byte, 32),
 			BlockHash:        make([]byte, 32),
 			TransactionsRoot: make([]byte, 32),

beacon-chain/das/BUILD.bazel

@@ -0,0 +1,49 @@
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "availability.go",
+        "cache.go",
+        "iface.go",
+        "mock.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/v4/beacon-chain/das",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//beacon-chain/db/filesystem:go_default_library",
+        "//beacon-chain/verification:go_default_library",
+        "//config/fieldparams:go_default_library",
+        "//config/params:go_default_library",
+        "//consensus-types/blocks:go_default_library",
+        "//consensus-types/primitives:go_default_library",
+        "//runtime/logging:go_default_library",
+        "//runtime/version:go_default_library",
+        "//time/slots:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "availability_test.go",
+        "cache_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = [
+        "//beacon-chain/db/filesystem:go_default_library",
+        "//beacon-chain/verification:go_default_library",
+        "//config/fieldparams:go_default_library",
+        "//config/params:go_default_library",
+        "//consensus-types/blocks:go_default_library",
+        "//consensus-types/primitives:go_default_library",
+        "//encoding/bytesutil:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
+        "//testing/require:go_default_library",
+        "//testing/util:go_default_library",
+        "//time/slots:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+    ],
+)

beacon-chain/das/availability.go

@@ -0,0 +1,149 @@
+package das
+
+import (
+	"context"
+	"fmt"
+
+	errors "github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/verification"
+	"github.com/prysmaticlabs/prysm/v4/config/params"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
+	"github.com/prysmaticlabs/prysm/v4/runtime/logging"
+	"github.com/prysmaticlabs/prysm/v4/runtime/version"
+	"github.com/prysmaticlabs/prysm/v4/time/slots"
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	errMixedRoots = errors.New("BlobSidecars must all be for the same block")
+)
+
+// LazilyPersistentStore is an implementation of AvailabilityStore to be used when batch syncing.
+// This implementation will hold any blobs passed to Persist until the IsDataAvailable is called for their
+// block, at which time they will undergo full verification and be saved to the disk.
+type LazilyPersistentStore struct {
+	store    *filesystem.BlobStorage
+	cache    *cache
+	verifier BlobBatchVerifier
+}
+
+var _ AvailabilityStore = &LazilyPersistentStore{}
+
+// BlobBatchVerifier enables LazyAvailabilityStore to manage the verification process
+// going from ROBlob->VerifiedROBlob, while avoiding the decision of which individual verifications
+// to run and in what order. Since LazilyPersistentStore always tries to verify and save blobs only when
+// they are all available, the interface takes a slice of blobs, enabling the implementation to optimize
+// batch verification.
+type BlobBatchVerifier interface {
+	VerifiedROBlobs(ctx context.Context, blk blocks.ROBlock, sc []blocks.ROBlob) ([]blocks.VerifiedROBlob, error)
+}
+
+// NewLazilyPersistentStore creates a new LazilyPersistentStore. This constructor should always be used
+// when creating a LazilyPersistentStore because it needs to initialize the cache under the hood.
+func NewLazilyPersistentStore(store *filesystem.BlobStorage, verifier BlobBatchVerifier) *LazilyPersistentStore {
+	return &LazilyPersistentStore{
+		store:    store,
+		cache:    newCache(),
+		verifier: verifier,
+	}
+}
+
+// Persist adds blobs to the working blob cache. Blobs stored in this cache will be persisted
+// for at least as long as the node is running. Once IsDataAvailable succeeds, all blobs referenced
+// by the given block are guaranteed to be persisted for the remainder of the retention period.
+func (s *LazilyPersistentStore) Persist(current primitives.Slot, sc ...blocks.ROBlob) error {
+	if len(sc) == 0 {
+		return nil
+	}
+	if len(sc) > 1 {
+		first := sc[0].BlockRoot()
+		for i := 1; i < len(sc); i++ {
+			if first != sc[i].BlockRoot() {
+				return errMixedRoots
+			}
+		}
+	}
+	if !params.WithinDAPeriod(slots.ToEpoch(sc[0].Slot()), slots.ToEpoch(current)) {
+		return nil
+	}
+	key := keyFromSidecar(sc[0])
+	entry := s.cache.ensure(key)
+	for i := range sc {
+		if err := entry.stash(&sc[i]); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// IsDataAvailable returns nil if all the commitments in the given block are persisted to the db and have been verified.
+// BlobSidecars already in the db are assumed to have been previously verified against the block.
+func (s *LazilyPersistentStore) IsDataAvailable(ctx context.Context, current primitives.Slot, b blocks.ROBlock) error {
+	blockCommitments, err := commitmentsToCheck(b, current)
+	if err != nil {
+		return errors.Wrapf(err, "could check data availability for block %#x", b.Root())
+	}
+	// Return early for blocks that are pre-deneb or which do not have any commitments.
+	if blockCommitments.count() == 0 {
+		return nil
+	}
+
+	key := keyFromBlock(b)
+	entry := s.cache.ensure(key)
+	defer s.cache.delete(key)
+	root := b.Root()
+	// Verify we have all the expected sidecars, and fail fast if any are missing or inconsistent.
+	// We don't try to salvage problematic batches because this indicates a misbehaving peer and we'd rather
+	// ignore their response and decrease their peer score.
+	sidecars, err := entry.filter(root, blockCommitments)
+	if err != nil {
+		return errors.Wrap(err, "incomplete BlobSidecar batch")
+	}
+	// Do thorough verifications of each BlobSidecar for the block.
+	// Same as above, we don't save BlobSidecars if there are any problems with the batch.
+	vscs, err := s.verifier.VerifiedROBlobs(ctx, b, sidecars)
+	if err != nil {
+		var me verification.VerificationMultiError
+		ok := errors.As(err, &me)
+		if ok {
+			fails := me.Failures()
+			lf := make(log.Fields, len(fails))
+			for i := range fails {
+				lf[fmt.Sprintf("fail_%d", i)] = fails[i].Error()
+			}
+			log.WithFields(lf).WithFields(logging.BlockFieldsFromBlob(sidecars[0])).
+				Debug("invalid BlobSidecars received")
+		}
+		return errors.Wrapf(err, "invalid BlobSidecars received for block %#x", root)
+	}
+	// Ensure that each BlobSidecar is written to disk.
+	for i := range vscs {
+		if err := s.store.Save(vscs[i]); err != nil {
+			return errors.Wrapf(err, "failed to save BlobSidecar index %d for block %#x", vscs[i].Index, root)
+		}
+	}
+	// All BlobSidecars are persisted - da check succeeds.
+	return nil
+}
+
+func commitmentsToCheck(b blocks.ROBlock, current primitives.Slot) (safeCommitmentArray, error) {
+	var ar safeCommitmentArray
+	if b.Version() < version.Deneb {
+		return ar, nil
+	}
+	// We are only required to check within MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS
+	if !params.WithinDAPeriod(slots.ToEpoch(b.Block().Slot()), slots.ToEpoch(current)) {
+		return ar, nil
+	}
+	kc, err := b.Block().Body().BlobKzgCommitments()
+	if err != nil {
+		return ar, err
+	}
+	if len(kc) > len(ar) {
+		return ar, errIndexOutOfBounds
+	}
+	copy(ar[:], kc)
+	return ar, nil
+}

beacon-chain/das/availability_test.go

@@ -0,0 +1,214 @@
+package das
+
+import (
+	"bytes"
+	"context"
+	"testing"
+
+	errors "github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/verification"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
+	"github.com/prysmaticlabs/prysm/v4/config/params"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
+	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
+	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/v4/testing/require"
+	"github.com/prysmaticlabs/prysm/v4/testing/util"
+	"github.com/prysmaticlabs/prysm/v4/time/slots"
+)
+
+func Test_commitmentsToCheck(t *testing.T) {
+	windowSlots, err := slots.EpochEnd(params.BeaconConfig().MinEpochsForBlobsSidecarsRequest)
+	require.NoError(t, err)
+	commits := [][]byte{
+		bytesutil.PadTo([]byte("a"), 48),
+		bytesutil.PadTo([]byte("b"), 48),
+		bytesutil.PadTo([]byte("c"), 48),
+		bytesutil.PadTo([]byte("d"), 48),
+	}
+	cases := []struct {
+		name    string
+		commits [][]byte
+		block   func(*testing.T) blocks.ROBlock
+		slot    primitives.Slot
+		err     error
+	}{
+		{
+			name: "pre deneb",
+			block: func(t *testing.T) blocks.ROBlock {
+				bb := util.NewBeaconBlockBellatrix()
+				sb, err := blocks.NewSignedBeaconBlock(bb)
+				require.NoError(t, err)
+				rb, err := blocks.NewROBlock(sb)
+				require.NoError(t, err)
+				return rb
+			},
+		},
+		{
+			name: "commitments within da",
+			block: func(t *testing.T) blocks.ROBlock {
+				d := util.NewBeaconBlockDeneb()
+				d.Block.Body.BlobKzgCommitments = commits
+				d.Block.Slot = 100
+				sb, err := blocks.NewSignedBeaconBlock(d)
+				require.NoError(t, err)
+				rb, err := blocks.NewROBlock(sb)
+				require.NoError(t, err)
+				return rb
+			},
+			commits: commits,
+			slot:    100,
+		},
+		{
+			name: "commitments outside da",
+			block: func(t *testing.T) blocks.ROBlock {
+				d := util.NewBeaconBlockDeneb()
+				// block is from slot 0, "current slot" is window size +1 (so outside the window)
+				d.Block.Body.BlobKzgCommitments = commits
+				sb, err := blocks.NewSignedBeaconBlock(d)
+				require.NoError(t, err)
+				rb, err := blocks.NewROBlock(sb)
+				require.NoError(t, err)
+				return rb
+			},
+			slot: windowSlots + 1,
+		},
+		{
+			name: "excessive commitments",
+			block: func(t *testing.T) blocks.ROBlock {
+				d := util.NewBeaconBlockDeneb()
+				d.Block.Slot = 100
+				// block is from slot 0, "current slot" is window size +1 (so outside the window)
+				d.Block.Body.BlobKzgCommitments = commits
+				// Double the number of commitments, assert that this is over the limit
+				d.Block.Body.BlobKzgCommitments = append(commits, d.Block.Body.BlobKzgCommitments...)
+				sb, err := blocks.NewSignedBeaconBlock(d)
+				require.NoError(t, err)
+				rb, err := blocks.NewROBlock(sb)
+				require.NoError(t, err)
+				c, err := rb.Block().Body().BlobKzgCommitments()
+				require.NoError(t, err)
+				require.Equal(t, true, len(c) > fieldparams.MaxBlobsPerBlock)
+				return rb
+			},
+			slot: windowSlots + 1,
+			err:  errIndexOutOfBounds,
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			b := c.block(t)
+			co, err := commitmentsToCheck(b, c.slot)
+			if c.err != nil {
+				require.ErrorIs(t, err, c.err)
+			} else {
+				require.NoError(t, err)
+			}
+			require.Equal(t, len(c.commits), co.count())
+			for i := 0; i < len(c.commits); i++ {
+				require.Equal(t, true, bytes.Equal(c.commits[i], co[i]))
+			}
+		})
+	}
+}
+
+func daAlwaysSucceeds(_ [][]byte, _ []*ethpb.BlobSidecar) error {
+	return nil
+}
+
+type mockDA struct {
+	t   *testing.T
+	scs []blocks.ROBlob
+	err error
+}
+
+func TestLazilyPersistent_Missing(t *testing.T) {
+	ctx := context.Background()
+	store := filesystem.NewEphemeralBlobStorage(t)
+
+	blk, scs := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, 1, 3)
+
+	mbv := &mockBlobBatchVerifier{t: t, scs: scs}
+	as := NewLazilyPersistentStore(store, mbv)
+
+	// Only one commitment persisted, should return error with other indices
+	require.NoError(t, as.Persist(1, scs[2]))
+	err := as.IsDataAvailable(ctx, 1, blk)
+	require.ErrorIs(t, err, errMissingSidecar)
+
+	// All but one persisted, return missing idx
+	require.NoError(t, as.Persist(1, scs[0]))
+	err = as.IsDataAvailable(ctx, 1, blk)
+	require.ErrorIs(t, err, errMissingSidecar)
+
+	// All persisted, return nil
+	require.NoError(t, as.Persist(1, scs...))
+
+	require.NoError(t, as.IsDataAvailable(ctx, 1, blk))
+}
+
+func TestLazilyPersistent_Mismatch(t *testing.T) {
+	ctx := context.Background()
+	store := filesystem.NewEphemeralBlobStorage(t)
+
+	blk, scs := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, 1, 3)
+
+	mbv := &mockBlobBatchVerifier{t: t, err: errors.New("kzg check should not run")}
+	scs[0].KzgCommitment = bytesutil.PadTo([]byte("nope"), 48)
+	as := NewLazilyPersistentStore(store, mbv)
+
+	// Only one commitment persisted, should return error with other indices
+	require.NoError(t, as.Persist(1, scs[0]))
+	err := as.IsDataAvailable(ctx, 1, blk)
+	require.NotNil(t, err)
+	require.ErrorIs(t, err, errCommitmentMismatch)
+}
+
+func TestLazyPersistOnceCommitted(t *testing.T) {
+	_, scs := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, 1, 6)
+	as := NewLazilyPersistentStore(filesystem.NewEphemeralBlobStorage(t), &mockBlobBatchVerifier{})
+	// stashes as expected
+	require.NoError(t, as.Persist(1, scs...))
+	// ignores duplicates
+	require.ErrorIs(t, as.Persist(1, scs...), ErrDuplicateSidecar)
+
+	// ignores index out of bound
+	scs[0].Index = 6
+	require.ErrorIs(t, as.Persist(1, scs[0]), errIndexOutOfBounds)
+
+	_, more := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, 1, 4)
+	// ignores sidecars before the retention period
+	slotOOB, err := slots.EpochStart(params.BeaconConfig().MinEpochsForBlobsSidecarsRequest)
+	require.NoError(t, err)
+	require.NoError(t, as.Persist(32+slotOOB, more[0]))
+
+	// doesn't ignore new sidecars with a different block root
+	require.NoError(t, as.Persist(1, more...))
+}
+
+type mockBlobBatchVerifier struct {
+	t        *testing.T
+	scs      []blocks.ROBlob
+	err      error
+	verified map[[32]byte]primitives.Slot
+}
+
+var _ BlobBatchVerifier = &mockBlobBatchVerifier{}
+
+func (m *mockBlobBatchVerifier) VerifiedROBlobs(_ context.Context, _ blocks.ROBlock, scs []blocks.ROBlob) ([]blocks.VerifiedROBlob, error) {
+	require.Equal(m.t, len(scs), len(m.scs))
+	for i := range m.scs {
+		require.Equal(m.t, m.scs[i], scs[i])
+	}
+	vscs := verification.FakeVerifySliceForTest(m.t, scs)
+	return vscs, m.err
+}
+
+func (m *mockBlobBatchVerifier) MarkVerified(root [32]byte, slot primitives.Slot) {
+	if m.verified == nil {
+		m.verified = make(map[[32]byte]primitives.Slot)
+	}
+	m.verified[root] = slot
+}

beacon-chain/das/cache.go

@@ -0,0 +1,117 @@
+package das
+
+import (
+	"bytes"
+
+	"github.com/pkg/errors"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
+)
+
+var (
+	ErrDuplicateSidecar   = errors.New("duplicate sidecar stashed in AvailabilityStore")
+	errIndexOutOfBounds   = errors.New("sidecar.index > MAX_BLOBS_PER_BLOCK")
+	errCommitmentMismatch = errors.New("KzgCommitment of sidecar in cache did not match block commitment")
+	errMissingSidecar     = errors.New("no sidecar in cache for block commitment")
+)
+
+// cacheKey includes the slot so that we can easily iterate through the cache and compare
+// slots for eviction purposes. Whether the input is the block or the sidecar, we always have
+// the root+slot when interacting with the cache, so it isn't an inconvenience to use both.
+type cacheKey struct {
+	slot primitives.Slot
+	root [32]byte
+}
+
+type cache struct {
+	entries map[cacheKey]*cacheEntry
+}
+
+func newCache() *cache {
+	return &cache{entries: make(map[cacheKey]*cacheEntry)}
+}
+
+// keyFromSidecar is a convenience method for constructing a cacheKey from a BlobSidecar value.
+func keyFromSidecar(sc blocks.ROBlob) cacheKey {
+	return cacheKey{slot: sc.Slot(), root: sc.BlockRoot()}
+}
+
+// keyFromBlock is a convenience method for constructing a cacheKey from a ROBlock value.
+func keyFromBlock(b blocks.ROBlock) cacheKey {
+	return cacheKey{slot: b.Block().Slot(), root: b.Root()}
+}
+
+// ensure returns the entry for the given key, creating it if it isn't already present.
+func (c *cache) ensure(key cacheKey) *cacheEntry {
+	e, ok := c.entries[key]
+	if !ok {
+		e = &cacheEntry{}
+		c.entries[key] = e
+	}
+	return e
+}
+
+// delete removes the cache entry from the cache.
+func (c *cache) delete(key cacheKey) {
+	delete(c.entries, key)
+}
+
+// cacheEntry holds a fixed-length cache of BlobSidecars.
+type cacheEntry struct {
+	scs [fieldparams.MaxBlobsPerBlock]*blocks.ROBlob
+}
+
+// stash adds an item to the in-memory cache of BlobSidecars.
+// Only the first BlobSidecar of a given Index will be kept in the cache.
+// stash will return an error if the given blob is already in the cache, or if the Index is out of bounds.
+func (e *cacheEntry) stash(sc *blocks.ROBlob) error {
+	if sc.Index >= fieldparams.MaxBlobsPerBlock {
+		return errors.Wrapf(errIndexOutOfBounds, "index=%d", sc.Index)
+	}
+	if e.scs[sc.Index] != nil {
+		return errors.Wrapf(ErrDuplicateSidecar, "root=%#x, index=%d, commitment=%#x", sc.BlockRoot(), sc.Index, sc.KzgCommitment)
+	}
+	e.scs[sc.Index] = sc
+	return nil
+}
+
+// filter evicts sidecars that are not committed to by the block and returns custom
+// errors if the cache is missing any of the commitments, or if the commitments in
+// the cache do not match those found in the block. If err is nil, then all expected
+// commitments were found in the cache and the sidecar slice return value can be used
+// to perform a DA check against the cached sidecars.
+func (e *cacheEntry) filter(root [32]byte, kc safeCommitmentArray) ([]blocks.ROBlob, error) {
+	scs := make([]blocks.ROBlob, kc.count())
+	for i := uint64(0); i < fieldparams.MaxBlobsPerBlock; i++ {
+		if kc[i] == nil {
+			if e.scs[i] != nil {
+				return nil, errors.Wrapf(errCommitmentMismatch, "root=%#x, index=%#x, commitment=%#x, no block commitment", root, i, e.scs[i].KzgCommitment)
+			}
+			continue
+		}
+
+		if e.scs[i] == nil {
+			return nil, errors.Wrapf(errMissingSidecar, "root=%#x, index=%#x", root, i)
+		}
+		if !bytes.Equal(kc[i], e.scs[i].KzgCommitment) {
+			return nil, errors.Wrapf(errCommitmentMismatch, "root=%#x, index=%#x, commitment=%#x, block commitment=%#x", root, i, e.scs[i].KzgCommitment, kc[i])
+		}
+		scs[i] = *e.scs[i]
+	}
+
+	return scs, nil
+}
+
+// safeCommitmentArray is a fixed size array of commitment byte slices. This is helpful for avoiding
+// gratuitous bounds checks.
+type safeCommitmentArray [fieldparams.MaxBlobsPerBlock][]byte
+
+func (s safeCommitmentArray) count() int {
+	for i := range s {
+		if s[i] == nil {
+			return i
+		}
+	}
+	return fieldparams.MaxBlobsPerBlock
+}

beacon-chain/das/cache_test.go

@@ -0,0 +1,25 @@
+package das
+
+import (
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
+	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
+	"github.com/prysmaticlabs/prysm/v4/testing/require"
+)
+
+func TestCacheEnsureDelete(t *testing.T) {
+	c := newCache()
+	require.Equal(t, 0, len(c.entries))
+	root := bytesutil.ToBytes32([]byte("root"))
+	slot := primitives.Slot(1234)
+	k := cacheKey{root: root, slot: slot}
+	entry := c.ensure(k)
+	require.Equal(t, 1, len(c.entries))
+	require.Equal(t, c.entries[k], entry)
+
+	c.delete(k)
+	require.Equal(t, 0, len(c.entries))
+	var nilEntry *cacheEntry
+	require.Equal(t, nilEntry, c.entries[k])
+}

beacon-chain/das/iface.go

@@ -0,0 +1,19 @@
+package das
+
+import (
+	"context"
+
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
+)
+
+// AvailabilityStore describes a component that can verify and save sidecars for a given block, and confirm previously
+// verified and saved sidecars.
+// Persist guarantees that the sidecar will be available to perform a DA check
+// for the life of the beacon node process.
+// IsDataAvailable guarantees that all blobs committed to in the block have been
+// durably persisted before returning a non-error value.
+type AvailabilityStore interface {
+	IsDataAvailable(ctx context.Context, current primitives.Slot, b blocks.ROBlock) error
+	Persist(current primitives.Slot, sc ...blocks.ROBlob) error
+}