Ignore blocks if their parent is older than justified

* scaffolding for verification package

* WIP blob verification methods

* lock wrapper for safer forkchoice sharing

* more solid cache and verification designs; adding tests

* more test coverage, adding missing cache files

* clearer func name

* remove forkchoice borrower (it's in another PR)

* revert temporary interface experiment

* lint

* nishant feedback

* add comments with spec text to all verifications

* some comments on public methods

* invert confusing verification name

* deep source

* remove cache from ProposerCache + gaz

* more consistently early return on error paths

* messed up the test with the wrong config value

* terence naming feedback

* tests on BeginsAt

* lint

* deep source...

* name errors after failure, not expectation

* deep sooource

* check len()==0 instead of nil so empty lists work

* update test for EIP-7044

---------

Co-authored-by: Kasey Kirkham <kasey@users.noreply.github.com>

.bazelversion

@@ -1 +1 @@
-6.3.2
+6.4.0

.buildkite-bazelrc

@@ -9,8 +9,8 @@
 #build:remote-cache --strategy=Genrule=standalone
 
 # Prysm specific remote-cache properties.
-#build:remote-cache --disk_cache=
-build:remote-cache --remote_download_toplevel
+build:remote-cache --remote_download_minimal
+build:remote-cache --experimental_remote_build_event_upload=minimal
 build:remote-cache --remote_cache=grpc://bazel-remote-cache:9092
 # Does not work with rules_oci. See https://github.com/bazel-contrib/rules_oci/issues/292
 #build:remote-cache --experimental_remote_downloader=grpc://bazel-remote-cache:9092
@@ -29,7 +29,10 @@ build --experimental_use_hermetic_linux_sandbox
 # Import workspace options.
 import %workspace%/.bazelrc
 
-startup --host_jvm_args=-Xmx4g --host_jvm_args=-Xms2g
+# Enable blake3 once it is supported in remote cache. See: https://github.com/buchgr/bazel-remote/issues/710
+# startup --digest_function=blake3
+
+startup --host_jvm_args=-Xmx8g --host_jvm_args=-Xms4g
 build --experimental_strict_action_env
 build --sandbox_tmpfs_path=/tmp
 build --verbose_failures
@@ -39,6 +42,7 @@ build --curses=no --color=no
 build --keep_going
 build --test_output=errors
 build --flaky_test_attempts=5
+build --build_runfile_links=false # Only build runfile symlink forest when required by local action, test, or run command.
 # Disabled race detection due to unstable test results under constrained environment build kite
 # build --features=race
 

.github/workflows/go.yml

@@ -5,6 +5,8 @@ on:
     branches: [ master ]
   pull_request:
     branches: [ '*' ]
+  merge_group: 
+    types: [checks_requested]
 
 jobs:
   formatting:
@@ -34,7 +36,7 @@ jobs:
         run: | # https://github.com/securego/gosec/issues/469
           export PATH=$PATH:$(go env GOPATH)/bin
           go install github.com/securego/gosec/v2/cmd/gosec@v2.15.0
-          gosec -exclude=G307 -exclude-dir=crypto/bls/herumi ./...
+          gosec -exclude-generated -exclude=G307 -exclude-dir=crypto/bls/herumi ./...
 
   lint:
     name: Lint

WORKSPACE

@@ -244,8 +244,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "91434d5fd5e1c6eb7b0174fed2afe25e09bddf00e1e4c431db931b2cee4e7773",
-    url = "https://github.com/eth-clients/slashing-protection-interchange-tests/archive/b8413ca42dc92308019d0d4db52c87e9e125c4e9.tar.gz",
+    sha256 = "516d551cfb3e50e4ac2f42db0992f4ceb573a7cb1616d727a725c8161485329f",
+    url = "https://github.com/eth-clients/slashing-protection-interchange-tests/archive/refs/tags/v5.3.0.tar.gz",
 )
 
 http_archive(

api/BUILD.bazel

@@ -2,7 +2,10 @@ load("@prysm//tools/go:def.bzl", "go_library")
 
 go_library(
     name = "go_default_library",
-    srcs = ["headers.go"],
+    srcs = [
+        "constants.go",
+        "headers.go",
+    ],
     importpath = "github.com/prysmaticlabs/prysm/v4/api",
     visibility = ["//visibility:public"],
 )

api/client/beacon/BUILD.bazel

@@ -12,10 +12,10 @@ go_library(
     deps = [
         "//api/client:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
-        "//beacon-chain/rpc/apimiddleware:go_default_library",
         "//beacon-chain/rpc/eth/beacon:go_default_library",
         "//beacon-chain/rpc/eth/config:go_default_library",
         "//beacon-chain/rpc/eth/shared:go_default_library",
+        "//beacon-chain/rpc/prysm/beacon:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//consensus-types/interfaces:go_default_library",
         "//consensus-types/primitives:go_default_library",

api/client/beacon/client.go

@@ -16,10 +16,10 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v4/api/client"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/apimiddleware"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/beacon"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/config"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
+	apibeacon "github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/prysm/beacon"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
 	"github.com/prysmaticlabs/prysm/v4/network/forks"
@@ -31,7 +31,7 @@ const (
 	getSignedBlockPath       = "/eth/v2/beacon/blocks"
 	getBlockRootPath         = "/eth/v1/beacon/blocks/{{.Id}}/root"
 	getForkForStatePath      = "/eth/v1/beacon/states/{{.Id}}/fork"
-	getWeakSubjectivityPath  = "/eth/v1/beacon/weak_subjectivity"
+	getWeakSubjectivityPath  = "/prysm/v1/beacon/weak_subjectivity"
 	getForkSchedulePath      = "/eth/v1/config/fork_schedule"
 	getConfigSpecPath        = "/eth/v1/config/spec"
 	getStatePath             = "/eth/v2/debug/beacon/states"
@@ -258,16 +258,16 @@ func (c *Client) GetWeakSubjectivity(ctx context.Context) (*WeakSubjectivityData
 	if err != nil {
 		return nil, err
 	}
-	v := &apimiddleware.WeakSubjectivityResponse{}
+	v := &apibeacon.GetWeakSubjectivityResponse{}
 	err = json.Unmarshal(body, v)
 	if err != nil {
 		return nil, err
 	}
-	epoch, err := strconv.ParseUint(v.Data.Checkpoint.Epoch, 10, 64)
+	epoch, err := strconv.ParseUint(v.Data.WsCheckpoint.Epoch, 10, 64)
 	if err != nil {
 		return nil, err
 	}
-	blockRoot, err := hexutil.Decode(v.Data.Checkpoint.Root)
+	blockRoot, err := hexutil.Decode(v.Data.WsCheckpoint.Root)
 	if err != nil {
 		return nil, err
 	}
@@ -305,7 +305,7 @@ func (c *Client) SubmitChangeBLStoExecution(ctx context.Context, request []*shar
 	if resp.StatusCode != http.StatusOK {
 		decoder := json.NewDecoder(resp.Body)
 		decoder.DisallowUnknownFields()
-		errorJson := &apimiddleware.IndexedVerificationFailureErrorJson{}
+		errorJson := &shared.IndexedVerificationFailureError{}
 		if err := decoder.Decode(errorJson); err != nil {
 			return errors.Wrapf(err, "failed to decode error JSON for %s", resp.Request.URL)
 		}

api/client/builder/BUILD.bazel

@@ -20,8 +20,6 @@ go_library(
         "//encoding/bytesutil:go_default_library",
         "//math:go_default_library",
         "//monitoring/tracing:go_default_library",
-        "//network:go_default_library",
-        "//network/authorization:go_default_library",
         "//proto/engine/v1:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//runtime/version:go_default_library",

api/client/builder/bid.go

@@ -6,7 +6,6 @@ import (
 	consensus_types "github.com/prysmaticlabs/prysm/v4/consensus-types"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
-	enginev1 "github.com/prysmaticlabs/prysm/v4/proto/engine/v1"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v4/runtime/version"
 )
@@ -22,7 +21,7 @@ type SignedBid interface {
 // Bid is an interface describing the method set of a builder bid.
 type Bid interface {
 	Header() (interfaces.ExecutionData, error)
-	BlindedBlobsBundle() (*enginev1.BlindedBlobsBundle, error)
+	BlobKzgCommitments() ([][]byte, error)
 	Value() []byte
 	Pubkey() []byte
 	Version() int
@@ -115,9 +114,9 @@ func (b builderBid) Header() (interfaces.ExecutionData, error) {
 	return blocks.WrappedExecutionPayloadHeader(b.p.Header)
 }
 
-// BlindedBlobsBundle --
-func (b builderBid) BlindedBlobsBundle() (*enginev1.BlindedBlobsBundle, error) {
-	return nil, errors.New("blinded blobs bundle not available before Deneb")
+// BlobKzgCommitments --
+func (b builderBid) BlobKzgCommitments() ([][]byte, error) {
+	return [][]byte{}, errors.New("blob kzg commitments not available before Deneb")
 }
 
 // Version --
@@ -169,9 +168,9 @@ func (b builderBidCapella) Header() (interfaces.ExecutionData, error) {
 	return blocks.WrappedExecutionPayloadHeaderCapella(b.p.Header, blocks.PayloadValueToGwei(b.p.Value))
 }
 
-// BlindedBlobsBundle --
-func (b builderBidCapella) BlindedBlobsBundle() (*enginev1.BlindedBlobsBundle, error) {
-	return nil, errors.New("blinded blobs bundle not available before Deneb")
+// BlobKzgCommitments --
+func (b builderBidCapella) BlobKzgCommitments() ([][]byte, error) {
+	return [][]byte{}, errors.New("blob kzg commitments not available before Deneb")
 }
 
 // Version --
@@ -253,9 +252,9 @@ func (b builderBidDeneb) Header() (interfaces.ExecutionData, error) {
 	return blocks.WrappedExecutionPayloadHeaderDeneb(b.p.Header, blocks.PayloadValueToGwei(b.p.Value))
 }
 
-// BlindedBlobsBundle --
-func (b builderBidDeneb) BlindedBlobsBundle() (*enginev1.BlindedBlobsBundle, error) {
-	return b.p.BlindedBlobsBundle, nil
+// BlobKzgCommitments --
+func (b builderBidDeneb) BlobKzgCommitments() ([][]byte, error) {
+	return b.p.BlobKzgCommitments, nil
 }
 
 type signedBuilderBidDeneb struct {

api/client/builder/client.go

@@ -19,8 +19,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
 	"github.com/prysmaticlabs/prysm/v4/monitoring/tracing"
-	"github.com/prysmaticlabs/prysm/v4/network"
-	"github.com/prysmaticlabs/prysm/v4/network/authorization"
 	v1 "github.com/prysmaticlabs/prysm/v4/proto/engine/v1"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v4/runtime/version"
@@ -89,7 +87,7 @@ type BuilderClient interface {
 	NodeURL() string
 	GetHeader(ctx context.Context, slot primitives.Slot, parentHash [32]byte, pubkey [48]byte) (SignedBid, error)
 	RegisterValidator(ctx context.Context, svr []*ethpb.SignedValidatorRegistrationV1) error
-	SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlySignedBeaconBlock, blobs []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error)
+	SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error)
 	Status(ctx context.Context) error
 }
 
@@ -104,8 +102,7 @@ type Client struct {
 // `host` is the base host + port used to construct request urls. This value can be
 // a URL string, or NewClient will assume an http endpoint if just `host:port` is used.
 func NewClient(host string, opts ...ClientOpt) (*Client, error) {
-	endpoint := covertEndPoint(host)
-	u, err := urlForHost(endpoint.Url)
+	u, err := urlForHost(host)
 	if err != nil {
 		return nil, err
 	}
@@ -121,8 +118,7 @@ func NewClient(host string, opts ...ClientOpt) (*Client, error) {
 
 func urlForHost(h string) (*url.URL, error) {
 	// try to parse as url (being permissive)
-	u, err := url.Parse(h)
-	if err == nil && u.Host != "" {
+	if u, err := url.Parse(h); err == nil && u.Host != "" {
 		return u, nil
 	}
 	// try to parse as host:port
@@ -140,7 +136,7 @@ func (c *Client) NodeURL() string {
 
 type reqOption func(*http.Request)
 
-// do is a generic, opinionated request function to reduce boilerplate amongst the methods in this package api/client/builder/types.go.
+// do is a generic, opinionated request function to reduce boilerplate amongst the methods in this package api/client/builder.
 func (c *Client) do(ctx context.Context, method string, path string, body io.Reader, opts ...reqOption) (res []byte, err error) {
 	ctx, span := trace.StartSpan(ctx, "builder.client.do")
 	defer func() {
@@ -291,7 +287,7 @@ func (c *Client) RegisterValidator(ctx context.Context, svr []*ethpb.SignedValid
 
 // SubmitBlindedBlock calls the builder API endpoint that binds the validator to the builder and submits the block.
 // The response is the full execution payload used to create the blinded block.
-func (c *Client) SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlySignedBeaconBlock, blobs []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
+func (c *Client) SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
 	if !sb.IsBlinded() {
 		return nil, nil, errNotBlinded
 	}
@@ -375,9 +371,9 @@ func (c *Client) SubmitBlindedBlock(ctx context.Context, sb interfaces.ReadOnlyS
 		if err != nil {
 			return nil, nil, errors.Wrapf(err, "could not get protobuf block")
 		}
-		b, err := shared.SignedBlindedBeaconBlockContentsDenebFromConsensus(&ethpb.SignedBlindedBeaconBlockAndBlobsDeneb{SignedBlindedBlock: psb, SignedBlindedBlobSidecars: blobs})
+		b, err := shared.SignedBlindedBeaconBlockDenebFromConsensus(&ethpb.SignedBlindedBeaconBlockDeneb{Message: psb.Message, Signature: bytesutil.SafeCopyBytes(psb.Signature)})
 		if err != nil {
-			return nil, nil, errors.Wrapf(err, "could not convert SignedBlindedBeaconBlockContentsDeneb to json marshalable type")
+			return nil, nil, errors.Wrapf(err, "could not convert SignedBlindedBeaconBlockDeneb to json marshalable type")
 		}
 		body, err := json.Marshal(b)
 		if err != nil {
@@ -457,12 +453,3 @@ func non200Err(response *http.Response) error {
 		return errors.Wrap(ErrNotOK, fmt.Sprintf("unsupported error code: %d", response.StatusCode))
 	}
 }
-
-func covertEndPoint(ep string) network.Endpoint {
-	return network.Endpoint{
-		Url: ep,
-		Auth: network.AuthorizationData{ // Auth is not used for builder.
-			Method: authorization.None,
-			Value:  "",
-		}}
-}

api/client/builder/client_test.go

@@ -12,10 +12,8 @@ import (
 	"strconv"
 	"testing"
 
-	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/prysmaticlabs/go-bitfield"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/eth/shared"
-	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	types "github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
@@ -270,55 +268,12 @@ func TestClient_GetHeader(t *testing.T) {
 		bidValue := bytesutil.ReverseByteOrder(bid.Value())
 		require.DeepEqual(t, bidValue, value.Bytes())
 		require.DeepEqual(t, big.NewInt(0).SetBytes(bidValue), value.Int)
-		bundle, err := bid.BlindedBlobsBundle()
+		kcgCommitments, err := bid.BlobKzgCommitments()
 		require.NoError(t, err)
-		require.Equal(t, len(bundle.BlobRoots) <= fieldparams.MaxBlobsPerBlock && len(bundle.BlobRoots) > 0, true)
-		for i := range bundle.BlobRoots {
-			require.Equal(t, len(bundle.BlobRoots[i]) == fieldparams.RootLength, true)
+		require.Equal(t, len(kcgCommitments) > 0, true)
+		for i := range kcgCommitments {
+			require.Equal(t, len(kcgCommitments[i]) == 48, true)
 		}
-		require.Equal(t, len(bundle.KzgCommitments) > 0, true)
-		for i := range bundle.KzgCommitments {
-			require.Equal(t, len(bundle.KzgCommitments[i]) == 48, true)
-		}
-		require.Equal(t, len(bundle.Proofs) > 0, true)
-		for i := range bundle.Proofs {
-			require.Equal(t, len(bundle.Proofs[i]) == 48, true)
-		}
-	})
-	t.Run("deneb, no bundle", func(t *testing.T) {
-		hc := &http.Client{
-			Transport: roundtrip(func(r *http.Request) (*http.Response, error) {
-				require.Equal(t, expectedPath, r.URL.Path)
-				return &http.Response{
-					StatusCode: http.StatusOK,
-					Body:       io.NopCloser(bytes.NewBufferString(testExampleHeaderResponseDenebNoBundle)),
-					Request:    r.Clone(ctx),
-				}, nil
-			}),
-		}
-		c := &Client{
-			hc:      hc,
-			baseURL: &url.URL{Host: "localhost:3500", Scheme: "http"},
-		}
-		h, err := c.GetHeader(ctx, slot, bytesutil.ToBytes32(parentHash), bytesutil.ToBytes48(pubkey))
-		require.NoError(t, err)
-		expectedWithdrawalsRoot := ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2")
-		bid, err := h.Message()
-		require.NoError(t, err)
-		bidHeader, err := bid.Header()
-		require.NoError(t, err)
-		withdrawalsRoot, err := bidHeader.WithdrawalsRoot()
-		require.NoError(t, err)
-		require.Equal(t, true, bytes.Equal(expectedWithdrawalsRoot, withdrawalsRoot))
-		value, err := stringToUint256("652312848583266388373324160190187140051835877600158453279131187530910662656")
-		require.NoError(t, err)
-		require.Equal(t, fmt.Sprintf("%#x", value.SSZBytes()), fmt.Sprintf("%#x", bid.Value()))
-		bidValue := bytesutil.ReverseByteOrder(bid.Value())
-		require.DeepEqual(t, bidValue, value.Bytes())
-		require.DeepEqual(t, big.NewInt(0).SetBytes(bidValue), value.Int)
-		bundle, err := bid.BlindedBlobsBundle()
-		require.NoError(t, err)
-		require.Equal(t, (*v1.BlindedBlobsBundle)(nil), bundle)
 	})
 
 	t.Run("unsupported version", func(t *testing.T) {
@@ -362,7 +317,7 @@ func TestSubmitBlindedBlock(t *testing.T) {
 		}
 		sbbb, err := blocks.NewSignedBeaconBlock(testSignedBlindedBeaconBlockBellatrix(t))
 		require.NoError(t, err)
-		ep, _, err := c.SubmitBlindedBlock(ctx, sbbb, nil)
+		ep, _, err := c.SubmitBlindedBlock(ctx, sbbb)
 		require.NoError(t, err)
 		require.Equal(t, true, bytes.Equal(ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"), ep.ParentHash()))
 		bfpg, err := stringToUint256("452312848583266388373324160190187140051835877600158453279131187530910662656")
@@ -388,7 +343,7 @@ func TestSubmitBlindedBlock(t *testing.T) {
 		}
 		sbb, err := blocks.NewSignedBeaconBlock(testSignedBlindedBeaconBlockCapella(t))
 		require.NoError(t, err)
-		ep, _, err := c.SubmitBlindedBlock(ctx, sbb, nil)
+		ep, _, err := c.SubmitBlindedBlock(ctx, sbb)
 		require.NoError(t, err)
 		withdrawals, err := ep.Withdrawals()
 		require.NoError(t, err)
@@ -399,18 +354,17 @@ func TestSubmitBlindedBlock(t *testing.T) {
 		assert.Equal(t, uint64(1), withdrawals[0].Amount)
 	})
 	t.Run("deneb", func(t *testing.T) {
-
-		test := testSignedBlindedBeaconBlockAndBlobsDeneb(t)
+		test := testSignedBlindedBeaconBlockDeneb(t)
 		hc := &http.Client{
 			Transport: roundtrip(func(r *http.Request) (*http.Response, error) {
 				require.Equal(t, postBlindedBeaconBlockPath, r.URL.Path)
 				require.Equal(t, "deneb", r.Header.Get("Eth-Consensus-Version"))
-				var req shared.SignedBlindedBeaconBlockContentsDeneb
+				var req shared.SignedBlindedBeaconBlockDeneb
 				err := json.NewDecoder(r.Body).Decode(&req)
 				require.NoError(t, err)
-				block, err := req.SignedBlindedBlock.ToConsensus()
+				block, err := req.ToConsensus()
 				require.NoError(t, err)
-				require.DeepEqual(t, block, test.SignedBlindedBlock)
+				require.DeepEqual(t, block, test)
 				return &http.Response{
 					StatusCode: http.StatusOK,
 					Body:       io.NopCloser(bytes.NewBufferString(testExampleExecutionPayloadDeneb)),
@@ -423,10 +377,10 @@ func TestSubmitBlindedBlock(t *testing.T) {
 			baseURL: &url.URL{Host: "localhost:3500", Scheme: "http"},
 		}
 
-		sbb, err := blocks.NewSignedBeaconBlock(test.SignedBlindedBlock)
+		sbb, err := blocks.NewSignedBeaconBlock(test)
 		require.NoError(t, err)
 
-		ep, blobBundle, err := c.SubmitBlindedBlock(ctx, sbb, test.SignedBlindedBlobSidecars)
+		ep, blobBundle, err := c.SubmitBlindedBlock(ctx, sbb)
 		require.NoError(t, err)
 		withdrawals, err := ep.Withdrawals()
 		require.NoError(t, err)
@@ -436,9 +390,6 @@ func TestSubmitBlindedBlock(t *testing.T) {
 		assert.DeepEqual(t, ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943"), withdrawals[0].Address)
 		assert.Equal(t, uint64(1), withdrawals[0].Amount)
 		require.NotNil(t, blobBundle)
-		require.Equal(t, hexutil.Encode(blobBundle.Blobs[0]), hexutil.Encode(make([]byte, fieldparams.BlobLength)))
-		require.Equal(t, hexutil.Encode(blobBundle.KzgCommitments[0]), "0x8dab030c51e16e84be9caab84ee3d0b8bbec1db4a0e4de76439da8424d9b957370a10a78851f97e4b54d2ce1ab0d686f")
-		require.Equal(t, hexutil.Encode(blobBundle.Proofs[0]), "0xb4021b0de10f743893d4f71e1bf830c019e832958efd6795baf2f83b8699a9eccc5dc99015d8d4d8ec370d0cc333c06a")
 	})
 	t.Run("mismatched versions, expected bellatrix got capella", func(t *testing.T) {
 		hc := &http.Client{
@@ -457,13 +408,13 @@ func TestSubmitBlindedBlock(t *testing.T) {
 		}
 		sbbb, err := blocks.NewSignedBeaconBlock(testSignedBlindedBeaconBlockBellatrix(t))
 		require.NoError(t, err)
-		_, _, err = c.SubmitBlindedBlock(ctx, sbbb, nil)
+		_, _, err = c.SubmitBlindedBlock(ctx, sbbb)
 		require.ErrorContains(t, "not a bellatrix payload", err)
 	})
 	t.Run("not blinded", func(t *testing.T) {
 		sbb, err := blocks.NewSignedBeaconBlock(&eth.SignedBeaconBlockBellatrix{Block: &eth.BeaconBlockBellatrix{Body: &eth.BeaconBlockBodyBellatrix{}}})
 		require.NoError(t, err)
-		_, _, err = (&Client{}).SubmitBlindedBlock(ctx, sbb, nil)
+		_, _, err = (&Client{}).SubmitBlindedBlock(ctx, sbb)
 		require.ErrorIs(t, err, errNotBlinded)
 	})
 }
@@ -753,91 +704,70 @@ func testSignedBlindedBeaconBlockCapella(t *testing.T) *eth.SignedBlindedBeaconB
 	}
 }
 
-func testSignedBlindedBeaconBlockAndBlobsDeneb(t *testing.T) *eth.SignedBlindedBeaconBlockAndBlobsDeneb {
+func testSignedBlindedBeaconBlockDeneb(t *testing.T) *eth.SignedBlindedBeaconBlockDeneb {
 	basebytes, err := shared.Uint256ToSSZBytes("14074904626401341155369551180448584754667373453244490859944217516317499064576")
 	if err != nil {
 		log.Error(err)
 	}
-	return &eth.SignedBlindedBeaconBlockAndBlobsDeneb{
-		SignedBlindedBlock: &eth.SignedBlindedBeaconBlockDeneb{
-			Message: &eth.BlindedBeaconBlockDeneb{
-				Slot:          1,
-				ProposerIndex: 1,
-				ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-				StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-				Body: &eth.BlindedBeaconBlockBodyDeneb{
-					RandaoReveal: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
-					Eth1Data: &eth.Eth1Data{
-						DepositRoot:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						DepositCount: 1,
-						BlockHash:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-					},
-					Graffiti: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-					ProposerSlashings: []*eth.ProposerSlashing{
-						{
-							Header_1: &eth.SignedBeaconBlockHeader{
-								Header: &eth.BeaconBlockHeader{
-									Slot:          1,
-									ProposerIndex: 1,
-									ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									BodyRoot:      ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-								},
-								Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+	return &eth.SignedBlindedBeaconBlockDeneb{
+		Message: &eth.BlindedBeaconBlockDeneb{
+			Slot:          1,
+			ProposerIndex: 1,
+			ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+			StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+			Body: &eth.BlindedBeaconBlockBodyDeneb{
+				RandaoReveal: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+				Eth1Data: &eth.Eth1Data{
+					DepositRoot:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					DepositCount: 1,
+					BlockHash:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+				},
+				Graffiti: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+				ProposerSlashings: []*eth.ProposerSlashing{
+					{
+						Header_1: &eth.SignedBeaconBlockHeader{
+							Header: &eth.BeaconBlockHeader{
+								Slot:          1,
+								ProposerIndex: 1,
+								ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+								StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+								BodyRoot:      ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
 							},
-							Header_2: &eth.SignedBeaconBlockHeader{
-								Header: &eth.BeaconBlockHeader{
-									Slot:          1,
-									ProposerIndex: 1,
-									ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									BodyRoot:      ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-								},
-								Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+							Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+						},
+						Header_2: &eth.SignedBeaconBlockHeader{
+							Header: &eth.BeaconBlockHeader{
+								Slot:          1,
+								ProposerIndex: 1,
+								ParentRoot:    ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+								StateRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+								BodyRoot:      ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
 							},
+							Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 						},
 					},
-					AttesterSlashings: []*eth.AttesterSlashing{
-						{
-							Attestation_1: &eth.IndexedAttestation{
-								AttestingIndices: []uint64{1},
-								Data: &eth.AttestationData{
-									Slot:            1,
-									CommitteeIndex:  1,
-									BeaconBlockRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									Source: &eth.Checkpoint{
-										Epoch: 1,
-										Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									},
-									Target: &eth.Checkpoint{
-										Epoch: 1,
-										Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									},
+				},
+				AttesterSlashings: []*eth.AttesterSlashing{
+					{
+						Attestation_1: &eth.IndexedAttestation{
+							AttestingIndices: []uint64{1},
+							Data: &eth.AttestationData{
+								Slot:            1,
+								CommitteeIndex:  1,
+								BeaconBlockRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+								Source: &eth.Checkpoint{
+									Epoch: 1,
+									Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
 								},
-								Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
-							},
-							Attestation_2: &eth.IndexedAttestation{
-								AttestingIndices: []uint64{1},
-								Data: &eth.AttestationData{
-									Slot:            1,
-									CommitteeIndex:  1,
-									BeaconBlockRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									Source: &eth.Checkpoint{
-										Epoch: 1,
-										Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									},
-									Target: &eth.Checkpoint{
-										Epoch: 1,
-										Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-									},
+								Target: &eth.Checkpoint{
+									Epoch: 1,
+									Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
 								},
-								Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 							},
+							Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 						},
-					},
-					Attestations: []*eth.Attestation{
-						{
-							AggregationBits: bitfield.Bitlist{0x01},
+						Attestation_2: &eth.IndexedAttestation{
+							AttestingIndices: []uint64{1},
 							Data: &eth.AttestationData{
 								Slot:            1,
 								CommitteeIndex:  1,
@@ -854,68 +784,72 @@ func testSignedBlindedBeaconBlockAndBlobsDeneb(t *testing.T) *eth.SignedBlindedB
 							Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 						},
 					},
-					Deposits: []*eth.Deposit{
-						{
-							Proof: [][]byte{ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2")},
-							Data: &eth.Deposit_Data{
-								PublicKey:             ezDecode(t, "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a"),
-								WithdrawalCredentials: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-								Amount:                1,
-								Signature:             ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+				},
+				Attestations: []*eth.Attestation{
+					{
+						AggregationBits: bitfield.Bitlist{0x01},
+						Data: &eth.AttestationData{
+							Slot:            1,
+							CommitteeIndex:  1,
+							BeaconBlockRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+							Source: &eth.Checkpoint{
+								Epoch: 1,
+								Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+							},
+							Target: &eth.Checkpoint{
+								Epoch: 1,
+								Root:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
 							},
 						},
+						Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 					},
-					VoluntaryExits: []*eth.SignedVoluntaryExit{
-						{
-							Exit: &eth.VoluntaryExit{
-								Epoch:          1,
-								ValidatorIndex: 1,
-							},
-							Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
+				},
+				Deposits: []*eth.Deposit{
+					{
+						Proof: [][]byte{ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2")},
+						Data: &eth.Deposit_Data{
+							PublicKey:             ezDecode(t, "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a"),
+							WithdrawalCredentials: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+							Amount:                1,
+							Signature:             ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 						},
 					},
-					SyncAggregate: &eth.SyncAggregate{
-						SyncCommitteeSignature: make([]byte, 96),
-						SyncCommitteeBits:      ezDecode(t, "0x6451e9f951ebf05edc01de67e593484b672877054f055903ff0df1a1a945cf30ca26bb4d4b154f94a1bc776bcf5d0efb3603e1f9b8ee2499ccdcfe2a18cef458"),
-					},
-					ExecutionPayloadHeader: &v1.ExecutionPayloadHeaderDeneb{
-						ParentHash:       ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						FeeRecipient:     ezDecode(t, "0xabcf8e0d4e9587369b2301d0790347320302cc09"),
-						StateRoot:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						ReceiptsRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						LogsBloom:        ezDecode(t, "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"),
-						PrevRandao:       ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						BlockNumber:      1,
-						GasLimit:         1,
-						GasUsed:          1,
-						Timestamp:        1,
-						ExtraData:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						BaseFeePerGas:    basebytes,
-						BlockHash:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						TransactionsRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						WithdrawalsRoot:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-						BlobGasUsed:      1,
-						ExcessBlobGas:    2,
+				},
+				VoluntaryExits: []*eth.SignedVoluntaryExit{
+					{
+						Exit: &eth.VoluntaryExit{
+							Epoch:          1,
+							ValidatorIndex: 1,
+						},
+						Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 					},
 				},
-			},
-			Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
-		},
-		SignedBlindedBlobSidecars: []*eth.SignedBlindedBlobSidecar{
-			{
-				Message: &eth.BlindedBlobSidecar{
-					BlockRoot:       ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-					Index:           0,
-					Slot:            1,
-					BlockParentRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
-					ProposerIndex:   1,
-					BlobRoot:        ezDecode(t, "0x24564723180fcb3d994104538d351c8dcbde12d541676bb736cf678018ca4739"),
-					KzgCommitment:   ezDecode(t, "0x8dab030c51e16e84be9caab84ee3d0b8bbec1db4a0e4de76439da8424d9b957370a10a78851f97e4b54d2ce1ab0d686f"),
-					KzgProof:        ezDecode(t, "0xb4021b0de10f743893d4f71e1bf830c019e832958efd6795baf2f83b8699a9eccc5dc99015d8d4d8ec370d0cc333c06a"),
+				SyncAggregate: &eth.SyncAggregate{
+					SyncCommitteeSignature: make([]byte, 96),
+					SyncCommitteeBits:      ezDecode(t, "0x6451e9f951ebf05edc01de67e593484b672877054f055903ff0df1a1a945cf30ca26bb4d4b154f94a1bc776bcf5d0efb3603e1f9b8ee2499ccdcfe2a18cef458"),
+				},
+				ExecutionPayloadHeader: &v1.ExecutionPayloadHeaderDeneb{
+					ParentHash:       ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					FeeRecipient:     ezDecode(t, "0xabcf8e0d4e9587369b2301d0790347320302cc09"),
+					StateRoot:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					ReceiptsRoot:     ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					LogsBloom:        ezDecode(t, "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"),
+					PrevRandao:       ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					BlockNumber:      1,
+					GasLimit:         1,
+					GasUsed:          1,
+					Timestamp:        1,
+					ExtraData:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					BaseFeePerGas:    basebytes,
+					BlockHash:        ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					TransactionsRoot: ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					WithdrawalsRoot:  ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"),
+					BlobGasUsed:      1,
+					ExcessBlobGas:    2,
 				},
-				Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 			},
 		},
+		Signature: ezDecode(t, "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"),
 	}
 }
 

api/client/builder/testing/mock.go

@@ -41,7 +41,7 @@ func (m MockClient) RegisterValidator(_ context.Context, svr []*ethpb.SignedVali
 }
 
 // SubmitBlindedBlock --
-func (MockClient) SubmitBlindedBlock(_ context.Context, _ interfaces.ReadOnlySignedBeaconBlock, _ []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
+func (MockClient) SubmitBlindedBlock(_ context.Context, _ interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
 	return nil, nil, nil
 }
 

api/client/builder/types.go

@@ -117,7 +117,7 @@ type VersionResponse struct {
 	Version string `json:"version"`
 }
 
-// ExecHeaderResponse is a JSON representation of  the builder API header response for Bellatrix.
+// ExecHeaderResponse is a JSON representation of the builder API header response for Bellatrix.
 type ExecHeaderResponse struct {
 	Version string `json:"version"`
 	Data    struct {
@@ -869,16 +869,16 @@ func (bb *BuilderBidDeneb) ToProto() (*eth.BuilderBidDeneb, error) {
 	if err != nil {
 		return nil, err
 	}
-	var bundle *v1.BlindedBlobsBundle
-	if bb.BlindedBlobsBundle != nil {
-		bundle, err = bb.BlindedBlobsBundle.ToProto()
-		if err != nil {
-			return nil, err
+	kzgCommitments := make([][]byte, len(bb.BlobKzgCommitments))
+	for i, commit := range bb.BlobKzgCommitments {
+		if len(commit) != fieldparams.BLSPubkeyLength {
+			return nil, fmt.Errorf("commitment length %d is not %d", len(commit), fieldparams.BLSPubkeyLength)
 		}
+		kzgCommitments[i] = bytesutil.SafeCopyBytes(commit)
 	}
 	return &eth.BuilderBidDeneb{
 		Header:             header,
-		BlindedBlobsBundle: bundle,
+		BlobKzgCommitments: kzgCommitments,
 		Value:              bytesutil.SafeCopyBytes(bb.Value.SSZBytes()),
 		Pubkey:             bytesutil.SafeCopyBytes(bb.Pubkey),
 	}, nil
@@ -887,42 +887,11 @@ func (bb *BuilderBidDeneb) ToProto() (*eth.BuilderBidDeneb, error) {
 // BuilderBidDeneb is a field of ExecHeaderResponseDeneb.
 type BuilderBidDeneb struct {
 	Header             *ExecutionPayloadHeaderDeneb `json:"header"`
-	BlindedBlobsBundle *BlindedBlobsBundle          `json:"blinded_blobs_bundle"`
+	BlobKzgCommitments []hexutil.Bytes              `json:"blob_kzg_commitments"`
 	Value              Uint256                      `json:"value"`
 	Pubkey             hexutil.Bytes                `json:"pubkey"`
 }
 
-// BlindedBlobsBundle is a field of BuilderBidDeneb and represents the blinded blobs of the associated header.
-type BlindedBlobsBundle struct {
-	KzgCommitments []hexutil.Bytes `json:"commitments"`
-	Proofs         []hexutil.Bytes `json:"proofs"`
-	BlobRoots      []hexutil.Bytes `json:"blob_roots"`
-}
-
-// ToProto creates a BlindedBlobsBundle Proto from BlindedBlobsBundle.
-func (r *BlindedBlobsBundle) ToProto() (*v1.BlindedBlobsBundle, error) {
-	kzg := make([][]byte, len(r.KzgCommitments))
-	for i := range kzg {
-		kzg[i] = bytesutil.SafeCopyBytes(r.KzgCommitments[i])
-	}
-
-	proofs := make([][]byte, len(r.Proofs))
-	for i := range proofs {
-		proofs[i] = bytesutil.SafeCopyBytes(r.Proofs[i])
-	}
-
-	blobRoots := make([][]byte, len(r.BlobRoots))
-	for i := range blobRoots {
-		blobRoots[i] = bytesutil.SafeCopyBytes(r.BlobRoots[i])
-	}
-
-	return &v1.BlindedBlobsBundle{
-		KzgCommitments: kzg,
-		Proofs:         proofs,
-		BlobRoots:      blobRoots,
-	}, nil
-}
-
 // ExecutionPayloadHeaderDeneb a field part of the BuilderBidDeneb.
 type ExecutionPayloadHeaderDeneb struct {
 	ParentHash       hexutil.Bytes `json:"parent_hash"`

api/client/builder/types_test.go

@@ -142,17 +142,9 @@ var testExampleHeaderResponseDeneb = `{
 		"blob_gas_used": "1",	
 		"excess_blob_gas": "2"	
       },
-      "blinded_blobs_bundle": {
-        "commitments": [
+      "blob_kzg_commitments": [
           "0x8dab030c51e16e84be9caab84ee3d0b8bbec1db4a0e4de76439da8424d9b957370a10a78851f97e4b54d2ce1ab0d686f"
-        ],
-        "proofs": [
-          "0xb4021b0de10f743893d4f71e1bf830c019e832958efd6795baf2f83b8699a9eccc5dc99015d8d4d8ec370d0cc333c06a"
-        ],
-        "blob_roots": [
-          "0x24564723180fcb3d994104538d351c8dcbde12d541676bb736cf678018ca4739"
-        ]
-      },
+	  ],
       "value": "652312848583266388373324160190187140051835877600158453279131187530910662656",
       "pubkey": "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a"
     },

api/constants.go

@@ -0,0 +1,3 @@
+package api
+
+const WebUrlPrefix = "/v2/validator/"

api/gateway/BUILD.bazel

@@ -14,7 +14,6 @@ go_library(
         "//validator:__subpackages__",
     ],
     deps = [
-        "//api/gateway/apimiddleware:go_default_library",
         "//runtime:go_default_library",
         "@com_github_gorilla_mux//:go_default_library",
         "@com_github_grpc_ecosystem_grpc_gateway_v2//runtime:go_default_library",
@@ -34,7 +33,6 @@ go_test(
     srcs = ["gateway_test.go"],
     embed = [":go_default_library"],
     deps = [
-        "//api/gateway/apimiddleware:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/require:go_default_library",

api/gateway/apimiddleware/BUILD.bazel

@@ -1,43 +0,0 @@
-load("@prysm//tools/go:def.bzl", "go_library", "go_test")
-
-go_library(
-    name = "go_default_library",
-    srcs = [
-        "api_middleware.go",
-        "log.go",
-        "param_handling.go",
-        "process_field.go",
-        "process_request.go",
-        "structs.go",
-    ],
-    importpath = "github.com/prysmaticlabs/prysm/v4/api/gateway/apimiddleware",
-    visibility = ["//visibility:public"],
-    deps = [
-        "//api:go_default_library",
-        "//api/grpc:go_default_library",
-        "//encoding/bytesutil:go_default_library",
-        "@com_github_ethereum_go_ethereum//common:go_default_library",
-        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
-        "@com_github_gorilla_mux//:go_default_library",
-        "@com_github_pkg_errors//:go_default_library",
-        "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_wealdtech_go_bytesutil//:go_default_library",
-    ],
-)
-
-go_test(
-    name = "go_default_test",
-    srcs = [
-        "param_handling_test.go",
-        "process_request_test.go",
-    ],
-    embed = [":go_default_library"],
-    deps = [
-        "//api:go_default_library",
-        "//api/grpc:go_default_library",
-        "//testing/assert:go_default_library",
-        "//testing/require:go_default_library",
-        "@com_github_gorilla_mux//:go_default_library",
-        "@com_github_sirupsen_logrus//hooks/test:go_default_library",
-    ],
-)

api/gateway/apimiddleware/api_middleware.go

@@ -1,265 +0,0 @@
-package apimiddleware
-
-import (
-	"net/http"
-	"reflect"
-	"time"
-
-	"github.com/gorilla/mux"
-)
-
-// ApiProxyMiddleware is a proxy between an Ethereum consensus API HTTP client and grpc-gateway.
-// The purpose of the proxy is to handle HTTP requests and gRPC responses in such a way that:
-//   - Ethereum consensus API requests can be handled by grpc-gateway correctly
-//   - gRPC responses can be returned as spec-compliant Ethereum consensus API responses
-type ApiProxyMiddleware struct {
-	GatewayAddress  string
-	EndpointCreator EndpointFactory
-	Timeout         time.Duration
-	router          *mux.Router
-}
-
-// EndpointFactory is responsible for creating new instances of Endpoint values.
-type EndpointFactory interface {
-	Create(path string) (*Endpoint, error)
-	Paths() []string
-	IsNil() bool
-}
-
-// Endpoint is a representation of an API HTTP endpoint that should be proxied by the middleware.
-type Endpoint struct {
-	Path               string          // The path of the HTTP endpoint.
-	GetResponse        interface{}     // The struct corresponding to the JSON structure used in a GET response.
-	PostRequest        interface{}     // The struct corresponding to the JSON structure used in a POST request.
-	PostResponse       interface{}     // The struct corresponding to the JSON structure used in a POST response.
-	DeleteRequest      interface{}     // The struct corresponding to the JSON structure used in a DELETE request.
-	DeleteResponse     interface{}     // The struct corresponding to the JSON structure used in a DELETE response.
-	RequestURLLiterals []string        // Names of URL parameters that should not be base64-encoded.
-	RequestQueryParams []QueryParam    // Query parameters of the request.
-	Err                ErrorJson       // The struct corresponding to the error that should be returned in case of a request failure.
-	Hooks              HookCollection  // A collection of functions that can be invoked at various stages of the request/response cycle.
-	CustomHandlers     []CustomHandler // Functions that will be executed instead of the default request/response behaviour.
-}
-
-// RunDefault expresses whether the default processing logic should be carried out after running a pre hook.
-type RunDefault bool
-
-// DefaultEndpoint returns an Endpoint with default configuration, e.g. DefaultErrorJson for error handling.
-func DefaultEndpoint() Endpoint {
-	return Endpoint{
-		Err: &DefaultErrorJson{},
-	}
-}
-
-// QueryParam represents a single query parameter's metadata.
-type QueryParam struct {
-	Name string
-	Hex  bool
-	Enum bool
-}
-
-// CustomHandler is a function that can be invoked at the very beginning of the request,
-// essentially replacing the whole default request/response logic with custom logic for a specific endpoint.
-type CustomHandler = func(m *ApiProxyMiddleware, endpoint Endpoint, w http.ResponseWriter, req *http.Request) (handled bool)
-
-// HookCollection contains hooks that can be used to amend the default request/response cycle with custom logic for a specific endpoint.
-type HookCollection struct {
-	OnPreDeserializeRequestBodyIntoContainer      func(endpoint *Endpoint, w http.ResponseWriter, req *http.Request) (RunDefault, ErrorJson)
-	OnPostDeserializeRequestBodyIntoContainer     func(endpoint *Endpoint, w http.ResponseWriter, req *http.Request) ErrorJson
-	OnPreDeserializeGrpcResponseBodyIntoContainer func([]byte, interface{}) (RunDefault, ErrorJson)
-	OnPreSerializeMiddlewareResponseIntoJson      func(interface{}) (RunDefault, []byte, ErrorJson)
-}
-
-// fieldProcessor applies the processing function f to a value when the tag is present on the field.
-type fieldProcessor struct {
-	tag string
-	f   func(value reflect.Value) error
-}
-
-// Run starts the proxy, registering all proxy endpoints.
-func (m *ApiProxyMiddleware) Run(gatewayRouter *mux.Router) {
-	for _, path := range m.EndpointCreator.Paths() {
-		gatewayRouter.HandleFunc(path, m.WithMiddleware(path))
-	}
-	m.router = gatewayRouter
-}
-
-// ServeHTTP for the proxy middleware.
-func (m *ApiProxyMiddleware) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-	m.router.ServeHTTP(w, req)
-}
-
-// WithMiddleware wraps the given endpoint handler with the middleware logic.
-func (m *ApiProxyMiddleware) WithMiddleware(path string) http.HandlerFunc {
-	return func(w http.ResponseWriter, req *http.Request) {
-		endpoint, err := m.EndpointCreator.Create(path)
-		if err != nil {
-			log.WithError(err).Errorf("Could not create endpoint for path: %s", path)
-			return
-		}
-
-		for _, handler := range endpoint.CustomHandlers {
-			if handler(m, *endpoint, w, req) {
-				return
-			}
-		}
-
-		if req.Method == "POST" {
-			if errJson := handlePostRequestForEndpoint(endpoint, w, req); errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-		}
-
-		if req.Method == "DELETE" && req.Body != http.NoBody {
-			if errJson := handleDeleteRequestForEndpoint(endpoint, req); errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-		}
-
-		if errJson := m.PrepareRequestForProxying(*endpoint, req); errJson != nil {
-			WriteError(w, errJson, nil)
-			return
-		}
-		grpcResp, errJson := m.ProxyRequest(req)
-		if errJson != nil {
-			WriteError(w, errJson, nil)
-			return
-		}
-		grpcRespBody, errJson := ReadGrpcResponseBody(grpcResp.Body)
-		if errJson != nil {
-			WriteError(w, errJson, nil)
-			return
-		}
-
-		var respJson []byte
-		if !GrpcResponseIsEmpty(grpcRespBody) {
-			respHasError, errJson := HandleGrpcResponseError(endpoint.Err, grpcResp, grpcRespBody, w)
-			if errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-			if respHasError {
-				return
-			}
-
-			var resp interface{}
-			if req.Method == "GET" {
-				resp = endpoint.GetResponse
-			} else if req.Method == "DELETE" {
-				resp = endpoint.DeleteResponse
-			} else {
-				resp = endpoint.PostResponse
-			}
-			if errJson := deserializeGrpcResponseBodyIntoContainerWrapped(endpoint, grpcRespBody, resp); errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-			if errJson := ProcessMiddlewareResponseFields(resp); errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-
-			respJson, errJson = serializeMiddlewareResponseIntoJsonWrapped(endpoint, respJson, resp)
-			if errJson != nil {
-				WriteError(w, errJson, nil)
-				return
-			}
-		}
-
-		if errJson := WriteMiddlewareResponseHeadersAndBody(grpcResp, respJson, w); errJson != nil {
-			WriteError(w, errJson, nil)
-			return
-		}
-		if errJson := Cleanup(grpcResp.Body); errJson != nil {
-			WriteError(w, errJson, nil)
-			return
-		}
-	}
-}
-
-func handlePostRequestForEndpoint(endpoint *Endpoint, w http.ResponseWriter, req *http.Request) ErrorJson {
-	if errJson := deserializeRequestBodyIntoContainerWrapped(endpoint, req, w); errJson != nil {
-		return errJson
-	}
-	if errJson := ProcessRequestContainerFields(endpoint.PostRequest); errJson != nil {
-		return errJson
-	}
-	return SetRequestBodyToRequestContainer(endpoint.PostRequest, req)
-}
-
-func handleDeleteRequestForEndpoint(endpoint *Endpoint, req *http.Request) ErrorJson {
-	if errJson := DeserializeRequestBodyIntoContainer(req.Body, endpoint.DeleteRequest); errJson != nil {
-		return errJson
-	}
-	if errJson := ProcessRequestContainerFields(endpoint.DeleteRequest); errJson != nil {
-		return errJson
-	}
-	return SetRequestBodyToRequestContainer(endpoint.DeleteRequest, req)
-}
-
-func deserializeRequestBodyIntoContainerWrapped(endpoint *Endpoint, req *http.Request, w http.ResponseWriter) ErrorJson {
-	runDefault := true
-	if endpoint.Hooks.OnPreDeserializeRequestBodyIntoContainer != nil {
-		run, errJson := endpoint.Hooks.OnPreDeserializeRequestBodyIntoContainer(endpoint, w, req)
-		if errJson != nil {
-			return errJson
-		}
-		if !run {
-			runDefault = false
-		}
-	}
-	if runDefault {
-		if errJson := DeserializeRequestBodyIntoContainer(req.Body, endpoint.PostRequest); errJson != nil {
-			return errJson
-		}
-	}
-	if endpoint.Hooks.OnPostDeserializeRequestBodyIntoContainer != nil {
-		if errJson := endpoint.Hooks.OnPostDeserializeRequestBodyIntoContainer(endpoint, w, req); errJson != nil {
-			return errJson
-		}
-	}
-	return nil
-}
-
-func deserializeGrpcResponseBodyIntoContainerWrapped(endpoint *Endpoint, grpcResponseBody []byte, resp interface{}) ErrorJson {
-	runDefault := true
-	if endpoint.Hooks.OnPreDeserializeGrpcResponseBodyIntoContainer != nil {
-		run, errJson := endpoint.Hooks.OnPreDeserializeGrpcResponseBodyIntoContainer(grpcResponseBody, resp)
-		if errJson != nil {
-			return errJson
-		}
-		if !run {
-			runDefault = false
-		}
-	}
-	if runDefault {
-		if errJson := DeserializeGrpcResponseBodyIntoContainer(grpcResponseBody, resp); errJson != nil {
-			return errJson
-		}
-	}
-	return nil
-}
-
-func serializeMiddlewareResponseIntoJsonWrapped(endpoint *Endpoint, respJson []byte, resp interface{}) ([]byte, ErrorJson) {
-	runDefault := true
-	var errJson ErrorJson
-	if endpoint.Hooks.OnPreSerializeMiddlewareResponseIntoJson != nil {
-		var run RunDefault
-		run, respJson, errJson = endpoint.Hooks.OnPreSerializeMiddlewareResponseIntoJson(resp)
-		if errJson != nil {
-			return nil, errJson
-		}
-		if !run {
-			runDefault = false
-		}
-	}
-	if runDefault {
-		respJson, errJson = SerializeMiddlewareResponseIntoJson(resp)
-		if errJson != nil {
-			return nil, errJson
-		}
-	}
-	return respJson, nil
-}

api/gateway/apimiddleware/log.go

@@ -1,5 +0,0 @@
-package apimiddleware
-
-import "github.com/sirupsen/logrus"
-
-var log = logrus.WithField("prefix", "apimiddleware")

api/gateway/apimiddleware/param_handling.go

@@ -1,103 +0,0 @@
-package apimiddleware
-
-import (
-	"encoding/base64"
-	"net/http"
-	"net/url"
-	"strings"
-
-	"github.com/gorilla/mux"
-	butil "github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
-	"github.com/wealdtech/go-bytesutil"
-)
-
-// HandleURLParameters processes URL parameters, allowing parameterized URLs to be safely and correctly proxied to grpc-gateway.
-func HandleURLParameters(url string, req *http.Request, literals []string) ErrorJson {
-	segments := strings.Split(url, "/")
-
-segmentsLoop:
-	for i, s := range segments {
-		// We only care about segments which are parameterized.
-		if isRequestParam(s) {
-			// Don't do anything with parameters which should be forwarded literally to gRPC.
-			for _, l := range literals {
-				if s == "{"+l+"}" {
-					continue segmentsLoop
-				}
-			}
-
-			routeVar := mux.Vars(req)[s[1:len(s)-1]]
-			bRouteVar := []byte(routeVar)
-			if butil.IsHex(bRouteVar) {
-				var err error
-				bRouteVar, err = bytesutil.FromHexString(string(bRouteVar))
-				if err != nil {
-					return InternalServerErrorWithMessage(err, "could not process URL parameter")
-				}
-			}
-			// Converting hex to base64 may result in a value which malforms the URL.
-			// We use URLEncoding to safely escape such values.
-			base64RouteVar := base64.URLEncoding.EncodeToString(bRouteVar)
-
-			// Merge segments back into the full URL.
-			splitPath := strings.Split(req.URL.Path, "/")
-			splitPath[i] = base64RouteVar
-			req.URL.Path = strings.Join(splitPath, "/")
-		}
-	}
-	return nil
-}
-
-// HandleQueryParameters processes query parameters, allowing them to be safely and correctly proxied to grpc-gateway.
-func HandleQueryParameters(req *http.Request, params []QueryParam) ErrorJson {
-	queryParams := req.URL.Query()
-
-	normalizeQueryValues(queryParams)
-
-	for key, vals := range queryParams {
-		for _, p := range params {
-			if key == p.Name {
-				if p.Hex {
-					queryParams.Del(key)
-					for _, v := range vals {
-						b := []byte(v)
-						if butil.IsHex(b) {
-							var err error
-							b, err = bytesutil.FromHexString(v)
-							if err != nil {
-								return InternalServerErrorWithMessage(err, "could not process query parameter")
-							}
-						}
-						queryParams.Add(key, base64.URLEncoding.EncodeToString(b))
-					}
-				}
-				if p.Enum {
-					queryParams.Del(key)
-					for _, v := range vals {
-						// gRPC expects uppercase enum values.
-						queryParams.Add(key, strings.ToUpper(v))
-					}
-				}
-			}
-		}
-	}
-	req.URL.RawQuery = queryParams.Encode()
-	return nil
-}
-
-// isRequestParam verifies whether the passed string is a request parameter.
-// Request parameters are enclosed in { and }.
-func isRequestParam(s string) bool {
-	return len(s) > 2 && s[0] == '{' && s[len(s)-1] == '}'
-}
-
-func normalizeQueryValues(queryParams url.Values) {
-	// Replace comma-separated values with individual values.
-	for key, vals := range queryParams {
-		splitVals := make([]string, 0)
-		for _, v := range vals {
-			splitVals = append(splitVals, strings.Split(v, ",")...)
-		}
-		queryParams[key] = splitVals
-	}
-}

api/gateway/apimiddleware/param_handling_test.go

@@ -1,124 +0,0 @@
-package apimiddleware
-
-import (
-	"bytes"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/gorilla/mux"
-	"github.com/prysmaticlabs/prysm/v4/testing/assert"
-	"github.com/prysmaticlabs/prysm/v4/testing/require"
-)
-
-func TestHandleURLParameters(t *testing.T) {
-	var body bytes.Buffer
-
-	t.Run("no_params", func(t *testing.T) {
-		request := httptest.NewRequest("GET", "http://foo.example/bar", &body)
-
-		errJson := HandleURLParameters("/not_param", request, []string{})
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "/bar", request.URL.Path)
-	})
-
-	t.Run("with_params", func(t *testing.T) {
-		muxVars := make(map[string]string)
-		muxVars["bar_param"] = "bar"
-		muxVars["quux_param"] = "quux"
-		request := httptest.NewRequest("GET", "http://foo.example/bar/baz/quux", &body)
-		request = mux.SetURLVars(request, muxVars)
-
-		errJson := HandleURLParameters("/{bar_param}/not_param/{quux_param}", request, []string{})
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "/YmFy/baz/cXV1eA==", request.URL.Path)
-	})
-
-	t.Run("with_literal", func(t *testing.T) {
-		muxVars := make(map[string]string)
-		muxVars["bar_param"] = "bar"
-		request := httptest.NewRequest("GET", "http://foo.example/bar/baz", &body)
-		request = mux.SetURLVars(request, muxVars)
-
-		errJson := HandleURLParameters("/{bar_param}/not_param/", request, []string{"bar_param"})
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "/bar/baz", request.URL.Path)
-	})
-
-	t.Run("with_hex", func(t *testing.T) {
-		muxVars := make(map[string]string)
-		muxVars["hex_param"] = "0x626172"
-		request := httptest.NewRequest("GET", "http://foo.example/0x626172/baz", &body)
-		request = mux.SetURLVars(request, muxVars)
-
-		errJson := HandleURLParameters("/{hex_param}/not_param/", request, []string{})
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "/YmFy/baz", request.URL.Path)
-	})
-}
-
-func TestHandleQueryParameters(t *testing.T) {
-	var body bytes.Buffer
-
-	t.Run("regular_params", func(t *testing.T) {
-		request := httptest.NewRequest("GET", "http://foo.example?bar=bar&baz=baz", &body)
-
-		errJson := HandleQueryParameters(request, []QueryParam{{Name: "bar"}, {Name: "baz"}})
-		require.Equal(t, true, errJson == nil)
-		query := request.URL.Query()
-		v, ok := query["bar"]
-		require.Equal(t, true, ok, "query param not found")
-		require.Equal(t, 1, len(v), "wrong number of query param values")
-		assert.Equal(t, "bar", v[0])
-		v, ok = query["baz"]
-		require.Equal(t, true, ok, "query param not found")
-		require.Equal(t, 1, len(v), "wrong number of query param values")
-		assert.Equal(t, "baz", v[0])
-	})
-
-	t.Run("hex_and_enum_params", func(t *testing.T) {
-		request := httptest.NewRequest("GET", "http://foo.example?hex=0x626172&baz=baz", &body)
-
-		errJson := HandleQueryParameters(request, []QueryParam{{Name: "hex", Hex: true}, {Name: "baz", Enum: true}})
-		require.Equal(t, true, errJson == nil)
-		query := request.URL.Query()
-		v, ok := query["hex"]
-		require.Equal(t, true, ok, "query param not found")
-		require.Equal(t, 1, len(v), "wrong number of query param values")
-		assert.Equal(t, "YmFy", v[0])
-		v, ok = query["baz"]
-		require.Equal(t, true, ok, "query param not found")
-		require.Equal(t, 1, len(v), "wrong number of query param values")
-		assert.Equal(t, "BAZ", v[0])
-	})
-}
-
-func TestIsRequestParam(t *testing.T) {
-	tests := []struct {
-		s string
-		b bool
-	}{
-		{"", false},
-		{"{", false},
-		{"}", false},
-		{"{}", false},
-		{"{x}", true},
-		{"{very_long_parameter_name_with_underscores}", true},
-	}
-	for _, tt := range tests {
-		b := isRequestParam(tt.s)
-		assert.Equal(t, tt.b, b)
-	}
-}
-
-func TestNormalizeQueryValues(t *testing.T) {
-	input := make(map[string][]string)
-	input["key"] = []string{"value1", "value2,value3,value4", "value5"}
-
-	normalizeQueryValues(input)
-	require.Equal(t, 5, len(input["key"]))
-	assert.Equal(t, "value1", input["key"][0])
-	assert.Equal(t, "value2", input["key"][1])
-	assert.Equal(t, "value3", input["key"][2])
-	assert.Equal(t, "value4", input["key"][3])
-	assert.Equal(t, "value5", input["key"][4])
-}

api/gateway/apimiddleware/process_field.go

@@ -1,179 +0,0 @@
-package apimiddleware
-
-import (
-	"encoding/base64"
-	"fmt"
-	"math/big"
-	"reflect"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/pkg/errors"
-	"github.com/wealdtech/go-bytesutil"
-)
-
-// processField calls each processor function on any field that has the matching tag set.
-// It is a recursive function.
-func processField(s interface{}, processors []fieldProcessor) error {
-	kind := reflect.TypeOf(s).Kind()
-	if kind != reflect.Ptr && kind != reflect.Slice && kind != reflect.Array {
-		return fmt.Errorf("processing fields of kind '%v' is unsupported", kind)
-	}
-
-	t := reflect.TypeOf(s).Elem()
-	v := reflect.Indirect(reflect.ValueOf(s))
-
-	for i := 0; i < t.NumField(); i++ {
-		switch v.Field(i).Kind() {
-		case reflect.Slice:
-			sliceElem := t.Field(i).Type.Elem()
-			kind := sliceElem.Kind()
-			// Recursively process slices to struct pointers.
-			switch {
-			case kind == reflect.Ptr && sliceElem.Elem().Kind() == reflect.Struct:
-				for j := 0; j < v.Field(i).Len(); j++ {
-					if err := processField(v.Field(i).Index(j).Interface(), processors); err != nil {
-						return errors.Wrapf(err, "could not process field '%s'", t.Field(i).Name)
-					}
-				}
-			// Process each string in string slices.
-			case kind == reflect.String:
-				for _, proc := range processors {
-					_, hasTag := t.Field(i).Tag.Lookup(proc.tag)
-					if !hasTag {
-						continue
-					}
-					for j := 0; j < v.Field(i).Len(); j++ {
-						if err := proc.f(v.Field(i).Index(j)); err != nil {
-							return errors.Wrapf(err, "could not process field '%s'", t.Field(i).Name)
-						}
-					}
-				}
-			}
-		// Recursively process struct pointers.
-		case reflect.Ptr:
-			if v.Field(i).Elem().Kind() == reflect.Struct {
-				if err := processField(v.Field(i).Interface(), processors); err != nil {
-					return errors.Wrapf(err, "could not process field '%s'", t.Field(i).Name)
-				}
-			}
-		default:
-			field := t.Field(i)
-			for _, proc := range processors {
-				if _, hasTag := field.Tag.Lookup(proc.tag); hasTag {
-					if err := proc.f(v.Field(i)); err != nil {
-						return errors.Wrapf(err, "could not process field '%s'", t.Field(i).Name)
-					}
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func hexToBase64Processor(v reflect.Value) error {
-	if v.String() == "0x" {
-		v.SetString("")
-		return nil
-	}
-	b, err := bytesutil.FromHexString(v.String())
-	if err != nil {
-		return err
-	}
-	v.SetString(base64.StdEncoding.EncodeToString(b))
-	return nil
-}
-
-func base64ToHexProcessor(v reflect.Value) error {
-	if v.String() == "" {
-		// Empty hex values are represented as "0x".
-		v.SetString("0x")
-		return nil
-	}
-	b, err := base64.StdEncoding.DecodeString(v.String())
-	if err != nil {
-		return err
-	}
-	v.SetString(hexutil.Encode(b))
-	return nil
-}
-
-func base64ToChecksumAddressProcessor(v reflect.Value) error {
-	if v.String() == "" {
-		// Empty hex values are represented as "0x".
-		v.SetString("0x")
-		return nil
-	}
-	b, err := base64.StdEncoding.DecodeString(v.String())
-	if err != nil {
-		return err
-	}
-	v.SetString(common.BytesToAddress(b).Hex())
-	return nil
-}
-
-func base64ToUint256Processor(v reflect.Value) error {
-	if v.String() == "" {
-		return nil
-	}
-	littleEndian, err := base64.StdEncoding.DecodeString(v.String())
-	if err != nil {
-		return err
-	}
-	if len(littleEndian) != 32 {
-		return errors.New("invalid length for Uint256")
-	}
-
-	// Integers are stored as little-endian, but
-	// big.Int expects big-endian. So we need to reverse
-	// the byte order before decoding.
-	var bigEndian [32]byte
-	for i := 0; i < len(littleEndian); i++ {
-		bigEndian[i] = littleEndian[len(littleEndian)-1-i]
-	}
-	var uint256 big.Int
-	uint256.SetBytes(bigEndian[:])
-	v.SetString(uint256.String())
-	return nil
-}
-
-func uint256ToBase64Processor(v reflect.Value) error {
-	if v.String() == "" {
-		return nil
-	}
-	uint256, ok := new(big.Int).SetString(v.String(), 10)
-	if !ok {
-		return fmt.Errorf("could not parse Uint256")
-	}
-	bigEndian := uint256.Bytes()
-	if len(bigEndian) > 32 {
-		return fmt.Errorf("number too big for Uint256")
-	}
-
-	// Integers are stored as little-endian, but
-	// big.Int gives big-endian. So we need to reverse
-	// the byte order before encoding.
-	var littleEndian [32]byte
-	for i := 0; i < len(bigEndian); i++ {
-		littleEndian[i] = bigEndian[len(bigEndian)-1-i]
-	}
-	v.SetString(base64.StdEncoding.EncodeToString(littleEndian[:]))
-	return nil
-}
-
-func enumToLowercaseProcessor(v reflect.Value) error {
-	v.SetString(strings.ToLower(v.String()))
-	return nil
-}
-
-func timeToUnixProcessor(v reflect.Value) error {
-	t, err := time.Parse(time.RFC3339, v.String())
-	if err != nil {
-		return err
-	}
-	v.SetString(strconv.FormatUint(uint64(t.Unix()), 10))
-	return nil
-}

api/gateway/apimiddleware/process_request.go

@@ -1,283 +0,0 @@
-package apimiddleware
-
-import (
-	"bytes"
-	"encoding/json"
-	"io"
-	"net"
-	"net/http"
-	"strconv"
-	"strings"
-
-	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/v4/api"
-	"github.com/prysmaticlabs/prysm/v4/api/grpc"
-)
-
-// DeserializeRequestBodyIntoContainer deserializes the request's body into an endpoint-specific struct.
-func DeserializeRequestBodyIntoContainer(body io.Reader, requestContainer interface{}) ErrorJson {
-	decoder := json.NewDecoder(body)
-	decoder.DisallowUnknownFields()
-	if err := decoder.Decode(&requestContainer); err != nil {
-		if strings.Contains(err.Error(), "json: unknown field") {
-			e := errors.Wrap(err, "could not decode request body")
-			return &DefaultErrorJson{
-				Message: e.Error(),
-				Code:    http.StatusBadRequest,
-			}
-		}
-		return InternalServerErrorWithMessage(err, "could not decode request body")
-	}
-	return nil
-}
-
-// ProcessRequestContainerFields processes fields of an endpoint-specific container according to field tags.
-func ProcessRequestContainerFields(requestContainer interface{}) ErrorJson {
-	if err := processField(requestContainer, []fieldProcessor{
-		{
-			tag: "hex",
-			f:   hexToBase64Processor,
-		},
-		{
-			tag: "uint256",
-			f:   uint256ToBase64Processor,
-		},
-	}); err != nil {
-		return InternalServerErrorWithMessage(err, "could not process request data")
-	}
-	return nil
-}
-
-// SetRequestBodyToRequestContainer makes the endpoint-specific container the new body of the request.
-func SetRequestBodyToRequestContainer(requestContainer interface{}, req *http.Request) ErrorJson {
-	// Serialize the struct, which now includes a base64-encoded value, into JSON.
-	j, err := json.Marshal(requestContainer)
-	if err != nil {
-		return InternalServerErrorWithMessage(err, "could not marshal request")
-	}
-	// Set the body to the new JSON.
-	req.Body = io.NopCloser(bytes.NewReader(j))
-	req.Header.Set("Content-Length", strconv.Itoa(len(j)))
-	req.ContentLength = int64(len(j))
-	return nil
-}
-
-// PrepareRequestForProxying applies additional logic to the request so that it can be correctly proxied to grpc-gateway.
-func (m *ApiProxyMiddleware) PrepareRequestForProxying(endpoint Endpoint, req *http.Request) ErrorJson {
-	req.URL.Scheme = "http"
-	req.URL.Host = m.GatewayAddress
-	req.RequestURI = ""
-	if errJson := HandleURLParameters(endpoint.Path, req, endpoint.RequestURLLiterals); errJson != nil {
-		return errJson
-	}
-	if errJson := HandleQueryParameters(req, endpoint.RequestQueryParams); errJson != nil {
-		return errJson
-	}
-	// We have to add the prefix after handling parameters because adding the prefix changes URL segment indexing.
-	req.URL.Path = "/internal" + req.URL.Path
-	return nil
-}
-
-// ProxyRequest proxies the request to grpc-gateway.
-func (m *ApiProxyMiddleware) ProxyRequest(req *http.Request) (*http.Response, ErrorJson) {
-	// We do not use http.DefaultClient because it does not have any timeout.
-	netClient := &http.Client{Timeout: m.Timeout}
-	grpcResp, err := netClient.Do(req)
-	if err != nil {
-		if err, ok := err.(net.Error); ok && err.Timeout() {
-			return nil, TimeoutError()
-		}
-		return nil, InternalServerErrorWithMessage(err, "could not proxy request")
-	}
-	if grpcResp == nil {
-		return nil, &DefaultErrorJson{Message: "nil response from gRPC-gateway", Code: http.StatusInternalServerError}
-	}
-	return grpcResp, nil
-}
-
-// ReadGrpcResponseBody reads the body from the grpc-gateway's response.
-func ReadGrpcResponseBody(r io.Reader) ([]byte, ErrorJson) {
-	body, err := io.ReadAll(r)
-	if err != nil {
-		return nil, InternalServerErrorWithMessage(err, "could not read response body")
-	}
-	return body, nil
-}
-
-// HandleGrpcResponseError acts on an error that resulted from a grpc-gateway's response.
-// Whether there was an error is indicated by the bool return value. In case of an error,
-// there is no need to write to the response because it's taken care of by the function.
-func HandleGrpcResponseError(errJson ErrorJson, resp *http.Response, respBody []byte, w http.ResponseWriter) (bool, ErrorJson) {
-	responseHasError := false
-	if err := json.Unmarshal(respBody, errJson); err != nil {
-		return false, InternalServerErrorWithMessage(err, "could not unmarshal error")
-	}
-	if errJson.Msg() != "" {
-		responseHasError = true
-		// Something went wrong, but the request completed, meaning we can write headers and the error message.
-		for h, vs := range resp.Header {
-			for _, v := range vs {
-				if strings.HasSuffix(h, api.VersionHeader) {
-					w.Header().Set(api.VersionHeader, v)
-				} else {
-					w.Header().Set(h, v)
-				}
-			}
-		}
-		// Handle gRPC timeout.
-		if resp.StatusCode == http.StatusGatewayTimeout {
-			WriteError(w, TimeoutError(), resp.Header)
-		} else {
-			// Set code to HTTP code because unmarshalled body contained gRPC code.
-			errJson.SetCode(resp.StatusCode)
-			WriteError(w, errJson, resp.Header)
-		}
-	}
-	return responseHasError, nil
-}
-
-// GrpcResponseIsEmpty determines whether the grpc-gateway's response body contains no data.
-func GrpcResponseIsEmpty(grpcResponseBody []byte) bool {
-	return len(grpcResponseBody) == 0 || string(grpcResponseBody) == "{}"
-}
-
-// DeserializeGrpcResponseBodyIntoContainer deserializes the grpc-gateway's response body into an endpoint-specific struct.
-func DeserializeGrpcResponseBodyIntoContainer(body []byte, responseContainer interface{}) ErrorJson {
-	if err := json.Unmarshal(body, &responseContainer); err != nil {
-		return InternalServerErrorWithMessage(err, "could not unmarshal response")
-	}
-	return nil
-}
-
-// ProcessMiddlewareResponseFields processes fields of an endpoint-specific container according to field tags.
-func ProcessMiddlewareResponseFields(responseContainer interface{}) ErrorJson {
-	if err := processField(responseContainer, []fieldProcessor{
-		{
-			tag: "hex",
-			f:   base64ToHexProcessor,
-		},
-		{
-			tag: "address",
-			f:   base64ToChecksumAddressProcessor,
-		},
-		{
-			tag: "enum",
-			f:   enumToLowercaseProcessor,
-		},
-		{
-			tag: "time",
-			f:   timeToUnixProcessor,
-		},
-		{
-			tag: "uint256",
-			f:   base64ToUint256Processor,
-		},
-	}); err != nil {
-		return InternalServerErrorWithMessage(err, "could not process response data")
-	}
-	return nil
-}
-
-// SerializeMiddlewareResponseIntoJson serializes the endpoint-specific response struct into a JSON representation.
-func SerializeMiddlewareResponseIntoJson(responseContainer interface{}) (jsonResponse []byte, errJson ErrorJson) {
-	j, err := json.Marshal(responseContainer)
-	if err != nil {
-		return nil, InternalServerErrorWithMessage(err, "could not marshal response")
-	}
-	return j, nil
-}
-
-// WriteMiddlewareResponseHeadersAndBody populates headers and the body of the final response.
-func WriteMiddlewareResponseHeadersAndBody(grpcResp *http.Response, responseJson []byte, w http.ResponseWriter) ErrorJson {
-	var statusCodeHeader string
-	for h, vs := range grpcResp.Header {
-		// We don't want to expose any gRPC metadata in the HTTP response, so we skip forwarding metadata headers.
-		if strings.HasPrefix(h, grpc.MetadataPrefix) {
-			if h == grpc.WithPrefix(grpc.HttpCodeMetadataKey) {
-				statusCodeHeader = vs[0]
-			} else if strings.HasSuffix(h, api.VersionHeader) {
-				w.Header().Set(api.VersionHeader, vs[0])
-			}
-		} else {
-			for _, v := range vs {
-				w.Header().Set(h, v)
-			}
-		}
-	}
-	if !GrpcResponseIsEmpty(responseJson) {
-		w.Header().Set("Content-Length", strconv.Itoa(len(responseJson)))
-		if statusCodeHeader != "" {
-			code, err := strconv.Atoi(statusCodeHeader)
-			if err != nil {
-				return InternalServerErrorWithMessage(err, "could not parse status code")
-			}
-			w.WriteHeader(code)
-		} else {
-			w.WriteHeader(grpcResp.StatusCode)
-		}
-		if _, err := io.Copy(w, io.NopCloser(bytes.NewReader(responseJson))); err != nil {
-			return InternalServerErrorWithMessage(err, "could not write response message")
-		}
-	} else {
-		w.Header().Set("Content-Length", "0")
-		w.WriteHeader(grpcResp.StatusCode)
-	}
-	return nil
-}
-
-// WriteError writes the error by manipulating headers and the body of the final response.
-func WriteError(w http.ResponseWriter, errJson ErrorJson, responseHeader http.Header) {
-	// Include custom error in the error JSON.
-	hasCustomError := false
-	if responseHeader != nil {
-		customError, ok := responseHeader[grpc.WithPrefix(grpc.CustomErrorMetadataKey)]
-		if ok {
-			hasCustomError = true
-			// Assume header has only one value and read the 0 index.
-			if err := json.Unmarshal([]byte(customError[0]), errJson); err != nil {
-				log.WithError(err).Error("Could not unmarshal custom error message")
-				return
-			}
-		}
-	}
-
-	var j []byte
-	if hasCustomError {
-		var err error
-		j, err = json.Marshal(errJson)
-		if err != nil {
-			log.WithError(err).Error("Could not marshal error message")
-			return
-		}
-	} else {
-		var err error
-		// We marshal the response body into a DefaultErrorJson if the custom error is not present.
-		// This is because the ErrorJson argument is the endpoint's error definition, which may contain custom fields.
-		// In such a scenario marhaling the endpoint's error would populate the resulting JSON
-		// with these fields even if they are not present in the gRPC header.
-		d := &DefaultErrorJson{
-			Message: errJson.Msg(),
-			Code:    errJson.StatusCode(),
-		}
-		j, err = json.Marshal(d)
-		if err != nil {
-			log.WithError(err).Error("Could not marshal error message")
-			return
-		}
-	}
-
-	w.Header().Set("Content-Length", strconv.Itoa(len(j)))
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(errJson.StatusCode())
-	if _, err := io.Copy(w, io.NopCloser(bytes.NewReader(j))); err != nil {
-		log.WithError(err).Error("Could not write error message")
-	}
-}
-
-// Cleanup performs final cleanup on the initial response from grpc-gateway.
-func Cleanup(grpcResponseBody io.ReadCloser) ErrorJson {
-	if err := grpcResponseBody.Close(); err != nil {
-		return InternalServerErrorWithMessage(err, "could not close response body")
-	}
-	return nil
-}

api/gateway/apimiddleware/process_request_test.go

@@ -1,435 +0,0 @@
-package apimiddleware
-
-import (
-	"bytes"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/prysmaticlabs/prysm/v4/api"
-	"github.com/prysmaticlabs/prysm/v4/api/grpc"
-	"github.com/prysmaticlabs/prysm/v4/testing/assert"
-	"github.com/prysmaticlabs/prysm/v4/testing/require"
-	"github.com/sirupsen/logrus/hooks/test"
-)
-
-type testRequestContainer struct {
-	TestString         string
-	TestHexString      string `hex:"true"`
-	TestEmptyHexString string `hex:"true"`
-	TestUint256String  string `uint256:"true"`
-}
-
-func defaultRequestContainer() *testRequestContainer {
-	return &testRequestContainer{
-		TestString:         "test string",
-		TestHexString:      "0x666F6F", // hex encoding of "foo"
-		TestEmptyHexString: "0x",
-		TestUint256String:  "4196",
-	}
-}
-
-type testResponseContainer struct {
-	TestString       string
-	TestHex          string `hex:"true"`
-	TestEmptyHex     string `hex:"true"`
-	TestAddress      string `address:"true"`
-	TestEmptyAddress string `address:"true"`
-	TestUint256      string `uint256:"true"`
-	TestEnum         string `enum:"true"`
-	TestTime         string `time:"true"`
-}
-
-func defaultResponseContainer() *testResponseContainer {
-	return &testResponseContainer{
-		TestString:       "test string",
-		TestHex:          "Zm9v", // base64 encoding of "foo"
-		TestEmptyHex:     "",
-		TestAddress:      "Zm9v",
-		TestEmptyAddress: "",
-		TestEnum:         "Test Enum",
-		TestTime:         "2006-01-02T15:04:05Z",
-
-		// base64 encoding of 4196 in little-endian
-		TestUint256: "ZBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
-	}
-}
-
-type testErrorJson struct {
-	Message     string
-	Code        int
-	CustomField string
-}
-
-// StatusCode returns the error's underlying error code.
-func (e *testErrorJson) StatusCode() int {
-	return e.Code
-}
-
-// Msg returns the error's underlying message.
-func (e *testErrorJson) Msg() string {
-	return e.Message
-}
-
-// SetCode sets the error's underlying error code.
-func (e *testErrorJson) SetCode(code int) {
-	e.Code = code
-}
-
-// SetMsg sets the error's underlying message.
-func (e *testErrorJson) SetMsg(msg string) {
-	e.Message = msg
-}
-
-func TestDeserializeRequestBodyIntoContainer(t *testing.T) {
-	t.Run("ok", func(t *testing.T) {
-		var bodyJson bytes.Buffer
-		err := json.NewEncoder(&bodyJson).Encode(defaultRequestContainer())
-		require.NoError(t, err)
-
-		container := &testRequestContainer{}
-		errJson := DeserializeRequestBodyIntoContainer(&bodyJson, container)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "test string", container.TestString)
-	})
-
-	t.Run("error", func(t *testing.T) {
-		var bodyJson bytes.Buffer
-		bodyJson.Write([]byte("foo"))
-		errJson := DeserializeRequestBodyIntoContainer(&bodyJson, &testRequestContainer{})
-		require.NotNil(t, errJson)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not decode request body"))
-		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
-	})
-
-	t.Run("unknown field", func(t *testing.T) {
-		var bodyJson bytes.Buffer
-		bodyJson.Write([]byte("{\"foo\":\"foo\"}"))
-		errJson := DeserializeRequestBodyIntoContainer(&bodyJson, &testRequestContainer{})
-		require.NotNil(t, errJson)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not decode request body"))
-		assert.Equal(t, http.StatusBadRequest, errJson.StatusCode())
-	})
-}
-
-func TestProcessRequestContainerFields(t *testing.T) {
-	t.Run("ok", func(t *testing.T) {
-		container := defaultRequestContainer()
-
-		errJson := ProcessRequestContainerFields(container)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "Zm9v", container.TestHexString)
-		assert.Equal(t, "", container.TestEmptyHexString)
-		assert.Equal(t, "ZBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", container.TestUint256String)
-	})
-
-	t.Run("error", func(t *testing.T) {
-		errJson := ProcessRequestContainerFields("foo")
-		require.NotNil(t, errJson)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not process request data"))
-		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
-	})
-}
-
-func TestSetRequestBodyToRequestContainer(t *testing.T) {
-	var body bytes.Buffer
-	request := httptest.NewRequest("GET", "http://foo.example", &body)
-
-	errJson := SetRequestBodyToRequestContainer(defaultRequestContainer(), request)
-	require.Equal(t, true, errJson == nil)
-	container := &testRequestContainer{}
-	require.NoError(t, json.NewDecoder(request.Body).Decode(container))
-	assert.Equal(t, "test string", container.TestString)
-	contentLengthHeader, ok := request.Header["Content-Length"]
-	require.Equal(t, true, ok)
-	require.Equal(t, 1, len(contentLengthHeader), "wrong number of header values")
-	assert.Equal(t, "108", contentLengthHeader[0])
-	assert.Equal(t, int64(108), request.ContentLength)
-}
-
-func TestPrepareRequestForProxying(t *testing.T) {
-	middleware := &ApiProxyMiddleware{
-		GatewayAddress: "http://gateway.example",
-	}
-	// We will set some params to make the request more interesting.
-	endpoint := Endpoint{
-		Path:               "/{url_param}",
-		RequestURLLiterals: []string{"url_param"},
-		RequestQueryParams: []QueryParam{{Name: "query_param"}},
-	}
-	var body bytes.Buffer
-	request := httptest.NewRequest("GET", "http://foo.example?query_param=bar", &body)
-
-	errJson := middleware.PrepareRequestForProxying(endpoint, request)
-	require.Equal(t, true, errJson == nil)
-	assert.Equal(t, "http", request.URL.Scheme)
-	assert.Equal(t, middleware.GatewayAddress, request.URL.Host)
-	assert.Equal(t, "", request.RequestURI)
-}
-
-func TestReadGrpcResponseBody(t *testing.T) {
-	var b bytes.Buffer
-	b.Write([]byte("foo"))
-
-	body, jsonErr := ReadGrpcResponseBody(&b)
-	require.Equal(t, true, jsonErr == nil)
-	assert.Equal(t, "foo", string(body))
-}
-
-func TestHandleGrpcResponseError(t *testing.T) {
-	response := &http.Response{
-		StatusCode: 400,
-		Header: http.Header{
-			"Foo": []string{"foo"},
-			"Bar": []string{"bar"},
-		},
-	}
-	writer := httptest.NewRecorder()
-	errJson := &testErrorJson{
-		Message: "foo",
-		Code:    400,
-	}
-	b, err := json.Marshal(errJson)
-	require.NoError(t, err)
-
-	hasError, e := HandleGrpcResponseError(errJson, response, b, writer)
-	require.Equal(t, true, e == nil)
-	assert.Equal(t, true, hasError)
-	v, ok := writer.Header()["Foo"]
-	require.Equal(t, true, ok, "header not found")
-	require.Equal(t, 1, len(v), "wrong number of header values")
-	assert.Equal(t, "foo", v[0])
-	v, ok = writer.Header()["Bar"]
-	require.Equal(t, true, ok, "header not found")
-	require.Equal(t, 1, len(v), "wrong number of header values")
-	assert.Equal(t, "bar", v[0])
-	assert.Equal(t, 400, errJson.StatusCode())
-}
-
-func TestGrpcResponseIsEmpty(t *testing.T) {
-	t.Run("nil", func(t *testing.T) {
-		assert.Equal(t, true, GrpcResponseIsEmpty(nil))
-	})
-	t.Run("empty_slice", func(t *testing.T) {
-		assert.Equal(t, true, GrpcResponseIsEmpty(make([]byte, 0)))
-	})
-	t.Run("empty_brackets", func(t *testing.T) {
-		assert.Equal(t, true, GrpcResponseIsEmpty([]byte("{}")))
-	})
-	t.Run("non_empty", func(t *testing.T) {
-		assert.Equal(t, false, GrpcResponseIsEmpty([]byte("{\"foo\":\"bar\"})")))
-	})
-}
-
-func TestDeserializeGrpcResponseBodyIntoContainer(t *testing.T) {
-	t.Run("ok", func(t *testing.T) {
-		body, err := json.Marshal(defaultRequestContainer())
-		require.NoError(t, err)
-
-		container := &testRequestContainer{}
-		errJson := DeserializeGrpcResponseBodyIntoContainer(body, container)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "test string", container.TestString)
-	})
-
-	t.Run("error", func(t *testing.T) {
-		var bodyJson bytes.Buffer
-		bodyJson.Write([]byte("foo"))
-		errJson := DeserializeGrpcResponseBodyIntoContainer(bodyJson.Bytes(), &testRequestContainer{})
-		require.NotNil(t, errJson)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not unmarshal response"))
-		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
-	})
-}
-
-func TestProcessMiddlewareResponseFields(t *testing.T) {
-	t.Run("Ok", func(t *testing.T) {
-		container := defaultResponseContainer()
-
-		errJson := ProcessMiddlewareResponseFields(container)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, "0x666f6f", container.TestHex)
-		assert.Equal(t, "0x", container.TestEmptyHex)
-		assert.Equal(t, "0x0000000000000000000000000000000000666F6f", container.TestAddress)
-		assert.Equal(t, "0x", container.TestEmptyAddress)
-		assert.Equal(t, "4196", container.TestUint256)
-		assert.Equal(t, "test enum", container.TestEnum)
-		assert.Equal(t, "1136214245", container.TestTime)
-	})
-
-	t.Run("error", func(t *testing.T) {
-		errJson := ProcessMiddlewareResponseFields("foo")
-		require.NotNil(t, errJson)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not process response data"))
-		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
-	})
-}
-
-func TestSerializeMiddlewareResponseIntoJson(t *testing.T) {
-	container := defaultResponseContainer()
-	j, errJson := SerializeMiddlewareResponseIntoJson(container)
-	assert.Equal(t, true, errJson == nil)
-	cToDeserialize := &testResponseContainer{}
-	require.NoError(t, json.Unmarshal(j, cToDeserialize))
-	assert.Equal(t, "test string", cToDeserialize.TestString)
-}
-
-func TestWriteMiddlewareResponseHeadersAndBody(t *testing.T) {
-	t.Run("GET", func(t *testing.T) {
-		response := &http.Response{
-			Header: http.Header{
-				"Foo": []string{"foo"},
-				grpc.WithPrefix(grpc.HttpCodeMetadataKey): []string{"204"},
-				grpc.WithPrefix(api.VersionHeader):        []string{"capella"},
-			},
-		}
-		container := defaultResponseContainer()
-		responseJson, err := json.Marshal(container)
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-		writer.Body = &bytes.Buffer{}
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, true, errJson == nil)
-		v, ok := writer.Header()["Foo"]
-		require.Equal(t, true, ok, "header not found")
-		require.Equal(t, 1, len(v), "wrong number of header values")
-		assert.Equal(t, "foo", v[0])
-		v, ok = writer.Header()["Content-Length"]
-		require.Equal(t, true, ok, "header not found")
-		require.Equal(t, 1, len(v), "wrong number of header values")
-		assert.Equal(t, "224", v[0])
-		v, ok = writer.Header()["Eth-Consensus-Version"]
-		require.Equal(t, true, ok, "header not found")
-		assert.Equal(t, "capella", v[0])
-		assert.Equal(t, 204, writer.Code)
-		assert.DeepEqual(t, responseJson, writer.Body.Bytes())
-	})
-
-	t.Run("GET_no_grpc_status_code_header", func(t *testing.T) {
-		response := &http.Response{
-			Header:     http.Header{},
-			StatusCode: 204,
-		}
-		container := defaultResponseContainer()
-		responseJson, err := json.Marshal(container)
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, 204, writer.Code)
-	})
-
-	t.Run("GET_invalid_status_code", func(t *testing.T) {
-		response := &http.Response{
-			Header: http.Header{"Grpc-Metadata-Eth-Consensus-Version": []string{"capella"}},
-		}
-
-		// Set invalid status code.
-		response.Header[grpc.WithPrefix(grpc.HttpCodeMetadataKey)] = []string{"invalid"}
-		response.Header[grpc.WithPrefix(api.VersionHeader)] = []string{"capella"}
-
-		container := defaultResponseContainer()
-		responseJson, err := json.Marshal(container)
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, false, errJson == nil)
-		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not parse status code"))
-		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
-	})
-
-	t.Run("POST", func(t *testing.T) {
-		response := &http.Response{
-			Header:     http.Header{},
-			StatusCode: 204,
-		}
-		container := defaultResponseContainer()
-		responseJson, err := json.Marshal(container)
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, 204, writer.Code)
-	})
-
-	t.Run("POST_with_response_body", func(t *testing.T) {
-		response := &http.Response{
-			Header:     http.Header{},
-			StatusCode: 204,
-		}
-		container := defaultResponseContainer()
-		responseJson, err := json.Marshal(container)
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-		writer.Body = &bytes.Buffer{}
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, 204, writer.Code)
-		assert.DeepEqual(t, responseJson, writer.Body.Bytes())
-	})
-
-	t.Run("POST_with_empty_json_body", func(t *testing.T) {
-		response := &http.Response{
-			Header:     http.Header{},
-			StatusCode: 204,
-		}
-		responseJson, err := json.Marshal(struct{}{})
-		require.NoError(t, err)
-		writer := httptest.NewRecorder()
-		writer.Body = &bytes.Buffer{}
-
-		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
-		require.Equal(t, true, errJson == nil)
-		assert.Equal(t, 204, writer.Code)
-		assert.DeepEqual(t, []byte(nil), writer.Body.Bytes())
-		assert.Equal(t, "0", writer.Header()["Content-Length"][0])
-	})
-}
-
-func TestWriteError(t *testing.T) {
-	t.Run("ok", func(t *testing.T) {
-		responseHeader := http.Header{
-			grpc.WithPrefix(grpc.CustomErrorMetadataKey): []string{"{\"CustomField\":\"bar\"}"},
-		}
-		errJson := &testErrorJson{
-			Message: "foo",
-			Code:    500,
-		}
-		writer := httptest.NewRecorder()
-		writer.Body = &bytes.Buffer{}
-
-		WriteError(writer, errJson, responseHeader)
-		v, ok := writer.Header()["Content-Length"]
-		require.Equal(t, true, ok, "header not found")
-		require.Equal(t, 1, len(v), "wrong number of header values")
-		assert.Equal(t, "48", v[0])
-		v, ok = writer.Header()["Content-Type"]
-		require.Equal(t, true, ok, "header not found")
-		require.Equal(t, 1, len(v), "wrong number of header values")
-		assert.Equal(t, "application/json", v[0])
-		assert.Equal(t, 500, writer.Code)
-		eDeserialize := &testErrorJson{}
-		require.NoError(t, json.Unmarshal(writer.Body.Bytes(), eDeserialize))
-		assert.Equal(t, "foo", eDeserialize.Message)
-		assert.Equal(t, 500, eDeserialize.Code)
-		assert.Equal(t, "bar", eDeserialize.CustomField)
-	})
-
-	t.Run("invalid_custom_error_header", func(t *testing.T) {
-		logHook := test.NewGlobal()
-
-		responseHeader := http.Header{
-			grpc.WithPrefix(grpc.CustomErrorMetadataKey): []string{"invalid"},
-		}
-
-		WriteError(httptest.NewRecorder(), &testErrorJson{}, responseHeader)
-		assert.LogsContain(t, logHook, "Could not unmarshal custom error message")
-	})
-}

api/gateway/apimiddleware/structs.go

@@ -1,69 +0,0 @@
-package apimiddleware
-
-import (
-	"net/http"
-
-	"github.com/pkg/errors"
-)
-
-// ---------------
-// Error handling.
-// ---------------
-
-// ErrorJson describes common functionality of all JSON error representations.
-type ErrorJson interface {
-	StatusCode() int
-	SetCode(code int)
-	Msg() string
-	SetMsg(msg string)
-}
-
-// DefaultErrorJson is a JSON representation of a simple error value, containing only a message and an error code.
-type DefaultErrorJson struct {
-	Message string `json:"message"`
-	Code    int    `json:"code"`
-}
-
-// InternalServerErrorWithMessage returns a DefaultErrorJson with 500 code and a custom message.
-func InternalServerErrorWithMessage(err error, message string) *DefaultErrorJson {
-	e := errors.Wrapf(err, message)
-	return &DefaultErrorJson{
-		Message: e.Error(),
-		Code:    http.StatusInternalServerError,
-	}
-}
-
-// InternalServerError returns a DefaultErrorJson with 500 code.
-func InternalServerError(err error) *DefaultErrorJson {
-	return &DefaultErrorJson{
-		Message: err.Error(),
-		Code:    http.StatusInternalServerError,
-	}
-}
-
-func TimeoutError() *DefaultErrorJson {
-	return &DefaultErrorJson{
-		Message: "Request timeout",
-		Code:    http.StatusRequestTimeout,
-	}
-}
-
-// StatusCode returns the error's underlying error code.
-func (e *DefaultErrorJson) StatusCode() int {
-	return e.Code
-}
-
-// Msg returns the error's underlying message.
-func (e *DefaultErrorJson) Msg() string {
-	return e.Message
-}
-
-// SetCode sets the error's underlying error code.
-func (e *DefaultErrorJson) SetCode(code int) {
-	e.Code = code
-}
-
-// SetMsg sets the error's underlying message.
-func (e *DefaultErrorJson) SetMsg(msg string) {
-	e.Message = msg
-}

api/gateway/gateway.go

@@ -6,14 +6,11 @@ import (
 	"fmt"
 	"net"
 	"net/http"
-	"path"
-	"strings"
 	"time"
 
 	"github.com/gorilla/mux"
 	gwruntime "github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/v4/api/gateway/apimiddleware"
 	"github.com/prysmaticlabs/prysm/v4/runtime"
 	"github.com/rs/cors"
 	"google.golang.org/grpc"
@@ -36,7 +33,6 @@ type PbHandlerRegistration func(context.Context, *gwruntime.ServeMux, *grpc.Clie
 
 // MuxHandler is a function that implements the mux handler functionality.
 type MuxHandler func(
-	apiMiddlewareHandler *apimiddleware.ApiProxyMiddleware,
 	h http.HandlerFunc,
 	w http.ResponseWriter,
 	req *http.Request,
@@ -44,16 +40,15 @@ type MuxHandler func(
 
 // Config parameters for setting up the gateway service.
 type config struct {
-	maxCallRecvMsgSize           uint64
-	remoteCert                   string
-	gatewayAddr                  string
-	remoteAddr                   string
-	allowedOrigins               []string
-	apiMiddlewareEndpointFactory apimiddleware.EndpointFactory
-	muxHandler                   MuxHandler
-	pbHandlers                   []*PbMux
-	router                       *mux.Router
-	timeout                      time.Duration
+	maxCallRecvMsgSize uint64
+	remoteCert         string
+	gatewayAddr        string
+	remoteAddr         string
+	allowedOrigins     []string
+	muxHandler         MuxHandler
+	pbHandlers         []*PbMux
+	router             *mux.Router
+	timeout            time.Duration
 }
 
 // Gateway is the gRPC gateway to serve HTTP JSON traffic as a proxy and forward it to the gRPC server.
@@ -62,7 +57,6 @@ type Gateway struct {
 	conn         *grpc.ClientConn
 	server       *http.Server
 	cancel       context.CancelFunc
-	proxy        *apimiddleware.ApiProxyMiddleware
 	ctx          context.Context
 	startFailure error
 }
@@ -112,13 +106,9 @@ func (g *Gateway) Start() {
 
 	corsMux := g.corsMiddleware(g.cfg.router)
 
-	if g.cfg.apiMiddlewareEndpointFactory != nil && !g.cfg.apiMiddlewareEndpointFactory.IsNil() {
-		g.registerApiMiddleware()
-	}
-
 	if g.cfg.muxHandler != nil {
 		g.cfg.router.PathPrefix("/").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			g.cfg.muxHandler(g.proxy, corsMux.ServeHTTP, w, r)
+			g.cfg.muxHandler(corsMux.ServeHTTP, w, r)
 		})
 	}
 
@@ -179,24 +169,6 @@ func (g *Gateway) corsMiddleware(h http.Handler) http.Handler {
 	return c.Handler(h)
 }
 
-const swaggerDir = "proto/prysm/v1alpha1/"
-
-// SwaggerServer returns swagger specification files located under "/swagger/"
-func SwaggerServer() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		if !strings.HasSuffix(r.URL.Path, ".swagger.json") {
-			log.Debugf("Not found: %s", r.URL.Path)
-			http.NotFound(w, r)
-			return
-		}
-
-		log.Debugf("Serving %s\n", r.URL.Path)
-		p := strings.TrimPrefix(r.URL.Path, "/swagger/")
-		p = path.Join(swaggerDir, p)
-		http.ServeFile(w, r, p)
-	}
-}
-
 // dial the gRPC server.
 func (g *Gateway) dial(ctx context.Context, network, addr string) (*grpc.ClientConn, error) {
 	switch network {
@@ -249,13 +221,3 @@ func (g *Gateway) dialUnix(ctx context.Context, addr string) (*grpc.ClientConn,
 	}
 	return grpc.DialContext(ctx, addr, opts...)
 }
-
-func (g *Gateway) registerApiMiddleware() {
-	g.proxy = &apimiddleware.ApiProxyMiddleware{
-		GatewayAddress:  g.cfg.gatewayAddr,
-		EndpointCreator: g.cfg.apiMiddlewareEndpointFactory,
-		Timeout:         g.cfg.timeout,
-	}
-	log.Info("Starting API middleware")
-	g.proxy.Run(g.cfg.router)
-}

api/gateway/gateway_test.go

@@ -10,7 +10,6 @@ import (
 	"testing"
 
 	"github.com/gorilla/mux"
-	"github.com/prysmaticlabs/prysm/v4/api/gateway/apimiddleware"
 	"github.com/prysmaticlabs/prysm/v4/cmd/beacon-chain/flags"
 	"github.com/prysmaticlabs/prysm/v4/testing/assert"
 	"github.com/prysmaticlabs/prysm/v4/testing/require"
@@ -18,36 +17,18 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
-type mockEndpointFactory struct {
-}
-
-func (*mockEndpointFactory) Paths() []string {
-	return []string{}
-}
-
-func (*mockEndpointFactory) Create(_ string) (*apimiddleware.Endpoint, error) {
-	return nil, nil
-}
-
-func (*mockEndpointFactory) IsNil() bool {
-	return false
-}
-
 func TestGateway_Customized(t *testing.T) {
 	r := mux.NewRouter()
 	cert := "cert"
 	origins := []string{"origin"}
 	size := uint64(100)
-	endpointFactory := &mockEndpointFactory{}
 
 	opts := []Option{
 		WithRouter(r),
 		WithRemoteCert(cert),
 		WithAllowedOrigins(origins),
 		WithMaxCallRecvMsgSize(size),
-		WithApiMiddleware(endpointFactory),
 		WithMuxHandler(func(
-			_ *apimiddleware.ApiProxyMiddleware,
 			_ http.HandlerFunc,
 			_ http.ResponseWriter,
 			_ *http.Request,
@@ -63,7 +44,6 @@ func TestGateway_Customized(t *testing.T) {
 	require.Equal(t, 1, len(g.cfg.allowedOrigins))
 	assert.Equal(t, origins[0], g.cfg.allowedOrigins[0])
 	assert.Equal(t, size, g.cfg.maxCallRecvMsgSize)
-	assert.Equal(t, endpointFactory, g.cfg.apiMiddlewareEndpointFactory)
 }
 
 func TestGateway_StartStop(t *testing.T) {
@@ -83,7 +63,6 @@ func TestGateway_StartStop(t *testing.T) {
 		WithGatewayAddr(gatewayAddress),
 		WithRemoteAddr(selfAddress),
 		WithMuxHandler(func(
-			_ *apimiddleware.ApiProxyMiddleware,
 			_ http.HandlerFunc,
 			_ http.ResponseWriter,
 			_ *http.Request,

api/gateway/options.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/gorilla/mux"
 	gwruntime "github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
-	"github.com/prysmaticlabs/prysm/v4/api/gateway/apimiddleware"
 )
 
 type Option func(g *Gateway) error
@@ -70,14 +69,6 @@ func WithMaxCallRecvMsgSize(size uint64) Option {
 	}
 }
 
-// WithApiMiddleware allows adding an API middleware proxy to the gateway.
-func WithApiMiddleware(endpointFactory apimiddleware.EndpointFactory) Option {
-	return func(g *Gateway) error {
-		g.cfg.apiMiddlewareEndpointFactory = endpointFactory
-		return nil
-	}
-}
-
 // WithTimeout allows changing the timeout value for API calls.
 func WithTimeout(seconds uint64) Option {
 	return func(g *Gateway) error {

beacon-chain/blockchain/BUILD.bazel

@@ -50,6 +50,7 @@ go_library(
         "//beacon-chain/core/time:go_default_library",
         "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/db/filesystem:go_default_library",
         "//beacon-chain/db/filters:go_default_library",
         "//beacon-chain/db/kv:go_default_library",
         "//beacon-chain/execution:go_default_library",
@@ -98,26 +99,22 @@ go_library(
     ],
 )
 
-test_suite(
-    name = "go_default_test",
-    tests = [
-        ":go_raceoff_test",
-        ":go_raceon_test",
-    ],
-)
-
 go_test(
-    name = "go_raceoff_test",
+    name = "go_default_test",
     size = "medium",
     srcs = [
         "blockchain_test.go",
+        "chain_info_norace_test.go",
         "chain_info_test.go",
         "checktags_test.go",
+        "error_test.go",
         "execution_engine_test.go",
         "forkchoice_update_execution_test.go",
         "head_sync_committee_info_test.go",
         "head_test.go",
+        "init_sync_process_block_test.go",
         "init_test.go",
+        "lightclient_test.go",
         "log_test.go",
         "metrics_test.go",
         "mock_test.go",
@@ -126,98 +123,66 @@ go_test(
         "process_block_test.go",
         "receive_attestation_test.go",
         "receive_block_test.go",
+        "service_norace_test.go",
         "service_test.go",
         "setup_test.go",
         "weak_subjectivity_checks_test.go",
     ],
     embed = [":go_default_library"],
     gotags = ["develop"],
+    tags = ["CI_race_detection"],
     deps = [
         "//async/event:go_default_library",
         "//beacon-chain/blockchain/testing:go_default_library",
+        "//beacon-chain/cache:go_default_library",
         "//beacon-chain/cache/depositcache:go_default_library",
         "//beacon-chain/core/blocks:go_default_library",
+        "//beacon-chain/core/feed/state:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
+        "//beacon-chain/core/signing:go_default_library",
         "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/db/filesystem:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/execution:go_default_library",
         "//beacon-chain/execution/testing:go_default_library",
+        "//beacon-chain/forkchoice:go_default_library",
+        "//beacon-chain/forkchoice/doubly-linked-tree:go_default_library",
         "//beacon-chain/forkchoice/types:go_default_library",
+        "//beacon-chain/operations/attestations:go_default_library",
+        "//beacon-chain/operations/blstoexec:go_default_library",
+        "//beacon-chain/operations/slashings:go_default_library",
+        "//beacon-chain/operations/voluntaryexits:go_default_library",
         "//beacon-chain/p2p:go_default_library",
+        "//beacon-chain/startup:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/state-native:go_default_library",
-        "//beacon-chain/state/stateutil:go_default_library",
+        "//beacon-chain/state/stategen:go_default_library",
+        "//config/features:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
         "//consensus-types/blocks:go_default_library",
-        "//consensus-types/blocks/testing:go_default_library",
+        "//consensus-types/interfaces:go_default_library",
+        "//consensus-types/primitives:go_default_library",
         "//container/trie:go_default_library",
+        "//crypto/bls:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//proto/engine/v1:go_default_library",
+        "//proto/eth/v1:go_default_library",
+        "//proto/eth/v2:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
+        "//runtime/version:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/require:go_default_library",
         "//testing/util:go_default_library",
-        "@com_github_ethereum_go_ethereum//:go_default_library",
-        "@com_github_ethereum_go_ethereum//common:go_default_library",
-        "@com_github_ethereum_go_ethereum//core/types:go_default_library",
-        "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_sirupsen_logrus//hooks/test:go_default_library",
-        "@in_gopkg_d4l3k_messagediff_v1//:go_default_library",
-        "@org_golang_google_protobuf//proto:go_default_library",
-        "@org_golang_x_net//context:go_default_library",
-    ],
-)
-
-go_test(
-    name = "go_raceon_test",
-    srcs = [
-        "chain_info_norace_test.go",
-        "checktags_test.go",
-        "init_test.go",
-        "mock_test.go",
-        "receive_block_test.go",
-        "service_norace_test.go",
-        "setup_test.go",
-    ],
-    embed = [":go_default_library"],
-    gc_goopts = [
-        # Go 1.14 enables checkptr by default when building with -race or -msan. There is a pointer
-        # issue in boltdb, so must disable checkptr at compile time. This flag can be removed once
-        # the project is migrated to etcd's version of boltdb and the issue has been fixed.
-        # See: https://github.com/etcd-io/bbolt/issues/187.
-        "-d=checkptr=0",
-    ],
-    gotags = ["develop"],
-    race = "on",
-    tags = ["race_on"],
-    deps = [
-        "//async/event:go_default_library",
-        "//beacon-chain/blockchain/testing:go_default_library",
-        "//beacon-chain/cache/depositcache:go_default_library",
-        "//beacon-chain/core/blocks:go_default_library",
-        "//beacon-chain/core/helpers:go_default_library",
-        "//beacon-chain/core/transition:go_default_library",
-        "//beacon-chain/db:go_default_library",
-        "//beacon-chain/db/testing:go_default_library",
-        "//beacon-chain/execution:go_default_library",
-        "//beacon-chain/execution/testing:go_default_library",
-        "//beacon-chain/forkchoice/types:go_default_library",
-        "//beacon-chain/p2p:go_default_library",
-        "//config/params:go_default_library",
-        "//consensus-types/blocks:go_default_library",
-        "//consensus-types/blocks/testing:go_default_library",
-        "//container/trie:go_default_library",
-        "//encoding/bytesutil:go_default_library",
-        "//proto/prysm/v1alpha1:go_default_library",
-        "//testing/assert:go_default_library",
-        "//testing/require:go_default_library",
-        "//testing/util:go_default_library",
-        "@com_github_ethereum_go_ethereum//:go_default_library",
+        "//time:go_default_library",
+        "//time/slots:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
+        "@com_github_holiman_uint256//:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
         "@org_golang_google_protobuf//proto:go_default_library",
-        "@org_golang_x_net//context:go_default_library",
     ],
 )

beacon-chain/blockchain/chain_info.go

@@ -464,6 +464,13 @@ func (s *Service) IsOptimisticForRoot(ctx context.Context, root [32]byte) (bool,
 	return !isCanonical, nil
 }
 
+// TargetRootForEpoch wraps the corresponding method in forkchoice
+func (s *Service) TargetRootForEpoch(root [32]byte, epoch primitives.Epoch) ([32]byte, error) {
+	s.cfg.ForkChoiceStore.RLock()
+	defer s.cfg.ForkChoiceStore.RUnlock()
+	return s.cfg.ForkChoiceStore.TargetRootForEpoch(root, epoch)
+}
+
 // Ancestor returns the block root of an ancestry block from the input block root.
 //
 // Spec pseudocode definition:
@@ -530,3 +537,10 @@ func (s *Service) recoverStateSummary(ctx context.Context, blockRoot [32]byte) (
 func (s *Service) BlockBeingSynced(root [32]byte) bool {
 	return s.blockBeingSynced.isSyncing(root)
 }
+
+// RecentBlockSlot returns block slot form fork choice store
+func (s *Service) RecentBlockSlot(root [32]byte) (primitives.Slot, error) {
+	s.cfg.ForkChoiceStore.RLock()
+	defer s.cfg.ForkChoiceStore.RUnlock()
+	return s.cfg.ForkChoiceStore.Slot(root)
+}

beacon-chain/blockchain/error.go

@@ -30,6 +30,8 @@ var (
 	ErrNotCheckpoint = errors.New("not a checkpoint in forkchoice")
 )
 
+var errMaxBlobsExceeded = errors.New("Expected commitments in block exceeds MAX_BLOBS_PER_BLOCK")
+
 // An invalid block is the block that fails state transition based on the core protocol rules.
 // The beacon node shall not be accepting nor building blocks that branch off from an invalid block.
 // Some examples of invalid blocks are:

beacon-chain/blockchain/kzg/BUILD.bazel

@@ -10,6 +10,7 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain/kzg",
     visibility = ["//visibility:public"],
     deps = [
+        "//consensus-types/blocks:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "@com_github_crate_crypto_go_kzg_4844//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
@@ -26,6 +27,8 @@ go_test(
     deps = [
         "//proto/prysm/v1alpha1:go_default_library",
         "//testing/require:go_default_library",
+        "@com_github_consensys_gnark_crypto//ecc/bls12-381/fr:go_default_library",
         "@com_github_crate_crypto_go_kzg_4844//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
     ],
 )

beacon-chain/blockchain/kzg/validation.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	GoKZG "github.com/crate-crypto/go-kzg-4844"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 )
 
@@ -31,6 +32,11 @@ func IsDataAvailable(commitments [][]byte, sidecars []*ethpb.DeprecatedBlobSidec
 	return kzgContext.VerifyBlobKZGProofBatch(blobs, cmts, proofs)
 }
 
+// VerifyROBlobCommitment is a helper that massages the fields of an ROBlob into the types needed to call VerifyBlobKZGProof.
+func VerifyROBlobCommitment(sc blocks.ROBlob) error {
+	return kzgContext.VerifyBlobKZGProof(bytesToBlob(sc.Blob), bytesToCommitment(sc.KzgCommitment), bytesToKZGProof(sc.KzgProof))
+}
+
 func bytesToBlob(blob []byte) (ret GoKZG.Blob) {
 	copy(ret[:], blob)
 	return

beacon-chain/blockchain/kzg/validation_test.go

@@ -1,13 +1,63 @@
 package kzg
 
 import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/binary"
 	"testing"
 
+	"github.com/consensys/gnark-crypto/ecc/bls12-381/fr"
 	GoKZG "github.com/crate-crypto/go-kzg-4844"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v4/testing/require"
+	"github.com/sirupsen/logrus"
 )
 
+func deterministicRandomness(seed int64) [32]byte {
+	// Converts an int64 to a byte slice
+	buf := new(bytes.Buffer)
+	err := binary.Write(buf, binary.BigEndian, seed)
+	if err != nil {
+		logrus.WithError(err).Error("Failed to write int64 to bytes buffer")
+		return [32]byte{}
+	}
+	bytes := buf.Bytes()
+
+	return sha256.Sum256(bytes)
+}
+
+// Returns a serialized random field element in big-endian
+func GetRandFieldElement(seed int64) [32]byte {
+	bytes := deterministicRandomness(seed)
+	var r fr.Element
+	r.SetBytes(bytes[:])
+
+	return GoKZG.SerializeScalar(r)
+}
+
+// Returns a random blob using the passed seed as entropy
+func GetRandBlob(seed int64) GoKZG.Blob {
+	var blob GoKZG.Blob
+	bytesPerBlob := GoKZG.ScalarsPerBlob * GoKZG.SerializedScalarSize
+	for i := 0; i < bytesPerBlob; i += GoKZG.SerializedScalarSize {
+		fieldElementBytes := GetRandFieldElement(seed + int64(i))
+		copy(blob[i:i+GoKZG.SerializedScalarSize], fieldElementBytes[:])
+	}
+	return blob
+}
+
+func GenerateCommitmentAndProof(blob GoKZG.Blob) (GoKZG.KZGCommitment, GoKZG.KZGProof, error) {
+	commitment, err := kzgContext.BlobToKZGCommitment(blob, 0)
+	if err != nil {
+		return GoKZG.KZGCommitment{}, GoKZG.KZGProof{}, err
+	}
+	proof, err := kzgContext.ComputeBlobKZGProof(blob, commitment, 0)
+	if err != nil {
+		return GoKZG.KZGCommitment{}, GoKZG.KZGProof{}, err
+	}
+	return commitment, proof, err
+}
+
 func TestIsDataAvailable(t *testing.T) {
 	sidecars := make([]*ethpb.DeprecatedBlobSidecar, 0)
 	commitments := make([][]byte, 0)
@@ -23,3 +73,13 @@ func TestBytesToAny(t *testing.T) {
 	require.DeepEqual(t, commitment, bytesToCommitment(bytes))
 	require.DeepEqual(t, proof, bytesToKZGProof(bytes))
 }
+
+func TestGenerateCommitmentAndProof(t *testing.T) {
+	blob := GetRandBlob(123)
+	commitment, proof, err := GenerateCommitmentAndProof(blob)
+	require.NoError(t, err)
+	expectedCommitment := GoKZG.KZGCommitment{180, 218, 156, 194, 59, 20, 10, 189, 186, 254, 132, 93, 7, 127, 104, 172, 238, 240, 237, 70, 83, 89, 1, 152, 99, 0, 165, 65, 143, 62, 20, 215, 230, 14, 205, 95, 28, 245, 54, 25, 160, 16, 178, 31, 232, 207, 38, 85}
+	expectedProof := GoKZG.KZGProof{128, 110, 116, 170, 56, 111, 126, 87, 229, 234, 211, 42, 110, 150, 129, 206, 73, 142, 167, 243, 90, 149, 240, 240, 236, 204, 143, 182, 229, 249, 81, 27, 153, 171, 83, 70, 144, 250, 42, 1, 188, 215, 71, 235, 30, 7, 175, 86}
+	require.Equal(t, expectedCommitment, commitment)
+	require.Equal(t, expectedProof, proof)
+}

beacon-chain/blockchain/log.go

@@ -147,15 +147,3 @@ func logPayload(block interfaces.ReadOnlyBeaconBlock) error {
 	log.WithFields(fields).Debug("Synced new payload")
 	return nil
 }
-
-func logBlobSidecar(scs []*ethpb.DeprecatedBlobSidecar, startTime time.Time) {
-	if len(scs) == 0 {
-		return
-	}
-	log.WithFields(logrus.Fields{
-		"count":          len(scs),
-		"slot":           scs[0].Slot,
-		"block":          hex.EncodeToString(scs[0].BlockRoot),
-		"validationTime": time.Since(startTime),
-	}).Debug("Synced new blob sidecars")
-}

beacon-chain/blockchain/options.go

@@ -5,6 +5,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
 	statefeed "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/execution"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/operations/attestations"
@@ -164,6 +165,8 @@ func WithFinalizedStateAtStartUp(st state.BeaconState) Option {
 	}
 }
 
+// WithClockSychronizer sets the ClockSetter/ClockWaiter values to be used by services that need to block until
+// the genesis timestamp is known (ClockWaiter) or which determine the genesis timestamp (ClockSetter).
 func WithClockSynchronizer(gs *startup.ClockSynchronizer) Option {
 	return func(s *Service) error {
 		s.clockSetter = gs
@@ -172,9 +175,18 @@ func WithClockSynchronizer(gs *startup.ClockSynchronizer) Option {
 	}
 }
 
+// WithSyncComplete sets a channel that is used to notify blockchain service that the node has synced to head.
 func WithSyncComplete(c chan struct{}) Option {
 	return func(s *Service) error {
 		s.syncComplete = c
 		return nil
 	}
 }
+
+// WithBlobStorage sets the blob storage backend for the blockchain service.
+func WithBlobStorage(b *filesystem.BlobStorage) Option {
+	return func(s *Service) error {
+		s.blobStorage = b
+		return nil
+	}
+}

beacon-chain/blockchain/process_attestation_test.go

@@ -2,6 +2,7 @@ package blockchain
 
 import (
 	"context"
+	"sync"
 	"testing"
 	"time"
 
@@ -145,6 +146,61 @@ func TestStore_OnAttestation_Ok_DoublyLinkedTree(t *testing.T) {
 	require.NoError(t, service.OnAttestation(ctx, att[0], 0))
 }
 
+func TestService_GetAttPreState_Concurrency(t *testing.T) {
+	service, _ := minimalTestService(t)
+	ctx := context.Background()
+
+	s, err := util.NewBeaconState()
+	require.NoError(t, err)
+	ckRoot := bytesutil.PadTo([]byte{'A'}, fieldparams.RootLength)
+	err = s.SetFinalizedCheckpoint(&ethpb.Checkpoint{Root: ckRoot})
+	require.NoError(t, err)
+	val := &ethpb.Validator{PublicKey: bytesutil.PadTo([]byte("foo"), 48), WithdrawalCredentials: bytesutil.PadTo([]byte("bar"), fieldparams.RootLength)}
+	err = s.SetValidators([]*ethpb.Validator{val})
+	require.NoError(t, err)
+	err = s.SetBalances([]uint64{0})
+	require.NoError(t, err)
+	r := [32]byte{'g'}
+	require.NoError(t, service.cfg.BeaconDB.SaveState(ctx, s, r))
+
+	cp1 := &ethpb.Checkpoint{Epoch: 1, Root: ckRoot}
+	require.NoError(t, service.cfg.BeaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'A'})))
+	require.NoError(t, service.cfg.BeaconDB.SaveStateSummary(ctx, &ethpb.StateSummary{Root: ckRoot}))
+
+	st, root, err := prepareForkchoiceState(ctx, 100, [32]byte(cp1.Root), [32]byte{}, [32]byte{'R'}, cp1, cp1)
+	require.NoError(t, err)
+	require.NoError(t, service.cfg.ForkChoiceStore.InsertNode(ctx, st, root))
+
+	var wg sync.WaitGroup
+	errChan := make(chan error, 1000)
+
+	for i := 0; i < 1000; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			cp1 := &ethpb.Checkpoint{Epoch: 1, Root: ckRoot}
+			_, err := service.getAttPreState(ctx, cp1)
+			if err != nil {
+				errChan <- err
+			}
+		}()
+	}
+
+	go func() {
+		wg.Wait()
+		close(errChan)
+	}()
+
+	select {
+	case <-time.After(10 * time.Second):
+		t.Fatal("Test timed out")
+	case err, ok := <-errChan:
+		if ok && err != nil {
+			require.ErrorContains(t, "not a checkpoint in forkchoice", err)
+		}
+	}
+}
+
 func TestStore_SaveCheckpointState(t *testing.T) {
 	service, tr := minimalTestService(t)
 	ctx := tr.ctx

beacon-chain/blockchain/process_block.go

@@ -6,17 +6,17 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain/kzg"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed"
 	statefeed "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/helpers"
 	coreTime "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/time"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/transition"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/v4/config/features"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	consensusblocks "github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
@@ -353,11 +353,15 @@ func (s *Service) databaseDACheck(ctx context.Context, b consensusblocks.ROBlock
 	if len(commitments) == 0 {
 		return nil
 	}
-	sidecars, err := s.cfg.BeaconDB.BlobSidecarsByRoot(ctx, b.Root())
+	missing, err := missingIndices(s.blobStorage, b.Root(), commitments)
 	if err != nil {
-		return errors.Wrap(err, "could not get blob sidecars")
+		return err
 	}
-	return kzg.IsDataAvailable(commitments, sidecars)
+	if len(missing) == 0 {
+		return nil
+	}
+	// TODO: don't worry that this error isn't informative, it will be superceded by a detailed sidecar cache error.
+	return errors.New("not all kzg commitments are available")
 }
 
 func (s *Service) updateEpochBoundaryCaches(ctx context.Context, st state.BeaconState) error {
@@ -529,11 +533,43 @@ func (s *Service) runLateBlockTasks() {
 	}
 }
 
+// missingIndices uses the expected commitments from the block to determine
+// which BlobSidecar indices would need to be in the database for DA success.
+// It returns a map where each key represents a missing BlobSidecar index.
+// An empty map means we have all indices; a non-empty map can be used to compare incoming
+// BlobSidecars against the set of known missing sidecars.
+func missingIndices(bs *filesystem.BlobStorage, root [32]byte, expected [][]byte) (map[uint64]struct{}, error) {
+	if len(expected) == 0 {
+		return nil, nil
+	}
+	if len(expected) > fieldparams.MaxBlobsPerBlock {
+		return nil, errMaxBlobsExceeded
+	}
+	indices, err := bs.Indices(root)
+	if err != nil {
+		return nil, err
+	}
+	missing := make(map[uint64]struct{}, len(expected))
+	for i := range expected {
+		ui := uint64(i)
+		if len(expected[i]) > 0 {
+			if !indices[i] {
+				missing[ui] = struct{}{}
+			}
+		}
+	}
+	return missing, nil
+}
+
+// isDataAvailable blocks until all BlobSidecars committed to in the block are available,
+// or an error or context cancellation occurs. A nil result means that the data availability check is successful.
+// The function will first check the database to see if all sidecars have been persisted. If any
+// sidecars are missing, it will then read from the blobNotifier channel for the given root until the channel is
+// closed, the context hits cancellation/timeout, or notifications have been received for all the missing sidecars.
 func (s *Service) isDataAvailable(ctx context.Context, root [32]byte, signed interfaces.ReadOnlySignedBeaconBlock) error {
 	if signed.Version() < version.Deneb {
 		return nil
 	}
-	t := time.Now()
 
 	block := signed.Block()
 	if block == nil {
@@ -552,59 +588,35 @@ func (s *Service) isDataAvailable(ctx context.Context, root [32]byte, signed int
 	if err != nil {
 		return errors.Wrap(err, "could not get KZG commitments")
 	}
+	// expected is the number of kzg commitments observed in the block.
 	expected := len(kzgCommitments)
 	if expected == 0 {
 		return nil
 	}
-
-	// Read first from db in case we have the blobs
-	sidecars, err := s.cfg.BeaconDB.BlobSidecarsByRoot(ctx, root)
-	switch {
-	case err == nil:
-		if len(sidecars) >= expected {
-			s.blobNotifiers.delete(root)
-			if err := kzg.IsDataAvailable(kzgCommitments, sidecars); err != nil {
-				log.WithField("root", fmt.Sprintf("%#x", root)).Warn("removing blob sidecars with invalid proofs")
-				if err2 := s.cfg.BeaconDB.DeleteBlobSidecars(ctx, root); err2 != nil {
-					log.WithError(err2).Error("could not delete sidecars")
-				}
-				return err
-			}
-			logBlobSidecar(sidecars, t)
-			return nil
-		}
-	case errors.Is(err, db.ErrNotFound):
-		// If the blob sidecars haven't arrived yet, the subsequent code will wait for them.
-		// Note: The system will not exit with an error in this scenario.
-	default:
-		log.WithError(err).Error("could not get blob sidecars from DB")
+	// get a map of BlobSidecar indices that are not currently available.
+	missing, err := missingIndices(s.blobStorage, root, kzgCommitments)
+	if err != nil {
+		return err
+	}
+	// If there are no missing indices, all BlobSidecars are available.
+	if len(missing) == 0 {
+		return nil
 	}
 
-	found := map[uint64]struct{}{}
-	for _, sc := range sidecars {
-		found[sc.Index] = struct{}{}
-	}
+	// The gossip handler for blobs writes the index of each verified blob referencing the given
+	// root to the channel returned by blobNotifiers.forRoot.
 	nc := s.blobNotifiers.forRoot(root)
 	for {
 		select {
 		case idx := <-nc:
-			found[idx] = struct{}{}
-			if len(found) != expected {
+			// Delete each index seen in the notification channel.
+			delete(missing, idx)
+			// Read from the channel until there are no more missing sidecars.
+			if len(missing) > 0 {
 				continue
 			}
+			// Once all sidecars have been observed, clean up the notification channel.
 			s.blobNotifiers.delete(root)
-			sidecars, err := s.cfg.BeaconDB.BlobSidecarsByRoot(ctx, root)
-			if err != nil {
-				return errors.Wrap(err, "could not get blob sidecars")
-			}
-			if err := kzg.IsDataAvailable(kzgCommitments, sidecars); err != nil {
-				log.WithField("root", fmt.Sprintf("%#x", root)).Warn("removing blob sidecars with invalid proofs")
-				if err2 := s.cfg.BeaconDB.DeleteBlobSidecars(ctx, root); err2 != nil {
-					log.WithError(err2).Error("could not delete sidecars")
-				}
-				return err
-			}
-			logBlobSidecar(sidecars, t)
 			return nil
 		case <-ctx.Done():
 			return errors.Wrap(ctx.Err(), "context deadline waiting for blob sidecars")

beacon-chain/blockchain/process_block_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/signing"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/transition"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	testDB "github.com/prysmaticlabs/prysm/v4/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/execution"
 	mockExecution "github.com/prysmaticlabs/prysm/v4/beacon-chain/execution/testing"
@@ -2115,3 +2116,99 @@ func Test_commitmentsToCheck(t *testing.T) {
 		})
 	}
 }
+
+func TestMissingIndices(t *testing.T) {
+	cases := []struct {
+		name     string
+		expected [][]byte
+		present  []uint64
+		result   map[uint64]struct{}
+		root     [32]byte
+		err      error
+	}{
+		{
+			name: "zero len",
+		},
+		{
+			name:     "expected exceeds max",
+			expected: fakeCommitments(fieldparams.MaxBlobsPerBlock + 1),
+			err:      errMaxBlobsExceeded,
+		},
+		{
+			name:     "first missing",
+			expected: fakeCommitments(fieldparams.MaxBlobsPerBlock),
+			present:  []uint64{1, 2, 3, 4, 5},
+			result:   fakeResult([]uint64{0}),
+		},
+		{
+			name:     "all missing",
+			expected: fakeCommitments(fieldparams.MaxBlobsPerBlock),
+			result:   fakeResult([]uint64{0, 1, 2, 3, 4, 5}),
+		},
+		{
+			name:     "none missing",
+			expected: fakeCommitments(fieldparams.MaxBlobsPerBlock),
+			present:  []uint64{0, 1, 2, 3, 4, 5},
+			result:   fakeResult([]uint64{}),
+		},
+		{
+			name:     "one commitment, missing",
+			expected: fakeCommitments(1),
+			present:  []uint64{},
+			result:   fakeResult([]uint64{0}),
+		},
+		{
+			name:     "3 commitments, 1 missing",
+			expected: fakeCommitments(3),
+			present:  []uint64{1},
+			result:   fakeResult([]uint64{0, 2}),
+		},
+		{
+			name:     "3 commitments, none missing",
+			expected: fakeCommitments(3),
+			present:  []uint64{0, 1, 2},
+			result:   fakeResult([]uint64{}),
+		},
+		{
+			name:     "3 commitments, all missing",
+			expected: fakeCommitments(3),
+			present:  []uint64{},
+			result:   fakeResult([]uint64{0, 1, 2}),
+		},
+	}
+
+	for _, c := range cases {
+		bm, bs := filesystem.NewEphemeralBlobStorageWithMocker(t)
+		t.Run(c.name, func(t *testing.T) {
+			require.NoError(t, bm.CreateFakeIndices(c.root, c.present))
+			missing, err := missingIndices(bs, c.root, c.expected)
+			if c.err != nil {
+				require.ErrorIs(t, err, c.err)
+				return
+			}
+			require.NoError(t, err)
+			require.Equal(t, len(c.result), len(missing))
+			for key := range c.result {
+				m, ok := missing[key]
+				require.Equal(t, true, ok)
+				require.Equal(t, c.result[key], m)
+			}
+		})
+	}
+}
+
+func fakeCommitments(n int) [][]byte {
+	f := make([][]byte, n)
+	for i := range f {
+		f[i] = make([]byte, 48)
+	}
+	return f
+}
+
+func fakeResult(missing []uint64) map[uint64]struct{} {
+	r := make(map[uint64]struct{}, len(missing))
+	for i := range missing {
+		r[missing[i]] = struct{}{}
+	}
+	return r
+}

beacon-chain/blockchain/receive_attestation.go

@@ -53,15 +53,11 @@ func (s *Service) AttestationTargetState(ctx context.Context, target *ethpb.Chec
 
 // VerifyLmdFfgConsistency verifies that attestation's LMD and FFG votes are consistency to each other.
 func (s *Service) VerifyLmdFfgConsistency(ctx context.Context, a *ethpb.Attestation) error {
-	targetSlot, err := slots.EpochStart(a.Data.Target.Epoch)
+	r, err := s.TargetRootForEpoch([32]byte(a.Data.BeaconBlockRoot), a.Data.Target.Epoch)
 	if err != nil {
 		return err
 	}
-	r, err := s.Ancestor(ctx, a.Data.BeaconBlockRoot, targetSlot)
-	if err != nil {
-		return err
-	}
-	if !bytes.Equal(a.Data.Target.Root, r) {
+	if !bytes.Equal(a.Data.Target.Root, r[:]) {
 		return fmt.Errorf("FFG and LMD votes are not consistent, block root: %#x, target root: %#x, canonical target root: %#x", a.Data.BeaconBlockRoot, a.Data.Target.Root, r)
 	}
 	return nil

beacon-chain/blockchain/receive_attestation_test.go

@@ -36,50 +36,28 @@ func TestAttestationCheckPtState_FarFutureSlot(t *testing.T) {
 	require.ErrorContains(t, "exceeds max allowed value relative to the local clock", err)
 }
 
-func TestVerifyLMDFFGConsistent_NotOK(t *testing.T) {
+func TestVerifyLMDFFGConsistent(t *testing.T) {
 	service, tr := minimalTestService(t)
 	ctx := tr.ctx
 
-	b32 := util.NewBeaconBlock()
-	b32.Block.Slot = 32
-	util.SaveBlock(t, ctx, service.cfg.BeaconDB, b32)
-	r32, err := b32.Block.HashTreeRoot()
+	f := service.cfg.ForkChoiceStore
+	fc := &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	state, r32, err := prepareForkchoiceState(ctx, 32, [32]byte{'a'}, params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, fc, fc)
 	require.NoError(t, err)
-	b33 := util.NewBeaconBlock()
-	b33.Block.Slot = 33
-	b33.Block.ParentRoot = r32[:]
-	util.SaveBlock(t, ctx, service.cfg.BeaconDB, b33)
-	r33, err := b33.Block.HashTreeRoot()
+	require.NoError(t, f.InsertNode(ctx, state, r32))
+
+	state, r33, err := prepareForkchoiceState(ctx, 33, [32]byte{'b'}, r32, params.BeaconConfig().ZeroHash, fc, fc)
 	require.NoError(t, err)
+	require.NoError(t, f.InsertNode(ctx, state, r33))
 
 	wanted := "FFG and LMD votes are not consistent"
 	a := util.NewAttestation()
 	a.Data.Target.Epoch = 1
-	a.Data.Target.Root = []byte{'a'}
+	a.Data.Target.Root = []byte{'c'}
 	a.Data.BeaconBlockRoot = r33[:]
 	require.ErrorContains(t, wanted, service.VerifyLmdFfgConsistency(context.Background(), a))
-}
 
-func TestVerifyLMDFFGConsistent_OK(t *testing.T) {
-	service, tr := minimalTestService(t)
-	ctx := tr.ctx
-
-	b32 := util.NewBeaconBlock()
-	b32.Block.Slot = 32
-	util.SaveBlock(t, ctx, service.cfg.BeaconDB, b32)
-	r32, err := b32.Block.HashTreeRoot()
-	require.NoError(t, err)
-	b33 := util.NewBeaconBlock()
-	b33.Block.Slot = 33
-	b33.Block.ParentRoot = r32[:]
-	util.SaveBlock(t, ctx, service.cfg.BeaconDB, b33)
-	r33, err := b33.Block.HashTreeRoot()
-	require.NoError(t, err)
-
-	a := util.NewAttestation()
-	a.Data.Target.Epoch = 1
 	a.Data.Target.Root = r32[:]
-	a.Data.BeaconBlockRoot = r33[:]
 	err = service.VerifyLmdFfgConsistency(context.Background(), a)
 	require.NoError(t, err, "Could not verify LMD and FFG votes to be consistent")
 }

beacon-chain/blockchain/receive_blob.go

@@ -3,7 +3,7 @@ package blockchain
 import (
 	"context"
 
-	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 )
 
 // SendNewBlobEvent sends a message to the BlobNotifier channel that the blob
@@ -13,11 +13,11 @@ func (s *Service) sendNewBlobEvent(root [32]byte, index uint64) {
 }
 
 // ReceiveBlob saves the blob to database and sends the new event
-func (s *Service) ReceiveBlob(ctx context.Context, b *ethpb.DeprecatedBlobSidecar) error {
-	if err := s.cfg.BeaconDB.SaveBlobSidecar(ctx, []*ethpb.DeprecatedBlobSidecar{b}); err != nil {
+func (s *Service) ReceiveBlob(ctx context.Context, b blocks.VerifiedROBlob) error {
+	if err := s.blobStorage.Save(b); err != nil {
 		return err
 	}
 
-	s.sendNewBlobEvent([32]byte(b.BlockRoot), b.Index)
+	s.sendNewBlobEvent(b.BlockRoot(), b.Index)
 	return nil
 }

beacon-chain/blockchain/receive_block.go

@@ -43,7 +43,7 @@ type BlockReceiver interface {
 // BlobReceiver interface defines the methods of chain service for receiving new
 // blobs
 type BlobReceiver interface {
-	ReceiveBlob(context.Context, *ethpb.DeprecatedBlobSidecar) error
+	ReceiveBlob(context.Context, blocks.VerifiedROBlob) error
 }
 
 // SlashingReceiver interface defines the methods of chain service for receiving validated slashing over the wire.
@@ -258,11 +258,6 @@ func (s *Service) HasBlock(ctx context.Context, root [32]byte) bool {
 	return s.hasBlockInInitSyncOrDB(ctx, root)
 }
 
-// RecentBlockSlot returns block slot form fork choice store
-func (s *Service) RecentBlockSlot(root [32]byte) (primitives.Slot, error) {
-	return s.cfg.ForkChoiceStore.Slot(root)
-}
-
 // ReceiveAttesterSlashing receives an attester slashing and inserts it to forkchoice
 func (s *Service) ReceiveAttesterSlashing(ctx context.Context, slashing *ethpb.AttesterSlashing) {
 	s.cfg.ForkChoiceStore.Lock()

beacon-chain/blockchain/service.go

@@ -20,6 +20,7 @@ import (
 	coreTime "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/time"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/core/transition"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/execution"
 	f "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice"
 	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
@@ -63,6 +64,7 @@ type Service struct {
 	syncComplete         chan struct{}
 	blobNotifiers        *blobNotifierMap
 	blockBeingSynced     *currentlySyncingBlock
+	blobStorage          *filesystem.BlobStorage
 }
 
 // config options for the service.

beacon-chain/blockchain/setup_test.go

@@ -56,7 +56,7 @@ func (mb *mockBroadcaster) BroadcastSyncCommitteeMessage(_ context.Context, _ ui
 	return nil
 }
 
-func (mb *mockBroadcaster) BroadcastBlob(_ context.Context, _ uint64, _ *ethpb.SignedBlobSidecar) error {
+func (mb *mockBroadcaster) BroadcastBlob(_ context.Context, _ uint64, _ *ethpb.BlobSidecar) error {
 	mb.broadcastCalled = true
 	return nil
 }

beacon-chain/blockchain/testing/mock.go

@@ -72,7 +72,7 @@ type ChainService struct {
 	OptimisticRoots             map[[32]byte]bool
 	BlockSlot                   primitives.Slot
 	SyncingRoot                 [32]byte
-	Blobs                       []*ethpb.DeprecatedBlobSidecar
+	Blobs                       []blocks.VerifiedROBlob
 }
 
 func (s *ChainService) Ancestor(ctx context.Context, root []byte, slot primitives.Slot) ([]byte, error) {
@@ -613,7 +613,7 @@ func (c *ChainService) BlockBeingSynced(root [32]byte) bool {
 }
 
 // ReceiveBlob implements the same method in the chain service
-func (c *ChainService) ReceiveBlob(_ context.Context, b *ethpb.DeprecatedBlobSidecar) error {
+func (c *ChainService) ReceiveBlob(_ context.Context, b blocks.VerifiedROBlob) error {
 	c.Blobs = append(c.Blobs, b)
 	return nil
 }

beacon-chain/builder/BUILD.bazel

@@ -15,7 +15,6 @@ go_library(
         "//beacon-chain/cache:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",
-        "//config/fieldparams:go_default_library",
         "//consensus-types/interfaces:go_default_library",
         "//consensus-types/primitives:go_default_library",
         "//encoding/bytesutil:go_default_library",

beacon-chain/builder/service.go

@@ -2,7 +2,6 @@ package builder
 
 import (
 	"context"
-	"fmt"
 	"reflect"
 	"time"
 
@@ -11,7 +10,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
-	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/interfaces"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
@@ -27,7 +25,7 @@ var ErrNoBuilder = errors.New("builder endpoint not configured")
 
 // BlockBuilder defines the interface for interacting with the block builder
 type BlockBuilder interface {
-	SubmitBlindedBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock, blobs []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error)
+	SubmitBlindedBlock(ctx context.Context, block interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error)
 	GetHeader(ctx context.Context, slot primitives.Slot, parentHash [32]byte, pubKey [48]byte) (builder.SignedBid, error)
 	RegisterValidator(ctx context.Context, reg []*ethpb.SignedValidatorRegistrationV1) error
 	RegistrationByValidatorID(ctx context.Context, id primitives.ValidatorIndex) (*ethpb.ValidatorRegistrationV1, error)
@@ -90,7 +88,7 @@ func (s *Service) Stop() error {
 }
 
 // SubmitBlindedBlock submits a blinded block to the builder relay network.
-func (s *Service) SubmitBlindedBlock(ctx context.Context, b interfaces.ReadOnlySignedBeaconBlock, blobs []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
+func (s *Service) SubmitBlindedBlock(ctx context.Context, b interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
 	ctx, span := trace.StartSpan(ctx, "builder.SubmitBlindedBlock")
 	defer span.End()
 	start := time.Now()
@@ -100,11 +98,8 @@ func (s *Service) SubmitBlindedBlock(ctx context.Context, b interfaces.ReadOnlyS
 	if s.c == nil {
 		return nil, nil, ErrNoBuilder
 	}
-	if uint64(len(blobs)) > fieldparams.MaxBlobsPerBlock {
-		return nil, nil, fmt.Errorf("blob count %d beyond max limit of %d", len(blobs), fieldparams.MaxBlobsPerBlock)
-	}
 
-	return s.c.SubmitBlindedBlock(ctx, b, blobs)
+	return s.c.SubmitBlindedBlock(ctx, b)
 }
 
 // GetHeader retrieves the header for a given slot and parent hash from the builder relay network.

beacon-chain/builder/service_test.go

@@ -62,7 +62,7 @@ func Test_BuilderMethodsWithouClient(t *testing.T) {
 	_, err = s.GetHeader(context.Background(), 0, [32]byte{}, [48]byte{})
 	assert.ErrorContains(t, ErrNoBuilder.Error(), err)
 
-	_, _, err = s.SubmitBlindedBlock(context.Background(), nil, nil)
+	_, _, err = s.SubmitBlindedBlock(context.Background(), nil)
 	assert.ErrorContains(t, ErrNoBuilder.Error(), err)
 
 	err = s.RegisterValidator(context.Background(), nil)

beacon-chain/builder/testing/mock.go

@@ -45,7 +45,7 @@ func (s *MockBuilderService) Configured() bool {
 }
 
 // SubmitBlindedBlock for mocking.
-func (s *MockBuilderService) SubmitBlindedBlock(_ context.Context, b interfaces.ReadOnlySignedBeaconBlock, _ []*ethpb.SignedBlindedBlobSidecar) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
+func (s *MockBuilderService) SubmitBlindedBlock(_ context.Context, b interfaces.ReadOnlySignedBeaconBlock) (interfaces.ExecutionData, *v1.BlobsBundle, error) {
 	switch b.Version() {
 	case version.Bellatrix:
 		w, err := blocks.WrappedExecutionPayload(s.Payload)

beacon-chain/core/blocks/exit_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/testing/assert"
 	"github.com/prysmaticlabs/prysm/v4/testing/require"
 	"github.com/prysmaticlabs/prysm/v4/testing/util"
+	"github.com/prysmaticlabs/prysm/v4/time/slots"
 )
 
 func TestProcessVoluntaryExits_NotActiveLongEnoughToExit(t *testing.T) {
@@ -134,6 +135,10 @@ func TestProcessVoluntaryExits_AppliesCorrectStatus(t *testing.T) {
 }
 
 func TestVerifyExitAndSignature(t *testing.T) {
+	undo := util.HackDenebMaxuint(t)
+	defer undo()
+	denebSlot, err := slots.EpochStart(params.BeaconConfig().DenebForkEpoch)
+	require.NoError(t, err)
 	tests := []struct {
 		name    string
 		setup   func() (*ethpb.Validator, *ethpb.SignedVoluntaryExit, state.ReadOnlyBeaconState, error)
@@ -241,11 +246,11 @@ func TestVerifyExitAndSignature(t *testing.T) {
 				fork := &ethpb.Fork{
 					PreviousVersion: params.BeaconConfig().CapellaForkVersion,
 					CurrentVersion:  params.BeaconConfig().DenebForkVersion,
-					Epoch:           primitives.Epoch(2),
+					Epoch:           params.BeaconConfig().DenebForkEpoch,
 				}
 				signedExit := &ethpb.SignedVoluntaryExit{
 					Exit: &ethpb.VoluntaryExit{
-						Epoch:          2,
+						Epoch:          params.BeaconConfig().CapellaForkEpoch,
 						ValidatorIndex: 0,
 					},
 				}
@@ -253,7 +258,7 @@ func TestVerifyExitAndSignature(t *testing.T) {
 				bs, err := state_native.InitializeFromProtoUnsafeDeneb(&ethpb.BeaconStateDeneb{
 					GenesisValidatorsRoot: bs.GenesisValidatorsRoot(),
 					Fork:                  fork,
-					Slot:                  (params.BeaconConfig().SlotsPerEpoch * 2) + 1,
+					Slot:                  denebSlot,
 					Validators:            bs.Validators(),
 				})
 				if err != nil {

beacon-chain/core/helpers/BUILD.bazel

@@ -58,8 +58,8 @@ go_test(
         "weak_subjectivity_test.go",
     ],
     embed = [":go_default_library"],
-    race = "on",
     shard_count = 2,
+    tags = ["CI_race_detection"],
     deps = [
         "//beacon-chain/cache:go_default_library",
         "//beacon-chain/core/time:go_default_library",

beacon-chain/core/signing/BUILD.bazel

@@ -16,7 +16,6 @@ go_library(
         "//crypto/bls:go_default_library",
         "//encoding/bytesutil:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
-        "//runtime/version:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prysmaticlabs_fastssz//:go_default_library",
     ],

beacon-chain/core/signing/signing_root.go

@@ -11,7 +11,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/crypto/bls"
 	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
-	"github.com/prysmaticlabs/prysm/v4/runtime/version"
 )
 
 // ForkVersionByteLength length of fork version byte array.
@@ -57,18 +56,22 @@ const (
 
 // ComputeDomainAndSign computes the domain and signing root and sign it using the passed in private key.
 func ComputeDomainAndSign(st state.ReadOnlyBeaconState, epoch primitives.Epoch, obj fssz.HashRoot, domain [4]byte, key bls.SecretKey) ([]byte, error) {
-	fork := st.Fork()
+	return ComputeDomainAndSignWithoutState(st.Fork(), epoch, domain, st.GenesisValidatorsRoot(), obj, key)
+}
+
+// ComputeDomainAndSignWithoutState offers the same functionalit as ComputeDomainAndSign without the need to provide a BeaconState.
+// This is particularly helpful for signing values in tests.
+func ComputeDomainAndSignWithoutState(fork *ethpb.Fork, epoch primitives.Epoch, domain [4]byte, vr []byte, obj fssz.HashRoot, key bls.SecretKey) ([]byte, error) {
 	// EIP-7044: Beginning in Deneb, fix the fork version to Capella for signed exits.
 	// This allows for signed validator exits to be valid forever.
-	if st.Version() >= version.Deneb && domain == params.BeaconConfig().DomainVoluntaryExit {
+	if domain == params.BeaconConfig().DomainVoluntaryExit && epoch >= params.BeaconConfig().DenebForkEpoch {
 		fork = &ethpb.Fork{
 			PreviousVersion: params.BeaconConfig().CapellaForkVersion,
 			CurrentVersion:  params.BeaconConfig().CapellaForkVersion,
 			Epoch:           params.BeaconConfig().CapellaForkEpoch,
 		}
 	}
-
-	d, err := Domain(fork, epoch, domain, st.GenesisValidatorsRoot())
+	d, err := Domain(fork, epoch, domain, vr)
 	if err != nil {
 		return nil, err
 	}
@@ -102,8 +105,14 @@ func Data(rootFunc func() ([32]byte, error), domain []byte) ([32]byte, error) {
 	if err != nil {
 		return [32]byte{}, err
 	}
+	return ComputeSigningRootForRoot(objRoot, domain)
+}
+
+// ComputeSigningRootForRoot works the same as ComputeSigningRoot,
+// except that gets the root from an argument instead of a callback.
+func ComputeSigningRootForRoot(root [32]byte, domain []byte) ([32]byte, error) {
 	container := &ethpb.SigningData{
-		ObjectRoot: objRoot[:],
+		ObjectRoot: root[:],
 		Domain:     domain,
 	}
 	return container.HashTreeRoot()

beacon-chain/db/db.go

@@ -3,16 +3,9 @@
 package db
 
 import (
-	"context"
-
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/kv"
 )
 
-// NewDB initializes a new DB.
-func NewDB(ctx context.Context, dirPath string) (Database, error) {
-	return kv.NewKVStore(ctx, dirPath)
-}
-
 // NewFileName uses the KVStoreDatafilePath so that if this layer of
 // indirection between db.NewDB->kv.NewKVStore ever changes, it will be easy to remember
 // to also change this filename indirection at the same time.

beacon-chain/db/errors.go

@@ -1,6 +1,11 @@
 package db
 
-import "github.com/prysmaticlabs/prysm/v4/beacon-chain/db/kv"
+import (
+	"errors"
+	"os"
+
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/kv"
+)
 
 // ErrNotFound can be used to determine if an error from a method in the database package
 // represents a "not found" error. These often require different handling than a low-level
@@ -19,3 +24,9 @@ var ErrNotFoundBackfillBlockRoot = kv.ErrNotFoundBackfillBlockRoot
 
 // ErrNotFoundGenesisBlockRoot means no genesis block root was found, indicating the db was not initialized with genesis
 var ErrNotFoundGenesisBlockRoot = kv.ErrNotFoundGenesisBlockRoot
+
+// IsNotFound allows callers to treat errors from a flat-file database, where the file record is missing,
+// as equivalent to db.ErrNotFound.
+func IsNotFound(err error) bool {
+	return errors.Is(err, ErrNotFound) || os.IsNotExist(err)
+}

beacon-chain/db/filesystem/BUILD.bazel

@@ -2,27 +2,43 @@ load("@prysm//tools/go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = ["save_blob.go"],
+    srcs = [
+        "blob.go",
+        "ephemeral.go",
+    ],
     importpath = "github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem",
-    visibility = ["//visibility:private"],
+    visibility = ["//visibility:public"],
     deps = [
+        "//beacon-chain/verification:go_default_library",
+        "//config/fieldparams:go_default_library",
+        "//config/params:go_default_library",
+        "//consensus-types/blocks:go_default_library",
+        "//consensus-types/primitives:go_default_library",
         "//io/file:go_default_library",
-        "//proto/eth/v2:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
+        "//runtime/logging:go_default_library",
+        "//time/slots:go_default_library",
+        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
-        "@com_github_prysmaticlabs_fastssz//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+        "@com_github_spf13_afero//:go_default_library",
     ],
 )
 
 go_test(
     name = "go_default_test",
-    srcs = ["save_blob_test.go"],
+    srcs = ["blob_test.go"],
     embed = [":go_default_library"],
     deps = [
+        "//beacon-chain/verification:go_default_library",
         "//config/fieldparams:go_default_library",
+        "//consensus-types/primitives:go_default_library",
         "//encoding/bytesutil:go_default_library",
-        "//io/file:go_default_library",
-        "//proto/eth/v2:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
         "//testing/require:go_default_library",
+        "//testing/util:go_default_library",
+        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_prysmaticlabs_fastssz//:go_default_library",
+        "@com_github_spf13_afero//:go_default_library",
     ],
 )

beacon-chain/db/filesystem/blob.go

@@ -0,0 +1,274 @@
+package filesystem
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/verification"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
+	"github.com/prysmaticlabs/prysm/v4/config/params"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
+	"github.com/prysmaticlabs/prysm/v4/io/file"
+	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/v4/runtime/logging"
+	"github.com/prysmaticlabs/prysm/v4/time/slots"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/afero"
+)
+
+var (
+	errIndexOutOfBounds = errors.New("blob index in file name > MaxBlobsPerBlock")
+)
+
+const (
+	sszExt  = "ssz"
+	partExt = "part"
+
+	bufferEpochs         = 2
+	directoryPermissions = 0700
+)
+
+// BlobStorageOption is a functional option for configuring a BlobStorage.
+type BlobStorageOption func(*BlobStorage)
+
+// WithBlobRetentionEpochs is an option that changes the number of epochs blobs will be persisted.
+func WithBlobRetentionEpochs(e primitives.Epoch) BlobStorageOption {
+	return func(b *BlobStorage) {
+		b.retentionEpochs = e
+	}
+}
+
+// NewBlobStorage creates a new instance of the BlobStorage object. Note that the implementation of BlobStorage may
+// attempt to hold a file lock to guarantee exclusive control of the blob storage directory, so this should only be
+// initialized once per beacon node.
+func NewBlobStorage(base string, opts ...BlobStorageOption) (*BlobStorage, error) {
+	base = path.Clean(base)
+	if err := file.MkdirAll(base); err != nil {
+		return nil, fmt.Errorf("failed to create blob storage at %s: %w", base, err)
+	}
+	fs := afero.NewBasePathFs(afero.NewOsFs(), base)
+	b := &BlobStorage{fs: fs, retentionEpochs: params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest}
+	for _, o := range opts {
+		o(b)
+	}
+	return b, nil
+}
+
+// BlobStorage is the concrete implementation of the filesystem backend for saving and retrieving BlobSidecars.
+type BlobStorage struct {
+	fs              afero.Fs
+	retentionEpochs primitives.Epoch
+}
+
+// Save saves blobs given a list of sidecars.
+func (bs *BlobStorage) Save(sidecar blocks.VerifiedROBlob) error {
+	fname := namerForSidecar(sidecar)
+	sszPath := fname.path()
+	exists, err := afero.Exists(bs.fs, sszPath)
+	if err != nil {
+		return err
+	}
+	if exists {
+		log.WithFields(logging.BlobFields(sidecar.ROBlob)).Debug("ignoring a duplicate blob sidecar Save attempt")
+		return nil
+	}
+
+	// Serialize the ethpb.BlobSidecar to binary data using SSZ.
+	sidecarData, err := sidecar.MarshalSSZ()
+	if err != nil {
+		return errors.Wrap(err, "failed to serialize sidecar data")
+	}
+	if err := bs.fs.MkdirAll(fname.dir(), directoryPermissions); err != nil {
+		return err
+	}
+	partPath := fname.partPath()
+	// Create a partial file and write the serialized data to it.
+	partialFile, err := bs.fs.Create(partPath)
+	if err != nil {
+		return errors.Wrap(err, "failed to create partial file")
+	}
+
+	_, err = partialFile.Write(sidecarData)
+	if err != nil {
+		closeErr := partialFile.Close()
+		if closeErr != nil {
+			return closeErr
+		}
+		return errors.Wrap(err, "failed to write to partial file")
+	}
+	err = partialFile.Close()
+	if err != nil {
+		return err
+	}
+
+	// Atomically rename the partial file to its final name.
+	err = bs.fs.Rename(partPath, sszPath)
+	if err != nil {
+		return errors.Wrap(err, "failed to rename partial file to final name")
+	}
+	return nil
+}
+
+// Get retrieves a single BlobSidecar by its root and index.
+// Since BlobStorage only writes blobs that have undergone full verification, the return
+// value is always a VerifiedROBlob.
+func (bs *BlobStorage) Get(root [32]byte, idx uint64) (blocks.VerifiedROBlob, error) {
+	expected := blobNamer{root: root, index: idx}
+	encoded, err := afero.ReadFile(bs.fs, expected.path())
+	var v blocks.VerifiedROBlob
+	if err != nil {
+		return v, err
+	}
+	s := &ethpb.BlobSidecar{}
+	if err := s.UnmarshalSSZ(encoded); err != nil {
+		return v, err
+	}
+	ro, err := blocks.NewROBlobWithRoot(s, root)
+	if err != nil {
+		return blocks.VerifiedROBlob{}, err
+	}
+	return verification.BlobSidecarNoop(ro)
+}
+
+// Indices generates a bitmap representing which BlobSidecar.Index values are present on disk for a given root.
+// This value can be compared to the commitments observed in a block to determine which indices need to be found
+// on the network to confirm data availability.
+func (bs *BlobStorage) Indices(root [32]byte) ([fieldparams.MaxBlobsPerBlock]bool, error) {
+	var mask [fieldparams.MaxBlobsPerBlock]bool
+	rootDir := blobNamer{root: root}.dir()
+	entries, err := afero.ReadDir(bs.fs, rootDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return mask, nil
+		}
+		return mask, err
+	}
+	for i := range entries {
+		if entries[i].IsDir() {
+			continue
+		}
+		name := entries[i].Name()
+		if !strings.HasSuffix(name, sszExt) {
+			continue
+		}
+		parts := strings.Split(name, ".")
+		if len(parts) != 2 {
+			continue
+		}
+		u, err := strconv.ParseUint(parts[0], 10, 64)
+		if err != nil {
+			return mask, errors.Wrapf(err, "unexpected directory entry breaks listing, %s", parts[0])
+		}
+		if u >= fieldparams.MaxBlobsPerBlock {
+			return mask, errIndexOutOfBounds
+		}
+		mask[u] = true
+	}
+	return mask, nil
+}
+
+type blobNamer struct {
+	root  [32]byte
+	index uint64
+}
+
+func namerForSidecar(sc blocks.VerifiedROBlob) blobNamer {
+	return blobNamer{root: sc.BlockRoot(), index: sc.Index}
+}
+
+func (p blobNamer) dir() string {
+	return fmt.Sprintf("%#x", p.root)
+}
+
+func (p blobNamer) fname(ext string) string {
+	return path.Join(p.dir(), fmt.Sprintf("%d.%s", p.index, ext))
+}
+
+func (p blobNamer) partPath() string {
+	return p.fname(partExt)
+}
+
+func (p blobNamer) path() string {
+	return p.fname(sszExt)
+}
+
+// Prune prunes blobs in the base directory based on the retention epoch.
+// It deletes blobs older than currentEpoch - (retentionEpochs+bufferEpochs).
+// This is so that we keep a slight buffer and blobs are deleted after n+2 epochs.
+func (bs *BlobStorage) Prune(currentSlot primitives.Slot) error {
+	retentionSlots, err := slots.EpochStart(bs.retentionEpochs + bufferEpochs)
+	if err != nil {
+		return err
+	}
+	if currentSlot < retentionSlots {
+		return nil // Overflow would occur
+	}
+
+	folders, err := afero.ReadDir(bs.fs, ".")
+	if err != nil {
+		return err
+	}
+	for _, folder := range folders {
+		if folder.IsDir() {
+			if err := bs.processFolder(folder, currentSlot, retentionSlots); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// processFolder will delete the folder of blobs if the blob slot is outside the
+// retention period. We determine the slot by looking at the first blob in the folder.
+func (bs *BlobStorage) processFolder(folder os.FileInfo, currentSlot, retentionSlots primitives.Slot) error {
+	f, err := bs.fs.Open(filepath.Join(folder.Name(), "0."+sszExt))
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			log.WithError(err).Errorf("Could not close blob file")
+		}
+	}()
+
+	slot, err := slotFromBlob(f)
+	if err != nil {
+		return err
+	}
+	if slot < (currentSlot - retentionSlots) {
+		if err = bs.fs.RemoveAll(folder.Name()); err != nil {
+			return errors.Wrapf(err, "failed to delete blob %s", f.Name())
+		}
+	}
+	return nil
+}
+
+// slotFromBlob reads the ssz data of a file at the specified offset (8 + 131072 + 48 + 48 = 131176 bytes),
+// which is calculated based on the size of the BlobSidecar struct and is based on the size of the fields
+// preceding the slot information within SignedBeaconBlockHeader.
+func slotFromBlob(at io.ReaderAt) (primitives.Slot, error) {
+	b := make([]byte, 8)
+	_, err := at.ReadAt(b, 131176)
+	if err != nil {
+		return 0, err
+	}
+	rawSlot := binary.LittleEndian.Uint64(b)
+	return primitives.Slot(rawSlot), nil
+}
+
+// Delete removes the directory matching the provided block root and all the blobs it contains.
+func (bs *BlobStorage) Delete(root [32]byte) error {
+	if err := bs.fs.RemoveAll(hexutil.Encode(root[:])); err != nil {
+		return fmt.Errorf("failed to delete blobs for root %#x: %w", root, err)
+	}
+	return nil
+}

beacon-chain/db/filesystem/blob_test.go

@@ -0,0 +1,208 @@
+package filesystem
+
+import (
+	"bytes"
+	"path"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	ssz "github.com/prysmaticlabs/fastssz"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/verification"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
+	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
+	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/v4/testing/require"
+	"github.com/prysmaticlabs/prysm/v4/testing/util"
+	"github.com/spf13/afero"
+)
+
+func TestBlobStorage_SaveBlobData(t *testing.T) {
+	_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, 1, fieldparams.MaxBlobsPerBlock)
+	testSidecars, err := verification.BlobSidecarSliceNoop(sidecars)
+	require.NoError(t, err)
+
+	t.Run("no error for duplicate", func(t *testing.T) {
+		fs, bs, err := NewEphemeralBlobStorageWithFs(t)
+		require.NoError(t, err)
+		existingSidecar := testSidecars[0]
+
+		blobPath := namerForSidecar(existingSidecar).path()
+		// Serialize the existing BlobSidecar to binary data.
+		existingSidecarData, err := ssz.MarshalSSZ(existingSidecar)
+		require.NoError(t, err)
+
+		require.NoError(t, bs.Save(existingSidecar))
+		// No error when attempting to write twice.
+		require.NoError(t, bs.Save(existingSidecar))
+
+		content, err := afero.ReadFile(fs, blobPath)
+		require.NoError(t, err)
+		require.Equal(t, true, bytes.Equal(existingSidecarData, content))
+
+		// Deserialize the BlobSidecar from the saved file data.
+		savedSidecar := &ethpb.BlobSidecar{}
+		err = savedSidecar.UnmarshalSSZ(content)
+		require.NoError(t, err)
+
+		// Compare the original Sidecar and the saved Sidecar.
+		require.DeepSSZEqual(t, existingSidecar.BlobSidecar, savedSidecar)
+
+	})
+	t.Run("indices", func(t *testing.T) {
+		bs := NewEphemeralBlobStorage(t)
+		sc := testSidecars[2]
+		require.NoError(t, bs.Save(sc))
+		actualSc, err := bs.Get(sc.BlockRoot(), sc.Index)
+		require.NoError(t, err)
+		expectedIdx := [fieldparams.MaxBlobsPerBlock]bool{false, false, true}
+		actualIdx, err := bs.Indices(actualSc.BlockRoot())
+		require.NoError(t, err)
+		require.Equal(t, expectedIdx, actualIdx)
+	})
+
+	t.Run("round trip write then read", func(t *testing.T) {
+		bs := NewEphemeralBlobStorage(t)
+		err := bs.Save(testSidecars[0])
+		require.NoError(t, err)
+
+		expected := testSidecars[0]
+		actual, err := bs.Get(expected.BlockRoot(), expected.Index)
+		require.NoError(t, err)
+		require.DeepSSZEqual(t, expected, actual)
+	})
+}
+
+func TestBlobIndicesBounds(t *testing.T) {
+	fs, bs, err := NewEphemeralBlobStorageWithFs(t)
+	require.NoError(t, err)
+	root := [32]byte{}
+
+	okIdx := uint64(fieldparams.MaxBlobsPerBlock - 1)
+	writeFakeSSZ(t, fs, root, okIdx)
+	indices, err := bs.Indices(root)
+	require.NoError(t, err)
+	var expected [fieldparams.MaxBlobsPerBlock]bool
+	expected[okIdx] = true
+	for i := range expected {
+		require.Equal(t, expected[i], indices[i])
+	}
+
+	oobIdx := uint64(fieldparams.MaxBlobsPerBlock)
+	writeFakeSSZ(t, fs, root, oobIdx)
+	_, err = bs.Indices(root)
+	require.ErrorIs(t, err, errIndexOutOfBounds)
+}
+
+func writeFakeSSZ(t *testing.T, fs afero.Fs, root [32]byte, idx uint64) {
+	namer := blobNamer{root: root, index: idx}
+	require.NoError(t, fs.MkdirAll(namer.dir(), 0700))
+	fh, err := fs.Create(namer.path())
+	require.NoError(t, err)
+	_, err = fh.Write([]byte("derp"))
+	require.NoError(t, err)
+	require.NoError(t, fh.Close())
+}
+
+func TestBlobStoragePrune(t *testing.T) {
+	currentSlot := primitives.Slot(200000)
+	fs, bs, err := NewEphemeralBlobStorageWithFs(t)
+	require.NoError(t, err)
+
+	t.Run("PruneOne", func(t *testing.T) {
+		_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, 300, fieldparams.MaxBlobsPerBlock)
+		testSidecars, err := verification.BlobSidecarSliceNoop(sidecars)
+		require.NoError(t, err)
+
+		for _, sidecar := range testSidecars {
+			require.NoError(t, bs.Save(sidecar))
+		}
+
+		require.NoError(t, bs.Prune(currentSlot))
+
+		remainingFolders, err := afero.ReadDir(fs, ".")
+		require.NoError(t, err)
+		require.Equal(t, 0, len(remainingFolders))
+	})
+	t.Run("PruneMany", func(t *testing.T) {
+		blockQty := 10
+		slot := primitives.Slot(0)
+
+		for j := 0; j <= blockQty; j++ {
+			root := bytesutil.ToBytes32(bytesutil.ToBytes(uint64(slot), 32))
+			_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, root, slot, fieldparams.MaxBlobsPerBlock)
+			testSidecars, err := verification.BlobSidecarSliceNoop(sidecars)
+			require.NoError(t, err)
+			require.NoError(t, bs.Save(testSidecars[0]))
+
+			slot += 10000
+		}
+
+		require.NoError(t, bs.Prune(currentSlot))
+
+		remainingFolders, err := afero.ReadDir(fs, ".")
+		require.NoError(t, err)
+		require.Equal(t, 4, len(remainingFolders))
+	})
+}
+
+func BenchmarkPruning(b *testing.B) {
+	var t *testing.T
+	_, bs, err := NewEphemeralBlobStorageWithFs(t)
+	require.NoError(t, err)
+
+	blockQty := 10000
+	currentSlot := primitives.Slot(150000)
+	slot := primitives.Slot(0)
+
+	for j := 0; j <= blockQty; j++ {
+		root := bytesutil.ToBytes32(bytesutil.ToBytes(uint64(slot), 32))
+		_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, root, slot, fieldparams.MaxBlobsPerBlock)
+		testSidecars, err := verification.BlobSidecarSliceNoop(sidecars)
+		require.NoError(t, err)
+		require.NoError(t, bs.Save(testSidecars[0]))
+
+		slot += 100
+	}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := bs.Prune(currentSlot)
+		require.NoError(b, err)
+	}
+}
+
+func TestBlobStorageDelete(t *testing.T) {
+	fs, bs, err := NewEphemeralBlobStorageWithFs(t)
+	require.NoError(t, err)
+	rawRoot := "0xcf9bb70c98f58092c9d6459227c9765f984d240be9690e85179bc5a6f60366ad"
+	blockRoot, err := hexutil.Decode(rawRoot)
+	require.NoError(t, err)
+
+	_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, 1, fieldparams.MaxBlobsPerBlock)
+	testSidecars, err := verification.BlobSidecarSliceNoop(sidecars)
+	require.NoError(t, err)
+	for _, sidecar := range testSidecars {
+		require.NoError(t, bs.Save(sidecar))
+	}
+
+	exists, err := afero.DirExists(fs, hexutil.Encode(blockRoot))
+	require.NoError(t, err)
+	require.Equal(t, true, exists)
+
+	// Delete the directory corresponding to the block root
+	require.NoError(t, bs.Delete(bytesutil.ToBytes32(blockRoot)))
+
+	// Ensure that the directory no longer exists after deletion
+	exists, err = afero.DirExists(fs, hexutil.Encode(blockRoot))
+	require.NoError(t, err)
+	require.Equal(t, false, exists)
+
+	// Deleting a non-existent root does not return an error.
+	require.NoError(t, bs.Delete(bytesutil.ToBytes32([]byte{0x1})))
+}
+
+func TestNewBlobStorage(t *testing.T) {
+	_, err := NewBlobStorage(path.Join(t.TempDir(), "good"))
+	require.NoError(t, err)
+}

beacon-chain/db/filesystem/ephemeral.go

@@ -0,0 +1,55 @@
+package filesystem
+
+import (
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/v4/config/params"
+	"github.com/spf13/afero"
+)
+
+// NewEphemeralBlobStorage should only be used for tests.
+// The instance of BlobStorage returned is backed by an in-memory virtual filesystem,
+// improving test performance and simplifying cleanup.
+func NewEphemeralBlobStorage(_ testing.TB) *BlobStorage {
+	return &BlobStorage{fs: afero.NewMemMapFs()}
+}
+
+// NewEphemeralBlobStorageWithFs can be used by tests that want access to the virtual filesystem
+// in order to interact with it outside the parameters of the BlobStorage api.
+func NewEphemeralBlobStorageWithFs(_ testing.TB) (afero.Fs, *BlobStorage, error) {
+	fs := afero.NewMemMapFs()
+	retentionEpoch := params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest
+	return fs, &BlobStorage{fs: fs, retentionEpochs: retentionEpoch}, nil
+}
+
+type BlobMocker struct {
+	fs afero.Fs
+	bs *BlobStorage
+}
+
+// CreateFakeIndices creates empty blob sidecar files at the expected path for the given
+// root and indices to influence the result of Indices().
+func (bm *BlobMocker) CreateFakeIndices(root [32]byte, indices []uint64) error {
+	for i := range indices {
+		n := blobNamer{root: root, index: indices[i]}
+		if err := bm.fs.MkdirAll(n.dir(), directoryPermissions); err != nil {
+			return err
+		}
+		f, err := bm.fs.Create(n.path())
+		if err != nil {
+			return err
+		}
+		if err := f.Close(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// NewEphemeralBlobStorageWithMocker returns a *BlobMocker value in addition to the BlobStorage value.
+// BlockMocker encapsulates things blob path construction to avoid leaking implementation details.
+func NewEphemeralBlobStorageWithMocker(_ testing.TB) (*BlobMocker, *BlobStorage) {
+	fs := afero.NewMemMapFs()
+	bs := &BlobStorage{fs: fs}
+	return &BlobMocker{fs: fs, bs: bs}, bs
+}

beacon-chain/db/filesystem/save_blob.go

@@ -1,98 +0,0 @@
-package filesystem
-
-import (
-	"crypto/sha256"
-	"encoding/hex"
-	"fmt"
-	"os"
-	"path"
-	"path/filepath"
-
-	"github.com/pkg/errors"
-	ssz "github.com/prysmaticlabs/fastssz"
-	"github.com/prysmaticlabs/prysm/v4/io/file"
-	"github.com/prysmaticlabs/prysm/v4/proto/eth/v2"
-)
-
-type BlobStorage struct {
-	baseDir string
-}
-
-// SaveBlobData saves blobs given a list of sidecars.
-func (bs *BlobStorage) SaveBlobData(sidecars []*eth.BlobSidecar) error {
-	if len(sidecars) == 0 {
-		return errors.New("no blob data to save")
-	}
-	for _, sidecar := range sidecars {
-		blobPath := bs.sidecarFileKey(sidecar)
-		exists := file.Exists(blobPath)
-		if exists {
-			if err := checkDataIntegrity(sidecar, blobPath); err != nil {
-				// This error should never happen, if it does then the
-				// file has most likely been tampered with.
-				return errors.Wrapf(err, "failed to save blob sidecar, tried to overwrite blob (%s) with different content", blobPath)
-			}
-			continue // Blob already exists, move to the next one
-		}
-
-		// Serialize the ethpb.BlobSidecar to binary data using SSZ.
-		sidecarData, err := ssz.MarshalSSZ(sidecar)
-		if err != nil {
-			return errors.Wrap(err, "failed to serialize sidecar data")
-		}
-
-		// Create a partial file and write the serialized data to it.
-		partialFilePath := blobPath + ".partial"
-		partialFile, err := os.Create(filepath.Clean(partialFilePath))
-		if err != nil {
-			return errors.Wrap(err, "failed to create partial file")
-		}
-
-		_, err = partialFile.Write(sidecarData)
-		if err != nil {
-			closeErr := partialFile.Close()
-			if closeErr != nil {
-				return closeErr
-			}
-			return errors.Wrap(err, "failed to write to partial file")
-		}
-		err = partialFile.Close()
-		if err != nil {
-			return err
-		}
-
-		// Atomically rename the partial file to its final name.
-		err = os.Rename(partialFilePath, blobPath)
-		if err != nil {
-			return errors.Wrap(err, "failed to rename partial file to final name")
-		}
-	}
-	return nil
-}
-
-func (bs *BlobStorage) sidecarFileKey(sidecar *eth.BlobSidecar) string {
-	return path.Join(bs.baseDir, fmt.Sprintf(
-		"%d_%x_%d_%x.blob",
-		sidecar.Slot,
-		sidecar.BlockRoot,
-		sidecar.Index,
-		sidecar.KzgCommitment,
-	))
-}
-
-// checkDataIntegrity checks the data integrity by comparing the original ethpb.BlobSidecar.
-func checkDataIntegrity(sidecar *eth.BlobSidecar, filePath string) error {
-	sidecarData, err := ssz.MarshalSSZ(sidecar)
-	if err != nil {
-		return errors.Wrap(err, "failed to serialize sidecar data")
-	}
-	originalChecksum := sha256.Sum256(sidecarData)
-	savedFileChecksum, err := file.HashFile(filePath)
-	if err != nil {
-		return errors.Wrap(err, "failed to calculate saved file checksum")
-	}
-	if hex.EncodeToString(originalChecksum[:]) != hex.EncodeToString(savedFileChecksum) {
-		return errors.New("data integrity check failed")
-	}
-	return nil
-}

beacon-chain/db/filesystem/save_blob_test.go

@@ -1,191 +0,0 @@
-package filesystem
-
-import (
-	"bytes"
-	"crypto/rand"
-	"crypto/sha256"
-	"encoding/hex"
-	"os"
-	"path"
-	"strings"
-	"testing"
-
-	ssz "github.com/prysmaticlabs/fastssz"
-	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
-	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
-	"github.com/prysmaticlabs/prysm/v4/io/file"
-	"github.com/prysmaticlabs/prysm/v4/proto/eth/v2"
-
-	"github.com/prysmaticlabs/prysm/v4/testing/require"
-)
-
-func TestBlobStorage_SaveBlobData(t *testing.T) {
-	testSidecars := generateBlobSidecars(t, fieldparams.MaxBlobsPerBlock)
-	t.Run("NoBlobData", func(t *testing.T) {
-		tempDir := t.TempDir()
-		bs := &BlobStorage{baseDir: tempDir}
-		err := bs.SaveBlobData([]*eth.BlobSidecar{})
-		require.ErrorContains(t, "no blob data to save", err)
-	})
-
-	t.Run("BlobExists", func(t *testing.T) {
-		tempDir := t.TempDir()
-		bs := &BlobStorage{baseDir: tempDir}
-		existingSidecar := testSidecars[0]
-
-		blobPath := bs.sidecarFileKey(existingSidecar)
-		// Serialize the existing BlobSidecar to binary data.
-		existingSidecarData, err := ssz.MarshalSSZ(existingSidecar)
-		require.NoError(t, err)
-
-		err = os.MkdirAll(path.Dir(blobPath), os.ModePerm)
-		require.NoError(t, err)
-
-		// Write the serialized data to the blob file.
-		err = os.WriteFile(blobPath, existingSidecarData, os.ModePerm)
-		require.NoError(t, err)
-
-		err = bs.SaveBlobData([]*eth.BlobSidecar{existingSidecar})
-		require.NoError(t, err)
-
-		content, err := os.ReadFile(blobPath)
-		require.NoError(t, err)
-
-		// Deserialize the BlobSidecar from the saved file data.
-		var savedSidecar ssz.Unmarshaler
-		savedSidecar = &eth.BlobSidecar{}
-		err = savedSidecar.UnmarshalSSZ(content)
-		require.NoError(t, err)
-
-		// Compare the original Sidecar and the saved Sidecar.
-		require.DeepSSZEqual(t, existingSidecar, savedSidecar)
-	})
-
-	t.Run("SaveBlobDataNoErrors", func(t *testing.T) {
-		tempDir := t.TempDir()
-		bs := &BlobStorage{baseDir: tempDir}
-		err := bs.SaveBlobData(testSidecars)
-		require.NoError(t, err)
-
-		// Check the number of files in the directory.
-		files, err := os.ReadDir(tempDir)
-		require.NoError(t, err)
-		require.Equal(t, len(testSidecars), len(files))
-
-		for _, f := range files {
-			content, err := os.ReadFile(path.Join(tempDir, f.Name()))
-			require.NoError(t, err)
-
-			// Deserialize the BlobSidecar from the saved file data.
-			var savedSidecar ssz.Unmarshaler
-			savedSidecar = &eth.BlobSidecar{}
-			err = savedSidecar.UnmarshalSSZ(content)
-			require.NoError(t, err)
-
-			// Find the corresponding test sidecar based on the file name.
-			sidecar := findTestSidecarsByFileName(t, testSidecars, f.Name())
-			require.NotNil(t, sidecar)
-			// Compare the original Sidecar and the saved Sidecar.
-			require.DeepSSZEqual(t, sidecar, savedSidecar)
-		}
-	})
-
-	t.Run("OverwriteBlobWithDifferentContent", func(t *testing.T) {
-		tempDir := t.TempDir()
-		bs := &BlobStorage{baseDir: tempDir}
-		originalSidecar := []*eth.BlobSidecar{testSidecars[0]}
-		// Save the original sidecar
-		err := bs.SaveBlobData(originalSidecar)
-		require.NoError(t, err)
-
-		// Modify the blob data
-		modifiedSidecar := originalSidecar
-		modifiedSidecar[0].Blob = []byte("Modified Blob Data")
-
-		err = bs.SaveBlobData(modifiedSidecar)
-		require.ErrorContains(t, "failed to save blob sidecar, tried to overwrite blob", err)
-	})
-}
-
-func findTestSidecarsByFileName(t *testing.T, testSidecars []*eth.BlobSidecar, fileName string) *eth.BlobSidecar {
-	parts := strings.SplitN(fileName, ".", 2)
-	require.Equal(t, 2, len(parts))
-	// parts[0] contains the substring before the first period
-	components := strings.Split(parts[0], "_")
-	if len(components) == 4 {
-		blockRoot, err := hex.DecodeString(components[1])
-		require.NoError(t, err)
-		kzgCommitment, err := hex.DecodeString(components[3])
-		require.NoError(t, err)
-		for _, sidecar := range testSidecars {
-			if bytes.Equal(sidecar.BlockRoot, blockRoot) && bytes.Equal(sidecar.KzgCommitment, kzgCommitment) {
-				return sidecar
-			}
-		}
-	}
-	return nil
-}
-
-func TestCheckDataIntegrity(t *testing.T) {
-	testSidecars := generateBlobSidecars(t, fieldparams.MaxBlobsPerBlock)
-	originalData, err := ssz.MarshalSSZ(testSidecars[0])
-	require.NoError(t, err)
-	originalChecksum := sha256.Sum256(originalData)
-
-	tempDir := t.TempDir()
-	tempfile, err := os.CreateTemp(tempDir, "testfile")
-	require.NoError(t, err)
-	_, err = tempfile.Write(originalData)
-	require.NoError(t, err)
-
-	err = checkDataIntegrity(testSidecars[0], tempfile.Name())
-	require.NoError(t, err)
-
-	// Modify the data in the file to simulate data corruption
-	corruptedData := []byte("corrupted data")
-	err = os.WriteFile(tempfile.Name(), corruptedData, os.ModePerm)
-	require.NoError(t, err)
-
-	// Test data integrity check with corrupted data
-	err = checkDataIntegrity(testSidecars[0], tempfile.Name())
-	require.ErrorContains(t, "data integrity check failed", err)
-
-	// Modify the calculated checksum to be incorrect
-	wrongChecksum := hex.EncodeToString(originalChecksum[:]) + "12345"
-	err = os.WriteFile(tempfile.Name(), []byte(wrongChecksum), os.ModePerm)
-	require.NoError(t, err)
-
-	checksum, err := file.HashFile(tempfile.Name())
-	require.NoError(t, err)
-	require.NotEqual(t, wrongChecksum, hex.EncodeToString(checksum))
-}
-
-func generateBlobSidecars(t *testing.T, n uint64) []*eth.BlobSidecar {
-	blobSidecars := make([]*eth.BlobSidecar, n)
-	for i := uint64(0); i < n; i++ {
-		blobSidecars[i] = generateBlobSidecar(t, i)
-	}
-	return blobSidecars
-}
-
-func generateBlobSidecar(t *testing.T, index uint64) *eth.BlobSidecar {
-	blob := make([]byte, 131072)
-	_, err := rand.Read(blob)
-	require.NoError(t, err)
-	kzgCommitment := make([]byte, 48)
-	_, err = rand.Read(kzgCommitment)
-	require.NoError(t, err)
-	kzgProof := make([]byte, 48)
-	_, err = rand.Read(kzgProof)
-	require.NoError(t, err)
-	return &eth.BlobSidecar{
-		BlockRoot:       bytesutil.PadTo([]byte{'a'}, 32),
-		Index:           index,
-		Slot:            100,
-		BlockParentRoot: bytesutil.PadTo([]byte{'b'}, 32),
-		ProposerIndex:   101,
-		Blob:            blob,
-		KzgCommitment:   kzgCommitment,
-		KzgProof:        kzgProof,
-	}
-}

beacon-chain/db/kv/BUILD.bazel

@@ -13,7 +13,6 @@ go_library(
         "error.go",
         "execution_chain.go",
         "finalized_block_roots.go",
-        "flags.go",
         "genesis.go",
         "key.go",
         "kv.go",
@@ -39,7 +38,6 @@ go_library(
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/genesis:go_default_library",
         "//beacon-chain/state/state-native:go_default_library",
-        "//cmd/beacon-chain/flags:go_default_library",
         "//config/features:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
@@ -67,7 +65,6 @@ go_library(
         "@com_github_prysmaticlabs_prombbolt//:go_default_library",
         "@com_github_schollz_progressbar_v3//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_urfave_cli_v2//:go_default_library",
         "@io_etcd_go_bbolt//:go_default_library",
         "@io_opencensus_go//trace:go_default_library",
         "@org_golang_google_protobuf//proto:go_default_library",
@@ -86,7 +83,6 @@ go_test(
         "encoding_test.go",
         "execution_chain_test.go",
         "finalized_block_roots_test.go",
-        "flags_test.go",
         "genesis_test.go",
         "init_test.go",
         "kv_test.go",
@@ -107,7 +103,6 @@ go_test(
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/genesis:go_default_library",
         "//beacon-chain/state/state-native:go_default_library",
-        "//cmd/beacon-chain/flags:go_default_library",
         "//config/features:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
@@ -125,7 +120,6 @@ go_test(
         "@com_github_golang_snappy//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_urfave_cli_v2//:go_default_library",
         "@io_bazel_rules_go//go/tools/bazel:go_default_library",
         "@io_etcd_go_bbolt//:go_default_library",
         "@org_golang_google_protobuf//proto:go_default_library",

beacon-chain/db/kv/blob.go

@@ -3,7 +3,6 @@ package kv
 import (
 	"bytes"
 	"context"
-	"fmt"
 	"sort"
 
 	"github.com/pkg/errors"
@@ -63,7 +62,7 @@ func (s *Store) SaveBlobSidecar(ctx context.Context, scs []*ethpb.DeprecatedBlob
 	defer span.End()
 
 	first := scs[0]
-	newKey := blobSidecarKey(first)
+	newKey := s.blobSidecarKey(first)
 	prefix := newKey.BufferPrefix()
 	var prune []blobRotatingKey
 	return s.db.Update(func(tx *bolt.Tx) error {
@@ -235,7 +234,7 @@ func (s *Store) BlobSidecarsBySlot(ctx context.Context, slot types.Slot, indices
 	defer span.End()
 
 	var enc []byte
-	sk := slotKey(slot)
+	sk := s.slotKey(slot)
 	if err := s.db.View(func(tx *bolt.Tx) error {
 		c := tx.Bucket(blobsBucket).Cursor()
 		// Bucket size is bounded and bolt cursors are fast. Moreover, a thin caching layer can be added.
@@ -281,32 +280,37 @@ func (s *Store) DeleteBlobSidecars(ctx context.Context, beaconBlockRoot [32]byte
 
 // We define a blob sidecar key as: bytes(slot_to_rotating_buffer(blob.slot)) ++ bytes(blob.slot) ++ blob.block_root
 // where slot_to_rotating_buffer(slot) = slot % MAX_SLOTS_TO_PERSIST_BLOBS.
-func blobSidecarKey(blob *ethpb.DeprecatedBlobSidecar) blobRotatingKey {
-	key := slotKey(blob.Slot)
+func (s *Store) blobSidecarKey(blob *ethpb.DeprecatedBlobSidecar) blobRotatingKey {
+	key := s.slotKey(blob.Slot)
 	key = append(key, bytesutil.SlotToBytesBigEndian(blob.Slot)...)
 	key = append(key, blob.BlockRoot...)
 	return key
 }
 
-func slotKey(slot types.Slot) []byte {
-	slotsPerEpoch := params.BeaconConfig().SlotsPerEpoch
-	maxSlotsToPersistBlobs := types.Slot(maxEpochsToPersistBlobs.Mul(uint64(slotsPerEpoch)))
-	return bytesutil.SlotToBytesBigEndian(slot.ModSlot(maxSlotsToPersistBlobs))
+func (s *Store) slotKey(slot types.Slot) []byte {
+	return bytesutil.SlotToBytesBigEndian(slot.ModSlot(s.blobRetentionSlots()))
 }
 
-func checkEpochsForBlobSidecarsRequestBucket(db *bolt.DB) error {
+func (s *Store) blobRetentionSlots() types.Slot {
+	return types.Slot(s.blobRetentionEpochs.Mul(uint64(params.BeaconConfig().SlotsPerEpoch)))
+}
+
+var errBlobRetentionEpochMismatch = errors.New("epochs for blobs request value in DB does not match runtime config")
+
+func (s *Store) checkEpochsForBlobSidecarsRequestBucket(db *bolt.DB) error {
+	uRetentionEpochs := uint64(s.blobRetentionEpochs)
 	if err := db.Update(func(tx *bolt.Tx) error {
 		b := tx.Bucket(chainMetadataBucket)
 		v := b.Get(blobRetentionEpochsKey)
 		if v == nil {
-			if err := b.Put(blobRetentionEpochsKey, bytesutil.Uint64ToBytesBigEndian(uint64(maxEpochsToPersistBlobs))); err != nil {
+			if err := b.Put(blobRetentionEpochsKey, bytesutil.Uint64ToBytesBigEndian(uRetentionEpochs)); err != nil {
 				return err
 			}
 			return nil
 		}
 		e := bytesutil.BytesToUint64BigEndian(v)
-		if e != uint64(maxEpochsToPersistBlobs) {
-			return fmt.Errorf("epochs for blobs request value in DB %d does not match config value %d", e, maxEpochsToPersistBlobs)
+		if e != uRetentionEpochs {
+			return errors.Wrapf(errBlobRetentionEpochMismatch, "db=%d, config=%d", e, uRetentionEpochs)
 		}
 		return nil
 	}); err != nil {

beacon-chain/db/kv/blob_test.go

@@ -3,13 +3,10 @@ package kv
 import (
 	"context"
 	"crypto/rand"
-	"flag"
 	"fmt"
-	"strconv"
 	"testing"
 
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/v4/cmd/beacon-chain/flags"
 	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
@@ -18,7 +15,6 @@ import (
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v4/testing/assertions"
 	"github.com/prysmaticlabs/prysm/v4/testing/require"
-	"github.com/urfave/cli/v2"
 	bolt "go.etcd.io/bbolt"
 )
 
@@ -493,7 +489,7 @@ func BenchmarkStore_BlobSidecarsByRoot(b *testing.B) {
 			scs := []*ethpb.DeprecatedBlobSidecar{
 				{BlockRoot: r, Slot: primitives.Slot(i)},
 			}
-			k := blobSidecarKey(scs[0])
+			k := s.blobSidecarKey(scs[0])
 			encodedBlobSidecar, err := encode(ctx, &ethpb.BlobSidecars{Sidecars: scs})
 			require.NoError(b, err)
 			require.NoError(b, bkt.Put(k, encodedBlobSidecar))
@@ -514,27 +510,23 @@ func BenchmarkStore_BlobSidecarsByRoot(b *testing.B) {
 }
 
 func Test_checkEpochsForBlobSidecarsRequestBucket(t *testing.T) {
-	dbStore := setupDB(t)
+	s := setupDB(t)
 
-	require.NoError(t, checkEpochsForBlobSidecarsRequestBucket(dbStore.db)) // First write
-	require.NoError(t, checkEpochsForBlobSidecarsRequestBucket(dbStore.db)) // First check
+	require.NoError(t, s.checkEpochsForBlobSidecarsRequestBucket(s.db)) // First write
+	require.NoError(t, s.checkEpochsForBlobSidecarsRequestBucket(s.db)) // First check
 
-	params.SetupTestConfigCleanup(t)
-	set := flag.NewFlagSet("test", 0)
-	set.Uint64(flags.BlobRetentionEpoch.Name, 0, "")
-	require.NoError(t, set.Set(flags.BlobRetentionEpoch.Name, strconv.FormatUint(42069, 10)))
-	cliCtx := cli.NewContext(&cli.App{}, set, nil)
-	require.NoError(t, ConfigureBlobRetentionEpoch(cliCtx))
-	require.ErrorContains(t, "epochs for blobs request value in DB 4096 does not match config value 42069", checkEpochsForBlobSidecarsRequestBucket(dbStore.db))
+	s.blobRetentionEpochs += 1
+	require.ErrorIs(t, s.checkEpochsForBlobSidecarsRequestBucket(s.db), errBlobRetentionEpochMismatch)
 }
 
 func TestBlobRotatingKey(t *testing.T) {
-	k := blobSidecarKey(&ethpb.DeprecatedBlobSidecar{
+	s := setupDB(t)
+	k := s.blobSidecarKey(&ethpb.DeprecatedBlobSidecar{
 		Slot:      1,
 		BlockRoot: []byte{2},
 	})
 
 	require.Equal(t, types.Slot(1), k.Slot())
 	require.DeepEqual(t, []byte{2}, k.BlockRoot())
-	require.DeepEqual(t, slotKey(types.Slot(1)), k.BufferPrefix())
+	require.DeepEqual(t, s.slotKey(types.Slot(1)), k.BufferPrefix())
 }

beacon-chain/db/kv/flags.go

@@ -1,33 +0,0 @@
-package kv
-
-import (
-	"fmt"
-
-	"github.com/prysmaticlabs/prysm/v4/cmd/beacon-chain/flags"
-	"github.com/prysmaticlabs/prysm/v4/config/params"
-	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
-	"github.com/urfave/cli/v2"
-)
-
-var maxEpochsToPersistBlobs = params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest
-
-// ConfigureBlobRetentionEpoch sets the for blob retention based on command-line context. It sets the local config `maxEpochsToPersistBlobs`.
-// If the flag is not set, the spec default `MinEpochsForBlobsSidecarsRequest` is used.
-// An error if the input epoch is smaller than the spec default value.
-func ConfigureBlobRetentionEpoch(cliCtx *cli.Context) error {
-	// Check if the blob retention epoch flag is set.
-	if cliCtx.IsSet(flags.BlobRetentionEpoch.Name) {
-		// Retrieve and cast the epoch value.
-		epochValue := cliCtx.Uint64(flags.BlobRetentionEpoch.Name)
-		e := primitives.Epoch(epochValue)
-
-		// Validate the epoch value against the spec default.
-		if e < params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest {
-			return fmt.Errorf("%s smaller than spec default, %d < %d", flags.BlobRetentionEpoch.Name, e, params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest)
-		}
-
-		maxEpochsToPersistBlobs = e
-	}
-
-	return nil
-}

beacon-chain/db/kv/flags_test.go

@@ -1,39 +0,0 @@
-package kv
-
-import (
-	"flag"
-	"strconv"
-	"testing"
-
-	"github.com/prysmaticlabs/prysm/v4/cmd/beacon-chain/flags"
-	"github.com/prysmaticlabs/prysm/v4/config/params"
-	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
-	"github.com/prysmaticlabs/prysm/v4/testing/require"
-	"github.com/urfave/cli/v2"
-)
-
-func TestConfigureBlobRetentionEpoch(t *testing.T) {
-	maxEpochsToPersistBlobs = params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest
-	params.SetupTestConfigCleanup(t)
-	app := cli.App{}
-	set := flag.NewFlagSet("test", 0)
-
-	// Test case: Spec default.
-	require.NoError(t, ConfigureBlobRetentionEpoch(cli.NewContext(&app, set, nil)))
-	require.Equal(t, params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest, maxEpochsToPersistBlobs)
-
-	set.Uint64(flags.BlobRetentionEpoch.Name, 0, "")
-	minEpochsForSidecarRequest := uint64(params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest)
-	require.NoError(t, set.Set(flags.BlobRetentionEpoch.Name, strconv.FormatUint(2*minEpochsForSidecarRequest, 10)))
-	cliCtx := cli.NewContext(&app, set, nil)
-
-	// Test case: Input epoch is greater than or equal to spec value.
-	require.NoError(t, ConfigureBlobRetentionEpoch(cliCtx))
-	require.Equal(t, primitives.Epoch(2*minEpochsForSidecarRequest), maxEpochsToPersistBlobs)
-
-	// Test case: Input epoch is less than spec value.
-	require.NoError(t, set.Set(flags.BlobRetentionEpoch.Name, strconv.FormatUint(minEpochsForSidecarRequest-1, 10)))
-	cliCtx = cli.NewContext(&app, set, nil)
-	err := ConfigureBlobRetentionEpoch(cliCtx)
-	require.ErrorContains(t, "blob-retention-epochs smaller than spec default", err)
-}

beacon-chain/db/kv/kv.go

@@ -18,6 +18,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/config/features"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/io/file"
 	bolt "go.etcd.io/bbolt"
 )
@@ -90,6 +91,7 @@ type Store struct {
 	validatorEntryCache *ristretto.Cache
 	stateSummaryCache   *stateSummaryCache
 	ctx                 context.Context
+	blobRetentionEpochs primitives.Epoch
 }
 
 // StoreDatafilePath is the canonical construction of a full
@@ -133,10 +135,20 @@ var Buckets = [][]byte{
 	blobsBucket,
 }
 
+// KVStoreOption is a functional option that modifies a kv.Store.
+type KVStoreOption func(*Store)
+
+// WithBlobRetentionEpochs sets the variable configuring the blob retention window.
+func WithBlobRetentionEpochs(e primitives.Epoch) KVStoreOption {
+	return func(s *Store) {
+		s.blobRetentionEpochs = e
+	}
+}
+
 // NewKVStore initializes a new boltDB key-value store at the directory
 // path specified, creates the kv-buckets based on the schema, and stores
 // an open connection db object as a property of the Store struct.
-func NewKVStore(ctx context.Context, dirPath string) (*Store, error) {
+func NewKVStore(ctx context.Context, dirPath string, opts ...KVStoreOption) (*Store, error) {
 	hasDir, err := file.HasDir(dirPath)
 	if err != nil {
 		return nil, err
@@ -189,6 +201,9 @@ func NewKVStore(ctx context.Context, dirPath string) (*Store, error) {
 		stateSummaryCache:   newStateSummaryCache(),
 		ctx:                 ctx,
 	}
+	for _, o := range opts {
+		o(kv)
+	}
 	if err := kv.db.Update(func(tx *bolt.Tx) error {
 		return createBuckets(tx, Buckets...)
 	}); err != nil {
@@ -202,10 +217,14 @@ func NewKVStore(ctx context.Context, dirPath string) (*Store, error) {
 		return nil, err
 	}
 
-	if err := checkEpochsForBlobSidecarsRequestBucket(boltDB); err != nil {
+	if err := kv.checkEpochsForBlobSidecarsRequestBucket(boltDB); err != nil {
 		return nil, errors.Wrap(err, "failed to check epochs for blob sidecars request bucket")
 	}
 
+	// set a default so that tests don't break
+	if kv.blobRetentionEpochs == 0 {
+		kv.blobRetentionEpochs = params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest
+	}
 	return kv, nil
 }
 

beacon-chain/db/kv/kv_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/v4/config/features"
+	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/blocks"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v4/testing/require"
@@ -15,7 +16,8 @@ import (
 
 // setupDB instantiates and returns a Store instance.
 func setupDB(t testing.TB) *Store {
-	db, err := NewKVStore(context.Background(), t.TempDir())
+	opt := WithBlobRetentionEpochs(params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest)
+	db, err := NewKVStore(context.Background(), t.TempDir(), opt)
 	require.NoError(t, err, "Failed to instantiate DB")
 	t.Cleanup(func() {
 		require.NoError(t, db.Close(), "Failed to close database")

beacon-chain/execution/options.go

@@ -108,3 +108,10 @@ func WithFinalizedStateAtStartup(st state.BeaconState) Option {
 		return nil
 	}
 }
+
+func WithJwtId(jwtId string) Option {
+	return func(s *Service) error {
+		s.cfg.jwtId = jwtId
+		return nil
+	}
+}

beacon-chain/execution/service.go

@@ -128,6 +128,7 @@ type config struct {
 	currHttpEndpoint        network.Endpoint
 	headers                 []string
 	finalizedStateAtStartup state.BeaconState
+	jwtId                   string
 }
 
 // Service fetches important information about the canonical

beacon-chain/forkchoice/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@prysm//tools/go:def.bzl", "go_library")
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -6,6 +6,7 @@ go_library(
         "doc.go",
         "error.go",
         "interfaces.go",
+        "ro.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice",
     visibility = [
@@ -22,3 +23,15 @@ go_library(
         "@com_github_pkg_errors//:go_default_library",
     ],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = ["ro_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//beacon-chain/forkchoice/types:go_default_library",
+        "//config/fieldparams:go_default_library",
+        "//consensus-types/primitives:go_default_library",
+        "//testing/require:go_default_library",
+    ],
+)

beacon-chain/forkchoice/doubly-linked-tree/forkchoice.go

@@ -475,7 +475,7 @@ func (f *ForkChoice) CommonAncestor(ctx context.Context, r1 [32]byte, r2 [32]byt
 // `blocks`. This slice must be ordered from child to parent. It includes all
 // blocks **except** the first one (that is the one with the highest slot
 // number). All blocks are assumed to be a strict chain
-// where blocks[i].Parent = blocks[i+1]. Also we assume that the parent of the
+// where blocks[i].Parent = blocks[i+1]. Also, we assume that the parent of the
 // last block in this list is already included in forkchoice store.
 func (f *ForkChoice) InsertChain(ctx context.Context, chain []*forkchoicetypes.BlockAndCheckpoints) error {
 	if len(chain) == 0 {
@@ -637,3 +637,41 @@ func (f *ForkChoice) Slot(root [32]byte) (primitives.Slot, error) {
 	}
 	return n.slot, nil
 }
+
+// TargetRootForEpoch returns the root of the target block for a given epoch.
+// The epoch parameter is crucial to identify the correct target root. For example:
+// When inserting a block at slot 63 with block root 0xA and target root 0xB (pointing to the block at slot 32),
+// and at slot 64, where the block is skipped, the attestation will reference the target root as 0xA (for slot 63), not 0xB (for slot 32).
+// This implies that if the input slot exceeds the block slot, the target root will be the same as the block root.
+// We also allow for the epoch to be below the current target for this root, in
+// which case we return the root of the checkpoint of the chain containing the
+// passed root, at the given epoch
+func (f *ForkChoice) TargetRootForEpoch(root [32]byte, epoch primitives.Epoch) ([32]byte, error) {
+	n, ok := f.store.nodeByRoot[root]
+	if !ok || n == nil {
+		return [32]byte{}, ErrNilNode
+	}
+	nodeEpoch := slots.ToEpoch(n.slot)
+	if epoch > nodeEpoch {
+		return n.root, nil
+	}
+	if n.target == nil {
+		return [32]byte{}, nil
+	}
+	targetRoot := n.target.root
+	if epoch == nodeEpoch {
+		return targetRoot, nil
+	}
+	targetNode, ok := f.store.nodeByRoot[targetRoot]
+	if !ok || targetNode == nil {
+		return [32]byte{}, ErrNilNode
+	}
+	// If slot 0 was not missed we consider a previous block to go back at least one epoch
+	if nodeEpoch == slots.ToEpoch(targetNode.slot) {
+		targetNode = targetNode.parent
+		if targetNode == nil {
+			return [32]byte{}, ErrNilNode
+		}
+	}
+	return f.TargetRootForEpoch(targetNode.root, epoch)
+}

beacon-chain/forkchoice/doubly-linked-tree/forkchoice_test.go

@@ -780,9 +780,9 @@ func TestForkChoiceIsViableForCheckpoint(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, true, viable)
 
-	st, broot, err := prepareForkchoiceState(ctx, 1, [32]byte{'b'}, root, [32]byte{'B'}, 0, 0)
+	st, bRoot, err := prepareForkchoiceState(ctx, 1, [32]byte{'b'}, root, [32]byte{'B'}, 0, 0)
 	require.NoError(t, err)
-	require.NoError(t, f.InsertNode(ctx, st, broot))
+	require.NoError(t, f.InsertNode(ctx, st, bRoot))
 
 	// Epoch start
 	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: root})
@@ -794,55 +794,55 @@ func TestForkChoiceIsViableForCheckpoint(t *testing.T) {
 	require.Equal(t, false, viable)
 
 	// No Children but impossible checkpoint
-	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: broot})
+	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: bRoot})
 	require.NoError(t, err)
 	require.Equal(t, false, viable)
 
-	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: broot, Epoch: 1})
+	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: bRoot, Epoch: 1})
 	require.NoError(t, err)
 	require.Equal(t, true, viable)
 
-	st, croot, err := prepareForkchoiceState(ctx, 2, [32]byte{'c'}, broot, [32]byte{'C'}, 0, 0)
+	st, cRoot, err := prepareForkchoiceState(ctx, 2, [32]byte{'c'}, bRoot, [32]byte{'C'}, 0, 0)
 	require.NoError(t, err)
-	require.NoError(t, f.InsertNode(ctx, st, croot))
+	require.NoError(t, f.InsertNode(ctx, st, cRoot))
 
 	// Children in same epoch
-	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: broot})
+	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: bRoot})
 	require.NoError(t, err)
 	require.Equal(t, false, viable)
 
-	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: broot, Epoch: 1})
+	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: bRoot, Epoch: 1})
 	require.NoError(t, err)
 	require.Equal(t, false, viable)
 
-	st, droot, err := prepareForkchoiceState(ctx, params.BeaconConfig().SlotsPerEpoch, [32]byte{'d'}, broot, [32]byte{'D'}, 0, 0)
+	st, dRoot, err := prepareForkchoiceState(ctx, params.BeaconConfig().SlotsPerEpoch, [32]byte{'d'}, bRoot, [32]byte{'D'}, 0, 0)
 	require.NoError(t, err)
-	require.NoError(t, f.InsertNode(ctx, st, droot))
+	require.NoError(t, f.InsertNode(ctx, st, dRoot))
 
 	// Children in next epoch but boundary
-	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: broot})
+	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: bRoot})
 	require.NoError(t, err)
 	require.Equal(t, false, viable)
 
-	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: broot, Epoch: 1})
+	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: bRoot, Epoch: 1})
 	require.NoError(t, err)
 	require.Equal(t, false, viable)
 
 	// Boundary block
-	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: droot, Epoch: 1})
+	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: dRoot, Epoch: 1})
 	require.NoError(t, err)
 	require.Equal(t, true, viable)
 
-	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: droot, Epoch: 0})
+	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: dRoot, Epoch: 0})
 	require.NoError(t, err)
 	require.Equal(t, false, viable)
 
 	// Children in next epoch
-	st, eroot, err := prepareForkchoiceState(ctx, params.BeaconConfig().SlotsPerEpoch+1, [32]byte{'e'}, broot, [32]byte{'E'}, 0, 0)
+	st, eRoot, err := prepareForkchoiceState(ctx, params.BeaconConfig().SlotsPerEpoch+1, [32]byte{'e'}, bRoot, [32]byte{'E'}, 0, 0)
 	require.NoError(t, err)
-	require.NoError(t, f.InsertNode(ctx, st, eroot))
+	require.NoError(t, f.InsertNode(ctx, st, eRoot))
 
-	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: broot, Epoch: 1})
+	viable, err = f.IsViableForCheckpoint(&forkchoicetypes.Checkpoint{Root: bRoot, Epoch: 1})
 	require.NoError(t, err)
 	require.Equal(t, true, viable)
 }

beacon-chain/forkchoice/doubly-linked-tree/node_test.go

@@ -86,7 +86,7 @@ func TestNode_UpdateBestDescendant_NonViableChild(t *testing.T) {
 func TestNode_UpdateBestDescendant_ViableChild(t *testing.T) {
 	f := setup(1, 1)
 	ctx := context.Background()
-	// Input child is best descendant
+	// Input child is the best descendant
 	state, blkRoot, err := prepareForkchoiceState(ctx, 1, indexToHash(1), params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, 1, 1)
 	require.NoError(t, err)
 	require.NoError(t, f.InsertNode(ctx, state, blkRoot))
@@ -99,7 +99,7 @@ func TestNode_UpdateBestDescendant_ViableChild(t *testing.T) {
 func TestNode_UpdateBestDescendant_HigherWeightChild(t *testing.T) {
 	f := setup(1, 1)
 	ctx := context.Background()
-	// Input child is best descendant
+	// Input child is the best descendant
 	state, blkRoot, err := prepareForkchoiceState(ctx, 1, indexToHash(1), params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, 1, 1)
 	require.NoError(t, err)
 	require.NoError(t, f.InsertNode(ctx, state, blkRoot))
@@ -119,7 +119,7 @@ func TestNode_UpdateBestDescendant_HigherWeightChild(t *testing.T) {
 func TestNode_UpdateBestDescendant_LowerWeightChild(t *testing.T) {
 	f := setup(1, 1)
 	ctx := context.Background()
-	// Input child is best descendant
+	// Input child is the best descendant
 	state, blkRoot, err := prepareForkchoiceState(ctx, 1, indexToHash(1), params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, 1, 1)
 	require.NoError(t, err)
 	require.NoError(t, f.InsertNode(ctx, state, blkRoot))

beacon-chain/forkchoice/doubly-linked-tree/proposer_boost_test.go

@@ -156,9 +156,9 @@ func TestForkChoice_BoostProposerRoot_PreventsExAnteAttack(t *testing.T) {
 		// Ancestors have the added weights of their children. Genesis is a special exception at 0 weight,
 		require.Equal(t, f.store.treeRootNode.weight, uint64(0))
 
-		// Proposer boost score with this tests parameters is 8
+		// Proposer boost score with these test parameters is 8
 		// Each of the nodes received one attestation accounting for 10.
-		// Node D is the only one with a proposer boost still applied:
+		// Node D is the only one with proposer boost still applied:
 		//
 		// (1: 48) -> (2: 38) -> (3: 10)
 		//		    \--------------->(4: 18)

beacon-chain/forkchoice/doubly-linked-tree/reorg_late_blocks.go

@@ -28,7 +28,7 @@ const orphanLateBlockProposingEarly = 2
 // 3- The beacon node is serving a validator that will propose during the next
 // slot.
 //
-// This function only applies an heuristic to decide if the beacon will update
+// This function only applies a heuristic to decide if the beacon will update
 // the engine's view of head with the parent block or the incoming block. It
 // does not guarantee an attempted reorg. This will only be decided later at
 // proposal time by calling GetProposerHead.

beacon-chain/forkchoice/doubly-linked-tree/store.go

@@ -61,7 +61,7 @@ func (s *Store) head(ctx context.Context) ([32]byte, error) {
 }
 
 // insert registers a new block node to the fork choice store's node list.
-// It then updates the new node's parent with best child and descendant node.
+// It then updates the new node's parent with the best child and descendant node.
 func (s *Store) insert(ctx context.Context,
 	slot primitives.Slot,
 	root, parentRoot, payloadHash [fieldparams.RootLength]byte,
@@ -75,7 +75,6 @@ func (s *Store) insert(ctx context.Context,
 	}
 
 	parent := s.nodeByRoot[parentRoot]
-
 	n := &Node{
 		slot:                     slot,
 		root:                     root,
@@ -89,6 +88,17 @@ func (s *Store) insert(ctx context.Context,
 		timestamp:                uint64(time.Now().Unix()),
 	}
 
+	// Set the node's target checkpoint
+	if slot%params.BeaconConfig().SlotsPerEpoch == 0 {
+		n.target = n
+	} else if parent != nil {
+		if slots.ToEpoch(slot) == slots.ToEpoch(parent.slot) {
+			n.target = parent.target
+		} else {
+			n.target = parent
+		}
+	}
+
 	s.nodeByPayload[payloadHash] = n
 	s.nodeByRoot[root] = n
 	if parent == nil {
@@ -145,6 +155,9 @@ func (s *Store) pruneFinalizedNodeByRootMap(ctx context.Context, node, finalized
 		return ctx.Err()
 	}
 	if node == finalizedNode {
+		if node.target != node {
+			node.target = nil
+		}
 		return nil
 	}
 	for _, child := range node.children {

beacon-chain/forkchoice/doubly-linked-tree/store_test.go

@@ -82,7 +82,7 @@ func TestStore_Head_Itself(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, f.InsertNode(context.Background(), state, blkRoot))
 
-	// Since the justified node does not have a best descendant so the best node
+	// Since the justified node does not have a best descendant, the best node
 	// is itself.
 	f.store.justifiedCheckpoint = &forkchoicetypes.Checkpoint{Epoch: 0, Root: indexToHash(1)}
 	h, err := f.store.head(context.Background())
@@ -213,7 +213,7 @@ func TestStore_Prune_ReturnEarly(t *testing.T) {
 	require.Equal(t, nodeCount, f.NodeCount())
 }
 
-// This unit tests starts with a simple branch like this
+// This unit test starts with a simple branch like this
 //
 //   - 1
 //     /
@@ -435,3 +435,90 @@ func TestForkChoice_ReceivedBlocksLastEpoch(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, uint64(0), count)
 }
+
+func TestStore_TargetRootForEpoch(t *testing.T) {
+	ctx := context.Background()
+	f := setup(1, 1)
+
+	state, blkRoot, err := prepareForkchoiceState(ctx, params.BeaconConfig().SlotsPerEpoch, [32]byte{'a'}, params.BeaconConfig().ZeroHash, params.BeaconConfig().ZeroHash, 1, 1)
+	require.NoError(t, err)
+	require.NoError(t, f.InsertNode(ctx, state, blkRoot))
+	target, err := f.TargetRootForEpoch(blkRoot, 1)
+	require.NoError(t, err)
+	require.Equal(t, target, blkRoot)
+
+	state, root1, err := prepareForkchoiceState(ctx, params.BeaconConfig().SlotsPerEpoch+1, [32]byte{'b'}, blkRoot, params.BeaconConfig().ZeroHash, 1, 1)
+	require.NoError(t, err)
+	require.NoError(t, f.InsertNode(ctx, state, root1))
+	target, err = f.TargetRootForEpoch(root1, 1)
+	require.NoError(t, err)
+	require.Equal(t, target, blkRoot)
+
+	// Insert a block for the next epoch (missed slot 0)
+
+	state, root2, err := prepareForkchoiceState(ctx, 2*params.BeaconConfig().SlotsPerEpoch+1, [32]byte{'c'}, root1, params.BeaconConfig().ZeroHash, 1, 1)
+	require.NoError(t, err)
+	require.NoError(t, f.InsertNode(ctx, state, root2))
+	target, err = f.TargetRootForEpoch(root2, 2)
+	require.NoError(t, err)
+	require.Equal(t, target, root1)
+
+	state, root3, err := prepareForkchoiceState(ctx, 2*params.BeaconConfig().SlotsPerEpoch+2, [32]byte{'d'}, root2, params.BeaconConfig().ZeroHash, 1, 1)
+	require.NoError(t, err)
+	require.NoError(t, f.InsertNode(ctx, state, root3))
+	target, err = f.TargetRootForEpoch(root2, 2)
+	require.NoError(t, err)
+	require.Equal(t, target, root1)
+
+	// Prune finalization
+	s := f.store
+	s.finalizedCheckpoint.Root = root1
+	require.NoError(t, s.prune(ctx))
+	target, err = f.TargetRootForEpoch(root1, 1)
+	require.NoError(t, err)
+	require.Equal(t, [32]byte{}, target)
+
+	// Insert a block for next epoch (slot 0 present)
+
+	state, root4, err := prepareForkchoiceState(ctx, 3*params.BeaconConfig().SlotsPerEpoch, [32]byte{'e'}, root1, params.BeaconConfig().ZeroHash, 1, 1)
+	require.NoError(t, err)
+	require.NoError(t, f.InsertNode(ctx, state, root4))
+	target, err = f.TargetRootForEpoch(root4, 3)
+	require.NoError(t, err)
+	require.Equal(t, target, root4)
+
+	state, root5, err := prepareForkchoiceState(ctx, 3*params.BeaconConfig().SlotsPerEpoch+1, [32]byte{'f'}, root4, params.BeaconConfig().ZeroHash, 1, 1)
+	require.NoError(t, err)
+	require.NoError(t, f.InsertNode(ctx, state, root5))
+	target, err = f.TargetRootForEpoch(root5, 3)
+	require.NoError(t, err)
+	require.Equal(t, target, root4)
+
+	// Target root where the target epoch is same or ahead of the block slot
+	target, err = f.TargetRootForEpoch(root5, 4)
+	require.NoError(t, err)
+	require.Equal(t, target, root5)
+
+	// Target root where the target epoch is two epochs ago
+	target, err = f.TargetRootForEpoch(root5, 2)
+	require.NoError(t, err)
+	require.Equal(t, root1, target) // the parent of root4 in epoch 3 is root 1 in epoch 1
+
+	// Target root where the target is two epochs ago, slot 0 was missed
+	state, root6, err := prepareForkchoiceState(ctx, 4*params.BeaconConfig().SlotsPerEpoch+1, [32]byte{'g'}, root5, params.BeaconConfig().ZeroHash, 1, 1)
+	require.NoError(t, err)
+	require.NoError(t, f.InsertNode(ctx, state, root6))
+	target, err = f.TargetRootForEpoch(root6, 4)
+	require.NoError(t, err)
+	require.Equal(t, target, root5)
+	target, err = f.TargetRootForEpoch(root6, 2)
+	require.NoError(t, err)
+	require.Equal(t, target, root1)
+
+	// Prune finalization
+	s.finalizedCheckpoint.Root = root4
+	require.NoError(t, s.prune(ctx))
+	target, err = f.TargetRootForEpoch(root4, 3)
+	require.NoError(t, err)
+	require.Equal(t, root4, target)
+}

beacon-chain/forkchoice/doubly-linked-tree/types.go

@@ -50,6 +50,7 @@ type Node struct {
 	root                     [fieldparams.RootLength]byte // root of the block converted to the node.
 	payloadHash              [fieldparams.RootLength]byte // payloadHash of the block converted to the node.
 	parent                   *Node                        // parent index of this node.
+	target                   *Node                        // target checkpoint for
 	children                 []*Node                      // the list of direct children of this Node
 	justifiedEpoch           primitives.Epoch             // justifiedEpoch of this node.
 	unrealizedJustifiedEpoch primitives.Epoch             // the epoch that would be justified if the block would be advanced to the next epoch.

beacon-chain/forkchoice/doubly-linked-tree/unrealized_justification_test.go

@@ -298,7 +298,7 @@ func TestStore_PullTips_Heuristics(t *testing.T) {
 		require.NoError(tt, err)
 		require.NoError(tt, f.InsertNode(ctx, st, root))
 		// Check that the justification point is not the parent's.
-		// This tests that the heuristics in pullTips did not apply and
+		// This test checks that the heuristics in pullTips did not apply and
 		// the test continues to compute a bogus unrealized
 		// justification
 		require.Equal(tt, primitives.Epoch(1), f.store.nodeByRoot[[32]byte{'h'}].unrealizedJustifiedEpoch)
@@ -315,7 +315,7 @@ func TestStore_PullTips_Heuristics(t *testing.T) {
 		require.NoError(tt, err)
 		require.NoError(tt, f.InsertNode(ctx, st, root))
 		// Check that the justification point is not the parent's.
-		// This tests that the heuristics in pullTips did not apply and
+		// This test checks that the heuristics in pullTips did not apply and
 		// the test continues to compute a bogus unrealized
 		// justification
 		require.Equal(tt, primitives.Epoch(1), f.store.nodeByRoot[[32]byte{'h'}].unrealizedJustifiedEpoch)
@@ -331,7 +331,7 @@ func TestStore_PullTips_Heuristics(t *testing.T) {
 		require.NoError(tt, err)
 		require.NoError(tt, f.InsertNode(ctx, st, root))
 		// Check that the justification point is not the parent's.
-		// This tests that the heuristics in pullTips did not apply and
+		// This test checks that the heuristics in pullTips did not apply and
 		// the test continues to compute a bogus unrealized
 		// justification
 		require.Equal(tt, primitives.Epoch(2), f.store.nodeByRoot[[32]byte{'h'}].unrealizedJustifiedEpoch)

beacon-chain/forkchoice/interfaces.go

@@ -16,10 +16,9 @@ type BalancesByRooter func(context.Context, [32]byte) ([]uint64, error)
 
 // ForkChoicer represents the full fork choice interface composed of all the sub-interfaces.
 type ForkChoicer interface {
+	RLocker // separate interface isolates  read locking for ROForkChoice.
 	Lock()
 	Unlock()
-	RLock()
-	RUnlock()
 	HeadRetriever        // to compute head.
 	BlockProcessor       // to track new block for fork choice.
 	AttestationProcessor // to track new attestation for fork choice.
@@ -27,6 +26,12 @@ type ForkChoicer interface {
 	Setter               // to set fork choice information.
 }
 
+// RLocker represents forkchoice's internal RWMutex read-only lock/unlock methods.
+type RLocker interface {
+	RLock()
+	RUnlock()
+}
+
 // HeadRetriever retrieves head root and optimistic info of the current chain.
 type HeadRetriever interface {
 	Head(context.Context) ([32]byte, error)
@@ -47,28 +52,33 @@ type AttestationProcessor interface {
 
 // Getter returns fork choice related information.
 type Getter interface {
-	HasNode([32]byte) bool
-	ProposerBoost() [fieldparams.RootLength]byte
+	FastGetter
 	AncestorRoot(ctx context.Context, root [32]byte, slot primitives.Slot) ([32]byte, error)
 	CommonAncestor(ctx context.Context, root1 [32]byte, root2 [32]byte) ([32]byte, primitives.Slot, error)
-	IsCanonical(root [32]byte) bool
-	FinalizedCheckpoint() *forkchoicetypes.Checkpoint
-	IsViableForCheckpoint(*forkchoicetypes.Checkpoint) (bool, error)
-	FinalizedPayloadBlockHash() [32]byte
-	JustifiedCheckpoint() *forkchoicetypes.Checkpoint
-	PreviousJustifiedCheckpoint() *forkchoicetypes.Checkpoint
-	JustifiedPayloadBlockHash() [32]byte
-	UnrealizedJustifiedPayloadBlockHash() [32]byte
-	NodeCount() int
-	HighestReceivedBlockSlot() primitives.Slot
-	ReceivedBlocksLastEpoch() (uint64, error)
 	ForkChoiceDump(context.Context) (*forkchoice2.Dump, error)
-	Weight(root [32]byte) (uint64, error)
 	Tips() ([][32]byte, []primitives.Slot)
+}
+
+type FastGetter interface {
+	FinalizedCheckpoint() *forkchoicetypes.Checkpoint
+	FinalizedPayloadBlockHash() [32]byte
+	HasNode([32]byte) bool
+	HighestReceivedBlockSlot() primitives.Slot
+	IsCanonical(root [32]byte) bool
 	IsOptimistic(root [32]byte) (bool, error)
+	IsViableForCheckpoint(*forkchoicetypes.Checkpoint) (bool, error)
+	JustifiedCheckpoint() *forkchoicetypes.Checkpoint
+	JustifiedPayloadBlockHash() [32]byte
+	LastRoot(primitives.Epoch) [32]byte
+	NodeCount() int
+	PreviousJustifiedCheckpoint() *forkchoicetypes.Checkpoint
+	ProposerBoost() [fieldparams.RootLength]byte
+	ReceivedBlocksLastEpoch() (uint64, error)
 	ShouldOverrideFCU() bool
 	Slot([32]byte) (primitives.Slot, error)
-	LastRoot(primitives.Epoch) [32]byte
+	TargetRootForEpoch([32]byte, primitives.Epoch) ([32]byte, error)
+	UnrealizedJustifiedPayloadBlockHash() [32]byte
+	Weight(root [32]byte) (uint64, error)
 }
 
 // Setter allows to set forkchoice information

beacon-chain/forkchoice/ro.go

@@ -0,0 +1,164 @@
+package forkchoice
+
+import (
+	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
+)
+
+// ROForkChoice is an implementation of forkchoice.Getter which calls `Rlock`/`RUnlock`
+// around a delegated method call to the underlying Getter implementation.
+type ROForkChoice struct {
+	getter FastGetter
+	l      RLocker
+}
+
+var _ FastGetter = &ROForkChoice{}
+
+// ROWrappable represents the subset of ForkChoicer a type needs to support
+// in order for ROForkChoice to wrap it. This simplifies the creation of a mock
+// type that can be used to assert that all of the wrapped methods are correctly
+// called between mutex acquire/release.
+type ROWrappable interface {
+	RLocker
+	FastGetter
+}
+
+// NewROForkChoice returns an ROForkChoice that delegates forkchoice.Getter calls to the
+// given value after first using its Locker methods to make sure it is correctly locked.
+func NewROForkChoice(w ROWrappable) *ROForkChoice {
+	return &ROForkChoice{getter: w, l: w}
+}
+
+// HasNode delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) HasNode(root [32]byte) bool {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.HasNode(root)
+}
+
+// ProposerBoost delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) ProposerBoost() [fieldparams.RootLength]byte {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.ProposerBoost()
+}
+
+// IsCanonical delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) IsCanonical(root [32]byte) bool {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.IsCanonical(root)
+}
+
+// FinalizedCheckpoint delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) FinalizedCheckpoint() *forkchoicetypes.Checkpoint {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.FinalizedCheckpoint()
+}
+
+// IsViableForCheckpoint delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) IsViableForCheckpoint(cp *forkchoicetypes.Checkpoint) (bool, error) {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.IsViableForCheckpoint(cp)
+}
+
+// FinalizedPayloadBlockHash delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) FinalizedPayloadBlockHash() [32]byte {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.FinalizedPayloadBlockHash()
+}
+
+// JustifiedCheckpoint delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) JustifiedCheckpoint() *forkchoicetypes.Checkpoint {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.JustifiedCheckpoint()
+}
+
+// PreviousJustifiedCheckpoint delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) PreviousJustifiedCheckpoint() *forkchoicetypes.Checkpoint {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.PreviousJustifiedCheckpoint()
+}
+
+// JustifiedPayloadBlockHash delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) JustifiedPayloadBlockHash() [32]byte {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.JustifiedPayloadBlockHash()
+}
+
+// UnrealizedJustifiedPayloadBlockHash delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) UnrealizedJustifiedPayloadBlockHash() [32]byte {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.UnrealizedJustifiedPayloadBlockHash()
+}
+
+// NodeCount delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) NodeCount() int {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.NodeCount()
+}
+
+// HighestReceivedBlockSlot delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) HighestReceivedBlockSlot() primitives.Slot {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.HighestReceivedBlockSlot()
+}
+
+// ReceivedBlocksLastEpoch delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) ReceivedBlocksLastEpoch() (uint64, error) {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.ReceivedBlocksLastEpoch()
+}
+
+// Weight delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) Weight(root [32]byte) (uint64, error) {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.Weight(root)
+}
+
+// IsOptimistic delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) IsOptimistic(root [32]byte) (bool, error) {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.IsOptimistic(root)
+}
+
+// ShouldOverrideFCU delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) ShouldOverrideFCU() bool {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.ShouldOverrideFCU()
+}
+
+// Slot delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) Slot(root [32]byte) (primitives.Slot, error) {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.Slot(root)
+}
+
+// LastRoot delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) LastRoot(e primitives.Epoch) [32]byte {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.LastRoot(e)
+}
+
+// TargetRootForEpoch delegates to the underlying forkchoice call, under a lock.
+func (ro *ROForkChoice) TargetRootForEpoch(root [32]byte, epoch primitives.Epoch) ([32]byte, error) {
+	ro.l.RLock()
+	defer ro.l.RUnlock()
+	return ro.getter.TargetRootForEpoch(root, epoch)
+}

beacon-chain/forkchoice/ro_test.go

@@ -0,0 +1,282 @@
+package forkchoice
+
+import (
+	"io"
+	"testing"
+
+	forkchoicetypes "github.com/prysmaticlabs/prysm/v4/beacon-chain/forkchoice/types"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
+	"github.com/prysmaticlabs/prysm/v4/testing/require"
+)
+
+type mockCall int
+
+const (
+	lockCalled mockCall = iota
+	unlockCalled
+	rlockCalled
+	runlockCalled
+	hasNodeCalled
+	proposerBoostCalled
+	isCanonicalCalled
+	finalizedCheckpointCalled
+	isViableForCheckpointCalled
+	finalizedPayloadBlockHashCalled
+	justifiedCheckpointCalled
+	previousJustifiedCheckpointCalled
+	justifiedPayloadBlockHashCalled
+	unrealizedJustifiedPayloadBlockHashCalled
+	nodeCountCalled
+	highestReceivedBlockSlotCalled
+	receivedBlocksLastEpochCalled
+	weightCalled
+	isOptimisticCalled
+	shouldOverrideFCUCalled
+	slotCalled
+	lastRootCalled
+	targetRootForEpochCalled
+)
+
+func _discard(t *testing.T, e error) {
+	if e != nil {
+		_, err := io.Discard.Write([]byte(e.Error()))
+		require.NoError(t, err)
+	}
+}
+
+// ensures that the correct func was called with the correct lock pattern
+// for each method in the interface.
+func TestROLocking(t *testing.T) {
+	cases := []struct {
+		name string
+		call mockCall
+		cb   func(FastGetter)
+	}{
+		{
+			name: "hasNodeCalled",
+			call: hasNodeCalled,
+			cb:   func(g FastGetter) { g.HasNode([32]byte{}) },
+		},
+		{
+			name: "proposerBoostCalled",
+			call: proposerBoostCalled,
+			cb:   func(g FastGetter) { g.ProposerBoost() },
+		},
+		{
+			name: "isCanonicalCalled",
+			call: isCanonicalCalled,
+			cb:   func(g FastGetter) { g.IsCanonical([32]byte{}) },
+		},
+		{
+			name: "finalizedCheckpointCalled",
+			call: finalizedCheckpointCalled,
+			cb:   func(g FastGetter) { g.FinalizedCheckpoint() },
+		},
+		{
+			name: "isViableForCheckpointCalled",
+			call: isViableForCheckpointCalled,
+			cb:   func(g FastGetter) { _, err := g.IsViableForCheckpoint(nil); _discard(t, err) },
+		},
+		{
+			name: "finalizedPayloadBlockHashCalled",
+			call: finalizedPayloadBlockHashCalled,
+			cb:   func(g FastGetter) { g.FinalizedPayloadBlockHash() },
+		},
+		{
+			name: "justifiedCheckpointCalled",
+			call: justifiedCheckpointCalled,
+			cb:   func(g FastGetter) { g.JustifiedCheckpoint() },
+		},
+		{
+			name: "previousJustifiedCheckpointCalled",
+			call: previousJustifiedCheckpointCalled,
+			cb:   func(g FastGetter) { g.PreviousJustifiedCheckpoint() },
+		},
+		{
+			name: "justifiedPayloadBlockHashCalled",
+			call: justifiedPayloadBlockHashCalled,
+			cb:   func(g FastGetter) { g.JustifiedPayloadBlockHash() },
+		},
+		{
+			name: "unrealizedJustifiedPayloadBlockHashCalled",
+			call: unrealizedJustifiedPayloadBlockHashCalled,
+			cb:   func(g FastGetter) { g.UnrealizedJustifiedPayloadBlockHash() },
+		},
+		{
+			name: "nodeCountCalled",
+			call: nodeCountCalled,
+			cb:   func(g FastGetter) { g.NodeCount() },
+		},
+		{
+			name: "highestReceivedBlockSlotCalled",
+			call: highestReceivedBlockSlotCalled,
+			cb:   func(g FastGetter) { g.HighestReceivedBlockSlot() },
+		},
+		{
+			name: "receivedBlocksLastEpochCalled",
+			call: receivedBlocksLastEpochCalled,
+			cb:   func(g FastGetter) { _, err := g.ReceivedBlocksLastEpoch(); _discard(t, err) },
+		},
+		{
+			name: "weightCalled",
+			call: weightCalled,
+			cb:   func(g FastGetter) { _, err := g.Weight([32]byte{}); _discard(t, err) },
+		},
+		{
+			name: "isOptimisticCalled",
+			call: isOptimisticCalled,
+			cb:   func(g FastGetter) { _, err := g.IsOptimistic([32]byte{}); _discard(t, err) },
+		},
+		{
+			name: "shouldOverrideFCUCalled",
+			call: shouldOverrideFCUCalled,
+			cb:   func(g FastGetter) { g.ShouldOverrideFCU() },
+		},
+		{
+			name: "slotCalled",
+			call: slotCalled,
+			cb:   func(g FastGetter) { _, err := g.Slot([32]byte{}); _discard(t, err) },
+		},
+		{
+			name: "lastRootCalled",
+			call: lastRootCalled,
+			cb:   func(g FastGetter) { g.LastRoot(0) },
+		},
+		{
+			name: "targetRootForEpochCalled",
+			call: targetRootForEpochCalled,
+			cb:   func(g FastGetter) { _, err := g.TargetRootForEpoch([32]byte{}, 0); _discard(t, err) },
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			m := &mockROForkchoice{}
+			ro := NewROForkChoice(m)
+			c.cb(ro)
+			require.Equal(t, rlockCalled, m.calls[0])
+			require.Equal(t, c.call, m.calls[1])
+			require.Equal(t, runlockCalled, m.calls[2])
+		})
+	}
+}
+
+type mockROForkchoice struct {
+	calls []mockCall
+}
+
+var _ FastGetter = &mockROForkchoice{}
+
+var _ RLocker = &mockROForkchoice{}
+
+func (ro *mockROForkchoice) Lock() {
+	ro.calls = append(ro.calls, lockCalled)
+}
+
+func (ro *mockROForkchoice) RLock() {
+	ro.calls = append(ro.calls, rlockCalled)
+}
+
+func (ro *mockROForkchoice) Unlock() {
+	ro.calls = append(ro.calls, unlockCalled)
+}
+
+func (ro *mockROForkchoice) RUnlock() {
+	ro.calls = append(ro.calls, runlockCalled)
+}
+
+func (ro *mockROForkchoice) HasNode(_ [32]byte) bool {
+	ro.calls = append(ro.calls, hasNodeCalled)
+	return false
+}
+
+func (ro *mockROForkchoice) ProposerBoost() [fieldparams.RootLength]byte {
+	ro.calls = append(ro.calls, proposerBoostCalled)
+	return [fieldparams.RootLength]byte{}
+}
+
+func (ro *mockROForkchoice) IsCanonical(_ [32]byte) bool {
+	ro.calls = append(ro.calls, isCanonicalCalled)
+	return false
+}
+
+func (ro *mockROForkchoice) FinalizedCheckpoint() *forkchoicetypes.Checkpoint {
+	ro.calls = append(ro.calls, finalizedCheckpointCalled)
+	return nil
+}
+
+func (ro *mockROForkchoice) IsViableForCheckpoint(_ *forkchoicetypes.Checkpoint) (bool, error) {
+	ro.calls = append(ro.calls, isViableForCheckpointCalled)
+	return false, nil
+}
+
+func (ro *mockROForkchoice) FinalizedPayloadBlockHash() [32]byte {
+	ro.calls = append(ro.calls, finalizedPayloadBlockHashCalled)
+	return [32]byte{}
+}
+
+func (ro *mockROForkchoice) JustifiedCheckpoint() *forkchoicetypes.Checkpoint {
+	ro.calls = append(ro.calls, justifiedCheckpointCalled)
+	return nil
+}
+
+func (ro *mockROForkchoice) PreviousJustifiedCheckpoint() *forkchoicetypes.Checkpoint {
+	ro.calls = append(ro.calls, previousJustifiedCheckpointCalled)
+	return nil
+}
+
+func (ro *mockROForkchoice) JustifiedPayloadBlockHash() [32]byte {
+	ro.calls = append(ro.calls, justifiedPayloadBlockHashCalled)
+	return [32]byte{}
+}
+
+func (ro *mockROForkchoice) UnrealizedJustifiedPayloadBlockHash() [32]byte {
+	ro.calls = append(ro.calls, unrealizedJustifiedPayloadBlockHashCalled)
+	return [32]byte{}
+}
+
+func (ro *mockROForkchoice) NodeCount() int {
+	ro.calls = append(ro.calls, nodeCountCalled)
+	return 0
+}
+
+func (ro *mockROForkchoice) HighestReceivedBlockSlot() primitives.Slot {
+	ro.calls = append(ro.calls, highestReceivedBlockSlotCalled)
+	return 0
+}
+
+func (ro *mockROForkchoice) ReceivedBlocksLastEpoch() (uint64, error) {
+	ro.calls = append(ro.calls, receivedBlocksLastEpochCalled)
+	return 0, nil
+}
+
+func (ro *mockROForkchoice) Weight(_ [32]byte) (uint64, error) {
+	ro.calls = append(ro.calls, weightCalled)
+	return 0, nil
+}
+
+func (ro *mockROForkchoice) IsOptimistic(_ [32]byte) (bool, error) {
+	ro.calls = append(ro.calls, isOptimisticCalled)
+	return false, nil
+}
+
+func (ro *mockROForkchoice) ShouldOverrideFCU() bool {
+	ro.calls = append(ro.calls, shouldOverrideFCUCalled)
+	return false
+}
+
+func (ro *mockROForkchoice) Slot(_ [32]byte) (primitives.Slot, error) {
+	ro.calls = append(ro.calls, slotCalled)
+	return 0, nil
+}
+
+func (ro *mockROForkchoice) LastRoot(_ primitives.Epoch) [32]byte {
+	ro.calls = append(ro.calls, lastRootCalled)
+	return [32]byte{}
+}
+
+// TargetRootForEpoch implements FastGetter.
+func (ro *mockROForkchoice) TargetRootForEpoch(_ [32]byte, _ primitives.Epoch) ([32]byte, error) {
+	ro.calls = append(ro.calls, targetRootForEpochCalled)
+	return [32]byte{}, nil
+}

beacon-chain/gateway/BUILD.bazel

@@ -8,7 +8,6 @@ go_library(
     deps = [
         "//api/gateway:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",
-        "//proto/eth/service:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "@com_github_grpc_ecosystem_grpc_gateway_v2//runtime:go_default_library",
         "@org_golang_google_protobuf//encoding/protojson:go_default_library",

beacon-chain/gateway/helpers.go

@@ -4,7 +4,6 @@ import (
 	gwruntime "github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/prysmaticlabs/prysm/v4/api/gateway"
 	"github.com/prysmaticlabs/prysm/v4/cmd/beacon-chain/flags"
-	ethpbservice "github.com/prysmaticlabs/prysm/v4/proto/eth/service"
 	ethpbalpha "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"google.golang.org/protobuf/encoding/protojson"
 )
@@ -51,10 +50,7 @@ func DefaultConfig(enableDebugRPCEndpoints bool, httpModules string) MuxConfig {
 		}
 	}
 	if flags.EnableHTTPEthAPI(httpModules) {
-		ethRegistrations := []gateway.PbHandlerRegistration{
-			ethpbservice.RegisterBeaconChainHandler,
-			ethpbservice.RegisterEventsHandler,
-		}
+		ethRegistrations := []gateway.PbHandlerRegistration{}
 		ethMux := gwruntime.NewServeMux(
 			gwruntime.WithMarshalerOption(gwruntime.MIMEWildcard, &gwruntime.HTTPBodyMarshaler{
 				Marshaler: &gwruntime.JSONPb{

beacon-chain/gateway/helpers_test.go

@@ -14,7 +14,7 @@ func TestDefaultConfig(t *testing.T) {
 		assert.NotNil(t, cfg.EthPbMux.Mux)
 		require.Equal(t, 2, len(cfg.EthPbMux.Patterns))
 		assert.Equal(t, "/internal/eth/v1/", cfg.EthPbMux.Patterns[0])
-		assert.Equal(t, 2, len(cfg.EthPbMux.Registrations))
+		assert.Equal(t, 0, len(cfg.EthPbMux.Registrations))
 		assert.Equal(t, (*gateway.PbMux)(nil), cfg.V1AlphaPbMux)
 	})
 	t.Run("Without Eth API", func(t *testing.T) {

beacon-chain/node/BUILD.bazel

@@ -24,6 +24,7 @@ go_library(
         "//beacon-chain/cache/depositcache:go_default_library",
         "//beacon-chain/cache/depositsnapshot:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/db/filesystem:go_default_library",
         "//beacon-chain/db/kv:go_default_library",
         "//beacon-chain/db/slasherkv:go_default_library",
         "//beacon-chain/deterministic-genesis:go_default_library",
@@ -40,7 +41,6 @@ go_library(
         "//beacon-chain/operations/voluntaryexits:go_default_library",
         "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/rpc:go_default_library",
-        "//beacon-chain/rpc/apimiddleware:go_default_library",
         "//beacon-chain/slasher:go_default_library",
         "//beacon-chain/startup:go_default_library",
         "//beacon-chain/state:go_default_library",
@@ -85,6 +85,7 @@ go_test(
         "//beacon-chain/blockchain:go_default_library",
         "//beacon-chain/builder:go_default_library",
         "//beacon-chain/core/feed/state:go_default_library",
+        "//beacon-chain/db/filesystem:go_default_library",
         "//beacon-chain/execution:go_default_library",
         "//beacon-chain/execution/testing:go_default_library",
         "//beacon-chain/monitor:go_default_library",

beacon-chain/node/config.go

@@ -40,7 +40,7 @@ func configureHistoricalSlasher(cliCtx *cli.Context) error {
 			return err
 		}
 		cmdConfig := cmd.Get()
-		// Allow up to 4096 attestations at a time to be requested from the beacon nde.
+		// Allow up to 4096 attestations at a time to be requested from the beacon node.
 		cmdConfig.MaxRPCPageSize = int(params.BeaconConfig().SlotsPerEpoch.Mul(params.BeaconConfig().MaxAttestations)) // lint:ignore uintcast -- Page size should not exceed int64 with these constants.
 		cmd.Init(cmdConfig)
 		log.Warnf(

beacon-chain/node/node.go

@@ -26,6 +26,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache/depositcache"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/cache/depositsnapshot"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/kv"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/slasherkv"
 	interopcoldstart "github.com/prysmaticlabs/prysm/v4/beacon-chain/deterministic-genesis"
@@ -42,7 +43,6 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/operations/voluntaryexits"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/p2p"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc"
-	"github.com/prysmaticlabs/prysm/v4/beacon-chain/rpc/apimiddleware"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/slasher"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/startup"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/state"
@@ -112,11 +112,13 @@ type BeaconNode struct {
 	forkChoicer             forkchoice.ForkChoicer
 	clockWaiter             startup.ClockWaiter
 	initialSyncComplete     chan struct{}
+	BlobStorage             *filesystem.BlobStorage
+	blobRetentionEpochs     primitives.Epoch
 }
 
 // New creates a new node instance, sets up configuration options, and registers
 // every required service to the node.
-func New(cliCtx *cli.Context, opts ...Option) (*BeaconNode, error) {
+func New(cliCtx *cli.Context, cancel context.CancelFunc, opts ...Option) (*BeaconNode, error) {
 	if err := configureTracing(cliCtx); err != nil {
 		return nil, err
 	}
@@ -154,9 +156,6 @@ func New(cliCtx *cli.Context, opts ...Option) (*BeaconNode, error) {
 	if err := configureExecutionSetting(cliCtx); err != nil {
 		return nil, err
 	}
-	if err := kv.ConfigureBlobRetentionEpoch(cliCtx); err != nil {
-		return nil, err
-	}
 	configureFastSSZHashingAlgorithm()
 
 	// Initializes any forks here.
@@ -164,7 +163,7 @@ func New(cliCtx *cli.Context, opts ...Option) (*BeaconNode, error) {
 
 	registry := runtime.NewServiceRegistry()
 
-	ctx, cancel := context.WithCancel(cliCtx.Context)
+	ctx := cliCtx.Context
 	beacon := &BeaconNode{
 		cliCtx:                  cliCtx,
 		ctx:                     ctx,
@@ -200,6 +199,7 @@ func New(cliCtx *cli.Context, opts ...Option) (*BeaconNode, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	log.Debugln("Starting DB")
 	if err := beacon.startDB(cliCtx, depositAddress); err != nil {
 		return nil, err
@@ -378,7 +378,7 @@ func (b *BeaconNode) startDB(cliCtx *cli.Context, depositAddress string) error {
 
 	log.WithField("database-path", dbPath).Info("Checking DB")
 
-	d, err := db.NewDB(b.ctx, dbPath)
+	d, err := kv.NewKVStore(b.ctx, dbPath, kv.WithBlobRetentionEpochs(b.blobRetentionEpochs))
 	if err != nil {
 		return err
 	}
@@ -400,7 +400,8 @@ func (b *BeaconNode) startDB(cliCtx *cli.Context, depositAddress string) error {
 		if err := d.ClearDB(); err != nil {
 			return errors.Wrap(err, "could not clear database")
 		}
-		d, err = db.NewDB(b.ctx, dbPath)
+
+		d, err = kv.NewKVStore(b.ctx, dbPath, kv.WithBlobRetentionEpochs(b.blobRetentionEpochs))
 		if err != nil {
 			return errors.Wrap(err, "could not create new database")
 		}
@@ -567,6 +568,7 @@ func (b *BeaconNode) registerP2P(cliCtx *cli.Context) error {
 		TCPPort:           cliCtx.Uint(cmd.P2PTCPPort.Name),
 		UDPPort:           cliCtx.Uint(cmd.P2PUDPPort.Name),
 		MaxPeers:          cliCtx.Uint(cmd.P2PMaxPeers.Name),
+		QueueSize:         cliCtx.Uint(cmd.PubsubQueueSize.Name),
 		AllowListCIDR:     cliCtx.String(cmd.P2PAllowList.Name),
 		DenyListCIDR:      slice.SplitCommaSeparated(cliCtx.StringSlice(cmd.P2PDenyList.Name)),
 		EnableUPnP:        cliCtx.Bool(cmd.EnableUPnPFlag.Name),
@@ -639,6 +641,7 @@ func (b *BeaconNode) registerBlockchainService(fc forkchoice.ForkChoicer, gs *st
 		blockchain.WithProposerIdsCache(b.proposerIdsCache),
 		blockchain.WithClockSynchronizer(gs),
 		blockchain.WithSyncComplete(syncComplete),
+		blockchain.WithBlobStorage(b.BlobStorage),
 	)
 
 	blockchainService, err := blockchain.NewService(b.ctx, opts...)
@@ -671,6 +674,7 @@ func (b *BeaconNode) registerPOWChainService() error {
 		execution.WithStateGen(b.stateGen),
 		execution.WithBeaconNodeStatsUpdater(bs),
 		execution.WithFinalizedStateAtStartup(b.finalizedStateAtStartUp),
+		execution.WithJwtId(b.cliCtx.String(flags.JwtId.Name)),
 	)
 	web3Service, err := execution.NewService(b.ctx, opts...)
 	if err != nil {
@@ -717,6 +721,7 @@ func (b *BeaconNode) registerSyncService(initialSyncComplete chan struct{}) erro
 		regularsync.WithClockWaiter(b.clockWaiter),
 		regularsync.WithInitialSyncComplete(initialSyncComplete),
 		regularsync.WithStateNotifier(b),
+		regularsync.WithBlobStorage(b.BlobStorage),
 	)
 	return b.services.RegisterService(rs)
 }
@@ -841,6 +846,7 @@ func (b *BeaconNode) registerRPCService(router *mux.Router) error {
 		ForkchoiceFetcher:             chainService,
 		FinalizationFetcher:           chainService,
 		BlockReceiver:                 chainService,
+		BlobReceiver:                  chainService,
 		AttestationReceiver:           chainService,
 		GenesisTimeFetcher:            chainService,
 		GenesisFetcher:                chainService,
@@ -932,9 +938,6 @@ func (b *BeaconNode) registerGRPCGateway(router *mux.Router) error {
 		apigateway.WithAllowedOrigins(allowedOrigins),
 		apigateway.WithTimeout(uint64(timeout)),
 	}
-	if flags.EnableHTTPEthAPI(httpModules) {
-		opts = append(opts, apigateway.WithApiMiddleware(&apimiddleware.BeaconEndpointFactory{}))
-	}
 	g, err := apigateway.New(b.ctx, opts...)
 	if err != nil {
 		return err

beacon-chain/node/node_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/builder"
 	statefeed "github.com/prysmaticlabs/prysm/v4/beacon-chain/core/feed/state"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/execution"
 	mockExecution "github.com/prysmaticlabs/prysm/v4/beacon-chain/execution/testing"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/monitor"
@@ -32,6 +33,12 @@ import (
 // Ensure BeaconNode implements interfaces.
 var _ statefeed.Notifier = (*BeaconNode)(nil)
 
+func newCliContextWithCancel(app *cli.App, set *flag.FlagSet) (*cli.Context, context.CancelFunc) {
+	context, cancel := context.WithCancel(context.Background())
+	parent := &cli.Context{Context: context}
+	return cli.NewContext(app, set, parent), cancel
+}
+
 // Test that beacon chain node can close.
 func TestNodeClose_OK(t *testing.T) {
 	hook := logTest.NewGlobal()
@@ -48,9 +55,9 @@ func TestNodeClose_OK(t *testing.T) {
 	require.NoError(t, set.Set("suggested-fee-recipient", "0x6e35733c5af9B61374A128e6F85f553aF09ff89A"))
 	cmd.ValidatorMonitorIndicesFlag.Value = &cli.IntSlice{}
 	cmd.ValidatorMonitorIndicesFlag.Value.SetInt(1)
-	ctx := cli.NewContext(&app, set, nil)
+	ctx, cancel := newCliContextWithCancel(&app, set)
 
-	node, err := New(ctx)
+	node, err := New(ctx, cancel)
 	require.NoError(t, err)
 
 	node.Close()
@@ -67,10 +74,11 @@ func TestNodeStart_Ok(t *testing.T) {
 	set.String("suggested-fee-recipient", "0x6e35733c5af9B61374A128e6F85f553aF09ff89A", "fee recipient")
 	require.NoError(t, set.Set("suggested-fee-recipient", "0x6e35733c5af9B61374A128e6F85f553aF09ff89A"))
 
-	ctx := cli.NewContext(&app, set, nil)
-	node, err := New(ctx, WithBlockchainFlagOptions([]blockchain.Option{}),
+	ctx, cancel := newCliContextWithCancel(&app, set)
+	node, err := New(ctx, cancel, WithBlockchainFlagOptions([]blockchain.Option{}),
 		WithBuilderFlagOptions([]builder.Option{}),
-		WithExecutionChainOptions([]execution.Option{}))
+		WithExecutionChainOptions([]execution.Option{}),
+		WithBlobStorage(filesystem.NewEphemeralBlobStorage(t)))
 	require.NoError(t, err)
 	node.services = &runtime.ServiceRegistry{}
 	go func() {
@@ -79,7 +87,6 @@ func TestNodeStart_Ok(t *testing.T) {
 	time.Sleep(3 * time.Second)
 	node.Close()
 	require.LogsContain(t, hook, "Starting beacon node")
-
 }
 
 func TestNodeStart_Ok_registerDeterministicGenesisService(t *testing.T) {
@@ -115,8 +122,8 @@ func TestNodeStart_Ok_registerDeterministicGenesisService(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, os.WriteFile("genesis_ssz.json", genesisBytes, 0666))
 	set.String("genesis-state", "genesis_ssz.json", "")
-	ctx := cli.NewContext(&app, set, nil)
-	node, err := New(ctx, WithBlockchainFlagOptions([]blockchain.Option{}),
+	ctx, cancel := newCliContextWithCancel(&app, set)
+	node, err := New(ctx, cancel, WithBlockchainFlagOptions([]blockchain.Option{}),
 		WithBuilderFlagOptions([]builder.Option{}),
 		WithExecutionChainOptions([]execution.Option{}))
 	require.NoError(t, err)
@@ -147,12 +154,13 @@ func TestClearDB(t *testing.T) {
 	set.Bool(cmd.ForceClearDB.Name, true, "force clear db")
 	set.String("suggested-fee-recipient", "0x6e35733c5af9B61374A128e6F85f553aF09ff89A", "fee recipient")
 	require.NoError(t, set.Set("suggested-fee-recipient", "0x6e35733c5af9B61374A128e6F85f553aF09ff89A"))
-	context := cli.NewContext(&app, set, nil)
-	_, err = New(context, WithExecutionChainOptions([]execution.Option{
-		execution.WithHttpEndpoint(endpoint),
-	}))
+	context, cancel := newCliContextWithCancel(&app, set)
+	options := []Option{
+		WithExecutionChainOptions([]execution.Option{execution.WithHttpEndpoint(endpoint)}),
+		WithBlobStorage(filesystem.NewEphemeralBlobStorage(t)),
+	}
+	_, err = New(context, cancel, options...)
 	require.NoError(t, err)
-
 	require.LogsContain(t, hook, "Removing database")
 }
 

beacon-chain/node/options.go

@@ -3,7 +3,9 @@ package node
 import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/blockchain"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/builder"
+	"github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem"
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/execution"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 )
 
 // Option for beacon node configuration.
@@ -32,3 +34,19 @@ func WithBuilderFlagOptions(opts []builder.Option) Option {
 		return nil
 	}
 }
+
+// WithBlobStorage sets the BlobStorage backend for the BeaconNode
+func WithBlobStorage(bs *filesystem.BlobStorage) Option {
+	return func(bn *BeaconNode) error {
+		bn.BlobStorage = bs
+		return nil
+	}
+}
+
+// WithBlobRetentionEpochs sets the blobRetentionEpochs value, used in kv store initialization.
+func WithBlobRetentionEpochs(e primitives.Epoch) Option {
+	return func(bn *BeaconNode) error {
+		bn.blobRetentionEpochs = e
+		return nil
+	}
+}

beacon-chain/operations/attestations/kv/unaggregated.go

@@ -121,15 +121,11 @@ func (c *AttCaches) DeleteSeenUnaggregatedAttestations() (int, error) {
 	defer c.unAggregateAttLock.Unlock()
 
 	count := 0
-	for _, att := range c.unAggregatedAtt {
+	for r, att := range c.unAggregatedAtt {
 		if att == nil || helpers.IsAggregated(att) {
 			continue
 		}
 		if seen, err := c.hasSeenBit(att); err == nil && seen {
-			r, err := hashFn(att)
-			if err != nil {
-				return count, errors.Wrap(err, "could not tree hash attestation")
-			}
 			delete(c.unAggregatedAtt, r)
 			count++
 		}

beacon-chain/p2p/broadcaster.go

@@ -204,7 +204,7 @@ func (s *Service) broadcastSyncCommittee(ctx context.Context, subnet uint64, sMs
 
 // BroadcastBlob broadcasts a blob to the p2p network, the message is assumed to be
 // broadcasted to the current fork and to the input subnet.
-func (s *Service) BroadcastBlob(ctx context.Context, subnet uint64, blob *ethpb.SignedBlobSidecar) error {
+func (s *Service) BroadcastBlob(ctx context.Context, subnet uint64, blob *ethpb.BlobSidecar) error {
 	ctx, span := trace.StartSpan(ctx, "p2p.BroadcastBlob")
 	defer span.End()
 	if blob == nil {
@@ -223,7 +223,7 @@ func (s *Service) BroadcastBlob(ctx context.Context, subnet uint64, blob *ethpb.
 	return nil
 }
 
-func (s *Service) broadcastBlob(ctx context.Context, subnet uint64, blobSidecar *ethpb.SignedBlobSidecar, forkDigest [4]byte) {
+func (s *Service) broadcastBlob(ctx context.Context, subnet uint64, blobSidecar *ethpb.BlobSidecar, forkDigest [4]byte) {
 	_, span := trace.StartSpan(ctx, "p2p.broadcastBlob")
 	defer span.End()
 	ctx = trace.NewContext(context.Background(), span) // clear parent context / deadline.

beacon-chain/p2p/broadcaster_test.go

@@ -469,18 +469,18 @@ func TestService_BroadcastBlob(t *testing.T) {
 		}),
 	}
 
-	blobSidecar := &ethpb.SignedBlobSidecar{
-		Message: &ethpb.DeprecatedBlobSidecar{
-			BlockRoot:       bytesutil.PadTo([]byte{'A'}, fieldparams.RootLength),
-			Index:           1,
-			Slot:            2,
-			BlockParentRoot: bytesutil.PadTo([]byte{'B'}, fieldparams.RootLength),
-			ProposerIndex:   3,
-			Blob:            bytesutil.PadTo([]byte{'C'}, fieldparams.BlobLength),
-			KzgCommitment:   bytesutil.PadTo([]byte{'D'}, fieldparams.BLSPubkeyLength),
-			KzgProof:        bytesutil.PadTo([]byte{'E'}, fieldparams.BLSPubkeyLength),
-		},
-		Signature: bytesutil.PadTo([]byte{'F'}, fieldparams.BLSSignatureLength),
+	header := util.HydrateSignedBeaconHeader(&ethpb.SignedBeaconBlockHeader{})
+	commitmentInclusionProof := make([][]byte, 17)
+	for i := range commitmentInclusionProof {
+		commitmentInclusionProof[i] = bytesutil.PadTo([]byte{}, 32)
+	}
+	blobSidecar := &ethpb.BlobSidecar{
+		Index:                    1,
+		Blob:                     bytesutil.PadTo([]byte{'C'}, fieldparams.BlobLength),
+		KzgCommitment:            bytesutil.PadTo([]byte{'D'}, fieldparams.BLSPubkeyLength),
+		KzgProof:                 bytesutil.PadTo([]byte{'E'}, fieldparams.BLSPubkeyLength),
+		SignedBlockHeader:        header,
+		CommitmentInclusionProof: commitmentInclusionProof,
 	}
 	subnet := uint64(0)
 
@@ -508,7 +508,7 @@ func TestService_BroadcastBlob(t *testing.T) {
 		incomingMessage, err := sub.Next(ctx)
 		require.NoError(t, err)
 
-		result := &ethpb.SignedBlobSidecar{}
+		result := &ethpb.BlobSidecar{}
 		require.NoError(t, p.Encoding().DecodeGossip(incomingMessage.Data, result))
 		require.DeepEqual(t, result, blobSidecar)
 	}(t)

beacon-chain/p2p/config.go

@@ -6,6 +6,9 @@ import (
 	"github.com/prysmaticlabs/prysm/v4/beacon-chain/startup"
 )
 
+// This is the default queue size used if we have specified an invalid one.
+const defaultPubsubQueueSize = 600
+
 // Config for the p2p service. These parameters are set from application level flags
 // to initialize the p2p service.
 type Config struct {
@@ -25,9 +28,20 @@ type Config struct {
 	TCPPort             uint
 	UDPPort             uint
 	MaxPeers            uint
+	QueueSize           uint
 	AllowListCIDR       string
 	DenyListCIDR        []string
 	StateNotifier       statefeed.Notifier
 	DB                  db.ReadOnlyDatabase
 	ClockWaiter         startup.ClockWaiter
 }
+
+// validateConfig validates whether the values provided are accurate and will set
+// the appropriate values for those that are invalid.
+func validateConfig(cfg *Config) *Config {
+	if cfg.QueueSize == 0 {
+		log.Warnf("Invalid pubsub queue size of %d initialized, setting the quese size as %d instead", cfg.QueueSize, defaultPubsubQueueSize)
+		cfg.QueueSize = defaultPubsubQueueSize
+	}
+	return cfg
+}

beacon-chain/p2p/connection_gater.go

@@ -21,7 +21,7 @@ const (
 
 	// High watermark buffer signifies the buffer till which
 	// we will handle inbound requests.
-	highWatermarkBuffer = 10
+	highWatermarkBuffer = 20
 )
 
 // InterceptPeerDial tests whether we're permitted to Dial the specified peer.

beacon-chain/p2p/gossip_topic_mappings.go

@@ -21,7 +21,7 @@ var gossipTopicMappings = map[string]proto.Message{
 	SyncContributionAndProofSubnetTopicFormat: &ethpb.SignedContributionAndProof{},
 	SyncCommitteeSubnetTopicFormat:            &ethpb.SyncCommitteeMessage{},
 	BlsToExecutionChangeSubnetTopicFormat:     &ethpb.SignedBLSToExecutionChange{},
-	BlobSubnetTopicFormat:                     &ethpb.SignedBlobSidecar{},
+	BlobSubnetTopicFormat:                     &ethpb.BlobSidecar{},
 }
 
 // GossipTopicMappings is a function to return the assigned data type

beacon-chain/p2p/interfaces.go

@@ -35,7 +35,7 @@ type Broadcaster interface {
 	Broadcast(context.Context, proto.Message) error
 	BroadcastAttestation(ctx context.Context, subnet uint64, att *ethpb.Attestation) error
 	BroadcastSyncCommitteeMessage(ctx context.Context, subnet uint64, sMsg *ethpb.SyncCommitteeMessage) error
-	BroadcastBlob(ctx context.Context, subnet uint64, blob *ethpb.SignedBlobSidecar) error
+	BroadcastBlob(ctx context.Context, subnet uint64, blob *ethpb.BlobSidecar) error
 }
 
 // SetStreamHandler configures p2p to handle streams of a certain topic ID.

beacon-chain/p2p/monitoring.go

@@ -113,7 +113,7 @@ var (
 		Name: "p2p_pubsub_reject_total",
 		Help: "The number of messages rejected of a particular topic",
 	},
-		[]string{"topic"})
+		[]string{"topic", "reason"})
 	pubsubPeerThrottle = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "p2p_pubsub_throttle_total",
 		Help: "The number of times a peer has been throttled for a particular topic",
@@ -121,31 +121,46 @@ var (
 		[]string{"topic"})
 	pubsubRPCRecv = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "p2p_pubsub_rpc_recv_total",
-		Help: "The number of messages received via rpc for a particular topic",
+		Help: "The number of messages received via rpc for a particular control message",
 	},
 		[]string{"control_message"})
 	pubsubRPCSubRecv = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "p2p_pubsub_rpc_recv_sub_total",
 		Help: "The number of subscription messages received via rpc",
 	})
+	pubsubRPCPubRecv = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "p2p_pubsub_rpc_recv_pub_total",
+		Help: "The number of publish messages received via rpc for a particular topic",
+	},
+		[]string{"topic"})
 	pubsubRPCDrop = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "p2p_pubsub_rpc_drop_total",
-		Help: "The number of messages dropped via rpc for a particular topic",
+		Help: "The number of messages dropped via rpc for a particular control message",
 	},
 		[]string{"control_message"})
 	pubsubRPCSubDrop = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "p2p_pubsub_rpc_drop_sub_total",
 		Help: "The number of subscription messages dropped via rpc",
 	})
+	pubsubRPCPubDrop = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "p2p_pubsub_rpc_drop_pub_total",
+		Help: "The number of publish messages dropped via rpc for a particular topic",
+	},
+		[]string{"topic"})
 	pubsubRPCSent = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "p2p_pubsub_rpc_sent_total",
-		Help: "The number of messages sent via rpc for a particular topic",
+		Help: "The number of messages sent via rpc for a particular control message",
 	},
 		[]string{"control_message"})
 	pubsubRPCSubSent = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "p2p_pubsub_rpc_sent_sub_total",
 		Help: "The number of subscription messages sent via rpc",
 	})
+	pubsubRPCPubSent = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "p2p_pubsub_rpc_sent_pub_total",
+		Help: "The number of publish messages sent via rpc for a particular topic",
+	},
+		[]string{"topic"})
 )
 
 func (s *Service) updateMetrics() {