Add metadata fields to getBlobSidecars (#14677)

* add metadata fields to getBlobSidecars

* gaz

* changelog

* Dhruv + Radek' reviews

CHANGELOG.md

@@ -67,6 +67,7 @@ The format is based on Keep a Changelog, and this project adheres to Semantic Ve
 - Fixed pending deposits processing on Electra.
 - Modified `ListAttestationsV2`, `GetAttesterSlashingsV2` and `GetAggregateAttestationV2` endpoints to use slot to determine fork version.
 - Improvements to HTTP response handling. [pr](https://github.com/prysmaticlabs/prysm/pull/14673)
+- Updated `Blobs` endpoint to return additional metadata fields.
 
 ### Deprecated
 

api/server/structs/endpoints_blob.go

@@ -1,7 +1,10 @@
 package structs
 
 type SidecarsResponse struct {
-	Data []*Sidecar `json:"data"`
+	Version             string     `json:"version"`
+	Data                []*Sidecar `json:"data"`
+	ExecutionOptimistic bool       `json:"execution_optimistic"`
+	Finalized           bool       `json:"finalized"`
 }
 
 type Sidecar struct {

beacon-chain/rpc/endpoints.go

@@ -165,9 +165,11 @@ func (s *Service) builderEndpoints(stater lookup.Stater) []endpoint {
 	}
 }
 
-func (*Service) blobEndpoints(blocker lookup.Blocker) []endpoint {
+func (s *Service) blobEndpoints(blocker lookup.Blocker) []endpoint {
 	server := &blob.Server{
-		Blocker: blocker,
+		Blocker:               blocker,
+		OptimisticModeFetcher: s.cfg.OptimisticModeFetcher,
+		FinalizationFetcher:   s.cfg.FinalizationFetcher,
 	}
 
 	const namespace = "blob"

beacon-chain/rpc/eth/blob/BUILD.bazel

@@ -10,12 +10,14 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//api/server/structs:go_default_library",
+        "//beacon-chain/blockchain:go_default_library",
         "//beacon-chain/rpc/core:go_default_library",
         "//beacon-chain/rpc/lookup:go_default_library",
         "//config/fieldparams:go_default_library",
         "//consensus-types/blocks:go_default_library",
         "//monitoring/tracing/trace:go_default_library",
         "//network/httputil:go_default_library",
+        "//runtime/version:go_default_library",
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
     ],

beacon-chain/rpc/eth/blob/handlers.go

@@ -15,6 +15,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v5/consensus-types/blocks"
 	"github.com/prysmaticlabs/prysm/v5/monitoring/tracing/trace"
 	"github.com/prysmaticlabs/prysm/v5/network/httputil"
+	"github.com/prysmaticlabs/prysm/v5/runtime/version"
 )
 
 // Blobs is an HTTP handler for Beacon API getBlobs.
@@ -59,7 +60,30 @@ func (s *Server) Blobs(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	httputil.WriteJson(w, buildSidecarsJsonResponse(verifiedBlobs))
+	blk, err := s.Blocker.Block(ctx, []byte(blockId))
+	if err != nil {
+		httputil.HandleError(w, "Could not fetch block: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+	blkRoot, err := blk.Block().HashTreeRoot()
+	if err != nil {
+		httputil.HandleError(w, "Could not hash block: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+	isOptimistic, err := s.OptimisticModeFetcher.IsOptimisticForRoot(ctx, blkRoot)
+	if err != nil {
+		httputil.HandleError(w, "Could not check if block is optimistic: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	data := buildSidecarsJsonResponse(verifiedBlobs)
+	resp := &structs.SidecarsResponse{
+		Version:             version.String(blk.Version()),
+		Data:                data,
+		ExecutionOptimistic: isOptimistic,
+		Finalized:           s.FinalizationFetcher.IsFinalized(ctx, blkRoot),
+	}
+	httputil.WriteJson(w, resp)
 }
 
 // parseIndices filters out invalid and duplicate blob indices
@@ -92,14 +116,14 @@ loop:
 	return indices, nil
 }
 
-func buildSidecarsJsonResponse(verifiedBlobs []*blocks.VerifiedROBlob) *structs.SidecarsResponse {
+func buildSidecarsJsonResponse(verifiedBlobs []*blocks.VerifiedROBlob) []*structs.Sidecar {
-	resp := &structs.SidecarsResponse{Data: make([]*structs.Sidecar, len(verifiedBlobs))}
+	sidecars := make([]*structs.Sidecar, len(verifiedBlobs))
 	for i, sc := range verifiedBlobs {
 		proofs := make([]string, len(sc.CommitmentInclusionProof))
 		for j := range sc.CommitmentInclusionProof {
 			proofs[j] = hexutil.Encode(sc.CommitmentInclusionProof[j])
 		}
-		resp.Data[i] = &structs.Sidecar{
+		sidecars[i] = &structs.Sidecar{
 			Index:                    strconv.FormatUint(sc.Index, 10),
 			Blob:                     hexutil.Encode(sc.Blob),
 			KzgCommitment:            hexutil.Encode(sc.KzgCommitment),
@@ -108,7 +132,7 @@ func buildSidecarsJsonResponse(verifiedBlobs []*blocks.VerifiedROBlob) *structs.
 			CommitmentInclusionProof: proofs,
 		}
 	}
-	return resp
+	return sidecars
 }
 
 func buildSidecarsSSZResponse(verifiedBlobs []*blocks.VerifiedROBlob) ([]byte, error) {

beacon-chain/rpc/eth/blob/handlers_test.go

@@ -46,16 +46,20 @@ func TestBlobs(t *testing.T) {
 	}
 	blockRoot := blobs[0].BlockRoot()
 
+	mockChainService := &mockChain.ChainService{
+		FinalizedRoots: map[[32]byte]bool{},
+	}
+	s := &Server{
+		OptimisticModeFetcher: mockChainService,
+		FinalizationFetcher:   mockChainService,
+	}
+
 	t.Run("genesis", func(t *testing.T) {
 		u := "http://foo.example/genesis"
 		request := httptest.NewRequest("GET", u, nil)
 		writer := httptest.NewRecorder()
 		writer.Body = &bytes.Buffer{}
-		blocker := &lookup.BeaconDbBlocker{}
+		s.Blocker = &lookup.BeaconDbBlocker{}
-		s := &Server{
-			Blocker: blocker,
-		}
-
 		s.Blobs(writer, request)
 
 		assert.Equal(t, http.StatusBadRequest, writer.Code)
@@ -69,18 +73,14 @@ func TestBlobs(t *testing.T) {
 		request := httptest.NewRequest("GET", u, nil)
 		writer := httptest.NewRecorder()
 		writer.Body = &bytes.Buffer{}
-		blocker := &lookup.BeaconDbBlocker{
+		s.Blocker = &lookup.BeaconDbBlocker{
-			ChainInfoFetcher: &mockChain.ChainService{Root: blockRoot[:]},
+			ChainInfoFetcher: &mockChain.ChainService{Root: blockRoot[:], Block: denebBlock},
 			GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
 				Genesis: time.Now(),
 			},
 			BeaconDB:    db,
 			BlobStorage: bs,
 		}
-		s := &Server{
-			Blocker: blocker,
-		}
-
 		s.Blobs(writer, request)
 
 		assert.Equal(t, http.StatusOK, writer.Code)
@@ -111,118 +111,96 @@ func TestBlobs(t *testing.T) {
 		assert.Equal(t, hexutil.Encode(blobs[3].Blob), sidecar.Blob)
 		assert.Equal(t, hexutil.Encode(blobs[3].KzgCommitment), sidecar.KzgCommitment)
 		assert.Equal(t, hexutil.Encode(blobs[3].KzgProof), sidecar.KzgProof)
+
+		require.Equal(t, "deneb", resp.Version)
+		require.Equal(t, false, resp.ExecutionOptimistic)
+		require.Equal(t, false, resp.Finalized)
 	})
 	t.Run("finalized", func(t *testing.T) {
 		u := "http://foo.example/finalized"
 		request := httptest.NewRequest("GET", u, nil)
 		writer := httptest.NewRecorder()
 		writer.Body = &bytes.Buffer{}
-		blocker := &lookup.BeaconDbBlocker{
+		s.Blocker = &lookup.BeaconDbBlocker{
-			ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: &eth.Checkpoint{Root: blockRoot[:]}},
+			ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: &eth.Checkpoint{Root: blockRoot[:]}, Block: denebBlock},
 			GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
 				Genesis: time.Now(),
 			},
 			BeaconDB:    db,
 			BlobStorage: bs,
 		}
-		s := &Server{
-			Blocker: blocker,
-		}
-
 		s.Blobs(writer, request)
 
 		assert.Equal(t, http.StatusOK, writer.Code)
 		resp := &structs.SidecarsResponse{}
 		require.NoError(t, json.Unmarshal(writer.Body.Bytes(), resp))
 		require.Equal(t, 4, len(resp.Data))
-	})
-	t.Run("justified", func(t *testing.T) {
-		u := "http://foo.example/justified"
-		request := httptest.NewRequest("GET", u, nil)
-		writer := httptest.NewRecorder()
-		writer.Body = &bytes.Buffer{}
-		blocker := &lookup.BeaconDbBlocker{
-			ChainInfoFetcher: &mockChain.ChainService{CurrentJustifiedCheckPoint: &eth.Checkpoint{Root: blockRoot[:]}},
-			GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
-				Genesis: time.Now(),
-			},
-			BeaconDB:    db,
-			BlobStorage: bs,
-		}
-		s := &Server{
-			Blocker: blocker,
-		}
 
-		s.Blobs(writer, request)
+		require.Equal(t, "deneb", resp.Version)
-
+		require.Equal(t, false, resp.ExecutionOptimistic)
-		assert.Equal(t, http.StatusOK, writer.Code)
+		require.Equal(t, false, resp.Finalized)
-		resp := &structs.SidecarsResponse{}
-		require.NoError(t, json.Unmarshal(writer.Body.Bytes(), resp))
-		require.Equal(t, 4, len(resp.Data))
 	})
 	t.Run("root", func(t *testing.T) {
 		u := "http://foo.example/" + hexutil.Encode(blockRoot[:])
 		request := httptest.NewRequest("GET", u, nil)
 		writer := httptest.NewRecorder()
 		writer.Body = &bytes.Buffer{}
-		blocker := &lookup.BeaconDbBlocker{
+		s.Blocker = &lookup.BeaconDbBlocker{
-			BeaconDB: db,
+			ChainInfoFetcher: &mockChain.ChainService{Block: denebBlock},
+			BeaconDB:         db,
 			GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
 				Genesis: time.Now(),
 			},
 			BlobStorage: bs,
 		}
-		s := &Server{
-			Blocker: blocker,
-		}
-
 		s.Blobs(writer, request)
 
 		assert.Equal(t, http.StatusOK, writer.Code)
 		resp := &structs.SidecarsResponse{}
 		require.NoError(t, json.Unmarshal(writer.Body.Bytes(), resp))
 		require.Equal(t, 4, len(resp.Data))
+
+		require.Equal(t, "deneb", resp.Version)
+		require.Equal(t, false, resp.ExecutionOptimistic)
+		require.Equal(t, false, resp.Finalized)
 	})
 	t.Run("slot", func(t *testing.T) {
 		u := "http://foo.example/123"
 		request := httptest.NewRequest("GET", u, nil)
 		writer := httptest.NewRecorder()
 		writer.Body = &bytes.Buffer{}
-		blocker := &lookup.BeaconDbBlocker{
+		s.Blocker = &lookup.BeaconDbBlocker{
-			BeaconDB: db,
+			ChainInfoFetcher: &mockChain.ChainService{Block: denebBlock},
+			BeaconDB:         db,
 			GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
 				Genesis: time.Now(),
 			},
 			BlobStorage: bs,
 		}
-		s := &Server{
-			Blocker: blocker,
-		}
-
 		s.Blobs(writer, request)
 
 		assert.Equal(t, http.StatusOK, writer.Code)
 		resp := &structs.SidecarsResponse{}
 		require.NoError(t, json.Unmarshal(writer.Body.Bytes(), resp))
 		require.Equal(t, 4, len(resp.Data))
+
+		require.Equal(t, "deneb", resp.Version)
+		require.Equal(t, false, resp.ExecutionOptimistic)
+		require.Equal(t, false, resp.Finalized)
 	})
 	t.Run("one blob only", func(t *testing.T) {
 		u := "http://foo.example/123?indices=2"
 		request := httptest.NewRequest("GET", u, nil)
 		writer := httptest.NewRecorder()
 		writer.Body = &bytes.Buffer{}
-		blocker := &lookup.BeaconDbBlocker{
+		s.Blocker = &lookup.BeaconDbBlocker{
-			ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: &eth.Checkpoint{Root: blockRoot[:]}},
+			ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: &eth.Checkpoint{Root: blockRoot[:]}, Block: denebBlock},
 			GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
 				Genesis: time.Now(),
 			},
 			BeaconDB:    db,
 			BlobStorage: bs,
 		}
-		s := &Server{
-			Blocker: blocker,
-		}
-
 		s.Blobs(writer, request)
 
 		assert.Equal(t, http.StatusOK, writer.Code)
@@ -235,45 +213,47 @@ func TestBlobs(t *testing.T) {
 		assert.Equal(t, hexutil.Encode(blobs[2].Blob), sidecar.Blob)
 		assert.Equal(t, hexutil.Encode(blobs[2].KzgCommitment), sidecar.KzgCommitment)
 		assert.Equal(t, hexutil.Encode(blobs[2].KzgProof), sidecar.KzgProof)
+
+		require.Equal(t, "deneb", resp.Version)
+		require.Equal(t, false, resp.ExecutionOptimistic)
+		require.Equal(t, false, resp.Finalized)
 	})
 	t.Run("no blobs returns an empty array", func(t *testing.T) {
 		u := "http://foo.example/123"
 		request := httptest.NewRequest("GET", u, nil)
 		writer := httptest.NewRecorder()
 		writer.Body = &bytes.Buffer{}
-		blocker := &lookup.BeaconDbBlocker{
+		s.Blocker = &lookup.BeaconDbBlocker{
-			ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: &eth.Checkpoint{Root: blockRoot[:]}},
+			ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: &eth.Checkpoint{Root: blockRoot[:]}, Block: denebBlock},
 			GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
 				Genesis: time.Now(),
 			},
 			BeaconDB:    db,
 			BlobStorage: filesystem.NewEphemeralBlobStorage(t), // new ephemeral storage
 		}
-		s := &Server{
-			Blocker: blocker,
-		}
 
 		s.Blobs(writer, request)
 		assert.Equal(t, http.StatusOK, writer.Code)
 		resp := &structs.SidecarsResponse{}
 		require.NoError(t, json.Unmarshal(writer.Body.Bytes(), resp))
 		require.Equal(t, len(resp.Data), 0)
+
+		require.Equal(t, "deneb", resp.Version)
+		require.Equal(t, false, resp.ExecutionOptimistic)
+		require.Equal(t, false, resp.Finalized)
 	})
 	t.Run("outside retention period returns 200 w/ empty list ", func(t *testing.T) {
 		u := "http://foo.example/123"
 		request := httptest.NewRequest("GET", u, nil)
 		writer := httptest.NewRecorder()
 		writer.Body = &bytes.Buffer{}
-		moc := &mockChain.ChainService{FinalizedCheckPoint: &eth.Checkpoint{Root: blockRoot[:]}}
+		moc := &mockChain.ChainService{FinalizedCheckPoint: &eth.Checkpoint{Root: blockRoot[:]}, Block: denebBlock}
-		blocker := &lookup.BeaconDbBlocker{
+		s.Blocker = &lookup.BeaconDbBlocker{
 			ChainInfoFetcher:   moc,
 			GenesisTimeFetcher: moc, // genesis time is set to 0 here, so it results in current epoch being extremely large
 			BeaconDB:           db,
 			BlobStorage:        bs,
 		}
-		s := &Server{
-			Blocker: blocker,
-		}
 
 		s.Blobs(writer, request)
 
@@ -281,6 +261,10 @@ func TestBlobs(t *testing.T) {
 		resp := &structs.SidecarsResponse{}
 		require.NoError(t, json.Unmarshal(writer.Body.Bytes(), resp))
 		require.Equal(t, 0, len(resp.Data))
+
+		require.Equal(t, "deneb", resp.Version)
+		require.Equal(t, false, resp.ExecutionOptimistic)
+		require.Equal(t, false, resp.Finalized)
 	})
 	t.Run("block without commitments returns 200 w/empty list ", func(t *testing.T) {
 		denebBlock, _ := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, 333, 0)
@@ -293,17 +277,14 @@ func TestBlobs(t *testing.T) {
 		request := httptest.NewRequest("GET", u, nil)
 		writer := httptest.NewRecorder()
 		writer.Body = &bytes.Buffer{}
-		blocker := &lookup.BeaconDbBlocker{
+		s.Blocker = &lookup.BeaconDbBlocker{
-			ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: &eth.Checkpoint{Root: blockRoot[:]}},
+			ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: &eth.Checkpoint{Root: blockRoot[:]}, Block: denebBlock},
 			GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
 				Genesis: time.Now(),
 			},
 			BeaconDB:    db,
 			BlobStorage: bs,
 		}
-		s := &Server{
-			Blocker: blocker,
-		}
 
 		s.Blobs(writer, request)
 
@@ -311,16 +292,17 @@ func TestBlobs(t *testing.T) {
 		resp := &structs.SidecarsResponse{}
 		require.NoError(t, json.Unmarshal(writer.Body.Bytes(), resp))
 		require.Equal(t, 0, len(resp.Data))
+
+		require.Equal(t, "deneb", resp.Version)
+		require.Equal(t, false, resp.ExecutionOptimistic)
+		require.Equal(t, false, resp.Finalized)
 	})
 	t.Run("slot before Deneb fork", func(t *testing.T) {
 		u := "http://foo.example/31"
 		request := httptest.NewRequest("GET", u, nil)
 		writer := httptest.NewRecorder()
 		writer.Body = &bytes.Buffer{}
-		blocker := &lookup.BeaconDbBlocker{}
+		s.Blocker = &lookup.BeaconDbBlocker{}
-		s := &Server{
-			Blocker: blocker,
-		}
 
 		s.Blobs(writer, request)
 
@@ -335,11 +317,7 @@ func TestBlobs(t *testing.T) {
 		request := httptest.NewRequest("GET", u, nil)
 		writer := httptest.NewRecorder()
 		writer.Body = &bytes.Buffer{}
-		blocker := &lookup.BeaconDbBlocker{}
+		s.Blocker = &lookup.BeaconDbBlocker{}
-		s := &Server{
-			Blocker: blocker,
-		}
-
 		s.Blobs(writer, request)
 
 		assert.Equal(t, http.StatusBadRequest, writer.Code)
@@ -354,7 +332,7 @@ func TestBlobs(t *testing.T) {
 		request.Header.Add("Accept", "application/octet-stream")
 		writer := httptest.NewRecorder()
 		writer.Body = &bytes.Buffer{}
-		blocker := &lookup.BeaconDbBlocker{
+		s.Blocker = &lookup.BeaconDbBlocker{
 			ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: &eth.Checkpoint{Root: blockRoot[:]}},
 			GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
 				Genesis: time.Now(),
@@ -362,10 +340,8 @@ func TestBlobs(t *testing.T) {
 			BeaconDB:    db,
 			BlobStorage: bs,
 		}
-		s := &Server{
-			Blocker: blocker,
-		}
 		s.Blobs(writer, request)
+
 		assert.Equal(t, http.StatusOK, writer.Code)
 		require.Equal(t, len(writer.Body.Bytes()), fieldparams.BlobSidecarSize) // size of each sidecar
 		// can directly unmarshal to sidecar since there's only 1
@@ -379,7 +355,7 @@ func TestBlobs(t *testing.T) {
 		request.Header.Add("Accept", "application/octet-stream")
 		writer := httptest.NewRecorder()
 		writer.Body = &bytes.Buffer{}
-		blocker := &lookup.BeaconDbBlocker{
+		s.Blocker = &lookup.BeaconDbBlocker{
 			ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: &eth.Checkpoint{Root: blockRoot[:]}},
 			GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
 				Genesis: time.Now(),
@@ -387,10 +363,8 @@ func TestBlobs(t *testing.T) {
 			BeaconDB:    db,
 			BlobStorage: bs,
 		}
-		s := &Server{
-			Blocker: blocker,
-		}
 		s.Blobs(writer, request)
+
 		assert.Equal(t, http.StatusOK, writer.Code)
 		require.Equal(t, len(writer.Body.Bytes()), fieldparams.BlobSidecarSize*4) // size of each sidecar
 	})

beacon-chain/rpc/eth/blob/server.go

@@ -1,9 +1,12 @@
 package blob
 
 import (
+	"github.com/prysmaticlabs/prysm/v5/beacon-chain/blockchain"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/rpc/lookup"
 )
 
 type Server struct {
-	Blocker lookup.Blocker
+	Blocker               lookup.Blocker
+	OptimisticModeFetcher blockchain.OptimisticModeFetcher
+	FinalizationFetcher   blockchain.FinalizationFetcher
 }