Debug data columns API endpoint (#15701)

* initial

* fixing from self review

* changelog

* fixing endpoints and adding test

* removed unneeded test

* self review

* fixing mock columns

* fixing tests

* gofmt

* fixing endpoint

* gaz

* gofmt

* fixing tests

* gofmt

* gaz

* radek comments

* gaz

* fixing formatting

* deduplicating and fixing an old bug, will break into separate PR

* better way for version

* optimizing post merge and fixing tests

* Update beacon-chain/rpc/eth/debug/handlers.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update beacon-chain/rpc/eth/debug/handlers.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update beacon-chain/rpc/lookup/blocker.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update beacon-chain/rpc/lookup/blocker.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* Update beacon-chain/rpc/lookup/blocker.go

Co-authored-by: Radosław Kapka <rkapka@wp.pl>

* adding some of radek's feedback

* reverting and gaz

---------

Co-authored-by: Radosław Kapka <rkapka@wp.pl>
This commit is contained in:
james-prysm
2025-09-19 10:20:43 -05:00
committed by GitHub
parent 9f9401e615
commit d67ee62efa
12 changed files with 945 additions and 22 deletions

View File

@@ -56,3 +56,19 @@ type ForkChoiceNodeExtraData struct {
TimeStamp string `json:"timestamp"`
Target string `json:"target"`
}
type GetDebugDataColumnSidecarsResponse struct {
Version string `json:"version"`
ExecutionOptimistic bool `json:"execution_optimistic"`
Finalized bool `json:"finalized"`
Data []*DataColumnSidecar `json:"data"`
}
type DataColumnSidecar struct {
Index string `json:"index"`
Column []string `json:"column"`
KzgCommitments []string `json:"kzg_commitments"`
KzgProofs []string `json:"kzg_proofs"`
SignedBeaconBlockHeader *SignedBeaconBlockHeader `json:"signed_block_header"`
KzgCommitmentsInclusionProof []string `json:"kzg_commitments_inclusion_proof"`
}

View File

@@ -106,7 +106,7 @@ func (s *Service) endpoints(
}
if enableDebug {
endpoints = append(endpoints, s.debugEndpoints(stater)...)
endpoints = append(endpoints, s.debugEndpoints(stater, blocker)...)
}
return endpoints
@@ -1097,7 +1097,7 @@ func (s *Service) lightClientEndpoints() []endpoint {
}
}
func (s *Service) debugEndpoints(stater lookup.Stater) []endpoint {
func (s *Service) debugEndpoints(stater lookup.Stater, blocker lookup.Blocker) []endpoint {
server := &debug.Server{
BeaconDB: s.cfg.BeaconDB,
HeadFetcher: s.cfg.HeadFetcher,
@@ -1107,6 +1107,8 @@ func (s *Service) debugEndpoints(stater lookup.Stater) []endpoint {
ForkchoiceFetcher: s.cfg.ForkchoiceFetcher,
FinalizationFetcher: s.cfg.FinalizationFetcher,
ChainInfoFetcher: s.cfg.ChainInfoFetcher,
GenesisTimeFetcher: s.cfg.GenesisTimeFetcher,
Blocker: blocker,
}
const namespace = "debug"
@@ -1141,6 +1143,16 @@ func (s *Service) debugEndpoints(stater lookup.Stater) []endpoint {
handler: server.GetForkChoice,
methods: []string{http.MethodGet},
},
{
template: "/eth/v1/debug/beacon/data_column_sidecars/{block_id}",
name: namespace + ".GetDataColumnSidecars",
middleware: []middleware.Middleware{
middleware.AcceptHeaderHandler([]string{api.JsonMediaType, api.OctetStreamMediaType}),
middleware.AcceptEncodingHeaderHandler(),
},
handler: server.DataColumnSidecars,
methods: []string{http.MethodGet},
},
}
}

View File

@@ -83,6 +83,7 @@ func Test_endpoints(t *testing.T) {
"/eth/v2/debug/beacon/states/{state_id}": {http.MethodGet},
"/eth/v2/debug/beacon/heads": {http.MethodGet},
"/eth/v1/debug/fork_choice": {http.MethodGet},
"/eth/v1/debug/beacon/data_column_sidecars/{block_id}": {http.MethodGet},
}
eventsRoutes := map[string][]string{

View File

@@ -13,13 +13,19 @@ go_library(
"//api/server/structs:go_default_library",
"//beacon-chain/blockchain:go_default_library",
"//beacon-chain/db:go_default_library",
"//beacon-chain/rpc/core:go_default_library",
"//beacon-chain/rpc/eth/helpers:go_default_library",
"//beacon-chain/rpc/eth/shared:go_default_library",
"//beacon-chain/rpc/lookup:go_default_library",
"//config/params:go_default_library",
"//consensus-types/blocks:go_default_library",
"//consensus-types/primitives:go_default_library",
"//monitoring/tracing/trace:go_default_library",
"//network/httputil:go_default_library",
"//proto/prysm/v1alpha1:go_default_library",
"//runtime/version:go_default_library",
"@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
"@com_github_pkg_errors//:go_default_library",
],
)
@@ -34,9 +40,13 @@ go_test(
"//beacon-chain/db/testing:go_default_library",
"//beacon-chain/forkchoice/doubly-linked-tree:go_default_library",
"//beacon-chain/forkchoice/types:go_default_library",
"//beacon-chain/rpc/core:go_default_library",
"//beacon-chain/rpc/testutil:go_default_library",
"//config/params:go_default_library",
"//consensus-types/blocks:go_default_library",
"//consensus-types/primitives:go_default_library",
"//encoding/bytesutil:go_default_library",
"//proto/prysm/v1alpha1:go_default_library",
"//runtime/version:go_default_library",
"//testing/assert:go_default_library",
"//testing/require:go_default_library",

View File

@@ -4,19 +4,31 @@ import (
"context"
"encoding/json"
"fmt"
"math"
"net/http"
"net/url"
"strconv"
"strings"
"github.com/OffchainLabs/prysm/v6/api"
"github.com/OffchainLabs/prysm/v6/api/server/structs"
"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/core"
"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/eth/helpers"
"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/eth/shared"
"github.com/OffchainLabs/prysm/v6/config/params"
"github.com/OffchainLabs/prysm/v6/consensus-types/blocks"
"github.com/OffchainLabs/prysm/v6/consensus-types/primitives"
"github.com/OffchainLabs/prysm/v6/monitoring/tracing/trace"
"github.com/OffchainLabs/prysm/v6/network/httputil"
ethpb "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
"github.com/OffchainLabs/prysm/v6/runtime/version"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/pkg/errors"
)
const errMsgStateFromConsensus = "Could not convert consensus state to response"
const (
errMsgStateFromConsensus = "Could not convert consensus state to response"
)
// GetBeaconStateV2 returns the full beacon state for a given state ID.
func (s *Server) GetBeaconStateV2(w http.ResponseWriter, r *http.Request) {
@@ -208,3 +220,181 @@ func (s *Server) GetForkChoice(w http.ResponseWriter, r *http.Request) {
}
httputil.WriteJson(w, resp)
}
// DataColumnSidecars retrieves data column sidecars for a given block id.
func (s *Server) DataColumnSidecars(w http.ResponseWriter, r *http.Request) {
ctx, span := trace.StartSpan(r.Context(), "debug.DataColumnSidecars")
defer span.End()
// Check if we're before Fulu fork - data columns are only available from Fulu onwards
fuluForkEpoch := params.BeaconConfig().FuluForkEpoch
if fuluForkEpoch == math.MaxUint64 {
httputil.HandleError(w, "Data columns are not supported - Fulu fork not configured", http.StatusBadRequest)
return
}
// Check if we're before Fulu fork based on current slot
currentSlot := s.GenesisTimeFetcher.CurrentSlot()
currentEpoch := primitives.Epoch(currentSlot / params.BeaconConfig().SlotsPerEpoch)
if currentEpoch < fuluForkEpoch {
httputil.HandleError(w, "Data columns are not supported - before Fulu fork", http.StatusBadRequest)
return
}
indices, err := parseDataColumnIndices(r.URL)
if err != nil {
httputil.HandleError(w, err.Error(), http.StatusBadRequest)
return
}
segments := strings.Split(r.URL.Path, "/")
blockId := segments[len(segments)-1]
verifiedDataColumns, rpcErr := s.Blocker.DataColumns(ctx, blockId, indices)
if rpcErr != nil {
code := core.ErrorReasonToHTTP(rpcErr.Reason)
switch code {
case http.StatusBadRequest:
httputil.HandleError(w, "Bad request: "+rpcErr.Err.Error(), code)
return
case http.StatusNotFound:
httputil.HandleError(w, "Not found: "+rpcErr.Err.Error(), code)
return
case http.StatusInternalServerError:
httputil.HandleError(w, "Internal server error: "+rpcErr.Err.Error(), code)
return
default:
httputil.HandleError(w, rpcErr.Err.Error(), code)
return
}
}
blk, err := s.Blocker.Block(ctx, []byte(blockId))
if err != nil {
httputil.HandleError(w, "Could not fetch block: "+err.Error(), http.StatusInternalServerError)
return
}
if blk == nil {
httputil.HandleError(w, "Block not found", http.StatusNotFound)
return
}
if httputil.RespondWithSsz(r) {
sszResp, err := buildDataColumnSidecarsSSZResponse(verifiedDataColumns)
if err != nil {
httputil.HandleError(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set(api.VersionHeader, version.String(blk.Version()))
httputil.WriteSsz(w, sszResp)
return
}
blkRoot, err := blk.Block().HashTreeRoot()
if err != nil {
httputil.HandleError(w, "Could not hash block: "+err.Error(), http.StatusInternalServerError)
return
}
isOptimistic, err := s.OptimisticModeFetcher.IsOptimisticForRoot(ctx, blkRoot)
if err != nil {
httputil.HandleError(w, "Could not check if block is optimistic: "+err.Error(), http.StatusInternalServerError)
return
}
data := buildDataColumnSidecarsJsonResponse(verifiedDataColumns)
resp := &structs.GetDebugDataColumnSidecarsResponse{
Version: version.String(blk.Version()),
Data: data,
ExecutionOptimistic: isOptimistic,
Finalized: s.FinalizationFetcher.IsFinalized(ctx, blkRoot),
}
w.Header().Set(api.VersionHeader, version.String(blk.Version()))
httputil.WriteJson(w, resp)
}
// parseDataColumnIndices filters out invalid and duplicate data column indices
func parseDataColumnIndices(url *url.URL) ([]int, error) {
numberOfColumns := params.BeaconConfig().NumberOfColumns
rawIndices := url.Query()["indices"]
indices := make([]int, 0, numberOfColumns)
invalidIndices := make([]string, 0)
loop:
for _, raw := range rawIndices {
ix, err := strconv.Atoi(raw)
if err != nil {
invalidIndices = append(invalidIndices, raw)
continue
}
if !(0 <= ix && uint64(ix) < numberOfColumns) {
invalidIndices = append(invalidIndices, raw)
continue
}
for i := range indices {
if ix == indices[i] {
continue loop
}
}
indices = append(indices, ix)
}
if len(invalidIndices) > 0 {
return nil, fmt.Errorf("requested data column indices %v are invalid", invalidIndices)
}
return indices, nil
}
func buildDataColumnSidecarsJsonResponse(verifiedDataColumns []blocks.VerifiedRODataColumn) []*structs.DataColumnSidecar {
sidecars := make([]*structs.DataColumnSidecar, len(verifiedDataColumns))
for i, dc := range verifiedDataColumns {
column := make([]string, len(dc.Column))
for j, cell := range dc.Column {
column[j] = hexutil.Encode(cell)
}
kzgCommitments := make([]string, len(dc.KzgCommitments))
for j, commitment := range dc.KzgCommitments {
kzgCommitments[j] = hexutil.Encode(commitment)
}
kzgProofs := make([]string, len(dc.KzgProofs))
for j, proof := range dc.KzgProofs {
kzgProofs[j] = hexutil.Encode(proof)
}
kzgCommitmentsInclusionProof := make([]string, len(dc.KzgCommitmentsInclusionProof))
for j, proof := range dc.KzgCommitmentsInclusionProof {
kzgCommitmentsInclusionProof[j] = hexutil.Encode(proof)
}
sidecars[i] = &structs.DataColumnSidecar{
Index: strconv.FormatUint(dc.Index, 10),
Column: column,
KzgCommitments: kzgCommitments,
KzgProofs: kzgProofs,
SignedBeaconBlockHeader: structs.SignedBeaconBlockHeaderFromConsensus(dc.SignedBlockHeader),
KzgCommitmentsInclusionProof: kzgCommitmentsInclusionProof,
}
}
return sidecars
}
// buildDataColumnSidecarsSSZResponse builds SSZ response for data column sidecars
func buildDataColumnSidecarsSSZResponse(verifiedDataColumns []blocks.VerifiedRODataColumn) ([]byte, error) {
if len(verifiedDataColumns) == 0 {
return []byte{}, nil
}
// Pre-allocate buffer for all sidecars using the known SSZ size
sizePerSidecar := (&ethpb.DataColumnSidecar{}).SizeSSZ()
ssz := make([]byte, 0, sizePerSidecar*len(verifiedDataColumns))
// Marshal and append each sidecar
for i, sidecar := range verifiedDataColumns {
sszrep, err := sidecar.MarshalSSZ()
if err != nil {
return nil, errors.Wrapf(err, "failed to marshal data column sidecar at index %d", i)
}
ssz = append(ssz, sszrep...)
}
return ssz, nil
}

View File

@@ -2,9 +2,13 @@ package debug
import (
"bytes"
"context"
"encoding/json"
"errors"
"math"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"github.com/OffchainLabs/prysm/v6/api"
@@ -13,9 +17,13 @@ import (
dbtest "github.com/OffchainLabs/prysm/v6/beacon-chain/db/testing"
doublylinkedtree "github.com/OffchainLabs/prysm/v6/beacon-chain/forkchoice/doubly-linked-tree"
forkchoicetypes "github.com/OffchainLabs/prysm/v6/beacon-chain/forkchoice/types"
"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/core"
"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/testutil"
"github.com/OffchainLabs/prysm/v6/config/params"
"github.com/OffchainLabs/prysm/v6/consensus-types/blocks"
"github.com/OffchainLabs/prysm/v6/consensus-types/primitives"
"github.com/OffchainLabs/prysm/v6/encoding/bytesutil"
ethpb "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
"github.com/OffchainLabs/prysm/v6/runtime/version"
"github.com/OffchainLabs/prysm/v6/testing/assert"
"github.com/OffchainLabs/prysm/v6/testing/require"
@@ -515,3 +523,285 @@ func TestGetForkChoice(t *testing.T) {
require.NoError(t, json.Unmarshal(writer.Body.Bytes(), resp))
require.Equal(t, "2", resp.FinalizedCheckpoint.Epoch)
}
func TestDataColumnSidecars(t *testing.T) {
t.Run("Fulu fork not configured", func(t *testing.T) {
// Save the original config
originalConfig := params.BeaconConfig()
defer func() { params.OverrideBeaconConfig(originalConfig) }()
// Set Fulu fork epoch to MaxUint64 (unconfigured)
config := params.BeaconConfig().Copy()
config.FuluForkEpoch = math.MaxUint64
params.OverrideBeaconConfig(config)
chainService := &blockchainmock.ChainService{}
// Create a mock blocker to avoid nil pointer
mockBlocker := &testutil.MockBlocker{}
s := &Server{
GenesisTimeFetcher: chainService,
Blocker: mockBlocker,
}
request := httptest.NewRequest(http.MethodGet, "http://example.com/eth/v1/debug/beacon/data_column_sidecars/head", nil)
writer := httptest.NewRecorder()
writer.Body = &bytes.Buffer{}
s.DataColumnSidecars(writer, request)
require.Equal(t, http.StatusBadRequest, writer.Code)
assert.StringContains(t, "Data columns are not supported - Fulu fork not configured", writer.Body.String())
})
t.Run("Before Fulu fork", func(t *testing.T) {
// Save the original config
originalConfig := params.BeaconConfig()
defer func() { params.OverrideBeaconConfig(originalConfig) }()
// Set Fulu fork epoch to 100
config := params.BeaconConfig().Copy()
config.FuluForkEpoch = 100
params.OverrideBeaconConfig(config)
chainService := &blockchainmock.ChainService{}
currentSlot := primitives.Slot(0) // Current slot 0 (epoch 0, before epoch 100)
chainService.Slot = &currentSlot
// Create a mock blocker to avoid nil pointer
mockBlocker := &testutil.MockBlocker{
DataColumnsFunc: func(ctx context.Context, id string, indices []int) ([]blocks.VerifiedRODataColumn, *core.RpcError) {
return nil, &core.RpcError{Err: errors.New("before Fulu fork"), Reason: core.BadRequest}
},
}
s := &Server{
GenesisTimeFetcher: chainService,
Blocker: mockBlocker,
}
request := httptest.NewRequest(http.MethodGet, "http://example.com/eth/v1/debug/beacon/data_column_sidecars/head", nil)
writer := httptest.NewRecorder()
writer.Body = &bytes.Buffer{}
s.DataColumnSidecars(writer, request)
require.Equal(t, http.StatusBadRequest, writer.Code)
assert.StringContains(t, "Data columns are not supported - before Fulu fork", writer.Body.String())
})
t.Run("Invalid indices", func(t *testing.T) {
// Save the original config
originalConfig := params.BeaconConfig()
defer func() { params.OverrideBeaconConfig(originalConfig) }()
// Set Fulu fork epoch to 0 (already activated)
config := params.BeaconConfig().Copy()
config.FuluForkEpoch = 0
params.OverrideBeaconConfig(config)
chainService := &blockchainmock.ChainService{}
currentSlot := primitives.Slot(0) // Current slot 0 (epoch 0, at fork)
chainService.Slot = &currentSlot
// Create a mock blocker to avoid nil pointer
mockBlocker := &testutil.MockBlocker{
DataColumnsFunc: func(ctx context.Context, id string, indices []int) ([]blocks.VerifiedRODataColumn, *core.RpcError) {
return nil, &core.RpcError{Err: errors.New("invalid index"), Reason: core.BadRequest}
},
}
s := &Server{
GenesisTimeFetcher: chainService,
Blocker: mockBlocker,
}
// Test with invalid index (out of range)
request := httptest.NewRequest(http.MethodGet, "http://example.com/eth/v1/debug/beacon/data_column_sidecars/head?indices=9999", nil)
writer := httptest.NewRecorder()
writer.Body = &bytes.Buffer{}
s.DataColumnSidecars(writer, request)
require.Equal(t, http.StatusBadRequest, writer.Code)
assert.StringContains(t, "requested data column indices [9999] are invalid", writer.Body.String())
})
t.Run("Block not found", func(t *testing.T) {
// Save the original config
originalConfig := params.BeaconConfig()
defer func() { params.OverrideBeaconConfig(originalConfig) }()
// Set Fulu fork epoch to 0 (already activated)
config := params.BeaconConfig().Copy()
config.FuluForkEpoch = 0
params.OverrideBeaconConfig(config)
chainService := &blockchainmock.ChainService{}
currentSlot := primitives.Slot(0) // Current slot 0
chainService.Slot = &currentSlot
// Create a mock blocker that returns block not found
mockBlocker := &testutil.MockBlocker{
DataColumnsFunc: func(ctx context.Context, id string, indices []int) ([]blocks.VerifiedRODataColumn, *core.RpcError) {
return nil, &core.RpcError{Err: errors.New("block not found"), Reason: core.NotFound}
},
BlockToReturn: nil, // Block not found
}
s := &Server{
GenesisTimeFetcher: chainService,
Blocker: mockBlocker,
}
request := httptest.NewRequest(http.MethodGet, "http://example.com/eth/v1/debug/beacon/data_column_sidecars/head", nil)
writer := httptest.NewRecorder()
writer.Body = &bytes.Buffer{}
s.DataColumnSidecars(writer, request)
require.Equal(t, http.StatusNotFound, writer.Code)
})
t.Run("Empty data columns", func(t *testing.T) {
// Save the original config
originalConfig := params.BeaconConfig()
defer func() { params.OverrideBeaconConfig(originalConfig) }()
// Set Fulu fork epoch to 0
config := params.BeaconConfig().Copy()
config.FuluForkEpoch = 0
params.OverrideBeaconConfig(config)
// Create a simple test block
signedTestBlock := util.NewBeaconBlock()
roBlock, err := blocks.NewSignedBeaconBlock(signedTestBlock)
require.NoError(t, err)
chainService := &blockchainmock.ChainService{}
currentSlot := primitives.Slot(0) // Current slot 0
chainService.Slot = &currentSlot
chainService.OptimisticRoots = make(map[[32]byte]bool)
chainService.FinalizedRoots = make(map[[32]byte]bool)
mockBlocker := &testutil.MockBlocker{
DataColumnsFunc: func(ctx context.Context, id string, indices []int) ([]blocks.VerifiedRODataColumn, *core.RpcError) {
return []blocks.VerifiedRODataColumn{}, nil // Empty data columns
},
BlockToReturn: roBlock,
}
s := &Server{
GenesisTimeFetcher: chainService,
OptimisticModeFetcher: chainService,
FinalizationFetcher: chainService,
Blocker: mockBlocker,
}
request := httptest.NewRequest(http.MethodGet, "http://example.com/eth/v1/debug/beacon/data_column_sidecars/head", nil)
writer := httptest.NewRecorder()
writer.Body = &bytes.Buffer{}
s.DataColumnSidecars(writer, request)
require.Equal(t, http.StatusOK, writer.Code)
resp := &structs.GetDebugDataColumnSidecarsResponse{}
require.NoError(t, json.Unmarshal(writer.Body.Bytes(), resp))
require.Equal(t, 0, len(resp.Data))
})
}
func TestParseDataColumnIndices(t *testing.T) {
// Save the original config
originalConfig := params.BeaconConfig()
defer func() { params.OverrideBeaconConfig(originalConfig) }()
// Set NumberOfColumns to 128 for testing
config := params.BeaconConfig().Copy()
config.NumberOfColumns = 128
params.OverrideBeaconConfig(config)
tests := []struct {
name string
queryParams map[string][]string
expected []int
expectError bool
}{
{
name: "no indices",
queryParams: map[string][]string{},
expected: []int{},
expectError: false,
},
{
name: "valid indices",
queryParams: map[string][]string{"indices": {"0", "1", "127"}},
expected: []int{0, 1, 127},
expectError: false,
},
{
name: "duplicate indices",
queryParams: map[string][]string{"indices": {"0", "1", "0"}},
expected: []int{0, 1},
expectError: false,
},
{
name: "invalid string index",
queryParams: map[string][]string{"indices": {"abc"}},
expected: nil,
expectError: true,
},
{
name: "negative index",
queryParams: map[string][]string{"indices": {"-1"}},
expected: nil,
expectError: true,
},
{
name: "index too large",
queryParams: map[string][]string{"indices": {"128"}}, // 128 >= NumberOfColumns (128)
expected: nil,
expectError: true,
},
{
name: "mixed valid and invalid",
queryParams: map[string][]string{"indices": {"0", "abc", "1"}},
expected: nil,
expectError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
u, err := url.Parse("http://example.com/test")
require.NoError(t, err)
q := u.Query()
for key, values := range tt.queryParams {
for _, value := range values {
q.Add(key, value)
}
}
u.RawQuery = q.Encode()
result, err := parseDataColumnIndices(u)
if tt.expectError {
assert.NotNil(t, err)
} else {
require.NoError(t, err)
assert.DeepEqual(t, tt.expected, result)
}
})
}
}
func TestBuildDataColumnSidecarsSSZResponse(t *testing.T) {
t.Run("empty data columns", func(t *testing.T) {
result, err := buildDataColumnSidecarsSSZResponse([]blocks.VerifiedRODataColumn{})
require.NoError(t, err)
require.DeepEqual(t, []byte{}, result)
})
t.Run("get SSZ size", func(t *testing.T) {
size := (&ethpb.DataColumnSidecar{}).SizeSSZ()
assert.Equal(t, true, size > 0)
})
}

View File

@@ -20,4 +20,6 @@ type Server struct {
ForkchoiceFetcher blockchain.ForkchoiceFetcher
FinalizationFetcher blockchain.FinalizationFetcher
ChainInfoFetcher blockchain.ChainInfoFetcher
GenesisTimeFetcher blockchain.TimeFetcher
Blocker lookup.Blocker
}

View File

@@ -25,7 +25,6 @@ go_library(
"//consensus-types/primitives:go_default_library",
"//encoding/bytesutil:go_default_library",
"//monitoring/tracing/trace:go_default_library",
"//runtime/version:go_default_library",
"//time/slots:go_default_library",
"@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
"@com_github_pkg_errors//:go_default_library",

View File

@@ -19,7 +19,6 @@ import (
"github.com/OffchainLabs/prysm/v6/consensus-types/interfaces"
"github.com/OffchainLabs/prysm/v6/consensus-types/primitives"
"github.com/OffchainLabs/prysm/v6/encoding/bytesutil"
"github.com/OffchainLabs/prysm/v6/runtime/version"
"github.com/OffchainLabs/prysm/v6/time/slots"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/pkg/errors"
@@ -62,6 +61,7 @@ func (e BlockIdParseError) Error() string {
type Blocker interface {
Block(ctx context.Context, id []byte) (interfaces.ReadOnlySignedBeaconBlock, error)
Blobs(ctx context.Context, id string, opts ...options.BlobsOption) ([]*blocks.VerifiedROBlob, *core.RpcError)
DataColumns(ctx context.Context, id string, indices []int) ([]blocks.VerifiedRODataColumn, *core.RpcError)
}
// BeaconDbBlocker is an implementation of Blocker. It retrieves blocks from the beacon chain database.
@@ -267,13 +267,9 @@ func (p *BeaconDbBlocker) Blobs(ctx context.Context, id string, opts ...options.
return nil, &core.RpcError{Err: err, Reason: core.ErrorReason(reason)}
}
// Validate fork epoch for Deneb (blobs)
if roSignedBlock != nil {
slot := roSignedBlock.Block().Slot()
if slots.ToEpoch(slot) < params.BeaconConfig().DenebForkEpoch {
forkName := version.String(slots.ToForkVersion(slot))
return nil, &core.RpcError{Err: fmt.Errorf("not supported before %s fork", forkName), Reason: core.BadRequest}
}
return nil, &core.RpcError{Err: errors.New("blobs are not supported before Deneb fork"), Reason: core.BadRequest}
}
roBlock := roSignedBlock.Block()
@@ -489,3 +485,93 @@ func (p *BeaconDbBlocker) neededDataColumnSidecars(root [fieldparams.RootLength]
return verifiedRoSidecars, nil
}
// DataColumns returns the data column sidecars for a given block id identifier and column indices. The identifier can be one of:
// - "head" (canonical head in node's view)
// - "genesis"
// - "finalized"
// - "justified"
// - <slot>
// - <hex encoded block root with '0x' prefix>
// - <block root>
//
// cases:
// - no block, 404
// - block exists, before Fulu fork, 400 (data columns are not supported before Fulu fork)
func (p *BeaconDbBlocker) DataColumns(ctx context.Context, id string, indices []int) ([]blocks.VerifiedRODataColumn, *core.RpcError) {
// Check for genesis block first (not supported for data columns)
if id == "genesis" {
return nil, &core.RpcError{Err: errors.New("data columns are not supported for Phase 0 fork"), Reason: core.BadRequest}
}
// Resolve block ID to root and block
root, roSignedBlock, err := p.resolveBlockID(ctx, id)
if err != nil {
var blockNotFound *BlockNotFoundError
var blockIdParseErr *BlockIdParseError
reason := core.Internal // Default to Internal for unexpected errors
if errors.As(err, &blockNotFound) {
reason = core.NotFound
} else if errors.As(err, &blockIdParseErr) {
reason = core.BadRequest
}
return nil, &core.RpcError{Err: err, Reason: core.ErrorReason(reason)}
}
slot := roSignedBlock.Block().Slot()
fuluForkEpoch := params.BeaconConfig().FuluForkEpoch
fuluForkSlot, err := slots.EpochStart(fuluForkEpoch)
if err != nil {
return nil, &core.RpcError{Err: errors.Wrap(err, "could not calculate Fulu start slot"), Reason: core.Internal}
}
if slot < fuluForkSlot {
return nil, &core.RpcError{Err: errors.New("data columns are not supported before Fulu fork"), Reason: core.BadRequest}
}
roBlock := roSignedBlock.Block()
commitments, err := roBlock.Body().BlobKzgCommitments()
if err != nil {
return nil, &core.RpcError{Err: errors.Wrapf(err, "failed to retrieve kzg commitments from block %#x", root), Reason: core.Internal}
}
// If there are no commitments return 200 w/ empty list
if len(commitments) == 0 {
return make([]blocks.VerifiedRODataColumn, 0), nil
}
// Get column indices to retrieve
columnIndices := make([]uint64, 0)
if len(indices) == 0 {
// If no indices specified, return all columns this node is custodying
summary := p.DataColumnStorage.Summary(root)
stored := summary.Stored()
for index := range stored {
columnIndices = append(columnIndices, index)
}
} else {
// Validate and convert indices
numberOfColumns := params.BeaconConfig().NumberOfColumns
for _, index := range indices {
if index < 0 || uint64(index) >= numberOfColumns {
return nil, &core.RpcError{
Err: fmt.Errorf("requested index %d is outside valid range [0, %d)", index, numberOfColumns),
Reason: core.BadRequest,
}
}
columnIndices = append(columnIndices, uint64(index))
}
}
// Retrieve data column sidecars from storage
verifiedRoDataColumns, err := p.DataColumnStorage.Get(root, columnIndices)
if err != nil {
return nil, &core.RpcError{
Err: errors.Wrapf(err, "could not retrieve data columns for block root %#x", root),
Reason: core.Internal,
}
}
return verifiedRoDataColumns, nil
}

View File

@@ -329,9 +329,9 @@ func TestGetBlob(t *testing.T) {
for _, blob := range fuluBlobSidecars {
var kzgBlob kzg.Blob
copy(kzgBlob[:], blob.Blob)
cellsAndProogs, err := kzg.ComputeCellsAndKZGProofs(&kzgBlob)
cellsAndProofs, err := kzg.ComputeCellsAndKZGProofs(&kzgBlob)
require.NoError(t, err)
cellsAndProofsList = append(cellsAndProofsList, cellsAndProogs)
cellsAndProofsList = append(cellsAndProofsList, cellsAndProofs)
}
roDataColumnSidecars, err := peerdas.DataColumnSidecars(cellsAndProofsList, peerdas.PopulateFromBlock(fuluBlock))
@@ -794,3 +794,300 @@ func TestBlobs_CommitmentOrdering(t *testing.T) {
require.StringContains(t, "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", rpcErr.Err.Error())
})
}
func TestGetDataColumns(t *testing.T) {
const (
blobCount = 4
fuluForkEpoch = 2
)
setupFulu := func(t *testing.T) {
params.SetupTestConfigCleanup(t)
cfg := params.BeaconConfig().Copy()
cfg.DenebForkEpoch = 1
cfg.FuluForkEpoch = fuluForkEpoch
params.OverrideBeaconConfig(cfg)
}
setupPreFulu := func(t *testing.T) {
params.SetupTestConfigCleanup(t)
cfg := params.BeaconConfig().Copy()
cfg.DenebForkEpoch = 1
cfg.FuluForkEpoch = 1000 // Set to a high epoch to ensure we're before Fulu
params.OverrideBeaconConfig(cfg)
}
ctx := t.Context()
db := testDB.SetupDB(t)
// Start the trusted setup.
err := kzg.Start()
require.NoError(t, err)
// Create Fulu block and convert blob sidecars to data column sidecars.
fuluForkSlot := fuluForkEpoch * params.BeaconConfig().SlotsPerEpoch
fuluBlock, fuluBlobSidecars := util.GenerateTestElectraBlockWithSidecar(t, [fieldparams.RootLength]byte{}, fuluForkSlot, blobCount)
fuluBlockRoot := fuluBlock.Root()
cellsAndProofsList := make([]kzg.CellsAndProofs, 0, len(fuluBlobSidecars))
for _, blob := range fuluBlobSidecars {
var kzgBlob kzg.Blob
copy(kzgBlob[:], blob.Blob)
cellsAndProofs, err := kzg.ComputeCellsAndKZGProofs(&kzgBlob)
require.NoError(t, err)
cellsAndProofsList = append(cellsAndProofsList, cellsAndProofs)
}
roDataColumnSidecars, err := peerdas.DataColumnSidecars(cellsAndProofsList, peerdas.PopulateFromBlock(fuluBlock))
require.NoError(t, err)
verifiedRoDataColumnSidecars := make([]blocks.VerifiedRODataColumn, 0, len(roDataColumnSidecars))
for _, roDataColumn := range roDataColumnSidecars {
verifiedRoDataColumn := blocks.NewVerifiedRODataColumn(roDataColumn)
verifiedRoDataColumnSidecars = append(verifiedRoDataColumnSidecars, verifiedRoDataColumn)
}
err = db.SaveBlock(t.Context(), fuluBlock)
require.NoError(t, err)
_, dataColumnStorage := filesystem.NewEphemeralDataColumnStorageAndFs(t)
err = dataColumnStorage.Save(verifiedRoDataColumnSidecars)
require.NoError(t, err)
t.Run("pre-fulu fork", func(t *testing.T) {
setupPreFulu(t)
// Create a block at slot 123 (before Fulu fork since FuluForkEpoch is set to MaxUint64)
preFuluBlock := util.NewBeaconBlock()
preFuluBlock.Block.Slot = 123
util.SaveBlock(t, ctx, db, preFuluBlock)
blocker := &BeaconDbBlocker{
GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
Genesis: time.Now(),
},
ChainInfoFetcher: &mockChain.ChainService{},
BeaconDB: db,
}
_, rpcErr := blocker.DataColumns(ctx, "123", nil)
require.NotNil(t, rpcErr)
require.Equal(t, core.ErrorReason(core.BadRequest), rpcErr.Reason)
require.StringContains(t, "not supported before Fulu fork", rpcErr.Err.Error())
})
t.Run("genesis", func(t *testing.T) {
setupFulu(t)
blocker := &BeaconDbBlocker{
GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
Genesis: time.Now(),
},
ChainInfoFetcher: &mockChain.ChainService{},
}
_, rpcErr := blocker.DataColumns(ctx, "genesis", nil)
require.NotNil(t, rpcErr)
require.Equal(t, http.StatusBadRequest, core.ErrorReasonToHTTP(rpcErr.Reason))
require.StringContains(t, "not supported for Phase 0 fork", rpcErr.Err.Error())
})
t.Run("head", func(t *testing.T) {
setupFulu(t)
blocker := &BeaconDbBlocker{
ChainInfoFetcher: &mockChain.ChainService{
Root: fuluBlockRoot[:],
Block: fuluBlock,
},
GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
Genesis: time.Now(),
},
BeaconDB: db,
DataColumnStorage: dataColumnStorage,
}
retrievedDataColumns, rpcErr := blocker.DataColumns(ctx, "head", nil)
require.IsNil(t, rpcErr)
require.Equal(t, len(verifiedRoDataColumnSidecars), len(retrievedDataColumns))
// Create a map of expected indices for easier verification
expectedIndices := make(map[uint64]bool)
for _, expected := range verifiedRoDataColumnSidecars {
expectedIndices[expected.RODataColumn.DataColumnSidecar.Index] = true
}
// Verify we got data columns with the expected indices
for _, actual := range retrievedDataColumns {
require.Equal(t, true, expectedIndices[actual.RODataColumn.DataColumnSidecar.Index])
}
})
t.Run("finalized", func(t *testing.T) {
setupFulu(t)
blocker := &BeaconDbBlocker{
ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: &ethpb.Checkpoint{Root: fuluBlockRoot[:]}},
GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
Genesis: time.Now(),
},
BeaconDB: db,
DataColumnStorage: dataColumnStorage,
}
retrievedDataColumns, rpcErr := blocker.DataColumns(ctx, "finalized", nil)
require.IsNil(t, rpcErr)
require.Equal(t, len(verifiedRoDataColumnSidecars), len(retrievedDataColumns))
})
t.Run("justified", func(t *testing.T) {
setupFulu(t)
blocker := &BeaconDbBlocker{
ChainInfoFetcher: &mockChain.ChainService{CurrentJustifiedCheckPoint: &ethpb.Checkpoint{Root: fuluBlockRoot[:]}},
GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
Genesis: time.Now(),
},
BeaconDB: db,
DataColumnStorage: dataColumnStorage,
}
retrievedDataColumns, rpcErr := blocker.DataColumns(ctx, "justified", nil)
require.IsNil(t, rpcErr)
require.Equal(t, len(verifiedRoDataColumnSidecars), len(retrievedDataColumns))
})
t.Run("root", func(t *testing.T) {
setupFulu(t)
blocker := &BeaconDbBlocker{
GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
Genesis: time.Now(),
},
BeaconDB: db,
DataColumnStorage: dataColumnStorage,
}
retrievedDataColumns, rpcErr := blocker.DataColumns(ctx, hexutil.Encode(fuluBlockRoot[:]), nil)
require.IsNil(t, rpcErr)
require.Equal(t, len(verifiedRoDataColumnSidecars), len(retrievedDataColumns))
})
t.Run("slot", func(t *testing.T) {
setupFulu(t)
blocker := &BeaconDbBlocker{
GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
Genesis: time.Now(),
},
ChainInfoFetcher: &mockChain.ChainService{},
BeaconDB: db,
DataColumnStorage: dataColumnStorage,
}
slotStr := fmt.Sprintf("%d", fuluForkSlot)
retrievedDataColumns, rpcErr := blocker.DataColumns(ctx, slotStr, nil)
require.IsNil(t, rpcErr)
require.Equal(t, len(verifiedRoDataColumnSidecars), len(retrievedDataColumns))
})
t.Run("specific indices", func(t *testing.T) {
setupFulu(t)
blocker := &BeaconDbBlocker{
GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
Genesis: time.Now(),
},
BeaconDB: db,
DataColumnStorage: dataColumnStorage,
}
// Request specific indices (first 3 data columns)
indices := []int{0, 1, 2}
retrievedDataColumns, rpcErr := blocker.DataColumns(ctx, hexutil.Encode(fuluBlockRoot[:]), indices)
require.IsNil(t, rpcErr)
require.Equal(t, 3, len(retrievedDataColumns))
for i, dataColumn := range retrievedDataColumns {
require.Equal(t, uint64(indices[i]), dataColumn.RODataColumn.DataColumnSidecar.Index)
}
})
t.Run("no data columns returns empty array", func(t *testing.T) {
setupFulu(t)
_, emptyDataColumnStorage := filesystem.NewEphemeralDataColumnStorageAndFs(t)
blocker := &BeaconDbBlocker{
GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
Genesis: time.Now(),
},
BeaconDB: db,
DataColumnStorage: emptyDataColumnStorage,
}
retrievedDataColumns, rpcErr := blocker.DataColumns(ctx, hexutil.Encode(fuluBlockRoot[:]), nil)
require.IsNil(t, rpcErr)
require.Equal(t, 0, len(retrievedDataColumns))
})
t.Run("index too big", func(t *testing.T) {
setupFulu(t)
blocker := &BeaconDbBlocker{
GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
Genesis: time.Now(),
},
BeaconDB: db,
DataColumnStorage: dataColumnStorage,
}
_, rpcErr := blocker.DataColumns(ctx, hexutil.Encode(fuluBlockRoot[:]), []int{0, math.MaxInt})
require.NotNil(t, rpcErr)
require.Equal(t, core.ErrorReason(core.BadRequest), rpcErr.Reason)
})
t.Run("outside retention period", func(t *testing.T) {
setupFulu(t)
// Create a data column storage with very short retention period
shortRetentionStorage, err := filesystem.NewDataColumnStorage(ctx,
filesystem.WithDataColumnBasePath(t.TempDir()),
filesystem.WithDataColumnRetentionEpochs(1), // Only 1 epoch retention
)
require.NoError(t, err)
// Mock genesis time to make current slot much later than the block slot
// This simulates being outside retention period
genesisTime := time.Now().Add(-time.Duration(fuluForkSlot+1000) * time.Duration(params.BeaconConfig().SecondsPerSlot) * time.Second)
blocker := &BeaconDbBlocker{
GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
Genesis: genesisTime,
},
BeaconDB: db,
DataColumnStorage: shortRetentionStorage,
}
// Since the block is outside retention period, should return empty array
retrievedDataColumns, rpcErr := blocker.DataColumns(ctx, hexutil.Encode(fuluBlockRoot[:]), nil)
require.IsNil(t, rpcErr)
require.Equal(t, 0, len(retrievedDataColumns))
})
t.Run("block not found", func(t *testing.T) {
setupFulu(t)
blocker := &BeaconDbBlocker{
GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{
Genesis: time.Now(),
},
BeaconDB: db,
DataColumnStorage: dataColumnStorage,
}
nonExistentRoot := bytesutil.PadTo([]byte("nonexistent"), 32)
_, rpcErr := blocker.DataColumns(ctx, hexutil.Encode(nonExistentRoot), nil)
require.NotNil(t, rpcErr)
require.Equal(t, core.ErrorReason(core.NotFound), rpcErr.Reason)
})
}

View File

@@ -18,6 +18,9 @@ type MockBlocker struct {
ErrorToReturn error
SlotBlockMap map[primitives.Slot]interfaces.ReadOnlySignedBeaconBlock
RootBlockMap map[[32]byte]interfaces.ReadOnlySignedBeaconBlock
DataColumnsFunc func(ctx context.Context, id string, indices []int) ([]blocks.VerifiedRODataColumn, *core.RpcError)
DataColumnsToReturn []blocks.VerifiedRODataColumn
DataColumnsErrorToReturn *core.RpcError
}
// Block --
@@ -40,3 +43,17 @@ func (m *MockBlocker) Block(_ context.Context, b []byte) (interfaces.ReadOnlySig
func (*MockBlocker) Blobs(_ context.Context, _ string, _ ...options.BlobsOption) ([]*blocks.VerifiedROBlob, *core.RpcError) {
return nil, &core.RpcError{}
}
// DataColumns --
func (m *MockBlocker) DataColumns(ctx context.Context, id string, indices []int) ([]blocks.VerifiedRODataColumn, *core.RpcError) {
if m.DataColumnsFunc != nil {
return m.DataColumnsFunc(ctx, id, indices)
}
if m.DataColumnsErrorToReturn != nil {
return nil, m.DataColumnsErrorToReturn
}
if m.DataColumnsToReturn != nil {
return m.DataColumnsToReturn, nil
}
return nil, &core.RpcError{}
}

View File

@@ -0,0 +1,3 @@
### Added
- Adding `/eth/v1/debug/beacon/data_column_sidecars/{block_id}` endpoint.