beacon-api: add data column sidecars debug endpoint

api/server/structs/BUILD.bazel

@@ -14,6 +14,7 @@ go_library(
         "endpoints_beacon.go",
         "endpoints_blob.go",
         "endpoints_builder.go",
+        "endpoints_column_sidecar.go",
         "endpoints_config.go",
         "endpoints_debug.go",
         "endpoints_events.go",

api/server/structs/endpoints_column_sidecar.go

@@ -0,0 +1,19 @@
+package structs
+
+// DataColumnSidecar represents a sidecar containing data columns for a specific index.
+type DataColumnSidecar struct {
+	Index                        string                   `json:"index"`
+	Column                       []string                 `json:"column"`
+	KZGCommitments               []string                 `json:"kzg_commitments"`
+	KZGProofs                    []string                 `json:"kzg_proofs"`
+	SignedBlockHeader            *SignedBeaconBlockHeader `json:"signed_block_header"`
+	KZGCommitmentsInclusionProof []string                 `json:"kzg_commitments_inclusion_proof"`
+}
+
+// DataColumnSidecarResponse represents the response structure for data column sidecars for beacon api endpoints.
+type DataColumnSidecarResponse struct {
+	Version             string               `json:"version"`
+	Data                []*DataColumnSidecar `json:"data"`
+	ExecutionOptimistic bool                 `json:"execution_optimistic"`
+	Finalized           bool                 `json:"finalized"`
+}

beacon-chain/rpc/BUILD.bazel

@@ -34,6 +34,7 @@ go_library(
         "//beacon-chain/rpc/eth/beacon:go_default_library",
         "//beacon-chain/rpc/eth/blob:go_default_library",
         "//beacon-chain/rpc/eth/builder:go_default_library",
+        "//beacon-chain/rpc/eth/column:go_default_library",
         "//beacon-chain/rpc/eth/config:go_default_library",
         "//beacon-chain/rpc/eth/debug:go_default_library",
         "//beacon-chain/rpc/eth/events:go_default_library",

beacon-chain/rpc/endpoints.go

@@ -9,6 +9,7 @@ import (
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/eth/beacon"
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/eth/blob"
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/eth/builder"
+	"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/eth/column"
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/eth/config"
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/eth/debug"
 	"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/eth/events"
@@ -100,6 +101,7 @@ func (s *Service) endpoints(
 	endpoints = append(endpoints, s.prysmBeaconEndpoints(ch, stater, coreService)...)
 	endpoints = append(endpoints, s.prysmNodeEndpoints()...)
 	endpoints = append(endpoints, s.prysmValidatorEndpoints(stater, coreService)...)
+	endpoints = append(endpoints, s.dataColumnSideCarEndPoints(blocker)...)
 
 	if features.Get().EnableLightClient {
 		endpoints = append(endpoints, s.lightClientEndpoints(blocker, stater)...)
@@ -201,6 +203,28 @@ func (s *Service) blobEndpoints(blocker lookup.Blocker) []endpoint {
 	}
 }
 
+func (s *Service) dataColumnSideCarEndPoints(blocker lookup.Blocker) []endpoint {
+	server := &column.Server{
+		Blocker:               blocker,
+		OptimisticModeFetcher: s.cfg.OptimisticModeFetcher,
+		FinalizationFetcher:   s.cfg.FinalizationFetcher,
+		TimeFetcher:           s.cfg.GenesisTimeFetcher,
+	}
+
+	const namespace = "debug"
+	return []endpoint{
+		{
+			template: "/eth/v1/debug/beacon/data_column_sidecars/{block_id}",
+			name:     namespace + ".DataColumnSidecars",
+			middleware: []middleware.Middleware{
+				middleware.AcceptHeaderHandler([]string{api.JsonMediaType, api.OctetStreamMediaType}),
+			},
+			handler: server.DataColumnSidecars,
+			methods: []string{http.MethodGet},
+		},
+	}
+}
+
 func (s *Service) validatorEndpoints(
 	validatorServer *validatorv1alpha1.Server,
 	stater lookup.Stater,

beacon-chain/rpc/endpoints_test.go

@@ -78,9 +78,10 @@ func Test_endpoints(t *testing.T) {
 	}
 
 	debugRoutes := map[string][]string{
-		"/eth/v2/debug/beacon/states/{state_id}": {http.MethodGet},
-		"/eth/v2/debug/beacon/heads":             {http.MethodGet},
-		"/eth/v1/debug/fork_choice":              {http.MethodGet},
+		"/eth/v2/debug/beacon/states/{state_id}":               {http.MethodGet},
+		"/eth/v2/debug/beacon/heads":                           {http.MethodGet},
+		"/eth/v1/debug/fork_choice":                            {http.MethodGet},
+		"/eth/v1/debug/beacon/data_column_sidecars/{block_id}": {http.MethodGet},
 	}
 
 	eventsRoutes := map[string][]string{

beacon-chain/rpc/eth/blob/handlers.go

@@ -26,7 +26,7 @@ func (s *Server) Blobs(w http.ResponseWriter, r *http.Request) {
 	ctx, span := trace.StartSpan(r.Context(), "beacon.Blobs")
 	defer span.End()
 
-	indices, err := parseIndices(r.URL, s.TimeFetcher.CurrentSlot())
+	indices, err := ParseIndices(r.URL, s.TimeFetcher.CurrentSlot())
 	if err != nil {
 		httputil.HandleError(w, err.Error(), http.StatusBadRequest)
 		return
@@ -96,8 +96,8 @@ func (s *Server) Blobs(w http.ResponseWriter, r *http.Request) {
 	httputil.WriteJson(w, resp)
 }
 
-// parseIndices filters out invalid and duplicate blob indices
-func parseIndices(url *url.URL, s primitives.Slot) ([]int, error) {
+// ParseIndices filters out invalid and duplicate blob indices
+func ParseIndices(url *url.URL, s primitives.Slot) ([]int, error) {
 	maxBlobsPerBlock := params.BeaconConfig().MaxBlobsPerBlock(s)
 	rawIndices := url.Query()["indices"]
 	indices := make([]int, 0, maxBlobsPerBlock)

beacon-chain/rpc/eth/blob/handlers_test.go

@@ -546,7 +546,7 @@ func Test_parseIndices(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := parseIndices(&url.URL{RawQuery: tt.query}, 0)
+			got, err := ParseIndices(&url.URL{RawQuery: tt.query}, 0)
 			if err != nil && tt.wantErr != "" {
 				require.StringContains(t, tt.wantErr, err.Error())
 				return

beacon-chain/rpc/eth/column/BUILD.bazel

@@ -0,0 +1,22 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["handler.go"],
+    importpath = "github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/eth/column",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//api:go_default_library",
+        "//api/server/structs:go_default_library",
+        "//beacon-chain/blockchain:go_default_library",
+        "//beacon-chain/rpc/core:go_default_library",
+        "//beacon-chain/rpc/eth/blob:go_default_library",
+        "//beacon-chain/rpc/lookup:go_default_library",
+        "//consensus-types/blocks:go_default_library",
+        "//monitoring/tracing/trace:go_default_library",
+        "//network/httputil:go_default_library",
+        "//runtime/version:go_default_library",
+        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+    ],
+)

beacon-chain/rpc/eth/column/handler.go

@@ -0,0 +1,143 @@
+package column
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/OffchainLabs/prysm/v6/api"
+	"github.com/OffchainLabs/prysm/v6/api/server/structs"
+	"github.com/OffchainLabs/prysm/v6/beacon-chain/blockchain"
+	"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/core"
+	"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/eth/blob"
+	"github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/lookup"
+	"github.com/OffchainLabs/prysm/v6/consensus-types/blocks"
+	"github.com/OffchainLabs/prysm/v6/monitoring/tracing/trace"
+	"github.com/OffchainLabs/prysm/v6/network/httputil"
+	"github.com/OffchainLabs/prysm/v6/runtime/version"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/pkg/errors"
+)
+
+// Server is the HTTP server for handling requests related to data column sidecars.
+type Server struct {
+	Blocker               lookup.Blocker
+	OptimisticModeFetcher blockchain.OptimisticModeFetcher
+	FinalizationFetcher   blockchain.FinalizationFetcher
+	TimeFetcher           blockchain.TimeFetcher
+}
+
+// DataColumnSidecars handles requests for data column sidecars associated with a specific block ID.
+func (s *Server) DataColumnSidecars(w http.ResponseWriter, r *http.Request) {
+	ctx, span := trace.StartSpan(r.Context(), "beacon.DataColumnSidecars")
+	defer span.End()
+
+	indices, err := blob.ParseIndices(r.URL, s.TimeFetcher.CurrentSlot())
+	if err != nil {
+		httputil.HandleError(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	segments := strings.Split(r.URL.Path, "/")
+	blockID := segments[len(segments)-1]
+
+	sidecars, rpcErr := s.Blocker.DataColumnSidecars(ctx, blockID, indices)
+	if rpcErr != nil {
+		code := core.ErrorReasonToHTTP(rpcErr.Reason)
+		msg := rpcErr.Err.Error()
+		switch code {
+		case http.StatusBadRequest:
+			httputil.HandleError(w, "Invalid block ID: "+msg, code)
+		case http.StatusNotFound:
+			httputil.HandleError(w, "Block not found: "+msg, code)
+		case http.StatusInternalServerError:
+			httputil.HandleError(w, "Internal server error: "+msg, code)
+		default:
+			httputil.HandleError(w, msg, code)
+		}
+		return
+	}
+
+	blk, err := s.Blocker.Block(ctx, []byte(blockID))
+	if err != nil {
+		httputil.HandleError(w, "Could not fetch block: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+	if blk == nil {
+		httputil.HandleError(w, "Block not found", http.StatusNotFound)
+		return
+	}
+
+	versionStr := version.String(blk.Version())
+	w.Header().Set(api.VersionHeader, versionStr)
+
+	if httputil.RespondWithSsz(r) {
+		sszResp, err := buildColumnsSszResponse(sidecars)
+		if err != nil {
+			httputil.HandleError(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		httputil.WriteSsz(w, sszResp)
+		return
+	}
+
+	blkRoot, err := blk.Block().HashTreeRoot()
+	if err != nil {
+		httputil.HandleError(w, "Could not hash block: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	isOptimistic, err := s.OptimisticModeFetcher.IsOptimisticForRoot(ctx, blkRoot)
+	if err != nil {
+		httputil.HandleError(w, "Could not check optimistic status: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	resp := &structs.DataColumnSidecarResponse{
+		Version:             versionStr,
+		Data:                buildColumnJSONResponse(sidecars),
+		ExecutionOptimistic: isOptimistic,
+		Finalized:           s.FinalizationFetcher.IsFinalized(ctx, blkRoot),
+	}
+	httputil.WriteJson(w, resp)
+}
+
+func buildColumnsSszResponse(columns []blocks.VerifiedRODataColumn) ([]byte, error) {
+	buf := make([]byte, 0)
+	for _, col := range columns {
+		b, err := col.MarshalSSZ()
+		if err != nil {
+			return nil, errors.Wrap(err, "marshal sidecar ssz")
+		}
+		buf = append(buf, b...)
+	}
+	return buf, nil
+}
+
+func buildColumnJSONResponse(columns []blocks.VerifiedRODataColumn) []*structs.DataColumnSidecar {
+	out := make([]*structs.DataColumnSidecar, len(columns))
+	for i, col := range columns {
+		column := encodeByteSlices(col.Column)
+		kzgCommitments := encodeByteSlices(col.KzgCommitments)
+		kzgProofs := encodeByteSlices(col.KzgProofs)
+		inclusionProofs := encodeByteSlices(col.KzgCommitmentsInclusionProof)
+
+		out[i] = &structs.DataColumnSidecar{
+			Index:                        strconv.FormatUint(col.Index, 10),
+			Column:                       column,
+			KZGCommitments:               kzgCommitments,
+			KZGProofs:                    kzgProofs,
+			SignedBlockHeader:            structs.SignedBeaconBlockHeaderFromConsensus(col.SignedBlockHeader),
+			KZGCommitmentsInclusionProof: inclusionProofs,
+		}
+	}
+	return out
+}
+
+func encodeByteSlices(items [][]byte) []string {
+	out := make([]string, len(items))
+	for i := range items {
+		out[i] = hexutil.Encode(items[i])
+	}
+	return out
+}

beacon-chain/rpc/lookup/blocker.go

@@ -41,6 +41,7 @@ func (e BlockIdParseError) Error() string {
 type Blocker interface {
 	Block(ctx context.Context, id []byte) (interfaces.ReadOnlySignedBeaconBlock, error)
 	Blobs(ctx context.Context, id string, indices []int) ([]*blocks.VerifiedROBlob, *core.RpcError)
+	DataColumnSidecars(ctx context.Context, id string, indices []int) ([]blocks.VerifiedRODataColumn, *core.RpcError)
 }
 
 // BeaconDbBlocker is an implementation of Blocker. It retrieves blocks from the beacon chain database.
@@ -49,6 +50,7 @@ type BeaconDbBlocker struct {
 	ChainInfoFetcher   blockchain.ChainInfoFetcher
 	GenesisTimeFetcher blockchain.TimeFetcher
 	BlobStorage        *filesystem.BlobStorage
+	DataColumnStorage  *filesystem.DataColumnStorage
 }
 
 // Block returns the beacon block for a given identifier. The identifier can be one of:
@@ -273,3 +275,103 @@ func (p *BeaconDbBlocker) Blobs(ctx context.Context, id string, indices []int) (
 
 	return blobs, nil
 }
+
+// DataColumnSidecars returns the verified data columns for a given block id identifier and indices. The identifier can be one of:
+//   - "head" (canonical head in node's view)
+//   - "genesis"
+//   - "finalized"
+//   - "justified"
+//   - <slot>
+//   - <hex encoded block root with '0x' prefix>
+//   - <block root>
+//
+// cases:
+//   - no block, 404
+//   - block exists, no commitment, 200 w/ empty list
+//   - block exists, has commitments, inside retention period (greater of protocol- or user-specified) serve then w/ 200 unless we hit an error reading them.
+//     we are technically not supposed to import a block to forkchoice unless we have the blobs, so the nuance here is if we can't find the file and we are inside the protocol-defined retention period, then it's actually a 500.
+//   - block exists, has commitments, outside retention period (greater of protocol- or user-specified) - ie just like block exists, no commitment
+func (p *BeaconDbBlocker) DataColumnSidecars(ctx context.Context, id string, indices []int) ([]blocks.VerifiedRODataColumn, *core.RpcError) {
+	var rootSlice []byte
+
+	switch id {
+	case "genesis":
+		return nil, &core.RpcError{Err: errors.New("data columns not supported at genesis"), Reason: core.BadRequest}
+	case "head":
+		var err error
+		rootSlice, err = p.ChainInfoFetcher.HeadRoot(ctx)
+		if err != nil {
+			return nil, &core.RpcError{Err: errors.Wrap(err, "could not retrieve head root"), Reason: core.Internal}
+		}
+	case "finalized":
+		fcp := p.ChainInfoFetcher.FinalizedCheckpt()
+		if fcp == nil {
+			return nil, &core.RpcError{Err: errors.New("received nil finalized checkpoint"), Reason: core.Internal}
+		}
+		rootSlice = fcp.Root
+	case "justified":
+		jcp := p.ChainInfoFetcher.CurrentJustifiedCheckpt()
+		if jcp == nil {
+			return nil, &core.RpcError{Err: errors.New("received nil justified checkpoint"), Reason: core.Internal}
+		}
+		rootSlice = jcp.Root
+	default:
+		if bytesutil.IsHex([]byte(id)) {
+			var err error
+			rootSlice, err = bytesutil.DecodeHexWithLength(id, fieldparams.RootLength)
+			if err != nil {
+				return nil, &core.RpcError{Err: NewBlockIdParseError(err), Reason: core.BadRequest}
+			}
+		} else {
+			slot, err := strconv.ParseUint(id, 10, 64)
+			if err != nil {
+				return nil, &core.RpcError{Err: NewBlockIdParseError(err), Reason: core.BadRequest}
+			}
+
+			ok, roots, err := p.BeaconDB.BlockRootsBySlot(ctx, primitives.Slot(slot))
+			if !ok {
+				return nil, &core.RpcError{Err: fmt.Errorf("no block roots at slot %d", slot), Reason: core.NotFound}
+			}
+			if err != nil {
+				return nil, &core.RpcError{Err: errors.Wrapf(err, "failed to get block roots for slot %d", slot), Reason: core.Internal}
+			}
+			rootSlice = roots[0][:]
+			if len(roots) > 1 {
+				for _, blockRoot := range roots {
+					canonical, err := p.ChainInfoFetcher.IsCanonical(ctx, blockRoot)
+					if err != nil {
+						return nil, &core.RpcError{Err: errors.Wrapf(err, "could not determine if block %#x is canonical", blockRoot), Reason: core.Internal}
+					}
+					if canonical {
+						rootSlice = blockRoot[:]
+						break
+					}
+				}
+			}
+		}
+	}
+
+	root := bytesutil.ToBytes32(rootSlice)
+
+	block, err := p.BeaconDB.Block(ctx, root)
+	if err != nil {
+		return nil, &core.RpcError{Err: errors.Wrapf(err, "failed to retrieve block %#x from db", rootSlice), Reason: core.Internal}
+	}
+	if block == nil {
+		return nil, &core.RpcError{Err: fmt.Errorf("block %#x not found in db", rootSlice), Reason: core.NotFound}
+	}
+
+	uintIndices := make([]uint64, len(indices))
+	for i, v := range indices {
+		uintIndices[i] = uint64(v)
+	}
+	columns, err := p.DataColumnStorage.Get(root, uintIndices)
+	if err != nil {
+		return nil, &core.RpcError{
+			Err:    fmt.Errorf("could not retrieve data column for block root %#x", rootSlice),
+			Reason: core.Internal,
+		}
+	}
+
+	return columns, nil
+}

beacon-chain/rpc/testutil/mock_blocker.go

@@ -39,3 +39,8 @@ func (m *MockBlocker) Block(_ context.Context, b []byte) (interfaces.ReadOnlySig
 func (*MockBlocker) Blobs(_ context.Context, _ string, _ []int) ([]*blocks.VerifiedROBlob, *core.RpcError) {
 	return nil, &core.RpcError{}
 }
+
+// DataColumnSidecars mocks the DataColumnSidecars method of the Blocker interface.
+func (*MockBlocker) DataColumnSidecars(_ context.Context, _ string, _ []int) ([]blocks.VerifiedRODataColumn, *core.RpcError) {
+	return nil, &core.RpcError{}
+}