optimize get blobs (#15902)

* init

* reverting some functions

* rolling back a change and fixing linting

* wip

* wip

* fixing test

* breaking up proofs and cells for cleaner code

* fixing test and type

* fixing safe conversion

* fixing test

* fixing more tests

* fixing even more tests

* fix the 0 indices option

* adding a test for coverage

* small test update

* changelog

* radek's suggestions

* Update beacon-chain/core/peerdas/validator.go

Co-authored-by: Manu NALEPA <enalepa@offchainlabs.com>

* addressing comments on kzg package

* addressing suggestions for reconstruction

* more manu feedback items

* removing unneeded files

* removing unneeded setter

---------

Co-authored-by: james-prysm <jhe@offchainlabs.com>
Co-authored-by: Manu NALEPA <enalepa@offchainlabs.com>
This commit is contained in:
james-prysm
2025-11-12 11:53:39 -08:00
committed by GitHub
parent eae15697da
commit e1b98a4ca1
23 changed files with 1181 additions and 407 deletions

View File

@@ -23,6 +23,7 @@ go_library(
go_test(
name = "go_default_test",
srcs = [
"kzg_test.go",
"trusted_setup_test.go",
"validation_test.go",
],

View File

@@ -34,12 +34,6 @@ type Bytes48 = ckzg4844.Bytes48
// Bytes32 is a 32-byte array.
type Bytes32 = ckzg4844.Bytes32
// CellsAndProofs represents the Cells and Proofs corresponding to a single blob.
type CellsAndProofs struct {
Cells []Cell
Proofs []Proof
}
// BlobToKZGCommitment computes a KZG commitment from a given blob.
func BlobToKZGCommitment(blob *Blob) (Commitment, error) {
var kzgBlob kzg4844.Blob
@@ -65,7 +59,7 @@ func ComputeCells(blob *Blob) ([]Cell, error) {
cells := make([]Cell, len(ckzgCells))
for i := range ckzgCells {
cells[i] = Cell(ckzgCells[i])
copy(cells[i][:], ckzgCells[i][:])
}
return cells, nil
@@ -78,22 +72,35 @@ func ComputeBlobKZGProof(blob *Blob, commitment Commitment) (Proof, error) {
proof, err := kzg4844.ComputeBlobProof(&kzgBlob, kzg4844.Commitment(commitment))
if err != nil {
return [48]byte{}, err
return Proof{}, err
}
return Proof(proof), nil
var result Proof
copy(result[:], proof[:])
return result, nil
}
// ComputeCellsAndKZGProofs computes the cells and cells KZG proofs from a given blob.
func ComputeCellsAndKZGProofs(blob *Blob) (CellsAndProofs, error) {
func ComputeCellsAndKZGProofs(blob *Blob) ([]Cell, []Proof, error) {
var ckzgBlob ckzg4844.Blob
copy(ckzgBlob[:], blob[:])
ckzgCells, ckzgProofs, err := ckzg4844.ComputeCellsAndKZGProofs(&ckzgBlob)
if err != nil {
return CellsAndProofs{}, err
return nil, nil, err
}
return makeCellsAndProofs(ckzgCells[:], ckzgProofs[:])
if len(ckzgCells) != len(ckzgProofs) {
return nil, nil, errors.New("mismatched cells and proofs length")
}
cells := make([]Cell, len(ckzgCells))
proofs := make([]Proof, len(ckzgProofs))
for i := range ckzgCells {
copy(cells[i][:], ckzgCells[i][:])
copy(proofs[i][:], ckzgProofs[i][:])
}
return cells, proofs, nil
}
// VerifyCellKZGProofBatch verifies the KZG proofs for a given slice of commitments, cells indices, cells and proofs.
@@ -103,44 +110,57 @@ func VerifyCellKZGProofBatch(commitmentsBytes []Bytes48, cellIndices []uint64, c
ckzgCells := make([]ckzg4844.Cell, len(cells))
for i := range cells {
ckzgCells[i] = ckzg4844.Cell(cells[i])
copy(ckzgCells[i][:], cells[i][:])
}
return ckzg4844.VerifyCellKZGProofBatch(commitmentsBytes, cellIndices, ckzgCells, proofsBytes)
}
// RecoverCellsAndKZGProofs recovers the complete cells and KZG proofs from a given set of cell indices and partial cells.
// RecoverCells recovers the complete cells from a given set of cell indices and partial cells.
// Note: `len(cellIndices)` must be equal to `len(partialCells)` and `cellIndices` must be sorted in ascending order.
func RecoverCellsAndKZGProofs(cellIndices []uint64, partialCells []Cell) (CellsAndProofs, error) {
func RecoverCells(cellIndices []uint64, partialCells []Cell) ([]Cell, error) {
// Convert `Cell` type to `ckzg4844.Cell`
ckzgPartialCells := make([]ckzg4844.Cell, len(partialCells))
for i := range partialCells {
ckzgPartialCells[i] = ckzg4844.Cell(partialCells[i])
copy(ckzgPartialCells[i][:], partialCells[i][:])
}
ckzgCells, err := ckzg4844.RecoverCells(cellIndices, ckzgPartialCells)
if err != nil {
return nil, errors.Wrap(err, "recover cells")
}
cells := make([]Cell, len(ckzgCells))
for i := range ckzgCells {
copy(cells[i][:], ckzgCells[i][:])
}
return cells, nil
}
// RecoverCellsAndKZGProofs recovers the complete cells and KZG proofs from a given set of cell indices and partial cells.
// Note: `len(cellIndices)` must be equal to `len(partialCells)` and `cellIndices` must be sorted in ascending order.
func RecoverCellsAndKZGProofs(cellIndices []uint64, partialCells []Cell) ([]Cell, []Proof, error) {
// Convert `Cell` type to `ckzg4844.Cell`
ckzgPartialCells := make([]ckzg4844.Cell, len(partialCells))
for i := range partialCells {
copy(ckzgPartialCells[i][:], partialCells[i][:])
}
ckzgCells, ckzgProofs, err := ckzg4844.RecoverCellsAndKZGProofs(cellIndices, ckzgPartialCells)
if err != nil {
return CellsAndProofs{}, errors.Wrap(err, "recover cells and KZG proofs")
return nil, nil, errors.Wrap(err, "recover cells and KZG proofs")
}
return makeCellsAndProofs(ckzgCells[:], ckzgProofs[:])
}
// makeCellsAndProofs converts cells/proofs to the CellsAndProofs type defined in this package.
func makeCellsAndProofs(ckzgCells []ckzg4844.Cell, ckzgProofs []ckzg4844.KZGProof) (CellsAndProofs, error) {
if len(ckzgCells) != len(ckzgProofs) {
return CellsAndProofs{}, errors.New("different number of cells/proofs")
return nil, nil, errors.New("mismatched cells and proofs length")
}
cells := make([]Cell, 0, len(ckzgCells))
proofs := make([]Proof, 0, len(ckzgProofs))
cells := make([]Cell, len(ckzgCells))
proofs := make([]Proof, len(ckzgProofs))
for i := range ckzgCells {
cells = append(cells, Cell(ckzgCells[i]))
proofs = append(proofs, Proof(ckzgProofs[i]))
copy(cells[i][:], ckzgCells[i][:])
copy(proofs[i][:], ckzgProofs[i][:])
}
return CellsAndProofs{
Cells: cells,
Proofs: proofs,
}, nil
return cells, proofs, nil
}

View File

@@ -0,0 +1,236 @@
package kzg
import (
"testing"
"github.com/OffchainLabs/prysm/v7/crypto/random"
"github.com/OffchainLabs/prysm/v7/testing/require"
)
func TestComputeCells(t *testing.T) {
require.NoError(t, Start())
t.Run("valid blob", func(t *testing.T) {
randBlob := random.GetRandBlob(123)
var blob Blob
copy(blob[:], randBlob[:])
cells, err := ComputeCells(&blob)
require.NoError(t, err)
require.Equal(t, 128, len(cells))
})
}
func TestComputeBlobKZGProof(t *testing.T) {
require.NoError(t, Start())
t.Run("valid blob and commitment", func(t *testing.T) {
randBlob := random.GetRandBlob(123)
var blob Blob
copy(blob[:], randBlob[:])
commitment, err := BlobToKZGCommitment(&blob)
require.NoError(t, err)
proof, err := ComputeBlobKZGProof(&blob, commitment)
require.NoError(t, err)
require.Equal(t, BytesPerProof, len(proof))
require.NotEqual(t, Proof{}, proof, "proof should not be empty")
})
}
func TestComputeCellsAndKZGProofs(t *testing.T) {
require.NoError(t, Start())
t.Run("valid blob returns matching cells and proofs", func(t *testing.T) {
randBlob := random.GetRandBlob(123)
var blob Blob
copy(blob[:], randBlob[:])
cells, proofs, err := ComputeCellsAndKZGProofs(&blob)
require.NoError(t, err)
require.Equal(t, 128, len(cells))
require.Equal(t, 128, len(proofs))
require.Equal(t, len(cells), len(proofs), "cells and proofs should have matching lengths")
})
}
func TestVerifyCellKZGProofBatch(t *testing.T) {
require.NoError(t, Start())
t.Run("valid proof batch", func(t *testing.T) {
randBlob := random.GetRandBlob(123)
var blob Blob
copy(blob[:], randBlob[:])
commitment, err := BlobToKZGCommitment(&blob)
require.NoError(t, err)
cells, proofs, err := ComputeCellsAndKZGProofs(&blob)
require.NoError(t, err)
// Verify a subset of cells
cellIndices := []uint64{0, 1, 2, 3, 4}
selectedCells := make([]Cell, len(cellIndices))
commitmentsBytes := make([]Bytes48, len(cellIndices))
proofsBytes := make([]Bytes48, len(cellIndices))
for i, idx := range cellIndices {
selectedCells[i] = cells[idx]
copy(commitmentsBytes[i][:], commitment[:])
copy(proofsBytes[i][:], proofs[idx][:])
}
valid, err := VerifyCellKZGProofBatch(commitmentsBytes, cellIndices, selectedCells, proofsBytes)
require.NoError(t, err)
require.Equal(t, true, valid)
})
t.Run("invalid proof should fail", func(t *testing.T) {
randBlob := random.GetRandBlob(123)
var blob Blob
copy(blob[:], randBlob[:])
commitment, err := BlobToKZGCommitment(&blob)
require.NoError(t, err)
cells, _, err := ComputeCellsAndKZGProofs(&blob)
require.NoError(t, err)
// Use invalid proofs
cellIndices := []uint64{0}
selectedCells := []Cell{cells[0]}
commitmentsBytes := make([]Bytes48, 1)
copy(commitmentsBytes[0][:], commitment[:])
// Create an invalid proof
invalidProof := Bytes48{}
proofsBytes := []Bytes48{invalidProof}
valid, err := VerifyCellKZGProofBatch(commitmentsBytes, cellIndices, selectedCells, proofsBytes)
require.NotNil(t, err)
require.Equal(t, false, valid)
})
}
func TestRecoverCells(t *testing.T) {
require.NoError(t, Start())
t.Run("recover from partial cells", func(t *testing.T) {
randBlob := random.GetRandBlob(123)
var blob Blob
copy(blob[:], randBlob[:])
cells, err := ComputeCells(&blob)
require.NoError(t, err)
// Use half of the cells
partialIndices := make([]uint64, 64)
partialCells := make([]Cell, 64)
for i := range 64 {
partialIndices[i] = uint64(i)
partialCells[i] = cells[i]
}
recoveredCells, err := RecoverCells(partialIndices, partialCells)
require.NoError(t, err)
require.Equal(t, 128, len(recoveredCells))
// Verify recovered cells match original
for i := range cells {
require.Equal(t, cells[i], recoveredCells[i])
}
})
t.Run("insufficient cells should fail", func(t *testing.T) {
randBlob := random.GetRandBlob(123)
var blob Blob
copy(blob[:], randBlob[:])
cells, err := ComputeCells(&blob)
require.NoError(t, err)
// Use only 32 cells (less than 50% required)
partialIndices := make([]uint64, 32)
partialCells := make([]Cell, 32)
for i := range 32 {
partialIndices[i] = uint64(i)
partialCells[i] = cells[i]
}
_, err = RecoverCells(partialIndices, partialCells)
require.NotNil(t, err)
})
}
func TestRecoverCellsAndKZGProofs(t *testing.T) {
require.NoError(t, Start())
t.Run("recover cells and proofs from partial cells", func(t *testing.T) {
randBlob := random.GetRandBlob(123)
var blob Blob
copy(blob[:], randBlob[:])
cells, proofs, err := ComputeCellsAndKZGProofs(&blob)
require.NoError(t, err)
// Use half of the cells
partialIndices := make([]uint64, 64)
partialCells := make([]Cell, 64)
for i := range 64 {
partialIndices[i] = uint64(i)
partialCells[i] = cells[i]
}
recoveredCells, recoveredProofs, err := RecoverCellsAndKZGProofs(partialIndices, partialCells)
require.NoError(t, err)
require.Equal(t, 128, len(recoveredCells))
require.Equal(t, 128, len(recoveredProofs))
require.Equal(t, len(recoveredCells), len(recoveredProofs), "recovered cells and proofs should have matching lengths")
// Verify recovered cells match original
for i := range cells {
require.Equal(t, cells[i], recoveredCells[i])
require.Equal(t, proofs[i], recoveredProofs[i])
}
})
t.Run("insufficient cells should fail", func(t *testing.T) {
randBlob := random.GetRandBlob(123)
var blob Blob
copy(blob[:], randBlob[:])
cells, err := ComputeCells(&blob)
require.NoError(t, err)
// Use only 32 cells (less than 50% required)
partialIndices := make([]uint64, 32)
partialCells := make([]Cell, 32)
for i := range 32 {
partialIndices[i] = uint64(i)
partialCells[i] = cells[i]
}
_, _, err = RecoverCellsAndKZGProofs(partialIndices, partialCells)
require.NotNil(t, err)
})
}
func TestBlobToKZGCommitment(t *testing.T) {
require.NoError(t, Start())
t.Run("valid blob", func(t *testing.T) {
randBlob := random.GetRandBlob(123)
var blob Blob
copy(blob[:], randBlob[:])
commitment, err := BlobToKZGCommitment(&blob)
require.NoError(t, err)
require.Equal(t, 48, len(commitment))
// Verify commitment is deterministic
commitment2, err := BlobToKZGCommitment(&blob)
require.NoError(t, err)
require.Equal(t, commitment, commitment2)
})
}

View File

@@ -203,13 +203,13 @@ func TestVerifyCellKZGProofBatchFromBlobData(t *testing.T) {
require.NoError(t, err)
// Compute cells and proofs
cellsAndProofs, err := ComputeCellsAndKZGProofs(&blob)
_, proofs, err := ComputeCellsAndKZGProofs(&blob)
require.NoError(t, err)
// Create flattened cell proofs (like execution client format)
cellProofs := make([][]byte, numberOfColumns)
for i := range numberOfColumns {
cellProofs[i] = cellsAndProofs.Proofs[i][:]
cellProofs[i] = proofs[i][:]
}
blobs := [][]byte{blob[:]}
@@ -236,7 +236,7 @@ func TestVerifyCellKZGProofBatchFromBlobData(t *testing.T) {
require.NoError(t, err)
// Compute cells and proofs
cellsAndProofs, err := ComputeCellsAndKZGProofs(&blob)
_, proofs, err := ComputeCellsAndKZGProofs(&blob)
require.NoError(t, err)
blobs[i] = blob[:]
@@ -244,7 +244,7 @@ func TestVerifyCellKZGProofBatchFromBlobData(t *testing.T) {
// Add cell proofs for this blob
for j := range numberOfColumns {
allCellProofs = append(allCellProofs, cellsAndProofs.Proofs[j][:])
allCellProofs = append(allCellProofs, proofs[j][:])
}
}
@@ -319,7 +319,7 @@ func TestVerifyCellKZGProofBatchFromBlobData(t *testing.T) {
randBlob := random.GetRandBlob(123)
var blob Blob
copy(blob[:], randBlob[:])
cellsAndProofs, err := ComputeCellsAndKZGProofs(&blob)
_, proofs, err := ComputeCellsAndKZGProofs(&blob)
require.NoError(t, err)
// Generate wrong commitment from different blob
@@ -331,7 +331,7 @@ func TestVerifyCellKZGProofBatchFromBlobData(t *testing.T) {
cellProofs := make([][]byte, numberOfColumns)
for i := range numberOfColumns {
cellProofs[i] = cellsAndProofs.Proofs[i][:]
cellProofs[i] = proofs[i][:]
}
blobs := [][]byte{blob[:]}

View File

@@ -43,6 +43,7 @@ go_test(
"das_core_test.go",
"info_test.go",
"p2p_interface_test.go",
"reconstruction_helpers_test.go",
"reconstruction_test.go",
"utils_test.go",
"validator_test.go",

View File

@@ -387,10 +387,10 @@ func generateRandomSidecars(t testing.TB, seed, blobCount int64) []blocks.ROData
sBlock, err := blocks.NewSignedBeaconBlock(dbBlock)
require.NoError(t, err)
cellsAndProofs := util.GenerateCellsAndProofs(t, blobs)
cellsPerBlob, proofsPerBlob := util.GenerateCellsAndProofs(t, blobs)
rob, err := blocks.NewROBlock(sBlock)
require.NoError(t, err)
sidecars, err := peerdas.DataColumnSidecars(cellsAndProofs, peerdas.PopulateFromBlock(rob))
sidecars, err := peerdas.DataColumnSidecars(cellsPerBlob, proofsPerBlob, peerdas.PopulateFromBlock(rob))
require.NoError(t, err)
return sidecars

View File

@@ -2,6 +2,7 @@ package peerdas
import (
"sort"
"sync"
"github.com/OffchainLabs/prysm/v7/beacon-chain/blockchain/kzg"
fieldparams "github.com/OffchainLabs/prysm/v7/config/fieldparams"
@@ -28,6 +29,80 @@ func MinimumColumnCountToReconstruct() uint64 {
return (params.BeaconConfig().NumberOfColumns + 1) / 2
}
// recoverCellsForBlobs reconstructs cells for specified blobs from the given data column sidecars.
// This is optimized to only recover cells without computing proofs.
// Returns a map from blob index to recovered cells.
func recoverCellsForBlobs(verifiedRoSidecars []blocks.VerifiedRODataColumn, blobIndices []int) (map[int][]kzg.Cell, error) {
sidecarCount := len(verifiedRoSidecars)
var wg errgroup.Group
cellsPerBlob := make(map[int][]kzg.Cell, len(blobIndices))
var mu sync.Mutex
for _, blobIndex := range blobIndices {
wg.Go(func() error {
cellsIndices := make([]uint64, 0, sidecarCount)
cells := make([]kzg.Cell, 0, sidecarCount)
for _, sidecar := range verifiedRoSidecars {
cell := sidecar.Column[blobIndex]
cells = append(cells, kzg.Cell(cell))
cellsIndices = append(cellsIndices, sidecar.Index)
}
recoveredCells, err := kzg.RecoverCells(cellsIndices, cells)
if err != nil {
return errors.Wrapf(err, "recover cells for blob %d", blobIndex)
}
mu.Lock()
cellsPerBlob[blobIndex] = recoveredCells
mu.Unlock()
return nil
})
}
if err := wg.Wait(); err != nil {
return nil, errors.Wrap(err, "wait for RecoverCells")
}
return cellsPerBlob, nil
}
// recoverCellsAndProofsForBlobs reconstructs both cells and proofs for specified blobs from the given data column sidecars.
func recoverCellsAndProofsForBlobs(verifiedRoSidecars []blocks.VerifiedRODataColumn, blobIndices []int) ([][]kzg.Cell, [][]kzg.Proof, error) {
sidecarCount := len(verifiedRoSidecars)
var wg errgroup.Group
cellsPerBlob := make([][]kzg.Cell, len(blobIndices))
proofsPerBlob := make([][]kzg.Proof, len(blobIndices))
for i, blobIndex := range blobIndices {
wg.Go(func() error {
cellsIndices := make([]uint64, 0, sidecarCount)
cells := make([]kzg.Cell, 0, sidecarCount)
for _, sidecar := range verifiedRoSidecars {
cell := sidecar.Column[blobIndex]
cells = append(cells, kzg.Cell(cell))
cellsIndices = append(cellsIndices, sidecar.Index)
}
recoveredCells, recoveredProofs, err := kzg.RecoverCellsAndKZGProofs(cellsIndices, cells)
if err != nil {
return errors.Wrapf(err, "recover cells and KZG proofs for blob %d", blobIndex)
}
cellsPerBlob[i] = recoveredCells
proofsPerBlob[i] = recoveredProofs
return nil
})
}
if err := wg.Wait(); err != nil {
return nil, nil, errors.Wrap(err, "wait for RecoverCellsAndKZGProofs")
}
return cellsPerBlob, proofsPerBlob, nil
}
// ReconstructDataColumnSidecars reconstructs all the data column sidecars from the given input data column sidecars.
// All input sidecars must be committed to the same block.
// `inVerifiedRoSidecars` should contain enough sidecars to reconstruct the missing columns, and should not contain any duplicate.
@@ -66,38 +141,16 @@ func ReconstructDataColumnSidecars(verifiedRoSidecars []blocks.VerifiedRODataCol
})
// Recover cells and compute proofs in parallel.
var wg errgroup.Group
cellsAndProofs := make([]kzg.CellsAndProofs, blobCount)
for blobIndex := range uint64(blobCount) {
wg.Go(func() error {
cellsIndices := make([]uint64, 0, sidecarCount)
cells := make([]kzg.Cell, 0, sidecarCount)
for _, sidecar := range verifiedRoSidecars {
cell := sidecar.Column[blobIndex]
cells = append(cells, kzg.Cell(cell))
cellsIndices = append(cellsIndices, sidecar.Index)
}
// Recover the cells and proofs for the corresponding blob
cellsAndProofsForBlob, err := kzg.RecoverCellsAndKZGProofs(cellsIndices, cells)
if err != nil {
return errors.Wrapf(err, "recover cells and KZG proofs for blob %d", blobIndex)
}
// It is safe for multiple goroutines to concurrently write to the same slice,
// as long as they are writing to different indices, which is the case here.
cellsAndProofs[blobIndex] = cellsAndProofsForBlob
return nil
})
blobIndices := make([]int, blobCount)
for i := range blobIndices {
blobIndices[i] = i
}
cellsPerBlob, proofsPerBlob, err := recoverCellsAndProofsForBlobs(verifiedRoSidecars, blobIndices)
if err != nil {
return nil, errors.Wrap(err, "recover cells and proofs for blobs")
}
if err := wg.Wait(); err != nil {
return nil, errors.Wrap(err, "wait for RecoverCellsAndKZGProofs")
}
outSidecars, err := DataColumnSidecars(cellsAndProofs, PopulateFromSidecar(referenceSidecar))
outSidecars, err := DataColumnSidecars(cellsPerBlob, proofsPerBlob, PopulateFromSidecar(referenceSidecar))
if err != nil {
return nil, errors.Wrap(err, "data column sidecars from items")
}
@@ -113,18 +166,192 @@ func ReconstructDataColumnSidecars(verifiedRoSidecars []blocks.VerifiedRODataCol
return reconstructedVerifiedRoSidecars, nil
}
// ReconstructBlobs constructs verified read only blobs sidecars from verified read only blob sidecars.
// reconstructIfNeeded validates the input data column sidecars and returns the prepared sidecars
// (reconstructed if necessary). This function performs common validation and reconstruction logic used by
// both ReconstructBlobs and ReconstructBlobSidecars.
func reconstructIfNeeded(verifiedDataColumnSidecars []blocks.VerifiedRODataColumn) ([]blocks.VerifiedRODataColumn, error) {
if len(verifiedDataColumnSidecars) == 0 {
return nil, ErrNotEnoughDataColumnSidecars
}
// Check if the sidecars are sorted by index and do not contain duplicates.
previousColumnIndex := verifiedDataColumnSidecars[0].Index
for _, dataColumnSidecar := range verifiedDataColumnSidecars[1:] {
columnIndex := dataColumnSidecar.Index
if columnIndex <= previousColumnIndex {
return nil, ErrDataColumnSidecarsNotSortedByIndex
}
previousColumnIndex = columnIndex
}
// Check if we have enough columns.
cellsPerBlob := fieldparams.CellsPerBlob
if len(verifiedDataColumnSidecars) < cellsPerBlob {
return nil, ErrNotEnoughDataColumnSidecars
}
// If all column sidecars corresponding to (non-extended) blobs are present, no need to reconstruct.
if verifiedDataColumnSidecars[cellsPerBlob-1].Index == uint64(cellsPerBlob-1) {
return verifiedDataColumnSidecars, nil
}
// We need to reconstruct the data column sidecars.
return ReconstructDataColumnSidecars(verifiedDataColumnSidecars)
}
// ReconstructBlobSidecars constructs verified read only blobs sidecars from verified read only blob sidecars.
// The following constraints must be satisfied:
// - All `dataColumnSidecars` has to be committed to the same block, and
// - `dataColumnSidecars` must be sorted by index and should not contain duplicates.
// - `dataColumnSidecars` must contain either all sidecars corresponding to (non-extended) blobs,
// or either enough sidecars to reconstruct the blobs.
func ReconstructBlobs(block blocks.ROBlock, verifiedDataColumnSidecars []blocks.VerifiedRODataColumn, indices []int) ([]*blocks.VerifiedROBlob, error) {
// - either enough sidecars to reconstruct the blobs.
func ReconstructBlobSidecars(block blocks.ROBlock, verifiedDataColumnSidecars []blocks.VerifiedRODataColumn, indices []int) ([]*blocks.VerifiedROBlob, error) {
// Return early if no blobs are requested.
if len(indices) == 0 {
return nil, nil
}
// Validate and prepare data columns (reconstruct if necessary).
// This also checks if input is empty.
preparedDataColumnSidecars, err := reconstructIfNeeded(verifiedDataColumnSidecars)
if err != nil {
return nil, err
}
// Check if the blob index is too high.
commitments, err := block.Block().Body().BlobKzgCommitments()
if err != nil {
return nil, errors.Wrap(err, "blob KZG commitments")
}
for _, blobIndex := range indices {
if blobIndex >= len(commitments) {
return nil, ErrBlobIndexTooHigh
}
}
// Check if the data column sidecars are aligned with the block.
dataColumnSidecars := make([]blocks.RODataColumn, 0, len(preparedDataColumnSidecars))
for _, verifiedDataColumnSidecar := range preparedDataColumnSidecars {
dataColumnSidecar := verifiedDataColumnSidecar.RODataColumn
dataColumnSidecars = append(dataColumnSidecars, dataColumnSidecar)
}
if err := DataColumnsAlignWithBlock(block, dataColumnSidecars); err != nil {
return nil, errors.Wrap(err, "data columns align with block")
}
// Convert verified data column sidecars to verified blob sidecars.
blobSidecars, err := blobSidecarsFromDataColumnSidecars(block, preparedDataColumnSidecars, indices)
if err != nil {
return nil, errors.Wrap(err, "blob sidecars from data column sidecars")
}
return blobSidecars, nil
}
// ComputeCellsAndProofsFromFlat computes the cells and proofs from blobs and cell flat proofs.
func ComputeCellsAndProofsFromFlat(blobs [][]byte, cellProofs [][]byte) ([][]kzg.Cell, [][]kzg.Proof, error) {
numberOfColumns := params.BeaconConfig().NumberOfColumns
blobCount := uint64(len(blobs))
cellProofsCount := uint64(len(cellProofs))
cellsCount := blobCount * numberOfColumns
if cellsCount != cellProofsCount {
return nil, nil, ErrBlobsCellsProofsMismatch
}
cellsPerBlob := make([][]kzg.Cell, 0, blobCount)
proofsPerBlob := make([][]kzg.Proof, 0, blobCount)
for i, blob := range blobs {
var kzgBlob kzg.Blob
if copy(kzgBlob[:], blob) != len(kzgBlob) {
return nil, nil, errors.New("wrong blob size - should never happen")
}
// Compute the extended cells from the (non-extended) blob.
cells, err := kzg.ComputeCells(&kzgBlob)
if err != nil {
return nil, nil, errors.Wrap(err, "compute cells")
}
var proofs []kzg.Proof
for idx := uint64(i) * numberOfColumns; idx < (uint64(i)+1)*numberOfColumns; idx++ {
var kzgProof kzg.Proof
if copy(kzgProof[:], cellProofs[idx]) != len(kzgProof) {
return nil, nil, errors.New("wrong KZG proof size - should never happen")
}
proofs = append(proofs, kzgProof)
}
cellsPerBlob = append(cellsPerBlob, cells)
proofsPerBlob = append(proofsPerBlob, proofs)
}
return cellsPerBlob, proofsPerBlob, nil
}
// ComputeCellsAndProofsFromStructured computes the cells and proofs from blobs and cell proofs.
func ComputeCellsAndProofsFromStructured(blobsAndProofs []*pb.BlobAndProofV2) ([][]kzg.Cell, [][]kzg.Proof, error) {
numberOfColumns := params.BeaconConfig().NumberOfColumns
cellsPerBlob := make([][]kzg.Cell, 0, len(blobsAndProofs))
proofsPerBlob := make([][]kzg.Proof, 0, len(blobsAndProofs))
for _, blobAndProof := range blobsAndProofs {
if blobAndProof == nil {
return nil, nil, ErrNilBlobAndProof
}
var kzgBlob kzg.Blob
if copy(kzgBlob[:], blobAndProof.Blob) != len(kzgBlob) {
return nil, nil, errors.New("wrong blob size - should never happen")
}
// Compute the extended cells from the (non-extended) blob.
cells, err := kzg.ComputeCells(&kzgBlob)
if err != nil {
return nil, nil, errors.Wrap(err, "compute cells")
}
kzgProofs := make([]kzg.Proof, 0, numberOfColumns)
for _, kzgProofBytes := range blobAndProof.KzgProofs {
if len(kzgProofBytes) != kzg.BytesPerProof {
return nil, nil, errors.New("wrong KZG proof size - should never happen")
}
var kzgProof kzg.Proof
if copy(kzgProof[:], kzgProofBytes) != len(kzgProof) {
return nil, nil, errors.New("wrong copied KZG proof size - should never happen")
}
kzgProofs = append(kzgProofs, kzgProof)
}
cellsPerBlob = append(cellsPerBlob, cells)
proofsPerBlob = append(proofsPerBlob, kzgProofs)
}
return cellsPerBlob, proofsPerBlob, nil
}
// ReconstructBlobs reconstructs blobs from data column sidecars without computing KZG proofs or creating sidecars.
// This is an optimized version for when only the blob data is needed (e.g., for the GetBlobs endpoint).
// The following constraints must be satisfied:
// - All `dataColumnSidecars` must be committed to the same block, and
// - `dataColumnSidecars` must be sorted by index and should not contain duplicates.
// - `dataColumnSidecars` must contain either all sidecars corresponding to (non-extended) blobs,
// - or enough sidecars to reconstruct the blobs.
func ReconstructBlobs(verifiedDataColumnSidecars []blocks.VerifiedRODataColumn, indices []int, blobCount int) ([][]byte, error) {
// If no specific indices are requested, populate with all blob indices.
if len(indices) == 0 {
indices = make([]int, blobCount)
for i := range indices {
indices[i] = i
}
}
if len(verifiedDataColumnSidecars) == 0 {
return nil, ErrNotEnoughDataColumnSidecars
}
@@ -146,136 +373,70 @@ func ReconstructBlobs(block blocks.ROBlock, verifiedDataColumnSidecars []blocks.
return nil, ErrNotEnoughDataColumnSidecars
}
// Check if the blob index is too high.
commitments, err := block.Block().Body().BlobKzgCommitments()
if err != nil {
return nil, errors.Wrap(err, "blob KZG commitments")
// Verify that the actual blob count from the first sidecar matches the expected count
referenceSidecar := verifiedDataColumnSidecars[0]
actualBlobCount := len(referenceSidecar.Column)
if actualBlobCount != blobCount {
return nil, errors.Errorf("blob count mismatch: expected %d, got %d", blobCount, actualBlobCount)
}
// Check if the blob index is too high.
for _, blobIndex := range indices {
if blobIndex >= len(commitments) {
if blobIndex >= blobCount {
return nil, ErrBlobIndexTooHigh
}
}
// Check if the data column sidecars are aligned with the block.
dataColumnSidecars := make([]blocks.RODataColumn, 0, len(verifiedDataColumnSidecars))
for _, verifiedDataColumnSidecar := range verifiedDataColumnSidecars {
dataColumnSidecar := verifiedDataColumnSidecar.RODataColumn
dataColumnSidecars = append(dataColumnSidecars, dataColumnSidecar)
// Check if all columns have the same length and are committed to the same block.
blockRoot := referenceSidecar.BlockRoot()
for _, sidecar := range verifiedDataColumnSidecars[1:] {
if len(sidecar.Column) != blobCount {
return nil, ErrColumnLengthsDiffer
}
if sidecar.BlockRoot() != blockRoot {
return nil, ErrBlockRootMismatch
}
}
if err := DataColumnsAlignWithBlock(block, dataColumnSidecars); err != nil {
return nil, errors.Wrap(err, "data columns align with block")
}
// Check if we have all non-extended columns (0..63) - if so, no reconstruction needed.
hasAllNonExtendedColumns := verifiedDataColumnSidecars[cellsPerBlob-1].Index == uint64(cellsPerBlob-1)
// If all column sidecars corresponding to (non-extended) blobs are present, no need to reconstruct.
if verifiedDataColumnSidecars[cellsPerBlob-1].Index == uint64(cellsPerBlob-1) {
// Convert verified data column sidecars to verified blob sidecars.
blobSidecars, err := blobSidecarsFromDataColumnSidecars(block, verifiedDataColumnSidecars, indices)
var reconstructedCells map[int][]kzg.Cell
if !hasAllNonExtendedColumns {
// Need to reconstruct cells (but NOT proofs) for the requested blobs only.
var err error
reconstructedCells, err = recoverCellsForBlobs(verifiedDataColumnSidecars, indices)
if err != nil {
return nil, errors.Wrap(err, "blob sidecars from data column sidecars")
return nil, errors.Wrap(err, "recover cells")
}
return blobSidecars, nil
}
// We need to reconstruct the data column sidecars.
reconstructedDataColumnSidecars, err := ReconstructDataColumnSidecars(verifiedDataColumnSidecars)
if err != nil {
return nil, errors.Wrap(err, "reconstruct data column sidecars")
}
// Extract blob data without computing proofs.
blobs := make([][]byte, 0, len(indices))
for _, blobIndex := range indices {
var blob kzg.Blob
// Convert verified data column sidecars to verified blob sidecars.
blobSidecars, err := blobSidecarsFromDataColumnSidecars(block, reconstructedDataColumnSidecars, indices)
if err != nil {
return nil, errors.Wrap(err, "blob sidecars from data column sidecars")
}
return blobSidecars, nil
}
// ComputeCellsAndProofsFromFlat computes the cells and proofs from blobs and cell flat proofs.
func ComputeCellsAndProofsFromFlat(blobs [][]byte, cellProofs [][]byte) ([]kzg.CellsAndProofs, error) {
numberOfColumns := params.BeaconConfig().NumberOfColumns
blobCount := uint64(len(blobs))
cellProofsCount := uint64(len(cellProofs))
cellsCount := blobCount * numberOfColumns
if cellsCount != cellProofsCount {
return nil, ErrBlobsCellsProofsMismatch
}
cellsAndProofs := make([]kzg.CellsAndProofs, 0, blobCount)
for i, blob := range blobs {
var kzgBlob kzg.Blob
if copy(kzgBlob[:], blob) != len(kzgBlob) {
return nil, errors.New("wrong blob size - should never happen")
}
// Compute the extended cells from the (non-extended) blob.
cells, err := kzg.ComputeCells(&kzgBlob)
if err != nil {
return nil, errors.Wrap(err, "compute cells")
}
var proofs []kzg.Proof
for idx := uint64(i) * numberOfColumns; idx < (uint64(i)+1)*numberOfColumns; idx++ {
var kzgProof kzg.Proof
if copy(kzgProof[:], cellProofs[idx]) != len(kzgProof) {
return nil, errors.New("wrong KZG proof size - should never happen")
// Compute the content of the blob.
for columnIndex := range cellsPerBlob {
var cell []byte
if hasAllNonExtendedColumns {
// Use existing cells from sidecars
cell = verifiedDataColumnSidecars[columnIndex].Column[blobIndex]
} else {
// Use reconstructed cells
cell = reconstructedCells[blobIndex][columnIndex][:]
}
proofs = append(proofs, kzgProof)
if copy(blob[kzg.BytesPerCell*columnIndex:], cell) != kzg.BytesPerCell {
return nil, errors.New("wrong cell size - should never happen")
}
}
cellsProofs := kzg.CellsAndProofs{Cells: cells, Proofs: proofs}
cellsAndProofs = append(cellsAndProofs, cellsProofs)
blobs = append(blobs, blob[:])
}
return cellsAndProofs, nil
}
// ComputeCellsAndProofs computes the cells and proofs from blobs and cell proofs.
func ComputeCellsAndProofsFromStructured(blobsAndProofs []*pb.BlobAndProofV2) ([]kzg.CellsAndProofs, error) {
numberOfColumns := params.BeaconConfig().NumberOfColumns
cellsAndProofs := make([]kzg.CellsAndProofs, 0, len(blobsAndProofs))
for _, blobAndProof := range blobsAndProofs {
if blobAndProof == nil {
return nil, ErrNilBlobAndProof
}
var kzgBlob kzg.Blob
if copy(kzgBlob[:], blobAndProof.Blob) != len(kzgBlob) {
return nil, errors.New("wrong blob size - should never happen")
}
// Compute the extended cells from the (non-extended) blob.
cells, err := kzg.ComputeCells(&kzgBlob)
if err != nil {
return nil, errors.Wrap(err, "compute cells")
}
kzgProofs := make([]kzg.Proof, 0, numberOfColumns)
for _, kzgProofBytes := range blobAndProof.KzgProofs {
if len(kzgProofBytes) != kzg.BytesPerProof {
return nil, errors.New("wrong KZG proof size - should never happen")
}
var kzgProof kzg.Proof
if copy(kzgProof[:], kzgProofBytes) != len(kzgProof) {
return nil, errors.New("wrong copied KZG proof size - should never happen")
}
kzgProofs = append(kzgProofs, kzgProof)
}
cellsProofs := kzg.CellsAndProofs{Cells: cells, Proofs: kzgProofs}
cellsAndProofs = append(cellsAndProofs, cellsProofs)
}
return cellsAndProofs, nil
return blobs, nil
}
// blobSidecarsFromDataColumnSidecars converts verified data column sidecars to verified blob sidecars.

View File

@@ -0,0 +1,79 @@
package peerdas_test
// Test helpers for reconstruction tests
import (
"testing"
"github.com/OffchainLabs/prysm/v7/beacon-chain/blockchain/kzg"
"github.com/OffchainLabs/prysm/v7/beacon-chain/core/peerdas"
"github.com/OffchainLabs/prysm/v7/config/params"
"github.com/OffchainLabs/prysm/v7/consensus-types/blocks"
"github.com/OffchainLabs/prysm/v7/consensus-types/primitives"
"github.com/OffchainLabs/prysm/v7/testing/require"
"github.com/OffchainLabs/prysm/v7/testing/util"
)
// testBlobSetup holds common test data for blob reconstruction tests.
type testBlobSetup struct {
blobCount int
blobs []kzg.Blob
roBlock blocks.ROBlock
roDataColumnSidecars []blocks.RODataColumn
verifiedRoDataColumnSidecars []blocks.VerifiedRODataColumn
}
// setupTestBlobs creates a complete test setup with blobs, cells, proofs, and data column sidecars.
func setupTestBlobs(t *testing.T, blobCount int) *testBlobSetup {
_, roBlobSidecars := util.GenerateTestElectraBlockWithSidecar(t, [32]byte{}, 42, blobCount)
blobs := make([]kzg.Blob, blobCount)
for i := range blobCount {
copy(blobs[i][:], roBlobSidecars[i].Blob)
}
cellsPerBlob, proofsPerBlob := util.GenerateCellsAndProofs(t, blobs)
fs := util.SlotAtEpoch(t, params.BeaconConfig().FuluForkEpoch)
roBlock, _, _ := util.GenerateTestFuluBlockWithSidecars(t, blobCount, util.WithSlot(fs))
roDataColumnSidecars, err := peerdas.DataColumnSidecars(cellsPerBlob, proofsPerBlob, peerdas.PopulateFromBlock(roBlock))
require.NoError(t, err)
verifiedRoSidecars := toVerifiedSidecars(roDataColumnSidecars)
return &testBlobSetup{
blobCount: blobCount,
blobs: blobs,
roBlock: roBlock,
roDataColumnSidecars: roDataColumnSidecars,
verifiedRoDataColumnSidecars: verifiedRoSidecars,
}
}
// toVerifiedSidecars converts a slice of RODataColumn to VerifiedRODataColumn.
func toVerifiedSidecars(roDataColumnSidecars []blocks.RODataColumn) []blocks.VerifiedRODataColumn {
verifiedRoSidecars := make([]blocks.VerifiedRODataColumn, 0, len(roDataColumnSidecars))
for _, roDataColumnSidecar := range roDataColumnSidecars {
verifiedRoSidecar := blocks.NewVerifiedRODataColumn(roDataColumnSidecar)
verifiedRoSidecars = append(verifiedRoSidecars, verifiedRoSidecar)
}
return verifiedRoSidecars
}
// filterEvenIndexedSidecars returns only the even-indexed sidecars (0, 2, 4, ...).
// This is useful for forcing reconstruction in tests.
func filterEvenIndexedSidecars(sidecars []blocks.VerifiedRODataColumn) []blocks.VerifiedRODataColumn {
filtered := make([]blocks.VerifiedRODataColumn, 0, len(sidecars)/2)
for i := 0; i < len(sidecars); i += 2 {
filtered = append(filtered, sidecars[i])
}
return filtered
}
// setupFuluForkEpoch sets up the test configuration with Fulu fork after Electra.
func setupFuluForkEpoch(t *testing.T) primitives.Slot {
params.SetupTestConfigCleanup(t)
params.BeaconConfig().FuluForkEpoch = params.BeaconConfig().ElectraForkEpoch + 4096*2
return util.SlotAtEpoch(t, params.BeaconConfig().FuluForkEpoch)
}

View File

@@ -124,7 +124,7 @@ func TestReconstructDataColumnSidecars(t *testing.T) {
})
}
func TestReconstructBlobs(t *testing.T) {
func TestReconstructBlobSidecars(t *testing.T) {
params.SetupTestConfigCleanup(t)
params.BeaconConfig().FuluForkEpoch = params.BeaconConfig().ElectraForkEpoch + 4096*2
@@ -133,13 +133,13 @@ func TestReconstructBlobs(t *testing.T) {
fs := util.SlotAtEpoch(t, params.BeaconConfig().FuluForkEpoch)
t.Run("no index", func(t *testing.T) {
actual, err := peerdas.ReconstructBlobs(emptyBlock, nil, nil)
actual, err := peerdas.ReconstructBlobSidecars(emptyBlock, nil, nil)
require.NoError(t, err)
require.IsNil(t, actual)
})
t.Run("empty input", func(t *testing.T) {
_, err := peerdas.ReconstructBlobs(emptyBlock, nil, []int{0})
_, err := peerdas.ReconstructBlobSidecars(emptyBlock, nil, []int{0})
require.ErrorIs(t, err, peerdas.ErrNotEnoughDataColumnSidecars)
})
@@ -149,7 +149,7 @@ func TestReconstructBlobs(t *testing.T) {
// Arbitrarily change the order of the sidecars.
verifiedRoSidecars[3], verifiedRoSidecars[2] = verifiedRoSidecars[2], verifiedRoSidecars[3]
_, err := peerdas.ReconstructBlobs(emptyBlock, verifiedRoSidecars, []int{0})
_, err := peerdas.ReconstructBlobSidecars(emptyBlock, verifiedRoSidecars, []int{0})
require.ErrorIs(t, err, peerdas.ErrDataColumnSidecarsNotSortedByIndex)
})
@@ -159,7 +159,7 @@ func TestReconstructBlobs(t *testing.T) {
// [0, 1, 1, 3, 4, ...]
verifiedRoSidecars[2] = verifiedRoSidecars[1]
_, err := peerdas.ReconstructBlobs(emptyBlock, verifiedRoSidecars, []int{0})
_, err := peerdas.ReconstructBlobSidecars(emptyBlock, verifiedRoSidecars, []int{0})
require.ErrorIs(t, err, peerdas.ErrDataColumnSidecarsNotSortedByIndex)
})
@@ -169,7 +169,7 @@ func TestReconstructBlobs(t *testing.T) {
// [0, 1, 2, 1, 4, ...]
verifiedRoSidecars[3] = verifiedRoSidecars[1]
_, err := peerdas.ReconstructBlobs(emptyBlock, verifiedRoSidecars, []int{0})
_, err := peerdas.ReconstructBlobSidecars(emptyBlock, verifiedRoSidecars, []int{0})
require.ErrorIs(t, err, peerdas.ErrDataColumnSidecarsNotSortedByIndex)
})
@@ -177,7 +177,7 @@ func TestReconstructBlobs(t *testing.T) {
_, _, verifiedRoSidecars := util.GenerateTestFuluBlockWithSidecars(t, 3)
inputSidecars := verifiedRoSidecars[:fieldparams.CellsPerBlob-1]
_, err := peerdas.ReconstructBlobs(emptyBlock, inputSidecars, []int{0})
_, err := peerdas.ReconstructBlobSidecars(emptyBlock, inputSidecars, []int{0})
require.ErrorIs(t, err, peerdas.ErrNotEnoughDataColumnSidecars)
})
@@ -186,7 +186,7 @@ func TestReconstructBlobs(t *testing.T) {
roBlock, _, verifiedRoSidecars := util.GenerateTestFuluBlockWithSidecars(t, blobCount)
_, err := peerdas.ReconstructBlobs(roBlock, verifiedRoSidecars, []int{1, blobCount})
_, err := peerdas.ReconstructBlobSidecars(roBlock, verifiedRoSidecars, []int{1, blobCount})
require.ErrorIs(t, err, peerdas.ErrBlobIndexTooHigh)
})
@@ -194,7 +194,7 @@ func TestReconstructBlobs(t *testing.T) {
_, _, verifiedRoSidecars := util.GenerateTestFuluBlockWithSidecars(t, 3, util.WithParentRoot([fieldparams.RootLength]byte{1}), util.WithSlot(fs))
roBlock, _, _ := util.GenerateTestFuluBlockWithSidecars(t, 3, util.WithParentRoot([fieldparams.RootLength]byte{2}), util.WithSlot(fs))
_, err := peerdas.ReconstructBlobs(roBlock, verifiedRoSidecars, []int{0})
_, err := peerdas.ReconstructBlobSidecars(roBlock, verifiedRoSidecars, []int{0})
require.ErrorContains(t, peerdas.ErrRootMismatch.Error(), err)
})
@@ -207,7 +207,8 @@ func TestReconstructBlobs(t *testing.T) {
// Compute cells and proofs from blob sidecars.
var wg errgroup.Group
blobs := make([][]byte, blobCount)
inputCellsAndProofs := make([]kzg.CellsAndProofs, blobCount)
inputCellsPerBlob := make([][]kzg.Cell, blobCount)
inputProofsPerBlob := make([][]kzg.Proof, blobCount)
for i := range blobCount {
blob := roBlobSidecars[i].Blob
blobs[i] = blob
@@ -217,14 +218,15 @@ func TestReconstructBlobs(t *testing.T) {
count := copy(kzgBlob[:], blob)
require.Equal(t, len(kzgBlob), count)
cp, err := kzg.ComputeCellsAndKZGProofs(&kzgBlob)
cells, proofs, err := kzg.ComputeCellsAndKZGProofs(&kzgBlob)
if err != nil {
return errors.Wrapf(err, "compute cells and kzg proofs for blob %d", i)
}
// It is safe for multiple goroutines to concurrently write to the same slice,
// as long as they are writing to different indices, which is the case here.
inputCellsAndProofs[i] = cp
inputCellsPerBlob[i] = cells
inputProofsPerBlob[i] = proofs
return nil
})
@@ -235,18 +237,18 @@ func TestReconstructBlobs(t *testing.T) {
// Flatten proofs.
cellProofs := make([][]byte, 0, blobCount*numberOfColumns)
for _, cp := range inputCellsAndProofs {
for _, proof := range cp.Proofs {
for _, proofs := range inputProofsPerBlob {
for _, proof := range proofs {
cellProofs = append(cellProofs, proof[:])
}
}
// Compute celles and proofs from the blobs and cell proofs.
cellsAndProofs, err := peerdas.ComputeCellsAndProofsFromFlat(blobs, cellProofs)
cellsPerBlob, proofsPerBlob, err := peerdas.ComputeCellsAndProofsFromFlat(blobs, cellProofs)
require.NoError(t, err)
// Construct data column sidears from the signed block and cells and proofs.
roDataColumnSidecars, err := peerdas.DataColumnSidecars(cellsAndProofs, peerdas.PopulateFromBlock(roBlock))
roDataColumnSidecars, err := peerdas.DataColumnSidecars(cellsPerBlob, proofsPerBlob, peerdas.PopulateFromBlock(roBlock))
require.NoError(t, err)
// Convert to verified data column sidecars.
@@ -260,7 +262,7 @@ func TestReconstructBlobs(t *testing.T) {
t.Run("no reconstruction needed", func(t *testing.T) {
// Reconstruct blobs.
reconstructedVerifiedRoBlobSidecars, err := peerdas.ReconstructBlobs(roBlock, verifiedRoSidecars, indices)
reconstructedVerifiedRoBlobSidecars, err := peerdas.ReconstructBlobSidecars(roBlock, verifiedRoSidecars, indices)
require.NoError(t, err)
// Compare blobs.
@@ -280,7 +282,7 @@ func TestReconstructBlobs(t *testing.T) {
}
// Reconstruct blobs.
reconstructedVerifiedRoBlobSidecars, err := peerdas.ReconstructBlobs(roBlock, filteredSidecars, indices)
reconstructedVerifiedRoBlobSidecars, err := peerdas.ReconstructBlobSidecars(roBlock, filteredSidecars, indices)
require.NoError(t, err)
// Compare blobs.
@@ -296,6 +298,135 @@ func TestReconstructBlobs(t *testing.T) {
}
func TestReconstructBlobs(t *testing.T) {
setupFuluForkEpoch(t)
require.NoError(t, kzg.Start())
t.Run("empty indices with blobCount > 0", func(t *testing.T) {
setup := setupTestBlobs(t, 3)
// Call with empty indices - should return all blobs
reconstructedBlobs, err := peerdas.ReconstructBlobs(setup.verifiedRoDataColumnSidecars, []int{}, setup.blobCount)
require.NoError(t, err)
require.Equal(t, setup.blobCount, len(reconstructedBlobs))
// Verify each blob matches
for i := 0; i < setup.blobCount; i++ {
require.DeepEqual(t, setup.blobs[i][:], reconstructedBlobs[i])
}
})
t.Run("specific indices", func(t *testing.T) {
setup := setupTestBlobs(t, 3)
// Request only blobs at indices 0 and 2
indices := []int{0, 2}
reconstructedBlobs, err := peerdas.ReconstructBlobs(setup.verifiedRoDataColumnSidecars, indices, setup.blobCount)
require.NoError(t, err)
require.Equal(t, len(indices), len(reconstructedBlobs))
// Verify requested blobs match
for i, blobIndex := range indices {
require.DeepEqual(t, setup.blobs[blobIndex][:], reconstructedBlobs[i])
}
})
t.Run("blob count mismatch", func(t *testing.T) {
setup := setupTestBlobs(t, 3)
// Pass wrong blob count
wrongBlobCount := 5
_, err := peerdas.ReconstructBlobs(setup.verifiedRoDataColumnSidecars, []int{0}, wrongBlobCount)
require.ErrorContains(t, "blob count mismatch", err)
})
t.Run("empty data columns", func(t *testing.T) {
_, err := peerdas.ReconstructBlobs([]blocks.VerifiedRODataColumn{}, []int{0}, 1)
require.ErrorIs(t, err, peerdas.ErrNotEnoughDataColumnSidecars)
})
t.Run("index too high", func(t *testing.T) {
setup := setupTestBlobs(t, 3)
// Request blob index that's too high
_, err := peerdas.ReconstructBlobs(setup.verifiedRoDataColumnSidecars, []int{setup.blobCount}, setup.blobCount)
require.ErrorIs(t, err, peerdas.ErrBlobIndexTooHigh)
})
t.Run("not enough columns", func(t *testing.T) {
setup := setupTestBlobs(t, 3)
// Only provide 63 columns (need at least 64)
inputSidecars := setup.verifiedRoDataColumnSidecars[:fieldparams.CellsPerBlob-1]
_, err := peerdas.ReconstructBlobs(inputSidecars, []int{0}, setup.blobCount)
require.ErrorIs(t, err, peerdas.ErrNotEnoughDataColumnSidecars)
})
t.Run("not sorted", func(t *testing.T) {
setup := setupTestBlobs(t, 3)
// Swap two sidecars to make them unsorted
setup.verifiedRoDataColumnSidecars[3], setup.verifiedRoDataColumnSidecars[2] = setup.verifiedRoDataColumnSidecars[2], setup.verifiedRoDataColumnSidecars[3]
_, err := peerdas.ReconstructBlobs(setup.verifiedRoDataColumnSidecars, []int{0}, setup.blobCount)
require.ErrorIs(t, err, peerdas.ErrDataColumnSidecarsNotSortedByIndex)
})
t.Run("with reconstruction needed", func(t *testing.T) {
setup := setupTestBlobs(t, 3)
// Keep only even-indexed columns (will need reconstruction)
filteredSidecars := filterEvenIndexedSidecars(setup.verifiedRoDataColumnSidecars)
// Reconstruct all blobs
reconstructedBlobs, err := peerdas.ReconstructBlobs(filteredSidecars, []int{}, setup.blobCount)
require.NoError(t, err)
require.Equal(t, setup.blobCount, len(reconstructedBlobs))
// Verify all blobs match
for i := range setup.blobCount {
require.DeepEqual(t, setup.blobs[i][:], reconstructedBlobs[i])
}
})
t.Run("no reconstruction needed - all non-extended columns present", func(t *testing.T) {
setup := setupTestBlobs(t, 3)
// Use all columns (no reconstruction needed since we have all non-extended columns 0-63)
reconstructedBlobs, err := peerdas.ReconstructBlobs(setup.verifiedRoDataColumnSidecars, []int{1}, setup.blobCount)
require.NoError(t, err)
require.Equal(t, 1, len(reconstructedBlobs))
// Verify blob matches
require.DeepEqual(t, setup.blobs[1][:], reconstructedBlobs[0])
})
t.Run("reconstruct only requested blob indices", func(t *testing.T) {
// This test verifies the optimization: when reconstruction is needed and specific
// blob indices are requested, we only reconstruct those blobs, not all of them.
setup := setupTestBlobs(t, 6)
// Keep only even-indexed columns (will need reconstruction)
// This ensures we don't have all non-extended columns (0-63)
filteredSidecars := filterEvenIndexedSidecars(setup.verifiedRoDataColumnSidecars)
// Request only specific blob indices (not all of them)
requestedIndices := []int{1, 3, 5}
reconstructedBlobs, err := peerdas.ReconstructBlobs(filteredSidecars, requestedIndices, setup.blobCount)
require.NoError(t, err)
// Should only get the requested blobs back (not all 6)
require.Equal(t, len(requestedIndices), len(reconstructedBlobs),
"should only reconstruct requested blobs, not all blobs")
// Verify each requested blob matches the original
for i, blobIndex := range requestedIndices {
require.DeepEqual(t, setup.blobs[blobIndex][:], reconstructedBlobs[i],
"blob at index %d should match", blobIndex)
}
})
}
func TestComputeCellsAndProofsFromFlat(t *testing.T) {
// Start the trusted setup.
err := kzg.Start()
@@ -310,7 +441,7 @@ func TestComputeCellsAndProofsFromFlat(t *testing.T) {
// Create proofs for 2 blobs worth of columns
cellProofs := make([][]byte, 2*numberOfColumns)
_, err := peerdas.ComputeCellsAndProofsFromFlat(blobs, cellProofs)
_, _, err := peerdas.ComputeCellsAndProofsFromFlat(blobs, cellProofs)
require.ErrorIs(t, err, peerdas.ErrBlobsCellsProofsMismatch)
})
@@ -323,7 +454,8 @@ func TestComputeCellsAndProofsFromFlat(t *testing.T) {
// Extract blobs and compute expected cells and proofs
blobs := make([][]byte, blobCount)
expectedCellsAndProofs := make([]kzg.CellsAndProofs, blobCount)
expectedCellsPerBlob := make([][]kzg.Cell, blobCount)
expectedProofsPerBlob := make([][]kzg.Proof, blobCount)
var wg errgroup.Group
for i := range blobCount {
@@ -335,12 +467,13 @@ func TestComputeCellsAndProofsFromFlat(t *testing.T) {
count := copy(kzgBlob[:], blob)
require.Equal(t, len(kzgBlob), count)
cp, err := kzg.ComputeCellsAndKZGProofs(&kzgBlob)
cells, proofs, err := kzg.ComputeCellsAndKZGProofs(&kzgBlob)
if err != nil {
return errors.Wrapf(err, "compute cells and kzg proofs for blob %d", i)
}
expectedCellsAndProofs[i] = cp
expectedCellsPerBlob[i] = cells
expectedProofsPerBlob[i] = proofs
return nil
})
}
@@ -350,30 +483,30 @@ func TestComputeCellsAndProofsFromFlat(t *testing.T) {
// Flatten proofs
cellProofs := make([][]byte, 0, blobCount*numberOfColumns)
for _, cp := range expectedCellsAndProofs {
for _, proof := range cp.Proofs {
for _, proofs := range expectedProofsPerBlob {
for _, proof := range proofs {
cellProofs = append(cellProofs, proof[:])
}
}
// Test ComputeCellsAndProofs
actualCellsAndProofs, err := peerdas.ComputeCellsAndProofsFromFlat(blobs, cellProofs)
actualCellsPerBlob, actualProofsPerBlob, err := peerdas.ComputeCellsAndProofsFromFlat(blobs, cellProofs)
require.NoError(t, err)
require.Equal(t, blobCount, len(actualCellsAndProofs))
require.Equal(t, blobCount, len(actualCellsPerBlob))
// Verify the results match expected
for i := range blobCount {
require.Equal(t, len(expectedCellsAndProofs[i].Cells), len(actualCellsAndProofs[i].Cells))
require.Equal(t, len(expectedCellsAndProofs[i].Proofs), len(actualCellsAndProofs[i].Proofs))
require.Equal(t, len(expectedCellsPerBlob[i]), len(actualCellsPerBlob[i]))
require.Equal(t, len(expectedProofsPerBlob[i]), len(actualProofsPerBlob[i]))
// Compare cells
for j, expectedCell := range expectedCellsAndProofs[i].Cells {
require.Equal(t, expectedCell, actualCellsAndProofs[i].Cells[j])
for j, expectedCell := range expectedCellsPerBlob[i] {
require.Equal(t, expectedCell, actualCellsPerBlob[i][j])
}
// Compare proofs
for j, expectedProof := range expectedCellsAndProofs[i].Proofs {
require.Equal(t, expectedProof, actualCellsAndProofs[i].Proofs[j])
for j, expectedProof := range expectedProofsPerBlob[i] {
require.Equal(t, expectedProof, actualProofsPerBlob[i][j])
}
}
})
@@ -381,7 +514,7 @@ func TestComputeCellsAndProofsFromFlat(t *testing.T) {
func TestComputeCellsAndProofsFromStructured(t *testing.T) {
t.Run("nil blob and proof", func(t *testing.T) {
_, err := peerdas.ComputeCellsAndProofsFromStructured([]*pb.BlobAndProofV2{nil})
_, _, err := peerdas.ComputeCellsAndProofsFromStructured([]*pb.BlobAndProofV2{nil})
require.ErrorIs(t, err, peerdas.ErrNilBlobAndProof)
})
@@ -397,7 +530,8 @@ func TestComputeCellsAndProofsFromStructured(t *testing.T) {
// Extract blobs and compute expected cells and proofs
blobsAndProofs := make([]*pb.BlobAndProofV2, blobCount)
expectedCellsAndProofs := make([]kzg.CellsAndProofs, blobCount)
expectedCellsPerBlob := make([][]kzg.Cell, blobCount)
expectedProofsPerBlob := make([][]kzg.Proof, blobCount)
var wg errgroup.Group
for i := range blobCount {
@@ -408,14 +542,15 @@ func TestComputeCellsAndProofsFromStructured(t *testing.T) {
count := copy(kzgBlob[:], blob)
require.Equal(t, len(kzgBlob), count)
cellsAndProofs, err := kzg.ComputeCellsAndKZGProofs(&kzgBlob)
cells, proofs, err := kzg.ComputeCellsAndKZGProofs(&kzgBlob)
if err != nil {
return errors.Wrapf(err, "compute cells and kzg proofs for blob %d", i)
}
expectedCellsAndProofs[i] = cellsAndProofs
expectedCellsPerBlob[i] = cells
expectedProofsPerBlob[i] = proofs
kzgProofs := make([][]byte, 0, len(cellsAndProofs.Proofs))
for _, proof := range cellsAndProofs.Proofs {
kzgProofs := make([][]byte, 0, len(proofs))
for _, proof := range proofs {
kzgProofs = append(kzgProofs, proof[:])
}
@@ -433,24 +568,24 @@ func TestComputeCellsAndProofsFromStructured(t *testing.T) {
require.NoError(t, err)
// Test ComputeCellsAndProofs
actualCellsAndProofs, err := peerdas.ComputeCellsAndProofsFromStructured(blobsAndProofs)
actualCellsPerBlob, actualProofsPerBlob, err := peerdas.ComputeCellsAndProofsFromStructured(blobsAndProofs)
require.NoError(t, err)
require.Equal(t, blobCount, len(actualCellsAndProofs))
require.Equal(t, blobCount, len(actualCellsPerBlob))
// Verify the results match expected
for i := range blobCount {
require.Equal(t, len(expectedCellsAndProofs[i].Cells), len(actualCellsAndProofs[i].Cells))
require.Equal(t, len(expectedCellsAndProofs[i].Proofs), len(actualCellsAndProofs[i].Proofs))
require.Equal(t, len(expectedCellsAndProofs[i].Proofs), cap(actualCellsAndProofs[i].Proofs))
require.Equal(t, len(expectedCellsPerBlob[i]), len(actualCellsPerBlob[i]))
require.Equal(t, len(expectedProofsPerBlob[i]), len(actualProofsPerBlob[i]))
require.Equal(t, len(expectedProofsPerBlob[i]), cap(actualProofsPerBlob[i]))
// Compare cells
for j, expectedCell := range expectedCellsAndProofs[i].Cells {
require.Equal(t, expectedCell, actualCellsAndProofs[i].Cells[j])
for j, expectedCell := range expectedCellsPerBlob[i] {
require.Equal(t, expectedCell, actualCellsPerBlob[i][j])
}
// Compare proofs
for j, expectedProof := range expectedCellsAndProofs[i].Proofs {
require.Equal(t, expectedProof, actualCellsAndProofs[i].Proofs[j])
for j, expectedProof := range expectedProofsPerBlob[i] {
require.Equal(t, expectedProof, actualProofsPerBlob[i][j])
}
}
})

View File

@@ -93,19 +93,20 @@ func ValidatorsCustodyRequirement(state beaconState.ReadOnlyBeaconState, validat
return min(max(count, validatorCustodyRequirement), numberOfCustodyGroups), nil
}
// DataColumnSidecars, given ConstructionPopulator and the cells/proofs associated with each blob in the
// DataColumnSidecars given ConstructionPopulator and the cells/proofs associated with each blob in the
// block, assembles sidecars which can be distributed to peers.
// cellsPerBlob and proofsPerBlob are parallel slices where each index represents a blob sidecar.
// This is an adapted version of
// https://github.com/ethereum/consensus-specs/blob/master/specs/fulu/validator.md#get_data_column_sidecars,
// which is designed to be used both when constructing sidecars from a block and from a sidecar, replacing
// https://github.com/ethereum/consensus-specs/blob/master/specs/fulu/validator.md#get_data_column_sidecars_from_block and
// https://github.com/ethereum/consensus-specs/blob/master/specs/fulu/validator.md#get_data_column_sidecars_from_column_sidecar
func DataColumnSidecars(rows []kzg.CellsAndProofs, src ConstructionPopulator) ([]blocks.RODataColumn, error) {
if len(rows) == 0 {
func DataColumnSidecars(cellsPerBlob [][]kzg.Cell, proofsPerBlob [][]kzg.Proof, src ConstructionPopulator) ([]blocks.RODataColumn, error) {
if len(cellsPerBlob) == 0 {
return nil, nil
}
start := time.Now()
cells, proofs, err := rotateRowsToCols(rows, params.BeaconConfig().NumberOfColumns)
cells, proofs, err := rotateRowsToCols(cellsPerBlob, proofsPerBlob, params.BeaconConfig().NumberOfColumns)
if err != nil {
return nil, errors.Wrap(err, "rotate cells and proofs")
}
@@ -197,26 +198,31 @@ func (b *BlockReconstructionSource) extract() (*blockInfo, error) {
// rotateRowsToCols takes a 2D slice of cells and proofs, where the x is rows (blobs) and y is columns,
// and returns a 2D slice where x is columns and y is rows.
func rotateRowsToCols(rows []kzg.CellsAndProofs, numCols uint64) ([][][]byte, [][][]byte, error) {
if len(rows) == 0 {
func rotateRowsToCols(cellsPerBlob [][]kzg.Cell, proofsPerBlob [][]kzg.Proof, numCols uint64) ([][][]byte, [][][]byte, error) {
if len(cellsPerBlob) == 0 {
return nil, nil, nil
}
if len(cellsPerBlob) != len(proofsPerBlob) {
return nil, nil, errors.New("cells and proofs length mismatch")
}
cellCols := make([][][]byte, numCols)
proofCols := make([][][]byte, numCols)
for i, cp := range rows {
if uint64(len(cp.Cells)) != numCols {
for i := range cellsPerBlob {
cells := cellsPerBlob[i]
proofs := proofsPerBlob[i]
if uint64(len(cells)) != numCols {
return nil, nil, errors.Wrap(ErrNotEnoughDataColumnSidecars, "not enough cells")
}
if len(cp.Cells) != len(cp.Proofs) {
if len(cells) != len(proofs) {
return nil, nil, errors.Wrap(ErrNotEnoughDataColumnSidecars, "not enough proofs")
}
for j := uint64(0); j < numCols; j++ {
if i == 0 {
cellCols[j] = make([][]byte, len(rows))
proofCols[j] = make([][]byte, len(rows))
cellCols[j] = make([][]byte, len(cellsPerBlob))
proofCols[j] = make([][]byte, len(cellsPerBlob))
}
cellCols[j][i] = cp.Cells[j][:]
proofCols[j][i] = cp.Proofs[j][:]
cellCols[j][i] = cells[j][:]
proofCols[j][i] = proofs[j][:]
}
}
return cellCols, proofCols, nil

View File

@@ -68,16 +68,16 @@ func TestDataColumnSidecars(t *testing.T) {
require.NoError(t, err)
// Create cells and proofs.
cellsAndProofs := []kzg.CellsAndProofs{
{
Cells: make([]kzg.Cell, params.BeaconConfig().NumberOfColumns),
Proofs: make([]kzg.Proof, params.BeaconConfig().NumberOfColumns),
},
cellsPerBlob := [][]kzg.Cell{
make([]kzg.Cell, params.BeaconConfig().NumberOfColumns),
}
proofsPerBlob := [][]kzg.Proof{
make([]kzg.Proof, params.BeaconConfig().NumberOfColumns),
}
rob, err := blocks.NewROBlock(signedBeaconBlock)
require.NoError(t, err)
_, err = peerdas.DataColumnSidecars(cellsAndProofs, peerdas.PopulateFromBlock(rob))
_, err = peerdas.DataColumnSidecars(cellsPerBlob, proofsPerBlob, peerdas.PopulateFromBlock(rob))
require.ErrorIs(t, err, peerdas.ErrSizeMismatch)
})
@@ -92,18 +92,18 @@ func TestDataColumnSidecars(t *testing.T) {
// Create cells and proofs with insufficient cells for the number of columns.
// This simulates a scenario where cellsAndProofs has fewer cells than expected columns.
cellsAndProofs := []kzg.CellsAndProofs{
{
Cells: make([]kzg.Cell, 10), // Only 10 cells
Proofs: make([]kzg.Proof, 10), // Only 10 proofs
},
cellsPerBlob := [][]kzg.Cell{
make([]kzg.Cell, 10), // Only 10 cells
}
proofsPerBlob := [][]kzg.Proof{
make([]kzg.Proof, 10), // Only 10 proofs
}
// This should fail because the function will try to access columns up to NumberOfColumns
// but we only have 10 cells/proofs.
rob, err := blocks.NewROBlock(signedBeaconBlock)
require.NoError(t, err)
_, err = peerdas.DataColumnSidecars(cellsAndProofs, peerdas.PopulateFromBlock(rob))
_, err = peerdas.DataColumnSidecars(cellsPerBlob, proofsPerBlob, peerdas.PopulateFromBlock(rob))
require.ErrorIs(t, err, peerdas.ErrNotEnoughDataColumnSidecars)
})
@@ -118,17 +118,17 @@ func TestDataColumnSidecars(t *testing.T) {
// Create cells and proofs with sufficient cells but insufficient proofs.
numberOfColumns := params.BeaconConfig().NumberOfColumns
cellsAndProofs := []kzg.CellsAndProofs{
{
Cells: make([]kzg.Cell, numberOfColumns),
Proofs: make([]kzg.Proof, 5), // Only 5 proofs, less than columns
},
cellsPerBlob := [][]kzg.Cell{
make([]kzg.Cell, numberOfColumns),
}
proofsPerBlob := [][]kzg.Proof{
make([]kzg.Proof, 5), // Only 5 proofs, less than columns
}
// This should fail when trying to access proof beyond index 4.
rob, err := blocks.NewROBlock(signedBeaconBlock)
require.NoError(t, err)
_, err = peerdas.DataColumnSidecars(cellsAndProofs, peerdas.PopulateFromBlock(rob))
_, err = peerdas.DataColumnSidecars(cellsPerBlob, proofsPerBlob, peerdas.PopulateFromBlock(rob))
require.ErrorIs(t, err, peerdas.ErrNotEnoughDataColumnSidecars)
require.ErrorContains(t, "not enough proofs", err)
})
@@ -150,28 +150,26 @@ func TestDataColumnSidecars(t *testing.T) {
// Create cells and proofs with correct dimensions.
numberOfColumns := params.BeaconConfig().NumberOfColumns
cellsAndProofs := []kzg.CellsAndProofs{
{
Cells: make([]kzg.Cell, numberOfColumns),
Proofs: make([]kzg.Proof, numberOfColumns),
},
{
Cells: make([]kzg.Cell, numberOfColumns),
Proofs: make([]kzg.Proof, numberOfColumns),
},
cellsPerBlob := [][]kzg.Cell{
make([]kzg.Cell, numberOfColumns),
make([]kzg.Cell, numberOfColumns),
}
proofsPerBlob := [][]kzg.Proof{
make([]kzg.Proof, numberOfColumns),
make([]kzg.Proof, numberOfColumns),
}
// Set distinct values in cells and proofs for testing
for i := range numberOfColumns {
cellsAndProofs[0].Cells[i][0] = byte(i)
cellsAndProofs[0].Proofs[i][0] = byte(i)
cellsAndProofs[1].Cells[i][0] = byte(i + 128)
cellsAndProofs[1].Proofs[i][0] = byte(i + 128)
cellsPerBlob[0][i][0] = byte(i)
proofsPerBlob[0][i][0] = byte(i)
cellsPerBlob[1][i][0] = byte(i + 128)
proofsPerBlob[1][i][0] = byte(i + 128)
}
rob, err := blocks.NewROBlock(signedBeaconBlock)
require.NoError(t, err)
sidecars, err := peerdas.DataColumnSidecars(cellsAndProofs, peerdas.PopulateFromBlock(rob))
sidecars, err := peerdas.DataColumnSidecars(cellsPerBlob, proofsPerBlob, peerdas.PopulateFromBlock(rob))
require.NoError(t, err)
require.NotNil(t, sidecars)
require.Equal(t, int(numberOfColumns), len(sidecars))
@@ -215,28 +213,26 @@ func TestReconstructionSource(t *testing.T) {
// Create cells and proofs with correct dimensions.
numberOfColumns := params.BeaconConfig().NumberOfColumns
cellsAndProofs := []kzg.CellsAndProofs{
{
Cells: make([]kzg.Cell, numberOfColumns),
Proofs: make([]kzg.Proof, numberOfColumns),
},
{
Cells: make([]kzg.Cell, numberOfColumns),
Proofs: make([]kzg.Proof, numberOfColumns),
},
cellsPerBlob := [][]kzg.Cell{
make([]kzg.Cell, numberOfColumns),
make([]kzg.Cell, numberOfColumns),
}
proofsPerBlob := [][]kzg.Proof{
make([]kzg.Proof, numberOfColumns),
make([]kzg.Proof, numberOfColumns),
}
// Set distinct values in cells and proofs for testing
for i := range numberOfColumns {
cellsAndProofs[0].Cells[i][0] = byte(i)
cellsAndProofs[0].Proofs[i][0] = byte(i)
cellsAndProofs[1].Cells[i][0] = byte(i + 128)
cellsAndProofs[1].Proofs[i][0] = byte(i + 128)
cellsPerBlob[0][i][0] = byte(i)
proofsPerBlob[0][i][0] = byte(i)
cellsPerBlob[1][i][0] = byte(i + 128)
proofsPerBlob[1][i][0] = byte(i + 128)
}
rob, err := blocks.NewROBlock(signedBeaconBlock)
require.NoError(t, err)
sidecars, err := peerdas.DataColumnSidecars(cellsAndProofs, peerdas.PopulateFromBlock(rob))
sidecars, err := peerdas.DataColumnSidecars(cellsPerBlob, proofsPerBlob, peerdas.PopulateFromBlock(rob))
require.NoError(t, err)
require.NotNil(t, sidecars)
require.Equal(t, int(numberOfColumns), len(sidecars))

View File

@@ -660,18 +660,18 @@ func (s *Service) ConstructDataColumnSidecars(ctx context.Context, populator pee
return nil, wrapWithBlockRoot(err, root, "commitments")
}
cellsAndProofs, err := s.fetchCellsAndProofsFromExecution(ctx, commitments)
cellsPerBlob, proofsPerBlob, err := s.fetchCellsAndProofsFromExecution(ctx, commitments)
if err != nil {
return nil, wrapWithBlockRoot(err, root, "fetch cells and proofs from execution client")
}
// Return early if nothing is returned from the EL.
if len(cellsAndProofs) == 0 {
if len(cellsPerBlob) == 0 {
return nil, nil
}
// Construct data column sidears from the signed block and cells and proofs.
roSidecars, err := peerdas.DataColumnSidecars(cellsAndProofs, populator)
roSidecars, err := peerdas.DataColumnSidecars(cellsPerBlob, proofsPerBlob, populator)
if err != nil {
return nil, wrapWithBlockRoot(err, populator.Root(), "data column sidcars from column sidecar")
}
@@ -684,7 +684,7 @@ func (s *Service) ConstructDataColumnSidecars(ctx context.Context, populator pee
}
// fetchCellsAndProofsFromExecution fetches cells and proofs from the execution client (using engine_getBlobsV2 execution API method)
func (s *Service) fetchCellsAndProofsFromExecution(ctx context.Context, kzgCommitments [][]byte) ([]kzg.CellsAndProofs, error) {
func (s *Service) fetchCellsAndProofsFromExecution(ctx context.Context, kzgCommitments [][]byte) ([][]kzg.Cell, [][]kzg.Proof, error) {
// Collect KZG hashes for all blobs.
versionedHashes := make([]common.Hash, 0, len(kzgCommitments))
for _, commitment := range kzgCommitments {
@@ -695,21 +695,21 @@ func (s *Service) fetchCellsAndProofsFromExecution(ctx context.Context, kzgCommi
// Fetch all blobsAndCellsProofs from the execution client.
blobAndProofV2s, err := s.GetBlobsV2(ctx, versionedHashes)
if err != nil {
return nil, errors.Wrapf(err, "get blobs V2")
return nil, nil, errors.Wrapf(err, "get blobs V2")
}
// Return early if nothing is returned from the EL.
if len(blobAndProofV2s) == 0 {
return nil, nil
return nil, nil, nil
}
// Compute cells and proofs from the blobs and cell proofs.
cellsAndProofs, err := peerdas.ComputeCellsAndProofsFromStructured(blobAndProofV2s)
cellsPerBlob, proofsPerBlob, err := peerdas.ComputeCellsAndProofsFromStructured(blobAndProofV2s)
if err != nil {
return nil, errors.Wrap(err, "compute cells and proofs")
return nil, nil, errors.Wrap(err, "compute cells and proofs")
}
return cellsAndProofs, nil
return cellsPerBlob, proofsPerBlob, nil
}
// upgradeSidecarsToVerifiedSidecars upgrades a list of data column sidecars into verified data column sidecars.

View File

@@ -3786,12 +3786,12 @@ func Test_validateBlobs(t *testing.T) {
numberOfColumns := params.BeaconConfig().NumberOfColumns
cellProofs := make([][]byte, uint64(blobCount)*numberOfColumns)
for blobIdx := 0; blobIdx < blobCount; blobIdx++ {
cellsAndProofs, err := kzg.ComputeCellsAndKZGProofs(&kzgBlobs[blobIdx])
_, proofs, err := kzg.ComputeCellsAndKZGProofs(&kzgBlobs[blobIdx])
require.NoError(t, err)
for colIdx := uint64(0); colIdx < numberOfColumns; colIdx++ {
cellProofIdx := uint64(blobIdx)*numberOfColumns + colIdx
cellProofs[cellProofIdx] = cellsAndProofs.Proofs[colIdx][:]
cellProofs[cellProofIdx] = proofs[colIdx][:]
}
}

View File

@@ -38,7 +38,7 @@ func (s *Server) Blobs(w http.ResponseWriter, r *http.Request) {
segments := strings.Split(r.URL.Path, "/")
blockId := segments[len(segments)-1]
verifiedBlobs, rpcErr := s.Blocker.Blobs(ctx, blockId, options.WithIndices(indices))
verifiedBlobs, rpcErr := s.Blocker.BlobSidecars(ctx, blockId, options.WithIndices(indices))
if rpcErr != nil {
code := core.ErrorReasonToHTTP(rpcErr.Reason)
switch code {
@@ -134,9 +134,6 @@ func (s *Server) GetBlobs(w http.ResponseWriter, r *http.Request) {
segments := strings.Split(r.URL.Path, "/")
blockId := segments[len(segments)-1]
var verifiedBlobs []*blocks.VerifiedROBlob
var rpcErr *core.RpcError
// Check if versioned_hashes parameter is provided
versionedHashesStr := r.URL.Query()["versioned_hashes"]
versionedHashes := make([][]byte, len(versionedHashesStr))
@@ -149,7 +146,7 @@ func (s *Server) GetBlobs(w http.ResponseWriter, r *http.Request) {
versionedHashes[i] = hash
}
}
verifiedBlobs, rpcErr = s.Blocker.Blobs(ctx, blockId, options.WithVersionedHashes(versionedHashes))
blobsData, rpcErr := s.Blocker.Blobs(ctx, blockId, options.WithVersionedHashes(versionedHashes))
if rpcErr != nil {
code := core.ErrorReasonToHTTP(rpcErr.Reason)
switch code {
@@ -175,9 +172,9 @@ func (s *Server) GetBlobs(w http.ResponseWriter, r *http.Request) {
if httputil.RespondWithSsz(r) {
sszLen := fieldparams.BlobSize
sszData := make([]byte, len(verifiedBlobs)*sszLen)
for i := range verifiedBlobs {
copy(sszData[i*sszLen:(i+1)*sszLen], verifiedBlobs[i].Blob)
sszData := make([]byte, len(blobsData)*sszLen)
for i := range blobsData {
copy(sszData[i*sszLen:(i+1)*sszLen], blobsData[i])
}
w.Header().Set(api.VersionHeader, version.String(blk.Version()))
@@ -196,9 +193,9 @@ func (s *Server) GetBlobs(w http.ResponseWriter, r *http.Request) {
return
}
data := make([]string, len(verifiedBlobs))
for i, v := range verifiedBlobs {
data[i] = hexutil.Encode(v.Blob)
data := make([]string, len(blobsData))
for i, blob := range blobsData {
data[i] = hexutil.Encode(blob)
}
resp := &structs.GetBlobsResponse{
Data: data,

View File

@@ -60,7 +60,8 @@ func (e BlockIdParseError) Error() string {
// Blocker is responsible for retrieving blocks.
type Blocker interface {
Block(ctx context.Context, id []byte) (interfaces.ReadOnlySignedBeaconBlock, error)
Blobs(ctx context.Context, id string, opts ...options.BlobsOption) ([]*blocks.VerifiedROBlob, *core.RpcError)
BlobSidecars(ctx context.Context, id string, opts ...options.BlobsOption) ([]*blocks.VerifiedROBlob, *core.RpcError)
Blobs(ctx context.Context, id string, opts ...options.BlobsOption) ([][]byte, *core.RpcError)
DataColumns(ctx context.Context, id string, indices []int) ([]blocks.VerifiedRODataColumn, *core.RpcError)
}
@@ -224,23 +225,18 @@ func (p *BeaconDbBlocker) Block(ctx context.Context, id []byte) (interfaces.Read
return blk, nil
}
// Blobs returns the fetched blobs for a given block ID with configurable options.
// Options can specify either blob indices or versioned hashes for retrieval.
// The identifier can be one of:
// - "head" (canonical head in node's view)
// - "genesis"
// - "finalized"
// - "justified"
// - <slot>
// - <hex encoded block root with '0x' prefix>
// - <block root>
//
// cases:
// - no block, 404
// - block exists, has commitments, inside retention period (greater of protocol- or user-specified) serve then w/ 200 unless we hit an error reading them.
// we are technically not supposed to import a block to forkchoice unless we have the blobs, so the nuance here is if we can't find the file and we are inside the protocol-defined retention period, then it's actually a 500.
// - block exists, has commitments, outside retention period (greater of protocol- or user-specified) - ie just like block exists, no commitment
func (p *BeaconDbBlocker) Blobs(ctx context.Context, id string, opts ...options.BlobsOption) ([]*blocks.VerifiedROBlob, *core.RpcError) {
// blobsContext holds common information needed for blob retrieval
type blobsContext struct {
root [fieldparams.RootLength]byte
roBlock blocks.ROBlock
commitments [][]byte
indices []int
postFulu bool
}
// resolveBlobsContext extracts common blob retrieval logic including block resolution,
// validation, and index conversion from versioned hashes.
func (p *BeaconDbBlocker) resolveBlobsContext(ctx context.Context, id string, opts ...options.BlobsOption) (*blobsContext, *core.RpcError) {
// Apply options
cfg := &options.BlobsConfig{}
for _, opt := range opts {
@@ -279,11 +275,6 @@ func (p *BeaconDbBlocker) Blobs(ctx context.Context, id string, opts ...options.
return nil, &core.RpcError{Err: errors.Wrapf(err, "failed to retrieve kzg commitments from block %#x", root), Reason: core.Internal}
}
// If there are no commitments return 200 w/ empty list
if len(commitments) == 0 {
return make([]*blocks.VerifiedROBlob, 0), nil
}
// Compute the first Fulu slot.
fuluForkSlot := primitives.Slot(math.MaxUint64)
if fuluForkEpoch := params.BeaconConfig().FuluForkEpoch; fuluForkEpoch != primitives.Epoch(math.MaxUint64) {
@@ -333,16 +324,156 @@ func (p *BeaconDbBlocker) Blobs(ctx context.Context, id string, opts ...options.
}
}
isPostFulu := false
// Create ROBlock with root for post-Fulu blocks
var roBlockWithRoot blocks.ROBlock
if roBlock.Slot() >= fuluForkSlot {
roBlock, err := blocks.NewROBlockWithRoot(roSignedBlock, root)
roBlockWithRoot, err = blocks.NewROBlockWithRoot(roSignedBlock, root)
if err != nil {
return nil, &core.RpcError{Err: errors.Wrapf(err, "failed to create roBlock with root %#x", root), Reason: core.Internal}
}
return p.blobsFromStoredDataColumns(roBlock, indices)
isPostFulu = true
}
return p.blobsFromStoredBlobs(commitments, root, indices)
return &blobsContext{
root: root,
roBlock: roBlockWithRoot,
commitments: commitments,
indices: indices,
postFulu: isPostFulu,
}, nil
}
// BlobSidecars returns the fetched blob sidecars (with full KZG proofs) for a given block ID.
// Options can specify either blob indices or versioned hashes for retrieval.
// The identifier can be one of:
// - "head" (canonical head in node's view)
// - "genesis"
// - "finalized"
// - "justified"
// - <slot>
// - <hex encoded block root with '0x' prefix>
func (p *BeaconDbBlocker) BlobSidecars(ctx context.Context, id string, opts ...options.BlobsOption) ([]*blocks.VerifiedROBlob, *core.RpcError) {
bctx, rpcErr := p.resolveBlobsContext(ctx, id, opts...)
if rpcErr != nil {
return nil, rpcErr
}
// If there are no commitments return 200 w/ empty list
if len(bctx.commitments) == 0 {
return make([]*blocks.VerifiedROBlob, 0), nil
}
// Check if this is a post-Fulu block (uses data columns)
if bctx.postFulu {
return p.blobSidecarsFromStoredDataColumns(bctx.roBlock, bctx.indices)
}
// Pre-Fulu block (uses blob sidecars)
return p.blobsFromStoredBlobs(bctx.commitments, bctx.root, bctx.indices)
}
// Blobs returns just the blob data without computing KZG proofs or creating full sidecars.
// This is an optimized endpoint for when only blob data is needed (e.g., GetBlobs endpoint).
// The identifier can be one of:
// - "head" (canonical head in node's view)
// - "genesis"
// - "finalized"
// - "justified"
// - <slot>
// - <hex encoded block root with '0x' prefix>
func (p *BeaconDbBlocker) Blobs(ctx context.Context, id string, opts ...options.BlobsOption) ([][]byte, *core.RpcError) {
bctx, rpcErr := p.resolveBlobsContext(ctx, id, opts...)
if rpcErr != nil {
return nil, rpcErr
}
// If there are no commitments return 200 w/ empty list
if len(bctx.commitments) == 0 {
return make([][]byte, 0), nil
}
// Check if this is a post-Fulu block (uses data columns)
if bctx.postFulu {
return p.blobsDataFromStoredDataColumns(bctx.root, bctx.indices, len(bctx.commitments))
}
// Pre-Fulu block (uses blob sidecars)
return p.blobsDataFromStoredBlobs(bctx.root, bctx.indices)
}
// blobsDataFromStoredBlobs retrieves just blob data (without proofs) from stored blob sidecars.
func (p *BeaconDbBlocker) blobsDataFromStoredBlobs(root [fieldparams.RootLength]byte, indices []int) ([][]byte, *core.RpcError) {
summary := p.BlobStorage.Summary(root)
// If no indices are provided, use all indices that are available in the summary.
if len(indices) == 0 {
maxBlobCount := summary.MaxBlobsForEpoch()
for index := 0; uint64(index) < maxBlobCount; index++ { // needed for safe conversion
if summary.HasIndex(uint64(index)) {
indices = append(indices, index)
}
}
}
// Retrieve blob sidecars from the store and extract just the blob data.
blobsData := make([][]byte, 0, len(indices))
for _, index := range indices {
if !summary.HasIndex(uint64(index)) {
return nil, &core.RpcError{
Err: fmt.Errorf("requested index %d not found", index),
Reason: core.NotFound,
}
}
blobSidecar, err := p.BlobStorage.Get(root, uint64(index))
if err != nil {
return nil, &core.RpcError{
Err: fmt.Errorf("could not retrieve blob for block root %#x at index %d", root, index),
Reason: core.Internal,
}
}
blobsData = append(blobsData, blobSidecar.Blob)
}
return blobsData, nil
}
// blobsDataFromStoredDataColumns retrieves blob data from stored data columns without computing KZG proofs.
func (p *BeaconDbBlocker) blobsDataFromStoredDataColumns(root [fieldparams.RootLength]byte, indices []int, blobCount int) ([][]byte, *core.RpcError) {
// Count how many columns we have in the store.
summary := p.DataColumnStorage.Summary(root)
stored := summary.Stored()
count := uint64(len(stored))
if count < peerdas.MinimumColumnCountToReconstruct() {
// There is no way to reconstruct the data columns.
return nil, &core.RpcError{
Err: errors.Errorf("the node does not custody enough data columns to reconstruct blobs - please start the beacon node with the `--%s` flag to ensure this call to succeed, or retry later if it is already the case", flags.SubscribeAllDataSubnets.Name),
Reason: core.NotFound,
}
}
// Retrieve from the database needed data columns.
verifiedRoDataColumnSidecars, err := p.neededDataColumnSidecars(root, stored)
if err != nil {
return nil, &core.RpcError{
Err: errors.Wrap(err, "needed data column sidecars"),
Reason: core.Internal,
}
}
// Use optimized path to get just blob data without computing proofs.
blobsData, err := peerdas.ReconstructBlobs(verifiedRoDataColumnSidecars, indices, blobCount)
if err != nil {
return nil, &core.RpcError{
Err: errors.Wrap(err, "reconstruct blobs data"),
Reason: core.Internal,
}
}
return blobsData, nil
}
// blobsFromStoredBlobs retrieves blob sidercars corresponding to `indices` and `root` from the store.
@@ -393,13 +524,12 @@ func (p *BeaconDbBlocker) blobsFromStoredBlobs(commitments [][]byte, root [field
return blobs, nil
}
// blobsFromStoredDataColumns retrieves data column sidecars from the store,
// reconstructs the whole matrix if needed, converts the matrix to blobs,
// and then returns converted blobs corresponding to `indices` and `root`.
// blobSidecarsFromStoredDataColumns retrieves data column sidecars from the store,
// reconstructs the whole matrix if needed, converts the matrix to blob sidecars with full KZG proofs.
// This function expects data column sidecars to be stored (aka. no blob sidecars).
// If not enough data column sidecars are available to convert blobs from them
// (either directly or after reconstruction), an error is returned.
func (p *BeaconDbBlocker) blobsFromStoredDataColumns(block blocks.ROBlock, indices []int) ([]*blocks.VerifiedROBlob, *core.RpcError) {
func (p *BeaconDbBlocker) blobSidecarsFromStoredDataColumns(block blocks.ROBlock, indices []int) ([]*blocks.VerifiedROBlob, *core.RpcError) {
root := block.Root()
// Use all indices if none are provided.
@@ -439,8 +569,8 @@ func (p *BeaconDbBlocker) blobsFromStoredDataColumns(block blocks.ROBlock, indic
}
}
// Reconstruct blob sidecars from data column sidecars.
verifiedRoBlobSidecars, err := peerdas.ReconstructBlobs(block, verifiedRoDataColumnSidecars, indices)
// Reconstruct blob sidecars with full KZG proofs.
verifiedRoBlobSidecars, err := peerdas.ReconstructBlobSidecars(block, verifiedRoDataColumnSidecars, indices)
if err != nil {
return nil, &core.RpcError{
Err: errors.Wrap(err, "blobs from data columns"),

View File

@@ -182,7 +182,7 @@ func TestBlobsErrorHandling(t *testing.T) {
BeaconDB: db,
}
_, rpcErr := blocker.Blobs(ctx, "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef")
_, rpcErr := blocker.BlobSidecars(ctx, "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef")
require.NotNil(t, rpcErr)
require.Equal(t, core.ErrorReason(core.NotFound), rpcErr.Reason)
require.StringContains(t, "not found", rpcErr.Err.Error())
@@ -194,7 +194,7 @@ func TestBlobsErrorHandling(t *testing.T) {
ChainInfoFetcher: &mockChain.ChainService{},
}
_, rpcErr := blocker.Blobs(ctx, "999999")
_, rpcErr := blocker.BlobSidecars(ctx, "999999")
require.NotNil(t, rpcErr)
require.Equal(t, core.ErrorReason(core.NotFound), rpcErr.Reason)
require.StringContains(t, "no blocks found at slot", rpcErr.Err.Error())
@@ -206,7 +206,7 @@ func TestBlobsErrorHandling(t *testing.T) {
}
// Note: genesis blocks don't support blobs, so this returns BadRequest
_, rpcErr := blocker.Blobs(ctx, "genesis")
_, rpcErr := blocker.BlobSidecars(ctx, "genesis")
require.NotNil(t, rpcErr)
require.Equal(t, core.ErrorReason(core.BadRequest), rpcErr.Reason)
require.StringContains(t, "not supported for Phase 0", rpcErr.Err.Error())
@@ -222,7 +222,7 @@ func TestBlobsErrorHandling(t *testing.T) {
},
}
_, rpcErr := blocker.Blobs(ctx, "finalized")
_, rpcErr := blocker.BlobSidecars(ctx, "finalized")
require.NotNil(t, rpcErr)
require.Equal(t, core.ErrorReason(core.NotFound), rpcErr.Reason)
require.StringContains(t, "finalized block", rpcErr.Err.Error())
@@ -239,7 +239,7 @@ func TestBlobsErrorHandling(t *testing.T) {
},
}
_, rpcErr := blocker.Blobs(ctx, "justified")
_, rpcErr := blocker.BlobSidecars(ctx, "justified")
require.NotNil(t, rpcErr)
require.Equal(t, core.ErrorReason(core.NotFound), rpcErr.Reason)
require.StringContains(t, "justified block", rpcErr.Err.Error())
@@ -251,7 +251,7 @@ func TestBlobsErrorHandling(t *testing.T) {
BeaconDB: db,
}
_, rpcErr := blocker.Blobs(ctx, "invalid-hex")
_, rpcErr := blocker.BlobSidecars(ctx, "invalid-hex")
require.NotNil(t, rpcErr)
require.Equal(t, core.ErrorReason(core.BadRequest), rpcErr.Reason)
require.StringContains(t, "could not parse block ID", rpcErr.Err.Error())
@@ -268,7 +268,7 @@ func TestBlobsErrorHandling(t *testing.T) {
BeaconDB: db,
}
_, rpcErr := blocker.Blobs(ctx, "100")
_, rpcErr := blocker.BlobSidecars(ctx, "100")
require.NotNil(t, rpcErr)
require.Equal(t, core.ErrorReason(core.Internal), rpcErr.Reason)
})
@@ -306,16 +306,18 @@ func TestGetBlob(t *testing.T) {
fuluBlock, fuluBlobSidecars := util.GenerateTestElectraBlockWithSidecar(t, [fieldparams.RootLength]byte{}, fs, blobCount)
fuluBlockRoot := fuluBlock.Root()
cellsAndProofsList := make([]kzg.CellsAndProofs, 0, len(fuluBlobSidecars))
cellsPerBlobList := make([][]kzg.Cell, 0, len(fuluBlobSidecars))
proofsPerBlobList := make([][]kzg.Proof, 0, len(fuluBlobSidecars))
for _, blob := range fuluBlobSidecars {
var kzgBlob kzg.Blob
copy(kzgBlob[:], blob.Blob)
cellsAndProofs, err := kzg.ComputeCellsAndKZGProofs(&kzgBlob)
cells, proofs, err := kzg.ComputeCellsAndKZGProofs(&kzgBlob)
require.NoError(t, err)
cellsAndProofsList = append(cellsAndProofsList, cellsAndProofs)
cellsPerBlobList = append(cellsPerBlobList, cells)
proofsPerBlobList = append(proofsPerBlobList, proofs)
}
roDataColumnSidecars, err := peerdas.DataColumnSidecars(cellsAndProofsList, peerdas.PopulateFromBlock(fuluBlock))
roDataColumnSidecars, err := peerdas.DataColumnSidecars(cellsPerBlobList, proofsPerBlobList, peerdas.PopulateFromBlock(fuluBlock))
require.NoError(t, err)
verifiedRoDataColumnSidecars := make([]blocks.VerifiedRODataColumn, 0, len(roDataColumnSidecars))
@@ -329,7 +331,7 @@ func TestGetBlob(t *testing.T) {
t.Run("genesis", func(t *testing.T) {
blocker := &BeaconDbBlocker{}
_, rpcErr := blocker.Blobs(ctx, "genesis")
_, rpcErr := blocker.BlobSidecars(ctx, "genesis")
require.Equal(t, http.StatusBadRequest, core.ErrorReasonToHTTP(rpcErr.Reason))
require.StringContains(t, "not supported for Phase 0 fork", rpcErr.Err.Error())
})
@@ -347,7 +349,7 @@ func TestGetBlob(t *testing.T) {
BlobStorage: blobStorage,
}
retrievedVerifiedSidecars, rpcErr := blocker.Blobs(ctx, "head")
retrievedVerifiedSidecars, rpcErr := blocker.BlobSidecars(ctx, "head")
require.IsNil(t, rpcErr)
require.Equal(t, blobCount, len(retrievedVerifiedSidecars))
@@ -374,7 +376,7 @@ func TestGetBlob(t *testing.T) {
BlobStorage: blobStorage,
}
verifiedSidecars, rpcErr := blocker.Blobs(ctx, "finalized")
verifiedSidecars, rpcErr := blocker.BlobSidecars(ctx, "finalized")
require.IsNil(t, rpcErr)
require.Equal(t, blobCount, len(verifiedSidecars))
})
@@ -389,7 +391,7 @@ func TestGetBlob(t *testing.T) {
BlobStorage: blobStorage,
}
verifiedSidecars, rpcErr := blocker.Blobs(ctx, "justified")
verifiedSidecars, rpcErr := blocker.BlobSidecars(ctx, "justified")
require.IsNil(t, rpcErr)
require.Equal(t, blobCount, len(verifiedSidecars))
})
@@ -403,7 +405,7 @@ func TestGetBlob(t *testing.T) {
BlobStorage: blobStorage,
}
verifiedBlobs, rpcErr := blocker.Blobs(ctx, hexutil.Encode(denebBlockRoot[:]))
verifiedBlobs, rpcErr := blocker.BlobSidecars(ctx, hexutil.Encode(denebBlockRoot[:]))
require.IsNil(t, rpcErr)
require.Equal(t, blobCount, len(verifiedBlobs))
})
@@ -418,7 +420,7 @@ func TestGetBlob(t *testing.T) {
BlobStorage: blobStorage,
}
verifiedBlobs, rpcErr := blocker.Blobs(ctx, dsStr)
verifiedBlobs, rpcErr := blocker.BlobSidecars(ctx, dsStr)
require.IsNil(t, rpcErr)
require.Equal(t, blobCount, len(verifiedBlobs))
})
@@ -435,7 +437,7 @@ func TestGetBlob(t *testing.T) {
BlobStorage: blobStorage,
}
retrievedVerifiedSidecars, rpcErr := blocker.Blobs(ctx, dsStr, options.WithIndices([]int{index}))
retrievedVerifiedSidecars, rpcErr := blocker.BlobSidecars(ctx, dsStr, options.WithIndices([]int{index}))
require.IsNil(t, rpcErr)
require.Equal(t, 1, len(retrievedVerifiedSidecars))
@@ -459,7 +461,7 @@ func TestGetBlob(t *testing.T) {
BlobStorage: filesystem.NewEphemeralBlobStorage(t),
}
verifiedBlobs, rpcErr := blocker.Blobs(ctx, dsStr)
verifiedBlobs, rpcErr := blocker.BlobSidecars(ctx, dsStr)
require.IsNil(t, rpcErr)
require.Equal(t, 0, len(verifiedBlobs))
})
@@ -475,7 +477,7 @@ func TestGetBlob(t *testing.T) {
}
noBlobIndex := len(storedBlobSidecars) + 1
_, rpcErr := blocker.Blobs(ctx, dsStr, options.WithIndices([]int{0, noBlobIndex}))
_, rpcErr := blocker.BlobSidecars(ctx, dsStr, options.WithIndices([]int{0, noBlobIndex}))
require.NotNil(t, rpcErr)
require.Equal(t, core.ErrorReason(core.NotFound), rpcErr.Reason)
})
@@ -489,7 +491,7 @@ func TestGetBlob(t *testing.T) {
BeaconDB: db,
BlobStorage: blobStorage,
}
_, rpcErr := blocker.Blobs(ctx, dsStr, options.WithIndices([]int{0, math.MaxInt}))
_, rpcErr := blocker.BlobSidecars(ctx, dsStr, options.WithIndices([]int{0, math.MaxInt}))
require.NotNil(t, rpcErr)
require.Equal(t, core.ErrorReason(core.BadRequest), rpcErr.Reason)
})
@@ -508,7 +510,7 @@ func TestGetBlob(t *testing.T) {
DataColumnStorage: dataColumnStorage,
}
_, rpcErr := blocker.Blobs(ctx, hexutil.Encode(fuluBlockRoot[:]))
_, rpcErr := blocker.BlobSidecars(ctx, hexutil.Encode(fuluBlockRoot[:]))
require.NotNil(t, rpcErr)
require.Equal(t, core.ErrorReason(core.NotFound), rpcErr.Reason)
})
@@ -527,7 +529,7 @@ func TestGetBlob(t *testing.T) {
DataColumnStorage: dataColumnStorage,
}
retrievedVerifiedRoBlobs, rpcErr := blocker.Blobs(ctx, hexutil.Encode(fuluBlockRoot[:]))
retrievedVerifiedRoBlobs, rpcErr := blocker.BlobSidecars(ctx, hexutil.Encode(fuluBlockRoot[:]))
require.IsNil(t, rpcErr)
require.Equal(t, len(fuluBlobSidecars), len(retrievedVerifiedRoBlobs))
@@ -552,7 +554,7 @@ func TestGetBlob(t *testing.T) {
DataColumnStorage: dataColumnStorage,
}
retrievedVerifiedRoBlobs, rpcErr := blocker.Blobs(ctx, hexutil.Encode(fuluBlockRoot[:]))
retrievedVerifiedRoBlobs, rpcErr := blocker.BlobSidecars(ctx, hexutil.Encode(fuluBlockRoot[:]))
require.IsNil(t, rpcErr)
require.Equal(t, len(fuluBlobSidecars), len(retrievedVerifiedRoBlobs))
@@ -581,7 +583,7 @@ func TestGetBlob(t *testing.T) {
BeaconDB: db,
}
_, rpcErr := blocker.Blobs(ctx, hexutil.Encode(predenebBlockRoot[:]))
_, rpcErr := blocker.BlobSidecars(ctx, hexutil.Encode(predenebBlockRoot[:]))
require.NotNil(t, rpcErr)
require.Equal(t, core.ErrorReason(core.BadRequest), rpcErr.Reason)
require.Equal(t, http.StatusBadRequest, core.ErrorReasonToHTTP(rpcErr.Reason))
@@ -621,7 +623,7 @@ func TestGetBlob(t *testing.T) {
}
// Should successfully retrieve blobs even when FuluForkEpoch is not set
retrievedBlobs, rpcErr := blocker.Blobs(ctx, hexutil.Encode(denebBlockRoot[:]))
retrievedBlobs, rpcErr := blocker.BlobSidecars(ctx, hexutil.Encode(denebBlockRoot[:]))
require.IsNil(t, rpcErr)
require.Equal(t, 2, len(retrievedBlobs))
@@ -665,16 +667,18 @@ func TestBlobs_CommitmentOrdering(t *testing.T) {
require.Equal(t, 3, len(commitments))
// Convert blob sidecars to data column sidecars for Fulu
cellsAndProofsList := make([]kzg.CellsAndProofs, 0, len(fuluBlobs))
cellsPerBlobList := make([][]kzg.Cell, 0, len(fuluBlobs))
proofsPerBlobList := make([][]kzg.Proof, 0, len(fuluBlobs))
for _, blob := range fuluBlobs {
var kzgBlob kzg.Blob
copy(kzgBlob[:], blob.Blob)
cellsAndProofs, err := kzg.ComputeCellsAndKZGProofs(&kzgBlob)
cells, proofs, err := kzg.ComputeCellsAndKZGProofs(&kzgBlob)
require.NoError(t, err)
cellsAndProofsList = append(cellsAndProofsList, cellsAndProofs)
cellsPerBlobList = append(cellsPerBlobList, cells)
proofsPerBlobList = append(proofsPerBlobList, proofs)
}
dataColumnSidecarPb, err := peerdas.DataColumnSidecars(cellsAndProofsList, peerdas.PopulateFromBlock(fuluBlock))
dataColumnSidecarPb, err := peerdas.DataColumnSidecars(cellsPerBlobList, proofsPerBlobList, peerdas.PopulateFromBlock(fuluBlock))
require.NoError(t, err)
verifiedRoDataColumnSidecars := make([]blocks.VerifiedRODataColumn, 0, len(dataColumnSidecarPb))
@@ -713,7 +717,7 @@ func TestBlobs_CommitmentOrdering(t *testing.T) {
// Request versioned hashes in reverse order: 2, 1, 0
requestedHashes := [][]byte{hash2[:], hash1[:], hash0[:]}
verifiedBlobs, rpcErr := blocker.Blobs(ctx, "finalized", options.WithVersionedHashes(requestedHashes))
verifiedBlobs, rpcErr := blocker.BlobSidecars(ctx, "finalized", options.WithVersionedHashes(requestedHashes))
if rpcErr != nil {
t.Errorf("RPC Error: %v (reason: %v)", rpcErr.Err, rpcErr.Reason)
return
@@ -738,7 +742,7 @@ func TestBlobs_CommitmentOrdering(t *testing.T) {
// Request hashes for indices 1 and 0 (out of order)
requestedHashes := [][]byte{hash1[:], hash0[:]}
verifiedBlobs, rpcErr := blocker.Blobs(ctx, "finalized", options.WithVersionedHashes(requestedHashes))
verifiedBlobs, rpcErr := blocker.BlobSidecars(ctx, "finalized", options.WithVersionedHashes(requestedHashes))
if rpcErr != nil {
t.Errorf("RPC Error: %v (reason: %v)", rpcErr.Err, rpcErr.Reason)
return
@@ -764,7 +768,7 @@ func TestBlobs_CommitmentOrdering(t *testing.T) {
// Request only the fake hash
requestedHashes := [][]byte{fakeHash}
_, rpcErr := blocker.Blobs(ctx, "finalized", options.WithVersionedHashes(requestedHashes))
_, rpcErr := blocker.BlobSidecars(ctx, "finalized", options.WithVersionedHashes(requestedHashes))
require.NotNil(t, rpcErr)
require.Equal(t, core.ErrorReason(core.NotFound), rpcErr.Reason)
require.StringContains(t, "versioned hash(es) not found in block", rpcErr.Err.Error())
@@ -784,7 +788,7 @@ func TestBlobs_CommitmentOrdering(t *testing.T) {
// Request valid hash with two fake hashes
requestedHashes := [][]byte{fakeHash1, hash0[:], fakeHash2}
_, rpcErr := blocker.Blobs(ctx, "finalized", options.WithVersionedHashes(requestedHashes))
_, rpcErr := blocker.BlobSidecars(ctx, "finalized", options.WithVersionedHashes(requestedHashes))
require.NotNil(t, rpcErr)
require.Equal(t, core.ErrorReason(core.NotFound), rpcErr.Reason)
require.StringContains(t, "versioned hash(es) not found in block", rpcErr.Err.Error())
@@ -829,16 +833,18 @@ func TestGetDataColumns(t *testing.T) {
fuluBlock, fuluBlobSidecars := util.GenerateTestElectraBlockWithSidecar(t, [fieldparams.RootLength]byte{}, fuluForkSlot, blobCount)
fuluBlockRoot := fuluBlock.Root()
cellsAndProofsList := make([]kzg.CellsAndProofs, 0, len(fuluBlobSidecars))
cellsPerBlobList := make([][]kzg.Cell, 0, len(fuluBlobSidecars))
proofsPerBlobList := make([][]kzg.Proof, 0, len(fuluBlobSidecars))
for _, blob := range fuluBlobSidecars {
var kzgBlob kzg.Blob
copy(kzgBlob[:], blob.Blob)
cellsAndProofs, err := kzg.ComputeCellsAndKZGProofs(&kzgBlob)
cells, proofs, err := kzg.ComputeCellsAndKZGProofs(&kzgBlob)
require.NoError(t, err)
cellsAndProofsList = append(cellsAndProofsList, cellsAndProofs)
cellsPerBlobList = append(cellsPerBlobList, cells)
proofsPerBlobList = append(proofsPerBlobList, proofs)
}
roDataColumnSidecars, err := peerdas.DataColumnSidecars(cellsAndProofsList, peerdas.PopulateFromBlock(fuluBlock))
roDataColumnSidecars, err := peerdas.DataColumnSidecars(cellsPerBlobList, proofsPerBlobList, peerdas.PopulateFromBlock(fuluBlock))
require.NoError(t, err)
verifiedRoDataColumnSidecars := make([]blocks.VerifiedRODataColumn, 0, len(roDataColumnSidecars))

View File

@@ -413,13 +413,13 @@ func (vs *Server) handleUnblindedBlock(
if block.Version() >= version.Fulu {
// Compute cells and proofs from the blobs and cell proofs.
cellsAndProofs, err := peerdas.ComputeCellsAndProofsFromFlat(rawBlobs, proofs)
cellsPerBlob, proofsPerBlob, err := peerdas.ComputeCellsAndProofsFromFlat(rawBlobs, proofs)
if err != nil {
return nil, nil, errors.Wrap(err, "compute cells and proofs")
}
// Construct data column sidecars from the signed block and cells and proofs.
roDataColumnSidecars, err := peerdas.DataColumnSidecars(cellsAndProofs, peerdas.PopulateFromBlock(block))
roDataColumnSidecars, err := peerdas.DataColumnSidecars(cellsPerBlob, proofsPerBlob, peerdas.PopulateFromBlock(block))
if err != nil {
return nil, nil, errors.Wrap(err, "data column sidcars")
}

View File

@@ -39,8 +39,13 @@ func (m *MockBlocker) Block(_ context.Context, b []byte) (interfaces.ReadOnlySig
return m.SlotBlockMap[primitives.Slot(slotNumber)], nil
}
// BlobSidecars --
func (*MockBlocker) BlobSidecars(_ context.Context, _ string, _ ...options.BlobsOption) ([]*blocks.VerifiedROBlob, *core.RpcError) {
return nil, &core.RpcError{}
}
// Blobs --
func (*MockBlocker) Blobs(_ context.Context, _ string, _ ...options.BlobsOption) ([]*blocks.VerifiedROBlob, *core.RpcError) {
func (*MockBlocker) Blobs(_ context.Context, _ string, _ ...options.BlobsOption) ([][]byte, *core.RpcError) {
return nil, &core.RpcError{}
}

View File

@@ -28,8 +28,8 @@ func GenerateTestDataColumns(t *testing.T, parent [fieldparams.RootLength]byte,
blobs = append(blobs, kzg.Blob(roBlobs[i].Blob))
}
cellsAndProofs := util.GenerateCellsAndProofs(t, blobs)
roDataColumnSidecars, err := peerdas.DataColumnSidecars(cellsAndProofs, peerdas.PopulateFromBlock(roBlock))
cellsPerBlob, proofsPerBlob := util.GenerateCellsAndProofs(t, blobs)
roDataColumnSidecars, err := peerdas.DataColumnSidecars(cellsPerBlob, proofsPerBlob, peerdas.PopulateFromBlock(roBlock))
require.NoError(t, err)
return roDataColumnSidecars

View File

@@ -0,0 +1,3 @@
### Ignored
- optimization to remove cell and blob proof computation on blob rest api.

View File

@@ -42,18 +42,16 @@ func TestComputeCellsAndKzgProofs(t *testing.T) {
}
b := kzgPrysm.Blob(blob)
cellsAndProofsForBlob, err := kzgPrysm.ComputeCellsAndKZGProofs(&b)
cells, proofs, err := kzgPrysm.ComputeCellsAndKZGProofs(&b)
if test.Output != nil {
require.NoError(t, err)
var combined [][]string
cs := cellsAndProofsForBlob.Cells
csRaw := make([]string, 0, len(cs))
for _, c := range cs {
csRaw := make([]string, 0, len(cells))
for _, c := range cells {
csRaw = append(csRaw, hexutil.Encode(c[:]))
}
ps := cellsAndProofsForBlob.Proofs
psRaw := make([]string, 0, len(ps))
for _, p := range ps {
psRaw := make([]string, 0, len(proofs))
for _, p := range proofs {
psRaw = append(psRaw, hexutil.Encode(p[:]))
}
combined = append(combined, csRaw)

View File

@@ -69,18 +69,16 @@ func TestRecoverCellsAndKzgProofs(t *testing.T) {
}
// Recover the cells and proofs for the corresponding blob
cellsAndProofsForBlob, err := kzgPrysm.RecoverCellsAndKZGProofs(cellIndices, cells)
recoveredCells, recoveredProofs, err := kzgPrysm.RecoverCellsAndKZGProofs(cellIndices, cells)
if test.Output != nil {
require.NoError(t, err)
var combined [][]string
cs := cellsAndProofsForBlob.Cells
csRaw := make([]string, 0, len(cs))
for _, c := range cs {
csRaw := make([]string, 0, len(recoveredCells))
for _, c := range recoveredCells {
csRaw = append(csRaw, hexutil.Encode(c[:]))
}
ps := cellsAndProofsForBlob.Proofs
psRaw := make([]string, 0, len(ps))
for _, p := range ps {
psRaw := make([]string, 0, len(recoveredProofs))
for _, p := range recoveredProofs {
psRaw = append(psRaw, hexutil.Encode(p[:]))
}
combined = append(combined, csRaw)

View File

@@ -146,11 +146,11 @@ func GenerateTestFuluBlockWithSidecars(t *testing.T, blobCount int, options ...F
signedBeaconBlock, err := blocks.NewSignedBeaconBlock(block)
require.NoError(t, err)
cellsAndProofs := GenerateCellsAndProofs(t, blobs)
cellsPerBlob, proofsPerBlob := GenerateCellsAndProofs(t, blobs)
rob, err := blocks.NewROBlockWithRoot(signedBeaconBlock, root)
require.NoError(t, err)
roSidecars, err := peerdas.DataColumnSidecars(cellsAndProofs, peerdas.PopulateFromBlock(rob))
roSidecars, err := peerdas.DataColumnSidecars(cellsPerBlob, proofsPerBlob, peerdas.PopulateFromBlock(rob))
require.NoError(t, err)
verifiedRoSidecars := make([]blocks.VerifiedRODataColumn, 0, len(roSidecars))
@@ -167,12 +167,14 @@ func GenerateTestFuluBlockWithSidecars(t *testing.T, blobCount int, options ...F
return roBlock, roSidecars, verifiedRoSidecars
}
func GenerateCellsAndProofs(t testing.TB, blobs []kzg.Blob) []kzg.CellsAndProofs {
cellsAndProofs := make([]kzg.CellsAndProofs, len(blobs))
func GenerateCellsAndProofs(t testing.TB, blobs []kzg.Blob) ([][]kzg.Cell, [][]kzg.Proof) {
cellsPerBlob := make([][]kzg.Cell, len(blobs))
proofsPerBlob := make([][]kzg.Proof, len(blobs))
for i := range blobs {
cp, err := kzg.ComputeCellsAndKZGProofs(&blobs[i])
cells, proofs, err := kzg.ComputeCellsAndKZGProofs(&blobs[i])
require.NoError(t, err)
cellsAndProofs[i] = cp
cellsPerBlob[i] = cells
proofsPerBlob[i] = proofs
}
return cellsAndProofs
return cellsPerBlob, proofsPerBlob
}