diff --git a/beacon-chain/rpc/lookup/BUILD.bazel b/beacon-chain/rpc/lookup/BUILD.bazel index 7a1c522efb..30a33b0898 100644 --- a/beacon-chain/rpc/lookup/BUILD.bazel +++ b/beacon-chain/rpc/lookup/BUILD.bazel @@ -10,11 +10,13 @@ go_library( visibility = ["//visibility:public"], deps = [ "//beacon-chain/blockchain:go_default_library", + "//beacon-chain/core/peerdas:go_default_library", "//beacon-chain/db:go_default_library", "//beacon-chain/db/filesystem:go_default_library", "//beacon-chain/rpc/core:go_default_library", "//beacon-chain/state:go_default_library", "//beacon-chain/state/stategen:go_default_library", + "//cmd/beacon-chain/flags:go_default_library", "//config/fieldparams:go_default_library", "//config/params:go_default_library", "//consensus-types/blocks:go_default_library", @@ -36,7 +38,9 @@ go_test( ], embed = [":go_default_library"], deps = [ + "//beacon-chain/blockchain/kzg:go_default_library", "//beacon-chain/blockchain/testing:go_default_library", + "//beacon-chain/core/peerdas:go_default_library", "//beacon-chain/db/filesystem:go_default_library", "//beacon-chain/db/testing:go_default_library", "//beacon-chain/rpc/core:go_default_library", @@ -45,6 +49,7 @@ go_test( "//beacon-chain/state/stategen:go_default_library", "//beacon-chain/state/stategen/mock:go_default_library", "//beacon-chain/verification:go_default_library", + "//config/fieldparams:go_default_library", "//config/params:go_default_library", "//consensus-types/blocks:go_default_library", "//consensus-types/primitives:go_default_library", diff --git a/beacon-chain/rpc/lookup/blocker.go b/beacon-chain/rpc/lookup/blocker.go index 7bd36c23ea..fd5cef9bca 100644 --- a/beacon-chain/rpc/lookup/blocker.go +++ b/beacon-chain/rpc/lookup/blocker.go @@ -3,12 +3,15 @@ package lookup import ( "context" "fmt" + "math" "strconv" "github.com/OffchainLabs/prysm/v6/beacon-chain/blockchain" + "github.com/OffchainLabs/prysm/v6/beacon-chain/core/peerdas" "github.com/OffchainLabs/prysm/v6/beacon-chain/db" "github.com/OffchainLabs/prysm/v6/beacon-chain/db/filesystem" "github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/core" + "github.com/OffchainLabs/prysm/v6/cmd/beacon-chain/flags" fieldparams "github.com/OffchainLabs/prysm/v6/config/fieldparams" "github.com/OffchainLabs/prysm/v6/config/params" "github.com/OffchainLabs/prysm/v6/consensus-types/blocks" @@ -49,6 +52,7 @@ type BeaconDbBlocker struct { ChainInfoFetcher blockchain.ChainInfoFetcher GenesisTimeFetcher blockchain.TimeFetcher BlobStorage *filesystem.BlobStorage + DataColumnStorage *filesystem.DataColumnStorage } // Block returns the beacon block for a given identifier. The identifier can be one of: @@ -212,64 +216,190 @@ func (p *BeaconDbBlocker) Blobs(ctx context.Context, id string, indices []int) ( root := bytesutil.ToBytes32(rootSlice) - b, err := p.BeaconDB.Block(ctx, root) + roSignedBlock, err := p.BeaconDB.Block(ctx, root) if err != nil { return nil, &core.RpcError{Err: errors.Wrapf(err, "failed to retrieve block %#x from db", rootSlice), Reason: core.Internal} } - if b == nil { + + if roSignedBlock == nil { return nil, &core.RpcError{Err: fmt.Errorf("block %#x not found in db", rootSlice), Reason: core.NotFound} } - // if block is not in the retention window, return 200 w/ empty list - if !p.BlobStorage.WithinRetentionPeriod(slots.ToEpoch(b.Block().Slot()), slots.ToEpoch(p.GenesisTimeFetcher.CurrentSlot())) { + // If block is not in the retention window, return 200 w/ empty list + if !p.BlobStorage.WithinRetentionPeriod(slots.ToEpoch(roSignedBlock.Block().Slot()), slots.ToEpoch(p.GenesisTimeFetcher.CurrentSlot())) { return make([]*blocks.VerifiedROBlob, 0), nil } - commitments, err := b.Block().Body().BlobKzgCommitments() + roBlock := roSignedBlock.Block() + + commitments, err := roBlock.Body().BlobKzgCommitments() if err != nil { return nil, &core.RpcError{Err: errors.Wrapf(err, "failed to retrieve kzg commitments from block %#x", rootSlice), Reason: core.Internal} } - // if there are no commitments return 200 w/ empty list + + // If there are no commitments return 200 w/ empty list if len(commitments) == 0 { return make([]*blocks.VerifiedROBlob, 0), nil } - sum := p.BlobStorage.Summary(root) + // Compute the first Fulu slot. + fuluForkEpoch := params.BeaconConfig().FuluForkEpoch + fuluForkSlot := primitives.Slot(math.MaxUint64) + if fuluForkEpoch != primitives.Epoch(math.MaxUint64) { + fuluForkSlot, err = slots.EpochStart(fuluForkEpoch) + if err != nil { + return nil, &core.RpcError{Err: errors.Wrap(err, "could not calculate peerDAS start slot"), Reason: core.Internal} + } + } - if len(indices) == 0 { - for i := range commitments { - if sum.HasIndex(uint64(i)) { - indices = append(indices, i) + if roBlock.Slot() >= fuluForkSlot { + roBlock, err := blocks.NewROBlockWithRoot(roSignedBlock, root) + if err != nil { + return nil, &core.RpcError{Err: errors.Wrapf(err, "failed to create roBlock with root %#x", root), Reason: core.Internal} + } + + return p.blobsFromStoredDataColumns(roBlock, indices) + } + + return p.blobsFromStoredBlobs(commitments, root, indices) +} + +// blobsFromStoredBlobs retrieves blob sidercars corresponding to `indices` and `root` from the store. +// This function expects blob sidecars to be stored (aka. no data column sidecars). +func (p *BeaconDbBlocker) blobsFromStoredBlobs(commitments [][]byte, root [fieldparams.RootLength]byte, indices []int) ([]*blocks.VerifiedROBlob, *core.RpcError) { + summary := p.BlobStorage.Summary(root) + maxBlobCount := summary.MaxBlobsForEpoch() + + for _, index := range indices { + if uint64(index) >= maxBlobCount { + return nil, &core.RpcError{ + Err: fmt.Errorf("requested index %d is bigger than the maximum possible blob count %d", index, maxBlobCount), + Reason: core.BadRequest, } } - } else { - for _, ix := range indices { - if uint64(ix) >= sum.MaxBlobsForEpoch() { - return nil, &core.RpcError{ - Err: fmt.Errorf("requested index %d is bigger than the maximum possible blob count %d", ix, sum.MaxBlobsForEpoch()), - Reason: core.BadRequest, - } - } - if !sum.HasIndex(uint64(ix)) { - return nil, &core.RpcError{ - Err: fmt.Errorf("requested index %d not found", ix), - Reason: core.NotFound, - } + + if !summary.HasIndex(uint64(index)) { + return nil, &core.RpcError{ + Err: fmt.Errorf("requested index %d not found", index), + Reason: core.NotFound, } } } - blobs := make([]*blocks.VerifiedROBlob, len(indices)) - for i, index := range indices { - vblob, err := p.BlobStorage.Get(root, uint64(index)) + // If no indices are provided, use all indices that are available in the summary. + if len(indices) == 0 { + for index := range commitments { + if summary.HasIndex(uint64(index)) { + indices = append(indices, index) + } + } + } + + // Retrieve blob sidecars from the store. + blobs := make([]*blocks.VerifiedROBlob, 0, len(indices)) + for _, index := range indices { + blobSidecar, err := p.BlobStorage.Get(root, uint64(index)) if err != nil { return nil, &core.RpcError{ - Err: fmt.Errorf("could not retrieve blob for block root %#x at index %d", rootSlice, index), + Err: fmt.Errorf("could not retrieve blob for block root %#x at index %d", root, index), Reason: core.Internal, } } - blobs[i] = &vblob + + blobs = append(blobs, &blobSidecar) } return blobs, nil } + +// blobsFromStoredDataColumns retrieves data column sidecars from the store, +// reconstructs the whole matrix if needed, converts the matrix to blobs, +// and then returns converted blobs corresponding to `indices` and `root`. +// This function expects data column sidecars to be stored (aka. no blob sidecars). +// If not enough data column sidecars are available to convert blobs from them +// (either directly or after reconstruction), an error is returned. +func (p *BeaconDbBlocker) blobsFromStoredDataColumns(block blocks.ROBlock, indices []int) ([]*blocks.VerifiedROBlob, *core.RpcError) { + root := block.Root() + + // Use all indices if none are provided. + if len(indices) == 0 { + commitments, err := block.Block().Body().BlobKzgCommitments() + if err != nil { + return nil, &core.RpcError{ + Err: errors.Wrap(err, "could not retrieve blob commitments"), + Reason: core.Internal, + } + } + + for index := range commitments { + indices = append(indices, index) + } + } + + // Count how many columns we have in the store. + summary := p.DataColumnStorage.Summary(root) + stored := summary.Stored() + count := uint64(len(stored)) + + if count < peerdas.MinimumColumnsCountToReconstruct() { + // There is no way to reconstruct the data columns. + return nil, &core.RpcError{ + Err: errors.Errorf("the node does not custody enough data columns to reconstruct blobs - please start the beacon node with the `--%s` flag to ensure this call to succeed, or retry later if it is already the case", flags.SubscribeAllDataSubnets.Name), + Reason: core.NotFound, + } + } + + // Retrieve from the database needed data columns. + verifiedRoDataColumnSidecars, err := p.neededDataColumnSidecars(root, stored) + if err != nil { + return nil, &core.RpcError{ + Err: errors.Wrap(err, "needed data column sidecars"), + Reason: core.Internal, + } + } + + // Reconstruct blob sidecars from data column sidecars. + verifiedRoBlobSidecars, err := peerdas.ReconstructBlobs(block, verifiedRoDataColumnSidecars, indices) + if err != nil { + return nil, &core.RpcError{ + Err: errors.Wrap(err, "blobs from data columns"), + Reason: core.Internal, + } + } + + return verifiedRoBlobSidecars, nil +} + +// neededDataColumnSidecars retrieves all data column sidecars corresponding to (non extended) blobs if available, +// else retrieves all data column sidecars from the store. +func (p *BeaconDbBlocker) neededDataColumnSidecars(root [fieldparams.RootLength]byte, stored map[uint64]bool) ([]blocks.VerifiedRODataColumn, error) { + // Check if we have all the non-extended data columns. + cellsPerBlob := fieldparams.CellsPerBlob + blobIndices := make([]uint64, 0, cellsPerBlob) + hasAllBlobColumns := true + for i := range uint64(cellsPerBlob) { + if !stored[i] { + hasAllBlobColumns = false + break + } + blobIndices = append(blobIndices, i) + } + + if hasAllBlobColumns { + // Retrieve only the non-extended data columns. + verifiedRoSidecars, err := p.DataColumnStorage.Get(root, blobIndices) + if err != nil { + return nil, errors.Wrap(err, "data columns storage get") + } + + return verifiedRoSidecars, nil + } + + // Retrieve all the data columns. + verifiedRoSidecars, err := p.DataColumnStorage.Get(root, nil) + if err != nil { + return nil, errors.Wrap(err, "data columns storage get") + } + + return verifiedRoSidecars, nil +} diff --git a/beacon-chain/rpc/lookup/blocker_test.go b/beacon-chain/rpc/lookup/blocker_test.go index 8251b19ae1..76d2f0344f 100644 --- a/beacon-chain/rpc/lookup/blocker_test.go +++ b/beacon-chain/rpc/lookup/blocker_test.go @@ -8,12 +8,15 @@ import ( "testing" "time" + "github.com/OffchainLabs/prysm/v6/beacon-chain/blockchain/kzg" mockChain "github.com/OffchainLabs/prysm/v6/beacon-chain/blockchain/testing" + "github.com/OffchainLabs/prysm/v6/beacon-chain/core/peerdas" "github.com/OffchainLabs/prysm/v6/beacon-chain/db/filesystem" testDB "github.com/OffchainLabs/prysm/v6/beacon-chain/db/testing" "github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/core" "github.com/OffchainLabs/prysm/v6/beacon-chain/rpc/testutil" "github.com/OffchainLabs/prysm/v6/beacon-chain/verification" + fieldparams "github.com/OffchainLabs/prysm/v6/config/fieldparams" "github.com/OffchainLabs/prysm/v6/config/params" "github.com/OffchainLabs/prysm/v6/consensus-types/blocks" "github.com/OffchainLabs/prysm/v6/encoding/bytesutil" @@ -158,172 +161,335 @@ func TestGetBlock(t *testing.T) { } func TestGetBlob(t *testing.T) { - params.SetupTestConfigCleanup(t) - cfg := params.BeaconConfig().Copy() - cfg.DenebForkEpoch = 1 - params.OverrideBeaconConfig(cfg) + const ( + slot = 123 + blobCount = 4 + denebForEpoch = 1 + fuluForkEpoch = 2 + ) + + setupDeneb := func(t *testing.T) { + params.SetupTestConfigCleanup(t) + cfg := params.BeaconConfig().Copy() + cfg.DenebForkEpoch = denebForEpoch + params.OverrideBeaconConfig(cfg) + } + + setupFulu := func(t *testing.T) { + params.SetupTestConfigCleanup(t) + cfg := params.BeaconConfig().Copy() + cfg.DenebForkEpoch = denebForEpoch + cfg.FuluForkEpoch = fuluForkEpoch + params.OverrideBeaconConfig(cfg) + } + ctx := t.Context() db := testDB.SetupDB(t) - denebBlock, blobs := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, 123, 4) - require.NoError(t, db.SaveBlock(t.Context(), denebBlock)) - _, bs := filesystem.NewEphemeralBlobStorageAndFs(t) - testSidecars := verification.FakeVerifySliceForTest(t, blobs) - for i := range testSidecars { - require.NoError(t, bs.Save(testSidecars[i])) + + // Start the trusted setup. + err := kzg.Start() + require.NoError(t, err) + + // Create and save Deneb block and blob sidecars. + _, blobStorage := filesystem.NewEphemeralBlobStorageAndFs(t) + + denebBlock, storedBlobSidecars := util.GenerateTestDenebBlockWithSidecar(t, [fieldparams.RootLength]byte{}, slot, blobCount) + denebBlockRoot := denebBlock.Root() + + verifiedStoredSidecars := verification.FakeVerifySliceForTest(t, storedBlobSidecars) + for i := range verifiedStoredSidecars { + err := blobStorage.Save(verifiedStoredSidecars[i]) + require.NoError(t, err) } - blockRoot := blobs[0].BlockRoot() + + err = db.SaveBlock(t.Context(), denebBlock) + require.NoError(t, err) + + // Create Electra block and blob sidecars. (Electra block = Fulu block), + // save the block, convert blob sidecars to data column sidecars and save the block. + fuluForkSlot := fuluForkEpoch * params.BeaconConfig().SlotsPerEpoch + fuluBlock, fuluBlobSidecars := util.GenerateTestElectraBlockWithSidecar(t, [fieldparams.RootLength]byte{}, fuluForkSlot, blobCount) + fuluBlockRoot := fuluBlock.Root() + + cellsAndProofsList := make([]kzg.CellsAndProofs, 0, len(fuluBlobSidecars)) + for _, blob := range fuluBlobSidecars { + var kzgBlob kzg.Blob + copy(kzgBlob[:], blob.Blob) + cellsAndProogs, err := kzg.ComputeCellsAndKZGProofs(&kzgBlob) + require.NoError(t, err) + cellsAndProofsList = append(cellsAndProofsList, cellsAndProogs) + } + + dataColumnSidecarPb, err := peerdas.DataColumnSidecars(fuluBlock, cellsAndProofsList) + require.NoError(t, err) + + verifiedRoDataColumnSidecars := make([]blocks.VerifiedRODataColumn, 0, len(dataColumnSidecarPb)) + for _, sidecarPb := range dataColumnSidecarPb { + roDataColumn, err := blocks.NewRODataColumnWithRoot(sidecarPb, fuluBlockRoot) + require.NoError(t, err) + + verifiedRoDataColumn := blocks.NewVerifiedRODataColumn(roDataColumn) + verifiedRoDataColumnSidecars = append(verifiedRoDataColumnSidecars, verifiedRoDataColumn) + } + + err = db.SaveBlock(t.Context(), fuluBlock) + require.NoError(t, err) + t.Run("genesis", func(t *testing.T) { + setupDeneb(t) + blocker := &BeaconDbBlocker{} _, rpcErr := blocker.Blobs(ctx, "genesis", nil) - assert.Equal(t, http.StatusBadRequest, core.ErrorReasonToHTTP(rpcErr.Reason)) - assert.StringContains(t, "blobs are not supported for Phase 0 fork", rpcErr.Err.Error()) + require.Equal(t, http.StatusBadRequest, core.ErrorReasonToHTTP(rpcErr.Reason)) + require.StringContains(t, "blobs are not supported for Phase 0 fork", rpcErr.Err.Error()) }) + t.Run("head", func(t *testing.T) { + setupDeneb(t) + blocker := &BeaconDbBlocker{ - ChainInfoFetcher: &mockChain.ChainService{Root: blockRoot[:]}, + ChainInfoFetcher: &mockChain.ChainService{Root: denebBlockRoot[:]}, GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{ Genesis: time.Now(), }, BeaconDB: db, - BlobStorage: bs, + BlobStorage: blobStorage, + } + + retrievedVerifiedSidecars, rpcErr := blocker.Blobs(ctx, "head", nil) + require.IsNil(t, rpcErr) + require.Equal(t, blobCount, len(retrievedVerifiedSidecars)) + + for i := range blobCount { + expected := verifiedStoredSidecars[i] + + actual := retrievedVerifiedSidecars[i].BlobSidecar + require.NotNil(t, actual) + + require.Equal(t, expected.Index, actual.Index) + require.DeepEqual(t, expected.Blob, actual.Blob) + require.DeepEqual(t, expected.KzgCommitment, actual.KzgCommitment) + require.DeepEqual(t, expected.KzgProof, actual.KzgProof) } - verifiedBlobs, rpcErr := blocker.Blobs(ctx, "head", nil) - assert.Equal(t, rpcErr == nil, true) - require.Equal(t, 4, len(verifiedBlobs)) - sidecar := verifiedBlobs[0].BlobSidecar - require.NotNil(t, sidecar) - assert.Equal(t, uint64(0), sidecar.Index) - assert.DeepEqual(t, blobs[0].Blob, sidecar.Blob) - assert.DeepEqual(t, blobs[0].KzgCommitment, sidecar.KzgCommitment) - assert.DeepEqual(t, blobs[0].KzgProof, sidecar.KzgProof) - sidecar = verifiedBlobs[1].BlobSidecar - require.NotNil(t, sidecar) - assert.Equal(t, uint64(1), sidecar.Index) - assert.DeepEqual(t, blobs[1].Blob, sidecar.Blob) - assert.DeepEqual(t, blobs[1].KzgCommitment, sidecar.KzgCommitment) - assert.DeepEqual(t, blobs[1].KzgProof, sidecar.KzgProof) - sidecar = verifiedBlobs[2].BlobSidecar - require.NotNil(t, sidecar) - assert.Equal(t, uint64(2), sidecar.Index) - assert.DeepEqual(t, blobs[2].Blob, sidecar.Blob) - assert.DeepEqual(t, blobs[2].KzgCommitment, sidecar.KzgCommitment) - assert.DeepEqual(t, blobs[2].KzgProof, sidecar.KzgProof) - sidecar = verifiedBlobs[3].BlobSidecar - require.NotNil(t, sidecar) - assert.Equal(t, uint64(3), sidecar.Index) - assert.DeepEqual(t, blobs[3].Blob, sidecar.Blob) - assert.DeepEqual(t, blobs[3].KzgCommitment, sidecar.KzgCommitment) - assert.DeepEqual(t, blobs[3].KzgProof, sidecar.KzgProof) }) + t.Run("finalized", func(t *testing.T) { + setupDeneb(t) + blocker := &BeaconDbBlocker{ - ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: ðpb.Checkpoint{Root: blockRoot[:]}}, + ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: ðpb.Checkpoint{Root: denebBlockRoot[:]}}, GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{ Genesis: time.Now(), }, BeaconDB: db, - BlobStorage: bs, + BlobStorage: blobStorage, } - verifiedBlobs, rpcErr := blocker.Blobs(ctx, "finalized", nil) - assert.Equal(t, rpcErr == nil, true) - require.Equal(t, 4, len(verifiedBlobs)) + verifiedSidecars, rpcErr := blocker.Blobs(ctx, "finalized", nil) + require.IsNil(t, rpcErr) + require.Equal(t, blobCount, len(verifiedSidecars)) }) + t.Run("justified", func(t *testing.T) { + setupDeneb(t) + blocker := &BeaconDbBlocker{ - ChainInfoFetcher: &mockChain.ChainService{CurrentJustifiedCheckPoint: ðpb.Checkpoint{Root: blockRoot[:]}}, + ChainInfoFetcher: &mockChain.ChainService{CurrentJustifiedCheckPoint: ðpb.Checkpoint{Root: denebBlockRoot[:]}}, GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{ Genesis: time.Now(), }, BeaconDB: db, - BlobStorage: bs, + BlobStorage: blobStorage, } - verifiedBlobs, rpcErr := blocker.Blobs(ctx, "justified", nil) - assert.Equal(t, rpcErr == nil, true) - require.Equal(t, 4, len(verifiedBlobs)) + verifiedSidecars, rpcErr := blocker.Blobs(ctx, "justified", nil) + require.IsNil(t, rpcErr) + require.Equal(t, blobCount, len(verifiedSidecars)) }) + t.Run("root", func(t *testing.T) { + setupDeneb(t) + blocker := &BeaconDbBlocker{ GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{ Genesis: time.Now(), }, BeaconDB: db, - BlobStorage: bs, + BlobStorage: blobStorage, } - verifiedBlobs, rpcErr := blocker.Blobs(ctx, hexutil.Encode(blockRoot[:]), nil) - assert.Equal(t, rpcErr == nil, true) - require.Equal(t, 4, len(verifiedBlobs)) + + verifiedBlobs, rpcErr := blocker.Blobs(ctx, hexutil.Encode(denebBlockRoot[:]), nil) + require.IsNil(t, rpcErr) + require.Equal(t, blobCount, len(verifiedBlobs)) }) + t.Run("slot", func(t *testing.T) { + setupDeneb(t) + blocker := &BeaconDbBlocker{ GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{ Genesis: time.Now(), }, BeaconDB: db, - BlobStorage: bs, + BlobStorage: blobStorage, } + verifiedBlobs, rpcErr := blocker.Blobs(ctx, "123", nil) - assert.Equal(t, rpcErr == nil, true) - require.Equal(t, 4, len(verifiedBlobs)) + require.IsNil(t, rpcErr) + require.Equal(t, blobCount, len(verifiedBlobs)) }) + t.Run("one blob only", func(t *testing.T) { + const index = 2 + + setupDeneb(t) + blocker := &BeaconDbBlocker{ - ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: ðpb.Checkpoint{Root: blockRoot[:]}}, + ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: ðpb.Checkpoint{Root: denebBlockRoot[:]}}, GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{ Genesis: time.Now(), }, BeaconDB: db, - BlobStorage: bs, + BlobStorage: blobStorage, } - verifiedBlobs, rpcErr := blocker.Blobs(ctx, "123", []int{2}) - assert.Equal(t, rpcErr == nil, true) - require.Equal(t, 1, len(verifiedBlobs)) - sidecar := verifiedBlobs[0].BlobSidecar - require.NotNil(t, sidecar) - assert.Equal(t, uint64(2), sidecar.Index) - assert.DeepEqual(t, blobs[2].Blob, sidecar.Blob) - assert.DeepEqual(t, blobs[2].KzgCommitment, sidecar.KzgCommitment) - assert.DeepEqual(t, blobs[2].KzgProof, sidecar.KzgProof) + + retrievedVerifiedSidecars, rpcErr := blocker.Blobs(ctx, "123", []int{index}) + require.IsNil(t, rpcErr) + require.Equal(t, 1, len(retrievedVerifiedSidecars)) + + expected := verifiedStoredSidecars[index] + actual := retrievedVerifiedSidecars[0].BlobSidecar + require.NotNil(t, actual) + + require.Equal(t, uint64(index), actual.Index) + require.DeepEqual(t, expected.Blob, actual.Blob) + require.DeepEqual(t, expected.KzgCommitment, actual.KzgCommitment) + require.DeepEqual(t, expected.KzgProof, actual.KzgProof) }) + t.Run("no blobs returns an empty array", func(t *testing.T) { + setupDeneb(t) + blocker := &BeaconDbBlocker{ - ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: ðpb.Checkpoint{Root: blockRoot[:]}}, + ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: ðpb.Checkpoint{Root: denebBlockRoot[:]}}, GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{ Genesis: time.Now(), }, BeaconDB: db, BlobStorage: filesystem.NewEphemeralBlobStorage(t), } + verifiedBlobs, rpcErr := blocker.Blobs(ctx, "123", nil) - assert.Equal(t, rpcErr == nil, true) + require.IsNil(t, rpcErr) require.Equal(t, 0, len(verifiedBlobs)) }) + t.Run("no blob at index", func(t *testing.T) { + setupDeneb(t) + blocker := &BeaconDbBlocker{ - ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: ðpb.Checkpoint{Root: blockRoot[:]}}, + ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: ðpb.Checkpoint{Root: denebBlockRoot[:]}}, GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{ Genesis: time.Now(), }, BeaconDB: db, - BlobStorage: bs, + BlobStorage: blobStorage, } - noBlobIndex := len(blobs) + 1 + + noBlobIndex := len(storedBlobSidecars) + 1 _, rpcErr := blocker.Blobs(ctx, "123", []int{0, noBlobIndex}) require.NotNil(t, rpcErr) - assert.Equal(t, core.ErrorReason(core.NotFound), rpcErr.Reason) + require.Equal(t, core.ErrorReason(core.NotFound), rpcErr.Reason) }) + t.Run("index too big", func(t *testing.T) { + setupDeneb(t) + blocker := &BeaconDbBlocker{ - ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: ðpb.Checkpoint{Root: blockRoot[:]}}, + ChainInfoFetcher: &mockChain.ChainService{FinalizedCheckPoint: ðpb.Checkpoint{Root: denebBlockRoot[:]}}, GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{ Genesis: time.Now(), }, BeaconDB: db, - BlobStorage: bs, + BlobStorage: blobStorage, } _, rpcErr := blocker.Blobs(ctx, "123", []int{0, math.MaxInt}) require.NotNil(t, rpcErr) - assert.Equal(t, core.ErrorReason(core.BadRequest), rpcErr.Reason) + require.Equal(t, core.ErrorReason(core.BadRequest), rpcErr.Reason) + }) + + t.Run("not enough stored data column sidecars", func(t *testing.T) { + setupFulu(t) + + _, dataColumnStorage := filesystem.NewEphemeralDataColumnStorageAndFs(t) + err = dataColumnStorage.Save(verifiedRoDataColumnSidecars[:fieldparams.CellsPerBlob-1]) + require.NoError(t, err) + + blocker := &BeaconDbBlocker{ + GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{ + Genesis: time.Now(), + }, + BeaconDB: db, + BlobStorage: blobStorage, + DataColumnStorage: dataColumnStorage, + } + + _, rpcErr := blocker.Blobs(ctx, hexutil.Encode(fuluBlockRoot[:]), nil) + require.NotNil(t, rpcErr) + require.Equal(t, core.ErrorReason(core.NotFound), rpcErr.Reason) + }) + + t.Run("reconstruction needed", func(t *testing.T) { + setupFulu(t) + + _, dataColumnStorage := filesystem.NewEphemeralDataColumnStorageAndFs(t) + err = dataColumnStorage.Save(verifiedRoDataColumnSidecars[1 : peerdas.MinimumColumnsCountToReconstruct()+1]) + require.NoError(t, err) + + blocker := &BeaconDbBlocker{ + GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{ + Genesis: time.Now(), + }, + BeaconDB: db, + BlobStorage: blobStorage, + DataColumnStorage: dataColumnStorage, + } + + retrievedVerifiedRoBlobs, rpcErr := blocker.Blobs(ctx, hexutil.Encode(fuluBlockRoot[:]), nil) + require.IsNil(t, rpcErr) + require.Equal(t, len(fuluBlobSidecars), len(retrievedVerifiedRoBlobs)) + + for i, retrievedVerifiedRoBlob := range retrievedVerifiedRoBlobs { + retrievedBlobSidecarPb := retrievedVerifiedRoBlob.BlobSidecar + initialBlobSidecarPb := fuluBlobSidecars[i].BlobSidecar + require.DeepSSZEqual(t, initialBlobSidecarPb, retrievedBlobSidecarPb) + } + }) + + t.Run("no reconstruction needed", func(t *testing.T) { + setupFulu(t) + + _, dataColumnStorage := filesystem.NewEphemeralDataColumnStorageAndFs(t) + err = dataColumnStorage.Save(verifiedRoDataColumnSidecars) + require.NoError(t, err) + + blocker := &BeaconDbBlocker{ + GenesisTimeFetcher: &testutil.MockGenesisTimeFetcher{ + Genesis: time.Now(), + }, + BeaconDB: db, + BlobStorage: blobStorage, + DataColumnStorage: dataColumnStorage, + } + + retrievedVerifiedRoBlobs, rpcErr := blocker.Blobs(ctx, hexutil.Encode(fuluBlockRoot[:]), nil) + require.IsNil(t, rpcErr) + require.Equal(t, len(fuluBlobSidecars), len(retrievedVerifiedRoBlobs)) + + for i, retrievedVerifiedRoBlob := range retrievedVerifiedRoBlobs { + retrievedBlobSidecarPb := retrievedVerifiedRoBlob.BlobSidecar + initialBlobSidecarPb := fuluBlobSidecars[i].BlobSidecar + require.DeepSSZEqual(t, initialBlobSidecarPb, retrievedBlobSidecarPb) + } }) } diff --git a/changelog/manu-peerdas-beacon-api.md b/changelog/manu-peerdas-beacon-api.md new file mode 100644 index 0000000000..ccd0e56228 --- /dev/null +++ b/changelog/manu-peerdas-beacon-api.md @@ -0,0 +1,2 @@ +### Added +- Implement beacon API blob sidecar enpoint for Fulu. diff --git a/testing/util/fulu.go b/testing/util/fulu.go index dc8ea44474..c463a9f85b 100644 --- a/testing/util/fulu.go +++ b/testing/util/fulu.go @@ -1,7 +1,6 @@ package util import ( - "encoding/binary" "math/big" "testing" @@ -110,14 +109,21 @@ func GenerateTestFuluBlockWithSidecars(t *testing.T, blobCount int, options ...F block.Block.ParentRoot = generator.parent[:] block.Block.ProposerIndex = generator.proposer - block.Block.Body.BlobKzgCommitments = make([][]byte, blobCount) - for i := range blobCount { - var commitment [fieldparams.KzgCommitmentSize]byte - binary.LittleEndian.PutUint16(commitment[:16], uint16(i)) - binary.LittleEndian.PutUint16(commitment[16:32], uint16(generator.slot)) - block.Block.Body.BlobKzgCommitments[i] = commitment[:] + blobs := make([]kzg.Blob, 0, generator.blobCount) + commitments := make([][]byte, 0, generator.blobCount) + + for i := range generator.blobCount { + blob := kzg.Blob{uint8(i)} + + commitment, err := kzg.BlobToKZGCommitment(&blob) + require.NoError(t, err) + + blobs = append(blobs, blob) + commitments = append(commitments, commitment[:]) } + block.Block.Body.BlobKzgCommitments = commitments + body, err := blocks.NewBeaconBlockBody(block.Block.Body) require.NoError(t, err) @@ -149,39 +155,30 @@ func GenerateTestFuluBlockWithSidecars(t *testing.T, blobCount int, options ...F root, err := block.Block.HashTreeRoot() require.NoError(t, err) - sbb, err := blocks.NewSignedBeaconBlock(block) + signedBeaconBlock, err := blocks.NewSignedBeaconBlock(block) require.NoError(t, err) - sh, err := sbb.Header() - require.NoError(t, err) - - blobs := make([]kzg.Blob, blobCount) - for i, commitment := range block.Block.Body.BlobKzgCommitments { - roSidecars := GenerateTestDenebBlobSidecar(t, root, sh, i, commitment, inclusion[i]) - blobs[i] = kzg.Blob(roSidecars.Blob) - } - cellsAndProofs := GenerateCellsAndProofs(t, blobs) - dataColumns, err := peerdas.DataColumnSidecars(sbb, cellsAndProofs) + sidecars, err := peerdas.DataColumnSidecars(signedBeaconBlock, cellsAndProofs) require.NoError(t, err) - roSidecars := make([]blocks.RODataColumn, 0, len(dataColumns)) - roVerifiedSidecars := make([]blocks.VerifiedRODataColumn, 0, len(dataColumns)) - for _, dataColumn := range dataColumns { - roSidecar, err := blocks.NewRODataColumnWithRoot(dataColumn, root) + roSidecars := make([]blocks.RODataColumn, 0, len(sidecars)) + verifiedRoSidecars := make([]blocks.VerifiedRODataColumn, 0, len(sidecars)) + for _, sidecar := range sidecars { + roSidecar, err := blocks.NewRODataColumnWithRoot(sidecar, root) require.NoError(t, err) roVerifiedSidecar := blocks.NewVerifiedRODataColumn(roSidecar) roSidecars = append(roSidecars, roSidecar) - roVerifiedSidecars = append(roVerifiedSidecars, roVerifiedSidecar) + verifiedRoSidecars = append(verifiedRoSidecars, roVerifiedSidecar) } - rob, err := blocks.NewROBlockWithRoot(sbb, root) + roBlock, err := blocks.NewROBlockWithRoot(signedBeaconBlock, root) require.NoError(t, err) - return rob, roSidecars, roVerifiedSidecars + return roBlock, roSidecars, verifiedRoSidecars } func GenerateCellsAndProofs(t testing.TB, blobs []kzg.Blob) []kzg.CellsAndProofs {