diff --git a/beacon-chain/db/iface/interface.go b/beacon-chain/db/iface/interface.go index 90f2500bf7..da97ee87b8 100644 --- a/beacon-chain/db/iface/interface.go +++ b/beacon-chain/db/iface/interface.go @@ -22,6 +22,8 @@ type ReadOnlyDatabase interface { Block(ctx context.Context, blockRoot [32]byte) (*eth.SignedBeaconBlock, error) Blocks(ctx context.Context, f *filters.QueryFilter) ([]*eth.SignedBeaconBlock, [][32]byte, error) BlockRoots(ctx context.Context, f *filters.QueryFilter) ([][32]byte, error) + BlocksBySlot(ctx context.Context, slot uint64) (bool, []*eth.SignedBeaconBlock, error) + BlockRootsBySlot(ctx context.Context, slot uint64) (bool, [][32]byte, error) HasBlock(ctx context.Context, blockRoot [32]byte) bool GenesisBlock(ctx context.Context) (*eth.SignedBeaconBlock, error) IsFinalizedBlock(ctx context.Context, blockRoot [32]byte) bool diff --git a/beacon-chain/db/kafka/passthrough.go b/beacon-chain/db/kafka/passthrough.go index 0879f94b37..f21f0b1e9a 100644 --- a/beacon-chain/db/kafka/passthrough.go +++ b/beacon-chain/db/kafka/passthrough.go @@ -46,6 +46,16 @@ func (e Exporter) BlockRoots(ctx context.Context, f *filters.QueryFilter) ([][32 return e.db.BlockRoots(ctx, f) } +// BlocksBySlot -- passthrough. +func (e Exporter) BlocksBySlot(ctx context.Context, slot uint64) (bool, []*eth.SignedBeaconBlock, error) { + return e.db.BlocksBySlot(ctx, slot) +} + +// BlockRootsBySlot -- passthrough. +func (e Exporter) BlockRootsBySlot(ctx context.Context, slot uint64) (bool, [][32]byte, error) { + return e.db.BlockRootsBySlot(ctx, slot) +} + // HasBlock -- passthrough. func (e Exporter) HasBlock(ctx context.Context, blockRoot [32]byte) bool { return e.db.HasBlock(ctx, blockRoot) diff --git a/beacon-chain/db/kv/blocks.go b/beacon-chain/db/kv/blocks.go index 140cdeab96..f93883d4d7 100644 --- a/beacon-chain/db/kv/blocks.go +++ b/beacon-chain/db/kv/blocks.go @@ -134,6 +134,55 @@ func (s *Store) HasBlock(ctx context.Context, blockRoot [32]byte) bool { return exists } +// BlocksBySlot retrieves a list of beacon blocks and its respective roots by slot. +func (s *Store) BlocksBySlot(ctx context.Context, slot uint64) (bool, []*ethpb.SignedBeaconBlock, error) { + ctx, span := trace.StartSpan(ctx, "BeaconDB.BlocksBySlot") + defer span.End() + blocks := make([]*ethpb.SignedBeaconBlock, 0) + + err := s.db.View(func(tx *bolt.Tx) error { + bkt := tx.Bucket(blocksBucket) + + keys, err := getBlockRootsBySlot(ctx, tx, slot) + if err != nil { + return err + } + + for i := 0; i < len(keys); i++ { + encoded := bkt.Get(keys[i]) + block := ðpb.SignedBeaconBlock{} + if err := decode(ctx, encoded, block); err != nil { + return err + } + blocks = append(blocks, block) + } + return nil + }) + return len(blocks) > 0, blocks, err +} + +// BlockRootsBySlot retrieves a list of beacon block roots by slot +func (s *Store) BlockRootsBySlot(ctx context.Context, slot uint64) (bool, [][32]byte, error) { + ctx, span := trace.StartSpan(ctx, "BeaconDB.BlockRootsBySlot") + defer span.End() + blockRoots := make([][32]byte, 0) + err := s.db.View(func(tx *bolt.Tx) error { + keys, err := getBlockRootsBySlot(ctx, tx, slot) + if err != nil { + return err + } + + for i := 0; i < len(keys); i++ { + blockRoots = append(blockRoots, bytesutil.ToBytes32(keys[i])) + } + return nil + }) + if err != nil { + return false, nil, errors.Wrap(err, "could not retrieve block roots by slot") + } + return len(blockRoots) > 0, blockRoots, nil +} + // deleteBlock by block root. func (s *Store) deleteBlock(ctx context.Context, blockRoot [32]byte) error { ctx, span := trace.StartSpan(ctx, "BeaconDB.deleteBlock") @@ -451,6 +500,24 @@ func fetchBlockRootsBySlotRange( return roots, nil } +// getBlockRootsByFilter retrieves the block roots by slot +func getBlockRootsBySlot(ctx context.Context, tx *bolt.Tx, slot uint64) ([][]byte, error) { + ctx, span := trace.StartSpan(ctx, "BeaconDB.getBlockRootsBySlot") + defer span.End() + + roots := make([][]byte, 0) + bkt := tx.Bucket(blockSlotIndicesBucket) + key := bytesutil.Uint64ToBytesBigEndian(slot) + c := bkt.Cursor() + k, v := c.Seek(key) + if k != nil && bytes.Equal(k, key) { + for i := 0; i < len(v); i += 32 { + roots = append(roots, v[i:i+32]) + } + } + return roots, nil +} + // createBlockIndicesFromBlock takes in a beacon block and returns // a map of bolt DB index buckets corresponding to each particular key for indices for // data, such as (shard indices bucket -> shard 5). diff --git a/beacon-chain/db/kv/blocks_test.go b/beacon-chain/db/kv/blocks_test.go index fb3c5b37cd..150dcc0b60 100644 --- a/beacon-chain/db/kv/blocks_test.go +++ b/beacon-chain/db/kv/blocks_test.go @@ -450,3 +450,54 @@ func TestStore_SaveBlocks_HasRootsMatched(t *testing.T) { assert.Equal(t, roots[i], rt, "mismatch of block roots") } } + +func TestStore_BlocksBySlot_BlockRootsBySlot(t *testing.T) { + db := setupDB(t) + ctx := context.Background() + + b1 := testutil.NewBeaconBlock() + b1.Block.Slot = 20 + require.NoError(t, db.SaveBlock(ctx, b1)) + b2 := testutil.NewBeaconBlock() + b2.Block.Slot = 100 + b2.Block.ParentRoot = bytesutil.PadTo([]byte("parent1"), 32) + require.NoError(t, db.SaveBlock(ctx, b2)) + b3 := testutil.NewBeaconBlock() + b3.Block.Slot = 100 + b3.Block.ParentRoot = bytesutil.PadTo([]byte("parent2"), 32) + require.NoError(t, db.SaveBlock(ctx, b3)) + + r1, err := b1.Block.HashTreeRoot() + require.NoError(t, err) + r2, err := b2.Block.HashTreeRoot() + require.NoError(t, err) + r3, err := b3.Block.HashTreeRoot() + require.NoError(t, err) + + hasBlocks, retrievedBlocks, err := db.BlocksBySlot(ctx, 1) + require.NoError(t, err) + assert.Equal(t, 0, len(retrievedBlocks), "Unexpected number of blocks received, expected none") + assert.Equal(t, false, hasBlocks, "Expected no blocks") + hasBlocks, retrievedBlocks, err = db.BlocksBySlot(ctx, 20) + require.NoError(t, err) + assert.Equal(t, true, proto.Equal(b1, retrievedBlocks[0]), "Wanted: %v, received: %v", b1, retrievedBlocks[0]) + assert.Equal(t, true, hasBlocks, "Expected to have blocks") + hasBlocks, retrievedBlocks, err = db.BlocksBySlot(ctx, 100) + require.NoError(t, err) + assert.Equal(t, true, proto.Equal(b2, retrievedBlocks[0]), "Wanted: %v, received: %v", b2, retrievedBlocks[0]) + assert.Equal(t, true, proto.Equal(b3, retrievedBlocks[1]), "Wanted: %v, received: %v", b3, retrievedBlocks[1]) + assert.Equal(t, true, hasBlocks, "Expected to have blocks") + + hasBlockRoots, retrievedBlockRoots, err := db.BlockRootsBySlot(ctx, 1) + require.NoError(t, err) + assert.DeepEqual(t, [][32]byte{}, retrievedBlockRoots) + assert.Equal(t, false, hasBlockRoots, "Expected no block roots") + hasBlockRoots, retrievedBlockRoots, err = db.BlockRootsBySlot(ctx, 20) + require.NoError(t, err) + assert.DeepEqual(t, [][32]byte{r1}, retrievedBlockRoots) + assert.Equal(t, true, hasBlockRoots, "Expected no block roots") + hasBlockRoots, retrievedBlockRoots, err = db.BlockRootsBySlot(ctx, 100) + require.NoError(t, err) + assert.DeepEqual(t, [][32]byte{r2, r3}, retrievedBlockRoots) + assert.Equal(t, true, hasBlockRoots, "Expected no block roots") +} diff --git a/beacon-chain/rpc/beacon/blocks.go b/beacon-chain/rpc/beacon/blocks.go index 587cd61e8a..2cea0ecb37 100644 --- a/beacon-chain/rpc/beacon/blocks.go +++ b/beacon-chain/rpc/beacon/blocks.go @@ -100,13 +100,11 @@ func (bs *Server) ListBlocks( }, nil case *ethpb.ListBlocksRequest_Slot: - blks, _, err := bs.BeaconDB.Blocks(ctx, filters.NewFilter().SetStartSlot(q.Slot).SetEndSlot(q.Slot)) + hasBlocks, blks, err := bs.BeaconDB.BlocksBySlot(ctx, q.Slot) if err != nil { return nil, status.Errorf(codes.Internal, "Could not retrieve blocks for slot %d: %v", q.Slot, err) } - - numBlks := len(blks) - if numBlks == 0 { + if !hasBlocks { return ðpb.ListBlocksResponse{ BlockContainers: make([]*ethpb.BeaconBlockContainer, 0), TotalSize: 0, @@ -114,6 +112,8 @@ func (bs *Server) ListBlocks( }, nil } + numBlks := len(blks) + start, end, nextPageToken, err := pagination.StartAndEndPage(req.PageToken, int(req.PageSize), numBlks) if err != nil { return nil, status.Errorf(codes.Internal, "Could not paginate blocks: %v", err) diff --git a/beacon-chain/rpc/beacon/validators.go b/beacon-chain/rpc/beacon/validators.go index 5ea2b27860..11649b8f8c 100644 --- a/beacon-chain/rpc/beacon/validators.go +++ b/beacon-chain/rpc/beacon/validators.go @@ -13,7 +13,6 @@ import ( "github.com/prysmaticlabs/prysm/beacon-chain/core/helpers" "github.com/prysmaticlabs/prysm/beacon-chain/core/state" "github.com/prysmaticlabs/prysm/beacon-chain/core/validators" - "github.com/prysmaticlabs/prysm/beacon-chain/db/filters" statetrie "github.com/prysmaticlabs/prysm/beacon-chain/state" "github.com/prysmaticlabs/prysm/shared/bytesutil" "github.com/prysmaticlabs/prysm/shared/cmd" @@ -860,12 +859,11 @@ func (bs *Server) GetIndividualVotes( // if the input slot has a skip block, false is returned, // if the input slot has more than one block, an error is returned. func (bs *Server) isSlotCanonical(ctx context.Context, slot uint64) (bool, error) { - filter := filters.NewFilter().SetStartSlot(slot).SetEndSlot(slot) - roots, err := bs.BeaconDB.BlockRoots(ctx, filter) + hasBlockRoots, roots, err := bs.BeaconDB.BlockRootsBySlot(ctx, slot) if err != nil { return false, err } - if len(roots) == 0 { + if !hasBlockRoots { return false, nil } if len(roots) != 1 { diff --git a/beacon-chain/rpc/beacon/validators_test.go b/beacon-chain/rpc/beacon/validators_test.go index e195954f0e..1e9dde5681 100644 --- a/beacon-chain/rpc/beacon/validators_test.go +++ b/beacon-chain/rpc/beacon/validators_test.go @@ -1598,6 +1598,11 @@ func TestServer_GetValidatorParticipation_OrphanedUntilGenesis(t *testing.T) { GenesisTimeFetcher: &mock.ChainService{ Genesis: timeutils.Now().Add(time.Duration(-1*int64(params.BeaconConfig().SlotsPerEpoch*params.BeaconConfig().SecondsPerSlot)) * time.Second), }, + CanonicalFetcher: &mock.ChainService{ + CanonicalRoots: map[[32]byte]bool{ + bRoot: true, + }, + }, FinalizationFetcher: &mock.ChainService{FinalizedCheckPoint: ðpb.Checkpoint{Epoch: 100}}, } diff --git a/beacon-chain/rpc/beaconv1/blocks.go b/beacon-chain/rpc/beaconv1/blocks.go index c2ad4f5535..38a50cb060 100644 --- a/beacon-chain/rpc/beaconv1/blocks.go +++ b/beacon-chain/rpc/beaconv1/blocks.go @@ -69,10 +69,14 @@ func (bs *Server) ListBlockHeaders(ctx context.Context, req *ethpb.BlockHeadersR return nil, status.Errorf(codes.Internal, "Could not retrieve blocks: %v", err) } } else { - blks, blkRoots, err = bs.BeaconDB.Blocks(ctx, filters.NewFilter().SetStartSlot(req.Slot).SetEndSlot(req.Slot)) + _, blks, err = bs.BeaconDB.BlocksBySlot(ctx, req.Slot) if err != nil { return nil, status.Errorf(codes.Internal, "Could not retrieve blocks for slot %d: %v", req.Slot, err) } + _, blkRoots, err = bs.BeaconDB.BlockRootsBySlot(ctx, req.Slot) + if err != nil { + return nil, status.Errorf(codes.Internal, "Could not retrieve block roots for slot %d: %v", req.Slot, err) + } } if blks == nil { return nil, status.Error(codes.NotFound, "Could not find requested blocks") @@ -213,17 +217,16 @@ func (bs *Server) GetBlockRoot(ctx context.Context, req *ethpb.BlockRequest) (*e if err != nil { return nil, status.Errorf(codes.Internal, "could not decode block id: %v", err) } - roots, err := bs.BeaconDB.BlockRoots(ctx, filters.NewFilter().SetStartSlot(slot).SetEndSlot(slot)) + hasRoots, roots, err := bs.BeaconDB.BlockRootsBySlot(ctx, slot) if err != nil { return nil, status.Errorf(codes.Internal, "Could not retrieve blocks for slot %d: %v", slot, err) } - numRoots := len(roots) - if numRoots == 0 { + if !hasRoots { return nil, status.Error(codes.NotFound, "Could not find any blocks with given slot") } root = roots[0][:] - if numRoots == 1 { + if len(roots) == 1 { break } for _, blockRoot := range roots { @@ -297,10 +300,14 @@ func (bs *Server) blockFromBlockID(ctx context.Context, blockId []byte) (*ethpb_ if err != nil { return nil, errors.Wrap(err, "could not decode block id") } - blks, roots, err := bs.BeaconDB.Blocks(ctx, filters.NewFilter().SetStartSlot(slot).SetEndSlot(slot)) + _, blks, err := bs.BeaconDB.BlocksBySlot(ctx, slot) if err != nil { return nil, errors.Wrapf(err, "could not retrieve blocks for slot %d", slot) } + _, roots, err := bs.BeaconDB.BlockRootsBySlot(ctx, slot) + if err != nil { + return nil, errors.Wrapf(err, "could not retrieve block roots for slot %d", slot) + } numBlks := len(blks) if numBlks == 0 { diff --git a/tools/blocktree/main.go b/tools/blocktree/main.go index d6e9afa095..d3cdd42a12 100644 --- a/tools/blocktree/main.go +++ b/tools/blocktree/main.go @@ -69,8 +69,7 @@ func main() { // If the state is not available, roll back for state == nil { slot-- - filter := filters.NewFilter().SetStartSlot(slot).SetEndSlot(slot) - rts, err := db.BlockRoots(context.Background(), filter) + _, rts, err := db.BlockRootsBySlot(context.Background(), slot) if err != nil { panic(err) } diff --git a/tools/extractor/BUILD.bazel b/tools/extractor/BUILD.bazel index a90e8e31d8..15e73a3cdc 100644 --- a/tools/extractor/BUILD.bazel +++ b/tools/extractor/BUILD.bazel @@ -9,7 +9,6 @@ go_library( deps = [ "//beacon-chain/core/state/interop:go_default_library", "//beacon-chain/db:go_default_library", - "//beacon-chain/db/filters:go_default_library", "//shared/featureconfig:go_default_library", ], ) diff --git a/tools/extractor/main.go b/tools/extractor/main.go index 10ba6648c3..0c5ce71b4f 100644 --- a/tools/extractor/main.go +++ b/tools/extractor/main.go @@ -7,7 +7,6 @@ import ( "github.com/prysmaticlabs/prysm/beacon-chain/core/state/interop" "github.com/prysmaticlabs/prysm/beacon-chain/db" - "github.com/prysmaticlabs/prysm/beacon-chain/db/filters" "github.com/prysmaticlabs/prysm/shared/featureconfig" ) @@ -29,7 +28,7 @@ func main() { } ctx := context.Background() slot := uint64(*state) - roots, err := d.BlockRoots(ctx, filters.NewFilter().SetStartSlot(slot).SetEndSlot(slot)) + _, roots, err := d.BlockRootsBySlot(ctx, slot) if err != nil { panic(err) }