From 2e9c3895f479446534be2f535b6cb229b1ba8707 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Wed, 11 Dec 2019 20:27:19 -0600 Subject: [PATCH] Bring Back Epoch Filtering for ListBlocks API (#4262) * bring back the epochs! * fix up * Merge refs/heads/master into bring-back-epoch-filter * add in patch * Merge branch 'bring-back-epoch-filter' of github.com:prysmaticlabs/prysm into bring-back-epoch-filter * import spacing * lint * build * gaz * Merge refs/heads/master into bring-back-epoch-filter * gaz * Merge branch 'bring-back-epoch-filter' of github.com:prysmaticlabs/prysm into bring-back-epoch-filter * move back perf * update ethapis * fix build * Merge refs/heads/master into bring-back-epoch-filter --- WORKSPACE | 2 +- beacon-chain/rpc/beacon/attestations.go | 2 +- beacon-chain/rpc/beacon/blocks.go | 41 ++++++++ beacon-chain/rpc/beacon/blocks_test.go | 85 ++++++++++++---- beacon-chain/rpc/validator/server.go | 98 +++++++++---------- ...thub_prysmaticlabs_ethereumapis-tags.patch | 42 ++++---- 6 files changed, 184 insertions(+), 86 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index e373cc87d1..67e4d2a501 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1249,7 +1249,7 @@ go_repository( go_repository( name = "com_github_prysmaticlabs_ethereumapis", - commit = "5f21afe48ab14bd0d5311cf5d33853a3e23d2fda", + commit = "2a889fed542ad00e4bd3caf723f871b6a4eff63d", importpath = "github.com/prysmaticlabs/ethereumapis", patch_args = ["-p1"], patches = [ diff --git a/beacon-chain/rpc/beacon/attestations.go b/beacon-chain/rpc/beacon/attestations.go index ad86260bab..36d59bc1d8 100644 --- a/beacon-chain/rpc/beacon/attestations.go +++ b/beacon-chain/rpc/beacon/attestations.go @@ -116,7 +116,7 @@ func (bs *Server) ListAttestations( // StreamAttestations to clients every single time a new attestation is received. // TODO(#4184): Implement. func (bs *Server) StreamAttestations( - _ *ethpb.ListAttestationsRequest, _ ethpb.BeaconChain_StreamAttestationsServer, + _ *ptypes.Empty, _ ethpb.BeaconChain_StreamAttestationsServer, ) error { return status.Error(codes.Unimplemented, "Not yet implemented") } diff --git a/beacon-chain/rpc/beacon/blocks.go b/beacon-chain/rpc/beacon/blocks.go index 14393505c3..53ba07a2a9 100644 --- a/beacon-chain/rpc/beacon/blocks.go +++ b/beacon-chain/rpc/beacon/blocks.go @@ -33,6 +33,47 @@ func (bs *Server) ListBlocks( } switch q := req.QueryFilter.(type) { + case *ethpb.ListBlocksRequest_Epoch: + startSlot := q.Epoch * params.BeaconConfig().SlotsPerEpoch + endSlot := startSlot + params.BeaconConfig().SlotsPerEpoch - 1 + + blks, err := bs.BeaconDB.Blocks(ctx, filters.NewFilter().SetStartSlot(startSlot).SetEndSlot(endSlot)) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to get blocks: %v", err) + } + + numBlks := len(blks) + if numBlks == 0 { + return ðpb.ListBlocksResponse{ + BlockContainers: make([]*ethpb.BeaconBlockContainer, 0), + TotalSize: 0, + NextPageToken: strconv.Itoa(0), + }, nil + } + + start, end, nextPageToken, err := pagination.StartAndEndPage(req.PageToken, int(req.PageSize), numBlks) + if err != nil { + return nil, status.Errorf(codes.Internal, "Could not paginate blocks: %v", err) + } + + returnedBlks := blks[start:end] + containers := make([]*ethpb.BeaconBlockContainer, len(returnedBlks)) + for i, b := range returnedBlks { + root, err := ssz.SigningRoot(b) + if err != nil { + return nil, err + } + containers[i] = ðpb.BeaconBlockContainer{ + Block: b, + BlockRoot: root[:], + } + } + + return ðpb.ListBlocksResponse{ + BlockContainers: containers, + TotalSize: int32(numBlks), + NextPageToken: nextPageToken, + }, nil case *ethpb.ListBlocksRequest_Root: blk, err := bs.BeaconDB.Block(ctx, bytesutil.ToBytes32(q.Root)) if err != nil { diff --git a/beacon-chain/rpc/beacon/blocks_test.go b/beacon-chain/rpc/beacon/blocks_test.go index 142e92a141..a79400b869 100644 --- a/beacon-chain/rpc/beacon/blocks_test.go +++ b/beacon-chain/rpc/beacon/blocks_test.go @@ -143,7 +143,7 @@ func TestServer_ListBlocks_Pagination(t *testing.T) { defer dbTest.TeardownDB(t, db) ctx := context.Background() - count := uint64(6) + count := uint64(100) blks := make([]*ethpb.BeaconBlock, count) blkContainers := make([]*ethpb.BeaconBlockContainer, count) for i := uint64(0); i < count; i++ { @@ -165,25 +165,76 @@ func TestServer_ListBlocks_Pagination(t *testing.T) { BeaconDB: db, } - req := ðpb.ListBlocksRequest{ - PageToken: strconv.Itoa(0), - QueryFilter: ðpb.ListBlocksRequest_Slot{Slot: 5}, - PageSize: 3, - } - want := ðpb.ListBlocksResponse{ - BlockContainers: []*ethpb.BeaconBlockContainer{{ - Block: ðpb.BeaconBlock{Slot: 5}, - BlockRoot: blkContainers[5].BlockRoot, - }}, - NextPageToken: "", - TotalSize: 1, - } - res, err := bs.ListBlocks(ctx, req) + root6, err := ssz.SigningRoot(ðpb.BeaconBlock{Slot: 6}) if err != nil { t.Fatal(err) } - if !proto.Equal(res, want) { - t.Errorf("Incorrect blocks response, wanted %v, received %v", want, res) + + tests := []struct { + req *ethpb.ListBlocksRequest + res *ethpb.ListBlocksResponse + }{ + {req: ðpb.ListBlocksRequest{ + PageToken: strconv.Itoa(0), + QueryFilter: ðpb.ListBlocksRequest_Slot{Slot: 5}, + PageSize: 3}, + res: ðpb.ListBlocksResponse{ + BlockContainers: []*ethpb.BeaconBlockContainer{{Block: ðpb.BeaconBlock{Slot: 5}, BlockRoot: blkContainers[5].BlockRoot}}, + NextPageToken: "", + TotalSize: 1}}, + {req: ðpb.ListBlocksRequest{ + PageToken: strconv.Itoa(0), + QueryFilter: ðpb.ListBlocksRequest_Root{Root: root6[:]}, + PageSize: 3}, + res: ðpb.ListBlocksResponse{ + BlockContainers: []*ethpb.BeaconBlockContainer{{Block: ðpb.BeaconBlock{Slot: 6}, BlockRoot: blkContainers[6].BlockRoot}}, + TotalSize: 1}}, + {req: ðpb.ListBlocksRequest{QueryFilter: ðpb.ListBlocksRequest_Root{Root: root6[:]}}, + res: ðpb.ListBlocksResponse{ + BlockContainers: []*ethpb.BeaconBlockContainer{{Block: ðpb.BeaconBlock{Slot: 6}, BlockRoot: blkContainers[6].BlockRoot}}, + TotalSize: 1}}, + {req: ðpb.ListBlocksRequest{ + PageToken: strconv.Itoa(0), + QueryFilter: ðpb.ListBlocksRequest_Epoch{Epoch: 0}, + PageSize: 100}, + res: ðpb.ListBlocksResponse{ + BlockContainers: blkContainers[0:params.BeaconConfig().SlotsPerEpoch], + NextPageToken: "", + TotalSize: int32(params.BeaconConfig().SlotsPerEpoch)}}, + {req: ðpb.ListBlocksRequest{ + PageToken: strconv.Itoa(1), + QueryFilter: ðpb.ListBlocksRequest_Epoch{Epoch: 5}, + PageSize: 3}, + res: ðpb.ListBlocksResponse{ + BlockContainers: blkContainers[43:46], + NextPageToken: "2", + TotalSize: int32(params.BeaconConfig().SlotsPerEpoch)}}, + {req: ðpb.ListBlocksRequest{ + PageToken: strconv.Itoa(1), + QueryFilter: ðpb.ListBlocksRequest_Epoch{Epoch: 11}, + PageSize: 7}, + res: ðpb.ListBlocksResponse{ + BlockContainers: blkContainers[95:96], + NextPageToken: "", + TotalSize: int32(params.BeaconConfig().SlotsPerEpoch)}}, + {req: ðpb.ListBlocksRequest{ + PageToken: strconv.Itoa(0), + QueryFilter: ðpb.ListBlocksRequest_Epoch{Epoch: 12}, + PageSize: 4}, + res: ðpb.ListBlocksResponse{ + BlockContainers: blkContainers[96:100], + NextPageToken: "1", + TotalSize: int32(params.BeaconConfig().SlotsPerEpoch / 2)}}, + } + + for _, test := range tests { + res, err := bs.ListBlocks(ctx, test.req) + if err != nil { + t.Fatal(err) + } + if !proto.Equal(res, test.res) { + t.Errorf("Incorrect blocks response, wanted %v, received %v", test.res, res) + } } } diff --git a/beacon-chain/rpc/validator/server.go b/beacon-chain/rpc/validator/server.go index 710d067349..3ac1808472 100644 --- a/beacon-chain/rpc/validator/server.go +++ b/beacon-chain/rpc/validator/server.go @@ -6,6 +6,11 @@ import ( ptypes "github.com/gogo/protobuf/types" ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1" + pbp2p "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" + pb "github.com/prysmaticlabs/prysm/proto/beacon/rpc/v1" + "github.com/sirupsen/logrus" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/prysmaticlabs/prysm/beacon-chain/blockchain" "github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache" "github.com/prysmaticlabs/prysm/beacon-chain/core/feed" @@ -15,12 +20,7 @@ import ( "github.com/prysmaticlabs/prysm/beacon-chain/db" "github.com/prysmaticlabs/prysm/beacon-chain/powchain" "github.com/prysmaticlabs/prysm/beacon-chain/sync" - pbp2p "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" - pb "github.com/prysmaticlabs/prysm/proto/beacon/rpc/v1" "github.com/prysmaticlabs/prysm/shared/bytesutil" - "github.com/sirupsen/logrus" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) var log logrus.FieldLogger @@ -102,6 +102,49 @@ func (vs *Server) ValidatorIndex(ctx context.Context, req *pb.ValidatorIndexRequ return &pb.ValidatorIndexResponse{Index: index}, nil } +// ExitedValidators queries validator statuses for a give list of validators +// and returns a filtered list of validator keys that are exited. +func (vs *Server) ExitedValidators( + ctx context.Context, + req *pb.ExitedValidatorsRequest) (*pb.ExitedValidatorsResponse, error) { + + _, statuses, err := vs.multipleValidatorStatus(ctx, req.PublicKeys) + if err != nil { + return nil, status.Errorf(codes.Internal, "Could not retrieve validator statuses: %v", err) + } + + exitedKeys := make([][]byte, 0) + for _, st := range statuses { + s := st.Status.Status + if s == pb.ValidatorStatus_EXITED || + s == pb.ValidatorStatus_EXITED_SLASHED || + s == pb.ValidatorStatus_INITIATED_EXIT { + exitedKeys = append(exitedKeys, st.PublicKey) + } + } + + resp := &pb.ExitedValidatorsResponse{ + PublicKeys: exitedKeys, + } + + return resp, nil +} + +// DomainData fetches the current domain version information from the beacon state. +func (vs *Server) DomainData(ctx context.Context, request *pb.DomainRequest) (*pb.DomainResponse, error) { + fork := vs.ForkFetcher.CurrentFork() + dv := helpers.Domain(fork, request.Epoch, request.Domain) + return &pb.DomainResponse{ + SignatureDomain: dv, + }, nil +} + +// CanonicalHead of the current beacon chain. This method is requested on-demand +// by a validator when it is their time to propose or attest. +func (vs *Server) CanonicalHead(ctx context.Context, req *ptypes.Empty) (*ethpb.BeaconBlock, error) { + return vs.HeadFetcher.HeadBlock(), nil +} + // ValidatorPerformance reports the validator's latest balance along with other important metrics on // rewards and penalties throughout its lifecycle in the beacon chain. func (vs *Server) ValidatorPerformance( @@ -148,53 +191,10 @@ func (vs *Server) ValidatorPerformance( AverageActiveValidatorBalance: avgBalance, MissingValidators: missingValidators, TotalValidators: uint64(len(headState.Validators)), - TotalActiveValidators: uint64(activeCount), + TotalActiveValidators: activeCount, }, nil } -// ExitedValidators queries validator statuses for a give list of validators -// and returns a filtered list of validator keys that are exited. -func (vs *Server) ExitedValidators( - ctx context.Context, - req *pb.ExitedValidatorsRequest) (*pb.ExitedValidatorsResponse, error) { - - _, statuses, err := vs.multipleValidatorStatus(ctx, req.PublicKeys) - if err != nil { - return nil, status.Errorf(codes.Internal, "Could not retrieve validator statuses: %v", err) - } - - exitedKeys := make([][]byte, 0) - for _, st := range statuses { - s := st.Status.Status - if s == pb.ValidatorStatus_EXITED || - s == pb.ValidatorStatus_EXITED_SLASHED || - s == pb.ValidatorStatus_INITIATED_EXIT { - exitedKeys = append(exitedKeys, st.PublicKey) - } - } - - resp := &pb.ExitedValidatorsResponse{ - PublicKeys: exitedKeys, - } - - return resp, nil -} - -// DomainData fetches the current domain version information from the beacon state. -func (vs *Server) DomainData(ctx context.Context, request *pb.DomainRequest) (*pb.DomainResponse, error) { - fork := vs.ForkFetcher.CurrentFork() - dv := helpers.Domain(fork, request.Epoch, request.Domain) - return &pb.DomainResponse{ - SignatureDomain: dv, - }, nil -} - -// CanonicalHead of the current beacon chain. This method is requested on-demand -// by a validator when it is their time to propose or attest. -func (vs *Server) CanonicalHead(ctx context.Context, req *ptypes.Empty) (*ethpb.BeaconBlock, error) { - return vs.HeadFetcher.HeadBlock(), nil -} - // WaitForChainStart queries the logs of the Deposit Contract in order to verify the beacon chain // has started its runtime and validators begin their responsibilities. If it has not, it then // subscribes to an event stream triggered by the powchain service whenever the ChainStart log does diff --git a/third_party/com_github_prysmaticlabs_ethereumapis-tags.patch b/third_party/com_github_prysmaticlabs_ethereumapis-tags.patch index 68a8e4ac2a..782eaeb6e6 100644 --- a/third_party/com_github_prysmaticlabs_ethereumapis-tags.patch +++ b/third_party/com_github_prysmaticlabs_ethereumapis-tags.patch @@ -255,7 +255,7 @@ index 69a148a..1b6ac18 100644 + bytes signature = 4 [(gogoproto.moretags) = "ssz-size:\"96\""]; } diff --git a/eth/v1alpha1/beacon_chain.proto b/eth/v1alpha1/beacon_chain.proto -index 5389a4e..2029f0e 100644 +index b4d1638..4bf7ee9 100644 --- a/eth/v1alpha1/beacon_chain.proto +++ b/eth/v1alpha1/beacon_chain.proto @@ -15,6 +15,7 @@ syntax = "proto3"; @@ -279,7 +279,7 @@ index 5389a4e..2029f0e 100644 // This includes the head block slot and root as well as information about // the most recent finalized and justified slots. rpc StreamChainHead(google.protobuf.Empty) returns (stream ChainHead) { -@@ -290,7 +291,7 @@ message ChainHead { +@@ -299,7 +300,7 @@ message ChainHead { uint64 head_epoch = 2; // 32 byte merkle tree root of the canonical head block in the beacon node. @@ -288,7 +288,7 @@ index 5389a4e..2029f0e 100644 // Most recent slot that contains the finalized block. uint64 finalized_slot = 4; -@@ -299,7 +300,7 @@ message ChainHead { +@@ -308,7 +309,7 @@ message ChainHead { uint64 finalized_epoch = 5; // Most recent 32 byte finalized block root. @@ -297,7 +297,7 @@ index 5389a4e..2029f0e 100644 // Most recent slot that contains the justified block. uint64 justified_slot = 7; -@@ -308,7 +309,7 @@ message ChainHead { +@@ -317,7 +318,7 @@ message ChainHead { uint64 justified_epoch = 8; // Most recent 32 byte justified block root. @@ -306,7 +306,7 @@ index 5389a4e..2029f0e 100644 // Most recent slot that contains the previous justified block. uint64 previous_justified_slot = 10; -@@ -317,7 +318,7 @@ message ChainHead { +@@ -326,7 +327,7 @@ message ChainHead { uint64 previous_justified_epoch = 11; // Previous 32 byte justified block root. @@ -315,7 +315,7 @@ index 5389a4e..2029f0e 100644 } message ListCommitteesRequest { -@@ -362,7 +363,7 @@ message ListValidatorBalancesRequest { +@@ -371,7 +372,7 @@ message ListValidatorBalancesRequest { // Validator 48 byte BLS public keys to filter validators for the given // epoch. @@ -324,7 +324,7 @@ index 5389a4e..2029f0e 100644 // Validator indices to filter validators for the given epoch. repeated uint64 indices = 4; -@@ -383,7 +384,7 @@ message ValidatorBalances { +@@ -392,7 +393,7 @@ message ValidatorBalances { message Balance { // Validator's 48 byte BLS public key. @@ -333,7 +333,7 @@ index 5389a4e..2029f0e 100644 // Validator's index in the validator set. uint64 index = 2; -@@ -432,7 +433,7 @@ message GetValidatorRequest { +@@ -441,7 +442,7 @@ message GetValidatorRequest { uint64 index = 1; // 48 byte validator public key. @@ -342,7 +342,7 @@ index 5389a4e..2029f0e 100644 } } -@@ -469,17 +470,17 @@ message ActiveSetChanges { +@@ -478,17 +479,17 @@ message ActiveSetChanges { uint64 epoch = 1; // 48 byte validator public keys that have been activated in this epoch. @@ -364,7 +364,7 @@ index 5389a4e..2029f0e 100644 } message ValidatorQueue { -@@ -489,11 +490,11 @@ message ValidatorQueue { +@@ -498,11 +499,11 @@ message ValidatorQueue { // Ordered list of 48 byte public keys awaiting activation. 0th index is the // next key to be processed. @@ -378,7 +378,7 @@ index 5389a4e..2029f0e 100644 } message ListValidatorAssignmentsRequest { -@@ -505,7 +506,7 @@ message ListValidatorAssignmentsRequest { +@@ -514,7 +515,7 @@ message ListValidatorAssignmentsRequest { bool genesis = 2; } // 48 byte validator public keys to filter assignments for the given epoch. @@ -387,7 +387,7 @@ index 5389a4e..2029f0e 100644 // Validator indicies to filter assignments for the given epoch. repeated uint64 indices = 4; -@@ -540,7 +541,7 @@ message ValidatorAssignments { +@@ -549,7 +550,7 @@ message ValidatorAssignments { uint64 proposer_slot = 4; // 48 byte BLS public key. @@ -397,7 +397,7 @@ index 5389a4e..2029f0e 100644 // The epoch for which this set of validator assignments is valid. diff --git a/eth/v1alpha1/validator.proto b/eth/v1alpha1/validator.proto -index 9f07458..e30cd3f 100644 +index 28a4f31..31e5ec0 100644 --- a/eth/v1alpha1/validator.proto +++ b/eth/v1alpha1/validator.proto @@ -15,6 +15,7 @@ syntax = "proto3"; @@ -408,7 +408,7 @@ index 9f07458..e30cd3f 100644 import "google/api/annotations.proto"; import "google/protobuf/empty.proto"; import "eth/v1alpha1/beacon_block.proto"; -@@ -106,14 +107,14 @@ message DutiesRequest { +@@ -257,14 +258,14 @@ message DutiesRequest { // Epoch at which validators should perform their duties. uint64 epoch = 1; // Array of byte encoded BLS public keys. @@ -425,16 +425,22 @@ index 9f07458..e30cd3f 100644 // Slot at which a validator must attest. uint64 attestation_slot = 2; // Shard at which a validator must attest. -@@ -130,7 +131,7 @@ message BlockRequest { +@@ -280,10 +281,12 @@ message DutiesResponse { + message BlockRequest { // Slot for which the block should be proposed. uint64 slot = 1; ++ // Validator's 32 byte randao reveal secret of the current epoch. - bytes randao_reveal = 2; -+ bytes randao_reveal = 2 [(gogoproto.moretags) = "ssz-size:\"48\""]; ++ bytes randao_reveal = 2 [(gogoproto.moretags) = "ssz-size:\"32\""]; ++ + // Validator's 32 byte graffiti message for the new block. +- bytes graffiti = 3; ++ bytes graffiti = 3 [(gogoproto.moretags) = "ssz-size:\"32\""]; } - message AttestationDataRequest { -@@ -147,10 +148,10 @@ message AttestationDataRequest { + message ProposeResponse { +@@ -309,10 +312,10 @@ message AttestResponse { // An Ethereum 2.0 validator. message Validator { // 48 byte BLS public key used for the validator's activities.