Bring Back Epoch Filtering for ListBlocks API (#4262)

* bring back the epochs!
* fix up
* Merge refs/heads/master into bring-back-epoch-filter
* add in patch
* Merge branch 'bring-back-epoch-filter' of github.com:prysmaticlabs/prysm into bring-back-epoch-filter
* import spacing
* lint
* build
* gaz
* Merge refs/heads/master into bring-back-epoch-filter
* gaz
* Merge branch 'bring-back-epoch-filter' of github.com:prysmaticlabs/prysm into bring-back-epoch-filter
* move back perf
* update ethapis
* fix build
* Merge refs/heads/master into bring-back-epoch-filter

WORKSPACE

@@ -1249,7 +1249,7 @@ go_repository(
 
 go_repository(
     name = "com_github_prysmaticlabs_ethereumapis",
-    commit = "5f21afe48ab14bd0d5311cf5d33853a3e23d2fda",
+    commit = "2a889fed542ad00e4bd3caf723f871b6a4eff63d",
     importpath = "github.com/prysmaticlabs/ethereumapis",
     patch_args = ["-p1"],
     patches = [

beacon-chain/rpc/beacon/attestations.go

@@ -116,7 +116,7 @@ func (bs *Server) ListAttestations(
 // StreamAttestations to clients every single time a new attestation is received.
 // TODO(#4184): Implement.
 func (bs *Server) StreamAttestations(
-	_ *ethpb.ListAttestationsRequest, _ ethpb.BeaconChain_StreamAttestationsServer,
+	_ *ptypes.Empty, _ ethpb.BeaconChain_StreamAttestationsServer,
 ) error {
 	return status.Error(codes.Unimplemented, "Not yet implemented")
 }

beacon-chain/rpc/beacon/blocks.go

@@ -33,6 +33,47 @@ func (bs *Server) ListBlocks(
 	}
 
 	switch q := req.QueryFilter.(type) {
+	case *ethpb.ListBlocksRequest_Epoch:
+		startSlot := q.Epoch * params.BeaconConfig().SlotsPerEpoch
+		endSlot := startSlot + params.BeaconConfig().SlotsPerEpoch - 1
+
+		blks, err := bs.BeaconDB.Blocks(ctx, filters.NewFilter().SetStartSlot(startSlot).SetEndSlot(endSlot))
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, "Failed to get blocks: %v", err)
+		}
+
+		numBlks := len(blks)
+		if numBlks == 0 {
+			return &ethpb.ListBlocksResponse{
+				BlockContainers: make([]*ethpb.BeaconBlockContainer, 0),
+				TotalSize:       0,
+				NextPageToken:   strconv.Itoa(0),
+			}, nil
+		}
+
+		start, end, nextPageToken, err := pagination.StartAndEndPage(req.PageToken, int(req.PageSize), numBlks)
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, "Could not paginate blocks: %v", err)
+		}
+
+		returnedBlks := blks[start:end]
+		containers := make([]*ethpb.BeaconBlockContainer, len(returnedBlks))
+		for i, b := range returnedBlks {
+			root, err := ssz.SigningRoot(b)
+			if err != nil {
+				return nil, err
+			}
+			containers[i] = &ethpb.BeaconBlockContainer{
+				Block:     b,
+				BlockRoot: root[:],
+			}
+		}
+
+		return &ethpb.ListBlocksResponse{
+			BlockContainers: containers,
+			TotalSize:       int32(numBlks),
+			NextPageToken:   nextPageToken,
+		}, nil
 	case *ethpb.ListBlocksRequest_Root:
 		blk, err := bs.BeaconDB.Block(ctx, bytesutil.ToBytes32(q.Root))
 		if err != nil {

beacon-chain/rpc/beacon/blocks_test.go

@@ -143,7 +143,7 @@ func TestServer_ListBlocks_Pagination(t *testing.T) {
 	defer dbTest.TeardownDB(t, db)
 	ctx := context.Background()
 
-	count := uint64(6)
+	count := uint64(100)
 	blks := make([]*ethpb.BeaconBlock, count)
 	blkContainers := make([]*ethpb.BeaconBlockContainer, count)
 	for i := uint64(0); i < count; i++ {
@@ -165,25 +165,76 @@ func TestServer_ListBlocks_Pagination(t *testing.T) {
 		BeaconDB: db,
 	}
 
-	req := &ethpb.ListBlocksRequest{
-		PageToken:   strconv.Itoa(0),
-		QueryFilter: &ethpb.ListBlocksRequest_Slot{Slot: 5},
-		PageSize:    3,
-	}
-	want := &ethpb.ListBlocksResponse{
-		BlockContainers: []*ethpb.BeaconBlockContainer{{
-			Block:     &ethpb.BeaconBlock{Slot: 5},
-			BlockRoot: blkContainers[5].BlockRoot,
-		}},
-		NextPageToken: "",
-		TotalSize:     1,
-	}
-	res, err := bs.ListBlocks(ctx, req)
+	root6, err := ssz.SigningRoot(&ethpb.BeaconBlock{Slot: 6})
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !proto.Equal(res, want) {
-		t.Errorf("Incorrect blocks response, wanted %v, received %v", want, res)
+
+	tests := []struct {
+		req *ethpb.ListBlocksRequest
+		res *ethpb.ListBlocksResponse
+	}{
+		{req: &ethpb.ListBlocksRequest{
+			PageToken:   strconv.Itoa(0),
+			QueryFilter: &ethpb.ListBlocksRequest_Slot{Slot: 5},
+			PageSize:    3},
+			res: &ethpb.ListBlocksResponse{
+				BlockContainers: []*ethpb.BeaconBlockContainer{{Block: &ethpb.BeaconBlock{Slot: 5}, BlockRoot: blkContainers[5].BlockRoot}},
+				NextPageToken:   "",
+				TotalSize:       1}},
+		{req: &ethpb.ListBlocksRequest{
+			PageToken:   strconv.Itoa(0),
+			QueryFilter: &ethpb.ListBlocksRequest_Root{Root: root6[:]},
+			PageSize:    3},
+			res: &ethpb.ListBlocksResponse{
+				BlockContainers: []*ethpb.BeaconBlockContainer{{Block: &ethpb.BeaconBlock{Slot: 6}, BlockRoot: blkContainers[6].BlockRoot}},
+				TotalSize:       1}},
+		{req: &ethpb.ListBlocksRequest{QueryFilter: &ethpb.ListBlocksRequest_Root{Root: root6[:]}},
+			res: &ethpb.ListBlocksResponse{
+				BlockContainers: []*ethpb.BeaconBlockContainer{{Block: &ethpb.BeaconBlock{Slot: 6}, BlockRoot: blkContainers[6].BlockRoot}},
+				TotalSize:       1}},
+		{req: &ethpb.ListBlocksRequest{
+			PageToken:   strconv.Itoa(0),
+			QueryFilter: &ethpb.ListBlocksRequest_Epoch{Epoch: 0},
+			PageSize:    100},
+			res: &ethpb.ListBlocksResponse{
+				BlockContainers: blkContainers[0:params.BeaconConfig().SlotsPerEpoch],
+				NextPageToken:   "",
+				TotalSize:       int32(params.BeaconConfig().SlotsPerEpoch)}},
+		{req: &ethpb.ListBlocksRequest{
+			PageToken:   strconv.Itoa(1),
+			QueryFilter: &ethpb.ListBlocksRequest_Epoch{Epoch: 5},
+			PageSize:    3},
+			res: &ethpb.ListBlocksResponse{
+				BlockContainers: blkContainers[43:46],
+				NextPageToken:   "2",
+				TotalSize:       int32(params.BeaconConfig().SlotsPerEpoch)}},
+		{req: &ethpb.ListBlocksRequest{
+			PageToken:   strconv.Itoa(1),
+			QueryFilter: &ethpb.ListBlocksRequest_Epoch{Epoch: 11},
+			PageSize:    7},
+			res: &ethpb.ListBlocksResponse{
+				BlockContainers: blkContainers[95:96],
+				NextPageToken:   "",
+				TotalSize:       int32(params.BeaconConfig().SlotsPerEpoch)}},
+		{req: &ethpb.ListBlocksRequest{
+			PageToken:   strconv.Itoa(0),
+			QueryFilter: &ethpb.ListBlocksRequest_Epoch{Epoch: 12},
+			PageSize:    4},
+			res: &ethpb.ListBlocksResponse{
+				BlockContainers: blkContainers[96:100],
+				NextPageToken:   "1",
+				TotalSize:       int32(params.BeaconConfig().SlotsPerEpoch / 2)}},
+	}
+
+	for _, test := range tests {
+		res, err := bs.ListBlocks(ctx, test.req)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !proto.Equal(res, test.res) {
+			t.Errorf("Incorrect blocks response, wanted %v, received %v", test.res, res)
+		}
 	}
 }
 

beacon-chain/rpc/validator/server.go

@@ -6,6 +6,11 @@ import (
 
 	ptypes "github.com/gogo/protobuf/types"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	pbp2p "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/rpc/v1"
+	"github.com/sirupsen/logrus"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"github.com/prysmaticlabs/prysm/beacon-chain/blockchain"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
@@ -15,12 +20,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
 	"github.com/prysmaticlabs/prysm/beacon-chain/sync"
-	pbp2p "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/rpc/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/sirupsen/logrus"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
 )
 
 var log logrus.FieldLogger
@@ -102,6 +102,49 @@ func (vs *Server) ValidatorIndex(ctx context.Context, req *pb.ValidatorIndexRequ
 	return &pb.ValidatorIndexResponse{Index: index}, nil
 }
 
+// ExitedValidators queries validator statuses for a give list of validators
+// and returns a filtered list of validator keys that are exited.
+func (vs *Server) ExitedValidators(
+	ctx context.Context,
+	req *pb.ExitedValidatorsRequest) (*pb.ExitedValidatorsResponse, error) {
+
+	_, statuses, err := vs.multipleValidatorStatus(ctx, req.PublicKeys)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "Could not retrieve validator statuses: %v", err)
+	}
+
+	exitedKeys := make([][]byte, 0)
+	for _, st := range statuses {
+		s := st.Status.Status
+		if s == pb.ValidatorStatus_EXITED ||
+			s == pb.ValidatorStatus_EXITED_SLASHED ||
+			s == pb.ValidatorStatus_INITIATED_EXIT {
+			exitedKeys = append(exitedKeys, st.PublicKey)
+		}
+	}
+
+	resp := &pb.ExitedValidatorsResponse{
+		PublicKeys: exitedKeys,
+	}
+
+	return resp, nil
+}
+
+// DomainData fetches the current domain version information from the beacon state.
+func (vs *Server) DomainData(ctx context.Context, request *pb.DomainRequest) (*pb.DomainResponse, error) {
+	fork := vs.ForkFetcher.CurrentFork()
+	dv := helpers.Domain(fork, request.Epoch, request.Domain)
+	return &pb.DomainResponse{
+		SignatureDomain: dv,
+	}, nil
+}
+
+// CanonicalHead of the current beacon chain. This method is requested on-demand
+// by a validator when it is their time to propose or attest.
+func (vs *Server) CanonicalHead(ctx context.Context, req *ptypes.Empty) (*ethpb.BeaconBlock, error) {
+	return vs.HeadFetcher.HeadBlock(), nil
+}
+
 // ValidatorPerformance reports the validator's latest balance along with other important metrics on
 // rewards and penalties throughout its lifecycle in the beacon chain.
 func (vs *Server) ValidatorPerformance(
@@ -148,53 +191,10 @@ func (vs *Server) ValidatorPerformance(
 		AverageActiveValidatorBalance: avgBalance,
 		MissingValidators:             missingValidators,
 		TotalValidators:               uint64(len(headState.Validators)),
-		TotalActiveValidators:         uint64(activeCount),
+		TotalActiveValidators:         activeCount,
 	}, nil
 }
 
-// ExitedValidators queries validator statuses for a give list of validators
-// and returns a filtered list of validator keys that are exited.
-func (vs *Server) ExitedValidators(
-	ctx context.Context,
-	req *pb.ExitedValidatorsRequest) (*pb.ExitedValidatorsResponse, error) {
-
-	_, statuses, err := vs.multipleValidatorStatus(ctx, req.PublicKeys)
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Could not retrieve validator statuses: %v", err)
-	}
-
-	exitedKeys := make([][]byte, 0)
-	for _, st := range statuses {
-		s := st.Status.Status
-		if s == pb.ValidatorStatus_EXITED ||
-			s == pb.ValidatorStatus_EXITED_SLASHED ||
-			s == pb.ValidatorStatus_INITIATED_EXIT {
-			exitedKeys = append(exitedKeys, st.PublicKey)
-		}
-	}
-
-	resp := &pb.ExitedValidatorsResponse{
-		PublicKeys: exitedKeys,
-	}
-
-	return resp, nil
-}
-
-// DomainData fetches the current domain version information from the beacon state.
-func (vs *Server) DomainData(ctx context.Context, request *pb.DomainRequest) (*pb.DomainResponse, error) {
-	fork := vs.ForkFetcher.CurrentFork()
-	dv := helpers.Domain(fork, request.Epoch, request.Domain)
-	return &pb.DomainResponse{
-		SignatureDomain: dv,
-	}, nil
-}
-
-// CanonicalHead of the current beacon chain. This method is requested on-demand
-// by a validator when it is their time to propose or attest.
-func (vs *Server) CanonicalHead(ctx context.Context, req *ptypes.Empty) (*ethpb.BeaconBlock, error) {
-	return vs.HeadFetcher.HeadBlock(), nil
-}
-
 // WaitForChainStart queries the logs of the Deposit Contract in order to verify the beacon chain
 // has started its runtime and validators begin their responsibilities. If it has not, it then
 // subscribes to an event stream triggered by the powchain service whenever the ChainStart log does

third_party/com_github_prysmaticlabs_ethereumapis-tags.patch

@@ -255,7 +255,7 @@ index 69a148a..1b6ac18 100644
 +    bytes signature = 4 [(gogoproto.moretags) = "ssz-size:\"96\""];
  }
 diff --git a/eth/v1alpha1/beacon_chain.proto b/eth/v1alpha1/beacon_chain.proto
-index 5389a4e..2029f0e 100644
+index b4d1638..4bf7ee9 100644
 --- a/eth/v1alpha1/beacon_chain.proto
 +++ b/eth/v1alpha1/beacon_chain.proto
 @@ -15,6 +15,7 @@ syntax = "proto3";
@@ -279,7 +279,7 @@ index 5389a4e..2029f0e 100644
      // This includes the head block slot and root as well as information about
      // the most recent finalized and justified slots.
      rpc StreamChainHead(google.protobuf.Empty) returns (stream ChainHead) {
-@@ -290,7 +291,7 @@ message ChainHead {
+@@ -299,7 +300,7 @@ message ChainHead {
      uint64 head_epoch = 2;
  
      // 32 byte merkle tree root of the canonical head block in the beacon node.
@@ -288,7 +288,7 @@ index 5389a4e..2029f0e 100644
  
      // Most recent slot that contains the finalized block.
      uint64 finalized_slot = 4;
-@@ -299,7 +300,7 @@ message ChainHead {
+@@ -308,7 +309,7 @@ message ChainHead {
      uint64 finalized_epoch = 5;
      
      // Most recent 32 byte finalized block root.
@@ -297,7 +297,7 @@ index 5389a4e..2029f0e 100644
  
      // Most recent slot that contains the justified block.
      uint64 justified_slot = 7;
-@@ -308,7 +309,7 @@ message ChainHead {
+@@ -317,7 +318,7 @@ message ChainHead {
      uint64 justified_epoch = 8;
      
      // Most recent 32 byte justified block root.
@@ -306,7 +306,7 @@ index 5389a4e..2029f0e 100644
  
      // Most recent slot that contains the previous justified block.
      uint64 previous_justified_slot = 10;
-@@ -317,7 +318,7 @@ message ChainHead {
+@@ -326,7 +327,7 @@ message ChainHead {
      uint64 previous_justified_epoch = 11;
  
      // Previous 32 byte justified block root.
@@ -315,7 +315,7 @@ index 5389a4e..2029f0e 100644
  }
  
  message ListCommitteesRequest {
-@@ -362,7 +363,7 @@ message ListValidatorBalancesRequest {
+@@ -371,7 +372,7 @@ message ListValidatorBalancesRequest {
  
      // Validator 48 byte BLS public keys to filter validators for the given
      // epoch.
@@ -324,7 +324,7 @@ index 5389a4e..2029f0e 100644
          
      // Validator indices to filter validators for the given epoch.
      repeated uint64 indices = 4;
-@@ -383,7 +384,7 @@ message ValidatorBalances {
+@@ -392,7 +393,7 @@ message ValidatorBalances {
  
      message Balance {
          // Validator's 48 byte BLS public key.
@@ -333,7 +333,7 @@ index 5389a4e..2029f0e 100644
  
          // Validator's index in the validator set.
          uint64 index = 2;
-@@ -432,7 +433,7 @@ message GetValidatorRequest {
+@@ -441,7 +442,7 @@ message GetValidatorRequest {
          uint64 index = 1;
  
          // 48 byte validator public key.
@@ -342,7 +342,7 @@ index 5389a4e..2029f0e 100644
      }
  }
  
-@@ -469,17 +470,17 @@ message ActiveSetChanges {
+@@ -478,17 +479,17 @@ message ActiveSetChanges {
      uint64 epoch = 1;
  
      // 48 byte validator public keys that have been activated in this epoch.
@@ -364,7 +364,7 @@ index 5389a4e..2029f0e 100644
  }
  
  message ValidatorQueue {
-@@ -489,11 +490,11 @@ message ValidatorQueue {
+@@ -498,11 +499,11 @@ message ValidatorQueue {
  
      // Ordered list of 48 byte public keys awaiting activation. 0th index is the
      // next key to be processed.
@@ -378,7 +378,7 @@ index 5389a4e..2029f0e 100644
  }
  
  message ListValidatorAssignmentsRequest {
-@@ -505,7 +506,7 @@ message ListValidatorAssignmentsRequest {
+@@ -514,7 +515,7 @@ message ListValidatorAssignmentsRequest {
          bool genesis = 2;
      }
      // 48 byte validator public keys to filter assignments for the given epoch.
@@ -387,7 +387,7 @@ index 5389a4e..2029f0e 100644
          
      // Validator indicies to filter assignments for the given epoch.
      repeated uint64 indices = 4;
-@@ -540,7 +541,7 @@ message ValidatorAssignments {
+@@ -549,7 +550,7 @@ message ValidatorAssignments {
          uint64 proposer_slot = 4;
  
          // 48 byte BLS public key.
@@ -397,7 +397,7 @@ index 5389a4e..2029f0e 100644
  
      // The epoch for which this set of validator assignments is valid.
 diff --git a/eth/v1alpha1/validator.proto b/eth/v1alpha1/validator.proto
-index 9f07458..e30cd3f 100644
+index 28a4f31..31e5ec0 100644
 --- a/eth/v1alpha1/validator.proto
 +++ b/eth/v1alpha1/validator.proto
 @@ -15,6 +15,7 @@ syntax = "proto3";
@@ -408,7 +408,7 @@ index 9f07458..e30cd3f 100644
  import "google/api/annotations.proto";
  import "google/protobuf/empty.proto";
  import "eth/v1alpha1/beacon_block.proto";
-@@ -106,14 +107,14 @@ message DutiesRequest {
+@@ -257,14 +258,14 @@ message DutiesRequest {
      // Epoch at which validators should perform their duties.
      uint64 epoch = 1;
      // Array of byte encoded BLS public keys.
@@ -425,16 +425,22 @@ index 9f07458..e30cd3f 100644
          // Slot at which a validator must attest.
          uint64 attestation_slot = 2;
          // Shard at which a validator must attest.
-@@ -130,7 +131,7 @@ message BlockRequest {
+@@ -280,10 +281,12 @@ message DutiesResponse {
+ message BlockRequest {
      // Slot for which the block should be proposed.
      uint64 slot = 1;
++
      // Validator's 32 byte randao reveal secret of the current epoch.
 -    bytes randao_reveal = 2;
-+    bytes randao_reveal = 2 [(gogoproto.moretags) = "ssz-size:\"48\""];
++    bytes randao_reveal = 2 [(gogoproto.moretags) = "ssz-size:\"32\""];
++
+     // Validator's 32 byte graffiti message for the new block.
+-    bytes graffiti = 3;
++    bytes graffiti = 3 [(gogoproto.moretags) = "ssz-size:\"32\""];
  }
  
- message AttestationDataRequest {
-@@ -147,10 +148,10 @@ message AttestationDataRequest {
+ message ProposeResponse {
+@@ -309,10 +312,10 @@ message AttestResponse {
  // An Ethereum 2.0 validator.
  message Validator {
      // 48 byte BLS public key used for the validator's activities.