Include Validator Index in GetDuties Response, Update EthereumAPIs (#4567)

* include new patch * add patch and validator indices to duties resp * test passing * move call to validator index * Merge branch 'master' into include-val-idx * do not use wait groups anymore * Merge branch 'include-val-idx' of github.com:prysmaticlabs/prysm into include-val-idx * Update beacon-chain/rpc/validator/assignments_test.go
2026-01-08 21:08:10 -05:00 · 2020-01-16 16:37:51 -06:00
parent ed529965af
commit eb429ab719
6 changed files with 112 additions and 103 deletions
--- a/2
+++ b/2
@@ -1255,7 +1255,7 @@ go_repository(

 go_repository(
    name = "com_github_prysmaticlabs_ethereumapis",
-    commit = "87118fb893cc6f32b25793d819790fd3bcce3221",
+    commit = "8a785e129627db2be96339a35fb2317b200160b1",
    importpath = "github.com/prysmaticlabs/ethereumapis",
    patch_args = ["-p1"],
    patches = [
--- a/beacon-chain/rpc/validator/assignments.go
+++ b/beacon-chain/rpc/validator/assignments.go
@@ -58,6 +58,7 @@ func (vs *Server) GetDuties(ctx context.Context, req *ethpb.DutiesRequest) (*eth
 			if ok {
 				assignment.Committee = ca.Committee
 				assignment.Status = ethpb.ValidatorStatus_ACTIVE
+				assignment.ValidatorIndex = idx
 				assignment.PublicKey = pubKey
 				assignment.AttesterSlot = ca.AttesterSlot
 				assignment.ProposerSlot = proposerIndexToSlot[idx]
--- a/beacon-chain/rpc/validator/assignments_test.go
+++ b/beacon-chain/rpc/validator/assignments_test.go
@@ -5,7 +5,6 @@ import (
 	"encoding/binary"
 	"fmt"
 	"strings"
-	"sync"
 	"testing"

 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
@@ -105,22 +104,14 @@ func TestGetDuties_OK(t *testing.T) {
 		t.Fatalf("Could not get signing root %v", err)
 	}

-	var wg sync.WaitGroup
-	numOfValidators := int(depChainStart)
-	errs := make(chan error, numOfValidators)
+	pubKeys := make([][]byte, len(deposits))
+	indices := make([]uint64, len(deposits))
 	for i := 0; i < len(deposits); i++ {
-		wg.Add(1)
-		go func(index int) {
-			errs <- db.SaveValidatorIndex(ctx, deposits[index].Data.PublicKey, uint64(index))
-			wg.Done()
-		}(i)
+		pubKeys[i] = deposits[i].Data.PublicKey
+		indices[i] = uint64(i)
 	}
-	wg.Wait()
-	close(errs)
-	for err := range errs {
-		if err != nil {
-			t.Fatalf("Could not save validator index: %v", err)
-		}
+	if err := db.SaveValidatorIndices(ctx, pubKeys, indices); err != nil {
+		t.Fatal(err)
 	}

 	vs := &Server{
@@ -157,6 +148,21 @@ func TestGetDuties_OK(t *testing.T) {
 		t.Errorf("Assigned slot %d can't be higher than %d",
 			res.Duties[0].AttesterSlot, state.Slot+params.BeaconConfig().SlotsPerEpoch)
 	}
+
+	// We request for duties for all validators.
+	req = &ethpb.DutiesRequest{
+		PublicKeys: pubKeys,
+		Epoch:      0,
+	}
+	res, err = vs.GetDuties(context.Background(), req)
+	if err != nil {
+		t.Fatalf("Could not call epoch committee assignment %v", err)
+	}
+	for i := 0; i < len(res.Duties); i++ {
+		if res.Duties[i].ValidatorIndex != uint64(i) {
+			t.Errorf("Wanted %d, received %d", i, res.Duties[i].ValidatorIndex)
+		}
+	}
 }

 func TestGetDuties_CurrentEpoch_ShouldNotFail(t *testing.T) {
@@ -182,22 +188,14 @@ func TestGetDuties_CurrentEpoch_ShouldNotFail(t *testing.T) {
 		t.Fatalf("Could not get signing root %v", err)
 	}

-	var wg sync.WaitGroup
-	numOfValidators := int(depChainStart)
-	errs := make(chan error, numOfValidators)
+	pubKeys := make([][]byte, len(deposits))
+	indices := make([]uint64, len(deposits))
 	for i := 0; i < len(deposits); i++ {
-		wg.Add(1)
-		go func(index int) {
-			errs <- db.SaveValidatorIndex(ctx, deposits[index].Data.PublicKey, uint64(index))
-			wg.Done()
-		}(i)
+		pubKeys[i] = deposits[i].Data.PublicKey
+		indices[i] = uint64(i)
 	}
-	wg.Wait()
-	close(errs)
-	for err := range errs {
-		if err != nil {
-			t.Fatalf("Could not save validator index: %v", err)
-		}
+	if err := db.SaveValidatorIndices(ctx, pubKeys, indices); err != nil {
+		t.Fatal(err)
 	}

 	vs := &Server{
@@ -241,22 +239,14 @@ func TestGetDuties_MultipleKeys_OK(t *testing.T) {
 		t.Fatalf("Could not get signing root %v", err)
 	}

-	var wg sync.WaitGroup
-	numOfValidators := int(depChainStart)
-	errs := make(chan error, numOfValidators)
-	for i := 0; i < numOfValidators; i++ {
-		wg.Add(1)
-		go func(index int) {
-			errs <- db.SaveValidatorIndex(ctx, deposits[index].Data.PublicKey, uint64(index))
-			wg.Done()
-		}(i)
+	pubKeys := make([][]byte, len(deposits))
+	indices := make([]uint64, len(deposits))
+	for i := 0; i < len(deposits); i++ {
+		pubKeys[i] = deposits[i].Data.PublicKey
+		indices[i] = uint64(i)
 	}
-	wg.Wait()
-	close(errs)
-	for err := range errs {
-		if err != nil {
-			t.Fatalf("Could not save validator index: %v", err)
-		}
+	if err := db.SaveValidatorIndices(ctx, pubKeys, indices); err != nil {
+		t.Fatal(err)
 	}

 	vs := &Server{
@@ -320,22 +310,14 @@ func BenchmarkCommitteeAssignment(b *testing.B) {
 		b.Fatalf("Could not get signing root %v", err)
 	}

-	var wg sync.WaitGroup
-	numOfValidators := int(depChainStart)
-	errs := make(chan error, numOfValidators)
-	for i := 0; i < numOfValidators; i++ {
-		wg.Add(1)
-		go func(index int) {
-			errs <- db.SaveValidatorIndex(ctx, deposits[index].Data.PublicKey, uint64(index))
-			wg.Done()
-		}(i)
+	pubKeys := make([][]byte, len(deposits))
+	indices := make([]uint64, len(deposits))
+	for i := 0; i < len(deposits); i++ {
+		pubKeys[i] = deposits[i].Data.PublicKey
+		indices[i] = uint64(i)
 	}
-	wg.Wait()
-	close(errs)
-	for err := range errs {
-		if err != nil {
-			b.Fatalf("Could not save validator index: %v", err)
-		}
+	if err := db.SaveValidatorIndices(ctx, pubKeys, indices); err != nil {
+		b.Fatal(err)
 	}

 	vs := &Server{
--- a/third_party/com_github_prysmaticlabs_ethereumapis-tags.patch
+++ b/third_party/com_github_prysmaticlabs_ethereumapis-tags.patch
@@ -1,8 +1,16 @@
 diff --git a/eth/v1alpha1/BUILD.bazel b/eth/v1alpha1/BUILD.bazel
-index a52dbad..33de299 100644
+index f5596a6..e209e30 100644
 --- a/eth/v1alpha1/BUILD.bazel
 +++ b/eth/v1alpha1/BUILD.bazel
-@@ -20,6 +20,7 @@ proto_library(
+@@ -10,14 +10,13 @@ proto_library(
+         "beacon_chain.proto",
+         "node.proto",
+         "validator.proto",
+-        ":generated_swagger_proto",
+     ],
+     visibility = ["//visibility:public"],
+     deps = [
+-        "@grpc_ecosystem_grpc_gateway//protoc-gen-swagger/options:options_proto",
         "@com_google_protobuf//:empty_proto",
         "@com_google_protobuf//:timestamp_proto",
         "@go_googleapis//google/api:annotations_proto",
@@ -10,7 +18,7 @@ index a52dbad..33de299 100644
     ],
 )
 
-@@ -46,12 +47,32 @@ load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
+@@ -39,12 +38,30 @@ load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
 
 go_proto_library(
     name = "go_proto",
@@ -20,7 +28,7 @@ index a52dbad..33de299 100644
     proto = ":proto",
     visibility = ["//visibility:public"],
     deps = [
-         "@grpc_ecosystem_grpc_gateway//protoc-gen-swagger/options:options_go_proto",
+-        "@grpc_ecosystem_grpc_gateway//protoc-gen-swagger/options:options_go_proto",
 +        "@com_github_prysmaticlabs_go_bitfield//:go_default_library",
 +        "@go_googleapis//google/api:annotations_go_proto",
 +    ],
@@ -36,7 +44,6 @@ index a52dbad..33de299 100644
 +    proto = ":proto",
 +    visibility = ["//visibility:public"],
 +    deps = [
-+        "@grpc_ecosystem_grpc_gateway//protoc-gen-swagger/options:options_go_proto",
 +        "@com_github_gogo_protobuf//gogoproto:go_default_library",
 +        "@com_github_golang_protobuf//descriptor:go_default_library",
 +        "@com_github_golang_protobuf//ptypes/empty:go_default_library",
@@ -44,13 +51,36 @@ index a52dbad..33de299 100644
         "@go_googleapis//google/api:annotations_go_proto",
     ],
 )
-@@ -74,4 +95,4 @@ protoc_gen_swagger(
+@@ -55,29 +72,3 @@ go_library(
+     importpath = "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1",
     visibility = ["//visibility:public"],
-     single_output = True,
-     json_names_for_fields = True,
+ )
+-
+-##############################################################################
+-# OpenAPI (Swagger) V2
+-##############################################################################
+-load("@grpc_ecosystem_grpc_gateway//protoc-gen-swagger:defs.bzl", "protoc_gen_swagger")
+-
+-protoc_gen_swagger(
+-    name = "swagger",
+-    proto = ":proto",
+-    visibility = ["//visibility:public"],
+-    single_output = True,
+-    json_names_for_fields = True,
+-)
+-
+-# Genrule for template subsitution.
+-# See documentation in //tools/replacer.
+-genrule(
+-    name = "generated_swagger_proto",
+-    srcs = [
+-        "swagger.proto", # A go template compatibile file.
+-        "swagger_description.md", # Replacement for description. 
+-    ],
+-    outs = ["swagger_generated.proto"],
+-    cmd = "$(location //tools/replacer) $(location swagger.proto) description=$(location swagger_description.md) > $(@)",
+-    tools = ["//tools/replacer"],
 -)
-+)
-\ No newline at end of file
 diff --git a/eth/v1alpha1/attestation.proto b/eth/v1alpha1/attestation.proto
 index b177b76..28b4b46 100644
 --- a/eth/v1alpha1/attestation.proto
@@ -246,7 +276,7 @@ index 2ce5c34..4cbb276 100644
 message SignedBeaconBlockHeader {
@@ -183,14 +184,14 @@ message SignedBeaconBlockHeader {
     BeaconBlockHeader header = 1;
-
+ 
     // 96 byte BLS signature from the validator that produced this block header.
 -    bytes signature = 2;
 +    bytes signature = 2 [(gogoproto.moretags) = "ssz-size:\"96\""];
@@ -263,7 +293,7 @@ index 2ce5c34..4cbb276 100644
 +    bytes signature = 3 [(gogoproto.moretags) = "ssz-size:\"96\""];
 }
 diff --git a/eth/v1alpha1/beacon_chain.proto b/eth/v1alpha1/beacon_chain.proto
-index 494b5d6..c87464b 100644
+index 8586530..e1b08b3 100644
 --- a/eth/v1alpha1/beacon_chain.proto
 +++ b/eth/v1alpha1/beacon_chain.proto
@@ -15,6 +15,7 @@ syntax = "proto3";
@@ -287,7 +317,7 @@ index 494b5d6..c87464b 100644
     // This includes the head block slot and root as well as information about
     // the most recent finalized and justified slots.
     rpc StreamChainHead(google.protobuf.Empty) returns (stream ChainHead) {
-@@ -309,7 +310,7 @@ message ChainHead {
+@@ -302,7 +303,7 @@ message ChainHead {
     uint64 head_epoch = 2;
 
     // 32 byte merkle tree root of the canonical head block in the beacon node.
@@ -296,7 +326,7 @@ index 494b5d6..c87464b 100644
 
     // Most recent slot that contains the finalized block.
     uint64 finalized_slot = 4;
-@@ -318,7 +319,7 @@ message ChainHead {
+@@ -311,7 +312,7 @@ message ChainHead {
     uint64 finalized_epoch = 5;
     
     // Most recent 32 byte finalized block root.
@@ -305,7 +335,7 @@ index 494b5d6..c87464b 100644
 
     // Most recent slot that contains the justified block.
     uint64 justified_slot = 7;
-@@ -327,7 +328,7 @@ message ChainHead {
+@@ -320,7 +321,7 @@ message ChainHead {
     uint64 justified_epoch = 8;
     
     // Most recent 32 byte justified block root.
@@ -314,7 +344,7 @@ index 494b5d6..c87464b 100644
 
     // Most recent slot that contains the previous justified block.
     uint64 previous_justified_slot = 10;
-@@ -336,7 +337,7 @@ message ChainHead {
+@@ -329,7 +330,7 @@ message ChainHead {
     uint64 previous_justified_epoch = 11;
 
     // Previous 32 byte justified block root.
@@ -323,7 +353,7 @@ index 494b5d6..c87464b 100644
 }
 
 message ListCommitteesRequest {
-@@ -381,7 +382,7 @@ message ListValidatorBalancesRequest {
+@@ -374,7 +375,7 @@ message ListValidatorBalancesRequest {
 
     // Validator 48 byte BLS public keys to filter validators for the given
     // epoch.
@@ -332,7 +362,7 @@ index 494b5d6..c87464b 100644
         
     // Validator indices to filter validators for the given epoch.
     repeated uint64 indices = 4;
-@@ -402,7 +403,7 @@ message ValidatorBalances {
+@@ -395,7 +396,7 @@ message ValidatorBalances {
 
     message Balance {
         // Validator's 48 byte BLS public key.
@@ -341,7 +371,7 @@ index 494b5d6..c87464b 100644
 
         // Validator's index in the validator set.
         uint64 index = 2;
-@@ -451,7 +452,7 @@ message GetValidatorRequest {
+@@ -444,7 +445,7 @@ message GetValidatorRequest {
         uint64 index = 1;
 
         // 48 byte validator public key.
@@ -350,7 +380,7 @@ index 494b5d6..c87464b 100644
     }
 }
 
-@@ -493,26 +494,25 @@ message ActiveSetChanges {
+@@ -486,26 +487,25 @@ message ActiveSetChanges {
     uint64 epoch = 1;
 
     // 48 byte validator public keys that have been activated in the given epoch.
@@ -383,7 +413,7 @@ index 494b5d6..c87464b 100644
 
     // Indices of validators ejected in the given epoch.
     repeated uint64 ejected_indices = 9;
-@@ -548,11 +548,11 @@ message ValidatorQueue {
+@@ -541,11 +541,11 @@ message ValidatorQueue {
 
     // Ordered list of 48 byte public keys awaiting activation. 0th index is the
     // next key to be processed.
@@ -397,7 +427,7 @@ index 494b5d6..c87464b 100644
 }
 
 message ListValidatorAssignmentsRequest {
-@@ -564,7 +564,7 @@ message ListValidatorAssignmentsRequest {
+@@ -557,7 +557,7 @@ message ListValidatorAssignmentsRequest {
         bool genesis = 2;
     }
     // 48 byte validator public keys to filter assignments for the given epoch.
@@ -406,7 +436,7 @@ index 494b5d6..c87464b 100644
         
     // Validator indicies to filter assignments for the given epoch.
     repeated uint64 indices = 4;
-@@ -599,7 +599,7 @@ message ValidatorAssignments {
+@@ -592,7 +592,7 @@ message ValidatorAssignments {
         uint64 proposer_slot = 4;
 
         // 48 byte BLS public key.
@@ -416,7 +446,7 @@ index 494b5d6..c87464b 100644
 
     // The epoch for which this set of validator assignments is valid.
 diff --git a/eth/v1alpha1/validator.proto b/eth/v1alpha1/validator.proto
-index 4bd149a..09dac9d 100644
+index 599bd57..38e1553 100644
 --- a/eth/v1alpha1/validator.proto
 +++ b/eth/v1alpha1/validator.proto
@@ -15,6 +15,7 @@ syntax = "proto3";
@@ -472,7 +502,7 @@ index 4bd149a..09dac9d 100644
 
         // The current status of the validator assigned to perform the duty.
         ValidatorStatus status = 6;
-@@ -286,15 +287,16 @@ message BlockRequest {
+@@ -289,15 +290,16 @@ message BlockRequest {
     uint64 slot = 1;
 
     // Validator's 32 byte randao reveal secret of the current epoch.
@@ -492,7 +522,7 @@ index 4bd149a..09dac9d 100644
 }
 
 message AttestationDataRequest {
-@@ -307,16 +309,16 @@ message AttestationDataRequest {
+@@ -310,16 +312,16 @@ message AttestationDataRequest {
 
 message AttestResponse {
     // The root of the attestation data successfully submitted to the beacon node.
--- a/validator/client/validator.go
+++ b/validator/client/validator.go
@@ -259,17 +259,11 @@ func (v *validator) UpdateDuties(ctx context.Context, slot uint64) error {

 		for _, duty := range v.duties.Duties {
 			if _, ok := v.pubKeyToID[bytesutil.ToBytes48(duty.PublicKey)]; !ok {
-				// TODO(4379): Make validator index part of the assignment respond.
-				res, err := v.validatorClient.ValidatorIndex(ctx, &ethpb.ValidatorIndexRequest{PublicKey: duty.PublicKey})
-				if err != nil {
-					log.Warnf("Validator pub key %#x does not exist in beacon node", bytesutil.Trunc(duty.PublicKey))
-					continue
-				}
-				v.pubKeyToID[bytesutil.ToBytes48(duty.PublicKey)] = res.Index
+				v.pubKeyToID[bytesutil.ToBytes48(duty.PublicKey)] = duty.ValidatorIndex
 			}
 			lFields := logrus.Fields{
 				"pubKey":         fmt.Sprintf("%#x", bytesutil.Trunc(duty.PublicKey)),
-				"validatorIndex": v.pubKeyToID[bytesutil.ToBytes48(duty.PublicKey)],
+				"validatorIndex": duty.ValidatorIndex,
 				"committeeIndex": duty.CommitteeIndex,
 				"epoch":          slot / params.BeaconConfig().SlotsPerEpoch,
 				"status":         duty.Status,
--- a/validator/client/validator_test.go
+++ b/validator/client/validator_test.go
@@ -441,7 +441,7 @@ func TestWaitSync_Syncing(t *testing.T) {
 	}
 }

-func TestUpdateAssignments_DoesNothingWhenNotEpochStartAndAlreadyExistingAssignments(t *testing.T) {
+func TestUpdateDuties_DoesNothingWhenNotEpochStart_AlreadyExistingAssignments(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 	client := internal.NewMockBeaconNodeValidatorClient(ctrl)
@@ -470,7 +470,7 @@ func TestUpdateAssignments_DoesNothingWhenNotEpochStartAndAlreadyExistingAssignm
 	}
 }

-func TestUpdateAssignments_ReturnsError(t *testing.T) {
+func TestUpdateDuties_ReturnsError(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 	client := internal.NewMockBeaconNodeValidatorClient(ctrl)
@@ -502,7 +502,7 @@ func TestUpdateAssignments_ReturnsError(t *testing.T) {
 	}
 }

-func TestUpdateAssignments_OK(t *testing.T) {
+func TestUpdateDuties_OK(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 	client := internal.NewMockBeaconNodeValidatorClient(ctrl)
@@ -512,6 +512,7 @@ func TestUpdateAssignments_OK(t *testing.T) {
 		Duties: []*ethpb.DutiesResponse_Duty{
 			{
 				AttesterSlot:   params.BeaconConfig().SlotsPerEpoch,
+				ValidatorIndex: 200,
 				CommitteeIndex: 100,
 				Committee:      []uint64{0, 1, 2, 3},
 				PublicKey:      []byte("testPubKey_1"),
@@ -529,12 +530,6 @@ func TestUpdateAssignments_OK(t *testing.T) {
 		gomock.Any(),
 	).Return(resp, nil)

-	indexResp := &ethpb.ValidatorIndexResponse{Index: 100}
-	client.EXPECT().ValidatorIndex(
-		gomock.Any(),
-		gomock.Any(),
-	).Return(indexResp, nil)
-
 	if err := v.UpdateDuties(context.Background(), slot); err != nil {
 		t.Fatalf("Could not update assignments: %v", err)
 	}
@@ -559,6 +554,13 @@ func TestUpdateAssignments_OK(t *testing.T) {
 			v.duties.Duties[0].CommitteeIndex,
 		)
 	}
+	if v.duties.Duties[0].ValidatorIndex != resp.Duties[0].ValidatorIndex {
+		t.Errorf(
+			"Unexpected validator assignments. want=%v got=%v",
+			resp.Duties[0].ValidatorIndex,
+			v.duties.Duties[0].ValidatorIndex,
+		)
+	}
 }

 func TestRolesAt_OK(t *testing.T) {