Refetch validating keys if no keys are fetched (#8000)

* refetch validating keys every 30 seconds

* deduplicate error/log messages

* remove redundant break statement

* comment about execution flow

* move code to wait_for_activation.go

Co-authored-by: Victor Farazdagi <simple.square@gmail.com>
Co-authored-by: terence tsao <terence@prysmaticlabs.com>
Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>
Co-authored-by: Preston Van Loon <preston@prysmaticlabs.com>

validator/client/validator.go

@@ -40,6 +40,15 @@ import (
 // slasher connection when the slasher client connection is not ready.
 var reconnectPeriod = 5 * time.Second
 
+// keyFetchPeriod is the frequency that we try to refetch validating keys
+// in case no keys were fetched previously.
+var keyRefetchPeriod = 30 * time.Second
+
+var (
+	msgCouldNotFetchKeys = "could not fetch validating keys"
+	msgNoKeysFetched     = "No validating keys fetched. Trying again"
+)
+
 // ValidatorRole defines the validator role.
 type ValidatorRole int8
 
@@ -219,7 +228,7 @@ func (v *validator) SlasherReady(ctx context.Context) error {
 				return nil
 			case <-ctx.Done():
 				log.Debug("Context closed, exiting reconnect external protection")
-				return errors.New("context closed, no longer attempting to restart external protection")
+				return ctx.Err()
 			}
 		}
 	}

validator/client/validator_test.go

@@ -46,11 +46,18 @@ func genMockKeymanger(numKeys int) *mockKeymanager {
 }
 
 type mockKeymanager struct {
-	lock    sync.RWMutex
-	keysMap map[[48]byte]bls.SecretKey
+	lock        sync.RWMutex
+	keysMap     map[[48]byte]bls.SecretKey
+	fetchNoKeys bool
 }
 
 func (m *mockKeymanager) FetchValidatingPublicKeys(ctx context.Context) ([][48]byte, error) {
+	if m.fetchNoKeys {
+		// We set the value to `false` to fetch keys the next time.
+		m.fetchNoKeys = false
+		return make([][48]byte, 0), nil
+	}
+
 	m.lock.RLock()
 	defer m.lock.RUnlock()
 	keys := make([][48]byte, 0)

validator/client/wait_for_activation.go

@@ -28,6 +28,33 @@ func (v *validator) WaitForActivation(ctx context.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "could not fetch validating keys")
 	}
+	if len(validatingKeys) == 0 {
+		log.Warn(msgNoKeysFetched)
+
+		ticker := time.NewTicker(keyRefetchPeriod)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				keys, err := v.keyManager.FetchValidatingPublicKeys(ctx)
+				if err != nil {
+					return errors.Wrap(err, msgCouldNotFetchKeys)
+				}
+				if len(keys) == 0 {
+					log.Warn(msgNoKeysFetched)
+					continue
+				}
+				// after this statement we jump out of `select` and hit `break`,
+				// thus jumping out of `for` into the rest of the function
+				validatingKeys = keys
+			case <-ctx.Done():
+				log.Debug("Context closed, exiting fetching validating keys")
+				return ctx.Err()
+			}
+			break
+		}
+	}
+
 	req := &ethpb.ValidatorActivationRequest{
 		PublicKeys: bytesutil.FromBytes48Array(validatingKeys),
 	}

validator/client/wait_for_activation_test.go

@@ -3,6 +3,7 @@ package client
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/golang/mock/gomock"
 	"github.com/pkg/errors"
@@ -183,3 +184,47 @@ func TestWaitForActivation_Exiting(t *testing.T) {
 	)
 	require.NoError(t, v.WaitForActivation(context.Background()))
 }
+
+func TestWaitForActivation_RefetchKeys(t *testing.T) {
+	originalPeriod := keyRefetchPeriod
+	defer func() {
+		keyRefetchPeriod = originalPeriod
+	}()
+	keyRefetchPeriod = 5 * time.Second
+
+	hook := logTest.NewGlobal()
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	client := mock.NewMockBeaconNodeValidatorClient(ctrl)
+	privKey, err := bls.RandKey()
+	require.NoError(t, err)
+	pubKey := [48]byte{}
+	copy(pubKey[:], privKey.PublicKey().Marshal())
+	km := &mockKeymanager{
+		keysMap: map[[48]byte]bls.SecretKey{
+			pubKey: privKey,
+		},
+		fetchNoKeys: true,
+	}
+	v := validator{
+		validatorClient: client,
+		keyManager:      km,
+		genesisTime:     1,
+	}
+	resp := generateMockStatusResponse([][]byte{pubKey[:]})
+	resp.Statuses[0].Status.Status = ethpb.ValidatorStatus_ACTIVE
+	clientStream := mock.NewMockBeaconNodeValidator_WaitForActivationClient(ctrl)
+	client.EXPECT().WaitForActivation(
+		gomock.Any(),
+		&ethpb.ValidatorActivationRequest{
+			PublicKeys: [][]byte{pubKey[:]},
+		},
+	).Return(clientStream, nil)
+	clientStream.EXPECT().Recv().Return(
+		resp,
+		nil,
+	)
+	assert.NoError(t, v.WaitForActivation(context.Background()), "Could not wait for activation")
+	assert.LogsContain(t, hook, msgNoKeysFetched)
+	assert.LogsContain(t, hook, "Validator activated")
+}