Merge branch 'develop' into save-sync-tips

validator/client/BUILD.bazel

@@ -130,7 +130,9 @@ go_test(
         "//validator/client/testutil:go_default_library",
         "//validator/db/testing:go_default_library",
         "//validator/graffiti:go_default_library",
+        "//validator/keymanager:go_default_library",
         "//validator/keymanager/derived:go_default_library",
+        "//validator/keymanager/imported:go_default_library",
         "//validator/keymanager/remote-web3signer:go_default_library",
         "//validator/keymanager/remote/mock:go_default_library",
         "//validator/slashing-protection-history:go_default_library",

validator/client/service.go

@@ -200,6 +200,7 @@ func (v *ValidatorService) Start() {
 		eipImportBlacklistedPublicKeys: slashablePublicKeys,
 		logDutyCountDown:               v.logDutyCountDown,
 		Web3SignerConfig:               v.web3SignerConfig,
+		walletIntializedChannel:        make(chan *wallet.Wallet, 1),
 	}
 	// To resolve a race condition at startup due to the interface
 	// nature of the abstracted block type. We initialize

validator/client/validator.go

@@ -90,6 +90,7 @@ type validator struct {
 	graffiti                           []byte
 	voteStats                          voteStats
 	Web3SignerConfig                   *remote_web3signer.SetupConfig
+	walletIntializedChannel            chan *wallet.Wallet
 }
 
 type validatorStatus struct {
@@ -109,9 +110,10 @@ func (v *validator) WaitForKeymanagerInitialization(ctx context.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "unable to retrieve valid genesis validators root while initializing key manager")
 	}
+
 	if v.useWeb && v.wallet == nil {
 		// if wallet is not set, wait for it to be set through the UI
-		km, err := waitForWebWalletInitialization(ctx, v.walletInitializedFeed)
+		km, err := waitForWebWalletInitialization(ctx, v.walletInitializedFeed, v.walletIntializedChannel)
 		if err != nil {
 			return err
 		}
@@ -141,8 +143,11 @@ func (v *validator) WaitForKeymanagerInitialization(ctx context.Context) error {
 }
 
 // subscribe to channel for when the wallet is initialized
-func waitForWebWalletInitialization(ctx context.Context, walletInitializedEvent *event.Feed) (keymanager.IKeymanager, error) {
-	walletChan := make(chan *wallet.Wallet)
+func waitForWebWalletInitialization(
+	ctx context.Context,
+	walletInitializedEvent *event.Feed,
+	walletChan chan *wallet.Wallet,
+) (keymanager.IKeymanager, error) {
 	sub := walletInitializedEvent.Subscribe(walletChan)
 	defer sub.Unsubscribe()
 	for {

validator/client/validator_test.go

@@ -28,6 +28,8 @@ import (
 	"github.com/prysmaticlabs/prysm/validator/accounts/wallet"
 	"github.com/prysmaticlabs/prysm/validator/client/iface"
 	dbTest "github.com/prysmaticlabs/prysm/validator/db/testing"
+	"github.com/prysmaticlabs/prysm/validator/keymanager"
+	"github.com/prysmaticlabs/prysm/validator/keymanager/imported"
 	remote_web3signer "github.com/prysmaticlabs/prysm/validator/keymanager/remote-web3signer"
 	"github.com/sirupsen/logrus"
 	logTest "github.com/sirupsen/logrus/hooks/test"
@@ -1359,3 +1361,52 @@ func TestValidator_WaitForKeymanagerInitialization_web3Signer(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, km)
 }
+
+func TestValidator_WaitForKeymanagerInitialization_Web(t *testing.T) {
+	ctx := context.Background()
+	db := dbTest.SetupDB(t, [][fieldparams.BLSPubkeyLength]byte{})
+	root := make([]byte, 32)
+	copy(root[2:], "a")
+	err := db.SaveGenesisValidatorsRoot(ctx, root)
+	require.NoError(t, err)
+	walletChan := make(chan *wallet.Wallet, 1)
+	v := validator{
+		db:                      db,
+		useWeb:                  true,
+		walletInitializedFeed:   &event.Feed{},
+		walletIntializedChannel: walletChan,
+	}
+	go func() {
+		err = v.WaitForKeymanagerInitialization(ctx)
+		require.NoError(t, err)
+		km, err := v.Keymanager()
+		require.NoError(t, err)
+		require.NotNil(t, km)
+	}()
+
+	walletChan <- wallet.New(&wallet.Config{
+		KeymanagerKind: keymanager.Imported,
+	})
+}
+
+func TestValidator_WaitForKeymanagerInitialization_Interop(t *testing.T) {
+	ctx := context.Background()
+	db := dbTest.SetupDB(t, [][fieldparams.BLSPubkeyLength]byte{})
+	root := make([]byte, 32)
+	copy(root[2:], "a")
+	err := db.SaveGenesisValidatorsRoot(ctx, root)
+	require.NoError(t, err)
+	v := validator{
+		db:     db,
+		useWeb: false,
+		interopKeysConfig: &imported.InteropKeymanagerConfig{
+			NumValidatorKeys: 2,
+			Offset:           1,
+		},
+	}
+	err = v.WaitForKeymanagerInitialization(ctx)
+	require.NoError(t, err)
+	km, err := v.Keymanager()
+	require.NoError(t, err)
+	require.NotNil(t, km)
+}

validator/keymanager/remote-web3signer/BUILD.bazel

@@ -2,7 +2,10 @@ load("@prysm//tools/go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = ["keymanager.go"],
+    srcs = [
+        "keymanager.go",
+        "metrics.go",
+    ],
     importpath = "github.com/prysmaticlabs/prysm/validator/keymanager/remote-web3signer",
     visibility = [
         "//cmd/validator:__subpackages__",
@@ -19,6 +22,8 @@ go_library(
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_go_playground_validator_v10//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
     ],
 )
 

validator/keymanager/remote-web3signer/internal/BUILD.bazel

@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "client.go",
         "log.go",
+        "metrics.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/validator/keymanager/remote-web3signer/internal",
     visibility = ["//validator/keymanager/remote-web3signer:__subpackages__"],
@@ -12,9 +13,13 @@ go_library(
         "//config/fieldparams:go_default_library",
         "//crypto/bls:go_default_library",
         "//encoding/bytesutil:go_default_library",
+        "//monitoring/tracing:go_default_library",
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
+        "@io_opencensus_go//trace:go_default_library",
     ],
 )
 

validator/keymanager/remote-web3signer/internal/client.go

@@ -8,13 +8,18 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/http/httputil"
 	"net/url"
+	"strconv"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/pkg/errors"
 	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/crypto/bls"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	"github.com/prysmaticlabs/prysm/monitoring/tracing"
+	"go.opencensus.io/trace"
 )
 
 const (
@@ -116,29 +121,43 @@ func (client *ApiClient) GetServerStatus(ctx context.Context) (string, error) {
 
 // doRequest is a utility method for requests.
 func (client *ApiClient) doRequest(ctx context.Context, httpMethod, fullPath string, body io.Reader) (*http.Response, error) {
+	ctx, span := trace.StartSpan(ctx, "remote_web3signer.Client.doRequest")
+	defer span.End()
+	span.AddAttributes(
+		trace.StringAttribute("httpMethod", httpMethod),
+		trace.StringAttribute("fullPath", fullPath),
+		trace.BoolAttribute("hasBody", body != nil),
+	)
 	req, err := http.NewRequestWithContext(ctx, httpMethod, fullPath, body)
 	if err != nil {
 		return nil, errors.Wrap(err, "invalid format, failed to create new Post Request Object")
 	}
 	req.Header.Set("Content-Type", "application/json")
-	resp, err := client.RestClient.Do(req)
+	requestDump, err := httputil.DumpRequest(req, true)
 	if err != nil {
-		return resp, errors.Wrap(err, "failed to execute json request")
+		return nil, err
+	}
+	start := time.Now()
+	resp, err := client.RestClient.Do(req)
+	duration := time.Since(start)
+	if err != nil {
+		signRequestDurationSeconds.WithLabelValues(req.Method, "error").Observe(duration.Seconds())
+		err = errors.Wrap(err, "failed to execute json request")
+		tracing.AnnotateError(span, err)
+		return resp, err
+	} else {
+		signRequestDurationSeconds.WithLabelValues(req.Method, strconv.Itoa(resp.StatusCode)).Observe(duration.Seconds())
 	}
 	if resp.StatusCode == http.StatusInternalServerError {
-		b, err := io.ReadAll(body)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to read body")
-		}
-		return nil, fmt.Errorf("internal Web3Signer server error, Signing Request URL: %v, Signing Request Body: %s, Full Response: %v", fullPath, string(b), resp)
+		err = fmt.Errorf("internal Web3Signer server error, Signing Request URL: %v, Signing Request: %s, Full Response: %v", fullPath, string(requestDump), resp)
+		tracing.AnnotateError(span, err)
+		return nil, err
 	} else if resp.StatusCode == http.StatusBadRequest {
-		b, err := io.ReadAll(body)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to read body")
-		}
-		return nil, fmt.Errorf("bad request format, Signing Request URL: %v, Signing Request Body: %v, Full Response: %v", fullPath, string(b), resp)
+		err = fmt.Errorf("bad request format, Signing Request URL: %v, Signing Request: %v, Full Response: %v", fullPath, string(requestDump), resp)
+		tracing.AnnotateError(span, err)
+		return nil, err
 	}
-	return resp, err
+	return resp, nil
 }
 
 // unmarshalResponse is a utility method for unmarshalling responses.

validator/keymanager/remote-web3signer/internal/client_test.go

@@ -125,6 +125,23 @@ func TestClient_GetPublicKeys_HappyPath(t *testing.T) {
 	assert.EqualValues(t, "[162 181 170 173 156 110 254 254 123 185 177 36 58 4 52 4 243 54 41 55 207 182 179 24 51 146 152 51 23 63 71 102 48 234 44 254 176 217 221 241 95 151 202 134 133 148 136 32]", fmt.Sprintf("%v", resp[0][:]))
 }
 
+func TestClient_GetPublicKeys_EncodingError(t *testing.T) {
+	// public keys are returned hex encoded with 0x
+	json := `["a2b5aaad9c6efefe7bb9b1243a043404f3362937c","fb6b31833929833173f476630ea2cfe","b0d9ddf15fca8685948820"]`
+	// create a new reader with that JSON
+	r := ioutil.NopCloser(bytes.NewReader([]byte(json)))
+	mock := &mockTransport{mockResponse: &http.Response{
+		StatusCode: 200,
+		Body:       r,
+	}}
+	u, err := url.Parse("example.com")
+	assert.NoError(t, err)
+	cl := internal.ApiClient{BaseURL: u, RestClient: &http.Client{Transport: mock}}
+	resp, err := cl.GetPublicKeys(context.Background(), "example.com/api/publickeys")
+	assert.Equal(t, err.Error(), "failed to decode from Hex from the following public key index locations: 0, 1, 2, ")
+	assert.Nil(t, resp)
+}
+
 // TODO: not really in use, should be revisited
 func TestClient_ReloadSignerKeys_HappyPath(t *testing.T) {
 	mock := &mockTransport{mockResponse: &http.Response{

validator/keymanager/remote-web3signer/internal/metrics.go

@@ -0,0 +1,17 @@
+package internal
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	signRequestDurationSeconds = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "remote_web3signer_internal_client_request_duration_seconds",
+			Help:    "Time (in seconds) spent doing client HTTP requests",
+			Buckets: prometheus.DefBuckets,
+		},
+		[]string{"method", "status_code"},
+	)
+)

validator/keymanager/remote-web3signer/keymanager.go

@@ -77,6 +77,7 @@ func (km *Keymanager) FetchValidatingPublicKeys(ctx context.Context) ([][fieldpa
 	if km.publicKeysURL != "" && len(km.providedPublicKeys) == 0 {
 		providedPublicKeys, err := km.client.GetPublicKeys(ctx, km.publicKeysURL)
 		if err != nil {
+			erroredResponsesTotal.Inc()
 			return nil, err
 		}
 		km.providedPublicKeys = providedPublicKeys
@@ -88,8 +89,12 @@ func (km *Keymanager) FetchValidatingPublicKeys(ctx context.Context) ([][fieldpa
 func (km *Keymanager) Sign(ctx context.Context, request *validatorpb.SignRequest) (bls.Signature, error) {
 	signRequest, err := getSignRequestJson(ctx, km.validator, request, km.genesisValidatorsRoot)
 	if err != nil {
+		erroredResponsesTotal.Inc()
 		return nil, err
 	}
+
+	signRequestsTotal.Inc()
+
 	return km.client.Sign(ctx, hexutil.Encode(request.PublicKey), signRequest)
 }
 
@@ -110,6 +115,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		if err = validator.StructCtx(ctx, bockSignRequest); err != nil {
 			return nil, err
 		}
+		blockSignRequestsTotal.Inc()
 		return json.Marshal(bockSignRequest)
 	case *validatorpb.SignRequest_AttestationData:
 		attestationSignRequest, err := v1.GetAttestationSignRequest(request, genesisValidatorsRoot)
@@ -119,6 +125,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		if err = validator.StructCtx(ctx, attestationSignRequest); err != nil {
 			return nil, err
 		}
+		attestationSignRequestsTotal.Inc()
 		return json.Marshal(attestationSignRequest)
 	case *validatorpb.SignRequest_AggregateAttestationAndProof:
 		aggregateAndProofSignRequest, err := v1.GetAggregateAndProofSignRequest(request, genesisValidatorsRoot)
@@ -128,6 +135,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		if err = validator.StructCtx(ctx, aggregateAndProofSignRequest); err != nil {
 			return nil, err
 		}
+		aggregateAndProofSignRequestsTotal.Inc()
 		return json.Marshal(aggregateAndProofSignRequest)
 	case *validatorpb.SignRequest_Slot:
 		aggregationSlotSignRequest, err := v1.GetAggregationSlotSignRequest(request, genesisValidatorsRoot)
@@ -137,6 +145,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		if err = validator.StructCtx(ctx, aggregationSlotSignRequest); err != nil {
 			return nil, err
 		}
+		aggregationSlotSignRequestsTotal.Inc()
 		return json.Marshal(aggregationSlotSignRequest)
 	case *validatorpb.SignRequest_BlockV2:
 		blocv2AltairSignRequest, err := v1.GetBlockV2AltairSignRequest(request, genesisValidatorsRoot)
@@ -146,6 +155,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		if err = validator.StructCtx(ctx, blocv2AltairSignRequest); err != nil {
 			return nil, err
 		}
+		blockV2SignRequestsTotal.Inc()
 		return json.Marshal(blocv2AltairSignRequest)
 	// TODO(#10053): Need to add support for merge blocks.
 
@@ -168,6 +178,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		if err = validator.StructCtx(ctx, randaoRevealSignRequest); err != nil {
 			return nil, err
 		}
+		randaoRevealSignRequestsTotal.Inc()
 		return json.Marshal(randaoRevealSignRequest)
 	case *validatorpb.SignRequest_Exit:
 		voluntaryExitRequest, err := v1.GetVoluntaryExitSignRequest(request, genesisValidatorsRoot)
@@ -177,6 +188,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		if err = validator.StructCtx(ctx, voluntaryExitRequest); err != nil {
 			return nil, err
 		}
+		voluntaryExitSignRequestsTotal.Inc()
 		return json.Marshal(voluntaryExitRequest)
 	case *validatorpb.SignRequest_SyncMessageBlockRoot:
 		syncCommitteeMessageRequest, err := v1.GetSyncCommitteeMessageSignRequest(request, genesisValidatorsRoot)
@@ -186,6 +198,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		if err = validator.StructCtx(ctx, syncCommitteeMessageRequest); err != nil {
 			return nil, err
 		}
+		syncCommitteeMessageSignRequestsTotal.Inc()
 		return json.Marshal(syncCommitteeMessageRequest)
 	case *validatorpb.SignRequest_SyncAggregatorSelectionData:
 		syncCommitteeSelectionProofRequest, err := v1.GetSyncCommitteeSelectionProofSignRequest(request, genesisValidatorsRoot)
@@ -195,6 +208,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		if err = validator.StructCtx(ctx, syncCommitteeSelectionProofRequest); err != nil {
 			return nil, err
 		}
+		syncCommitteeSelectionProofSignRequestsTotal.Inc()
 		return json.Marshal(syncCommitteeSelectionProofRequest)
 	case *validatorpb.SignRequest_ContributionAndProof:
 		contributionAndProofRequest, err := v1.GetSyncCommitteeContributionAndProofSignRequest(request, genesisValidatorsRoot)
@@ -204,6 +218,7 @@ func getSignRequestJson(ctx context.Context, validator *validator.Validate, requ
 		if err = validator.StructCtx(ctx, contributionAndProofRequest); err != nil {
 			return nil, err
 		}
+		syncCommitteeContributionAndProofSignRequestsTotal.Inc()
 		return json.Marshal(contributionAndProofRequest)
 	default:
 		return nil, fmt.Errorf("web3signer sign request type %T not supported", request.Object)

validator/keymanager/remote-web3signer/metrics.go

@@ -0,0 +1,57 @@
+package remote_web3signer
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	signRequestsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "remote_web3signer_sign_requests_total",
+		Help: "Total number of sign requests",
+	})
+	erroredResponsesTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "remote_web3signer_errored_responses_total",
+		Help: "Total number of errored responses when calling web3signer",
+	})
+	blockSignRequestsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "remote_web3signer_block_sign_requests_total",
+		Help: "Total number of block sign requests",
+	})
+	aggregationSlotSignRequestsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "remote_web3signer_aggregation_slot_requests_total",
+		Help: "Total number of aggregation slot requests",
+	})
+	aggregateAndProofSignRequestsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "remote_web3signer_aggregate_and_proof_sign_requests_total",
+		Help: "Total number of aggregate and proof sign requests",
+	})
+	attestationSignRequestsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "remote_web3signer_attestation_sign_requests_total",
+		Help: "Total number of attestation sign requests",
+	})
+	blockV2SignRequestsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "remote_web3signer_block_v2_sign_requests_total",
+		Help: "Total number of block v2 sign requests",
+	})
+	randaoRevealSignRequestsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "remote_web3signer_randao_reveal_sign_requests_total",
+		Help: "Total number of randao reveal sign requests",
+	})
+	voluntaryExitSignRequestsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "remote_web3signer_voluntary_exit_sign_requests_total",
+		Help: "Total number of voluntary exit sign requests",
+	})
+	syncCommitteeMessageSignRequestsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "remote_web3signer_sync_committee_message_sign_requests_total",
+		Help: "Total number of sync committee message sign requests",
+	})
+	syncCommitteeSelectionProofSignRequestsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "remote_web3signer_sync_committee_selection_proof_sign_requests_total",
+		Help: "Total number of sync committee selection proof sign requests",
+	})
+	syncCommitteeContributionAndProofSignRequestsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "remote_web3signer_sync_committee_contribution_and_proof_sign_requests_total",
+		Help: "Total number of sync committee contribution and proof sign requests",
+	})
+)