Web3signer: Add an HTTP Client Wrapper to Interact with Remote Server (#9991)

* initial commit for web3signer code work in progress

* adding more functions for web3signer

* more improvements to unit tests and web3signer functions

* fixing unit test

* removing path construction

* fixing failing unit test

* adding more happy path unit tests

* fixing unit tests

* temp removing keymanagerfiles being wip

* removing some comments

* Update validator/keymanager/remote-web3signer/client.go

Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>

* Update validator/keymanager/remote-web3signer/client.go

Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>

* making errors lowercase

* addressing review comments

* missed resolving a conflict

* addressing deepsource issues

* bazel test and gazelle

* no lint

* deadcode

* addressing comments

* fixing comments

* small fix for readability

Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>

validator/keymanager/remote-web3signer/BUILD.bazel

@@ -0,0 +1,26 @@
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "client.go",
+        "log.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/validator/keymanager/remote-web3signer",
+    visibility = [
+        "//cmd/validator:__subpackages__",
+        "//validator:__subpackages__",
+    ],
+    deps = [
+        "//crypto/bls:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["client_test.go"],
+    embed = [":go_default_library"],
+    deps = ["@com_github_stretchr_testify//assert:go_default_library"],
+)

validator/keymanager/remote-web3signer/client.go

@@ -0,0 +1,194 @@
+package remote_web3signer
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/crypto/bls"
+)
+
+const (
+	ethApiNamespace = "/api/v1/eth2/sign/"
+	maxTimeout      = 3 * time.Second
+)
+
+// client a wrapper object around web3signer APIs. API docs found here https://consensys.github.io/web3signer/web3signer-eth2.html.
+type client struct {
+	BasePath   string
+	restClient *http.Client
+}
+
+// newClient method instantiates a new client object.
+//nolint:unused,deadcode
+func newClient(endpoint string) (*client, error) {
+	u, err := url.Parse(endpoint)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid format, unable to parse url")
+	}
+	return &client{
+		BasePath: u.Host,
+		restClient: &http.Client{
+			Timeout: maxTimeout,
+		},
+	}, nil
+}
+
+// SignRequest is a request object for web3signer sign api.
+type SignRequest struct {
+	Type            string           `json:"type"`
+	ForkInfo        *ForkInfo        `json:"fork_info"`
+	SigningRoot     string           `json:"signingRoot"`
+	AggregationSlot *AggregationSlot `json:"aggregation_slot"`
+}
+
+// ForkInfo a sub property object of the Sign request,in the future before the merge to remove the need to send the entire block body and just use the block_body_root.
+type ForkInfo struct {
+	Fork                  *Fork  `json:"fork"`
+	GenesisValidatorsRoot string `json:"genesis_validators_root"`
+}
+
+// Fork a sub property of ForkInfo.
+type Fork struct {
+	PreviousVersion string `json:"previous_version"`
+	CurrentVersion  string `json:"current_version"`
+	Epoch           string `json:"epoch"`
+}
+
+// AggregationSlot a sub property of SignRequest.
+type AggregationSlot struct {
+	Slot string `json:"slot"`
+}
+
+// signResponse the response object of the web3signer sign api.
+type signResponse struct {
+	Signature string `json:"signature"`
+}
+
+// Sign is a wrapper method around the web3signer sign api.
+func (client *client) Sign(pubKey string, request *SignRequest) (bls.Signature, error) {
+	requestPath := ethApiNamespace + pubKey
+	jsonRequest, err := json.Marshal(request)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid format, failed to marshal json request")
+	}
+	resp, err := client.doRequest(http.MethodPost, client.BasePath+requestPath, bytes.NewBuffer(jsonRequest))
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode == 404 {
+		return nil, errors.Wrap(err, "public key not found")
+	}
+	if resp.StatusCode == 412 {
+		return nil, errors.Wrap(err, "signing operation failed due to slashing protection rules")
+	}
+	signResp := &signResponse{}
+	if err := client.unmarshalResponse(resp.Body, &signResp); err != nil {
+		return nil, err
+	}
+	decoded, err := decodeHex(signResp.Signature)
+	if err != nil {
+		return nil, err
+	}
+	return bls.SignatureFromBytes(decoded)
+}
+
+// GetPublicKeys is a wrapper method around the web3signer publickeys api (this may be removed in the future or moved to another location due to its usage).
+func (client *client) GetPublicKeys() ([][]byte, error) {
+	const requestPath = "/publicKeys"
+	resp, err := client.doRequest(http.MethodGet, client.BasePath+requestPath, nil)
+	if err != nil {
+		return nil, err
+	}
+	var publicKeys []string
+	if err := client.unmarshalResponse(resp.Body, &publicKeys); err != nil {
+		return nil, err
+	}
+	decodedKeys := make([][]byte, len(publicKeys))
+	var errorKeyPositions string
+	for i, value := range publicKeys {
+		decodedKey, err := decodeHex(value)
+		if err != nil {
+			errorKeyPositions += fmt.Sprintf("%v, ", i)
+			continue
+		}
+		decodedKeys[i] = decodedKey
+	}
+	if errorKeyPositions != "" {
+		return nil, errors.New("failed to decode from Hex from the following public key index locations: " + errorKeyPositions)
+	}
+	return decodedKeys, nil
+}
+
+// ReloadSignerKeys is a wrapper method around the web3signer reload api.
+func (client *client) ReloadSignerKeys() error {
+	const requestPath = "/reload"
+	if _, err := client.doRequest(http.MethodPost, client.BasePath+requestPath, nil); err != nil {
+		return err
+	}
+	return nil
+}
+
+// GetServerStatus is a wrapper method around the web3signer upcheck api
+func (client *client) GetServerStatus() (string, error) {
+	const requestPath = "/upcheck"
+	resp, err := client.doRequest(http.MethodGet, client.BasePath+requestPath, nil)
+	if err != nil {
+		return "", err
+	}
+	var status string
+	if err := client.unmarshalResponse(resp.Body, &status); err != nil {
+		return "", err
+	}
+	return status, nil
+}
+
+// doRequest is a utility method for requests.
+func (client *client) doRequest(httpMethod, fullPath string, body io.Reader) (*http.Response, error) {
+	req, err := http.NewRequest(httpMethod, fullPath, body)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid format, failed to create new Post Request Object")
+	}
+	resp, err := client.restClient.Do(req)
+	if err != nil {
+		return resp, errors.Wrap(err, "failed to execute json request")
+	}
+	if resp.StatusCode == 500 {
+		return nil, errors.Wrap(err, "internal Web3Signer server error")
+	} else if resp.StatusCode == 400 {
+		return nil, errors.Wrap(err, "bad request format")
+	}
+	return resp, err
+}
+
+// unmarshalResponse is a utility method for unmarshalling responses.
+func (*client) unmarshalResponse(responseBody io.ReadCloser, unmarshalledResponseObject interface{}) error {
+	defer closeBody(responseBody)
+	if err := json.NewDecoder(responseBody).Decode(&unmarshalledResponseObject); err != nil {
+		return errors.Wrap(err, "invalid format, unable to read response body as array of strings")
+	}
+	return nil
+}
+
+// decodeHex a utility method for decoding hex strings may be a duplicate in which case will be removed in the future.
+func decodeHex(signature string) ([]byte, error) {
+	decoded, err := hex.DecodeString(strings.TrimPrefix(signature, "0x"))
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid format, failed to unmarshal json response")
+	}
+	return decoded, nil
+}
+
+// closeBody a utility method to wrap an error for closing
+func closeBody(body io.Closer) {
+	if err := body.Close(); err != nil {
+		log.Errorf("could not close response body: %v", err)
+	}
+}

validator/keymanager/remote-web3signer/client_test.go

@@ -0,0 +1,96 @@
+package remote_web3signer
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// mockTransport is the mock Transport object
+type mockTransport struct {
+	mockResponse *http.Response
+}
+
+// RoundTrip is mocking my own implementation of the RoundTripper interface
+func (m *mockTransport) RoundTrip(*http.Request) (*http.Response, error) {
+	return m.mockResponse, nil
+}
+
+func TestClient_Sign_HappyPath(t *testing.T) {
+	json := `{
+  		"signature": "0xb3baa751d0a9132cfe93e4e3d5ff9075111100e3789dca219ade5a24d27e19d16b3353149da1833e9b691bb38634e8dc04469be7032132906c927d7e1a49b414730612877bc6b2810c8f202daf793d1ab0d6b5cb21d52f9e52e883859887a5d9"
+	}`
+	// create a new reader with that JSON
+	r := ioutil.NopCloser(bytes.NewReader([]byte(json)))
+	mock := &mockTransport{mockResponse: &http.Response{
+		StatusCode: 200,
+		Body:       r,
+	}}
+	cl := client{BasePath: "example.com", restClient: &http.Client{Transport: mock}}
+	forkData := &Fork{
+		PreviousVersion: "",
+		CurrentVersion:  "",
+		Epoch:           "",
+	}
+	forkInfoData := &ForkInfo{
+		Fork:                  forkData,
+		GenesisValidatorsRoot: "",
+	}
+
+	AggregationSlotData := &AggregationSlot{Slot: ""}
+	// remember to replace signing root with hex encoding remove 0x
+	web3SignerRequest := SignRequest{
+		Type:            "foo",
+		ForkInfo:        forkInfoData,
+		SigningRoot:     "0xfasd0fjsa0dfjas0dfjasdf",
+		AggregationSlot: AggregationSlotData,
+	}
+	resp, err := cl.Sign("a2b5aaad9c6efefe7bb9b1243a043404f3362937cfb6b31833929833173f476630ea2cfeb0d9ddf15f97ca8685948820", &web3SignerRequest)
+	assert.NotNil(t, resp)
+	assert.Nil(t, err)
+	assert.EqualValues(t, "0xb3baa751d0a9132cfe93e4e3d5ff9075111100e3789dca219ade5a24d27e19d16b3353149da1833e9b691bb38634e8dc04469be7032132906c927d7e1a49b414730612877bc6b2810c8f202daf793d1ab0d6b5cb21d52f9e52e883859887a5d9", fmt.Sprintf("%#x", resp.Marshal()))
+}
+
+func TestClient_GetPublicKeys_HappyPath(t *testing.T) {
+	// public keys are returned hex encoded with 0x
+	json := `["0x613262356161616439633665666566653762623962313234336130343334303466333336323933376366623662333138333339323938333331373366343736363330656132636665623064396464663135663937636138363835393438383230","0x613262356161616439633665666566653762623962313234336130343334303466333336323933376366623662333138333339323938333331373366343736363330656132636665623064396464663135663937636138363835393438383230"]`
+	// create a new reader with that JSON
+	r := ioutil.NopCloser(bytes.NewReader([]byte(json)))
+	mock := &mockTransport{mockResponse: &http.Response{
+		StatusCode: 200,
+		Body:       r,
+	}}
+	cl := client{BasePath: "example.com", restClient: &http.Client{Transport: mock}}
+	resp, err := cl.GetPublicKeys()
+	assert.NotNil(t, resp)
+	assert.Nil(t, err)
+	// we would like them as 48byte base64 without 0x
+	assert.EqualValues(t, "a2b5aaad9c6efefe7bb9b1243a043404f3362937cfb6b31833929833173f476630ea2cfeb0d9ddf15f97ca8685948820", string(resp[0]))
+}
+
+func TestClient_ReloadSignerKeys_HappyPath(t *testing.T) {
+	mock := &mockTransport{mockResponse: &http.Response{
+		StatusCode: 200,
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+	}}
+	cl := client{BasePath: "example.com", restClient: &http.Client{Transport: mock}}
+	err := cl.ReloadSignerKeys()
+	assert.Nil(t, err)
+}
+
+func TestClient_GetServerStatus_HappyPath(t *testing.T) {
+	json := `"some server status, not sure what it looks like, need to find some sample data"`
+	r := ioutil.NopCloser(bytes.NewReader([]byte(json)))
+	mock := &mockTransport{mockResponse: &http.Response{
+		StatusCode: 200,
+		Body:       r,
+	}}
+	cl := client{BasePath: "example.com", restClient: &http.Client{Transport: mock}}
+	resp, err := cl.GetServerStatus()
+	assert.NotNil(t, resp)
+	assert.Nil(t, err)
+}

validator/keymanager/remote-web3signer/log.go

@@ -0,0 +1,7 @@
+package remote_web3signer
+
+import (
+	"github.com/sirupsen/logrus"
+)
+
+var log = logrus.WithField("prefix", "remote_web3signer")