E2E: fee-recipient evaluator (#10528)

* testing out fee-recipient evaluator

* fixing bazel linter

* adjusting comparison

* typo on file rolling back

* adding fee recipient is present to minimal e2e

* fixing gofmt

* fixing gofmt

* fixing flag usage name

* adding in log to help debug

* fixing log build

* trying to figure out why suggested fee recipient isn't working in e2e, adding more logging temporarily

* rolling back logs

* making e2e test more dynamic

* fixing deepsource issue

* fixing bazel

* adding in condition for latest release

* duplicate condtion check.

* fixing gofmt

* rolling back changes

* adding fee recipient evaluator in new file

* fixing validator component logic

* testing rpc client addition

* testing fee recipient evaluator

* fixing bazel:

* testing casting

* test casting

* reverting

* testing casting

* testing casting

* testing log

* adding bazel fix

* switching mixed case and adding temp logging

* fixing gofmt

* rolling back changes

* removing fee recipient evaluator when web3signer is used

* test only minimal config

* reverting changes

* adding fee recipient evaluator to mainnet

* current version uses wrong flag name

* optimizing key usage

* making mining address a variable

* moving from global to local variable

* removing unneeded log

* removing redundant check

* make proposer settings mroe deterministic and also have the evaluator compare the wanting values

* fixing err return

* fixing bazel

* checking file too much moving it out

* fixing gosec

* trying to fix gosec error

* trying to fix address

* fixing linting

* trying to gerenate key and random address

* fixing linting

* fixing check for proposer config

* trying with multi config files

* fixing is dir check

* testing for older previous balance

* adding logging to help debug

* changing how i get the block numbers

* fixing missed error check

* adding gasused check

* adding log for current gas used

* taking suggestion to make fee recipient more deterministic

* fixing linting

* fixing check

* fixing the address check

* fixing format error

* logic to differentiate recipients

* fixing linting

config/validator/service/proposer-settings.go

@@ -7,11 +7,11 @@ import (
 )
 
 // ProposerSettingsPayload is the struct representation of the JSON or YAML payload set in the validator through the CLI.
-// ProposeConfig is the map of validator address to fee recipient options all in hex format.
+// ProposerConfig is the map of validator address to fee recipient options all in hex format.
 // DefaultConfig is the default fee recipient address for all validators unless otherwise specified in the propose config.required.
 type ProposerSettingsPayload struct {
-	ProposeConfig map[string]*ProposerOptionPayload `json:"proposer_config" yaml:"proposer_config"`
-	DefaultConfig *ProposerOptionPayload            `json:"default_config" yaml:"default_config"`
+	ProposerConfig map[string]*ProposerOptionPayload `json:"proposer_config" yaml:"proposer_config"`
+	DefaultConfig  *ProposerOptionPayload            `json:"default_config" yaml:"default_config"`
 }
 
 // ProposerOptionPayload is the struct representation of the JSON config file set in the validator through the CLI.

testing/endtoend/components/BUILD.bazel

@@ -24,7 +24,9 @@ go_library(
         "//cmd/beacon-chain/flags:go_default_library",
         "//cmd/validator/flags:go_default_library",
         "//config/features:go_default_library",
+        "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
+        "//config/validator/service:go_default_library",
         "//contracts/deposit:go_default_library",
         "//crypto/bls:go_default_library",
         "//encoding/bytesutil:go_default_library",
@@ -38,6 +40,7 @@ go_library(
         "//validator/keymanager:go_default_library",
         "@com_github_ethereum_go_ethereum//accounts/abi/bind:go_default_library",
         "@com_github_ethereum_go_ethereum//accounts/keystore:go_default_library",
+        "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_ethereum_go_ethereum//ethclient:go_default_library",
         "@com_github_ethereum_go_ethereum//rpc:go_default_library",

testing/endtoend/components/eth1/miner.go

@@ -27,6 +27,10 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+const (
+	EthAddress = "0x878705ba3f8bc32fcf7f4caa1a35e72af65cf766"
+)
+
 // Miner represents an ETH1 node which mines blocks.
 type Miner struct {
 	e2etypes.ComponentRunner
@@ -127,9 +131,9 @@ func (m *Miner) Start(ctx context.Context) error {
 		"--ipcdisable",
 		"--verbosity=4",
 		"--mine",
-		"--unlock=0x878705ba3f8bc32fcf7f4caa1a35e72af65cf766",
+		fmt.Sprintf("--unlock=%s", EthAddress),
 		"--allow-insecure-unlock",
-		"--txpool.locals=0x878705ba3f8bc32fcf7f4caa1a35e72af65cf766",
+		fmt.Sprintf("--txpool.locals=%s", EthAddress),
 		fmt.Sprintf("--password=%s", eth1Path+"/keystore/"+minerPasswordFile),
 	}
 

testing/endtoend/components/eth1/node.go

@@ -90,7 +90,7 @@ func (node *Node) Start(ctx context.Context) error {
 		"--ws.origins=\"*\"",
 		"--ipcdisable",
 		"--verbosity=4",
-		"--txpool.locals=0x878705ba3f8bc32fcf7f4caa1a35e72af65cf766",
+		fmt.Sprintf("--txpool.locals=%s", EthAddress),
 	}
 	// If we are testing sync, geth needs to be run via full sync as snap sync does not
 	// work in our setup.

testing/endtoend/components/validator.go

@@ -3,17 +3,20 @@ package components
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"math/big"
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"strings"
 	"syscall"
 
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -21,9 +24,12 @@ import (
 	cmdshared "github.com/prysmaticlabs/prysm/cmd"
 	"github.com/prysmaticlabs/prysm/cmd/validator/flags"
 	"github.com/prysmaticlabs/prysm/config/features"
+	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/config/params"
+	validator_service_config "github.com/prysmaticlabs/prysm/config/validator/service"
 	contracts "github.com/prysmaticlabs/prysm/contracts/deposit"
 	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	"github.com/prysmaticlabs/prysm/io/file"
 	"github.com/prysmaticlabs/prysm/runtime/interop"
 	"github.com/prysmaticlabs/prysm/testing/endtoend/components/eth1"
 	"github.com/prysmaticlabs/prysm/testing/endtoend/helpers"
@@ -33,6 +39,7 @@ import (
 )
 
 const depositGasLimit = 4000000
+const DefaultFeeRecipientAddress = "0x099FB65722e7b2455043bfebF6177f1D2E9738d9"
 
 var _ e2etypes.ComponentRunner = (*ValidatorNode)(nil)
 var _ e2etypes.ComponentRunner = (*ValidatorNodeSet)(nil)
@@ -170,6 +177,7 @@ func NewValidatorNode(config *e2etypes.E2EConfig, validatorNum, index, offset in
 
 // Start starts a validator client.
 func (v *ValidatorNode) Start(ctx context.Context) error {
+	validatorHexPubKeys := make([]string, 0)
 	var pkg, target string
 	if v.config.UsePrysmShValidator {
 		pkg = ""
@@ -198,6 +206,15 @@ func (v *ValidatorNode) Start(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+
+	_, pubs, err := interop.DeterministicallyGenerateKeys(uint64(offset), uint64(validatorNum))
+	if err != nil {
+		return err
+	}
+	for _, pub := range pubs {
+		validatorHexPubKeys = append(validatorHexPubKeys, hexutil.Encode(pub.Marshal()))
+	}
+
 	args := []string{
 		fmt.Sprintf("--%s=%s/eth2-val-%d", cmdshared.DataDirFlag.Name, e2e.TestParams.TestPath, index),
 		fmt.Sprintf("--%s=%s", cmdshared.LogFileName.Name, file.Name()),
@@ -219,20 +236,22 @@ func (v *ValidatorNode) Start(ctx context.Context) error {
 		args = append(args, fmt.Sprintf("--%s=http://localhost:%d", flags.Web3SignerURLFlag.Name, Web3RemoteSignerPort))
 		// Write the pubkeys as comma seperated hex strings with 0x prefix.
 		// See: https://docs.teku.consensys.net/en/latest/HowTo/External-Signer/Use-External-Signer/
-		_, pubs, err := interop.DeterministicallyGenerateKeys(uint64(offset), uint64(validatorNum))
-		if err != nil {
-			return err
-		}
-		var hexPubs []string
-		for _, pub := range pubs {
-			hexPubs = append(hexPubs, hexutil.Encode(pub.Marshal()))
-		}
-		args = append(args, fmt.Sprintf("--%s=%s", flags.Web3SignerPublicValidatorKeysFlag.Name, strings.Join(hexPubs, ",")))
+		args = append(args, fmt.Sprintf("--%s=%s", flags.Web3SignerPublicValidatorKeysFlag.Name, strings.Join(validatorHexPubKeys, ",")))
 	} else {
 		// When not using remote key signer, use interop keys.
 		args = append(args,
 			fmt.Sprintf("--%s=%d", flags.InteropNumValidators.Name, validatorNum),
-			fmt.Sprintf("--%s=%d", flags.InteropStartIndex.Name, offset))
+			fmt.Sprintf("--%s=%d", flags.InteropStartIndex.Name, offset),
+		)
+	}
+	//TODO: web3signer does not support validator registration signing currently, move this when support is there.
+	//TODO: current version of prysmsh still uses wrong flag name.
+	if !v.config.UsePrysmShValidator && !v.config.UseWeb3RemoteSigner {
+		proposerSettingsPathPath, err := createProposerSettingsPath(validatorHexPubKeys, index)
+		if err != nil {
+			return err
+		}
+		args = append(args, fmt.Sprintf("--%s=%s", flags.ProposerSettingsFlag.Name, proposerSettingsPathPath))
 	}
 	args = append(args, config.ValidatorFlags...)
 
@@ -378,3 +397,50 @@ func sendDeposits(web3 *ethclient.Client, keystoreBytes []byte, num, offset int,
 	}
 	return nil
 }
+
+func createProposerSettingsPath(pubkeys []string, validatorIndex int) (string, error) {
+	testNetDir := e2e.TestParams.TestPath + fmt.Sprintf("/proposer-settings/validator_%d", validatorIndex)
+	configPath := filepath.Join(testNetDir, "config.json")
+	if len(pubkeys) == 0 {
+		return "", errors.New("number of validators must be greater than 0")
+	}
+	var proposerSettingsPayload validator_service_config.ProposerSettingsPayload
+	if len(pubkeys) == 1 {
+		proposerSettingsPayload = validator_service_config.ProposerSettingsPayload{
+			DefaultConfig: &validator_service_config.ProposerOptionPayload{
+				FeeRecipient: DefaultFeeRecipientAddress,
+			},
+		}
+	} else {
+		config := make(map[string]*validator_service_config.ProposerOptionPayload)
+
+		for i, pubkey := range pubkeys {
+			// Create an account
+			byteval, err := hexutil.Decode(pubkey)
+			if err != nil {
+				return "", err
+			}
+			deterministicFeeRecipient := common.HexToAddress(hexutil.Encode(byteval[:fieldparams.FeeRecipientLength])).Hex()
+			config[pubkeys[i]] = &validator_service_config.ProposerOptionPayload{
+				FeeRecipient: deterministicFeeRecipient,
+			}
+		}
+		proposerSettingsPayload = validator_service_config.ProposerSettingsPayload{
+			ProposerConfig: config,
+			DefaultConfig: &validator_service_config.ProposerOptionPayload{
+				FeeRecipient: DefaultFeeRecipientAddress,
+			},
+		}
+	}
+	jsonBytes, err := json.Marshal(proposerSettingsPayload)
+	if err != nil {
+		return "", err
+	}
+	if err := file.MkdirAll(testNetDir); err != nil {
+		return "", err
+	}
+	if err := file.WriteFile(configPath, jsonBytes); err != nil {
+		return "", err
+	}
+	return configPath, nil
+}

testing/endtoend/endtoend_setup_test.go

@@ -82,6 +82,10 @@ func e2eMinimal(t *testing.T, cfgo ...types.E2EConfigOpt) *testRunner {
 	for _, o := range cfgo {
 		o(testConfig)
 	}
+	//TODO: web3signer does not currently support validator registration signing so evaluator will break code
+	if !testConfig.UseWeb3RemoteSigner {
+		testConfig.Evaluators = append(testConfig.Evaluators, ev.FeeRecipientIsPresent)
+	}
 
 	return newTestRunner(t, testConfig)
 }
@@ -158,7 +162,10 @@ func e2eMainnet(t *testing.T, usePrysmSh, useMultiClient bool, cfgo ...types.E2E
 	for _, o := range cfgo {
 		o(testConfig)
 	}
-
+	//TODO: web3signer does not currently support validator registration signing so evaluator will break code
+	if !testConfig.UseWeb3RemoteSigner {
+		testConfig.Evaluators = append(testConfig.Evaluators, ev.FeeRecipientIsPresent)
+	}
 	return newTestRunner(t, testConfig)
 }
 

testing/endtoend/evaluators/BUILD.bazel

@@ -8,6 +8,7 @@ go_library(
         "api_middleware.go",
         "data.go",
         "execution_engine.go",
+        "fee_recipient.go",
         "finality.go",
         "fork.go",
         "metrics.go",
@@ -37,15 +38,21 @@ go_library(
         "//proto/eth/v1:go_default_library",
         "//proto/eth/v2:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
+        "//runtime/interop:go_default_library",
+        "//testing/endtoend/components:go_default_library",
         "//testing/endtoend/helpers:go_default_library",
         "//testing/endtoend/params:go_default_library",
         "//testing/endtoend/policies:go_default_library",
         "//testing/endtoend/types:go_default_library",
         "//testing/util:go_default_library",
         "//time/slots:go_default_library",
+        "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
+        "@com_github_ethereum_go_ethereum//ethclient:go_default_library",
+        "@com_github_ethereum_go_ethereum//rpc:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prysmaticlabs_go_bitfield//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
         "@io_bazel_rules_go//proto/wkt:empty_go_proto",
         "@org_golang_google_grpc//:go_default_library",
         "@org_golang_google_protobuf//types/known/emptypb:go_default_library",

testing/endtoend/evaluators/fee_recipient.go

@@ -0,0 +1,123 @@
+package evaluators
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/pkg/errors"
+	fieldparams "github.com/prysmaticlabs/prysm/config/fieldparams"
+	"github.com/prysmaticlabs/prysm/config/params"
+	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/runtime/interop"
+	"github.com/prysmaticlabs/prysm/testing/endtoend/components"
+	"github.com/prysmaticlabs/prysm/testing/endtoend/helpers"
+	e2e "github.com/prysmaticlabs/prysm/testing/endtoend/params"
+	"github.com/prysmaticlabs/prysm/testing/endtoend/policies"
+	"github.com/prysmaticlabs/prysm/testing/endtoend/types"
+	log "github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
+	"google.golang.org/protobuf/types/known/emptypb"
+)
+
+var FeeRecipientIsPresent = types.Evaluator{
+	Name:       "Fee_Recipient_Is_Present_%d",
+	Policy:     policies.AfterNthEpoch(helpers.BellatrixE2EForkEpoch),
+	Evaluation: feeRecipientIsPresent,
+}
+
+func feeRecipientIsPresent(conns ...*grpc.ClientConn) error {
+	conn := conns[0]
+	client := ethpb.NewBeaconChainClient(conn)
+	chainHead, err := client.GetChainHead(context.Background(), &emptypb.Empty{})
+	if err != nil {
+		return errors.Wrap(err, "failed to get chain head")
+	}
+	req := &ethpb.ListBlocksRequest{QueryFilter: &ethpb.ListBlocksRequest_Epoch{Epoch: chainHead.HeadEpoch.Sub(1)}}
+	blks, err := client.ListBeaconBlocks(context.Background(), req)
+	if err != nil {
+		return errors.Wrap(err, "failed to list blocks")
+	}
+
+	rpcclient, err := rpc.DialHTTP(fmt.Sprintf("http://127.0.0.1:%d", e2e.TestParams.Ports.Eth1RPCPort))
+	if err != nil {
+		return err
+	}
+	defer rpcclient.Close()
+	web3 := ethclient.NewClient(rpcclient)
+	ctx := context.Background()
+
+	for _, ctr := range blks.BlockContainers {
+		switch ctr.Block.(type) {
+		case *ethpb.BeaconBlockContainer_BellatrixBlock:
+			var account common.Address
+
+			fr := ctr.GetBellatrixBlock().Block.Body.ExecutionPayload.FeeRecipient
+			if len(fr) != 0 && hexutil.Encode(fr) != params.BeaconConfig().EthBurnAddressHex {
+				account = common.BytesToAddress(fr)
+			} else {
+				return errors.New("fee recipient is not set")
+			}
+			validatorRequest := &ethpb.GetValidatorRequest{
+				QueryFilter: &ethpb.GetValidatorRequest_Index{
+					Index: ctr.GetBellatrixBlock().Block.ProposerIndex,
+				},
+			}
+			validator, err := client.GetValidator(context.Background(), validatorRequest)
+			if err != nil {
+				return errors.Wrap(err, "failed to get validators")
+			}
+			publickey := validator.GetPublicKey()
+			isDeterministicKey := false
+			validatorNum := int(params.BeaconConfig().MinGenesisActiveValidatorCount) // matches validator start in validator component
+			_, pubs, err := interop.DeterministicallyGenerateKeys(uint64(0), uint64(validatorNum))
+			if err != nil {
+				return err
+			}
+			for _, pub := range pubs {
+				if hexutil.Encode(publickey) == hexutil.Encode(pub.Marshal()) {
+					isDeterministicKey = true
+					break
+				}
+			}
+			// calculate deterministic fee recipient using first 20 bytes of public key
+			deterministicFeeRecipient := common.HexToAddress(hexutil.Encode(publickey[:fieldparams.FeeRecipientLength])).Hex()
+			if isDeterministicKey && deterministicFeeRecipient != account.Hex() {
+				return fmt.Errorf("publickey %s, fee recipient %s does not match the proposer settings fee recipient %s",
+					hexutil.Encode(publickey), account.Hex(), deterministicFeeRecipient)
+			}
+			if !isDeterministicKey && components.DefaultFeeRecipientAddress != account.Hex() {
+				return fmt.Errorf("publickey %s, fee recipient %s does not match the default fee recipient %s",
+					hexutil.Encode(publickey), account.Hex(), components.DefaultFeeRecipientAddress)
+			}
+			currentBlock, err := web3.BlockByHash(ctx, common.BytesToHash(ctr.GetBellatrixBlock().GetBlock().GetBody().GetExecutionPayload().BlockHash))
+			if err != nil {
+				return err
+			}
+
+			accountBalance, err := web3.BalanceAt(ctx, account, currentBlock.Number())
+			if err != nil {
+				return err
+			}
+			previousBlock, err := web3.BlockByHash(ctx, common.BytesToHash(ctr.GetBellatrixBlock().GetBlock().GetBody().GetExecutionPayload().ParentHash))
+			if err != nil {
+				return err
+			}
+			prevAccountBalance, err := web3.BalanceAt(ctx, account, previousBlock.Number())
+			if err != nil {
+				return err
+			}
+			if currentBlock.GasUsed() > 0 && accountBalance.Uint64() <= prevAccountBalance.Uint64() {
+				log.Infof("current block num: %d , previous block num: %d , account balance: %d,  pre account balance %d", currentBlock.Number(), previousBlock.Number(), accountBalance, prevAccountBalance)
+				return errors.Errorf("account balance didn't change after applying fee recipient for account: %s", account.Hex())
+			} else {
+				log.Infof("current gas used: %v current account balance %v ,increased from previous account balance %v ", currentBlock.GasUsed(), accountBalance, prevAccountBalance)
+			}
+		}
+	}
+
+	return nil
+}

validator/node/node.go

@@ -489,7 +489,7 @@ func proposerSettings(cliCtx *cli.Context) (*validatorServiceConfig.ProposerSett
 	if cliCtx.IsSet(flags.SuggestedFeeRecipientFlag.Name) {
 		suggestedFee := cliCtx.String(flags.SuggestedFeeRecipientFlag.Name)
 		fileConfig = &validatorServiceConfig.ProposerSettingsPayload{
-			ProposeConfig: nil,
+			ProposerConfig: nil,
 			DefaultConfig: &validatorServiceConfig.ProposerOptionPayload{
 				FeeRecipient: suggestedFee,
 				GasLimit:     params.BeaconConfig().DefaultBuilderGasLimit,
@@ -538,9 +538,9 @@ func proposerSettings(cliCtx *cli.Context) (*validatorServiceConfig.ProposerSett
 		GasLimit:     reviewGasLimit(fileConfig.DefaultConfig.GasLimit),
 	}
 
-	if fileConfig.ProposeConfig != nil {
+	if fileConfig.ProposerConfig != nil {
 		vpSettings.ProposeConfig = make(map[[fieldparams.BLSPubkeyLength]byte]*validatorServiceConfig.ProposerOption)
-		for key, option := range fileConfig.ProposeConfig {
+		for key, option := range fileConfig.ProposerConfig {
 			decodedKey, err := hexutil.Decode(key)
 			if err != nil {
 				return nil, errors.Wrapf(err, "could not decode public key %s", key)