persistent validator settings in validator client (#12354)

* WIP

* improving proposer settings store

* WIP persistent validator settings

* WIP persistent validator settings

* changing visibility level

* fixing some deepsource issues

* fixing more deepsource issues

* fixing json marshalling

* fix linting

* fixing tests

* fixing more tests

* fixing more tests

* fixing more tests

* fixing linting

* WIP fixing unit tests

* fixing remaining db tests

* converting json to protobuf

* fixing e2e

* k8s yaml library is used directly

* fixing linting

* fixing broken unit test

* reverting changes on e2e

* fixing linting

* fixing deepsource issue

* resolving some internal comments

* resolving some comments and adding more tests

* adding more unit tests

* gaz

* fixing flaking test

* fixing unit test contamination

* fixing deepsource issue

* resolving review item

* gaz

validator/db/iface/BUILD.bazel

@@ -5,9 +5,10 @@ go_library(
     srcs = ["interface.go"],
     importpath = "github.com/prysmaticlabs/prysm/v4/validator/db/iface",
     # Other packages must use github.com/prysmaticlabs/prysm/v4/validator/db.Database alias.
-    visibility = ["//validator/db:__subpackages__"],
+    visibility = ["//validator:__subpackages__"],
     deps = [
         "//config/fieldparams:go_default_library",
+        "//config/validator/service:go_default_library",
         "//consensus-types/primitives:go_default_library",
         "//monitoring/backup:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",

validator/db/iface/interface.go

@@ -6,6 +6,7 @@ import (
 	"io"
 
 	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
+	validatorServiceConfig "github.com/prysmaticlabs/prysm/v4/config/validator/service"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v4/monitoring/backup"
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
@@ -62,4 +63,11 @@ type ValidatorDB interface {
 	// Graffiti ordered index related methods
 	SaveGraffitiOrderedIndex(ctx context.Context, index uint64) error
 	GraffitiOrderedIndex(ctx context.Context, fileHash [32]byte) (uint64, error)
+
+	// ProposerSettings related methods
+	ProposerSettings(context.Context) (*validatorServiceConfig.ProposerSettings, error)
+	ProposerSettingsExists(ctx context.Context) (bool, error)
+	UpdateProposerSettingsDefault(context.Context, *validatorServiceConfig.ProposerOption) error
+	UpdateProposerSettingsForPubkey(context.Context, [fieldparams.BLSPubkeyLength]byte, *validatorServiceConfig.ProposerOption) error
+	SaveProposerSettings(ctx context.Context, settings *validatorServiceConfig.ProposerSettings) error
 }

validator/db/kv/BUILD.bazel

@@ -15,6 +15,7 @@ go_library(
         "migration_optimal_attester_protection.go",
         "migration_source_target_epochs_bucket.go",
         "proposer_protection.go",
+        "proposer_settings.go",
         "prune_attester_protection.go",
         "schema.go",
     ],
@@ -29,6 +30,7 @@ go_library(
         "//config/features:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
+        "//config/validator/service:go_default_library",
         "//consensus-types/primitives:go_default_library",
         "//encoding/bytesutil:go_default_library",
         "//io/file:go_default_library",
@@ -36,6 +38,7 @@ go_library(
         "//monitoring/tracing:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//proto/prysm/v1alpha1/slashings:go_default_library",
+        "//proto/prysm/v1alpha1/validator-client:go_default_library",
         "//time/slots:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
@@ -43,6 +46,7 @@ go_library(
         "@com_github_sirupsen_logrus//:go_default_library",
         "@io_etcd_go_bbolt//:go_default_library",
         "@io_opencensus_go//trace:go_default_library",
+        "@org_golang_google_protobuf//proto:go_default_library",
     ],
 )
 
@@ -59,18 +63,23 @@ go_test(
         "migration_optimal_attester_protection_test.go",
         "migration_source_target_epochs_bucket_test.go",
         "proposer_protection_test.go",
+        "proposer_settings_test.go",
         "prune_attester_protection_test.go",
     ],
     embed = [":go_default_library"],
     deps = [
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
+        "//config/validator/service:go_default_library",
         "//consensus-types/primitives:go_default_library",
+        "//consensus-types/validator:go_default_library",
         "//crypto/hash:go_default_library",
         "//encoding/bytesutil:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/require:go_default_library",
+        "@com_github_ethereum_go_ethereum//common:go_default_library",
+        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
         "@io_etcd_go_bbolt//:go_default_library",

validator/db/kv/db.go

@@ -152,6 +152,7 @@ func NewKVStore(ctx context.Context, dirPath string, config *Config) (*Store, er
 			pubKeysBucket,
 			migrationsBucket,
 			graffitiBucket,
+			proposerSettingsBucket,
 		)
 	}); err != nil {
 		return nil, err

validator/db/kv/proposer_settings.go

@@ -0,0 +1,132 @@
+package kv
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
+	validatorServiceConfig "github.com/prysmaticlabs/prysm/v4/config/validator/service"
+	validatorpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1/validator-client"
+	bolt "go.etcd.io/bbolt"
+	"go.opencensus.io/trace"
+	"google.golang.org/protobuf/proto"
+)
+
+// NoProposerSettingsFound is an error thrown when no settings are found in bucket
+var NoProposerSettingsFound = errors.New("no proposer settings found in bucket")
+
+// UpdateProposerSettingsForPubkey updates the existing settings for an internal representation of the proposers settings file at a particular public key
+func (s *Store) UpdateProposerSettingsForPubkey(ctx context.Context, pubkey [fieldparams.BLSPubkeyLength]byte, options *validatorServiceConfig.ProposerOption) error {
+	_, span := trace.StartSpan(ctx, "validator.db.UpdateProposerSettingsForPubkey")
+	defer span.End()
+	err := s.db.Update(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket(proposerSettingsBucket)
+		b := bkt.Get(proposerSettingsKey)
+		if len(b) == 0 {
+			return fmt.Errorf("no proposer settings found in bucket")
+		}
+		to := &validatorpb.ProposerSettingsPayload{}
+		if err := proto.Unmarshal(b, to); err != nil {
+			return errors.Wrap(err, "failed to unmarshal proposer settings")
+		}
+		settings, err := validatorServiceConfig.ToSettings(to)
+		if err != nil {
+			return errors.Wrap(err, "failed to convert payload to proposer settings")
+		}
+		if settings.ProposeConfig == nil {
+			settings.ProposeConfig = make(map[[fieldparams.BLSPubkeyLength]byte]*validatorServiceConfig.ProposerOption)
+		}
+		settings.ProposeConfig[pubkey] = options
+		m, err := proto.Marshal(settings.ToPayload())
+		if err != nil {
+			return errors.Wrap(err, "failed to marshal proposer settings")
+		}
+		return bkt.Put(proposerSettingsKey, m)
+	})
+	return err
+}
+
+// UpdateProposerSettingsDefault updates the existing default settings for proposer settings
+func (s *Store) UpdateProposerSettingsDefault(ctx context.Context, options *validatorServiceConfig.ProposerOption) error {
+	_, span := trace.StartSpan(ctx, "validator.db.UpdateProposerSettingsDefault")
+	defer span.End()
+	if options == nil {
+		return errors.New("proposer settings option was empty")
+	}
+	if options.FeeRecipientConfig == nil {
+		return errors.New("fee recipient cannot be empty")
+	}
+	err := s.db.Update(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket(proposerSettingsBucket)
+		b := bkt.Get(proposerSettingsKey)
+		if len(b) == 0 {
+			return NoProposerSettingsFound
+		}
+		to := &validatorpb.ProposerSettingsPayload{}
+		if err := proto.Unmarshal(b, to); err != nil {
+			return errors.Wrap(err, "failed to unmarshal proposer settings")
+		}
+		settings, err := validatorServiceConfig.ToSettings(to)
+		if err != nil {
+			return errors.Wrap(err, "failed to convert payload to proposer settings")
+		}
+		settings.DefaultConfig = options
+		m, err := proto.Marshal(settings.ToPayload())
+		if err != nil {
+			return errors.Wrap(err, "failed to marshal proposer settings")
+		}
+		return bkt.Put(proposerSettingsKey, m)
+	})
+	return err
+}
+
+// ProposerSettings gets the current proposer settings
+func (s *Store) ProposerSettings(ctx context.Context) (*validatorServiceConfig.ProposerSettings, error) {
+	_, span := trace.StartSpan(ctx, "validator.db.ProposerSettings")
+	defer span.End()
+	to := &validatorpb.ProposerSettingsPayload{}
+	if err := s.db.View(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket(proposerSettingsBucket)
+		b := bkt.Get(proposerSettingsKey)
+		if len(b) == 0 {
+			return NoProposerSettingsFound
+		}
+		if err := proto.Unmarshal(b, to); err != nil {
+			return errors.Wrap(err, "failed to unmarshal proposer settings")
+		}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return validatorServiceConfig.ToSettings(to)
+}
+
+// ProposerSettingsExists returns true or false if the settings exist or not
+func (s *Store) ProposerSettingsExists(ctx context.Context) (bool, error) {
+	ps, err := s.ProposerSettings(ctx)
+	if err != nil {
+		if errors.Is(err, NoProposerSettingsFound) {
+			return false, nil
+		}
+		return false, err
+	}
+	if ps == nil {
+		return false, nil
+	}
+	return true, nil
+}
+
+// SaveProposerSettings saves the entire proposer setting overriding the existing settings
+func (s *Store) SaveProposerSettings(ctx context.Context, settings *validatorServiceConfig.ProposerSettings) error {
+	_, span := trace.StartSpan(ctx, "validator.db.SaveProposerSettings")
+	defer span.End()
+	return s.db.Update(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket(proposerSettingsBucket)
+		m, err := proto.Marshal(settings.ToPayload())
+		if err != nil {
+			return errors.Wrap(err, "failed to marshal proposer settings")
+		}
+		return bkt.Put(proposerSettingsKey, m)
+	})
+}

validator/db/kv/proposer_settings_test.go

@@ -0,0 +1,106 @@
+package kv
+
+import (
+	"context"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
+	"github.com/prysmaticlabs/prysm/v4/config/params"
+	validatorServiceConfig "github.com/prysmaticlabs/prysm/v4/config/validator/service"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/validator"
+	"github.com/prysmaticlabs/prysm/v4/encoding/bytesutil"
+	"github.com/prysmaticlabs/prysm/v4/testing/require"
+)
+
+func TestStore_ProposerSettings_ReadAndWrite(t *testing.T) {
+	t.Run("save to db in full", func(t *testing.T) {
+		ctx := context.Background()
+		db := setupDB(t, [][fieldparams.BLSPubkeyLength]byte{})
+		key1, err := hexutil.Decode("0xa057816155ad77931185101128655c0191bd0214c201ca48ed887f6c4c6adf334070efcd75140eada5ac83a92506dd7a")
+		require.NoError(t, err)
+		settings := &validatorServiceConfig.ProposerSettings{
+			ProposeConfig: map[[fieldparams.BLSPubkeyLength]byte]*validatorServiceConfig.ProposerOption{
+				bytesutil.ToBytes48(key1): {
+					FeeRecipientConfig: &validatorServiceConfig.FeeRecipientConfig{
+						FeeRecipient: common.HexToAddress("0x50155530FCE8a85ec7055A5F8b2bE214B3DaeFd3"),
+					},
+					BuilderConfig: &validatorServiceConfig.BuilderConfig{
+						Enabled:  true,
+						GasLimit: validator.Uint64(40000000),
+					},
+				},
+			},
+			DefaultConfig: &validatorServiceConfig.ProposerOption{
+				FeeRecipientConfig: &validatorServiceConfig.FeeRecipientConfig{
+					FeeRecipient: common.HexToAddress("0x6e35733c5af9B61374A128e6F85f553aF09ff89A"),
+				},
+				BuilderConfig: &validatorServiceConfig.BuilderConfig{
+					Enabled:  false,
+					GasLimit: validator.Uint64(params.BeaconConfig().DefaultBuilderGasLimit),
+				},
+			},
+		}
+		err = db.SaveProposerSettings(ctx, settings)
+		require.NoError(t, err)
+
+		dbSettings, err := db.ProposerSettings(ctx)
+		require.NoError(t, err)
+		require.DeepEqual(t, settings, dbSettings)
+	})
+	t.Run("update default settings then update at specific key", func(t *testing.T) {
+		ctx := context.Background()
+		db := setupDB(t, [][fieldparams.BLSPubkeyLength]byte{})
+		key1, err := hexutil.Decode("0xa057816155ad77931185101128655c0191bd0214c201ca48ed887f6c4c6adf334070efcd75140eada5ac83a92506dd7a")
+		require.NoError(t, err)
+		settings := &validatorServiceConfig.ProposerSettings{
+			DefaultConfig: &validatorServiceConfig.ProposerOption{
+				FeeRecipientConfig: &validatorServiceConfig.FeeRecipientConfig{
+					FeeRecipient: common.HexToAddress("0x6e35733c5af9B61374A128e6F85f553aF09ff89A"),
+				},
+				BuilderConfig: &validatorServiceConfig.BuilderConfig{
+					Enabled:  false,
+					GasLimit: validator.Uint64(params.BeaconConfig().DefaultBuilderGasLimit),
+				},
+			},
+		}
+		err = db.SaveProposerSettings(ctx, settings)
+		require.NoError(t, err)
+		upatedDefault := &validatorServiceConfig.ProposerOption{
+			FeeRecipientConfig: &validatorServiceConfig.FeeRecipientConfig{
+				FeeRecipient: common.HexToAddress("0x9995733c5af9B61374A128e6F85f553aF09ff89B"),
+			},
+			BuilderConfig: &validatorServiceConfig.BuilderConfig{
+				Enabled:  true,
+				GasLimit: validator.Uint64(params.BeaconConfig().DefaultBuilderGasLimit),
+			},
+		}
+		err = db.UpdateProposerSettingsDefault(ctx, upatedDefault)
+		require.NoError(t, err)
+
+		dbSettings, err := db.ProposerSettings(ctx)
+		require.NoError(t, err)
+		require.NotNil(t, dbSettings)
+		require.DeepEqual(t, dbSettings.DefaultConfig, upatedDefault)
+		option := &validatorServiceConfig.ProposerOption{
+			FeeRecipientConfig: &validatorServiceConfig.FeeRecipientConfig{
+				FeeRecipient: common.HexToAddress("0x50155530FCE8a85ec7055A5F8b2bE214B3DaeFd3"),
+			},
+			BuilderConfig: &validatorServiceConfig.BuilderConfig{
+				Enabled:  true,
+				GasLimit: validator.Uint64(40000000),
+			},
+		}
+		err = db.UpdateProposerSettingsForPubkey(ctx, bytesutil.ToBytes48(key1), option)
+		require.NoError(t, err)
+
+		newSettings, err := db.ProposerSettings(ctx)
+		require.NoError(t, err)
+		require.NotNil(t, newSettings)
+		require.DeepEqual(t, newSettings.DefaultConfig, upatedDefault)
+		op, ok := newSettings.ProposeConfig[bytesutil.ToBytes48(key1)]
+		require.Equal(t, ok, true)
+		require.DeepEqual(t, op, option)
+	})
+}

validator/db/kv/schema.go

@@ -37,4 +37,8 @@ var (
 	// Graffiti ordered index and hash keys
 	graffitiOrderedIndexKey = []byte("graffiti-ordered-index")
 	graffitiFileHashKey     = []byte("graffiti-file-hash")
+
+	// ProposerSettings stores the encoded proposer settings file
+	proposerSettingsBucket = []byte("proposer-settings-bucket")
+	proposerSettingsKey    = []byte("proposer-settings")
 )