From 36b6a71af46475e00504bcbed39da7835928b452 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Mon, 15 Feb 2021 14:29:47 -0600 Subject: [PATCH] Configurable DB Mmap Size for Beacon Node and Validator Client (#8448) * add flag to beacon and validator * gaz * fuzz * add dep viz * add to tools --- beacon-chain/db/db.go | 4 ++-- beacon-chain/db/db_kafka_wrapped.go | 2 +- beacon-chain/db/kv/BUILD.bazel | 6 +++++- beacon-chain/db/kv/backup_test.go | 4 ++-- beacon-chain/db/kv/kv.go | 16 ++++++++++++++-- beacon-chain/db/kv/kv_test.go | 2 +- beacon-chain/db/restore_test.go | 4 ++-- beacon-chain/db/testing/setup_db.go | 2 +- beacon-chain/main.go | 1 + beacon-chain/node/node.go | 8 ++++++-- beacon-chain/sync/rpc_status_test.go | 2 +- beacon-chain/usage.go | 1 + fuzz/BUILD.bazel | 2 ++ fuzz/block_fuzz.go | 3 ++- shared/cmd/flags.go | 6 ++++++ tools/blocktree/BUILD.bazel | 1 + tools/blocktree/main.go | 3 ++- tools/extractor/BUILD.bazel | 1 + tools/extractor/main.go | 3 ++- tools/interop/export-genesis/BUILD.bazel | 1 + tools/interop/export-genesis/main.go | 3 ++- validator/db/kv/attester_protection_test.go | 8 +++++--- validator/db/kv/backup_test.go | 2 +- validator/db/kv/db.go | 16 ++++++++++++---- validator/db/kv/db_test.go | 4 +++- validator/db/kv/proposer_protection_test.go | 2 +- validator/db/migrate.go | 4 ++-- validator/db/restore_test.go | 4 ++-- validator/db/testing/setup_db.go | 4 +++- validator/db/testing/setup_db_test.go | 4 +++- validator/main.go | 1 + validator/node/node.go | 12 +++++++++--- validator/slashing-protection/cli_export.go | 2 +- validator/slashing-protection/cli_import.go | 2 +- validator/usage.go | 1 + 35 files changed, 101 insertions(+), 40 deletions(-) diff --git a/beacon-chain/db/db.go b/beacon-chain/db/db.go index 98d4db4c1e..b062257115 100644 --- a/beacon-chain/db/db.go +++ b/beacon-chain/db/db.go @@ -9,6 +9,6 @@ import ( ) // NewDB initializes a new DB. -func NewDB(ctx context.Context, dirPath string) (Database, error) { - return kv.NewKVStore(ctx, dirPath) +func NewDB(ctx context.Context, dirPath string, config *kv.Config) (Database, error) { + return kv.NewKVStore(ctx, dirPath, config) } diff --git a/beacon-chain/db/db_kafka_wrapped.go b/beacon-chain/db/db_kafka_wrapped.go index 593fbde525..d97ecc6de7 100644 --- a/beacon-chain/db/db_kafka_wrapped.go +++ b/beacon-chain/db/db_kafka_wrapped.go @@ -9,7 +9,7 @@ import ( // NewDB initializes a new DB with kafka wrapper. func NewDB(dirPath string, stateSummaryCache *kv.stateSummaryCache) (Database, error) { - db, err := kv.NewKVStore(dirPath, stateSummaryCache) + db, err := kv.NewKVStore(dirPath, stateSummaryCache, &kv.Config{}) if err != nil { return nil, err } diff --git a/beacon-chain/db/kv/BUILD.bazel b/beacon-chain/db/kv/BUILD.bazel index 0ce029264f..9234b1fcec 100644 --- a/beacon-chain/db/kv/BUILD.bazel +++ b/beacon-chain/db/kv/BUILD.bazel @@ -26,7 +26,11 @@ go_library( "utils.go", ], importpath = "github.com/prysmaticlabs/prysm/beacon-chain/db/kv", - visibility = ["//beacon-chain:__subpackages__"], + visibility = [ + "//beacon-chain:__subpackages__", + "//fuzz:__pkg__", + "//tools:__subpackages__", + ], deps = [ "//beacon-chain/core/helpers:go_default_library", "//beacon-chain/db/filters:go_default_library", diff --git a/beacon-chain/db/kv/backup_test.go b/beacon-chain/db/kv/backup_test.go index b434b8e4b4..527191419e 100644 --- a/beacon-chain/db/kv/backup_test.go +++ b/beacon-chain/db/kv/backup_test.go @@ -12,7 +12,7 @@ import ( ) func TestStore_Backup(t *testing.T) { - db, err := NewKVStore(context.Background(), t.TempDir()) + db, err := NewKVStore(context.Background(), t.TempDir(), &Config{}) require.NoError(t, err, "Failed to instantiate DB") ctx := context.Background() @@ -41,7 +41,7 @@ func TestStore_Backup(t *testing.T) { // our NewKVStore function expects when opening a database. require.NoError(t, os.Rename(oldFilePath, newFilePath)) - backedDB, err := NewKVStore(ctx, backupsPath) + backedDB, err := NewKVStore(ctx, backupsPath, &Config{}) require.NoError(t, err, "Failed to instantiate DB") t.Cleanup(func() { require.NoError(t, backedDB.Close(), "Failed to close database") diff --git a/beacon-chain/db/kv/kv.go b/beacon-chain/db/kv/kv.go index 13beeff8f2..504b4742ce 100644 --- a/beacon-chain/db/kv/kv.go +++ b/beacon-chain/db/kv/kv.go @@ -48,6 +48,11 @@ var blockedBuckets = [][]byte{ finalizedBlockRootsIndexBucket, } +// Config for the bolt db kv store. +type Config struct { + InitialMMapSize int +} + // Store defines an implementation of the Prysm Database interface // using BoltDB as the underlying persistent kv-store for eth2. type Store struct { @@ -62,7 +67,7 @@ type Store struct { // NewKVStore initializes a new boltDB key-value store at the directory // path specified, creates the kv-buckets based on the schema, and stores // an open connection db object as a property of the Store struct. -func NewKVStore(ctx context.Context, dirPath string) (*Store, error) { +func NewKVStore(ctx context.Context, dirPath string, config *Config) (*Store, error) { hasDir, err := fileutil.HasDir(dirPath) if err != nil { return nil, err @@ -73,7 +78,14 @@ func NewKVStore(ctx context.Context, dirPath string) (*Store, error) { } } datafile := path.Join(dirPath, DatabaseFileName) - boltDB, err := bolt.Open(datafile, params.BeaconIoConfig().ReadWritePermissions, &bolt.Options{Timeout: 1 * time.Second, InitialMmapSize: 10e6}) + boltDB, err := bolt.Open( + datafile, + params.BeaconIoConfig().ReadWritePermissions, + &bolt.Options{ + Timeout: 1 * time.Second, + InitialMmapSize: config.InitialMMapSize, + }, + ) if err != nil { if errors.Is(err, bolt.ErrTimeout) { return nil, errors.New("cannot obtain database lock, database may be in use by another process") diff --git a/beacon-chain/db/kv/kv_test.go b/beacon-chain/db/kv/kv_test.go index cd90d55708..674ebd39a6 100644 --- a/beacon-chain/db/kv/kv_test.go +++ b/beacon-chain/db/kv/kv_test.go @@ -9,7 +9,7 @@ import ( // setupDB instantiates and returns a Store instance. func setupDB(t testing.TB) *Store { - db, err := NewKVStore(context.Background(), t.TempDir()) + db, err := NewKVStore(context.Background(), t.TempDir(), &Config{}) require.NoError(t, err, "Failed to instantiate DB") t.Cleanup(func() { require.NoError(t, db.Close(), "Failed to close database") diff --git a/beacon-chain/db/restore_test.go b/beacon-chain/db/restore_test.go index e3b0262131..b4a9a03b8a 100644 --- a/beacon-chain/db/restore_test.go +++ b/beacon-chain/db/restore_test.go @@ -21,7 +21,7 @@ func TestRestore(t *testing.T) { logHook := logTest.NewGlobal() ctx := context.Background() - backupDb, err := kv.NewKVStore(context.Background(), t.TempDir()) + backupDb, err := kv.NewKVStore(context.Background(), t.TempDir(), &kv.Config{}) defer func() { require.NoError(t, backupDb.Close()) }() @@ -58,7 +58,7 @@ func TestRestore(t *testing.T) { require.NoError(t, err) assert.Equal(t, 1, len(files)) assert.Equal(t, kv.DatabaseFileName, files[0].Name()) - restoredDb, err := kv.NewKVStore(context.Background(), path.Join(restoreDir, kv.BeaconNodeDbDirName)) + restoredDb, err := kv.NewKVStore(context.Background(), path.Join(restoreDir, kv.BeaconNodeDbDirName), &kv.Config{}) defer func() { require.NoError(t, restoredDb.Close()) }() diff --git a/beacon-chain/db/testing/setup_db.go b/beacon-chain/db/testing/setup_db.go index 93d8e4b37c..321fff177c 100644 --- a/beacon-chain/db/testing/setup_db.go +++ b/beacon-chain/db/testing/setup_db.go @@ -12,7 +12,7 @@ import ( // SetupDB instantiates and returns database backed by key value store. func SetupDB(t testing.TB) db.Database { - s, err := kv.NewKVStore(context.Background(), t.TempDir()) + s, err := kv.NewKVStore(context.Background(), t.TempDir(), &kv.Config{}) if err != nil { t.Fatal(err) } diff --git a/beacon-chain/main.go b/beacon-chain/main.go index 921aad10e2..1ef50c9b87 100644 --- a/beacon-chain/main.go +++ b/beacon-chain/main.go @@ -108,6 +108,7 @@ var appFlags = []cli.Flag{ cmd.AcceptTosFlag, cmd.RestoreSourceFileFlag, cmd.RestoreTargetDirFlag, + cmd.BoltMMapInitialSizeFlag, } func init() { diff --git a/beacon-chain/node/node.go b/beacon-chain/node/node.go index ac193c0028..9e2040c520 100644 --- a/beacon-chain/node/node.go +++ b/beacon-chain/node/node.go @@ -298,7 +298,9 @@ func (b *BeaconNode) startDB(cliCtx *cli.Context) error { log.WithField("database-path", dbPath).Info("Checking DB") - d, err := db.NewDB(b.ctx, dbPath) + d, err := db.NewDB(b.ctx, dbPath, &kv.Config{ + InitialMMapSize: cliCtx.Int(cmd.BoltMMapInitialSizeFlag.Name), + }) if err != nil { return err } @@ -320,7 +322,9 @@ func (b *BeaconNode) startDB(cliCtx *cli.Context) error { if err := d.ClearDB(); err != nil { return errors.Wrap(err, "could not clear database") } - d, err = db.NewDB(b.ctx, dbPath) + d, err = db.NewDB(b.ctx, dbPath, &kv.Config{ + InitialMMapSize: cliCtx.Int(cmd.BoltMMapInitialSizeFlag.Name), + }) if err != nil { return errors.Wrap(err, "could not create new database") } diff --git a/beacon-chain/sync/rpc_status_test.go b/beacon-chain/sync/rpc_status_test.go index 56fd580ad4..2c47c9bc6c 100644 --- a/beacon-chain/sync/rpc_status_test.go +++ b/beacon-chain/sync/rpc_status_test.go @@ -528,7 +528,7 @@ func TestStatusRPCRequest_FinalizedBlockExists(t *testing.T) { } func TestStatusRPCRequest_FinalizedBlockSkippedSlots(t *testing.T) { - db, err := kv.NewKVStore(context.Background(), t.TempDir()) + db, err := kv.NewKVStore(context.Background(), t.TempDir(), &kv.Config{}) require.NoError(t, err) bState, err := state.GenesisBeaconState(nil, 0, ðpb.Eth1Data{DepositRoot: make([]byte, 32), BlockHash: make([]byte, 32)}) require.NoError(t, err) diff --git a/beacon-chain/usage.go b/beacon-chain/usage.go index 58a864e708..158dbdfdce 100644 --- a/beacon-chain/usage.go +++ b/beacon-chain/usage.go @@ -72,6 +72,7 @@ var appHelpFlagGroups = []flagGroup{ cmd.AcceptTosFlag, cmd.RestoreSourceFileFlag, cmd.RestoreTargetDirFlag, + cmd.BoltMMapInitialSizeFlag, }, }, { diff --git a/fuzz/BUILD.bazel b/fuzz/BUILD.bazel index f91b3c02fe..b0b63a241a 100644 --- a/fuzz/BUILD.bazel +++ b/fuzz/BUILD.bazel @@ -70,6 +70,7 @@ go_fuzz_test( "//beacon-chain/p2p/testing:go_default_library", "//beacon-chain/sync:go_default_library", "//beacon-chain/db:go_default_library", + "//beacon-chain/db/kv:go_default_library", "//beacon-chain/operations/voluntaryexits:go_default_library", "//beacon-chain/blockchain:go_default_library", "//beacon-chain/operations/slashings:go_default_library", @@ -169,6 +170,7 @@ go_library( "//beacon-chain/core/helpers:go_default_library", "//beacon-chain/core/state:go_default_library", "//beacon-chain/db:go_default_library", + "//beacon-chain/db/kv:go_default_library", "//beacon-chain/forkchoice/protoarray:go_default_library", "//beacon-chain/operations/attestations:go_default_library", "//beacon-chain/operations/slashings:go_default_library", diff --git a/fuzz/block_fuzz.go b/fuzz/block_fuzz.go index 9f8f2813e2..2b37ad6c8c 100644 --- a/fuzz/block_fuzz.go +++ b/fuzz/block_fuzz.go @@ -17,6 +17,7 @@ import ( "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing" "github.com/prysmaticlabs/prysm/beacon-chain/core/state" "github.com/prysmaticlabs/prysm/beacon-chain/db" + beaconkv "github.com/prysmaticlabs/prysm/beacon-chain/db/kv" "github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray" "github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations" "github.com/prysmaticlabs/prysm/beacon-chain/operations/slashings" @@ -51,7 +52,7 @@ func init() { var err error - db1, err = db.NewDB(context.Background(), dbPath) + db1, err = db.NewDB(context.Background(), dbPath, &beaconkv.Config{}) if err != nil { panic(err) } diff --git a/shared/cmd/flags.go b/shared/cmd/flags.go index a1e8da046b..c7b85532af 100644 --- a/shared/cmd/flags.go +++ b/shared/cmd/flags.go @@ -231,6 +231,12 @@ var ( Usage: "Target directory of the restored database", Value: DefaultDataDir(), } + // BoltMMapInitialSizeFlag specifies the initial size in bytes of boltdb's mmap syscall. + BoltMMapInitialSizeFlag = &cli.IntFlag{ + Name: "bolt-mmap-initial-size", + Usage: "Specifies the size in bytes of bolt db's mmap syscall allocation", + Value: 536870912, // 512 Mb as a default value. + } ) // LoadFlagsFromConfig sets flags values from config file if ConfigFileFlag is set. diff --git a/tools/blocktree/BUILD.bazel b/tools/blocktree/BUILD.bazel index a506ff7a6d..6ef3565279 100644 --- a/tools/blocktree/BUILD.bazel +++ b/tools/blocktree/BUILD.bazel @@ -9,6 +9,7 @@ go_library( deps = [ "//beacon-chain/db:go_default_library", "//beacon-chain/db/filters:go_default_library", + "//beacon-chain/db/kv:go_default_library", "//shared/bytesutil:go_default_library", "@com_github_emicklei_dot//:go_default_library", ], diff --git a/tools/blocktree/main.go b/tools/blocktree/main.go index d3cdd42a12..c2c04616ae 100644 --- a/tools/blocktree/main.go +++ b/tools/blocktree/main.go @@ -18,6 +18,7 @@ import ( "github.com/emicklei/dot" "github.com/prysmaticlabs/prysm/beacon-chain/db" "github.com/prysmaticlabs/prysm/beacon-chain/db/filters" + "github.com/prysmaticlabs/prysm/beacon-chain/db/kv" "github.com/prysmaticlabs/prysm/shared/bytesutil" ) @@ -37,7 +38,7 @@ type node struct { func main() { flag.Parse() - db, err := db.NewDB(context.Background(), *datadir) + db, err := db.NewDB(context.Background(), *datadir, &kv.Config{}) if err != nil { panic(err) } diff --git a/tools/extractor/BUILD.bazel b/tools/extractor/BUILD.bazel index 15e73a3cdc..04daaf8a5a 100644 --- a/tools/extractor/BUILD.bazel +++ b/tools/extractor/BUILD.bazel @@ -9,6 +9,7 @@ go_library( deps = [ "//beacon-chain/core/state/interop:go_default_library", "//beacon-chain/db:go_default_library", + "//beacon-chain/db/kv:go_default_library", "//shared/featureconfig:go_default_library", ], ) diff --git a/tools/extractor/main.go b/tools/extractor/main.go index 0c5ce71b4f..9b134b3215 100644 --- a/tools/extractor/main.go +++ b/tools/extractor/main.go @@ -7,6 +7,7 @@ import ( "github.com/prysmaticlabs/prysm/beacon-chain/core/state/interop" "github.com/prysmaticlabs/prysm/beacon-chain/db" + "github.com/prysmaticlabs/prysm/beacon-chain/db/kv" "github.com/prysmaticlabs/prysm/shared/featureconfig" ) @@ -22,7 +23,7 @@ func main() { defer resetCfg() flag.Parse() fmt.Println("Starting process...") - d, err := db.NewDB(context.Background(), *datadir) + d, err := db.NewDB(context.Background(), *datadir, &kv.Config{}) if err != nil { panic(err) } diff --git a/tools/interop/export-genesis/BUILD.bazel b/tools/interop/export-genesis/BUILD.bazel index 874ca541ac..bf9b06cdec 100644 --- a/tools/interop/export-genesis/BUILD.bazel +++ b/tools/interop/export-genesis/BUILD.bazel @@ -8,6 +8,7 @@ go_library( visibility = ["//visibility:private"], deps = [ "//beacon-chain/db:go_default_library", + "//beacon-chain/db/kv:go_default_library", "//shared/fileutil:go_default_library", ], ) diff --git a/tools/interop/export-genesis/main.go b/tools/interop/export-genesis/main.go index f9bb99e989..42764fb4cb 100644 --- a/tools/interop/export-genesis/main.go +++ b/tools/interop/export-genesis/main.go @@ -6,6 +6,7 @@ import ( "os" "github.com/prysmaticlabs/prysm/beacon-chain/db" + "github.com/prysmaticlabs/prysm/beacon-chain/db/kv" "github.com/prysmaticlabs/prysm/shared/fileutil" ) @@ -20,7 +21,7 @@ func main() { fmt.Printf("Reading db at %s and writing ssz output to %s.\n", os.Args[1], os.Args[2]) - d, err := db.NewDB(context.Background(), os.Args[1]) + d, err := db.NewDB(context.Background(), os.Args[1], &kv.Config{}) if err != nil { panic(err) } diff --git a/validator/db/kv/attester_protection_test.go b/validator/db/kv/attester_protection_test.go index 08bec8dcc2..6f27322f55 100644 --- a/validator/db/kv/attester_protection_test.go +++ b/validator/db/kv/attester_protection_test.go @@ -213,7 +213,7 @@ func TestStore_CheckSlashableAttestation_SurroundVote_54kEpochs(t *testing.T) { func TestLowestSignedSourceEpoch_SaveRetrieve(t *testing.T) { ctx := context.Background() - validatorDB, err := NewKVStore(ctx, t.TempDir(), nil) + validatorDB, err := NewKVStore(ctx, t.TempDir(), &Config{}) require.NoError(t, err, "Failed to instantiate DB") t.Cleanup(func() { require.NoError(t, validatorDB.Close(), "Failed to close database") @@ -272,7 +272,7 @@ func TestLowestSignedSourceEpoch_SaveRetrieve(t *testing.T) { func TestLowestSignedTargetEpoch_SaveRetrieveReplace(t *testing.T) { ctx := context.Background() - validatorDB, err := NewKVStore(ctx, t.TempDir(), nil) + validatorDB, err := NewKVStore(ctx, t.TempDir(), &Config{}) require.NoError(t, err, "Failed to instantiate DB") t.Cleanup(func() { require.NoError(t, validatorDB.Close(), "Failed to close database") @@ -500,7 +500,9 @@ func benchCheckSurroundVote( shouldSurround bool, ) { ctx := context.Background() - validatorDB, err := NewKVStore(ctx, filepath.Join(os.TempDir(), "benchsurroundvote"), pubKeys) + validatorDB, err := NewKVStore(ctx, filepath.Join(os.TempDir(), "benchsurroundvote"), &Config{ + PubKeys: pubKeys, + }) require.NoError(b, err, "Failed to instantiate DB") defer func() { require.NoError(b, validatorDB.Close(), "Failed to close database") diff --git a/validator/db/kv/backup_test.go b/validator/db/kv/backup_test.go index eb6870eb77..9664bc4a6a 100644 --- a/validator/db/kv/backup_test.go +++ b/validator/db/kv/backup_test.go @@ -29,7 +29,7 @@ func TestStore_Backup(t *testing.T) { // our NewKVStore function expects when opening a database. require.NoError(t, os.Rename(oldFilePath, newFilePath)) - backedDB, err := NewKVStore(ctx, backupsPath, nil) + backedDB, err := NewKVStore(ctx, backupsPath, &Config{}) require.NoError(t, err, "Failed to instantiate DB") t.Cleanup(func() { require.NoError(t, backedDB.Close(), "Failed to close database") diff --git a/validator/db/kv/db.go b/validator/db/kv/db.go index 4e68eb5a9b..1d09a3f0ca 100644 --- a/validator/db/kv/db.go +++ b/validator/db/kv/db.go @@ -48,6 +48,11 @@ var blockedBuckets = [][]byte{ attestationSourceEpochsBucket, } +type Config struct { + PubKeys [][48]byte + InitialMMapSize int +} + // Store defines an implementation of the Prysm Database interface // using BoltDB as the underlying persistent kv-store for eth2. type Store struct { @@ -98,7 +103,7 @@ func createBuckets(tx *bolt.Tx, buckets ...[]byte) error { // NewKVStore initializes a new boltDB key-value store at the directory // path specified, creates the kv-buckets based on the schema, and stores // an open connection db object as a property of the Store struct. -func NewKVStore(ctx context.Context, dirPath string, pubKeys [][48]byte) (*Store, error) { +func NewKVStore(ctx context.Context, dirPath string, config *Config) (*Store, error) { hasDir, err := fileutil.HasDir(dirPath) if err != nil { return nil, err @@ -109,7 +114,10 @@ func NewKVStore(ctx context.Context, dirPath string, pubKeys [][48]byte) (*Store } } datafile := filepath.Join(dirPath, ProtectionDbFileName) - boltDB, err := bolt.Open(datafile, params.BeaconIoConfig().ReadWritePermissions, &bolt.Options{Timeout: params.BeaconIoConfig().BoltTimeout}) + boltDB, err := bolt.Open(datafile, params.BeaconIoConfig().ReadWritePermissions, &bolt.Options{ + Timeout: params.BeaconIoConfig().BoltTimeout, + InitialMmapSize: config.InitialMMapSize, + }) if err != nil { if errors.Is(err, bolt.ErrTimeout) { return nil, errors.New("cannot obtain database lock, database may be in use by another process") @@ -144,8 +152,8 @@ func NewKVStore(ctx context.Context, dirPath string, pubKeys [][48]byte) (*Store } // Initialize the required public keys into the DB to ensure they're not empty. - if pubKeys != nil { - if err := kv.UpdatePublicKeysBuckets(pubKeys); err != nil { + if config != nil { + if err := kv.UpdatePublicKeysBuckets(config.PubKeys); err != nil { return nil, err } } diff --git a/validator/db/kv/db_test.go b/validator/db/kv/db_test.go index 7ada22592d..9e7e3a39c5 100644 --- a/validator/db/kv/db_test.go +++ b/validator/db/kv/db_test.go @@ -18,7 +18,9 @@ func TestMain(m *testing.M) { // setupDB instantiates and returns a DB instance for the validator client. func setupDB(t testing.TB, pubkeys [][48]byte) *Store { - db, err := NewKVStore(context.Background(), t.TempDir(), pubkeys) + db, err := NewKVStore(context.Background(), t.TempDir(), &Config{ + PubKeys: pubkeys, + }) require.NoError(t, err, "Failed to instantiate DB") err = db.UpdatePublicKeysBuckets(pubkeys) require.NoError(t, err, "Failed to create old buckets for public keys") diff --git a/validator/db/kv/proposer_protection_test.go b/validator/db/kv/proposer_protection_test.go index 266fb1c304..1a4d5d0357 100644 --- a/validator/db/kv/proposer_protection_test.go +++ b/validator/db/kv/proposer_protection_test.go @@ -178,7 +178,7 @@ func TestPruneProposalHistoryBySlot_OK(t *testing.T) { func TestStore_ProposedPublicKeys(t *testing.T) { ctx := context.Background() - validatorDB, err := NewKVStore(ctx, t.TempDir(), nil) + validatorDB, err := NewKVStore(ctx, t.TempDir(), &Config{}) require.NoError(t, err, "Failed to instantiate DB") t.Cleanup(func() { require.NoError(t, validatorDB.Close(), "Failed to close database") diff --git a/validator/db/migrate.go b/validator/db/migrate.go index e3ab982aa4..d401f34b23 100644 --- a/validator/db/migrate.go +++ b/validator/db/migrate.go @@ -19,7 +19,7 @@ func migrateUp(cliCtx *cli.Context) error { } ctx := context.Background() - validatorDB, err := kv.NewKVStore(ctx, dataDir, nil) + validatorDB, err := kv.NewKVStore(ctx, dataDir, &kv.Config{}) if err != nil { return err } @@ -34,7 +34,7 @@ func migrateDown(cliCtx *cli.Context) error { } ctx := context.Background() - validatorDB, err := kv.NewKVStore(ctx, dataDir, nil) + validatorDB, err := kv.NewKVStore(ctx, dataDir, &kv.Config{}) if err != nil { return err } diff --git a/validator/db/restore_test.go b/validator/db/restore_test.go index efe4d42ddb..cafe851f65 100644 --- a/validator/db/restore_test.go +++ b/validator/db/restore_test.go @@ -21,7 +21,7 @@ func TestRestore(t *testing.T) { logHook := logTest.NewGlobal() ctx := context.Background() - backupDb, err := kv.NewKVStore(ctx, t.TempDir(), nil) + backupDb, err := kv.NewKVStore(ctx, t.TempDir(), &kv.Config{}) defer func() { require.NoError(t, backupDb.Close()) }() @@ -51,7 +51,7 @@ func TestRestore(t *testing.T) { require.NoError(t, err) assert.Equal(t, 1, len(files)) assert.Equal(t, kv.ProtectionDbFileName, files[0].Name()) - restoredDb, err := kv.NewKVStore(ctx, restoreDir, nil) + restoredDb, err := kv.NewKVStore(ctx, restoreDir, &kv.Config{}) defer func() { require.NoError(t, restoredDb.Close()) }() diff --git a/validator/db/testing/setup_db.go b/validator/db/testing/setup_db.go index 378d1f1799..ba1ee601f2 100644 --- a/validator/db/testing/setup_db.go +++ b/validator/db/testing/setup_db.go @@ -10,7 +10,9 @@ import ( // SetupDB instantiates and returns a DB instance for the validator client. func SetupDB(t testing.TB, pubkeys [][48]byte) iface.ValidatorDB { - db, err := kv.NewKVStore(context.Background(), t.TempDir(), pubkeys) + db, err := kv.NewKVStore(context.Background(), t.TempDir(), &kv.Config{ + PubKeys: pubkeys, + }) if err != nil { t.Fatalf("Failed to instantiate DB: %v", err) } diff --git a/validator/db/testing/setup_db_test.go b/validator/db/testing/setup_db_test.go index 99bf92b9f1..584cc52141 100644 --- a/validator/db/testing/setup_db_test.go +++ b/validator/db/testing/setup_db_test.go @@ -12,7 +12,9 @@ import ( func TestClearDB(t *testing.T) { // Setting up manually is required, since SetupDB() will also register a teardown procedure. - testDB, err := kv.NewKVStore(context.Background(), t.TempDir(), [][48]byte{}) + testDB, err := kv.NewKVStore(context.Background(), t.TempDir(), &kv.Config{ + PubKeys: nil, + }) require.NoError(t, err, "Failed to instantiate DB") require.NoError(t, testDB.ClearDB()) diff --git a/validator/main.go b/validator/main.go index 6d4c5c9b67..d3d7f1aa1a 100644 --- a/validator/main.go +++ b/validator/main.go @@ -89,6 +89,7 @@ var appFlags = []cli.Flag{ cmd.ConfigFileFlag, cmd.ChainConfigFileFlag, cmd.GrpcMaxCallRecvMsgSizeFlag, + cmd.BoltMMapInitialSizeFlag, debug.PProfFlag, debug.PProfAddrFlag, debug.PProfPortFlag, diff --git a/validator/node/node.go b/validator/node/node.go index 52a91bda8c..31de630d18 100644 --- a/validator/node/node.go +++ b/validator/node/node.go @@ -224,7 +224,10 @@ func (c *ValidatorClient) initializeFromCLI(cliCtx *cli.Context) error { } log.WithField("databasePath", dataDir).Info("Checking DB") - valDB, err := kv.NewKVStore(cliCtx.Context, dataDir, nil) + valDB, err := kv.NewKVStore(cliCtx.Context, dataDir, &kv.Config{ + PubKeys: nil, + InitialMMapSize: cliCtx.Int(cmd.BoltMMapInitialSizeFlag.Name), + }) if err != nil { return errors.Wrap(err, "could not initialize db") } @@ -310,7 +313,10 @@ func (c *ValidatorClient) initializeForWeb(cliCtx *cli.Context) error { } } log.WithField("databasePath", dataDir).Info("Checking DB") - valDB, err := kv.NewKVStore(cliCtx.Context, dataDir, make([][48]byte, 0)) + valDB, err := kv.NewKVStore(cliCtx.Context, dataDir, &kv.Config{ + PubKeys: nil, + InitialMMapSize: cliCtx.Int(cmd.BoltMMapInitialSizeFlag.Name), + }) if err != nil { return errors.Wrap(err, "could not initialize db") } @@ -551,7 +557,7 @@ func clearDB(ctx context.Context, dataDir string, force bool) error { } if clearDBConfirmed { - valDB, err := kv.NewKVStore(ctx, dataDir, nil) + valDB, err := kv.NewKVStore(ctx, dataDir, &kv.Config{}) if err != nil { return errors.Wrapf(err, "Could not create DB in dir %s", dataDir) } diff --git a/validator/slashing-protection/cli_export.go b/validator/slashing-protection/cli_export.go index 404c81cfce..ef70f9f2eb 100644 --- a/validator/slashing-protection/cli_export.go +++ b/validator/slashing-protection/cli_export.go @@ -37,7 +37,7 @@ func ExportSlashingProtectionJSONCli(cliCtx *cli.Context) error { return err } } - validatorDB, err := kv.NewKVStore(cliCtx.Context, dataDir, nil) + validatorDB, err := kv.NewKVStore(cliCtx.Context, dataDir, &kv.Config{}) if err != nil { return errors.Wrapf(err, "could not access validator database at path %s", dataDir) } diff --git a/validator/slashing-protection/cli_import.go b/validator/slashing-protection/cli_import.go index 095a6ab35d..76e7c5d0f3 100644 --- a/validator/slashing-protection/cli_import.go +++ b/validator/slashing-protection/cli_import.go @@ -32,7 +32,7 @@ func ImportSlashingProtectionCLI(cliCtx *cli.Context) error { return err } } - valDB, err := kv.NewKVStore(cliCtx.Context, dataDir, make([][48]byte, 0)) + valDB, err := kv.NewKVStore(cliCtx.Context, dataDir, &kv.Config{}) if err != nil { return errors.Wrapf(err, "could not access validator database at path: %s", dataDir) } diff --git a/validator/usage.go b/validator/usage.go index 10024dfff6..a228118330 100644 --- a/validator/usage.go +++ b/validator/usage.go @@ -65,6 +65,7 @@ var appHelpFlagGroups = []flagGroup{ cmd.ChainConfigFileFlag, cmd.GrpcMaxCallRecvMsgSizeFlag, cmd.AcceptTosFlag, + cmd.BoltMMapInitialSizeFlag, }, }, {