github/prysm
1
1
Fork 0
mirror of https://github.com/OffchainLabs/prysm.git synced 2026-01-10 07:58:22 -05:00
Code Issues Packages Projects Releases Wiki Activity

Remove Bolt Mmap Flag (#11236)

Browse Source 
* remove mmap flag

* fix mmap

* fix all build

* gaz

* config

* fix
This commit is contained in:
Nishant Das
2022-08-17 20:22:41 +08:00
committed by GitHub
parent 2377d6d6ea
commit a9ccabf6c9
24 changed files with 37 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
beacon-chain/db/db.go
View File

@@ -9,8 +9,8 @@ import (
)
// NewDB initializes a new DB.
func NewDB(ctx context.Context, dirPath string, config *kv.Config) (Database, error) {
return kv.NewKVStore(ctx, dirPath, config)
func NewDB(ctx context.Context, dirPath string) (Database, error) {
return kv.NewKVStore(ctx, dirPath)
}
// NewDBFilename uses the KVStoreDatafilePath so that if this layer of

8
beacon-chain/db/kv/backup_test.go
View File

@@ -13,7 +13,7 @@ import (
)
func TestStore_Backup(t *testing.T) {
db, err := NewKVStore(context.Background(), t.TempDir(), &Config{})
db, err := NewKVStore(context.Background(), t.TempDir())
require.NoError(t, err, "Failed to instantiate DB")
ctx := context.Background()
@@ -44,7 +44,7 @@ func TestStore_Backup(t *testing.T) {
// our NewKVStore function expects when opening a database.
require.NoError(t, os.Rename(oldFilePath, newFilePath))
backedDB, err := NewKVStore(ctx, backupsPath, &Config{})
backedDB, err := NewKVStore(ctx, backupsPath)
require.NoError(t, err, "Failed to instantiate DB")
t.Cleanup(func() {
require.NoError(t, backedDB.Close(), "Failed to close database")
@@ -53,7 +53,7 @@ func TestStore_Backup(t *testing.T) {
}
func TestStore_BackupMultipleBuckets(t *testing.T) {
db, err := NewKVStore(context.Background(), t.TempDir(), &Config{})
db, err := NewKVStore(context.Background(), t.TempDir())
require.NoError(t, err, "Failed to instantiate DB")
ctx := context.Background()
@@ -88,7 +88,7 @@ func TestStore_BackupMultipleBuckets(t *testing.T) {
// our NewKVStore function expects when opening a database.
require.NoError(t, os.Rename(oldFilePath, newFilePath))
backedDB, err := NewKVStore(ctx, backupsPath, &Config{})
backedDB, err := NewKVStore(ctx, backupsPath)
require.NoError(t, err, "Failed to instantiate DB")
t.Cleanup(func() {
require.NoError(t, backedDB.Close(), "Failed to close database")

11
beacon-chain/db/kv/kv.go
View File

@@ -37,6 +37,8 @@ const (
boltAllocSize = 8 * 1024 * 1024
// The size of hash length in bytes
hashLength = 32
// Specifies the initial mmap size of bolt.
mmapSize = 536870912
)
var (
@@ -70,11 +72,6 @@ var blockedBuckets = [][]byte{
finalizedBlockRootsIndexBucket,
}
// Config for the bolt db kv store.
type Config struct {
InitialMMapSize int
}
// Store defines an implementation of the Prysm Database interface
// using BoltDB as the underlying persistent kv-store for Ethereum Beacon Nodes.
type Store struct {
@@ -96,7 +93,7 @@ func KVStoreDatafilePath(dirPath string) string {
// NewKVStore initializes a new boltDB key-value store at the directory
// path specified, creates the kv-buckets based on the schema, and stores
// an open connection db object as a property of the Store struct.
func NewKVStore(ctx context.Context, dirPath string, config *Config) (*Store, error) {
func NewKVStore(ctx context.Context, dirPath string) (*Store, error) {
hasDir, err := file.HasDir(dirPath)
if err != nil {
return nil, err
@@ -113,7 +110,7 @@ func NewKVStore(ctx context.Context, dirPath string, config *Config) (*Store, er
params.BeaconIoConfig().ReadWritePermissions,
&bolt.Options{
Timeout: 1 * time.Second,
InitialMmapSize: config.InitialMMapSize,
InitialMmapSize: mmapSize,
},
)
if err != nil {

2
beacon-chain/db/kv/kv_test.go
View File

@@ -11,7 +11,7 @@ import (
// setupDB instantiates and returns a Store instance.
func setupDB(t testing.TB) *Store {
db, err := NewKVStore(context.Background(), t.TempDir(), &Config{})
db, err := NewKVStore(context.Background(), t.TempDir())
require.NoError(t, err, "Failed to instantiate DB")
t.Cleanup(func() {
require.NoError(t, db.Close(), "Failed to close database")

4
beacon-chain/db/restore_test.go
View File

@@ -22,7 +22,7 @@ func TestRestore(t *testing.T) {
logHook := logTest.NewGlobal()
ctx := context.Background()
backupDb, err := kv.NewKVStore(context.Background(), t.TempDir(), &kv.Config{})
backupDb, err := kv.NewKVStore(context.Background(), t.TempDir())
require.NoError(t, err)
head := util.NewBeaconBlock()
head.Block.Slot = 5000
@@ -58,7 +58,7 @@ func TestRestore(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, 1, len(files))
assert.Equal(t, kv.DatabaseFileName, files[0].Name())
restoredDb, err := kv.NewKVStore(context.Background(), path.Join(restoreDir, kv.BeaconNodeDbDirName), &kv.Config{})
restoredDb, err := kv.NewKVStore(context.Background(), path.Join(restoreDir, kv.BeaconNodeDbDirName))
defer func() {
require.NoError(t, restoredDb.Close())
}()

11
beacon-chain/db/slasherkv/kv.go
View File

@@ -21,13 +21,10 @@ const (
// DatabaseFileName is the name of the beacon node database.
DatabaseFileName = "slasher.db"
boltAllocSize = 8 * 1024 * 1024
// Specifies the initial mmap size of bolt.
mmapSize = 536870912
)
// Config for the bolt db kv store.
type Config struct {
InitialMMapSize int
}
// Store defines an implementation of the Prysm Database interface
// using BoltDB as the underlying persistent kv-store for Ethereum consensus.
type Store struct {
@@ -39,7 +36,7 @@ type Store struct {
// NewKVStore initializes a new boltDB key-value store at the directory
// path specified, creates the kv-buckets based on the schema, and stores
// an open connection db object as a property of the Store struct.
func NewKVStore(ctx context.Context, dirPath string, config *Config) (*Store, error) {
func NewKVStore(ctx context.Context, dirPath string) (*Store, error) {
hasDir, err := file.HasDir(dirPath)
if err != nil {
return nil, err
@@ -55,7 +52,7 @@ func NewKVStore(ctx context.Context, dirPath string, config *Config) (*Store, er
params.BeaconIoConfig().ReadWritePermissions,
&bolt.Options{
Timeout: 1 * time.Second,
InitialMmapSize: config.InitialMMapSize,
InitialMmapSize: mmapSize,
},
)
if err != nil {

2
beacon-chain/db/slasherkv/kv_test.go
View File

@@ -9,7 +9,7 @@ import (
// setupDB instantiates and returns a Store instance.
func setupDB(t testing.TB) *Store {
db, err := NewKVStore(context.Background(), t.TempDir(), &Config{})
db, err := NewKVStore(context.Background(), t.TempDir())
require.NoError(t, err, "Failed to instantiate DB")
t.Cleanup(func() {
require.NoError(t, db.Close(), "Failed to close database")

4
beacon-chain/db/testing/setup_db.go
View File

@@ -14,7 +14,7 @@ import (
// SetupDB instantiates and returns database backed by key value store.
func SetupDB(t testing.TB) db.Database {
s, err := kv.NewKVStore(context.Background(), t.TempDir(), &kv.Config{})
s, err := kv.NewKVStore(context.Background(), t.TempDir())
if err != nil {
t.Fatal(err)
}
@@ -28,7 +28,7 @@ func SetupDB(t testing.TB) db.Database {
// SetupSlasherDB --
func SetupSlasherDB(t testing.TB) iface.SlasherDatabase {
s, err := slasherkv.NewKVStore(context.Background(), t.TempDir(), &slasherkv.Config{})
s, err := slasherkv.NewKVStore(context.Background(), t.TempDir())
if err != nil {
t.Fatal(err)
}

16
beacon-chain/node/node.go
View File

@@ -371,9 +371,7 @@ func (b *BeaconNode) startDB(cliCtx *cli.Context, depositAddress string) error {
log.WithField("database-path", dbPath).Info("Checking DB")
d, err := db.NewDB(b.ctx, dbPath, &kv.Config{
InitialMMapSize: cliCtx.Int(cmd.BoltMMapInitialSizeFlag.Name),
})
d, err := db.NewDB(b.ctx, dbPath)
if err != nil {
return err
}
@@ -395,9 +393,7 @@ func (b *BeaconNode) startDB(cliCtx *cli.Context, depositAddress string) error {
if err := d.ClearDB(); err != nil {
return errors.Wrap(err, "could not clear database")
}
d, err = db.NewDB(b.ctx, dbPath, &kv.Config{
InitialMMapSize: cliCtx.Int(cmd.BoltMMapInitialSizeFlag.Name),
})
d, err = db.NewDB(b.ctx, dbPath)
if err != nil {
return errors.Wrap(err, "could not create new database")
}
@@ -467,9 +463,7 @@ func (b *BeaconNode) startSlasherDB(cliCtx *cli.Context) error {
log.WithField("database-path", dbPath).Info("Checking DB")
d, err := slasherkv.NewKVStore(b.ctx, dbPath, &slasherkv.Config{
InitialMMapSize: cliCtx.Int(cmd.BoltMMapInitialSizeFlag.Name),
})
d, err := slasherkv.NewKVStore(b.ctx, dbPath)
if err != nil {
return err
}
@@ -491,9 +485,7 @@ func (b *BeaconNode) startSlasherDB(cliCtx *cli.Context) error {
if err := d.ClearDB(); err != nil {
return errors.Wrap(err, "could not clear database")
}
d, err = slasherkv.NewKVStore(b.ctx, dbPath, &slasherkv.Config{
InitialMMapSize: cliCtx.Int(cmd.BoltMMapInitialSizeFlag.Name),
})
d, err = slasherkv.NewKVStore(b.ctx, dbPath)
if err != nil {
return errors.Wrap(err, "could not create new database")
}

2
beacon-chain/sync/rpc_status_test.go
View File

@@ -563,7 +563,7 @@ func TestStatusRPCRequest_FinalizedBlockExists(t *testing.T) {
}
func TestStatusRPCRequest_FinalizedBlockSkippedSlots(t *testing.T) {
db, err := kv.NewKVStore(context.Background(), t.TempDir(), &kv.Config{})
db, err := kv.NewKVStore(context.Background(), t.TempDir())
require.NoError(t, err)
bState, err := transition.GenesisBeaconState(context.Background(), nil, 0, &ethpb.Eth1Data{DepositRoot: make([]byte, 32), BlockHash: make([]byte, 32)})
require.NoError(t, err)

1
cmd/beacon-chain/main.go
View File

@@ -121,7 +121,6 @@ var appFlags = []cli.Flag{
cmd.AcceptTosFlag,
cmd.RestoreSourceFileFlag,
cmd.RestoreTargetDirFlag,
cmd.BoltMMapInitialSizeFlag,
cmd.ValidatorMonitorIndicesFlag,
cmd.ApiTimeoutFlag,
checkpoint.BlockPath,

1
cmd/beacon-chain/usage.go
View File

@@ -73,7 +73,6 @@ var appHelpFlagGroups = []flagGroup{
cmd.AcceptTosFlag,
cmd.RestoreSourceFileFlag,
cmd.RestoreTargetDirFlag,
cmd.BoltMMapInitialSizeFlag,
cmd.ValidatorMonitorIndicesFlag,
cmd.ApiTimeoutFlag,
},

6
cmd/flags.go
View File

@@ -243,12 +243,6 @@ var (
Usage: "Target directory of the restored database",
Value: DefaultDataDir(),
}
// BoltMMapInitialSizeFlag specifies the initial size in bytes of boltdb's mmap syscall.
BoltMMapInitialSizeFlag = &cli.IntFlag{
Name: "bolt-mmap-initial-size",
Usage: "Specifies the size in bytes of bolt db's mmap syscall allocation",
Value: 536870912, // 512 Mb as a default value.
}
// ApiTimeoutFlag specifies the timeout value for API requests in seconds. A timeout of zero means no timeout.
ApiTimeoutFlag = &cli.IntFlag{
Name: "api-timeout",

1
cmd/validator/main.go
View File

@@ -99,7 +99,6 @@ var appFlags = []cli.Flag{
cmd.ConfigFileFlag,
cmd.ChainConfigFileFlag,
cmd.GrpcMaxCallRecvMsgSizeFlag,
cmd.BoltMMapInitialSizeFlag,
cmd.ApiTimeoutFlag,
debug.PProfFlag,
debug.PProfAddrFlag,

1
cmd/validator/usage.go
View File

@@ -65,7 +65,6 @@ var appHelpFlagGroups = []flagGroup{
cmd.ChainConfigFileFlag,
cmd.GrpcMaxCallRecvMsgSizeFlag,
cmd.AcceptTosFlag,
cmd.BoltMMapInitialSizeFlag,
cmd.ApiTimeoutFlag,
},
},

1
tools/blocktree/BUILD.bazel
View File

@@ -9,7 +9,6 @@ go_library(
deps = [
"//beacon-chain/db:go_default_library",
"//beacon-chain/db/filters:go_default_library",
"//beacon-chain/db/kv:go_default_library",
"//consensus-types/primitives:go_default_library",
"//encoding/bytesutil:go_default_library",
"@com_github_emicklei_dot//:go_default_library",

3
tools/blocktree/main.go
View File

@@ -18,7 +18,6 @@ import (
"github.com/emicklei/dot"
"github.com/prysmaticlabs/prysm/v3/beacon-chain/db"
"github.com/prysmaticlabs/prysm/v3/beacon-chain/db/filters"
"github.com/prysmaticlabs/prysm/v3/beacon-chain/db/kv"
types "github.com/prysmaticlabs/prysm/v3/consensus-types/primitives"
"github.com/prysmaticlabs/prysm/v3/encoding/bytesutil"
)
@@ -39,7 +38,7 @@ type node struct {
func main() {
flag.Parse()
database, err := db.NewDB(context.Background(), *datadir, &kv.Config{})
database, err := db.NewDB(context.Background(), *datadir)
if err != nil {
panic(err)
}

6
tools/exploredb/main.go
View File

@@ -131,7 +131,7 @@ func printBucketContents(dbNameWithPath string, rowLimit uint64, bucketName stri
// create a new KV Store.
dbDirectory := filepath.Dir(dbNameWithPath)
db, openErr := kv.NewKVStore(context.Background(), dbDirectory, &kv.Config{})
db, openErr := kv.NewKVStore(context.Background(), dbDirectory)
if openErr != nil {
log.WithError(openErr).Fatal("could not open db")
}
@@ -379,13 +379,13 @@ func checkValidatorMigration(dbNameWithPath, destDbNameWithPath string) {
// create the source and destination KV stores.
sourceDbDirectory := filepath.Dir(dbNameWithPath)
sourceDB, openErr := kv.NewKVStore(context.Background(), sourceDbDirectory, &kv.Config{})
sourceDB, openErr := kv.NewKVStore(context.Background(), sourceDbDirectory)
if openErr != nil {
log.WithError(openErr).Fatal("could not open sourceDB")
}
destinationDbDirectory := filepath.Dir(destDbNameWithPath)
destDB, openErr := kv.NewKVStore(context.Background(), destinationDbDirectory, &kv.Config{})
destDB, openErr := kv.NewKVStore(context.Background(), destinationDbDirectory)
if openErr != nil {
// dirty hack alert: Ignore this prometheus error as we are opening two DB with same metric name
// if you want to avoid this then we should pass the metric name when opening the DB which touches

1
tools/extractor/BUILD.bazel
View File

@@ -9,7 +9,6 @@ go_library(
deps = [
"//beacon-chain/core/transition/interop:go_default_library",
"//beacon-chain/db:go_default_library",
"//beacon-chain/db/kv:go_default_library",
"//config/features:go_default_library",
"//consensus-types/primitives:go_default_library",
],

3
tools/extractor/main.go
View File

@@ -7,7 +7,6 @@ import (
"github.com/prysmaticlabs/prysm/v3/beacon-chain/core/transition/interop"
"github.com/prysmaticlabs/prysm/v3/beacon-chain/db"
"github.com/prysmaticlabs/prysm/v3/beacon-chain/db/kv"
"github.com/prysmaticlabs/prysm/v3/config/features"
types "github.com/prysmaticlabs/prysm/v3/consensus-types/primitives"
)
@@ -24,7 +23,7 @@ func main() {
defer resetCfg()
flag.Parse()
fmt.Println("Starting process...")
d, err := db.NewDB(context.Background(), *datadir, &kv.Config{})
d, err := db.NewDB(context.Background(), *datadir)
if err != nil {
panic(err)
}

1
tools/interop/export-genesis/BUILD.bazel
View File

@@ -8,7 +8,6 @@ go_library(
visibility = ["//visibility:private"],
deps = [
"//beacon-chain/db:go_default_library",
"//beacon-chain/db/kv:go_default_library",
"//io/file:go_default_library",
],
)

3
tools/interop/export-genesis/main.go
View File

@@ -6,7 +6,6 @@ import (
"os"
"github.com/prysmaticlabs/prysm/v3/beacon-chain/db"
"github.com/prysmaticlabs/prysm/v3/beacon-chain/db/kv"
"github.com/prysmaticlabs/prysm/v3/io/file"
)
@@ -21,7 +20,7 @@ func main() {
fmt.Printf("Reading db at %s and writing ssz output to %s.\n", os.Args[1], os.Args[2])
d, err := db.NewDB(context.Background(), os.Args[1], &kv.Config{})
d, err := db.NewDB(context.Background(), os.Args[1])
if err != nil {
panic(err)
}

7
validator/db/kv/db.go
View File

@@ -29,6 +29,8 @@ const (
// Time interval after which we flush attestation records to the database
// from a batch kept in memory for slashing protection.
attestationBatchWriteInterval = time.Millisecond * 100
// Specifies the initial mmap size of bolt.
mmapSize = 536870912
)
// ProtectionDbFileName Validator slashing protection db file name.
@@ -53,8 +55,7 @@ var blockedBuckets = [][]byte{
// Config represents store's config object.
type Config struct {
PubKeys [][fieldparams.BLSPubkeyLength]byte
InitialMMapSize int
PubKeys [][fieldparams.BLSPubkeyLength]byte
}
// Store defines an implementation of the Prysm Database interface
@@ -120,7 +121,7 @@ func NewKVStore(ctx context.Context, dirPath string, config *Config) (*Store, er
datafile := filepath.Join(dirPath, ProtectionDbFileName)
boltDB, err := bolt.Open(datafile, params.BeaconIoConfig().ReadWritePermissions, &bolt.Options{
Timeout: params.BeaconIoConfig().BoltTimeout,
InitialMmapSize: config.InitialMMapSize,
InitialMmapSize: mmapSize,
})
if err != nil {
if errors.Is(err, bolt.ErrTimeout) {

6
validator/node/node.go
View File

@@ -241,8 +241,7 @@ func (c *ValidatorClient) initializeFromCLI(cliCtx *cli.Context) error {
log.WithField("databasePath", dataDir).Info("Checking DB")
valDB, err := kv.NewKVStore(cliCtx.Context, dataDir, &kv.Config{
PubKeys: nil,
InitialMMapSize: cliCtx.Int(cmd.BoltMMapInitialSizeFlag.Name),
PubKeys: nil,
})
if err != nil {
return errors.Wrap(err, "could not initialize db")
@@ -318,8 +317,7 @@ func (c *ValidatorClient) initializeForWeb(cliCtx *cli.Context) error {
}
log.WithField("databasePath", dataDir).Info("Checking DB")
valDB, err := kv.NewKVStore(cliCtx.Context, dataDir, &kv.Config{
PubKeys: nil,
InitialMMapSize: cliCtx.Int(cmd.BoltMMapInitialSizeFlag.Name),
PubKeys: nil,
})
if err != nil {
return errors.Wrap(err, "could not initialize db")