Merge branch 'develop' into addArchivalFlag

beacon-chain/node/BUILD.bazel

@@ -55,6 +55,8 @@ go_library(
         "//beacon-chain/verification:go_default_library",
         "//cmd:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",
+        "//cmd/beacon-chain/storage/flags:go_default_library",
+        "//cmd/beacon-chain/sync/backfill/flags:go_default_library",
         "//config/features:go_default_library",
         "//config/params:go_default_library",
         "//consensus-types/primitives:go_default_library",
@@ -94,6 +96,8 @@ go_test(
         "//beacon-chain/monitor:go_default_library",
         "//cmd:go_default_library",
         "//cmd/beacon-chain/flags:go_default_library",
+        "//cmd/beacon-chain/storage/flags:go_default_library",
+        "//cmd/beacon-chain/sync/backfill/flags:go_default_library",
         "//config/features:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",

beacon-chain/node/config.go

@@ -2,17 +2,26 @@ package node
 
 import (
 	"fmt"
+	"math"
 
 	"github.com/ethereum/go-ethereum/common"
 	fastssz "github.com/prysmaticlabs/fastssz"
 	"github.com/prysmaticlabs/prysm/v5/cmd"
 	"github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/flags"
+	storageFlags "github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/storage/flags"
+	backfill "github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/sync/backfill/flags"
+	"github.com/prysmaticlabs/prysm/v5/config/features"
 	"github.com/prysmaticlabs/prysm/v5/config/params"
 	"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
 	tracing2 "github.com/prysmaticlabs/prysm/v5/monitoring/tracing"
 	"github.com/urfave/cli/v2"
 )
 
+const (
+	maxBlobRetentionEpoch         = math.MaxUint32
+	archivalSlotsPerArchivedPoint = 32
+)
+
 func configureTracing(cliCtx *cli.Context) error {
 	return tracing2.Setup(
 		"beacon-chain", // service name
@@ -198,6 +207,37 @@ func configureExecutionSetting(cliCtx *cli.Context) error {
 	return params.SetActive(c)
 }
 
+func configureArchivalNode(cliCtx *cli.Context) error {
+	if cliCtx.IsSet(flags.ArchivalNodeFlag.Name) {
+		log.Info("Enabling Archival mode on the beacon node")
+		if cliCtx.IsSet(flags.SlotsPerArchivedPoint.Name) {
+			log.Infof("Changing slots per archived point from %d to %d", cliCtx.Int(flags.SlotsPerArchivedPoint.Name), archivalSlotsPerArchivedPoint)
+		}
+		if err := cliCtx.Set(flags.SlotsPerArchivedPoint.Name, fmt.Sprintf("%d", archivalSlotsPerArchivedPoint)); err != nil {
+			return err
+		}
+		if !cliCtx.IsSet(features.SaveFullExecutionPayloads.Name) {
+			log.Info("Saving full execution payloads")
+			if err := cliCtx.Set(features.SaveFullExecutionPayloads.Name, "true"); err != nil {
+				return err
+			}
+		}
+		if !cliCtx.IsSet(backfill.EnableExperimentalBackfill.Name) {
+			log.Info("Enabling backfill on nodes")
+			if err := cliCtx.Set(backfill.EnableExperimentalBackfill.Name, "true"); err != nil {
+				return err
+			}
+		}
+		if cliCtx.IsSet(storageFlags.BlobRetentionEpochFlag.Name) {
+			log.Infof("Changing blob retention epochs from %d to %d", cliCtx.Uint64(storageFlags.BlobRetentionEpochFlag.Name), maxBlobRetentionEpoch)
+		}
+		if err := cliCtx.Set(storageFlags.BlobRetentionEpochFlag.Name, fmt.Sprintf("%d", maxBlobRetentionEpoch)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func configureFastSSZHashingAlgorithm() {
 	fastssz.EnableVectorizedHTR = true
 }

beacon-chain/node/config_test.go

@@ -11,6 +11,9 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/prysmaticlabs/prysm/v5/cmd"
 	"github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/flags"
+	storageFlags "github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/storage/flags"
+	backfill "github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/sync/backfill/flags"
+	"github.com/prysmaticlabs/prysm/v5/config/features"
 	"github.com/prysmaticlabs/prysm/v5/config/params"
 	"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v5/testing/assert"
@@ -228,3 +231,38 @@ func TestConfigureInterop(t *testing.T) {
 		})
 	}
 }
+
+func TestConfigureArchivalNode(t *testing.T) {
+	params.SetupTestConfigCleanup(t)
+	hook := logTest.NewGlobal()
+
+	app := cli.App{}
+	set := flag.NewFlagSet("test", 0)
+	set.Bool(flags.ArchivalNodeFlag.Name, false, "")
+	set.Int(flags.SlotsPerArchivedPoint.Name, 2048, "")
+	set.Bool(features.SaveFullExecutionPayloads.Name, false, "")
+	set.Bool(backfill.EnableExperimentalBackfill.Name, false, "")
+	set.Uint64(storageFlags.BlobRetentionEpochFlag.Name, 4096, "")
+
+	require.NoError(t, set.Set(flags.ArchivalNodeFlag.Name, "true"))
+	cliCtx := cli.NewContext(&app, set, nil)
+
+	require.NoError(t, configureArchivalNode(cliCtx))
+	assert.LogsContain(t, hook, "Enabling Archival mode on the beacon node")
+	assert.LogsContain(t, hook, "Saving full execution payloads")
+	assert.LogsContain(t, hook, "Enabling backfill on nodes")
+
+	hook.Reset()
+
+	require.NoError(t, set.Set(flags.SlotsPerArchivedPoint.Name, "256"))
+	require.NoError(t, set.Set(features.SaveFullExecutionPayloads.Name, "true"))
+	require.NoError(t, set.Set(backfill.EnableExperimentalBackfill.Name, "true"))
+	require.NoError(t, set.Set(storageFlags.BlobRetentionEpochFlag.Name, "2048"))
+
+	cliCtx = cli.NewContext(&app, set, nil)
+	require.NoError(t, configureArchivalNode(cliCtx))
+	assert.LogsContain(t, hook, "Enabling Archival mode on the beacon node")
+	assert.LogsContain(t, hook, "Changing slots per archived point from 256 to 32")
+	assert.LogsContain(t, hook, "Changing blob retention epochs from 2048 to 4294967295")
+
+}

beacon-chain/node/node.go

@@ -134,6 +134,9 @@ func New(cliCtx *cli.Context, cancel context.CancelFunc, opts ...Option) (*Beaco
 	if hasNetworkFlag(cliCtx) && cliCtx.IsSet(cmd.ChainConfigFileFlag.Name) {
 		return nil, fmt.Errorf("%s cannot be passed concurrently with network flag", cmd.ChainConfigFileFlag.Name)
 	}
+	if err := configureArchivalNode(cliCtx); err != nil {
+		return nil, err
+	}
 	if err := features.ConfigureBeaconChain(cliCtx); err != nil {
 		return nil, err
 	}

cmd/beacon-chain/BUILD.bazel

@@ -21,6 +21,7 @@ go_library(
         "//cmd/beacon-chain/flags:go_default_library",
         "//cmd/beacon-chain/jwt:go_default_library",
         "//cmd/beacon-chain/storage:go_default_library",
+        "//cmd/beacon-chain/storage/flags:go_default_library",
         "//cmd/beacon-chain/sync/backfill:go_default_library",
         "//cmd/beacon-chain/sync/backfill/flags:go_default_library",
         "//cmd/beacon-chain/sync/checkpoint:go_default_library",

cmd/beacon-chain/flags/base.go

@@ -259,4 +259,9 @@ var (
 		Usage: "Directory for the slasher database",
 		Value: cmd.DefaultDataDir(),
 	}
+	// ArchivalNodeFlag defines the flag that enables archival mode on a prysm beacon node.
+	ArchivalNodeFlag = &cli.BoolFlag{
+		Name:  "archive",
+		Usage: "Enables archival mode on a prysm beacon node.",
+	}
 )

cmd/beacon-chain/main.go

@@ -20,6 +20,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/flags"
 	jwtcommands "github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/jwt"
 	"github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/storage"
+	storageFlags "github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/storage/flags"
 	backfill "github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/sync/backfill"
 	bflags "github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/sync/backfill/flags"
 	"github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/sync/checkpoint"
@@ -138,8 +139,9 @@ var appFlags = []cli.Flag{
 	genesis.BeaconAPIURL,
 	flags.SlasherDirFlag,
 	flags.JwtId,
-	storage.BlobStoragePathFlag,
-	storage.BlobRetentionEpochFlag,
+	flags.ArchivalNodeFlag,
+	storageFlags.BlobStoragePathFlag,
+	storageFlags.BlobRetentionEpochFlag,
 	bflags.EnableExperimentalBackfill,
 	bflags.BackfillBatchSize,
 	bflags.BackfillWorkerCount,

cmd/beacon-chain/storage/BUILD.bazel

@@ -9,6 +9,7 @@ go_library(
         "//beacon-chain/db/filesystem:go_default_library",
         "//beacon-chain/node:go_default_library",
         "//cmd:go_default_library",
+        "//cmd/beacon-chain/storage/flags:go_default_library",
         "//config/params:go_default_library",
         "//consensus-types/primitives:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
@@ -22,6 +23,7 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//cmd:go_default_library",
+        "//cmd/beacon-chain/storage/flags:go_default_library",
         "//config/params:go_default_library",
         "//consensus-types/primitives:go_default_library",
         "//testing/assert:go_default_library",

cmd/beacon-chain/storage/flags/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["flags.go"],
+    importpath = "github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/storage/flags",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//config/params:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
+    ],
+)

cmd/beacon-chain/storage/flags/flags.go

@@ -0,0 +1,20 @@
+package flags
+
+import (
+	"github.com/prysmaticlabs/prysm/v5/config/params"
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	// BlobStoragePathFlag defines a flag to start the beacon chain from a give genesis state file.
+	BlobStoragePathFlag = &cli.PathFlag{
+		Name:  "blob-path",
+		Usage: "Location for blob storage. Default location will be a 'blobs' directory next to the beacon db.",
+	}
+	BlobRetentionEpochFlag = &cli.Uint64Flag{
+		Name:    "blob-retention-epochs",
+		Usage:   "Override the default blob retention period (measured in epochs). The node will exit with an error at startup if the value is less than the default of 4096 epochs.",
+		Value:   uint64(params.BeaconConfig().MinEpochsForBlobsSidecarsRequest),
+		Aliases: []string{"extend-blob-retention-epoch"},
+	}
+)

cmd/beacon-chain/storage/options.go

@@ -7,25 +7,12 @@ import (
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/db/filesystem"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/node"
 	"github.com/prysmaticlabs/prysm/v5/cmd"
+	"github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/storage/flags"
 	"github.com/prysmaticlabs/prysm/v5/config/params"
 	"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
 	"github.com/urfave/cli/v2"
 )
 
-var (
-	// BlobStoragePathFlag defines a flag to start the beacon chain from a give genesis state file.
-	BlobStoragePathFlag = &cli.PathFlag{
-		Name:  "blob-path",
-		Usage: "Location for blob storage. Default location will be a 'blobs' directory next to the beacon db.",
-	}
-	BlobRetentionEpochFlag = &cli.Uint64Flag{
-		Name:    "blob-retention-epochs",
-		Usage:   "Override the default blob retention period (measured in epochs). The node will exit with an error at startup if the value is less than the default of 4096 epochs.",
-		Value:   uint64(params.BeaconConfig().MinEpochsForBlobsSidecarsRequest),
-		Aliases: []string{"extend-blob-retention-epoch"},
-	}
-)
-
 // BeaconNodeOptions sets configuration values on the node.BeaconNode value at node startup.
 // Note: we can't get the right context from cli.Context, because the beacon node setup code uses this context to
 // create a cancellable context. If we switch to using App.RunContext, we can set up this cancellation in the cmd
@@ -42,7 +29,7 @@ func BeaconNodeOptions(c *cli.Context) ([]node.Option, error) {
 }
 
 func blobStoragePath(c *cli.Context) string {
-	blobsPath := c.Path(BlobStoragePathFlag.Name)
+	blobsPath := c.Path(flags.BlobStoragePathFlag.Name)
 	if blobsPath == "" {
 		// append a "blobs" subdir to the end of the data dir path
 		blobsPath = path.Join(c.String(cmd.DataDirFlag.Name), "blobs")
@@ -57,14 +44,14 @@ var errInvalidBlobRetentionEpochs = errors.New("value is smaller than spec minim
 // smaller than the spec default, an error will be returned.
 func blobRetentionEpoch(cliCtx *cli.Context) (primitives.Epoch, error) {
 	spec := params.BeaconConfig().MinEpochsForBlobsSidecarsRequest
-	if !cliCtx.IsSet(BlobRetentionEpochFlag.Name) {
+	if !cliCtx.IsSet(flags.BlobRetentionEpochFlag.Name) {
 		return spec, nil
 	}
 
-	re := primitives.Epoch(cliCtx.Uint64(BlobRetentionEpochFlag.Name))
+	re := primitives.Epoch(cliCtx.Uint64(flags.BlobRetentionEpochFlag.Name))
 	// Validate the epoch value against the spec default.
 	if re < params.BeaconConfig().MinEpochsForBlobsSidecarsRequest {
-		return spec, errors.Wrapf(errInvalidBlobRetentionEpochs, "%s=%d, spec=%d", BlobRetentionEpochFlag.Name, re, spec)
+		return spec, errors.Wrapf(errInvalidBlobRetentionEpochs, "%s=%d, spec=%d", flags.BlobRetentionEpochFlag.Name, re, spec)
 	}
 
 	return re, nil

cmd/beacon-chain/storage/options_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/v5/cmd"
+	"github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/storage/flags"
 	"github.com/prysmaticlabs/prysm/v5/config/params"
 	"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v5/testing/assert"
@@ -26,7 +27,7 @@ func TestBlobStoragePath_NoFlagSpecified(t *testing.T) {
 func TestBlobStoragePath_FlagSpecified(t *testing.T) {
 	app := cli.App{}
 	set := flag.NewFlagSet("test", 0)
-	set.String(BlobStoragePathFlag.Name, "/blah/blah", BlobStoragePathFlag.Usage)
+	set.String(flags.BlobStoragePathFlag.Name, "/blah/blah", flags.BlobStoragePathFlag.Usage)
 	cliCtx := cli.NewContext(&app, set, nil)
 	storagePath := blobStoragePath(cliCtx)
 
@@ -46,18 +47,18 @@ func TestConfigureBlobRetentionEpoch(t *testing.T) {
 	require.Equal(t, specMinEpochs, epochs)
 
 	// manually define the flag in the set, so the following code can use set.Set
-	set.Uint64(BlobRetentionEpochFlag.Name, 0, "")
+	set.Uint64(flags.BlobRetentionEpochFlag.Name, 0, "")
 
 	// Test case: Input epoch is greater than or equal to spec value.
 	expectedChange := specMinEpochs + 1
-	require.NoError(t, set.Set(BlobRetentionEpochFlag.Name, fmt.Sprintf("%d", expectedChange)))
+	require.NoError(t, set.Set(flags.BlobRetentionEpochFlag.Name, fmt.Sprintf("%d", expectedChange)))
 	epochs, err = blobRetentionEpoch(cliCtx)
 	require.NoError(t, err)
 	require.Equal(t, primitives.Epoch(expectedChange), epochs)
 
 	// Test case: Input epoch is less than spec value.
 	expectedChange = specMinEpochs - 1
-	require.NoError(t, set.Set(BlobRetentionEpochFlag.Name, fmt.Sprintf("%d", expectedChange)))
+	require.NoError(t, set.Set(flags.BlobRetentionEpochFlag.Name, fmt.Sprintf("%d", expectedChange)))
 	_, err = blobRetentionEpoch(cliCtx)
 	require.ErrorIs(t, err, errInvalidBlobRetentionEpochs)
 }

cmd/beacon-chain/usage.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/prysmaticlabs/prysm/v5/cmd"
 	"github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/flags"
-	"github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/storage"
+	storageFlags "github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/storage/flags"
 	backfill "github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/sync/backfill/flags"
 	"github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/sync/checkpoint"
 	"github.com/prysmaticlabs/prysm/v5/cmd/beacon-chain/sync/genesis"
@@ -130,13 +130,14 @@ var appHelpFlagGroups = []flagGroup{
 			flags.SlasherDirFlag,
 			flags.LocalBlockValueBoost,
 			flags.JwtId,
+			flags.ArchivalNodeFlag,
 			checkpoint.BlockPath,
 			checkpoint.StatePath,
 			checkpoint.RemoteURL,
 			genesis.StatePath,
 			genesis.BeaconAPIURL,
-			storage.BlobStoragePathFlag,
-			storage.BlobRetentionEpochFlag,
+			storageFlags.BlobStoragePathFlag,
+			storageFlags.BlobRetentionEpochFlag,
 			backfill.EnableExperimentalBackfill,
 			backfill.BackfillWorkerCount,
 			backfill.BackfillBatchSize,