Add a flag to configure blob retention epoch period (#12941)

* Add a flag to configure blob retention epoch period

* Add test

* Kasey's feedback

* More Kasey's feedback

* Fix lint and tests

---------

Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com>

beacon-chain/db/kv/BUILD.bazel

@@ -13,6 +13,7 @@ go_library(
         "error.go",
         "execution_chain.go",
         "finalized_block_roots.go",
+        "flags.go",
         "genesis.go",
         "key.go",
         "kv.go",
@@ -38,6 +39,7 @@ go_library(
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/genesis:go_default_library",
         "//beacon-chain/state/state-native:go_default_library",
+        "//cmd/beacon-chain/flags:go_default_library",
         "//config/features:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
@@ -65,6 +67,7 @@ go_library(
         "@com_github_prysmaticlabs_prombbolt//:go_default_library",
         "@com_github_schollz_progressbar_v3//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
         "@io_etcd_go_bbolt//:go_default_library",
         "@io_opencensus_go//trace:go_default_library",
         "@org_golang_google_protobuf//proto:go_default_library",
@@ -83,6 +86,7 @@ go_test(
         "encoding_test.go",
         "execution_chain_test.go",
         "finalized_block_roots_test.go",
+        "flags_test.go",
         "genesis_test.go",
         "init_test.go",
         "kv_test.go",
@@ -103,6 +107,7 @@ go_test(
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/genesis:go_default_library",
         "//beacon-chain/state/state-native:go_default_library",
+        "//cmd/beacon-chain/flags:go_default_library",
         "//config/features:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
@@ -119,6 +124,7 @@ go_test(
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_golang_snappy//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
         "@io_bazel_rules_go//go/tools/bazel:go_default_library",
         "@io_etcd_go_bbolt//:go_default_library",
         "@org_golang_google_protobuf//proto:go_default_library",

beacon-chain/db/kv/blob.go

@@ -289,7 +289,6 @@ func blobSidecarKey(blob *ethpb.BlobSidecar) blobRotatingKey {
 
 func slotKey(slot types.Slot) []byte {
 	slotsPerEpoch := params.BeaconConfig().SlotsPerEpoch
-	maxEpochsToPersistBlobs := params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest
 	maxSlotsToPersistBlobs := types.Slot(maxEpochsToPersistBlobs.Mul(uint64(slotsPerEpoch)))
 	return bytesutil.SlotToBytesBigEndian(slot.ModSlot(maxSlotsToPersistBlobs))
 }
@@ -299,14 +298,14 @@ func checkEpochsForBlobSidecarsRequestBucket(db *bolt.DB) error {
 		b := tx.Bucket(chainMetadataBucket)
 		v := b.Get(blobRetentionEpochsKey)
 		if v == nil {
-			if err := b.Put(blobRetentionEpochsKey, bytesutil.Uint64ToBytesBigEndian(uint64(params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest))); err != nil {
+			if err := b.Put(blobRetentionEpochsKey, bytesutil.Uint64ToBytesBigEndian(uint64(maxEpochsToPersistBlobs))); err != nil {
 				return err
 			}
 			return nil
 		}
 		e := bytesutil.BytesToUint64BigEndian(v)
-		if e != uint64(params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest) {
+		if e != uint64(maxEpochsToPersistBlobs) {
-			return fmt.Errorf("epochs for blobs request value in DB %d does not match config value %d", e, params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest)
+			return fmt.Errorf("epochs for blobs request value in DB %d does not match config value %d", e, maxEpochsToPersistBlobs)
 		}
 		return nil
 	}); err != nil {

beacon-chain/db/kv/blob_test.go

@@ -3,10 +3,13 @@ package kv
 import (
 	"context"
 	"crypto/rand"
+	"flag"
 	"fmt"
+	"strconv"
 	"testing"
 
 	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/v4/cmd/beacon-chain/flags"
 	fieldparams "github.com/prysmaticlabs/prysm/v4/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v4/config/params"
 	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
@@ -15,6 +18,7 @@ import (
 	ethpb "github.com/prysmaticlabs/prysm/v4/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v4/testing/assertions"
 	"github.com/prysmaticlabs/prysm/v4/testing/require"
+	"github.com/urfave/cli/v2"
 	bolt "go.etcd.io/bbolt"
 )
 
@@ -515,9 +519,12 @@ func Test_checkEpochsForBlobSidecarsRequestBucket(t *testing.T) {
 	require.NoError(t, checkEpochsForBlobSidecarsRequestBucket(dbStore.db)) // First write
 	require.NoError(t, checkEpochsForBlobSidecarsRequestBucket(dbStore.db)) // First check
 
-	nConfig := params.BeaconNetworkConfig()
+	params.SetupTestConfigCleanup(t)
-	nConfig.MinEpochsForBlobsSidecarsRequest = 42069
+	set := flag.NewFlagSet("test", 0)
-	params.OverrideBeaconNetworkConfig(nConfig)
+	set.Uint64(flags.BlobRetentionEpoch.Name, 0, "")
+	require.NoError(t, set.Set(flags.BlobRetentionEpoch.Name, strconv.FormatUint(42069, 10)))
+	cliCtx := cli.NewContext(&cli.App{}, set, nil)
+	require.NoError(t, ConfigureBlobRetentionEpoch(cliCtx))
 	require.ErrorContains(t, "epochs for blobs request value in DB 4096 does not match config value 42069", checkEpochsForBlobSidecarsRequestBucket(dbStore.db))
 }
 

beacon-chain/db/kv/flags.go

@@ -0,0 +1,33 @@
+package kv
+
+import (
+	"fmt"
+
+	"github.com/prysmaticlabs/prysm/v4/cmd/beacon-chain/flags"
+	"github.com/prysmaticlabs/prysm/v4/config/params"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
+	"github.com/urfave/cli/v2"
+)
+
+var maxEpochsToPersistBlobs = params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest
+
+// ConfigureBlobRetentionEpoch sets the for blob retention based on command-line context. It sets the local config `maxEpochsToPersistBlobs`.
+// If the flag is not set, the spec default `MinEpochsForBlobsSidecarsRequest` is used.
+// An error if the input epoch is smaller than the spec default value.
+func ConfigureBlobRetentionEpoch(cliCtx *cli.Context) error {
+	// Check if the blob retention epoch flag is set.
+	if cliCtx.IsSet(flags.BlobRetentionEpoch.Name) {
+		// Retrieve and cast the epoch value.
+		epochValue := cliCtx.Uint64(flags.BlobRetentionEpoch.Name)
+		e := primitives.Epoch(epochValue)
+
+		// Validate the epoch value against the spec default.
+		if e < params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest {
+			return fmt.Errorf("%s smaller than spec default, %d < %d", flags.BlobRetentionEpoch.Name, e, params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest)
+		}
+
+		maxEpochsToPersistBlobs = e
+	}
+
+	return nil
+}

beacon-chain/db/kv/flags_test.go

@@ -0,0 +1,39 @@
+package kv
+
+import (
+	"flag"
+	"strconv"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/v4/cmd/beacon-chain/flags"
+	"github.com/prysmaticlabs/prysm/v4/config/params"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
+	"github.com/prysmaticlabs/prysm/v4/testing/require"
+	"github.com/urfave/cli/v2"
+)
+
+func TestConfigureBlobRetentionEpoch(t *testing.T) {
+	maxEpochsToPersistBlobs = params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest
+	params.SetupTestConfigCleanup(t)
+	app := cli.App{}
+	set := flag.NewFlagSet("test", 0)
+
+	// Test case: Spec default.
+	require.NoError(t, ConfigureBlobRetentionEpoch(cli.NewContext(&app, set, nil)))
+	require.Equal(t, params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest, maxEpochsToPersistBlobs)
+
+	set.Uint64(flags.BlobRetentionEpoch.Name, 0, "")
+	minEpochsForSidecarRequest := uint64(params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest)
+	require.NoError(t, set.Set(flags.BlobRetentionEpoch.Name, strconv.FormatUint(2*minEpochsForSidecarRequest, 10)))
+	cliCtx := cli.NewContext(&app, set, nil)
+
+	// Test case: Input epoch is greater than or equal to spec value.
+	require.NoError(t, ConfigureBlobRetentionEpoch(cliCtx))
+	require.Equal(t, primitives.Epoch(2*minEpochsForSidecarRequest), maxEpochsToPersistBlobs)
+
+	// Test case: Input epoch is less than spec value.
+	require.NoError(t, set.Set(flags.BlobRetentionEpoch.Name, strconv.FormatUint(minEpochsForSidecarRequest-1, 10)))
+	cliCtx = cli.NewContext(&app, set, nil)
+	err := ConfigureBlobRetentionEpoch(cliCtx)
+	require.ErrorContains(t, "extend-blob-retention-epoch smaller than spec default", err)
+}

beacon-chain/node/node.go

@@ -153,6 +153,9 @@ func New(cliCtx *cli.Context, opts ...Option) (*BeaconNode, error) {
 	if err := configureExecutionSetting(cliCtx); err != nil {
 		return nil, err
 	}
+	if err := kv.ConfigureBlobRetentionEpoch(cliCtx); err != nil {
+		return nil, err
+	}
 	configureFastSSZHashingAlgorithm()
 
 	// Initializes any forks here.

cmd/beacon-chain/flags/base.go

@@ -263,4 +263,9 @@ var (
 		Usage: "Directory for the slasher database",
 		Value: cmd.DefaultDataDir(),
 	}
+	BlobRetentionEpoch = &cli.Uint64Flag{
+		Name:  "extend-blob-retention-epoch",
+		Usage: "Extend blob retention epoch period to beyond default 4096 epochs (~18 days). The node will error at start if input value is less than 4096 epochs.",
+		Value: uint64(params.BeaconNetworkConfig().MinEpochsForBlobsSidecarsRequest),
+	}
 )

cmd/beacon-chain/main.go

@@ -76,6 +76,7 @@ var appFlags = []cli.Flag{
 	flags.MaxBuilderConsecutiveMissedSlots,
 	flags.EngineEndpointTimeoutSeconds,
 	flags.LocalBlockValueBoost,
+	flags.BlobRetentionEpoch,
 	cmd.BackupWebhookOutputDir,
 	cmd.MinimalConfigFlag,
 	cmd.E2EConfigFlag,

cmd/beacon-chain/usage.go

@@ -128,6 +128,7 @@ var appHelpFlagGroups = []flagGroup{
 			flags.EngineEndpointTimeoutSeconds,
 			flags.SlasherDirFlag,
 			flags.LocalBlockValueBoost,
+			flags.BlobRetentionEpoch,
 			checkpoint.BlockPath,
 			checkpoint.StatePath,
 			checkpoint.RemoteURL,