Move skip-regen-historical-states to deprecated (#6512)

* Remove unused test
* Move flag to be deprecated
* Remove usages
* Update BUILD.bazel
* Merge refs/heads/master into rm-skip-stategen-usages

beacon-chain/blockchain/BUILD.bazel

@@ -74,7 +74,6 @@ go_test(
     srcs = [
         "chain_info_test.go",
         "head_test.go",
-        "init_sync_process_block_test.go",
         "process_attestation_test.go",
         "process_block_test.go",
         "receive_attestation_test.go",

beacon-chain/blockchain/init_sync_process_block.go

@@ -1,64 +1,9 @@
 package blockchain
 
 import (
-	"context"
-
-	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 )
 
-func (s *Service) generateState(ctx context.Context, startRoot [32]byte, endRoot [32]byte) (*stateTrie.BeaconState, error) {
-	preState, err := s.beaconDB.State(ctx, startRoot)
-	if err != nil {
-		return nil, err
-	}
-	if preState == nil {
-		if !s.stateGen.HasState(ctx, startRoot) {
-			if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
-				return nil, errors.Wrap(err, "could not save initial sync blocks")
-			}
-			s.clearInitSyncBlocks()
-		}
-		preState, err = s.stateGen.StateByRoot(ctx, startRoot)
-		if err != nil {
-			return nil, err
-		}
-		if preState == nil {
-			return nil, errors.New("finalized state does not exist in db")
-		}
-	}
-
-	if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
-		return nil, err
-	}
-	var endBlock *ethpb.SignedBeaconBlock
-	if s.hasInitSyncBlock(endRoot) {
-		endBlock = s.getInitSyncBlock(endRoot)
-		s.clearInitSyncBlocks()
-	} else {
-		endBlock, err = s.beaconDB.Block(ctx, endRoot)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	if endBlock == nil {
-		return nil, errors.New("provided block root does not have block saved in the db")
-	}
-	log.Warnf("Generating missing state of slot %d and root %#x", endBlock.Block.Slot, endRoot)
-
-	blocks, err := s.stateGen.LoadBlocks(ctx, preState.Slot()+1, endBlock.Block.Slot, endRoot)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not load the required blocks")
-	}
-	postState, err := s.stateGen.ReplayBlocks(ctx, preState, blocks, endBlock.Block.Slot)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not replay the blocks to generate the resultant state")
-	}
-	return postState, nil
-}
-
 // This saves a beacon block to the initial sync blocks cache.
 func (s *Service) saveInitSyncBlock(r [32]byte, b *ethpb.SignedBeaconBlock) {
 	s.initSyncBlocksLock.Lock()

beacon-chain/blockchain/init_sync_process_block_test.go

@@ -1,97 +0,0 @@
-package blockchain
-
-import (
-	"context"
-	"testing"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
-	"gopkg.in/d4l3k/messagediff.v1"
-)
-
-func TestGenerateState_CorrectlyGenerated(t *testing.T) {
-	db, sc := testDB.SetupDB(t)
-	cfg := &Config{BeaconDB: db, StateGen: stategen.New(db, sc)}
-	service, err := NewService(context.Background(), cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	beaconState, privs := testutil.DeterministicGenesisState(t, 32)
-	genesisBlock := testutil.NewBeaconBlock()
-	bodyRoot, err := stateutil.BlockRoot(genesisBlock.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = beaconState.SetLatestBlockHeader(&ethpb.BeaconBlockHeader{
-		Slot:       genesisBlock.Block.Slot,
-		ParentRoot: genesisBlock.Block.ParentRoot,
-		StateRoot:  params.BeaconConfig().ZeroHash[:],
-		BodyRoot:   bodyRoot[:],
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := beaconState.SetSlashings(make([]uint64, params.BeaconConfig().EpochsPerSlashingsVector)); err != nil {
-		t.Fatal(err)
-	}
-	cp := beaconState.CurrentJustifiedCheckpoint()
-	mockRoot := [32]byte{}
-	copy(mockRoot[:], "hello-world")
-	cp.Root = mockRoot[:]
-	if err := beaconState.SetCurrentJustifiedCheckpoint(cp); err != nil {
-		t.Fatal(err)
-	}
-	if err := beaconState.SetCurrentEpochAttestations([]*pb.PendingAttestation{}); err != nil {
-		t.Fatal(err)
-	}
-	err = db.SaveBlock(context.Background(), genesisBlock)
-	if err != nil {
-		t.Fatal(err)
-	}
-	genRoot, err := stateutil.BlockRoot(genesisBlock.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = db.SaveState(context.Background(), beaconState, genRoot)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	lastBlock := &ethpb.SignedBeaconBlock{}
-	for i := uint64(1); i < 10; i++ {
-		block, err := testutil.GenerateFullBlock(beaconState, privs, testutil.DefaultBlockGenConfig(), i)
-		if err != nil {
-			t.Fatal(err)
-		}
-		beaconState, err = state.ExecuteStateTransition(context.Background(), beaconState, block)
-		if err != nil {
-			t.Fatal(err)
-		}
-		err = db.SaveBlock(context.Background(), block)
-		if err != nil {
-			t.Fatal(err)
-		}
-		lastBlock = block
-	}
-	root, err := stateutil.BlockRoot(lastBlock.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	newState, err := service.generateState(context.Background(), genRoot, root)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !ssz.DeepEqual(newState.InnerStateUnsafe(), beaconState.InnerStateUnsafe()) {
-		diff, _ := messagediff.PrettyDiff(newState.InnerStateUnsafe(), beaconState.InnerStateUnsafe())
-		t.Errorf("Generated state is different from what is expected: %s", diff)
-	}
-}

beacon-chain/blockchain/service.go

@@ -33,7 +33,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/slotutil"
 	"go.opencensus.io/trace"
@@ -421,12 +420,9 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "could not get finalized state from db")
 	}
-	if !featureconfig.Get().SkipRegenHistoricalStates {
-		// Since historical states were skipped, the node should start from last finalized check point.
-		finalizedRoot = s.beaconDB.LastArchivedIndexRoot(ctx)
-		if finalizedRoot == params.BeaconConfig().ZeroHash {
-			finalizedRoot = bytesutil.ToBytes32(finalized.Root)
-		}
+	finalizedRoot = s.beaconDB.LastArchivedIndexRoot(ctx)
+	if finalizedRoot == params.BeaconConfig().ZeroHash {
+		finalizedRoot = bytesutil.ToBytes32(finalized.Root)
 	}
 
 	finalizedBlock, err := s.beaconDB.Block(ctx, finalizedRoot)
@@ -434,20 +430,6 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 		return errors.Wrap(err, "could not get finalized block from db")
 	}
 
-	// To skip the regeneration of historical state, the node has to generate the parent of the last finalized state.
-	// We don't need to do this for genesis.
-	atGenesis := s.CurrentSlot() == 0
-	if featureconfig.Get().SkipRegenHistoricalStates && !atGenesis {
-		parentRoot := bytesutil.ToBytes32(finalizedBlock.Block.ParentRoot)
-		parentState, err := s.generateState(ctx, finalizedRoot, parentRoot)
-		if err != nil {
-			return err
-		}
-		if s.beaconDB.SaveState(ctx, parentState, parentRoot) != nil {
-			return err
-		}
-	}
-
 	if finalizedState == nil || finalizedBlock == nil {
 		return errors.New("finalized state and block can't be nil")
 	}

beacon-chain/node/node.go

@@ -292,12 +292,10 @@ func (b *BeaconNode) startDB(cliCtx *cli.Context) error {
 			return errors.Wrap(err, "could not create new database")
 		}
 	} else {
-		if !featureconfig.Get().SkipRegenHistoricalStates {
-			// Only check if historical states were deleted and needed to recompute when
-			// user doesn't want to skip.
-			if err := d.HistoricalStatesDeleted(b.ctx); err != nil {
-				return err
-			}
+		// Only check if historical states were deleted and needed to recompute when
+		// user doesn't want to skip.
+		if err := d.HistoricalStatesDeleted(b.ctx); err != nil {
+			return err
 		}
 	}
 

shared/featureconfig/config.go

@@ -53,7 +53,6 @@ type Flags struct {
 	DontPruneStateStartUp                      bool // DontPruneStateStartUp disables pruning state upon beacon node start up.
 	NewStateMgmt                               bool // NewStateMgmt enables the new state mgmt service.
 	WaitForSynced                              bool // WaitForSynced uses WaitForSynced in validator startup to ensure it can communicate with the beacon node as soon as possible.
-	SkipRegenHistoricalStates                  bool // SkipRegenHistoricalState skips regenerating historical states from genesis to last finalized. This enables a quick switch over to using new-state-mgmt.
 	ReduceAttesterStateCopy                    bool // ReduceAttesterStateCopy reduces head state copies for attester rpc.
 	BatchBlockVerify                           bool // BatchBlockVerify performs batched verification of block batches that we receive when syncing.
 	// DisableForkChoice disables using LMD-GHOST fork choice to update
@@ -193,10 +192,6 @@ func ConfigureBeaconChain(ctx *cli.Context) {
 		log.Warn("Disabling slashing broadcasting to p2p network")
 		cfg.DisableBroadcastSlashings = true
 	}
-	if ctx.Bool(skipRegenHistoricalStates.Name) {
-		log.Warn("Enabling skipping of historical states regen")
-		cfg.SkipRegenHistoricalStates = true
-	}
 	if ctx.IsSet(deprecatedP2PWhitelist.Name) {
 		log.Warnf("--%s is deprecated, please use --%s", deprecatedP2PWhitelist.Name, cmd.P2PAllowList.Name)
 		if err := ctx.Set(cmd.P2PAllowList.Name, ctx.String(deprecatedP2PWhitelist.Name)); err != nil {

shared/featureconfig/flags.go

@@ -119,10 +119,6 @@ var (
 		Name:  "disable-lookback",
 		Usage: "Disables use of the lookback feature and updates attestation history for validators from head to epoch 0",
 	}
-	skipRegenHistoricalStates = &cli.BoolFlag{
-		Name:  "skip-regen-historical-states",
-		Usage: "Skips regeneration and saving of historical states from genesis to last finalized. This enables a quick switch-over to using `--enable-new-state-mgmt`",
-	}
 	disableReduceAttesterStateCopy = &cli.BoolFlag{
 		Name:  "disable-reduce-attester-state-copy",
 		Usage: "Disables the feature to reduce the amount of state copies for attester rpc",
@@ -481,6 +477,11 @@ var (
 		Usage:  deprecatedUsage,
 		Hidden: true,
 	}
+	deprecatedSkipRegenHistoricalStates = &cli.BoolFlag{
+		Name:   "skip-regen-historical-states",
+		Usage:  deprecatedUsage,
+		Hidden: true,
+	}
 )
 
 var deprecatedFlags = []cli.Flag{
@@ -547,6 +548,7 @@ var deprecatedFlags = []cli.Flag{
 	deprecatedEnableProtectProposerFlag,
 	deprecatedEnableProtectAttesterFlag,
 	deprecatedInitSyncVerifyEverythingFlag,
+	deprecatedSkipRegenHistoricalStates,
 }
 
 // ValidatorFlags contains a list of all the feature flags that apply to the validator client.
@@ -591,7 +593,6 @@ var BeaconChainFlags = append(deprecatedFlags, []cli.Flag{
 	dontPruneStateStartUp,
 	disableBroadcastSlashingFlag,
 	waitForSyncedFlag,
-	skipRegenHistoricalStates,
 	disableNewStateMgmt,
 	disableReduceAttesterStateCopy,
 	disableGRPCConnectionLogging,