hdiff restart-support: validate on startup by default

beacon-chain/db/kv/error.go

@@ -26,6 +26,15 @@ var ErrNotFoundMetadataSeqNum = errors.Wrap(ErrNotFound, "metadata sequence numb
 // but the database was created without state-diff support.
 var ErrStateDiffIncompatible = errors.New("state-diff feature enabled but database was created without state-diff support")
 
+// ErrStateDiffCorrupted is returned when state-diff metadata or data is missing or invalid.
+var ErrStateDiffCorrupted = errors.New("state-diff database corrupted")
+
+// ErrStateDiffExponentMismatch is returned when configured exponents differ from stored metadata.
+var ErrStateDiffExponentMismatch = errors.New("state-diff exponents mismatch")
+
+// ErrStateDiffMissingSnapshot is returned when the offset snapshot is missing.
+var ErrStateDiffMissingSnapshot = errors.New("state-diff offset snapshot missing")
+
 var errEmptyBlockSlice = errors.New("[]blocks.ROBlock is empty")
 var errIncorrectBlockParent = errors.New("unexpected missing or forked blocks in a []ROBlock")
 var errFinalizedChildNotFound = errors.New("unable to find finalized root descending from backfill batch")

beacon-chain/db/kv/kv.go

@@ -7,9 +7,11 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"slices"
 	"time"
 
 	"github.com/OffchainLabs/prysm/v7/beacon-chain/db/iface"
+	"github.com/OffchainLabs/prysm/v7/cmd/beacon-chain/flags"
 	"github.com/OffchainLabs/prysm/v7/config/features"
 	"github.com/OffchainLabs/prysm/v7/config/params"
 	"github.com/OffchainLabs/prysm/v7/consensus-types/blocks"
@@ -21,6 +23,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	prombolt "github.com/prysmaticlabs/prombbolt"
+	logrus "github.com/sirupsen/logrus"
 	bolt "go.etcd.io/bbolt"
 )
 
@@ -223,8 +226,42 @@ func (kv *Store) startStateDiff(ctx context.Context) error {
 	}
 
 	if hasOffset {
-		// Existing state-diff database - restarts not yet supported.
-		return errors.New("restarting with existing state-diff database not yet supported")
+		storedExponents, err := kv.loadStateDiffExponents()
+		if err != nil {
+			return fmt.Errorf("%w: state-diff metadata missing or invalid; re-sync required: %v", ErrStateDiffCorrupted, err)
+		}
+		currentExponents := flags.Get().StateDiffExponents
+		if !slices.Equal(storedExponents, currentExponents) {
+			return errors.Wrapf(
+				ErrStateDiffExponentMismatch,
+				"state-diff exponents changed; database incompatible. "+
+					"Database was initialized with: %v. "+
+					"Current configuration: %v. "+
+					"Options: use original exponents (--state-diff-exponents=%s) or delete database and re-sync from genesis/checkpoint.",
+				storedExponents,
+				currentExponents,
+				formatStateDiffExponents(storedExponents),
+			)
+		}
+		offset, err := kv.loadOffset()
+		if err != nil {
+			return err
+		}
+		cache, err := populateStateDiffCacheFromDB(kv, offset)
+		if err != nil {
+			return err
+		}
+		kv.stateDiffCache = cache
+		if flags.Get().StateDiffValidateOnStartup {
+			if err := validateStateDiffCache(ctx, kv, cache); err != nil {
+				return err
+			}
+		}
+		log.WithFields(logrus.Fields{
+			"offset":    offset,
+			"exponents": storedExponents,
+		}).Info("State-diff cache initialized from existing database")
+		return nil
 	}
 
 	// Check if this is a new database (no head block).

beacon-chain/db/kv/kv_test.go

@@ -3,12 +3,15 @@ package kv
 import (
 	"bytes"
 	"context"
+	"encoding/binary"
 	"fmt"
 	"testing"
 
+	"github.com/OffchainLabs/prysm/v7/cmd/beacon-chain/flags"
 	"github.com/OffchainLabs/prysm/v7/config/features"
 	"github.com/OffchainLabs/prysm/v7/consensus-types/blocks"
 	ethpb "github.com/OffchainLabs/prysm/v7/proto/prysm/v1alpha1"
+	"github.com/OffchainLabs/prysm/v7/runtime/version"
 	"github.com/OffchainLabs/prysm/v7/testing/require"
 	"github.com/OffchainLabs/prysm/v7/testing/util"
 	bolt "go.etcd.io/bbolt"
@@ -27,6 +30,108 @@ func setupDB(t testing.TB) *Store {
 	return db
 }
 
+func TestStartStateDiff_ExponentMismatch(t *testing.T) {
+	resetCfg := features.InitWithReset(&features.Flags{EnableStateDiff: true})
+	defer resetCfg()
+	setDefaultStateDiffExponents()
+
+	store := setupDB(t)
+	require.NoError(t, store.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket(stateDiffBucket)
+		if bucket == nil {
+			return bolt.ErrBucketNotFound
+		}
+		offsetBytes := make([]byte, 8)
+		binary.LittleEndian.PutUint64(offsetBytes, 0)
+		if err := bucket.Put(offsetKey, offsetBytes); err != nil {
+			return err
+		}
+		encoded, err := encodeStateDiffExponents([]int{20, 10})
+		if err != nil {
+			return err
+		}
+		return bucket.Put(exponentsKey, encoded)
+	}))
+
+	ctx := t.Context()
+	err := store.startStateDiff(ctx)
+	require.ErrorContains(t, "state-diff exponents changed", err)
+}
+
+func TestStartStateDiff_MissingOffsetSnapshot(t *testing.T) {
+	resetCfg := features.InitWithReset(&features.Flags{EnableStateDiff: true})
+	defer resetCfg()
+	setDefaultStateDiffExponents()
+
+	store := setupDB(t)
+	require.NoError(t, store.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket(stateDiffBucket)
+		if bucket == nil {
+			return bolt.ErrBucketNotFound
+		}
+		offsetBytes := make([]byte, 8)
+		binary.LittleEndian.PutUint64(offsetBytes, 0)
+		if err := bucket.Put(offsetKey, offsetBytes); err != nil {
+			return err
+		}
+		encoded, err := encodeStateDiffExponents(flags.Get().StateDiffExponents)
+		if err != nil {
+			return err
+		}
+		return bucket.Put(exponentsKey, encoded)
+	}))
+
+	ctx := t.Context()
+	err := store.startStateDiff(ctx)
+	require.ErrorContains(t, "missing offset snapshot", err)
+}
+
+func TestStartStateDiff_ValidateOnStartup(t *testing.T) {
+	resetCfg := features.InitWithReset(&features.Flags{EnableStateDiff: true})
+	defer resetCfg()
+	setDefaultStateDiffExponents()
+
+	globalFlags := flags.GlobalFlags{
+		StateDiffExponents:         flags.Get().StateDiffExponents,
+		StateDiffValidateOnStartup: true,
+	}
+	flags.Init(&globalFlags)
+
+	store := setupDB(t)
+	require.NoError(t, store.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket(stateDiffBucket)
+		if bucket == nil {
+			return bolt.ErrBucketNotFound
+		}
+		st, _ := createState(t, 0, version.Phase0)
+		stateBytes, err := st.MarshalSSZ()
+		if err != nil {
+			return err
+		}
+		enc, err := addKey(st.Version(), stateBytes)
+		if err != nil {
+			return err
+		}
+		offsetBytes := make([]byte, 8)
+		binary.LittleEndian.PutUint64(offsetBytes, 0)
+		if err := bucket.Put(offsetKey, offsetBytes); err != nil {
+			return err
+		}
+		encoded, err := encodeStateDiffExponents(flags.Get().StateDiffExponents)
+		if err != nil {
+			return err
+		}
+		if err := bucket.Put(exponentsKey, encoded); err != nil {
+			return err
+		}
+		key := makeKeyForStateDiffTree(0, 0)
+		return bucket.Put(key, enc)
+	}))
+
+	err := store.startStateDiff(t.Context())
+	require.NoError(t, err)
+}
+
 func Test_setupBlockStorageType(t *testing.T) {
 	ctx := t.Context()
 	t.Run("fresh database with feature enabled to store full blocks should store full blocks", func(t *testing.T) {

beacon-chain/db/kv/state_diff.go

@@ -132,6 +132,9 @@ func (s *Store) saveHdiff(lvl int, anchor, st state.ReadOnlyBeaconState) error {
 			return err
 		}
 	}
+	if err := s.stateDiffCache.setLevelHasData(lvl); err != nil {
+		return err
+	}
 
 	return nil
 }
@@ -171,6 +174,9 @@ func (s *Store) saveFullSnapshot(st state.ReadOnlyBeaconState) error {
 	if err != nil {
 		return err
 	}
+	if err := s.stateDiffCache.setLevelHasData(0); err != nil {
+		return err
+	}
 
 	return nil
 }

beacon-chain/db/kv/state_diff_cache.go

@@ -1,19 +1,131 @@
 package kv
 
 import (
+	"context"
 	"encoding/binary"
 	"errors"
 	"sync"
 
 	"github.com/OffchainLabs/prysm/v7/beacon-chain/state"
 	"github.com/OffchainLabs/prysm/v7/cmd/beacon-chain/flags"
+	"github.com/OffchainLabs/prysm/v7/consensus-types/primitives"
+	pkgerrors "github.com/pkg/errors"
 	"go.etcd.io/bbolt"
 )
 
 type stateDiffCache struct {
 	sync.RWMutex
-	anchors []state.ReadOnlyBeaconState
-	offset  uint64
+	anchors        []state.ReadOnlyBeaconState
+	levelsWithData []bool
+	offset         uint64
+}
+
+func populateStateDiffCacheFromDB(s *Store, offset uint64) (*stateDiffCache, error) {
+	cache := &stateDiffCache{
+		anchors:        make([]state.ReadOnlyBeaconState, len(flags.Get().StateDiffExponents)-1),
+		levelsWithData: make([]bool, len(flags.Get().StateDiffExponents)),
+		offset:         offset,
+	}
+
+	if err := s.db.View(func(tx *bbolt.Tx) error {
+		bucket := tx.Bucket(stateDiffBucket)
+		if bucket == nil {
+			return bbolt.ErrBucketNotFound
+		}
+		for level := range cache.levelsWithData {
+			if level == 0 {
+				if bucket.Get(makeKeyForStateDiffTree(0, offset)) != nil {
+					cache.levelsWithData[level] = true
+				}
+				continue
+			}
+			cursor := bucket.Cursor()
+			prefix := []byte{byte(level)}
+			key, _ := cursor.Seek(prefix)
+			if key != nil && key[0] == byte(level) {
+				slot, ok := slotFromStateDiffKey(key)
+				if !ok {
+					return ErrStateDiffCorrupted
+				}
+				if slot < offset {
+					return ErrStateDiffCorrupted
+				}
+				if level == 0 && slot != offset {
+					return ErrStateDiffCorrupted
+				}
+				if computeLevel(offset, primitives.Slot(slot)) != level {
+					return ErrStateDiffCorrupted
+				}
+				cache.levelsWithData[level] = true
+			}
+		}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	anchor0, err := s.getFullSnapshot(offset)
+	if err != nil {
+		return nil, pkgerrors.Wrapf(ErrStateDiffMissingSnapshot, "state diff cache: missing offset snapshot at %d", offset)
+	}
+	cache.anchors[0] = anchor0
+	cache.levelsWithData[0] = true
+
+	return cache, nil
+}
+
+func validateStateDiffCache(ctx context.Context, s *Store, cache *stateDiffCache) error {
+	for level, hasData := range cache.levelsWithData {
+		if !hasData || level == 0 {
+			continue
+		}
+		maxSlot, err := latestSlotForLevel(s, level)
+		if err != nil {
+			return err
+		}
+		if _, err := s.stateByDiff(ctx, primitives.Slot(maxSlot)); err != nil {
+			return pkgerrors.Wrapf(ErrStateDiffCorrupted, "state diff validation failed for level %d slot %d: %v", level, maxSlot, err)
+		}
+	}
+	return nil
+}
+
+func latestSlotForLevel(s *Store, level int) (uint64, error) {
+	var maxSlot uint64
+	found := false
+	err := s.db.View(func(tx *bbolt.Tx) error {
+		bucket := tx.Bucket(stateDiffBucket)
+		if bucket == nil {
+			return bbolt.ErrBucketNotFound
+		}
+		cursor := bucket.Cursor()
+		prefix := []byte{byte(level)}
+		for key, _ := cursor.Seek(prefix); key != nil && key[0] == byte(level); key, _ = cursor.Next() {
+			slot, ok := slotFromStateDiffKey(key)
+			if !ok {
+				return ErrStateDiffCorrupted
+			}
+			if !found || slot > maxSlot {
+				maxSlot = slot
+				found = true
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return 0, err
+	}
+	if !found {
+		return 0, ErrStateDiffCorrupted
+	}
+	return maxSlot, nil
+}
+
+func slotFromStateDiffKey(key []byte) (uint64, bool) {
+	if len(key) < 9 {
+		return 0, false
+	}
+	return binary.LittleEndian.Uint64(key[1:9]), true
 }
 
 func newStateDiffCache(s *Store) (*stateDiffCache, error) {
@@ -37,8 +149,9 @@ func newStateDiffCache(s *Store) (*stateDiffCache, error) {
 	}
 
 	return &stateDiffCache{
-		anchors: make([]state.ReadOnlyBeaconState, len(flags.Get().StateDiffExponents)-1), // -1 because last level doesn't need to be cached
-		offset:  offset,
+		anchors:        make([]state.ReadOnlyBeaconState, len(flags.Get().StateDiffExponents)-1), // -1 because last level doesn't need to be cached
+		levelsWithData: make([]bool, len(flags.Get().StateDiffExponents)),
+		offset:         offset,
 	}, nil
 }
 
@@ -58,6 +171,25 @@ func (c *stateDiffCache) setAnchor(level int, anchor state.ReadOnlyBeaconState)
 	return nil
 }
 
+func (c *stateDiffCache) levelHasData(level int) bool {
+	c.RLock()
+	defer c.RUnlock()
+	if level < 0 || level >= len(c.levelsWithData) {
+		return false
+	}
+	return c.levelsWithData[level]
+}
+
+func (c *stateDiffCache) setLevelHasData(level int) error {
+	c.Lock()
+	defer c.Unlock()
+	if level < 0 || level >= len(c.levelsWithData) {
+		return errors.New("state diff cache: level data index out of range")
+	}
+	c.levelsWithData[level] = true
+	return nil
+}
+
 func (c *stateDiffCache) getOffset() uint64 {
 	c.RLock()
 	defer c.RUnlock()

beacon-chain/db/kv/state_diff_helpers.go

@@ -5,6 +5,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"strings"
 
 	"github.com/OffchainLabs/prysm/v7/beacon-chain/state"
 	statenative "github.com/OffchainLabs/prysm/v7/beacon-chain/state/state-native"
@@ -21,9 +22,78 @@ import (
 
 var (
 	offsetKey           = []byte("offset")
+	exponentsKey        = []byte("exponents")
 	ErrSlotBeforeOffset = errors.New("slot is before state-diff root offset")
 )
 
+func encodeStateDiffExponents(exponents []int) ([]byte, error) {
+	if len(exponents) == 0 {
+		return nil, errors.New("state diff exponents cannot be empty")
+	}
+	if len(exponents) > 255 {
+		return nil, fmt.Errorf("state diff exponents length %d exceeds max 255", len(exponents))
+	}
+	encoded := make([]byte, len(exponents)+1)
+	encoded[0] = byte(len(exponents))
+	for i, exp := range exponents {
+		if exp < 2 || exp > flags.MaxStateDiffExponent {
+			return nil, fmt.Errorf("state diff exponent %d out of range for encoding", exp)
+		}
+		encoded[i+1] = byte(exp)
+	}
+	return encoded, nil
+}
+
+func decodeStateDiffExponents(encoded []byte) ([]int, error) {
+	if len(encoded) == 0 {
+		return nil, errors.New("state diff exponents missing length prefix")
+	}
+	count := int(encoded[0])
+	if count == 0 {
+		return nil, errors.New("state diff exponents length cannot be zero")
+	}
+	if len(encoded) != count+1 {
+		return nil, fmt.Errorf("state diff exponents length mismatch: expected %d got %d", count, len(encoded)-1)
+	}
+	exponents := make([]int, count)
+	for i := range count {
+		exponents[i] = int(encoded[i+1])
+	}
+	return exponents, nil
+}
+
+func formatStateDiffExponents(exponents []int) string {
+	if len(exponents) == 0 {
+		return ""
+	}
+	parts := make([]string, len(exponents))
+	for i, exp := range exponents {
+		parts[i] = fmt.Sprintf("%d", exp)
+	}
+	return strings.Join(parts, ",")
+}
+
+func (s *Store) loadStateDiffExponents() ([]int, error) {
+	var encoded []byte
+	err := s.db.View(func(tx *bbolt.Tx) error {
+		bucket := tx.Bucket(stateDiffBucket)
+		if bucket == nil {
+			return bbolt.ErrBucketNotFound
+		}
+		value := bucket.Get(exponentsKey)
+		if value == nil {
+			return errors.New("state diff exponents not found")
+		}
+		encoded = make([]byte, len(value))
+		copy(encoded, value)
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return decodeStateDiffExponents(encoded)
+}
+
 func makeKeyForStateDiffTree(level int, slot uint64) []byte {
 	buf := make([]byte, 16)
 	buf[0] = byte(level)
@@ -124,6 +194,29 @@ func (s *Store) getOffset() uint64 {
 	return s.stateDiffCache.getOffset()
 }
 
+func (s *Store) loadOffset() (uint64, error) {
+	var offset uint64
+	err := s.db.View(func(tx *bbolt.Tx) error {
+		bucket := tx.Bucket(stateDiffBucket)
+		if bucket == nil {
+			return bbolt.ErrBucketNotFound
+		}
+		offsetBytes := bucket.Get(offsetKey)
+		if offsetBytes == nil {
+			return errors.New("state diff offset not found")
+		}
+		if len(offsetBytes) != 8 {
+			return fmt.Errorf("state diff offset has invalid length %d", len(offsetBytes))
+		}
+		offset = binary.LittleEndian.Uint64(offsetBytes)
+		return nil
+	})
+	if err != nil {
+		return 0, err
+	}
+	return offset, nil
+}
+
 // hasStateDiffOffset checks if the state-diff offset has been set in the database.
 // This is used to detect if an existing database has state-diff enabled.
 func (s *Store) hasStateDiffOffset() (bool, error) {
@@ -153,8 +246,13 @@ func (s *Store) initializeStateDiff(slot primitives.Slot, initialState state.Rea
 			return nil
 		}
 	}
-	// Write offset directly to the database (without using cache which doesn't exist yet).
-	err := s.db.Update(func(tx *bbolt.Tx) error {
+	exponentsBytes, err := encodeStateDiffExponents(flags.Get().StateDiffExponents)
+	if err != nil {
+		return pkgerrors.Wrap(err, "failed to encode state diff exponents")
+	}
+
+	// Write metadata directly to the database (without using cache which doesn't exist yet).
+	err = s.db.Update(func(tx *bbolt.Tx) error {
 		bucket := tx.Bucket(stateDiffBucket)
 		if bucket == nil {
 			return bbolt.ErrBucketNotFound
@@ -162,7 +260,10 @@ func (s *Store) initializeStateDiff(slot primitives.Slot, initialState state.Rea
 
 		offsetBytes := make([]byte, 8)
 		binary.LittleEndian.PutUint64(offsetBytes, uint64(slot))
-		return bucket.Put(offsetKey, offsetBytes)
+		if err := bucket.Put(offsetKey, offsetBytes); err != nil {
+			return err
+		}
+		return bucket.Put(exponentsKey, exponentsBytes)
 	})
 	if err != nil {
 		return pkgerrors.Wrap(err, "failed to set offset")
@@ -293,7 +394,12 @@ func (s *Store) getBaseAndDiffChain(offset uint64, slot primitives.Slot) (state.
 		if diffSlot == lastSeenAnchorSlot {
 			continue
 		}
-		diffChainItems = append(diffChainItems, diffItem{level: i + 1, slot: diffSlot + offset})
+		level := i + 1
+		if s.stateDiffCache != nil && !s.stateDiffCache.levelHasData(level) {
+			lastSeenAnchorSlot = diffSlot
+			continue
+		}
+		diffChainItems = append(diffChainItems, diffItem{level: level, slot: diffSlot + offset})
 		lastSeenAnchorSlot = diffSlot
 	}
 

beacon-chain/db/kv/state_diff_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/OffchainLabs/prysm/v7/beacon-chain/state"
 	"github.com/OffchainLabs/prysm/v7/cmd/beacon-chain/flags"
+	"github.com/OffchainLabs/prysm/v7/config/features"
 	"github.com/OffchainLabs/prysm/v7/config/params"
 	"github.com/OffchainLabs/prysm/v7/consensus-types/primitives"
 	"github.com/OffchainLabs/prysm/v7/math"
@@ -34,6 +35,81 @@ func TestStateDiff_LoadOrInitOffset(t *testing.T) {
 	require.Equal(t, uint64(10), offset)
 }
 
+func TestStateDiff_LoadOffset(t *testing.T) {
+	setDefaultStateDiffExponents()
+
+	db := setupDB(t)
+	_, err := db.loadOffset()
+	require.ErrorContains(t, "offset not found", err)
+
+	err = setOffsetInDB(db, 10)
+	require.NoError(t, err)
+	offset, err := db.loadOffset()
+	require.NoError(t, err)
+	require.Equal(t, uint64(10), offset)
+}
+
+func TestStateDiff_EncodeDecodeExponents(t *testing.T) {
+	t.Run("roundtrip", func(t *testing.T) {
+		exponents := []int{21, 18, 16, 13}
+		encoded, err := encodeStateDiffExponents(exponents)
+		require.NoError(t, err)
+		decoded, err := decodeStateDiffExponents(encoded)
+		require.NoError(t, err)
+		require.DeepEqual(t, exponents, decoded)
+	})
+
+	t.Run("encode-empty", func(t *testing.T) {
+		_, err := encodeStateDiffExponents(nil)
+		require.ErrorContains(t, "cannot be empty", err)
+	})
+
+	t.Run("encode-negative", func(t *testing.T) {
+		_, err := encodeStateDiffExponents([]int{21, -1})
+		require.ErrorContains(t, "out of range", err)
+	})
+
+	t.Run("encode-too-large", func(t *testing.T) {
+		_, err := encodeStateDiffExponents([]int{flags.MaxStateDiffExponent + 1})
+		require.ErrorContains(t, "out of range", err)
+	})
+
+	t.Run("decode-empty", func(t *testing.T) {
+		_, err := decodeStateDiffExponents(nil)
+		require.ErrorContains(t, "missing length prefix", err)
+	})
+
+	t.Run("decode-zero-length", func(t *testing.T) {
+		_, err := decodeStateDiffExponents([]byte{0})
+		require.ErrorContains(t, "length cannot be zero", err)
+	})
+
+	t.Run("decode-length-mismatch", func(t *testing.T) {
+		_, err := decodeStateDiffExponents([]byte{2, 10})
+		require.ErrorContains(t, "length mismatch", err)
+	})
+}
+
+func TestStateDiff_InitializeStoresExponents(t *testing.T) {
+	setDefaultStateDiffExponents()
+	resetCfg := features.InitWithReset(&features.Flags{EnableStateDiff: true})
+	defer resetCfg()
+
+	db := setupDB(t)
+	st, _ := createState(t, 0, version.Phase0)
+	require.NoError(t, db.initializeStateDiff(0, st))
+
+	stored, err := db.loadStateDiffExponents()
+	require.NoError(t, err)
+	require.DeepEqual(t, flags.Get().StateDiffExponents, stored)
+}
+
+func TestStateDiff_LoadExponentsMissing(t *testing.T) {
+	db := setupDB(t)
+	_, err := db.loadStateDiffExponents()
+	require.ErrorContains(t, "exponents not found", err)
+}
+
 func TestStateDiff_ComputeLevel(t *testing.T) {
 	db := setupDB(t)
 	setDefaultStateDiffExponents()
@@ -154,6 +230,94 @@ func TestStateDiff_SaveFullSnapshot(t *testing.T) {
 	}
 }
 
+func TestStateDiff_PopulateStateDiffCacheFromDB(t *testing.T) {
+	setDefaultStateDiffExponents()
+
+	db := setupDB(t)
+	_, err := populateStateDiffCacheFromDB(db, 0)
+	require.ErrorContains(t, "missing offset snapshot", err)
+
+	st, _ := createState(t, 0, version.Phase0)
+	require.NoError(t, setOffsetInDB(db, 0))
+	require.NoError(t, db.saveStateByDiff(context.Background(), st))
+
+	err = db.db.Update(func(tx *bbolt.Tx) error {
+		bucket := tx.Bucket(stateDiffBucket)
+		if bucket == nil {
+			return bbolt.ErrBucketNotFound
+		}
+		key := makeKeyForStateDiffTree(2, math.PowerOf2(16))
+		return bucket.Put(append(key, stateSuffix...), []byte{1})
+	})
+	require.NoError(t, err)
+
+	cache, err := populateStateDiffCacheFromDB(db, 0)
+	require.NoError(t, err)
+	require.NotNil(t, cache)
+	require.Equal(t, uint64(0), cache.getOffset())
+	require.NotNil(t, cache.getAnchor(0))
+	require.Equal(t, true, cache.levelHasData(0))
+	require.Equal(t, false, cache.levelHasData(1))
+	require.Equal(t, true, cache.levelHasData(2))
+}
+
+func TestStateDiff_PopulateStateDiffCacheFromDB_InvalidLevelKey(t *testing.T) {
+	setDefaultStateDiffExponents()
+
+	db := setupDB(t)
+	st, _ := createState(t, 0, version.Phase0)
+	require.NoError(t, setOffsetInDB(db, 0))
+	require.NoError(t, db.saveStateByDiff(context.Background(), st))
+
+	require.NoError(t, db.db.Update(func(tx *bbolt.Tx) error {
+		bucket := tx.Bucket(stateDiffBucket)
+		if bucket == nil {
+			return bbolt.ErrBucketNotFound
+		}
+		key := makeKeyForStateDiffTree(2, 1)
+		return bucket.Put(append(key, stateSuffix...), []byte{1})
+	}))
+
+	_, err := populateStateDiffCacheFromDB(db, 0)
+	require.ErrorIs(t, ErrStateDiffCorrupted, err)
+}
+
+func TestStateDiff_GetBaseAndDiffChainSkipsEmptyLevels(t *testing.T) {
+	setDefaultStateDiffExponents()
+
+	db := setupDB(t)
+	require.NoError(t, setOffsetInDB(db, 0))
+	st, _ := createState(t, 0, version.Phase0)
+	require.NoError(t, db.saveFullSnapshot(st))
+
+	cache, err := populateStateDiffCacheFromDB(db, 0)
+	require.NoError(t, err)
+	cache.levelsWithData[0] = true
+	cache.levelsWithData[1] = false
+	cache.levelsWithData[2] = true
+	db.stateDiffCache = cache
+
+	slot := primitives.Slot(math.PowerOf2(18) + math.PowerOf2(16))
+	key := makeKeyForStateDiffTree(2, uint64(slot))
+	require.NoError(t, db.db.Update(func(tx *bbolt.Tx) error {
+		bucket := tx.Bucket(stateDiffBucket)
+		if bucket == nil {
+			return bbolt.ErrBucketNotFound
+		}
+		if err := bucket.Put(append(key, stateSuffix...), []byte{1}); err != nil {
+			return err
+		}
+		if err := bucket.Put(append(key, validatorSuffix...), []byte{2}); err != nil {
+			return err
+		}
+		return bucket.Put(append(key, balancesSuffix...), []byte{3})
+	}))
+
+	_, diffChain, err := db.getBaseAndDiffChain(0, slot)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(diffChain))
+}
+
 func TestStateDiff_SaveAndReadFullSnapshot(t *testing.T) {
 	setDefaultStateDiffExponents()
 

beacon-chain/node/node.go

@@ -550,6 +550,12 @@ func openDB(ctx context.Context, dbPath string, clearer *dbClearer) (*kv.Store,
 		cfg := features.Get()
 		cfg.EnableStateDiff = false
 		features.Init(cfg)
+	} else if errors.Is(err, kv.ErrStateDiffExponentMismatch) {
+		log.WithError(err).Error("State-diff configuration mismatch; restart aborted. Use the stored exponents or re-sync the database.")
+		return nil, err
+	} else if errors.Is(err, kv.ErrStateDiffMissingSnapshot) || errors.Is(err, kv.ErrStateDiffCorrupted) {
+		log.WithError(err).Error("State-diff database corrupted; restart aborted. Delete database and re-sync from genesis/checkpoint.")
+		return nil, err
 	} else if err != nil {
 		return nil, errors.Wrapf(err, "could not create database at %s", dbPath)
 	}

cmd/beacon-chain/flags/base.go

@@ -356,6 +356,11 @@ var (
 		Usage: "A comma-separated list of exponents (of 2) in decreasing order, defining the state diff hierarchy levels. The last exponent must be greater than or equal to 5.",
 		Value: cli.NewIntSlice(21, 18, 16, 13, 11, 9, 5),
 	}
+	// StateDiffValidateOnStartup validates state diff data on startup.
+	StateDiffValidateOnStartup = &cli.BoolFlag{
+		Name:  "disable-hdiff-validate-on-startup",
+		Usage: "Disables state-diff validation on startup (enabled by default).",
+	}
 	// DisableEphemeralLogFile disables the 24 hour debug log file.
 	DisableEphemeralLogFile = &cli.BoolFlag{
 		Name:  "disable-ephemeral-log-file",

cmd/beacon-chain/flags/config.go

@@ -9,24 +9,25 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
-const maxStateDiffExponents = 30
+const MaxStateDiffExponent = 30
 
 // GlobalFlags specifies all the global flags for the
 // beacon node.
 type GlobalFlags struct {
-	SubscribeToAllSubnets           bool
+	StateDiffValidateOnStartup      bool
 	Supernode                       bool
-	SemiSupernode                   bool
 	DisableGetBlobsV2               bool
-	MinimumSyncPeers                int
-	MinimumPeersPerSubnet           int
-	MaxConcurrentDials              int
-	BlockBatchLimit                 int
-	BlockBatchLimitBurstFactor      int
-	BlobBatchLimit                  int
+	SemiSupernode                   bool
+	SubscribeToAllSubnets           bool
 	BlobBatchLimitBurstFactor       int
 	DataColumnBatchLimit            int
+	BlockBatchLimit                 int
+	MaxConcurrentDials              int
+	MinimumPeersPerSubnet           int
+	MinimumSyncPeers                int
 	DataColumnBatchLimitBurstFactor int
+	BlockBatchLimitBurstFactor      int
+	BlobBatchLimit                  int
 	StateDiffExponents              []int
 }
 
@@ -80,6 +81,7 @@ func ConfigureGlobalFlags(ctx *cli.Context) error {
 
 	// State-diff-exponents
 	cfg.StateDiffExponents = ctx.IntSlice(StateDiffExponents.Name)
+	cfg.StateDiffValidateOnStartup = !ctx.Bool(StateDiffValidateOnStartup.Name)
 	if features.Get().EnableStateDiff {
 		if err := validateStateDiffExponents(cfg.StateDiffExponents); err != nil {
 			return err
@@ -88,6 +90,9 @@ func ConfigureGlobalFlags(ctx *cli.Context) error {
 		if ctx.IsSet(StateDiffExponents.Name) {
 			log.Warn("--state-diff-exponents is set but --enable-state-diff is not; the value will be ignored.")
 		}
+		if ctx.IsSet(StateDiffValidateOnStartup.Name) {
+			log.Warn("--disable-hdiff-validate-on-startup is set but --enable-state-diff is not; the value will be ignored.")
+		}
 	}
 
 	cfg.BlockBatchLimit = ctx.Int(BlockBatchLimit.Name)
@@ -132,7 +137,7 @@ func validateStateDiffExponents(exponents []int) error {
 	if exponents[length-1] < 5 {
 		return errors.New("the last state diff exponent must be at least 5")
 	}
-	prev := maxStateDiffExponents + 1
+	prev := MaxStateDiffExponent + 1
 	for _, exp := range exponents {
 		if exp >= prev {
 			return errors.New("state diff exponents must be in strictly decreasing order, and each exponent must be <= 30")

cmd/beacon-chain/main.go

@@ -161,6 +161,7 @@ var appFlags = []cli.Flag{
 	dasFlags.BlobRetentionEpochFlag,
 	flags.BatchVerifierLimit,
 	flags.StateDiffExponents,
+	flags.StateDiffValidateOnStartup,
 	flags.DisableEphemeralLogFile,
 }
 

cmd/beacon-chain/usage.go

@@ -75,6 +75,7 @@ var appHelpFlagGroups = []flagGroup{
 			flags.RPCPort,
 			flags.BatchVerifierLimit,
 			flags.StateDiffExponents,
+			flags.StateDiffValidateOnStartup,
 		},
 	},
 	{