Remove finalized validator index to pubkey cache (#14497)

* remove the cache

* linting

* changelog

* fixing unit tests

CHANGELOG.md

@@ -71,9 +71,10 @@ The format is based on Keep a Changelog, and this project adheres to Semantic Ve
 
 ### Removed
 
-- removed gRPC Gateway
-- Removed unused blobs bundle cache
+- Removed gRPC Gateway.
+- Removed unused blobs bundle cache.
 - Removed consolidation signing domain from params. The Electra design changed such that EL handles consolidation signature verification.
+- Removed finalized validator index cache, no longer needed.
 
 ### Fixed
 

beacon-chain/blockchain/receive_block.go

@@ -273,7 +273,6 @@ func (s *Service) reportPostBlockProcessing(
 func (s *Service) executePostFinalizationTasks(ctx context.Context, finalizedState state.BeaconState) {
 	finalized := s.cfg.ForkChoiceStore.FinalizedCheckpoint()
 	go func() {
-		finalizedState.SaveValidatorIndices() // used to handle Validator index invariant from EIP6110
 		s.sendNewFinalizedEvent(ctx, finalizedState)
 	}()
 	depCtx, cancel := context.WithTimeout(context.Background(), depositDeadline)

beacon-chain/blockchain/service.go

@@ -329,8 +329,6 @@ func (s *Service) StartFromSavedState(saved state.BeaconState) error {
 		return errors.Wrap(err, "failed to initialize blockchain service")
 	}
 
-	saved.SaveValidatorIndices() // used to handle Validator index invariant from EIP6110
-
 	return nil
 }
 

beacon-chain/core/electra/deposits_test.go

@@ -177,7 +177,6 @@ func TestProcessPendingDeposits(t *testing.T) {
 				dep := stateTesting.GeneratePendingDeposit(t, sk, excessBalance, bytesutil.ToBytes32(wc), 0)
 				dep.Signature = common.InfiniteSignature[:]
 				require.NoError(t, st.SetValidators(validators))
-				st.SaveValidatorIndices()
 				require.NoError(t, st.SetPendingDeposits([]*eth.PendingDeposit{dep}))
 				return st
 			}(),
@@ -508,7 +507,6 @@ func stateWithPendingDeposits(t *testing.T, balETH uint64, numDeposits, amount u
 		deps[i] = stateTesting.GeneratePendingDeposit(t, sk, amount, bytesutil.ToBytes32(wc), 0)
 	}
 	require.NoError(t, st.SetValidators(validators))
-	st.SaveValidatorIndices()
 	require.NoError(t, st.SetPendingDeposits(deps))
 	return st
 }
@@ -527,7 +525,6 @@ func TestApplyPendingDeposit_TopUp(t *testing.T) {
 	dep := stateTesting.GeneratePendingDeposit(t, sk, excessBalance, bytesutil.ToBytes32(wc), 0)
 	dep.Signature = common.InfiniteSignature[:]
 	require.NoError(t, st.SetValidators(validators))
-	st.SaveValidatorIndices()
 
 	require.NoError(t, electra.ApplyPendingDeposit(context.Background(), st, dep))
 

beacon-chain/state/interfaces.go

@@ -105,7 +105,6 @@ type WriteOnlyBeaconState interface {
 	AppendHistoricalRoots(root [32]byte) error
 	AppendHistoricalSummaries(*ethpb.HistoricalSummary) error
 	SetLatestExecutionPayloadHeader(payload interfaces.ExecutionData) error
-	SaveValidatorIndices()
 }
 
 // ReadOnlyValidator defines a struct which only has read access to validator methods.

beacon-chain/state/state-native/BUILD.bazel

@@ -46,7 +46,6 @@ go_library(
         "ssz.go",
         "state_trie.go",
         "types.go",
-        "validator_index_cache.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/v5/beacon-chain/state/state-native",
     visibility = ["//visibility:public"],
@@ -121,7 +120,6 @@ go_test(
         "state_test.go",
         "state_trie_test.go",
         "types_test.go",
-        "validator_index_cache_test.go",
     ],
     data = glob(["testdata/**"]) + [
         "@consensus_spec_tests_mainnet//:test_data",

beacon-chain/state/state-native/beacon_state.go

@@ -76,7 +76,6 @@ type BeaconState struct {
 	stateFieldLeaves      map[types.FieldIndex]*fieldtrie.FieldTrie
 	rebuildTrie           map[types.FieldIndex]bool
 	valMapHandler         *stateutil.ValidatorMapHandler
-	validatorIndexCache   *finalizedValidatorIndexCache
 	merkleLayers          [][][]byte
 	sharedFieldReferences map[types.FieldIndex]*stateutil.Reference
 }

beacon-chain/state/state-native/getters_validator.go

@@ -180,10 +180,6 @@ func (b *BeaconState) ValidatorIndexByPubkey(key [fieldparams.BLSPubkeyLength]by
 	b.lock.RLock()
 	defer b.lock.RUnlock()
 
-	if b.Version() >= version.Electra {
-		return b.getValidatorIndex(key)
-	}
-
 	var numOfVals int
 	if features.Get().EnableExperimentalState {
 		numOfVals = b.validatorsMultiValue.Len(b)

beacon-chain/state/state-native/readonly_validator.go

@@ -70,11 +70,6 @@ func (v readOnlyValidator) PublicKey() [fieldparams.BLSPubkeyLength]byte {
 	return pubkey
 }
 
-// publicKeySlice returns the public key in the slice form for the read only validator.
-func (v readOnlyValidator) publicKeySlice() []byte {
-	return v.validator.PublicKey
-}
-
 // WithdrawalCredentials returns the withdrawal credentials of the
 // read only validator.
 func (v readOnlyValidator) GetWithdrawalCredentials() []byte {

beacon-chain/state/state-native/setters_misc.go

@@ -107,22 +107,6 @@ func (b *BeaconState) SetHistoricalRoots(val [][]byte) error {
 	return nil
 }
 
-// SaveValidatorIndices save validator indices of beacon chain to cache
-func (b *BeaconState) SaveValidatorIndices() {
-	if b.Version() < version.Electra {
-		return
-	}
-
-	b.lock.Lock()
-	defer b.lock.Unlock()
-
-	if b.validatorIndexCache == nil {
-		b.validatorIndexCache = newFinalizedValidatorIndexCache()
-	}
-
-	b.saveValidatorIndices()
-}
-
 // AppendHistoricalRoots for the beacon state. Appends the new value
 // to the end of list.
 func (b *BeaconState) AppendHistoricalRoots(root [32]byte) error {

beacon-chain/state/state-native/state_trie.go

@@ -189,12 +189,11 @@ func InitializeFromProtoUnsafePhase0(st *ethpb.BeaconState) (state.BeaconState,
 
 		id: types.Enumerator.Inc(),
 
-		dirtyFields:         make(map[types.FieldIndex]bool, fieldCount),
-		dirtyIndices:        make(map[types.FieldIndex][]uint64, fieldCount),
-		stateFieldLeaves:    make(map[types.FieldIndex]*fieldtrie.FieldTrie, fieldCount),
-		rebuildTrie:         make(map[types.FieldIndex]bool, fieldCount),
-		valMapHandler:       stateutil.NewValMapHandler(st.Validators),
-		validatorIndexCache: newFinalizedValidatorIndexCache(),
+		dirtyFields:      make(map[types.FieldIndex]bool, fieldCount),
+		dirtyIndices:     make(map[types.FieldIndex][]uint64, fieldCount),
+		stateFieldLeaves: make(map[types.FieldIndex]*fieldtrie.FieldTrie, fieldCount),
+		rebuildTrie:      make(map[types.FieldIndex]bool, fieldCount),
+		valMapHandler:    stateutil.NewValMapHandler(st.Validators),
 	}
 
 	if features.Get().EnableExperimentalState {
@@ -296,12 +295,11 @@ func InitializeFromProtoUnsafeAltair(st *ethpb.BeaconStateAltair) (state.BeaconS
 
 		id: types.Enumerator.Inc(),
 
-		dirtyFields:         make(map[types.FieldIndex]bool, fieldCount),
-		dirtyIndices:        make(map[types.FieldIndex][]uint64, fieldCount),
-		stateFieldLeaves:    make(map[types.FieldIndex]*fieldtrie.FieldTrie, fieldCount),
-		rebuildTrie:         make(map[types.FieldIndex]bool, fieldCount),
-		valMapHandler:       stateutil.NewValMapHandler(st.Validators),
-		validatorIndexCache: newFinalizedValidatorIndexCache(),
+		dirtyFields:      make(map[types.FieldIndex]bool, fieldCount),
+		dirtyIndices:     make(map[types.FieldIndex][]uint64, fieldCount),
+		stateFieldLeaves: make(map[types.FieldIndex]*fieldtrie.FieldTrie, fieldCount),
+		rebuildTrie:      make(map[types.FieldIndex]bool, fieldCount),
+		valMapHandler:    stateutil.NewValMapHandler(st.Validators),
 	}
 
 	if features.Get().EnableExperimentalState {
@@ -407,12 +405,11 @@ func InitializeFromProtoUnsafeBellatrix(st *ethpb.BeaconStateBellatrix) (state.B
 
 		id: types.Enumerator.Inc(),
 
-		dirtyFields:         make(map[types.FieldIndex]bool, fieldCount),
-		dirtyIndices:        make(map[types.FieldIndex][]uint64, fieldCount),
-		stateFieldLeaves:    make(map[types.FieldIndex]*fieldtrie.FieldTrie, fieldCount),
-		rebuildTrie:         make(map[types.FieldIndex]bool, fieldCount),
-		valMapHandler:       stateutil.NewValMapHandler(st.Validators),
-		validatorIndexCache: newFinalizedValidatorIndexCache(),
+		dirtyFields:      make(map[types.FieldIndex]bool, fieldCount),
+		dirtyIndices:     make(map[types.FieldIndex][]uint64, fieldCount),
+		stateFieldLeaves: make(map[types.FieldIndex]*fieldtrie.FieldTrie, fieldCount),
+		rebuildTrie:      make(map[types.FieldIndex]bool, fieldCount),
+		valMapHandler:    stateutil.NewValMapHandler(st.Validators),
 	}
 
 	if features.Get().EnableExperimentalState {
@@ -522,12 +519,11 @@ func InitializeFromProtoUnsafeCapella(st *ethpb.BeaconStateCapella) (state.Beaco
 
 		id: types.Enumerator.Inc(),
 
-		dirtyFields:         make(map[types.FieldIndex]bool, fieldCount),
-		dirtyIndices:        make(map[types.FieldIndex][]uint64, fieldCount),
-		stateFieldLeaves:    make(map[types.FieldIndex]*fieldtrie.FieldTrie, fieldCount),
-		rebuildTrie:         make(map[types.FieldIndex]bool, fieldCount),
-		valMapHandler:       stateutil.NewValMapHandler(st.Validators),
-		validatorIndexCache: newFinalizedValidatorIndexCache(),
+		dirtyFields:      make(map[types.FieldIndex]bool, fieldCount),
+		dirtyIndices:     make(map[types.FieldIndex][]uint64, fieldCount),
+		stateFieldLeaves: make(map[types.FieldIndex]*fieldtrie.FieldTrie, fieldCount),
+		rebuildTrie:      make(map[types.FieldIndex]bool, fieldCount),
+		valMapHandler:    stateutil.NewValMapHandler(st.Validators),
 	}
 
 	if features.Get().EnableExperimentalState {
@@ -636,12 +632,11 @@ func InitializeFromProtoUnsafeDeneb(st *ethpb.BeaconStateDeneb) (state.BeaconSta
 		nextWithdrawalValidatorIndex:      st.NextWithdrawalValidatorIndex,
 		historicalSummaries:               st.HistoricalSummaries,
 
-		dirtyFields:         make(map[types.FieldIndex]bool, fieldCount),
-		dirtyIndices:        make(map[types.FieldIndex][]uint64, fieldCount),
-		stateFieldLeaves:    make(map[types.FieldIndex]*fieldtrie.FieldTrie, fieldCount),
-		rebuildTrie:         make(map[types.FieldIndex]bool, fieldCount),
-		valMapHandler:       stateutil.NewValMapHandler(st.Validators),
-		validatorIndexCache: newFinalizedValidatorIndexCache(),
+		dirtyFields:      make(map[types.FieldIndex]bool, fieldCount),
+		dirtyIndices:     make(map[types.FieldIndex][]uint64, fieldCount),
+		stateFieldLeaves: make(map[types.FieldIndex]*fieldtrie.FieldTrie, fieldCount),
+		rebuildTrie:      make(map[types.FieldIndex]bool, fieldCount),
+		valMapHandler:    stateutil.NewValMapHandler(st.Validators),
 	}
 
 	if features.Get().EnableExperimentalState {
@@ -759,12 +754,11 @@ func InitializeFromProtoUnsafeElectra(st *ethpb.BeaconStateElectra) (state.Beaco
 		pendingPartialWithdrawals:         st.PendingPartialWithdrawals,
 		pendingConsolidations:             st.PendingConsolidations,
 
-		dirtyFields:         make(map[types.FieldIndex]bool, fieldCount),
-		dirtyIndices:        make(map[types.FieldIndex][]uint64, fieldCount),
-		stateFieldLeaves:    make(map[types.FieldIndex]*fieldtrie.FieldTrie, fieldCount),
-		rebuildTrie:         make(map[types.FieldIndex]bool, fieldCount),
-		valMapHandler:       stateutil.NewValMapHandler(st.Validators),
-		validatorIndexCache: newFinalizedValidatorIndexCache(), //only used in post-electra and only populates when finalizing, otherwise it falls back to processing the full validator set
+		dirtyFields:      make(map[types.FieldIndex]bool, fieldCount),
+		dirtyIndices:     make(map[types.FieldIndex][]uint64, fieldCount),
+		stateFieldLeaves: make(map[types.FieldIndex]*fieldtrie.FieldTrie, fieldCount),
+		rebuildTrie:      make(map[types.FieldIndex]bool, fieldCount),
+		valMapHandler:    stateutil.NewValMapHandler(st.Validators),
 	}
 
 	if features.Get().EnableExperimentalState {
@@ -925,8 +919,7 @@ func (b *BeaconState) Copy() state.BeaconState {
 		stateFieldLeaves: make(map[types.FieldIndex]*fieldtrie.FieldTrie, fieldCount),
 
 		// Share the reference to validator index map.
-		valMapHandler:       b.valMapHandler,
-		validatorIndexCache: b.validatorIndexCache,
+		valMapHandler: b.valMapHandler,
 	}
 
 	if features.Get().EnableExperimentalState {

beacon-chain/state/state-native/validator_index_cache.go

@@ -1,96 +0,0 @@
-package state_native
-
-import (
-	"bytes"
-	"sync"
-
-	"github.com/prysmaticlabs/prysm/v5/config/features"
-	fieldparams "github.com/prysmaticlabs/prysm/v5/config/fieldparams"
-	"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
-	"github.com/prysmaticlabs/prysm/v5/encoding/bytesutil"
-	ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
-)
-
-// finalizedValidatorIndexCache maintains a mapping from validator public keys to their indices within the beacon state.
-// It includes a lastFinalizedIndex to track updates up to the last finalized validator index,
-// and uses a mutex for concurrent read/write access to the cache.
-type finalizedValidatorIndexCache struct {
-	indexMap map[[fieldparams.BLSPubkeyLength]byte]primitives.ValidatorIndex // Maps finalized BLS public keys to validator indices.
-	sync.RWMutex
-}
-
-// newFinalizedValidatorIndexCache initializes a new validator index cache with an empty index map.
-func newFinalizedValidatorIndexCache() *finalizedValidatorIndexCache {
-	return &finalizedValidatorIndexCache{
-		indexMap: make(map[[fieldparams.BLSPubkeyLength]byte]primitives.ValidatorIndex),
-	}
-}
-
-// getValidatorIndex retrieves the validator index for a given public key from the cache.
-// If the public key is not found in the cache, it searches through the state starting from the last finalized index.
-func (b *BeaconState) getValidatorIndex(pubKey [fieldparams.BLSPubkeyLength]byte) (primitives.ValidatorIndex, bool) {
-	b.validatorIndexCache.RLock()
-	defer b.validatorIndexCache.RUnlock()
-	index, found := b.validatorIndexCache.indexMap[pubKey]
-	if found {
-		return index, true
-	}
-
-	validatorCount := len(b.validatorIndexCache.indexMap)
-	vals := b.validatorsReadOnlySinceIndex(validatorCount)
-	for i, val := range vals {
-		if bytes.Equal(bytesutil.PadTo(val.publicKeySlice(), 48), pubKey[:]) {
-			index := primitives.ValidatorIndex(validatorCount + i)
-			return index, true
-		}
-	}
-	return 0, false
-}
-
-// saveValidatorIndices updates the validator index cache with new indices.
-// It processes validator indices starting after the last finalized index and updates the tracker.
-func (b *BeaconState) saveValidatorIndices() {
-	b.validatorIndexCache.Lock()
-	defer b.validatorIndexCache.Unlock()
-
-	validatorCount := len(b.validatorIndexCache.indexMap)
-	vals := b.validatorsReadOnlySinceIndex(validatorCount)
-	for i, val := range vals {
-		b.validatorIndexCache.indexMap[val.PublicKey()] = primitives.ValidatorIndex(validatorCount + i)
-	}
-}
-
-// validatorsReadOnlySinceIndex constructs a list of read only validator references after a specified index.
-// The indices in the returned list correspond to their respective validator indices in the state.
-// It returns nil if the specified index is out of bounds. This function is read-only and does not use locks.
-func (b *BeaconState) validatorsReadOnlySinceIndex(index int) []readOnlyValidator {
-	totalValidators := b.validatorsLen()
-	if index >= totalValidators {
-		return nil
-	}
-
-	var v []*ethpb.Validator
-	if features.Get().EnableExperimentalState {
-		if b.validatorsMultiValue == nil {
-			return nil
-		}
-		v = b.validatorsMultiValue.Value(b)
-	} else {
-		if b.validators == nil {
-			return nil
-		}
-		v = b.validators
-	}
-
-	result := make([]readOnlyValidator, totalValidators-index)
-	for i := 0; i < len(result); i++ {
-		val := v[i+index]
-		if val == nil {
-			continue
-		}
-		result[i] = readOnlyValidator{
-			validator: val,
-		}
-	}
-	return result
-}

beacon-chain/state/state-native/validator_index_cache_test.go

@@ -1,105 +0,0 @@
-package state_native
-
-import (
-	"testing"
-
-	fieldparams "github.com/prysmaticlabs/prysm/v5/config/fieldparams"
-	"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
-	ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
-	"github.com/prysmaticlabs/prysm/v5/testing/require"
-)
-
-func Test_FinalizedValidatorIndexCache(t *testing.T) {
-	c := newFinalizedValidatorIndexCache()
-	b := &BeaconState{validatorIndexCache: c}
-
-	// What happens if you call getValidatorIndex with a public key that is not in the cache and state?
-	// The function will return 0 and false.
-	i, exists := b.getValidatorIndex([fieldparams.BLSPubkeyLength]byte{0})
-	require.Equal(t, primitives.ValidatorIndex(0), i)
-	require.Equal(t, false, exists)
-
-	// Validators are added to the state. They are [0, 1, 2]
-	b.validators = []*ethpb.Validator{
-		{PublicKey: []byte{1}},
-		{PublicKey: []byte{2}},
-		{PublicKey: []byte{3}},
-	}
-	// We should be able to retrieve these validators by public key even when they are not in the cache
-	i, exists = b.getValidatorIndex([fieldparams.BLSPubkeyLength]byte{1})
-	require.Equal(t, primitives.ValidatorIndex(0), i)
-	require.Equal(t, true, exists)
-	i, exists = b.getValidatorIndex([fieldparams.BLSPubkeyLength]byte{2})
-	require.Equal(t, primitives.ValidatorIndex(1), i)
-	require.Equal(t, true, exists)
-	i, exists = b.getValidatorIndex([fieldparams.BLSPubkeyLength]byte{3})
-	require.Equal(t, primitives.ValidatorIndex(2), i)
-	require.Equal(t, true, exists)
-
-	// State is finalized. We save [0, 1, 2 ] to the cache.
-	b.saveValidatorIndices()
-	require.Equal(t, 3, len(b.validatorIndexCache.indexMap))
-	i, exists = b.getValidatorIndex([fieldparams.BLSPubkeyLength]byte{1})
-	require.Equal(t, primitives.ValidatorIndex(0), i)
-	require.Equal(t, true, exists)
-	i, exists = b.getValidatorIndex([fieldparams.BLSPubkeyLength]byte{2})
-	require.Equal(t, primitives.ValidatorIndex(1), i)
-	require.Equal(t, true, exists)
-	i, exists = b.getValidatorIndex([fieldparams.BLSPubkeyLength]byte{3})
-	require.Equal(t, primitives.ValidatorIndex(2), i)
-	require.Equal(t, true, exists)
-
-	// New validators are added to the state. They are [4, 5]
-	b.validators = []*ethpb.Validator{
-		{PublicKey: []byte{1}},
-		{PublicKey: []byte{2}},
-		{PublicKey: []byte{3}},
-		{PublicKey: []byte{4}},
-		{PublicKey: []byte{5}},
-	}
-	// We should be able to retrieve these validators by public key even when they are not in the cache
-	i, exists = b.getValidatorIndex([fieldparams.BLSPubkeyLength]byte{4})
-	require.Equal(t, primitives.ValidatorIndex(3), i)
-	require.Equal(t, true, exists)
-	i, exists = b.getValidatorIndex([fieldparams.BLSPubkeyLength]byte{5})
-	require.Equal(t, primitives.ValidatorIndex(4), i)
-	require.Equal(t, true, exists)
-
-	// State is finalized. We save [4, 5] to the cache.
-	b.saveValidatorIndices()
-	require.Equal(t, 5, len(b.validatorIndexCache.indexMap))
-
-	// New validators are added to the state. They are [6]
-	b.validators = []*ethpb.Validator{
-		{PublicKey: []byte{1}},
-		{PublicKey: []byte{2}},
-		{PublicKey: []byte{3}},
-		{PublicKey: []byte{4}},
-		{PublicKey: []byte{5}},
-		{PublicKey: []byte{6}},
-	}
-	// We should be able to retrieve these validators by public key even when they are not in the cache
-	i, exists = b.getValidatorIndex([fieldparams.BLSPubkeyLength]byte{6})
-	require.Equal(t, primitives.ValidatorIndex(5), i)
-	require.Equal(t, true, exists)
-
-	// State is finalized. We save [6] to the cache.
-	b.saveValidatorIndices()
-	require.Equal(t, 6, len(b.validatorIndexCache.indexMap))
-
-	// Save a few more times.
-	b.saveValidatorIndices()
-	b.saveValidatorIndices()
-	require.Equal(t, 6, len(b.validatorIndexCache.indexMap))
-
-	// Can still retrieve the validators from the cache
-	i, exists = b.getValidatorIndex([fieldparams.BLSPubkeyLength]byte{1})
-	require.Equal(t, primitives.ValidatorIndex(0), i)
-	require.Equal(t, true, exists)
-	i, exists = b.getValidatorIndex([fieldparams.BLSPubkeyLength]byte{2})
-	require.Equal(t, primitives.ValidatorIndex(1), i)
-	require.Equal(t, true, exists)
-	i, exists = b.getValidatorIndex([fieldparams.BLSPubkeyLength]byte{3})
-	require.Equal(t, primitives.ValidatorIndex(2), i)
-	require.Equal(t, true, exists)
-}

beacon-chain/state/stategen/replay_test.go

@@ -230,7 +230,6 @@ func TestReplayBlocks_ProcessEpoch_Electra(t *testing.T) {
 			EffectiveBalance:      params.BeaconConfig().MinActivationBalance,
 		},
 	}))
-	beaconState.SaveValidatorIndices()
 
 	require.NoError(t, beaconState.SetPendingDeposits([]*ethpb.PendingDeposit{
 		stateTesting.GeneratePendingDeposit(t, sk, uint64(amountAvailForProcessing)/10, bytesutil.ToBytes32(withdrawalCredentials), genesisBlock.Block.Slot),