From 557c5be4335d7e02dc5a899830cba8323d8ef467 Mon Sep 17 00:00:00 2001 From: Jun Song <87601811+syjn99@users.noreply.github.com> Date: Fri, 7 Feb 2025 13:31:01 +0900 Subject: [PATCH] Prune pending deposits from the deposit cache post-Electra (#14829) * Add metrics for pruned proofs & pending deposits * Add PruneAllProofs & PruneAllPendingDeposits * Add simple unit tests * Add DepositPruner interface * Add pruning logic at post finalization task * Move pruner logic into new file(deposit_pruner.go) Rationale: As deposit_fetcher.go contains all pruning logics, it would be better to separate its interest into fetcher/inserter/pruner. * Gofmt * Add reference link for deprecating eth1 polling * Add changelog * Apply reviews from nisdas and james * add pre and post deposit request tests * nishant's comment --------- Co-authored-by: james-prysm <90280386+james-prysm@users.noreply.github.com> Co-authored-by: james-prysm --- .../blockchain/process_block_helpers.go | 20 +- beacon-chain/blockchain/process_block_test.go | 8 +- beacon-chain/blockchain/receive_block.go | 3 +- beacon-chain/blockchain/receive_block_test.go | 100 ++++-- .../cache/depositsnapshot/BUILD.bazel | 2 + .../depositsnapshot/deposit_cache_test.go | 183 ---------- .../cache/depositsnapshot/deposit_fetcher.go | 46 --- .../depositsnapshot/deposit_fetcher_test.go | 64 ---- .../cache/depositsnapshot/deposit_pruner.go | 88 +++++ .../depositsnapshot/deposit_pruner_test.go | 323 ++++++++++++++++++ beacon-chain/cache/interfaces.go | 11 +- changelog/syjn99_prune-deposit-cache.md | 3 + 12 files changed, 520 insertions(+), 331 deletions(-) create mode 100644 beacon-chain/cache/depositsnapshot/deposit_pruner.go create mode 100644 beacon-chain/cache/depositsnapshot/deposit_pruner_test.go create mode 100644 changelog/syjn99_prune-deposit-cache.md diff --git a/beacon-chain/blockchain/process_block_helpers.go b/beacon-chain/blockchain/process_block_helpers.go index e4a37ff4b2..d751f859ca 100644 --- a/beacon-chain/blockchain/process_block_helpers.go +++ b/beacon-chain/blockchain/process_block_helpers.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/prysmaticlabs/prysm/v5/beacon-chain/core/helpers" lightclient "github.com/prysmaticlabs/prysm/v5/beacon-chain/core/light-client" "github.com/ethereum/go-ethereum/common" @@ -552,7 +553,8 @@ func (s *Service) fillInForkChoiceMissingBlocks(ctx context.Context, signed inte // inserts finalized deposits into our finalized deposit trie, needs to be // called in the background -func (s *Service) insertFinalizedDeposits(ctx context.Context, fRoot [32]byte) { +// Post-Electra: prunes all proofs and pending deposits in the cache +func (s *Service) insertFinalizedDepositsAndPrune(ctx context.Context, fRoot [32]byte) { ctx, span := trace.StartSpan(ctx, "blockChain.insertFinalizedDeposits") defer span.End() startTime := time.Now() @@ -563,6 +565,16 @@ func (s *Service) insertFinalizedDeposits(ctx context.Context, fRoot [32]byte) { log.WithError(err).Error("could not fetch finalized state") return } + + // Check if we should prune all pending deposits. + // In post-Electra(after the legacy deposit mechanism is deprecated), + // we can prune all pending deposits in the deposit cache. + // See: https://eips.ethereum.org/EIPS/eip-6110#eth1data-poll-deprecation + if helpers.DepositRequestsStarted(finalizedState) { + s.pruneAllPendingDepositsAndProofs(ctx) + return + } + // We update the cache up to the last deposit index in the finalized block's state. // We can be confident that these deposits will be included in some block // because the Eth1 follow distance makes such long-range reorgs extremely unlikely. @@ -591,6 +603,12 @@ func (s *Service) insertFinalizedDeposits(ctx context.Context, fRoot [32]byte) { log.WithField("duration", time.Since(startTime).String()).Debugf("Finalized deposit insertion completed at index %d", finalizedEth1DepIdx) } +// pruneAllPendingDepositsAndProofs prunes all proofs and pending deposits in the cache. +func (s *Service) pruneAllPendingDepositsAndProofs(ctx context.Context) { + s.cfg.DepositCache.PruneAllPendingDeposits(ctx) + s.cfg.DepositCache.PruneAllProofs(ctx) +} + // This ensures that the input root defaults to using genesis root instead of zero hashes. This is needed for handling // fork choice justification routine. func (s *Service) ensureRootNotZeros(root [32]byte) [32]byte { diff --git a/beacon-chain/blockchain/process_block_test.go b/beacon-chain/blockchain/process_block_test.go index 36e8383050..a41e5a178d 100644 --- a/beacon-chain/blockchain/process_block_test.go +++ b/beacon-chain/blockchain/process_block_test.go @@ -723,7 +723,7 @@ func TestInsertFinalizedDeposits(t *testing.T) { Signature: zeroSig[:], }, Proof: [][]byte{root}}, 100+i, int64(i), bytesutil.ToBytes32(root))) } - service.insertFinalizedDeposits(ctx, [32]byte{'m', 'o', 'c', 'k'}) + service.insertFinalizedDepositsAndPrune(ctx, [32]byte{'m', 'o', 'c', 'k'}) fDeposits, err := depositCache.FinalizedDeposits(ctx) require.NoError(t, err) assert.Equal(t, 7, int(fDeposits.MerkleTrieIndex()), "Finalized deposits not inserted correctly") @@ -759,7 +759,7 @@ func TestInsertFinalizedDeposits_PrunePendingDeposits(t *testing.T) { Signature: zeroSig[:], }, Proof: [][]byte{root}}, 100+i, int64(i), bytesutil.ToBytes32(root)) } - service.insertFinalizedDeposits(ctx, [32]byte{'m', 'o', 'c', 'k'}) + service.insertFinalizedDepositsAndPrune(ctx, [32]byte{'m', 'o', 'c', 'k'}) fDeposits, err := depositCache.FinalizedDeposits(ctx) require.NoError(t, err) assert.Equal(t, 7, int(fDeposits.MerkleTrieIndex()), "Finalized deposits not inserted correctly") @@ -799,7 +799,7 @@ func TestInsertFinalizedDeposits_MultipleFinalizedRoutines(t *testing.T) { } // Insert 3 deposits before hand. require.NoError(t, depositCache.InsertFinalizedDeposits(ctx, 2, [32]byte{}, 0)) - service.insertFinalizedDeposits(ctx, [32]byte{'m', 'o', 'c', 'k'}) + service.insertFinalizedDepositsAndPrune(ctx, [32]byte{'m', 'o', 'c', 'k'}) fDeposits, err := depositCache.FinalizedDeposits(ctx) require.NoError(t, err) assert.Equal(t, 5, int(fDeposits.MerkleTrieIndex()), "Finalized deposits not inserted correctly") @@ -810,7 +810,7 @@ func TestInsertFinalizedDeposits_MultipleFinalizedRoutines(t *testing.T) { } // Insert New Finalized State with higher deposit count. - service.insertFinalizedDeposits(ctx, [32]byte{'m', 'o', 'c', 'k', '2'}) + service.insertFinalizedDepositsAndPrune(ctx, [32]byte{'m', 'o', 'c', 'k', '2'}) fDeposits, err = depositCache.FinalizedDeposits(ctx) require.NoError(t, err) assert.Equal(t, 12, int(fDeposits.MerkleTrieIndex()), "Finalized deposits not inserted correctly") diff --git a/beacon-chain/blockchain/receive_block.go b/beacon-chain/blockchain/receive_block.go index 30f89b538b..64c5444569 100644 --- a/beacon-chain/blockchain/receive_block.go +++ b/beacon-chain/blockchain/receive_block.go @@ -279,9 +279,10 @@ func (s *Service) executePostFinalizationTasks(ctx context.Context, finalizedSta go func() { s.sendNewFinalizedEvent(ctx, finalizedState) }() + depCtx, cancel := context.WithTimeout(context.Background(), depositDeadline) go func() { - s.insertFinalizedDeposits(depCtx, finalized.Root) + s.insertFinalizedDepositsAndPrune(depCtx, finalized.Root) cancel() }() } diff --git a/beacon-chain/blockchain/receive_block_test.go b/beacon-chain/blockchain/receive_block_test.go index d8d36051a0..1dae450b80 100644 --- a/beacon-chain/blockchain/receive_block_test.go +++ b/beacon-chain/blockchain/receive_block_test.go @@ -455,41 +455,81 @@ func Test_executePostFinalizationTasks(t *testing.T) { Root: headRoot[:], })) require.NoError(t, headState.SetGenesisValidatorsRoot(params.BeaconConfig().ZeroHash[:])) + t.Run("pre deposit request", func(t *testing.T) { + require.NoError(t, headState.SetEth1DepositIndex(1)) + s, tr := minimalTestService(t, WithFinalizedStateAtStartUp(headState)) + ctx, beaconDB, stateGen := tr.ctx, tr.db, tr.sg - s, tr := minimalTestService(t, WithFinalizedStateAtStartUp(headState)) - ctx, beaconDB, stateGen := tr.ctx, tr.db, tr.sg + require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, genesisRoot)) + util.SaveBlock(t, ctx, beaconDB, genesis) + require.NoError(t, beaconDB.SaveState(ctx, headState, headRoot)) + require.NoError(t, beaconDB.SaveState(ctx, headState, genesisRoot)) + util.SaveBlock(t, ctx, beaconDB, headBlock) + require.NoError(t, beaconDB.SaveFinalizedCheckpoint(ctx, ðpb.Checkpoint{Epoch: slots.ToEpoch(finalizedSlot), Root: headRoot[:]})) - require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, genesisRoot)) - util.SaveBlock(t, ctx, beaconDB, genesis) - require.NoError(t, beaconDB.SaveState(ctx, headState, headRoot)) - require.NoError(t, beaconDB.SaveState(ctx, headState, genesisRoot)) - util.SaveBlock(t, ctx, beaconDB, headBlock) - require.NoError(t, beaconDB.SaveFinalizedCheckpoint(ctx, ðpb.Checkpoint{Epoch: slots.ToEpoch(finalizedSlot), Root: headRoot[:]})) + require.NoError(t, err) + require.NoError(t, stateGen.SaveState(ctx, headRoot, headState)) + require.NoError(t, beaconDB.SaveLastValidatedCheckpoint(ctx, ðpb.Checkpoint{Epoch: slots.ToEpoch(finalizedSlot), Root: headRoot[:]})) - require.NoError(t, err) - require.NoError(t, stateGen.SaveState(ctx, headRoot, headState)) - require.NoError(t, beaconDB.SaveLastValidatedCheckpoint(ctx, ðpb.Checkpoint{Epoch: slots.ToEpoch(finalizedSlot), Root: headRoot[:]})) + notifier := &blockchainTesting.MockStateNotifier{RecordEvents: true} + s.cfg.StateNotifier = notifier + s.executePostFinalizationTasks(s.ctx, headState) - notifier := &blockchainTesting.MockStateNotifier{RecordEvents: true} - s.cfg.StateNotifier = notifier - s.executePostFinalizationTasks(s.ctx, headState) + time.Sleep(1 * time.Second) // sleep for a second because event is in a separate go routine + require.Equal(t, 1, len(notifier.ReceivedEvents())) + e := notifier.ReceivedEvents()[0] + assert.Equal(t, statefeed.FinalizedCheckpoint, int(e.Type)) + fc, ok := e.Data.(*ethpbv1.EventFinalizedCheckpoint) + require.Equal(t, true, ok, "event has wrong data type") + assert.Equal(t, primitives.Epoch(123), fc.Epoch) + assert.DeepEqual(t, headRoot[:], fc.Block) + assert.DeepEqual(t, finalizedStRoot[:], fc.State) + assert.Equal(t, false, fc.ExecutionOptimistic) - time.Sleep(1 * time.Second) // sleep for a second because event is in a separate go routine - require.Equal(t, 1, len(notifier.ReceivedEvents())) - e := notifier.ReceivedEvents()[0] - assert.Equal(t, statefeed.FinalizedCheckpoint, int(e.Type)) - fc, ok := e.Data.(*ethpbv1.EventFinalizedCheckpoint) - require.Equal(t, true, ok, "event has wrong data type") - assert.Equal(t, primitives.Epoch(123), fc.Epoch) - assert.DeepEqual(t, headRoot[:], fc.Block) - assert.DeepEqual(t, finalizedStRoot[:], fc.State) - assert.Equal(t, false, fc.ExecutionOptimistic) + // check the cache + index, ok := headState.ValidatorIndexByPubkey(bytesutil.ToBytes48(key)) + require.Equal(t, true, ok) + require.Equal(t, primitives.ValidatorIndex(0), index) // first index - // check the cache - index, ok := headState.ValidatorIndexByPubkey(bytesutil.ToBytes48(key)) - require.Equal(t, true, ok) - require.Equal(t, primitives.ValidatorIndex(0), index) // first index + // check deposit + require.LogsContain(t, logHook, "Finalized deposit insertion completed at index") + }) + t.Run("deposit requests started", func(t *testing.T) { + require.NoError(t, headState.SetEth1DepositIndex(1)) + require.NoError(t, headState.SetDepositRequestsStartIndex(1)) + s, tr := minimalTestService(t, WithFinalizedStateAtStartUp(headState)) + ctx, beaconDB, stateGen := tr.ctx, tr.db, tr.sg + + require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, genesisRoot)) + util.SaveBlock(t, ctx, beaconDB, genesis) + require.NoError(t, beaconDB.SaveState(ctx, headState, headRoot)) + require.NoError(t, beaconDB.SaveState(ctx, headState, genesisRoot)) + util.SaveBlock(t, ctx, beaconDB, headBlock) + require.NoError(t, beaconDB.SaveFinalizedCheckpoint(ctx, ðpb.Checkpoint{Epoch: slots.ToEpoch(finalizedSlot), Root: headRoot[:]})) + + require.NoError(t, err) + require.NoError(t, stateGen.SaveState(ctx, headRoot, headState)) + require.NoError(t, beaconDB.SaveLastValidatedCheckpoint(ctx, ðpb.Checkpoint{Epoch: slots.ToEpoch(finalizedSlot), Root: headRoot[:]})) + + notifier := &blockchainTesting.MockStateNotifier{RecordEvents: true} + s.cfg.StateNotifier = notifier + s.executePostFinalizationTasks(s.ctx, headState) + + time.Sleep(1 * time.Second) // sleep for a second because event is in a separate go routine + require.Equal(t, 1, len(notifier.ReceivedEvents())) + e := notifier.ReceivedEvents()[0] + assert.Equal(t, statefeed.FinalizedCheckpoint, int(e.Type)) + fc, ok := e.Data.(*ethpbv1.EventFinalizedCheckpoint) + require.Equal(t, true, ok, "event has wrong data type") + assert.Equal(t, primitives.Epoch(123), fc.Epoch) + assert.DeepEqual(t, headRoot[:], fc.Block) + assert.DeepEqual(t, finalizedStRoot[:], fc.State) + assert.Equal(t, false, fc.ExecutionOptimistic) + + // check the cache + index, ok := headState.ValidatorIndexByPubkey(bytesutil.ToBytes48(key)) + require.Equal(t, true, ok) + require.Equal(t, primitives.ValidatorIndex(0), index) // first index + }) - // check deposit - require.LogsContain(t, logHook, "Finalized deposit insertion completed at index") } diff --git a/beacon-chain/cache/depositsnapshot/BUILD.bazel b/beacon-chain/cache/depositsnapshot/BUILD.bazel index de6e52e5ee..12a1835540 100644 --- a/beacon-chain/cache/depositsnapshot/BUILD.bazel +++ b/beacon-chain/cache/depositsnapshot/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "deposit_fetcher.go", "deposit_inserter.go", + "deposit_pruner.go", "deposit_tree.go", "deposit_tree_snapshot.go", "merkle_tree.go", @@ -35,6 +36,7 @@ go_test( srcs = [ "deposit_cache_test.go", "deposit_fetcher_test.go", + "deposit_pruner_test.go", "deposit_tree_snapshot_test.go", "merkle_tree_test.go", "spec_test.go", diff --git a/beacon-chain/cache/depositsnapshot/deposit_cache_test.go b/beacon-chain/cache/depositsnapshot/deposit_cache_test.go index 2c17e8428b..34e2cae987 100644 --- a/beacon-chain/cache/depositsnapshot/deposit_cache_test.go +++ b/beacon-chain/cache/depositsnapshot/deposit_cache_test.go @@ -903,189 +903,6 @@ func TestMin(t *testing.T) { } -func TestPruneProofs_Ok(t *testing.T) { - dc, err := New() - require.NoError(t, err) - - deposits := []struct { - blkNum uint64 - deposit *ethpb.Deposit - index int64 - }{ - { - blkNum: 0, - deposit: ðpb.Deposit{Proof: makeDepositProof(), - Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}}, - index: 0, - }, - { - blkNum: 0, - deposit: ðpb.Deposit{Proof: makeDepositProof(), - Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}}, - index: 1, - }, - { - blkNum: 0, - deposit: ðpb.Deposit{Proof: makeDepositProof(), - Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}}, - index: 2, - }, - { - blkNum: 0, - deposit: ðpb.Deposit{Proof: makeDepositProof(), - Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}}, - index: 3, - }, - } - - for _, ins := range deposits { - assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{})) - } - - require.NoError(t, dc.PruneProofs(context.Background(), 1)) - - assert.DeepEqual(t, [][]byte(nil), dc.deposits[0].Deposit.Proof) - assert.DeepEqual(t, [][]byte(nil), dc.deposits[1].Deposit.Proof) - assert.NotNil(t, dc.deposits[2].Deposit.Proof) - assert.NotNil(t, dc.deposits[3].Deposit.Proof) -} - -func TestPruneProofs_SomeAlreadyPruned(t *testing.T) { - dc, err := New() - require.NoError(t, err) - - deposits := []struct { - blkNum uint64 - deposit *ethpb.Deposit - index int64 - }{ - { - blkNum: 0, - deposit: ðpb.Deposit{Proof: nil, Data: ðpb.Deposit_Data{ - PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}}, - index: 0, - }, - { - blkNum: 0, - deposit: ðpb.Deposit{Proof: nil, Data: ðpb.Deposit_Data{ - PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}}, index: 1, - }, - { - blkNum: 0, - deposit: ðpb.Deposit{Proof: makeDepositProof(), Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}}, - index: 2, - }, - { - blkNum: 0, - deposit: ðpb.Deposit{Proof: makeDepositProof(), - Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}}, - index: 3, - }, - } - - for _, ins := range deposits { - assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{})) - } - - require.NoError(t, dc.PruneProofs(context.Background(), 2)) - - assert.DeepEqual(t, [][]byte(nil), dc.deposits[2].Deposit.Proof) -} - -func TestPruneProofs_PruneAllWhenDepositIndexTooBig(t *testing.T) { - dc, err := New() - require.NoError(t, err) - - deposits := []struct { - blkNum uint64 - deposit *ethpb.Deposit - index int64 - }{ - { - blkNum: 0, - deposit: ðpb.Deposit{Proof: makeDepositProof(), - Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}}, - index: 0, - }, - { - blkNum: 0, - deposit: ðpb.Deposit{Proof: makeDepositProof(), - Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}}, - index: 1, - }, - { - blkNum: 0, - deposit: ðpb.Deposit{Proof: makeDepositProof(), - Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}}, - index: 2, - }, - { - blkNum: 0, - deposit: ðpb.Deposit{Proof: makeDepositProof(), - Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}}, - index: 3, - }, - } - - for _, ins := range deposits { - assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{})) - } - - require.NoError(t, dc.PruneProofs(context.Background(), 99)) - - assert.DeepEqual(t, [][]byte(nil), dc.deposits[0].Deposit.Proof) - assert.DeepEqual(t, [][]byte(nil), dc.deposits[1].Deposit.Proof) - assert.DeepEqual(t, [][]byte(nil), dc.deposits[2].Deposit.Proof) - assert.DeepEqual(t, [][]byte(nil), dc.deposits[3].Deposit.Proof) -} - -func TestPruneProofs_CorrectlyHandleLastIndex(t *testing.T) { - dc, err := New() - require.NoError(t, err) - - deposits := []struct { - blkNum uint64 - deposit *ethpb.Deposit - index int64 - }{ - { - blkNum: 0, - deposit: ðpb.Deposit{Proof: makeDepositProof(), - Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}}, - index: 0, - }, - { - blkNum: 0, - deposit: ðpb.Deposit{Proof: makeDepositProof(), - Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}}, - index: 1, - }, - { - blkNum: 0, - deposit: ðpb.Deposit{Proof: makeDepositProof(), - Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}}, - index: 2, - }, - { - blkNum: 0, - deposit: ðpb.Deposit{Proof: makeDepositProof(), - Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}}, - index: 3, - }, - } - - for _, ins := range deposits { - assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{})) - } - - require.NoError(t, dc.PruneProofs(context.Background(), 4)) - - assert.DeepEqual(t, [][]byte(nil), dc.deposits[0].Deposit.Proof) - assert.DeepEqual(t, [][]byte(nil), dc.deposits[1].Deposit.Proof) - assert.DeepEqual(t, [][]byte(nil), dc.deposits[2].Deposit.Proof) - assert.DeepEqual(t, [][]byte(nil), dc.deposits[3].Deposit.Proof) -} - func TestDepositMap_WorksCorrectly(t *testing.T) { dc, err := New() require.NoError(t, err) diff --git a/beacon-chain/cache/depositsnapshot/deposit_fetcher.go b/beacon-chain/cache/depositsnapshot/deposit_fetcher.go index e7b9f089ad..088e480bcf 100644 --- a/beacon-chain/cache/depositsnapshot/deposit_fetcher.go +++ b/beacon-chain/cache/depositsnapshot/deposit_fetcher.go @@ -178,52 +178,6 @@ func (c *Cache) NonFinalizedDeposits(ctx context.Context, lastFinalizedIndex int return deposits } -// PruneProofs removes proofs from all deposits whose index is equal or less than untilDepositIndex. -func (c *Cache) PruneProofs(ctx context.Context, untilDepositIndex int64) error { - _, span := trace.StartSpan(ctx, "Cache.PruneProofs") - defer span.End() - c.depositsLock.Lock() - defer c.depositsLock.Unlock() - - if untilDepositIndex >= int64(len(c.deposits)) { - untilDepositIndex = int64(len(c.deposits) - 1) - } - - for i := untilDepositIndex; i >= 0; i-- { - // Finding a nil proof means that all proofs up to this deposit have been already pruned. - if c.deposits[i].Deposit.Proof == nil { - break - } - c.deposits[i].Deposit.Proof = nil - } - - return nil -} - -// PrunePendingDeposits removes any deposit which is older than the given deposit merkle tree index. -func (c *Cache) PrunePendingDeposits(ctx context.Context, merkleTreeIndex int64) { - _, span := trace.StartSpan(ctx, "Cache.PrunePendingDeposits") - defer span.End() - - if merkleTreeIndex == 0 { - log.Debug("Ignoring 0 deposit removal") - return - } - - c.depositsLock.Lock() - defer c.depositsLock.Unlock() - - cleanDeposits := make([]*ethpb.DepositContainer, 0, len(c.pendingDeposits)) - for _, dp := range c.pendingDeposits { - if dp.Index >= merkleTreeIndex { - cleanDeposits = append(cleanDeposits, dp) - } - } - - c.pendingDeposits = cleanDeposits - pendingDepositsCount.Set(float64(len(c.pendingDeposits))) -} - // InsertPendingDeposit into the database. If deposit or block number are nil // then this method does nothing. func (c *Cache) InsertPendingDeposit(ctx context.Context, d *ethpb.Deposit, blockNum uint64, index int64, depositRoot [32]byte) { diff --git a/beacon-chain/cache/depositsnapshot/deposit_fetcher_test.go b/beacon-chain/cache/depositsnapshot/deposit_fetcher_test.go index 04c9493554..0c5fbf6ade 100644 --- a/beacon-chain/cache/depositsnapshot/deposit_fetcher_test.go +++ b/beacon-chain/cache/depositsnapshot/deposit_fetcher_test.go @@ -44,67 +44,3 @@ func TestPendingDeposits_OK(t *testing.T) { all := dc.PendingDeposits(context.Background(), nil) assert.Equal(t, len(dc.pendingDeposits), len(all), "PendingDeposits(ctx, nil) did not return all deposits") } - -func TestPrunePendingDeposits_ZeroMerkleIndex(t *testing.T) { - dc := Cache{} - - dc.pendingDeposits = []*ethpb.DepositContainer{ - {Eth1BlockHeight: 2, Index: 2}, - {Eth1BlockHeight: 4, Index: 4}, - {Eth1BlockHeight: 6, Index: 6}, - {Eth1BlockHeight: 8, Index: 8}, - {Eth1BlockHeight: 10, Index: 10}, - {Eth1BlockHeight: 12, Index: 12}, - } - - dc.PrunePendingDeposits(context.Background(), 0) - expected := []*ethpb.DepositContainer{ - {Eth1BlockHeight: 2, Index: 2}, - {Eth1BlockHeight: 4, Index: 4}, - {Eth1BlockHeight: 6, Index: 6}, - {Eth1BlockHeight: 8, Index: 8}, - {Eth1BlockHeight: 10, Index: 10}, - {Eth1BlockHeight: 12, Index: 12}, - } - assert.DeepEqual(t, expected, dc.pendingDeposits) -} - -func TestPrunePendingDeposits_OK(t *testing.T) { - dc := Cache{} - - dc.pendingDeposits = []*ethpb.DepositContainer{ - {Eth1BlockHeight: 2, Index: 2}, - {Eth1BlockHeight: 4, Index: 4}, - {Eth1BlockHeight: 6, Index: 6}, - {Eth1BlockHeight: 8, Index: 8}, - {Eth1BlockHeight: 10, Index: 10}, - {Eth1BlockHeight: 12, Index: 12}, - } - - dc.PrunePendingDeposits(context.Background(), 6) - expected := []*ethpb.DepositContainer{ - {Eth1BlockHeight: 6, Index: 6}, - {Eth1BlockHeight: 8, Index: 8}, - {Eth1BlockHeight: 10, Index: 10}, - {Eth1BlockHeight: 12, Index: 12}, - } - - assert.DeepEqual(t, expected, dc.pendingDeposits) - - dc.pendingDeposits = []*ethpb.DepositContainer{ - {Eth1BlockHeight: 2, Index: 2}, - {Eth1BlockHeight: 4, Index: 4}, - {Eth1BlockHeight: 6, Index: 6}, - {Eth1BlockHeight: 8, Index: 8}, - {Eth1BlockHeight: 10, Index: 10}, - {Eth1BlockHeight: 12, Index: 12}, - } - - dc.PrunePendingDeposits(context.Background(), 10) - expected = []*ethpb.DepositContainer{ - {Eth1BlockHeight: 10, Index: 10}, - {Eth1BlockHeight: 12, Index: 12}, - } - - assert.DeepEqual(t, expected, dc.pendingDeposits) -} diff --git a/beacon-chain/cache/depositsnapshot/deposit_pruner.go b/beacon-chain/cache/depositsnapshot/deposit_pruner.go new file mode 100644 index 0000000000..904a3a1b1f --- /dev/null +++ b/beacon-chain/cache/depositsnapshot/deposit_pruner.go @@ -0,0 +1,88 @@ +package depositsnapshot + +import ( + "context" + + "github.com/prysmaticlabs/prysm/v5/monitoring/tracing/trace" + ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1" +) + +// PruneProofs removes proofs from all deposits whose index is equal or less than untilDepositIndex. +func (c *Cache) PruneProofs(ctx context.Context, untilDepositIndex int64) error { + _, span := trace.StartSpan(ctx, "Cache.PruneProofs") + defer span.End() + c.depositsLock.Lock() + defer c.depositsLock.Unlock() + + if untilDepositIndex >= int64(len(c.deposits)) { + untilDepositIndex = int64(len(c.deposits) - 1) + } + + for i := untilDepositIndex; i >= 0; i-- { + // Finding a nil proof means that all proofs up to this deposit have been already pruned. + if c.deposits[i].Deposit.Proof == nil { + break + } + c.deposits[i].Deposit.Proof = nil + } + + return nil +} + +// PruneAllProofs removes proofs from all deposits. +// As EIP-6110 applies and the legacy deposit mechanism is deprecated, +// proofs in deposit snapshot are no longer needed. +// See: https://eips.ethereum.org/EIPS/eip-6110#eth1data-poll-deprecation +func (c *Cache) PruneAllProofs(ctx context.Context) { + _, span := trace.StartSpan(ctx, "Cache.PruneAllProofs") + defer span.End() + + c.depositsLock.Lock() + defer c.depositsLock.Unlock() + + for i := len(c.deposits) - 1; i >= 0; i-- { + if c.deposits[i].Deposit.Proof == nil { + break + } + c.deposits[i].Deposit.Proof = nil + } +} + +// PrunePendingDeposits removes any deposit which is older than the given deposit merkle tree index. +func (c *Cache) PrunePendingDeposits(ctx context.Context, merkleTreeIndex int64) { + _, span := trace.StartSpan(ctx, "Cache.PrunePendingDeposits") + defer span.End() + + if merkleTreeIndex == 0 { + log.Debug("Ignoring 0 deposit removal") + return + } + + c.depositsLock.Lock() + defer c.depositsLock.Unlock() + + cleanDeposits := make([]*ethpb.DepositContainer, 0, len(c.pendingDeposits)) + for _, dp := range c.pendingDeposits { + if dp.Index >= merkleTreeIndex { + cleanDeposits = append(cleanDeposits, dp) + } + } + + c.pendingDeposits = cleanDeposits + pendingDepositsCount.Set(float64(len(c.pendingDeposits))) +} + +// PruneAllPendingDeposits removes all pending deposits from the cache. +// As EIP-6110 applies and the legacy deposit mechanism is deprecated, +// pending deposits in deposit snapshot are no longer needed. +// See: https://eips.ethereum.org/EIPS/eip-6110#eth1data-poll-deprecation +func (c *Cache) PruneAllPendingDeposits(ctx context.Context) { + _, span := trace.StartSpan(ctx, "Cache.PruneAllPendingDeposits") + defer span.End() + + c.depositsLock.Lock() + defer c.depositsLock.Unlock() + + c.pendingDeposits = make([]*ethpb.DepositContainer, 0) + pendingDepositsCount.Set(float64(0)) +} diff --git a/beacon-chain/cache/depositsnapshot/deposit_pruner_test.go b/beacon-chain/cache/depositsnapshot/deposit_pruner_test.go new file mode 100644 index 0000000000..64821f1a14 --- /dev/null +++ b/beacon-chain/cache/depositsnapshot/deposit_pruner_test.go @@ -0,0 +1,323 @@ +package depositsnapshot + +import ( + "context" + "testing" + + "github.com/prysmaticlabs/prysm/v5/encoding/bytesutil" + ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1" + "github.com/prysmaticlabs/prysm/v5/testing/assert" + "github.com/prysmaticlabs/prysm/v5/testing/require" +) + +func TestPrunePendingDeposits_ZeroMerkleIndex(t *testing.T) { + dc := Cache{} + + dc.pendingDeposits = []*ethpb.DepositContainer{ + {Eth1BlockHeight: 2, Index: 2}, + {Eth1BlockHeight: 4, Index: 4}, + {Eth1BlockHeight: 6, Index: 6}, + {Eth1BlockHeight: 8, Index: 8}, + {Eth1BlockHeight: 10, Index: 10}, + {Eth1BlockHeight: 12, Index: 12}, + } + + dc.PrunePendingDeposits(context.Background(), 0) + expected := []*ethpb.DepositContainer{ + {Eth1BlockHeight: 2, Index: 2}, + {Eth1BlockHeight: 4, Index: 4}, + {Eth1BlockHeight: 6, Index: 6}, + {Eth1BlockHeight: 8, Index: 8}, + {Eth1BlockHeight: 10, Index: 10}, + {Eth1BlockHeight: 12, Index: 12}, + } + assert.DeepEqual(t, expected, dc.pendingDeposits) +} + +func TestPrunePendingDeposits_OK(t *testing.T) { + dc := Cache{} + + dc.pendingDeposits = []*ethpb.DepositContainer{ + {Eth1BlockHeight: 2, Index: 2}, + {Eth1BlockHeight: 4, Index: 4}, + {Eth1BlockHeight: 6, Index: 6}, + {Eth1BlockHeight: 8, Index: 8}, + {Eth1BlockHeight: 10, Index: 10}, + {Eth1BlockHeight: 12, Index: 12}, + } + + dc.PrunePendingDeposits(context.Background(), 6) + expected := []*ethpb.DepositContainer{ + {Eth1BlockHeight: 6, Index: 6}, + {Eth1BlockHeight: 8, Index: 8}, + {Eth1BlockHeight: 10, Index: 10}, + {Eth1BlockHeight: 12, Index: 12}, + } + + assert.DeepEqual(t, expected, dc.pendingDeposits) + + dc.pendingDeposits = []*ethpb.DepositContainer{ + {Eth1BlockHeight: 2, Index: 2}, + {Eth1BlockHeight: 4, Index: 4}, + {Eth1BlockHeight: 6, Index: 6}, + {Eth1BlockHeight: 8, Index: 8}, + {Eth1BlockHeight: 10, Index: 10}, + {Eth1BlockHeight: 12, Index: 12}, + } + + dc.PrunePendingDeposits(context.Background(), 10) + expected = []*ethpb.DepositContainer{ + {Eth1BlockHeight: 10, Index: 10}, + {Eth1BlockHeight: 12, Index: 12}, + } + + assert.DeepEqual(t, expected, dc.pendingDeposits) +} + +func TestPruneAllPendingDeposits(t *testing.T) { + dc := Cache{} + + dc.pendingDeposits = []*ethpb.DepositContainer{ + {Eth1BlockHeight: 2, Index: 2}, + {Eth1BlockHeight: 4, Index: 4}, + {Eth1BlockHeight: 6, Index: 6}, + {Eth1BlockHeight: 8, Index: 8}, + {Eth1BlockHeight: 10, Index: 10}, + {Eth1BlockHeight: 12, Index: 12}, + } + + dc.PruneAllPendingDeposits(context.Background()) + expected := []*ethpb.DepositContainer{} + + assert.DeepEqual(t, expected, dc.pendingDeposits) +} + +func TestPruneProofs_Ok(t *testing.T) { + dc, err := New() + require.NoError(t, err) + + deposits := []struct { + blkNum uint64 + deposit *ethpb.Deposit + index int64 + }{ + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), + Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}}, + index: 0, + }, + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), + Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}}, + index: 1, + }, + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), + Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}}, + index: 2, + }, + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), + Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}}, + index: 3, + }, + } + + for _, ins := range deposits { + assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{})) + } + + require.NoError(t, dc.PruneProofs(context.Background(), 1)) + + assert.DeepEqual(t, [][]byte(nil), dc.deposits[0].Deposit.Proof) + assert.DeepEqual(t, [][]byte(nil), dc.deposits[1].Deposit.Proof) + assert.NotNil(t, dc.deposits[2].Deposit.Proof) + assert.NotNil(t, dc.deposits[3].Deposit.Proof) +} + +func TestPruneProofs_SomeAlreadyPruned(t *testing.T) { + dc, err := New() + require.NoError(t, err) + + deposits := []struct { + blkNum uint64 + deposit *ethpb.Deposit + index int64 + }{ + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: nil, Data: ðpb.Deposit_Data{ + PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}}, + index: 0, + }, + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: nil, Data: ðpb.Deposit_Data{ + PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}}, index: 1, + }, + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}}, + index: 2, + }, + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), + Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}}, + index: 3, + }, + } + + for _, ins := range deposits { + assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{})) + } + + require.NoError(t, dc.PruneProofs(context.Background(), 2)) + + assert.DeepEqual(t, [][]byte(nil), dc.deposits[2].Deposit.Proof) +} + +func TestPruneProofs_PruneAllWhenDepositIndexTooBig(t *testing.T) { + dc, err := New() + require.NoError(t, err) + + deposits := []struct { + blkNum uint64 + deposit *ethpb.Deposit + index int64 + }{ + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), + Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}}, + index: 0, + }, + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), + Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}}, + index: 1, + }, + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), + Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}}, + index: 2, + }, + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), + Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}}, + index: 3, + }, + } + + for _, ins := range deposits { + assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{})) + } + + require.NoError(t, dc.PruneProofs(context.Background(), 99)) + + assert.DeepEqual(t, [][]byte(nil), dc.deposits[0].Deposit.Proof) + assert.DeepEqual(t, [][]byte(nil), dc.deposits[1].Deposit.Proof) + assert.DeepEqual(t, [][]byte(nil), dc.deposits[2].Deposit.Proof) + assert.DeepEqual(t, [][]byte(nil), dc.deposits[3].Deposit.Proof) +} + +func TestPruneProofs_CorrectlyHandleLastIndex(t *testing.T) { + dc, err := New() + require.NoError(t, err) + + deposits := []struct { + blkNum uint64 + deposit *ethpb.Deposit + index int64 + }{ + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), + Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}}, + index: 0, + }, + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), + Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}}, + index: 1, + }, + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), + Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}}, + index: 2, + }, + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), + Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}}, + index: 3, + }, + } + + for _, ins := range deposits { + assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{})) + } + + require.NoError(t, dc.PruneProofs(context.Background(), 4)) + + assert.DeepEqual(t, [][]byte(nil), dc.deposits[0].Deposit.Proof) + assert.DeepEqual(t, [][]byte(nil), dc.deposits[1].Deposit.Proof) + assert.DeepEqual(t, [][]byte(nil), dc.deposits[2].Deposit.Proof) + assert.DeepEqual(t, [][]byte(nil), dc.deposits[3].Deposit.Proof) +} + +func TestPruneAllProofs(t *testing.T) { + dc, err := New() + require.NoError(t, err) + + deposits := []struct { + blkNum uint64 + deposit *ethpb.Deposit + index int64 + }{ + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), + Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}}, + index: 0, + }, + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), + Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}}, + index: 1, + }, + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), + Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}}, + index: 2, + }, + { + blkNum: 0, + deposit: ðpb.Deposit{Proof: makeDepositProof(), + Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}}, + index: 3, + }, + } + + for _, ins := range deposits { + assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{})) + } + + dc.PruneAllProofs(context.Background()) + + assert.DeepEqual(t, [][]byte(nil), dc.deposits[0].Deposit.Proof) + assert.DeepEqual(t, [][]byte(nil), dc.deposits[1].Deposit.Proof) + assert.DeepEqual(t, [][]byte(nil), dc.deposits[2].Deposit.Proof) + assert.DeepEqual(t, [][]byte(nil), dc.deposits[3].Deposit.Proof) +} diff --git a/beacon-chain/cache/interfaces.go b/beacon-chain/cache/interfaces.go index 163dbd0ef7..338613e296 100644 --- a/beacon-chain/cache/interfaces.go +++ b/beacon-chain/cache/interfaces.go @@ -12,6 +12,7 @@ import ( type DepositCache interface { DepositFetcher DepositInserter + DepositPruner } // DepositFetcher defines a struct which can retrieve deposit information from a store. @@ -23,8 +24,6 @@ type DepositFetcher interface { InsertPendingDeposit(ctx context.Context, d *ethpb.Deposit, blockNum uint64, index int64, depositRoot [32]byte) PendingDeposits(ctx context.Context, untilBlk *big.Int) []*ethpb.Deposit PendingContainers(ctx context.Context, untilBlk *big.Int) []*ethpb.DepositContainer - PrunePendingDeposits(ctx context.Context, merkleTreeIndex int64) - PruneProofs(ctx context.Context, untilDepositIndex int64) error FinalizedFetcher } @@ -42,6 +41,14 @@ type FinalizedFetcher interface { NonFinalizedDeposits(ctx context.Context, lastFinalizedIndex int64, untilBlk *big.Int) []*ethpb.Deposit } +// DepositPruner is an interface for pruning deposits and proofs. +type DepositPruner interface { + PrunePendingDeposits(ctx context.Context, merkleTreeIndex int64) + PruneAllPendingDeposits(ctx context.Context) + PruneProofs(ctx context.Context, untilDepositIndex int64) error + PruneAllProofs(ctx context.Context) +} + // FinalizedDeposits defines a method to access a merkle tree containing deposits and their indexes. type FinalizedDeposits interface { Deposits() MerkleTree diff --git a/changelog/syjn99_prune-deposit-cache.md b/changelog/syjn99_prune-deposit-cache.md new file mode 100644 index 0000000000..64962ae683 --- /dev/null +++ b/changelog/syjn99_prune-deposit-cache.md @@ -0,0 +1,3 @@ +### Added + +- Prune all pending deposits and proofs in post-Electra. \ No newline at end of file