From 4fbee52d3204665d8d91c14dc1eafc7cc9b35c52 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Wed, 14 Feb 2024 16:16:05 -0500 Subject: [PATCH] cut unnecessary methods --- bin/reth/src/commands/stage/drop.rs | 23 +++- crates/node-core/src/init.rs | 9 +- .../bundle_state_with_receipts.rs | 49 +-------- .../storage/provider/src/bundle_state/mod.rs | 4 +- .../src/bundle_state/state_changes.rs | 82 -------------- .../src/bundle_state/state_reverts.rs | 104 +++--------------- crates/storage/provider/src/lib.rs | 2 +- .../provider/src/providers/database/mod.rs | 12 +- .../src/providers/database/provider.rs | 38 ++++++- 9 files changed, 81 insertions(+), 242 deletions(-) delete mode 100644 crates/storage/provider/src/bundle_state/state_changes.rs diff --git a/bin/reth/src/commands/stage/drop.rs b/bin/reth/src/commands/stage/drop.rs index 3187932c2a..649b6770b3 100644 --- a/bin/reth/src/commands/stage/drop.rs +++ b/bin/reth/src/commands/stage/drop.rs @@ -6,7 +6,6 @@ use crate::{ DatabaseArgs, StageEnum, }, dirs::{DataDirPath, MaybePlatformPath}, - utils::DbTool, }; use clap::Parser; use reth_db::{ @@ -14,6 +13,7 @@ use reth_db::{ }; use reth_node_core::init::{insert_genesis_header, insert_genesis_state}; use reth_primitives::{fs, stage::StageId, ChainSpec}; +use reth_provider::DatabaseProviderRW; use std::sync::Arc; use tracing::info; @@ -59,11 +59,13 @@ impl Command { let db = open_db(db_path.as_ref(), DatabaseArguments::default().log_level(self.db.log_level))?; - let tool = DbTool::new(&db, self.chain.clone())?; + let provider_rw = + DatabaseProviderRW::::with_tx(db.tx_mut()?, self.chain.clone(), &None)?; - tool.db.update(|tx| { + provider_rw.update(|provider| { match &self.stage { StageEnum::Bodies => { + let tx = provider.tx_mut(); tx.clear::()?; tx.clear::()?; tx.clear::()?; @@ -73,6 +75,7 @@ impl Command { insert_genesis_header::(tx, self.chain)?; } StageEnum::Senders => { + let tx = provider.tx_mut(); tx.clear::()?; tx.put::( StageId::SenderRecovery.to_string(), @@ -80,6 +83,7 @@ impl Command { )?; } StageEnum::Execution => { + let tx = provider.tx_mut(); tx.clear::()?; tx.clear::()?; tx.clear::()?; @@ -90,9 +94,11 @@ impl Command { StageId::Execution.to_string(), Default::default(), )?; - insert_genesis_state::(tx, self.chain.genesis())?; + + insert_genesis_state(provider, self.chain.genesis())?; } StageEnum::AccountHashing => { + let tx = provider.tx_mut(); tx.clear::()?; tx.put::( StageId::AccountHashing.to_string(), @@ -100,6 +106,7 @@ impl Command { )?; } StageEnum::StorageHashing => { + let tx = provider.tx_mut(); tx.clear::()?; tx.put::( StageId::StorageHashing.to_string(), @@ -107,6 +114,8 @@ impl Command { )?; } StageEnum::Hashing => { + let tx = provider.tx_mut(); + // Clear hashed accounts tx.clear::()?; tx.put::( @@ -122,6 +131,7 @@ impl Command { )?; } StageEnum::Merkle => { + let tx = provider.tx_mut(); tx.clear::()?; tx.clear::()?; tx.put::( @@ -138,6 +148,7 @@ impl Command { )?; } StageEnum::AccountHistory | StageEnum::StorageHistory => { + let tx = provider.tx_mut(); tx.clear::()?; tx.clear::()?; tx.put::( @@ -150,6 +161,8 @@ impl Command { )?; } StageEnum::TotalDifficulty => { + let tx = provider.tx_mut(); + tx.clear::()?; tx.put::( StageId::TotalDifficulty.to_string(), @@ -158,6 +171,7 @@ impl Command { insert_genesis_header::(tx, self.chain)?; } StageEnum::TxLookup => { + let tx = provider.tx_mut(); tx.clear::()?; tx.put::( StageId::TransactionLookup.to_string(), @@ -171,6 +185,7 @@ impl Command { } } + let tx = provider.tx_mut(); tx.put::(StageId::Finish.to_string(), Default::default())?; Ok::<_, eyre::Error>(()) diff --git a/crates/node-core/src/init.rs b/crates/node-core/src/init.rs index 4bf0f647b7..8abe252d3a 100644 --- a/crates/node-core/src/init.rs +++ b/crates/node-core/src/init.rs @@ -13,7 +13,7 @@ use reth_primitives::{ use reth_provider::{ bundle_state::{BundleStateInit, RevertsInit}, BundleStateWithReceipts, DatabaseProviderRW, HashingWriter, HistoryWriter, OriginalValuesKnown, - ProviderError, ProviderFactory, + ProviderError, ProviderFactory, StateWriter, }; use std::{ collections::{BTreeMap, HashMap}, @@ -75,14 +75,13 @@ pub fn init_genesis( let provider_rw = factory.provider_rw()?; insert_genesis_hashes(&provider_rw, genesis)?; insert_genesis_history(&provider_rw, genesis)?; + insert_genesis_state(&provider_rw, genesis)?; provider_rw.commit()?; // Insert header let tx = db.tx_mut()?; insert_genesis_header::(&tx, chain.clone())?; - insert_genesis_state::(&tx, genesis)?; - // insert sync stage for stage in StageId::ALL.iter() { tx.put::(stage.to_string(), Default::default())?; @@ -94,7 +93,7 @@ pub fn init_genesis( /// Inserts the genesis state into the database. pub fn insert_genesis_state( - tx: &::TXMut, + provider: &DatabaseProviderRW, genesis: &reth_primitives::Genesis, ) -> ProviderResult<()> { let mut state_init: BundleStateInit = HashMap::new(); @@ -153,7 +152,7 @@ pub fn insert_genesis_state( 0, ); - bundle.write_to_db(tx, OriginalValuesKnown::Yes)?; + provider.write_state(bundle, OriginalValuesKnown::Yes)?; Ok(()) } diff --git a/crates/storage/provider/src/bundle_state/bundle_state_with_receipts.rs b/crates/storage/provider/src/bundle_state/bundle_state_with_receipts.rs index b47a23c2ae..21d14a1528 100644 --- a/crates/storage/provider/src/bundle_state/bundle_state_with_receipts.rs +++ b/crates/storage/provider/src/bundle_state/bundle_state_with_receipts.rs @@ -1,10 +1,3 @@ -use crate::{StateChanges, StateReverts}; -use reth_db::{ - cursor::{DbCursorRO, DbCursorRW}, - tables, - transaction::{DbTx, DbTxMut}, -}; -use reth_interfaces::db::DatabaseError; use reth_primitives::{ logs_bloom, revm::compat::{into_reth_acc, into_revm_acc}, @@ -289,46 +282,6 @@ impl BundleStateWithReceipts { // swap bundles std::mem::swap(&mut self.bundle, &mut other) } - - /// Write the [BundleStateWithReceipts] to the database. - /// - /// `is_value_known` should be set to `Not` if the [BundleStateWithReceipts] has some of its - /// state detached, This would make some original values not known. - pub fn write_to_db( - self, - tx: &TX, - is_value_known: OriginalValuesKnown, - ) -> Result<(), DatabaseError> { - let (plain_state, reverts) = self.bundle.into_plain_state_and_reverts(is_value_known); - - StateReverts(reverts).write_to_db(tx, self.first_block)?; - - // write receipts - let mut bodies_cursor = tx.cursor_read::()?; - let mut receipts_cursor = tx.cursor_write::()?; - - for (idx, receipts) in self.receipts.into_iter().enumerate() { - if !receipts.is_empty() { - let block_number = self.first_block + idx as u64; - let (_, body_indices) = - bodies_cursor.seek_exact(block_number)?.unwrap_or_else(|| { - let last_available = bodies_cursor.last().ok().flatten().map(|(number, _)| number); - panic!("body indices for block {block_number} must exist. last available block number: {last_available:?}"); - }); - - let first_tx_index = body_indices.first_tx_num(); - for (tx_idx, receipt) in receipts.into_iter().enumerate() { - if let Some(receipt) = receipt { - receipts_cursor.append(first_tx_index + tx_idx as u64, receipt)?; - } - } - } - } - - StateChanges(plain_state).write_to_db(tx)?; - - Ok(()) - } } #[cfg(test)] @@ -344,7 +297,7 @@ mod tests { models::{AccountBeforeTx, BlockNumberAddress}, tables, test_utils::create_test_rw_db, - transaction::DbTx, + transaction::{DbTx, DbTxMut}, }; use reth_primitives::{ keccak256, revm::compat::into_reth_acc, Address, Receipt, Receipts, StorageEntry, B256, diff --git a/crates/storage/provider/src/bundle_state/mod.rs b/crates/storage/provider/src/bundle_state/mod.rs index 5df4a213aa..406ce3f398 100644 --- a/crates/storage/provider/src/bundle_state/mod.rs +++ b/crates/storage/provider/src/bundle_state/mod.rs @@ -2,12 +2,10 @@ //! This module contains all the logic related to bundle state. mod bundle_state_with_receipts; mod hashed_state_changes; -mod state_changes; mod state_reverts; pub use bundle_state_with_receipts::{ AccountRevertInit, BundleStateInit, BundleStateWithReceipts, OriginalValuesKnown, RevertsInit, }; pub use hashed_state_changes::HashedStateChanges; -pub use state_changes::StateChanges; -pub use state_reverts::{StateReverts, StorageRevertsIter}; +pub use state_reverts::StorageRevertsIter; diff --git a/crates/storage/provider/src/bundle_state/state_changes.rs b/crates/storage/provider/src/bundle_state/state_changes.rs deleted file mode 100644 index a62606dede..0000000000 --- a/crates/storage/provider/src/bundle_state/state_changes.rs +++ /dev/null @@ -1,82 +0,0 @@ -use rayon::slice::ParallelSliceMut; -use reth_db::{ - cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW}, - tables, - transaction::{DbTx, DbTxMut}, -}; -use reth_interfaces::db::DatabaseError; -use reth_primitives::{revm::compat::into_reth_acc, Bytecode, StorageEntry, U256}; -use revm::db::states::{PlainStorageChangeset, StateChangeset}; - -/// A change to the state of the world. -#[derive(Debug, Default)] -pub struct StateChanges(pub StateChangeset); - -impl From for StateChanges { - fn from(revm: StateChangeset) -> Self { - Self(revm) - } -} - -impl StateChanges { - /// Write the bundle state to the database. - pub fn write_to_db(mut self, tx: &TX) -> Result<(), DatabaseError> { - // sort all entries so they can be written to database in more performant way. - // and take smaller memory footprint. - self.0.accounts.par_sort_by_key(|a| a.0); - self.0.storage.par_sort_by_key(|a| a.address); - self.0.contracts.par_sort_by_key(|a| a.0); - - // Write new account state - tracing::trace!(target: "provider::bundle_state", len = self.0.accounts.len(), "Writing new account state"); - let mut accounts_cursor = tx.cursor_write::()?; - // write account to database. - for (address, account) in self.0.accounts.into_iter() { - if let Some(account) = account { - tracing::trace!(target: "provider::bundle_state", ?address, "Updating plain state account"); - accounts_cursor.upsert(address, into_reth_acc(account))?; - } else if accounts_cursor.seek_exact(address)?.is_some() { - tracing::trace!(target: "provider::bundle_state", ?address, "Deleting plain state account"); - accounts_cursor.delete_current()?; - } - } - - // Write bytecode - tracing::trace!(target: "provider::bundle_state", len = self.0.contracts.len(), "Writing bytecodes"); - let mut bytecodes_cursor = tx.cursor_write::()?; - for (hash, bytecode) in self.0.contracts.into_iter() { - bytecodes_cursor.upsert(hash, Bytecode(bytecode))?; - } - - // Write new storage state and wipe storage if needed. - tracing::trace!(target: "provider::bundle_state", len = self.0.storage.len(), "Writing new storage state"); - let mut storages_cursor = tx.cursor_dup_write::()?; - for PlainStorageChangeset { address, wipe_storage, storage } in self.0.storage.into_iter() { - // Wiping of storage. - if wipe_storage && storages_cursor.seek_exact(address)?.is_some() { - storages_cursor.delete_current_duplicates()?; - } - // cast storages to B256. - let mut storage = storage - .into_iter() - .map(|(k, value)| StorageEntry { key: k.into(), value }) - .collect::>(); - // sort storage slots by key. - storage.par_sort_unstable_by_key(|a| a.key); - - for entry in storage.into_iter() { - tracing::trace!(target: "provider::bundle_state", ?address, ?entry.key, "Updating plain state storage"); - if let Some(db_entry) = storages_cursor.seek_by_key_subkey(address, entry.key)? { - if db_entry.key == entry.key { - storages_cursor.delete_current()?; - } - } - - if entry.value != U256::ZERO { - storages_cursor.upsert(address, entry)?; - } - } - } - Ok(()) - } -} diff --git a/crates/storage/provider/src/bundle_state/state_reverts.rs b/crates/storage/provider/src/bundle_state/state_reverts.rs index 2066511c78..e8be3e372d 100644 --- a/crates/storage/provider/src/bundle_state/state_reverts.rs +++ b/crates/storage/provider/src/bundle_state/state_reverts.rs @@ -1,95 +1,7 @@ -use rayon::slice::ParallelSliceMut; -use reth_db::{ - cursor::{DbCursorRO, DbDupCursorRO, DbDupCursorRW}, - models::{AccountBeforeTx, BlockNumberAddress}, - tables, - transaction::{DbTx, DbTxMut}, -}; -use reth_interfaces::db::DatabaseError; -use reth_primitives::{revm::compat::into_reth_acc, BlockNumber, StorageEntry, B256, U256}; -use revm::db::states::{PlainStateReverts, PlainStorageRevert, RevertToSlot}; +use reth_primitives::{B256, U256}; +use revm::db::states::RevertToSlot; use std::iter::Peekable; -/// Revert of the state. -#[derive(Debug, Default)] -pub struct StateReverts(pub PlainStateReverts); - -impl From for StateReverts { - fn from(revm: PlainStateReverts) -> Self { - Self(revm) - } -} - -impl StateReverts { - /// Write reverts to database. - /// - /// Note:: Reverts will delete all wiped storage from plain state. - pub fn write_to_db( - self, - tx: &TX, - first_block: BlockNumber, - ) -> Result<(), DatabaseError> { - // Write storage changes - tracing::trace!(target: "provider::reverts", "Writing storage changes"); - let mut storages_cursor = tx.cursor_dup_write::()?; - let mut storage_changeset_cursor = tx.cursor_dup_write::()?; - for (block_index, mut storage_changes) in self.0.storage.into_iter().enumerate() { - let block_number = first_block + block_index as BlockNumber; - - tracing::trace!(target: "provider::reverts", block_number, "Writing block change"); - // sort changes by address. - storage_changes.par_sort_unstable_by_key(|a| a.address); - for PlainStorageRevert { address, wiped, storage_revert } in storage_changes.into_iter() - { - let storage_id = BlockNumberAddress((block_number, address)); - - let mut storage = storage_revert - .into_iter() - .map(|(k, v)| (B256::new(k.to_be_bytes()), v)) - .collect::>(); - // sort storage slots by key. - storage.par_sort_unstable_by_key(|a| a.0); - - // If we are writing the primary storage wipe transition, the pre-existing plain - // storage state has to be taken from the database and written to storage history. - // See [StorageWipe::Primary] for more details. - let mut wiped_storage = Vec::new(); - if wiped { - tracing::trace!(target: "provider::reverts", ?address, "Wiping storage"); - if let Some((_, entry)) = storages_cursor.seek_exact(address)? { - wiped_storage.push((entry.key, entry.value)); - while let Some(entry) = storages_cursor.next_dup_val()? { - wiped_storage.push((entry.key, entry.value)) - } - } - } - - tracing::trace!(target: "provider::reverts", ?address, ?storage, "Writing storage reverts"); - for (key, value) in StorageRevertsIter::new(storage, wiped_storage) { - storage_changeset_cursor.append_dup(storage_id, StorageEntry { key, value })?; - } - } - } - - // Write account changes - tracing::trace!(target: "provider::reverts", "Writing account changes"); - let mut account_changeset_cursor = tx.cursor_dup_write::()?; - for (block_index, mut account_block_reverts) in self.0.accounts.into_iter().enumerate() { - let block_number = first_block + block_index as BlockNumber; - // Sort accounts by address. - account_block_reverts.par_sort_by_key(|a| a.0); - for (address, info) in account_block_reverts { - account_changeset_cursor.append_dup( - block_number, - AccountBeforeTx { address, info: info.map(into_reth_acc) }, - )?; - } - } - - Ok(()) - } -} - /// Iterator over storage reverts. /// See [StorageRevertsIter::next] for more details. pub struct StorageRevertsIter { @@ -97,6 +9,18 @@ pub struct StorageRevertsIter { wiped: Peekable, } +impl std::fmt::Debug for StorageRevertsIter +where + R: Iterator, + W: Iterator, + A: std::fmt::Debug, + B: std::fmt::Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StorageRevertsIter").finish() + } +} + impl StorageRevertsIter where R: Iterator, diff --git a/crates/storage/provider/src/lib.rs b/crates/storage/provider/src/lib.rs index a1aa097475..743547ecf0 100644 --- a/crates/storage/provider/src/lib.rs +++ b/crates/storage/provider/src/lib.rs @@ -33,7 +33,7 @@ pub mod chain; pub use chain::{Chain, DisplayBlocksChain}; pub mod bundle_state; -pub use bundle_state::{BundleStateWithReceipts, OriginalValuesKnown, StateChanges, StateReverts}; +pub use bundle_state::{BundleStateWithReceipts, OriginalValuesKnown}; pub(crate) fn to_range>(bounds: R) -> std::ops::Range { let start = match bounds.start_bound() { diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index b9142efda1..9630e2b028 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -118,13 +118,11 @@ impl ProviderFactory { /// open. #[track_caller] pub fn provider_rw(&self) -> ProviderResult> { - let mut provider = DatabaseProvider::new_rw(self.db.tx_mut()?, self.chain_spec.clone()); - - if let Some(snapshot_provider) = &self.snapshot_provider { - provider = provider.with_snapshot_provider(snapshot_provider.clone()); - } - - Ok(DatabaseProviderRW(provider)) + DatabaseProviderRW::with_tx( + self.db.tx_mut()?, + self.chain_spec.clone(), + &self.snapshot_provider, + ) } /// Storage provider for latest block diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index c2ac3e7dcd..fa3b701e3a 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -74,6 +74,40 @@ pub type DatabaseProviderRO = DatabaseProvider<::TX>; #[derive(Debug)] pub struct DatabaseProviderRW(pub DatabaseProvider<::TXMut>); +impl DatabaseProviderRW +where + DB: Database, +{ + /// Returns a provider with the given `DbTxMut` inside, which allows fetching and updating + /// data from the database using different types of providers. + #[track_caller] + pub fn with_tx( + tx: ::TXMut, + chain_spec: Arc, + snapshot_provider: &Option>, + ) -> ProviderResult> { + let mut provider = DatabaseProvider::new_rw(tx, chain_spec.clone()); + + if let Some(snapshot_provider) = snapshot_provider { + provider = provider.with_snapshot_provider(snapshot_provider.clone()); + } + + Ok(DatabaseProviderRW(provider)) + } + + /// Takes a function and passes a write-read transaction into it, making sure it's committed at + /// the end of the execution. + pub fn update(mut self, f: F) -> Result + where + F: FnOnce(&mut Self) -> T, + { + let res = f(&mut self); + self.commit()?; + + Ok(res) + } +} + impl Deref for DatabaseProviderRW { type Target = DatabaseProvider<::TXMut>; @@ -90,7 +124,7 @@ impl DerefMut for DatabaseProviderRW { impl DatabaseProviderRW { /// Commit database transaction - pub fn commit(self) -> ProviderResult { + pub fn commit(self) -> Result { self.0.commit() } @@ -375,7 +409,7 @@ impl DatabaseProvider { impl DatabaseProvider { /// Commit database transaction. - pub fn commit(self) -> ProviderResult { + pub fn commit(self) -> Result { Ok(self.tx.commit()?) }