mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
082daa7fc1 |
136
Cargo.lock
generated
136
Cargo.lock
generated
@@ -7907,7 +7907,6 @@ dependencies = [
|
||||
"reth-stages-types",
|
||||
"reth-static-file",
|
||||
"reth-static-file-types",
|
||||
"reth-storage-api",
|
||||
"reth-tasks",
|
||||
"reth-trie",
|
||||
"reth-trie-common",
|
||||
@@ -14638,3 +14637,138 @@ dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-consensus"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-contract"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-eips"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-genesis"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-json-rpc"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-network"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-network-primitives"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-provider"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-pubsub"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-rpc-client"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-rpc-types"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-rpc-types-admin"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-rpc-types-anvil"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-rpc-types-beacon"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-rpc-types-debug"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-rpc-types-engine"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-rpc-types-eth"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-rpc-types-mev"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-rpc-types-trace"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-rpc-types-txpool"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-serde"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-signer"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-signer-local"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-transport"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-transport-http"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-transport-ipc"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
[[patch.unused]]
|
||||
name = "alloy-transport-ws"
|
||||
version = "1.5.1"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
|
||||
|
||||
29
Cargo.toml
29
Cargo.toml
@@ -797,3 +797,32 @@ ipnet = "2.11"
|
||||
|
||||
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
|
||||
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
|
||||
|
||||
# Patched by patch-alloy.sh
|
||||
alloy-consensus = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-contract = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-eips = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-genesis = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-network = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-provider = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-serde = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-signer = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-transport = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
|
||||
@@ -50,7 +50,6 @@ reth-stages-types = { workspace = true, optional = true }
|
||||
reth-static-file-types = { workspace = true, features = ["clap"] }
|
||||
reth-static-file.workspace = true
|
||||
reth-tasks.workspace = true
|
||||
reth-storage-api.workspace = true
|
||||
reth-trie = { workspace = true, features = ["metrics"] }
|
||||
reth-trie-db = { workspace = true, features = ["metrics"] }
|
||||
reth-trie-common.workspace = true
|
||||
|
||||
@@ -21,7 +21,6 @@ use reth_node_builder::NodeTypesWithDB;
|
||||
use reth_primitives_traits::ValueWithSubKey;
|
||||
use reth_provider::{providers::ProviderNodeTypes, ChangeSetReader, StaticFileProviderFactory};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::StorageChangeSetReader;
|
||||
use tracing::error;
|
||||
|
||||
/// The arguments for the `reth db get` command
|
||||
@@ -83,33 +82,6 @@ impl Command {
|
||||
table.view(&GetValueViewer { tool, key, subkey, end_key, end_subkey, raw })?
|
||||
}
|
||||
Subcommand::StaticFile { segment, key, subkey, raw } => {
|
||||
if let StaticFileSegment::StorageChangeSets = segment {
|
||||
let storage_key =
|
||||
table_subkey::<tables::StorageChangeSets>(subkey.as_deref()).ok();
|
||||
let key = table_key::<tables::StorageChangeSets>(&key)?;
|
||||
|
||||
let provider = tool.provider_factory.static_file_provider();
|
||||
|
||||
if let Some(storage_key) = storage_key {
|
||||
let entry = provider.get_storage_before_block(
|
||||
key.block_number(),
|
||||
key.address(),
|
||||
storage_key,
|
||||
)?;
|
||||
|
||||
if let Some(entry) = entry {
|
||||
println!("{}", serde_json::to_string_pretty(&entry)?);
|
||||
} else {
|
||||
error!(target: "reth::cli", "No content for the given table key.");
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let changesets = provider.storage_changeset(key.block_number())?;
|
||||
println!("{}", serde_json::to_string_pretty(&changesets)?);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (key, subkey, mask): (u64, _, _) = match segment {
|
||||
StaticFileSegment::Headers => (
|
||||
table_key::<tables::Headers>(&key)?,
|
||||
@@ -140,9 +112,6 @@ impl Command {
|
||||
AccountChangesetMask::MASK,
|
||||
)
|
||||
}
|
||||
StaticFileSegment::StorageChangeSets => {
|
||||
unreachable!("storage changesets handled above");
|
||||
}
|
||||
};
|
||||
|
||||
// handle account changesets differently if a subkey is provided.
|
||||
@@ -221,9 +190,6 @@ impl Command {
|
||||
StaticFileSegment::AccountChangeSets => {
|
||||
unreachable!("account changeset static files are special cased before this match")
|
||||
}
|
||||
StaticFileSegment::StorageChangeSets => {
|
||||
unreachable!("storage changeset static files are special cased before this match")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,7 +162,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
let access_rights =
|
||||
if command.dry_run { AccessRights::RO } else { AccessRights::RW };
|
||||
db_exec!(self.env, tool, N, access_rights, {
|
||||
command.execute(&tool, ctx.task_executor, &data_dir)?;
|
||||
command.execute(&tool, ctx.task_executor.clone(), &data_dir)?;
|
||||
});
|
||||
}
|
||||
Subcommands::StaticFileHeader(command) => {
|
||||
|
||||
@@ -69,11 +69,6 @@ pub enum SetCommand {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store storage changesets in static files instead of the database
|
||||
StorageChangesets {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
}
|
||||
|
||||
impl Command {
|
||||
@@ -120,7 +115,6 @@ impl Command {
|
||||
transaction_hash_numbers_in_rocksdb: _,
|
||||
account_history_in_rocksdb: _,
|
||||
account_changesets_in_static_files: _,
|
||||
storage_changesets_in_static_files: _,
|
||||
} = settings.unwrap_or_else(StorageSettings::legacy);
|
||||
|
||||
// Update the setting based on the key
|
||||
@@ -173,14 +167,6 @@ impl Command {
|
||||
settings.account_history_in_rocksdb = value;
|
||||
println!("Set account_history_in_rocksdb = {}", value);
|
||||
}
|
||||
SetCommand::StorageChangesets { value } => {
|
||||
if settings.storage_changesets_in_static_files == value {
|
||||
println!("storage_changesets_in_static_files is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.storage_changesets_in_static_files = value;
|
||||
println!("Set storage_changesets_in_static_files = {}", value);
|
||||
}
|
||||
}
|
||||
|
||||
// Write updated settings
|
||||
|
||||
@@ -15,8 +15,7 @@ use reth_db_common::{
|
||||
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
|
||||
use reth_node_core::args::StageEnum;
|
||||
use reth_provider::{
|
||||
DBProvider, RocksDBProviderFactory, StaticFileProviderFactory, StaticFileWriter,
|
||||
StorageSettingsCache,
|
||||
DBProvider, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter,
|
||||
};
|
||||
use reth_prune::PruneSegment;
|
||||
use reth_stages::StageId;
|
||||
@@ -91,14 +90,11 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
StaticFileSegment::AccountChangeSets => {
|
||||
writer.prune_account_changesets(highest_block)?;
|
||||
}
|
||||
StaticFileSegment::StorageChangeSets => {
|
||||
writer.prune_storage_changesets(highest_block)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let provider_rw = tool.provider_factory.unwind_provider_rw()?;
|
||||
let provider_rw = tool.provider_factory.database_provider_rw()?;
|
||||
let tx = provider_rw.tx_ref();
|
||||
|
||||
match self.stage {
|
||||
@@ -172,20 +168,8 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
)?;
|
||||
}
|
||||
StageEnum::AccountHistory | StageEnum::StorageHistory => {
|
||||
let settings = provider_rw.cached_storage_settings();
|
||||
let rocksdb = tool.provider_factory.rocksdb_provider();
|
||||
|
||||
if settings.account_history_in_rocksdb {
|
||||
rocksdb.clear::<tables::AccountsHistory>()?;
|
||||
} else {
|
||||
tx.clear::<tables::AccountsHistory>()?;
|
||||
}
|
||||
|
||||
if settings.storages_history_in_rocksdb {
|
||||
rocksdb.clear::<tables::StoragesHistory>()?;
|
||||
} else {
|
||||
tx.clear::<tables::StoragesHistory>()?;
|
||||
}
|
||||
tx.clear::<tables::AccountsHistory>()?;
|
||||
tx.clear::<tables::StoragesHistory>()?;
|
||||
|
||||
reset_stage_checkpoint(tx, StageId::IndexAccountHistory)?;
|
||||
reset_stage_checkpoint(tx, StageId::IndexStorageHistory)?;
|
||||
@@ -193,14 +177,7 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
insert_genesis_history(&provider_rw, self.env.chain.genesis().alloc.iter())?;
|
||||
}
|
||||
StageEnum::TxLookup => {
|
||||
if provider_rw.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
|
||||
tool.provider_factory
|
||||
.rocksdb_provider()
|
||||
.clear::<tables::TransactionHashNumbers>()?;
|
||||
} else {
|
||||
tx.clear::<tables::TransactionHashNumbers>()?;
|
||||
}
|
||||
|
||||
tx.clear::<tables::TransactionHashNumbers>()?;
|
||||
reset_prune_checkpoint(tx, PruneSegment::TransactionLookup)?;
|
||||
|
||||
reset_stage_checkpoint(tx, StageId::TransactionLookup)?;
|
||||
|
||||
@@ -438,8 +438,6 @@ pub struct BlocksPerFileConfig {
|
||||
pub transaction_senders: Option<u64>,
|
||||
/// Number of blocks per file for the account changesets segment.
|
||||
pub account_change_sets: Option<u64>,
|
||||
/// Number of blocks per file for the storage changesets segment.
|
||||
pub storage_change_sets: Option<u64>,
|
||||
}
|
||||
|
||||
impl StaticFilesConfig {
|
||||
@@ -453,7 +451,6 @@ impl StaticFilesConfig {
|
||||
receipts,
|
||||
transaction_senders,
|
||||
account_change_sets,
|
||||
storage_change_sets,
|
||||
} = self.blocks_per_file;
|
||||
eyre::ensure!(headers != Some(0), "Headers segment blocks per file must be greater than 0");
|
||||
eyre::ensure!(
|
||||
@@ -472,10 +469,6 @@ impl StaticFilesConfig {
|
||||
account_change_sets != Some(0),
|
||||
"Account changesets segment blocks per file must be greater than 0"
|
||||
);
|
||||
eyre::ensure!(
|
||||
storage_change_sets != Some(0),
|
||||
"Storage changesets segment blocks per file must be greater than 0"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -487,7 +480,6 @@ impl StaticFilesConfig {
|
||||
receipts,
|
||||
transaction_senders,
|
||||
account_change_sets,
|
||||
storage_change_sets,
|
||||
} = self.blocks_per_file;
|
||||
|
||||
let mut map = StaticFileMap::default();
|
||||
@@ -500,7 +492,6 @@ impl StaticFilesConfig {
|
||||
StaticFileSegment::Receipts => receipts,
|
||||
StaticFileSegment::TransactionSenders => transaction_senders,
|
||||
StaticFileSegment::AccountChangeSets => account_change_sets,
|
||||
StaticFileSegment::StorageChangeSets => storage_change_sets,
|
||||
};
|
||||
|
||||
if let Some(blocks_per_file) = blocks_per_file {
|
||||
|
||||
@@ -34,11 +34,6 @@ fn default_account_worker_count() -> usize {
|
||||
/// The size of proof targets chunk to spawn in one multiproof calculation.
|
||||
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE: usize = 60;
|
||||
|
||||
/// The size of proof targets chunk to spawn in one multiproof calculation when V2 proofs are
|
||||
/// enabled. This is 4x the default chunk size to take advantage of more efficient V2 proof
|
||||
/// computation.
|
||||
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2: usize = DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE * 4;
|
||||
|
||||
/// Default number of reserved CPU cores for non-reth processes.
|
||||
///
|
||||
/// This will be deducted from the thread count of main reth global threadpool.
|
||||
@@ -272,17 +267,6 @@ impl TreeConfig {
|
||||
self.multiproof_chunk_size
|
||||
}
|
||||
|
||||
/// Return the multiproof task chunk size, using the V2 default if V2 proofs are enabled
|
||||
/// and the chunk size is at the default value.
|
||||
pub const fn effective_multiproof_chunk_size(&self) -> usize {
|
||||
if self.enable_proof_v2 && self.multiproof_chunk_size == DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE
|
||||
{
|
||||
DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2
|
||||
} else {
|
||||
self.multiproof_chunk_size
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the number of reserved CPU cores for non-reth processes
|
||||
pub const fn reserved_cpu_cores(&self) -> usize {
|
||||
self.reserved_cpu_cores
|
||||
|
||||
@@ -32,8 +32,7 @@ use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedBlock, Sealed
|
||||
use reth_provider::{
|
||||
BlockExecutionOutput, BlockExecutionResult, BlockNumReader, BlockReader, ChangeSetReader,
|
||||
DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader,
|
||||
StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader,
|
||||
TransactionVariant,
|
||||
StateProviderBox, StateProviderFactory, StateReader, TransactionVariant,
|
||||
};
|
||||
use reth_revm::database::StateProviderDatabase;
|
||||
use reth_stages_api::ControlFlow;
|
||||
@@ -318,7 +317,6 @@ where
|
||||
<P as DatabaseProviderFactory>::Provider: BlockReader<Block = N::Block, Header = N::BlockHeader>
|
||||
+ StageCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader,
|
||||
C: ConfigureEvm<Primitives = N> + 'static,
|
||||
T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>,
|
||||
|
||||
@@ -247,9 +247,6 @@ where
|
||||
let (to_sparse_trie, sparse_trie_rx) = channel();
|
||||
let (to_multi_proof, from_multi_proof) = crossbeam_channel::unbounded();
|
||||
|
||||
// Extract V2 proofs flag early so we can pass it to prewarm
|
||||
let v2_proofs_enabled = config.enable_proof_v2();
|
||||
|
||||
// Handle BAL-based optimization if available
|
||||
let prewarm_handle = if let Some(bal) = bal {
|
||||
// When BAL is present, use BAL prewarming and send BAL to multiproof
|
||||
@@ -266,7 +263,6 @@ where
|
||||
provider_builder.clone(),
|
||||
None, // Don't send proof targets when BAL is present
|
||||
Some(bal),
|
||||
v2_proofs_enabled,
|
||||
)
|
||||
} else {
|
||||
// Normal path: spawn with transaction prewarming
|
||||
@@ -277,7 +273,6 @@ where
|
||||
provider_builder.clone(),
|
||||
Some(to_multi_proof.clone()),
|
||||
None,
|
||||
v2_proofs_enabled,
|
||||
)
|
||||
};
|
||||
|
||||
@@ -285,6 +280,7 @@ where
|
||||
let task_ctx = ProofTaskCtx::new(multiproof_provider_factory);
|
||||
let storage_worker_count = config.storage_worker_count();
|
||||
let account_worker_count = config.account_worker_count();
|
||||
let v2_proofs_enabled = config.enable_proof_v2();
|
||||
let proof_handle = ProofWorkerHandle::new(
|
||||
self.executor.handle().clone(),
|
||||
task_ctx,
|
||||
@@ -296,13 +292,10 @@ where
|
||||
let multi_proof_task = MultiProofTask::new(
|
||||
proof_handle.clone(),
|
||||
to_sparse_trie,
|
||||
config
|
||||
.multiproof_chunking_enabled()
|
||||
.then_some(config.effective_multiproof_chunk_size()),
|
||||
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size()),
|
||||
to_multi_proof.clone(),
|
||||
from_multi_proof,
|
||||
)
|
||||
.with_v2_proofs_enabled(v2_proofs_enabled);
|
||||
);
|
||||
|
||||
// spawn multi-proof task
|
||||
let parent_span = span.clone();
|
||||
@@ -351,9 +344,8 @@ where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
{
|
||||
let (prewarm_rx, execution_rx, size_hint) = self.spawn_tx_iterator(transactions);
|
||||
// This path doesn't use multiproof, so V2 proofs flag doesn't matter
|
||||
let prewarm_handle =
|
||||
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal, false);
|
||||
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal);
|
||||
PayloadHandle {
|
||||
to_multi_proof: None,
|
||||
prewarm_handle,
|
||||
@@ -420,7 +412,6 @@ where
|
||||
}
|
||||
|
||||
/// Spawn prewarming optionally wired to the multiproof task for target updates.
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
fn spawn_caching_with<P>(
|
||||
&self,
|
||||
env: ExecutionEnv<Evm>,
|
||||
@@ -429,7 +420,6 @@ where
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
bal: Option<Arc<BlockAccessList>>,
|
||||
v2_proofs_enabled: bool,
|
||||
) -> CacheTaskHandle<N::Receipt>
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
@@ -452,7 +442,6 @@ where
|
||||
terminate_execution: Arc::new(AtomicBool::new(false)),
|
||||
precompile_cache_disabled: self.precompile_cache_disabled,
|
||||
precompile_cache_map: self.precompile_cache_map.clone(),
|
||||
v2_proofs_enabled,
|
||||
};
|
||||
|
||||
let (prewarm_task, to_prewarm_task) = PrewarmCacheTask::new(
|
||||
|
||||
@@ -11,18 +11,14 @@ use reth_metrics::Metrics;
|
||||
use reth_provider::AccountReader;
|
||||
use reth_revm::state::EvmState;
|
||||
use reth_trie::{
|
||||
added_removed_keys::MultiAddedRemovedKeys, proof_v2, HashedPostState, HashedStorage,
|
||||
added_removed_keys::MultiAddedRemovedKeys, DecodedMultiProof, HashedPostState, HashedStorage,
|
||||
MultiProofTargets,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use reth_trie_parallel::stats::ParallelTrieTracker;
|
||||
use reth_trie_parallel::{
|
||||
proof::ParallelProof,
|
||||
proof_task::{
|
||||
AccountMultiproofInput, ProofResult, ProofResultContext, ProofResultMessage,
|
||||
ProofWorkerHandle,
|
||||
AccountMultiproofInput, ProofResultContext, ProofResultMessage, ProofWorkerHandle,
|
||||
},
|
||||
targets_v2::{ChunkedMultiProofTargetsV2, MultiProofTargetsV2},
|
||||
};
|
||||
use revm_primitives::map::{hash_map, B256Map};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::Instant};
|
||||
@@ -67,12 +63,12 @@ const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300;
|
||||
|
||||
/// A trie update that can be applied to sparse trie alongside the proofs for touched parts of the
|
||||
/// state.
|
||||
#[derive(Debug)]
|
||||
#[derive(Default, Debug)]
|
||||
pub struct SparseTrieUpdate {
|
||||
/// The state update that was used to calculate the proof
|
||||
pub(crate) state: HashedPostState,
|
||||
/// The calculated multiproof
|
||||
pub(crate) multiproof: ProofResult,
|
||||
pub(crate) multiproof: DecodedMultiProof,
|
||||
}
|
||||
|
||||
impl SparseTrieUpdate {
|
||||
@@ -84,11 +80,7 @@ impl SparseTrieUpdate {
|
||||
/// Construct update from multiproof.
|
||||
#[cfg(test)]
|
||||
pub(super) fn from_multiproof(multiproof: reth_trie::MultiProof) -> alloy_rlp::Result<Self> {
|
||||
let stats = ParallelTrieTracker::default().finish();
|
||||
Ok(Self {
|
||||
state: HashedPostState::default(),
|
||||
multiproof: ProofResult::Legacy(multiproof.try_into()?, stats),
|
||||
})
|
||||
Ok(Self { multiproof: multiproof.try_into()?, ..Default::default() })
|
||||
}
|
||||
|
||||
/// Extend update with contents of the other.
|
||||
@@ -102,7 +94,7 @@ impl SparseTrieUpdate {
|
||||
#[derive(Debug)]
|
||||
pub(super) enum MultiProofMessage {
|
||||
/// Prefetch proof targets
|
||||
PrefetchProofs(VersionedMultiProofTargets),
|
||||
PrefetchProofs(MultiProofTargets),
|
||||
/// New state update from transaction execution with its source
|
||||
StateUpdate(Source, EvmState),
|
||||
/// State update that can be applied to the sparse trie without any new proofs.
|
||||
@@ -231,155 +223,12 @@ pub(crate) fn evm_state_to_hashed_post_state(update: EvmState) -> HashedPostStat
|
||||
hashed_state
|
||||
}
|
||||
|
||||
/// Extends a `MultiProofTargets` with the contents of a `VersionedMultiProofTargets`,
|
||||
/// regardless of which variant the latter is.
|
||||
fn extend_multiproof_targets(dest: &mut MultiProofTargets, src: &VersionedMultiProofTargets) {
|
||||
match src {
|
||||
VersionedMultiProofTargets::Legacy(targets) => {
|
||||
dest.extend_ref(targets);
|
||||
}
|
||||
VersionedMultiProofTargets::V2(targets) => {
|
||||
// Add all account targets
|
||||
for target in &targets.account_targets {
|
||||
dest.entry(target.key()).or_default();
|
||||
}
|
||||
|
||||
// Add all storage targets
|
||||
for (hashed_address, slots) in &targets.storage_targets {
|
||||
let slot_set = dest.entry(*hashed_address).or_default();
|
||||
for slot in slots {
|
||||
slot_set.insert(slot.key());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A set of multiproof targets which can be either in the legacy or V2 representations.
|
||||
#[derive(Debug)]
|
||||
pub(super) enum VersionedMultiProofTargets {
|
||||
/// Legacy targets
|
||||
Legacy(MultiProofTargets),
|
||||
/// V2 targets
|
||||
V2(MultiProofTargetsV2),
|
||||
}
|
||||
|
||||
impl VersionedMultiProofTargets {
|
||||
/// Returns true if there are no account or storage targets.
|
||||
fn is_empty(&self) -> bool {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.is_empty(),
|
||||
Self::V2(targets) => targets.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of account targets in the multiproof target
|
||||
fn account_targets_len(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.len(),
|
||||
Self::V2(targets) => targets.account_targets.len(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of storage targets in the multiproof target
|
||||
fn storage_targets_len(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.values().map(|slots| slots.len()).sum::<usize>(),
|
||||
Self::V2(targets) => {
|
||||
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of accounts in the multiproof targets.
|
||||
fn len(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.len(),
|
||||
Self::V2(targets) => targets.account_targets.len(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the total storage slot count across all accounts.
|
||||
fn storage_count(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.values().map(|slots| slots.len()).sum(),
|
||||
Self::V2(targets) => targets.storage_targets.values().map(|slots| slots.len()).sum(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of items that will be considered during chunking.
|
||||
fn chunking_length(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.chunking_length(),
|
||||
Self::V2(targets) => {
|
||||
// For V2, count accounts + storage slots
|
||||
targets.account_targets.len() +
|
||||
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Retains the targets representing the difference with another `MultiProofTargets`.
|
||||
/// Removes all targets that are already present in `other`.
|
||||
fn retain_difference(&mut self, other: &MultiProofTargets) {
|
||||
match self {
|
||||
Self::Legacy(targets) => {
|
||||
targets.retain_difference(other);
|
||||
}
|
||||
Self::V2(targets) => {
|
||||
// Remove account targets that exist in other
|
||||
targets.account_targets.retain(|target| !other.contains_key(&target.key()));
|
||||
|
||||
// For each account in storage_targets, remove slots that exist in other
|
||||
targets.storage_targets.retain(|hashed_address, slots| {
|
||||
if let Some(other_slots) = other.get(hashed_address) {
|
||||
slots.retain(|slot| !other_slots.contains(&slot.key()));
|
||||
!slots.is_empty()
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extends this `VersionedMultiProofTargets` with the contents of another.
|
||||
///
|
||||
/// Panics if the variants do not match.
|
||||
fn extend(&mut self, other: Self) {
|
||||
match (self, other) {
|
||||
(Self::Legacy(dest), Self::Legacy(src)) => {
|
||||
dest.extend(src);
|
||||
}
|
||||
(Self::V2(dest), Self::V2(src)) => {
|
||||
dest.account_targets.extend(src.account_targets);
|
||||
for (addr, slots) in src.storage_targets {
|
||||
dest.storage_targets.entry(addr).or_default().extend(slots);
|
||||
}
|
||||
}
|
||||
_ => panic!("Cannot extend VersionedMultiProofTargets with mismatched variants"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Chunks this `VersionedMultiProofTargets` into smaller chunks of the given size.
|
||||
fn chunks(self, chunk_size: usize) -> Box<dyn Iterator<Item = Self>> {
|
||||
match self {
|
||||
Self::Legacy(targets) => {
|
||||
Box::new(MultiProofTargets::chunks(targets, chunk_size).map(Self::Legacy))
|
||||
}
|
||||
Self::V2(targets) => {
|
||||
Box::new(ChunkedMultiProofTargetsV2::new(targets, chunk_size).map(Self::V2))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Input parameters for dispatching a multiproof calculation.
|
||||
#[derive(Debug)]
|
||||
struct MultiproofInput {
|
||||
source: Option<Source>,
|
||||
hashed_state_update: HashedPostState,
|
||||
proof_targets: VersionedMultiProofTargets,
|
||||
proof_targets: MultiProofTargets,
|
||||
proof_sequence_number: u64,
|
||||
state_root_message_sender: CrossbeamSender<MultiProofMessage>,
|
||||
multi_added_removed_keys: Option<Arc<MultiAddedRemovedKeys>>,
|
||||
@@ -414,6 +263,8 @@ pub struct MultiproofManager {
|
||||
proof_result_tx: CrossbeamSender<ProofResultMessage>,
|
||||
/// Metrics
|
||||
metrics: MultiProofTaskMetrics,
|
||||
/// Whether to use V2 storage proofs
|
||||
v2_proofs_enabled: bool,
|
||||
}
|
||||
|
||||
impl MultiproofManager {
|
||||
@@ -427,7 +278,9 @@ impl MultiproofManager {
|
||||
metrics.max_storage_workers.set(proof_worker_handle.total_storage_workers() as f64);
|
||||
metrics.max_account_workers.set(proof_worker_handle.total_account_workers() as f64);
|
||||
|
||||
Self { metrics, proof_worker_handle, proof_result_tx }
|
||||
let v2_proofs_enabled = proof_worker_handle.v2_proofs_enabled();
|
||||
|
||||
Self { metrics, proof_worker_handle, proof_result_tx, v2_proofs_enabled }
|
||||
}
|
||||
|
||||
/// Dispatches a new multiproof calculation to worker pools.
|
||||
@@ -472,48 +325,41 @@ impl MultiproofManager {
|
||||
multi_added_removed_keys,
|
||||
} = multiproof_input;
|
||||
|
||||
let account_targets = proof_targets.len();
|
||||
let storage_targets = proof_targets.values().map(|slots| slots.len()).sum::<usize>();
|
||||
|
||||
trace!(
|
||||
target: "engine::tree::payload_processor::multiproof",
|
||||
proof_sequence_number,
|
||||
?proof_targets,
|
||||
account_targets = proof_targets.account_targets_len(),
|
||||
storage_targets = proof_targets.storage_targets_len(),
|
||||
account_targets,
|
||||
storage_targets,
|
||||
?source,
|
||||
"Dispatching multiproof to workers"
|
||||
);
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
// Workers will send ProofResultMessage directly to proof_result_rx
|
||||
let proof_result_sender = ProofResultContext::new(
|
||||
self.proof_result_tx.clone(),
|
||||
proof_sequence_number,
|
||||
hashed_state_update,
|
||||
start,
|
||||
);
|
||||
|
||||
let input = match proof_targets {
|
||||
VersionedMultiProofTargets::Legacy(proof_targets) => {
|
||||
// Extend prefix sets with targets
|
||||
let frozen_prefix_sets = ParallelProof::extend_prefix_sets_with_targets(
|
||||
&Default::default(),
|
||||
&proof_targets,
|
||||
);
|
||||
|
||||
AccountMultiproofInput::Legacy {
|
||||
targets: proof_targets,
|
||||
prefix_sets: frozen_prefix_sets,
|
||||
collect_branch_node_masks: true,
|
||||
multi_added_removed_keys,
|
||||
proof_result_sender,
|
||||
}
|
||||
}
|
||||
VersionedMultiProofTargets::V2(proof_targets) => {
|
||||
AccountMultiproofInput::V2 { targets: proof_targets, proof_result_sender }
|
||||
}
|
||||
};
|
||||
// Extend prefix sets with targets
|
||||
let frozen_prefix_sets =
|
||||
ParallelProof::extend_prefix_sets_with_targets(&Default::default(), &proof_targets);
|
||||
|
||||
// Dispatch account multiproof to worker pool with result sender
|
||||
let input = AccountMultiproofInput {
|
||||
targets: proof_targets,
|
||||
prefix_sets: frozen_prefix_sets,
|
||||
collect_branch_node_masks: true,
|
||||
multi_added_removed_keys,
|
||||
// Workers will send ProofResultMessage directly to proof_result_rx
|
||||
proof_result_sender: ProofResultContext::new(
|
||||
self.proof_result_tx.clone(),
|
||||
proof_sequence_number,
|
||||
hashed_state_update,
|
||||
start,
|
||||
),
|
||||
v2_proofs_enabled: self.v2_proofs_enabled,
|
||||
};
|
||||
|
||||
if let Err(e) = self.proof_worker_handle.dispatch_account_multiproof(input) {
|
||||
error!(target: "engine::tree::payload_processor::multiproof", ?e, "Failed to dispatch account multiproof");
|
||||
return;
|
||||
@@ -715,9 +561,6 @@ pub(super) struct MultiProofTask {
|
||||
/// there are any active workers and force chunking across workers. This is to prevent tasks
|
||||
/// which are very long from hitting a single worker.
|
||||
max_targets_for_chunking: usize,
|
||||
/// Whether or not V2 proof calculation is enabled. If enabled then [`MultiProofTargetsV2`]
|
||||
/// will be produced by state updates.
|
||||
v2_proofs_enabled: bool,
|
||||
}
|
||||
|
||||
impl MultiProofTask {
|
||||
@@ -749,16 +592,9 @@ impl MultiProofTask {
|
||||
),
|
||||
metrics,
|
||||
max_targets_for_chunking: DEFAULT_MAX_TARGETS_FOR_CHUNKING,
|
||||
v2_proofs_enabled: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enables V2 proof target generation on state updates.
|
||||
pub(super) const fn with_v2_proofs_enabled(mut self, v2_proofs_enabled: bool) -> Self {
|
||||
self.v2_proofs_enabled = v2_proofs_enabled;
|
||||
self
|
||||
}
|
||||
|
||||
/// Handles request for proof prefetch.
|
||||
///
|
||||
/// Returns how many multiproof tasks were dispatched for the prefetch request.
|
||||
@@ -766,29 +602,37 @@ impl MultiProofTask {
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor::multiproof",
|
||||
skip_all,
|
||||
fields(accounts = targets.account_targets_len(), chunks = 0)
|
||||
fields(accounts = targets.len(), chunks = 0)
|
||||
)]
|
||||
fn on_prefetch_proof(&mut self, mut targets: VersionedMultiProofTargets) -> u64 {
|
||||
fn on_prefetch_proof(&mut self, mut targets: MultiProofTargets) -> u64 {
|
||||
// Remove already fetched proof targets to avoid redundant work.
|
||||
targets.retain_difference(&self.fetched_proof_targets);
|
||||
extend_multiproof_targets(&mut self.fetched_proof_targets, &targets);
|
||||
self.fetched_proof_targets.extend_ref(&targets);
|
||||
|
||||
// For Legacy multiproofs, make sure all target accounts have an `AddedRemovedKeySet` in the
|
||||
// Make sure all target accounts have an `AddedRemovedKeySet` in the
|
||||
// [`MultiAddedRemovedKeys`]. Even if there are not any known removed keys for the account,
|
||||
// we still want to optimistically fetch extension children for the leaf addition case.
|
||||
// V2 multiproofs don't need this.
|
||||
let multi_added_removed_keys =
|
||||
if let VersionedMultiProofTargets::Legacy(legacy_targets) = &targets {
|
||||
self.multi_added_removed_keys.touch_accounts(legacy_targets.keys().copied());
|
||||
Some(Arc::new(self.multi_added_removed_keys.clone()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
self.multi_added_removed_keys.touch_accounts(targets.keys().copied());
|
||||
|
||||
// Clone+Arc MultiAddedRemovedKeys for sharing with the dispatched multiproof tasks
|
||||
let multi_added_removed_keys = Arc::new(MultiAddedRemovedKeys {
|
||||
account: self.multi_added_removed_keys.account.clone(),
|
||||
storages: targets
|
||||
.keys()
|
||||
.filter_map(|account| {
|
||||
self.multi_added_removed_keys
|
||||
.storages
|
||||
.get(account)
|
||||
.cloned()
|
||||
.map(|keys| (*account, keys))
|
||||
})
|
||||
.collect(),
|
||||
});
|
||||
|
||||
self.metrics.prefetch_proof_targets_accounts_histogram.record(targets.len() as f64);
|
||||
self.metrics
|
||||
.prefetch_proof_targets_storages_histogram
|
||||
.record(targets.storage_count() as f64);
|
||||
.record(targets.values().map(|slots| slots.len()).sum::<usize>() as f64);
|
||||
|
||||
let chunking_len = targets.chunking_length();
|
||||
let available_account_workers =
|
||||
@@ -802,7 +646,7 @@ impl MultiProofTask {
|
||||
self.max_targets_for_chunking,
|
||||
available_account_workers,
|
||||
available_storage_workers,
|
||||
VersionedMultiProofTargets::chunks,
|
||||
MultiProofTargets::chunks,
|
||||
|proof_targets| {
|
||||
self.multiproof_manager.dispatch(MultiproofInput {
|
||||
source: None,
|
||||
@@ -810,7 +654,7 @@ impl MultiProofTask {
|
||||
proof_targets,
|
||||
proof_sequence_number: self.proof_sequencer.next_sequence(),
|
||||
state_root_message_sender: self.tx.clone(),
|
||||
multi_added_removed_keys: multi_added_removed_keys.clone(),
|
||||
multi_added_removed_keys: Some(multi_added_removed_keys.clone()),
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -913,7 +757,6 @@ impl MultiProofTask {
|
||||
self.multiproof_manager.proof_worker_handle.available_account_workers();
|
||||
let available_storage_workers =
|
||||
self.multiproof_manager.proof_worker_handle.available_storage_workers();
|
||||
|
||||
let num_chunks = dispatch_with_chunking(
|
||||
not_fetched_state_update,
|
||||
chunking_len,
|
||||
@@ -927,9 +770,8 @@ impl MultiProofTask {
|
||||
&hashed_state_update,
|
||||
&self.fetched_proof_targets,
|
||||
&multi_added_removed_keys,
|
||||
self.v2_proofs_enabled,
|
||||
);
|
||||
extend_multiproof_targets(&mut spawned_proof_targets, &proof_targets);
|
||||
spawned_proof_targets.extend_ref(&proof_targets);
|
||||
|
||||
self.multiproof_manager.dispatch(MultiproofInput {
|
||||
source: Some(source),
|
||||
@@ -1029,10 +871,7 @@ impl MultiProofTask {
|
||||
batch_metrics.proofs_processed += 1;
|
||||
if let Some(combined_update) = self.on_proof(
|
||||
sequence_number,
|
||||
SparseTrieUpdate {
|
||||
state,
|
||||
multiproof: ProofResult::empty(self.v2_proofs_enabled),
|
||||
},
|
||||
SparseTrieUpdate { state, multiproof: Default::default() },
|
||||
) {
|
||||
let _ = self.to_sparse_trie.send(combined_update);
|
||||
}
|
||||
@@ -1059,7 +898,8 @@ impl MultiProofTask {
|
||||
}
|
||||
|
||||
let account_targets = merged_targets.len();
|
||||
let storage_targets = merged_targets.storage_count();
|
||||
let storage_targets =
|
||||
merged_targets.values().map(|slots| slots.len()).sum::<usize>();
|
||||
batch_metrics.prefetch_proofs_requested += self.on_prefetch_proof(merged_targets);
|
||||
trace!(
|
||||
target: "engine::tree::payload_processor::multiproof",
|
||||
@@ -1163,10 +1003,7 @@ impl MultiProofTask {
|
||||
|
||||
if let Some(combined_update) = self.on_proof(
|
||||
sequence_number,
|
||||
SparseTrieUpdate {
|
||||
state,
|
||||
multiproof: ProofResult::empty(self.v2_proofs_enabled),
|
||||
},
|
||||
SparseTrieUpdate { state, multiproof: Default::default() },
|
||||
) {
|
||||
let _ = self.to_sparse_trie.send(combined_update);
|
||||
}
|
||||
@@ -1269,7 +1106,7 @@ impl MultiProofTask {
|
||||
|
||||
let update = SparseTrieUpdate {
|
||||
state: proof_result.state,
|
||||
multiproof: proof_result_data,
|
||||
multiproof: proof_result_data.proof,
|
||||
};
|
||||
|
||||
if let Some(combined_update) =
|
||||
@@ -1359,7 +1196,7 @@ struct MultiproofBatchCtx {
|
||||
/// received.
|
||||
updates_finished_time: Option<Instant>,
|
||||
/// Reusable buffer for accumulating prefetch targets during batching.
|
||||
accumulated_prefetch_targets: Vec<VersionedMultiProofTargets>,
|
||||
accumulated_prefetch_targets: Vec<MultiProofTargets>,
|
||||
}
|
||||
|
||||
impl MultiproofBatchCtx {
|
||||
@@ -1405,77 +1242,40 @@ fn get_proof_targets(
|
||||
state_update: &HashedPostState,
|
||||
fetched_proof_targets: &MultiProofTargets,
|
||||
multi_added_removed_keys: &MultiAddedRemovedKeys,
|
||||
v2_enabled: bool,
|
||||
) -> VersionedMultiProofTargets {
|
||||
if v2_enabled {
|
||||
let mut targets = MultiProofTargetsV2::default();
|
||||
) -> MultiProofTargets {
|
||||
let mut targets = MultiProofTargets::default();
|
||||
|
||||
// first collect all new accounts (not previously fetched)
|
||||
for &hashed_address in state_update.accounts.keys() {
|
||||
if !fetched_proof_targets.contains_key(&hashed_address) {
|
||||
targets.account_targets.push(hashed_address.into());
|
||||
}
|
||||
// first collect all new accounts (not previously fetched)
|
||||
for hashed_address in state_update.accounts.keys() {
|
||||
if !fetched_proof_targets.contains_key(hashed_address) {
|
||||
targets.insert(*hashed_address, HashSet::default());
|
||||
}
|
||||
|
||||
// then process storage slots for all accounts in the state update
|
||||
for (hashed_address, storage) in &state_update.storages {
|
||||
let fetched = fetched_proof_targets.get(hashed_address);
|
||||
|
||||
// If the storage is wiped, we still need to fetch the account proof.
|
||||
if storage.wiped && fetched.is_none() {
|
||||
targets.account_targets.push(Into::<proof_v2::Target>::into(*hashed_address));
|
||||
continue
|
||||
}
|
||||
|
||||
let changed_slots = storage
|
||||
.storage
|
||||
.keys()
|
||||
.filter(|slot| !fetched.is_some_and(|f| f.contains(*slot)))
|
||||
.map(|slot| Into::<proof_v2::Target>::into(*slot))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if !changed_slots.is_empty() {
|
||||
targets.account_targets.push((*hashed_address).into());
|
||||
targets.storage_targets.insert(*hashed_address, changed_slots);
|
||||
}
|
||||
}
|
||||
|
||||
VersionedMultiProofTargets::V2(targets)
|
||||
} else {
|
||||
let mut targets = MultiProofTargets::default();
|
||||
|
||||
// first collect all new accounts (not previously fetched)
|
||||
for hashed_address in state_update.accounts.keys() {
|
||||
if !fetched_proof_targets.contains_key(hashed_address) {
|
||||
targets.insert(*hashed_address, HashSet::default());
|
||||
}
|
||||
}
|
||||
|
||||
// then process storage slots for all accounts in the state update
|
||||
for (hashed_address, storage) in &state_update.storages {
|
||||
let fetched = fetched_proof_targets.get(hashed_address);
|
||||
let storage_added_removed_keys = multi_added_removed_keys.get_storage(hashed_address);
|
||||
let mut changed_slots = storage
|
||||
.storage
|
||||
.keys()
|
||||
.filter(|slot| {
|
||||
!fetched.is_some_and(|f| f.contains(*slot)) ||
|
||||
storage_added_removed_keys.is_some_and(|k| k.is_removed(slot))
|
||||
})
|
||||
.peekable();
|
||||
|
||||
// If the storage is wiped, we still need to fetch the account proof.
|
||||
if storage.wiped && fetched.is_none() {
|
||||
targets.entry(*hashed_address).or_default();
|
||||
}
|
||||
|
||||
if changed_slots.peek().is_some() {
|
||||
targets.entry(*hashed_address).or_default().extend(changed_slots);
|
||||
}
|
||||
}
|
||||
|
||||
VersionedMultiProofTargets::Legacy(targets)
|
||||
}
|
||||
|
||||
// then process storage slots for all accounts in the state update
|
||||
for (hashed_address, storage) in &state_update.storages {
|
||||
let fetched = fetched_proof_targets.get(hashed_address);
|
||||
let storage_added_removed_keys = multi_added_removed_keys.get_storage(hashed_address);
|
||||
let mut changed_slots = storage
|
||||
.storage
|
||||
.keys()
|
||||
.filter(|slot| {
|
||||
!fetched.is_some_and(|f| f.contains(*slot)) ||
|
||||
storage_added_removed_keys.is_some_and(|k| k.is_removed(slot))
|
||||
})
|
||||
.peekable();
|
||||
|
||||
// If the storage is wiped, we still need to fetch the account proof.
|
||||
if storage.wiped && fetched.is_none() {
|
||||
targets.entry(*hashed_address).or_default();
|
||||
}
|
||||
|
||||
if changed_slots.peek().is_some() {
|
||||
targets.entry(*hashed_address).or_default().extend(changed_slots);
|
||||
}
|
||||
}
|
||||
|
||||
targets
|
||||
}
|
||||
|
||||
/// Dispatches work items as a single unit or in chunks based on target size and worker
|
||||
@@ -1523,7 +1323,7 @@ mod tests {
|
||||
use reth_provider::{
|
||||
providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory,
|
||||
BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, LatestStateProvider,
|
||||
PruneCheckpointReader, StageCheckpointReader, StateProviderBox, StorageChangeSetReader,
|
||||
PruneCheckpointReader, StageCheckpointReader, StateProviderBox,
|
||||
};
|
||||
use reth_trie::MultiProof;
|
||||
use reth_trie_db::ChangesetCache;
|
||||
@@ -1550,7 +1350,6 @@ mod tests {
|
||||
+ StageCheckpointReader
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader,
|
||||
> + Clone
|
||||
+ Send
|
||||
@@ -1682,24 +1481,12 @@ mod tests {
|
||||
state
|
||||
}
|
||||
|
||||
fn unwrap_legacy_targets(targets: VersionedMultiProofTargets) -> MultiProofTargets {
|
||||
match targets {
|
||||
VersionedMultiProofTargets::Legacy(targets) => targets,
|
||||
VersionedMultiProofTargets::V2(_) => panic!("Expected Legacy targets"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_proof_targets_new_account_targets() {
|
||||
let state = create_get_proof_targets_state();
|
||||
let fetched = MultiProofTargets::default();
|
||||
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&MultiAddedRemovedKeys::new(),
|
||||
false,
|
||||
));
|
||||
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
|
||||
|
||||
// should return all accounts as targets since nothing was fetched before
|
||||
assert_eq!(targets.len(), state.accounts.len());
|
||||
@@ -1713,12 +1500,7 @@ mod tests {
|
||||
let state = create_get_proof_targets_state();
|
||||
let fetched = MultiProofTargets::default();
|
||||
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&MultiAddedRemovedKeys::new(),
|
||||
false,
|
||||
));
|
||||
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
|
||||
|
||||
// verify storage slots are included for accounts with storage
|
||||
for (addr, storage) in &state.storages {
|
||||
@@ -1746,12 +1528,7 @@ mod tests {
|
||||
// mark the account as already fetched
|
||||
fetched.insert(*fetched_addr, HashSet::default());
|
||||
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&MultiAddedRemovedKeys::new(),
|
||||
false,
|
||||
));
|
||||
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
|
||||
|
||||
// should not include the already fetched account since it has no storage updates
|
||||
assert!(!targets.contains_key(fetched_addr));
|
||||
@@ -1771,12 +1548,7 @@ mod tests {
|
||||
fetched_slots.insert(fetched_slot);
|
||||
fetched.insert(*addr, fetched_slots);
|
||||
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&MultiAddedRemovedKeys::new(),
|
||||
false,
|
||||
));
|
||||
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
|
||||
|
||||
// should not include the already fetched storage slot
|
||||
let target_slots = &targets[addr];
|
||||
@@ -1789,12 +1561,7 @@ mod tests {
|
||||
let state = HashedPostState::default();
|
||||
let fetched = MultiProofTargets::default();
|
||||
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&MultiAddedRemovedKeys::new(),
|
||||
false,
|
||||
));
|
||||
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
|
||||
|
||||
assert!(targets.is_empty());
|
||||
}
|
||||
@@ -1821,12 +1588,7 @@ mod tests {
|
||||
fetched_slots.insert(slot1);
|
||||
fetched.insert(addr1, fetched_slots);
|
||||
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&MultiAddedRemovedKeys::new(),
|
||||
false,
|
||||
));
|
||||
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
|
||||
|
||||
assert!(targets.contains_key(&addr2));
|
||||
assert!(!targets[&addr1].contains(&slot1));
|
||||
@@ -1852,12 +1614,7 @@ mod tests {
|
||||
assert!(!state.accounts.contains_key(&addr));
|
||||
assert!(!fetched.contains_key(&addr));
|
||||
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&MultiAddedRemovedKeys::new(),
|
||||
false,
|
||||
));
|
||||
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
|
||||
|
||||
// verify that we still get the storage slots for the unmodified account
|
||||
assert!(targets.contains_key(&addr));
|
||||
@@ -1899,12 +1656,7 @@ mod tests {
|
||||
removed_state.storages.insert(addr, removed_storage);
|
||||
multi_added_removed_keys.update_with_state(&removed_state);
|
||||
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&multi_added_removed_keys,
|
||||
false,
|
||||
));
|
||||
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
|
||||
|
||||
// slot1 should be included despite being fetched, because it's marked as removed
|
||||
assert!(targets.contains_key(&addr));
|
||||
@@ -1931,12 +1683,7 @@ mod tests {
|
||||
storage.storage.insert(slot1, U256::from(100));
|
||||
state.storages.insert(addr, storage);
|
||||
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&multi_added_removed_keys,
|
||||
false,
|
||||
));
|
||||
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
|
||||
|
||||
// account should be included because storage is wiped and account wasn't fetched
|
||||
assert!(targets.contains_key(&addr));
|
||||
@@ -1979,12 +1726,7 @@ mod tests {
|
||||
removed_state.storages.insert(addr, removed_storage);
|
||||
multi_added_removed_keys.update_with_state(&removed_state);
|
||||
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&multi_added_removed_keys,
|
||||
false,
|
||||
));
|
||||
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
|
||||
|
||||
// only slots in the state update can be included, so slot3 should not appear
|
||||
assert!(!targets.contains_key(&addr));
|
||||
@@ -2011,12 +1753,9 @@ mod tests {
|
||||
targets3.insert(addr3, HashSet::default());
|
||||
|
||||
let tx = task.tx.clone();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets1)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets2)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets3)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(targets1)).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(targets2)).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(targets3)).unwrap();
|
||||
|
||||
let proofs_requested =
|
||||
if let Ok(MultiProofMessage::PrefetchProofs(targets)) = task.rx.recv() {
|
||||
@@ -2030,12 +1769,11 @@ mod tests {
|
||||
|
||||
assert_eq!(num_batched, 3);
|
||||
assert_eq!(merged_targets.len(), 3);
|
||||
let legacy_targets = unwrap_legacy_targets(merged_targets);
|
||||
assert!(legacy_targets.contains_key(&addr1));
|
||||
assert!(legacy_targets.contains_key(&addr2));
|
||||
assert!(legacy_targets.contains_key(&addr3));
|
||||
assert!(merged_targets.contains_key(&addr1));
|
||||
assert!(merged_targets.contains_key(&addr2));
|
||||
assert!(merged_targets.contains_key(&addr3));
|
||||
|
||||
task.on_prefetch_proof(VersionedMultiProofTargets::Legacy(legacy_targets))
|
||||
task.on_prefetch_proof(merged_targets)
|
||||
} else {
|
||||
panic!("Expected PrefetchProofs message");
|
||||
};
|
||||
@@ -2110,16 +1848,11 @@ mod tests {
|
||||
|
||||
// Queue: [PrefetchProofs1, PrefetchProofs2, StateUpdate1, StateUpdate2, PrefetchProofs3]
|
||||
let tx = task.tx.clone();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets1)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets2)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(targets1)).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(targets2)).unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update1)).unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update2)).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(
|
||||
targets3.clone(),
|
||||
)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(targets3.clone())).unwrap();
|
||||
|
||||
// Step 1: Receive and batch PrefetchProofs (should get targets1 + targets2)
|
||||
let mut pending_msg: Option<MultiProofMessage> = None;
|
||||
@@ -2145,10 +1878,9 @@ mod tests {
|
||||
// Should have batched exactly 2 PrefetchProofs (not 3!)
|
||||
assert_eq!(num_batched, 2, "Should batch only until different message type");
|
||||
assert_eq!(merged_targets.len(), 2);
|
||||
let legacy_targets = unwrap_legacy_targets(merged_targets);
|
||||
assert!(legacy_targets.contains_key(&addr1));
|
||||
assert!(legacy_targets.contains_key(&addr2));
|
||||
assert!(!legacy_targets.contains_key(&addr3), "addr3 should NOT be in first batch");
|
||||
assert!(merged_targets.contains_key(&addr1));
|
||||
assert!(merged_targets.contains_key(&addr2));
|
||||
assert!(!merged_targets.contains_key(&addr3), "addr3 should NOT be in first batch");
|
||||
} else {
|
||||
panic!("Expected PrefetchProofs message");
|
||||
}
|
||||
@@ -2173,8 +1905,7 @@ mod tests {
|
||||
match task.rx.try_recv() {
|
||||
Ok(MultiProofMessage::PrefetchProofs(targets)) => {
|
||||
assert_eq!(targets.len(), 1);
|
||||
let legacy_targets = unwrap_legacy_targets(targets);
|
||||
assert!(legacy_targets.contains_key(&addr3));
|
||||
assert!(targets.contains_key(&addr3));
|
||||
}
|
||||
_ => panic!("PrefetchProofs3 was lost!"),
|
||||
}
|
||||
@@ -2220,13 +1951,9 @@ mod tests {
|
||||
let source = StateChangeSource::Transaction(99);
|
||||
|
||||
let tx = task.tx.clone();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(prefetch1)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(prefetch1)).unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update)).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(
|
||||
prefetch2.clone(),
|
||||
)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(prefetch2.clone())).unwrap();
|
||||
|
||||
let mut ctx = MultiproofBatchCtx::new(Instant::now());
|
||||
let mut batch_metrics = MultiproofBatchMetrics::default();
|
||||
@@ -2259,8 +1986,7 @@ mod tests {
|
||||
match task.rx.try_recv() {
|
||||
Ok(MultiProofMessage::PrefetchProofs(targets)) => {
|
||||
assert_eq!(targets.len(), 1);
|
||||
let legacy_targets = unwrap_legacy_targets(targets);
|
||||
assert!(legacy_targets.contains_key(&prefetch_addr2));
|
||||
assert!(targets.contains_key(&prefetch_addr2));
|
||||
}
|
||||
other => panic!("Expected PrefetchProofs2 in channel, got {:?}", other),
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ use crate::tree::{
|
||||
payload_processor::{
|
||||
bal::{total_slots, BALSlotIter},
|
||||
executor::WorkloadExecutor,
|
||||
multiproof::{MultiProofMessage, VersionedMultiProofTargets},
|
||||
multiproof::MultiProofMessage,
|
||||
ExecutionCache as PayloadExecutionCache,
|
||||
},
|
||||
precompile_cache::{CachedPrecompile, PrecompileCacheMap},
|
||||
@@ -237,7 +237,7 @@ where
|
||||
}
|
||||
|
||||
/// If configured and the tx returned proof targets, emit the targets the transaction produced
|
||||
fn send_multi_proof_targets(&self, targets: Option<VersionedMultiProofTargets>) {
|
||||
fn send_multi_proof_targets(&self, targets: Option<MultiProofTargets>) {
|
||||
if self.is_execution_terminated() {
|
||||
// if execution is already terminated then we dont need to send more proof fetch
|
||||
// messages
|
||||
@@ -484,8 +484,6 @@ where
|
||||
pub(super) terminate_execution: Arc<AtomicBool>,
|
||||
pub(super) precompile_cache_disabled: bool,
|
||||
pub(super) precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
|
||||
/// Whether V2 proof calculation is enabled.
|
||||
pub(super) v2_proofs_enabled: bool,
|
||||
}
|
||||
|
||||
impl<N, P, Evm> PrewarmContext<N, P, Evm>
|
||||
@@ -494,12 +492,10 @@ where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
Evm: ConfigureEvm<Primitives = N> + 'static,
|
||||
{
|
||||
/// Splits this context into an evm, an evm config, metrics, the atomic bool for terminating
|
||||
/// execution, and whether V2 proofs are enabled.
|
||||
/// Splits this context into an evm, an evm config, metrics, and the atomic bool for terminating
|
||||
/// execution.
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
|
||||
fn evm_for_ctx(
|
||||
self,
|
||||
) -> Option<(EvmFor<Evm, impl Database>, PrewarmMetrics, Arc<AtomicBool>, bool)> {
|
||||
fn evm_for_ctx(self) -> Option<(EvmFor<Evm, impl Database>, PrewarmMetrics, Arc<AtomicBool>)> {
|
||||
let Self {
|
||||
env,
|
||||
evm_config,
|
||||
@@ -509,7 +505,6 @@ where
|
||||
terminate_execution,
|
||||
precompile_cache_disabled,
|
||||
precompile_cache_map,
|
||||
v2_proofs_enabled,
|
||||
} = self;
|
||||
|
||||
let mut state_provider = match provider.build() {
|
||||
@@ -559,7 +554,7 @@ where
|
||||
});
|
||||
}
|
||||
|
||||
Some((evm, metrics, terminate_execution, v2_proofs_enabled))
|
||||
Some((evm, metrics, terminate_execution))
|
||||
}
|
||||
|
||||
/// Accepts an [`mpsc::Receiver`] of transactions and a handle to prewarm task. Executes
|
||||
@@ -580,10 +575,7 @@ where
|
||||
) where
|
||||
Tx: ExecutableTxFor<Evm>,
|
||||
{
|
||||
let Some((mut evm, metrics, terminate_execution, v2_proofs_enabled)) = self.evm_for_ctx()
|
||||
else {
|
||||
return
|
||||
};
|
||||
let Some((mut evm, metrics, terminate_execution)) = self.evm_for_ctx() else { return };
|
||||
|
||||
while let Ok(IndexedTransaction { index, tx }) = {
|
||||
let _enter = debug_span!(target: "engine::tree::payload_processor::prewarm", "recv tx")
|
||||
@@ -646,8 +638,7 @@ where
|
||||
let _enter =
|
||||
debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm outcome", index, tx_hash=%tx.tx().tx_hash())
|
||||
.entered();
|
||||
let (targets, storage_targets) =
|
||||
multiproof_targets_from_state(res.state, v2_proofs_enabled);
|
||||
let (targets, storage_targets) = multiproof_targets_from_state(res.state);
|
||||
metrics.prefetch_storage_targets.record(storage_targets as f64);
|
||||
let _ = sender.send(PrewarmTaskEvent::Outcome { proof_targets: Some(targets) });
|
||||
drop(_enter);
|
||||
@@ -792,22 +783,9 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a set of [`VersionedMultiProofTargets`] and the total amount of storage targets, based
|
||||
/// on the given state.
|
||||
fn multiproof_targets_from_state(
|
||||
state: EvmState,
|
||||
v2_enabled: bool,
|
||||
) -> (VersionedMultiProofTargets, usize) {
|
||||
if v2_enabled {
|
||||
multiproof_targets_v2_from_state(state)
|
||||
} else {
|
||||
multiproof_targets_legacy_from_state(state)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns legacy [`MultiProofTargets`] and the total amount of storage targets, based on the
|
||||
/// Returns a set of [`MultiProofTargets`] and the total amount of storage targets, based on the
|
||||
/// given state.
|
||||
fn multiproof_targets_legacy_from_state(state: EvmState) -> (VersionedMultiProofTargets, usize) {
|
||||
fn multiproof_targets_from_state(state: EvmState) -> (MultiProofTargets, usize) {
|
||||
let mut targets = MultiProofTargets::with_capacity(state.len());
|
||||
let mut storage_targets = 0;
|
||||
for (addr, account) in state {
|
||||
@@ -837,50 +815,7 @@ fn multiproof_targets_legacy_from_state(state: EvmState) -> (VersionedMultiProof
|
||||
targets.insert(keccak256(addr), storage_set);
|
||||
}
|
||||
|
||||
(VersionedMultiProofTargets::Legacy(targets), storage_targets)
|
||||
}
|
||||
|
||||
/// Returns V2 [`reth_trie_parallel::targets_v2::MultiProofTargetsV2`] and the total amount of
|
||||
/// storage targets, based on the given state.
|
||||
fn multiproof_targets_v2_from_state(state: EvmState) -> (VersionedMultiProofTargets, usize) {
|
||||
use reth_trie::proof_v2;
|
||||
use reth_trie_parallel::targets_v2::MultiProofTargetsV2;
|
||||
|
||||
let mut targets = MultiProofTargetsV2::default();
|
||||
let mut storage_target_count = 0;
|
||||
for (addr, account) in state {
|
||||
// if the account was not touched, or if the account was selfdestructed, do not
|
||||
// fetch proofs for it
|
||||
//
|
||||
// Since selfdestruct can only happen in the same transaction, we can skip
|
||||
// prefetching proofs for selfdestructed accounts
|
||||
//
|
||||
// See: https://eips.ethereum.org/EIPS/eip-6780
|
||||
if !account.is_touched() || account.is_selfdestructed() {
|
||||
continue
|
||||
}
|
||||
|
||||
let hashed_address = keccak256(addr);
|
||||
targets.account_targets.push(hashed_address.into());
|
||||
|
||||
let mut storage_slots = Vec::with_capacity(account.storage.len());
|
||||
for (key, slot) in account.storage {
|
||||
// do nothing if unchanged
|
||||
if !slot.is_changed() {
|
||||
continue
|
||||
}
|
||||
|
||||
let hashed_slot = keccak256(B256::new(key.to_be_bytes()));
|
||||
storage_slots.push(proof_v2::Target::from(hashed_slot));
|
||||
}
|
||||
|
||||
storage_target_count += storage_slots.len();
|
||||
if !storage_slots.is_empty() {
|
||||
targets.storage_targets.insert(hashed_address, storage_slots);
|
||||
}
|
||||
}
|
||||
|
||||
(VersionedMultiProofTargets::V2(targets), storage_target_count)
|
||||
(targets, storage_targets)
|
||||
}
|
||||
|
||||
/// The events the pre-warm task can handle.
|
||||
@@ -905,7 +840,7 @@ pub(super) enum PrewarmTaskEvent<R> {
|
||||
/// The outcome of a pre-warm task
|
||||
Outcome {
|
||||
/// The prepared proof targets based on the evm state outcome
|
||||
proof_targets: Option<VersionedMultiProofTargets>,
|
||||
proof_targets: Option<MultiProofTargets>,
|
||||
},
|
||||
/// Finished executing all transactions
|
||||
FinishedTxExecution {
|
||||
|
||||
@@ -4,7 +4,7 @@ use crate::tree::payload_processor::multiproof::{MultiProofTaskMetrics, SparseTr
|
||||
use alloy_primitives::B256;
|
||||
use rayon::iter::{ParallelBridge, ParallelIterator};
|
||||
use reth_trie::{updates::TrieUpdates, Nibbles};
|
||||
use reth_trie_parallel::{proof_task::ProofResult, root::ParallelStateRootError};
|
||||
use reth_trie_parallel::root::ParallelStateRootError;
|
||||
use reth_trie_sparse::{
|
||||
errors::{SparseStateTrieResult, SparseTrieErrorKind},
|
||||
provider::{TrieNodeProvider, TrieNodeProviderFactory},
|
||||
@@ -97,8 +97,8 @@ where
|
||||
debug!(
|
||||
target: "engine::root",
|
||||
num_updates,
|
||||
account_proofs = update.multiproof.account_proofs_len(),
|
||||
storage_proofs = update.multiproof.storage_proofs_len(),
|
||||
account_proofs = update.multiproof.account_subtree.len(),
|
||||
storage_proofs = update.multiproof.storages.len(),
|
||||
"Updating sparse trie"
|
||||
);
|
||||
|
||||
@@ -157,14 +157,7 @@ where
|
||||
let started_at = Instant::now();
|
||||
|
||||
// Reveal new accounts and storage slots.
|
||||
match multiproof {
|
||||
ProofResult::Legacy(decoded, _) => {
|
||||
trie.reveal_decoded_multiproof(decoded)?;
|
||||
}
|
||||
ProofResult::V2(decoded_v2) => {
|
||||
trie.reveal_decoded_multiproof_v2(decoded_v2)?;
|
||||
}
|
||||
}
|
||||
trie.reveal_decoded_multiproof(multiproof)?;
|
||||
let reveal_multiproof_elapsed = started_at.elapsed();
|
||||
trace!(
|
||||
target: "engine::root::sparse",
|
||||
|
||||
@@ -39,7 +39,7 @@ use reth_provider::{
|
||||
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader,
|
||||
ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider,
|
||||
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
|
||||
StateProviderFactory, StateReader, StorageChangeSetReader,
|
||||
StateProviderFactory, StateReader,
|
||||
};
|
||||
use reth_revm::db::{states::bundle_state::BundleRetention, State};
|
||||
use reth_trie::{updates::TrieUpdates, HashedPostState, StateRoot};
|
||||
@@ -144,7 +144,6 @@ where
|
||||
+ StageCheckpointReader
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader,
|
||||
> + BlockReader<Header = N::BlockHeader>
|
||||
+ ChangeSetReader
|
||||
@@ -1337,7 +1336,6 @@ where
|
||||
+ StageCheckpointReader
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader,
|
||||
> + BlockReader<Header = N::BlockHeader>
|
||||
+ StateProviderFactory
|
||||
|
||||
@@ -1,357 +0,0 @@
|
||||
//! Tests for handling invalid payloads via Engine API.
|
||||
//!
|
||||
//! This module tests the scenario where a node receives invalid payloads (e.g., with modified
|
||||
//! state roots) before receiving valid ones, ensuring the node can recover and continue.
|
||||
|
||||
use crate::utils::eth_payload_attributes;
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rpc_types_engine::{ExecutionPayloadV3, PayloadStatusEnum};
|
||||
use rand::{rngs::StdRng, Rng, SeedableRng};
|
||||
use reth_chainspec::{ChainSpecBuilder, MAINNET};
|
||||
use reth_e2e_test_utils::{setup_engine, transaction::TransactionTestContext};
|
||||
use reth_node_ethereum::EthereumNode;
|
||||
|
||||
use reth_rpc_api::EngineApiClient;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Tests that a node can handle receiving an invalid payload (with wrong state root)
|
||||
/// followed by the correct payload, and continue operating normally.
|
||||
///
|
||||
/// Setup:
|
||||
/// - Node 1: Produces valid payloads and advances the chain
|
||||
/// - Node 2: Receives payloads from node 1, but we also inject modified payloads with invalid state
|
||||
/// roots in between to verify error handling
|
||||
#[tokio::test]
|
||||
async fn can_handle_invalid_payload_then_valid() -> eyre::Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let seed: [u8; 32] = rand::rng().random();
|
||||
let mut rng = StdRng::from_seed(seed);
|
||||
println!("Seed: {seed:?}");
|
||||
|
||||
let chain_spec = Arc::new(
|
||||
ChainSpecBuilder::default()
|
||||
.chain(MAINNET.chain)
|
||||
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
|
||||
.cancun_activated()
|
||||
.build(),
|
||||
);
|
||||
|
||||
let (mut nodes, _tasks, wallet) = setup_engine::<EthereumNode>(
|
||||
2,
|
||||
chain_spec.clone(),
|
||||
false,
|
||||
Default::default(),
|
||||
eth_payload_attributes,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut producer = nodes.pop().unwrap();
|
||||
let receiver = nodes.pop().unwrap();
|
||||
|
||||
// Get engine API client for the receiver node
|
||||
let receiver_engine = receiver.auth_server_handle().http_client();
|
||||
|
||||
// Inject a transaction to allow block building (advance_block waits for transactions)
|
||||
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
|
||||
producer.rpc.inject_tx(raw_tx).await?;
|
||||
|
||||
// Build a valid payload on the producer
|
||||
let payload = producer.advance_block().await?;
|
||||
let valid_block = payload.block().clone();
|
||||
|
||||
// Create valid payload first, then corrupt the state root
|
||||
let mut invalid_payload = ExecutionPayloadV3::from_block_unchecked(
|
||||
valid_block.hash(),
|
||||
&valid_block.clone().into_block(),
|
||||
);
|
||||
let original_state_root = invalid_payload.payload_inner.payload_inner.state_root;
|
||||
invalid_payload.payload_inner.payload_inner.state_root = B256::random_with(&mut rng);
|
||||
|
||||
// Send the invalid payload to the receiver - should be rejected
|
||||
let invalid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
|
||||
&receiver_engine,
|
||||
invalid_payload.clone(),
|
||||
vec![],
|
||||
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!(
|
||||
"Invalid payload response: {:?} (state_root changed from {original_state_root} to {})",
|
||||
invalid_result.status, invalid_payload.payload_inner.payload_inner.state_root
|
||||
);
|
||||
|
||||
// The invalid payload should be rejected
|
||||
assert!(
|
||||
matches!(
|
||||
invalid_result.status,
|
||||
PayloadStatusEnum::Invalid { .. } | PayloadStatusEnum::Syncing
|
||||
),
|
||||
"Expected INVALID or SYNCING status for invalid payload, got {:?}",
|
||||
invalid_result.status
|
||||
);
|
||||
|
||||
// Now send the valid payload - should be accepted
|
||||
let valid_payload = ExecutionPayloadV3::from_block_unchecked(
|
||||
valid_block.hash(),
|
||||
&valid_block.clone().into_block(),
|
||||
);
|
||||
|
||||
let valid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
|
||||
&receiver_engine,
|
||||
valid_payload,
|
||||
vec![],
|
||||
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!("Valid payload response: {:?}", valid_result.status);
|
||||
|
||||
// The valid payload should be accepted
|
||||
assert!(
|
||||
matches!(
|
||||
valid_result.status,
|
||||
PayloadStatusEnum::Valid | PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted
|
||||
),
|
||||
"Expected VALID/SYNCING/ACCEPTED status for valid payload, got {:?}",
|
||||
valid_result.status
|
||||
);
|
||||
|
||||
// Update forkchoice on receiver to the valid block
|
||||
receiver.update_forkchoice(valid_block.hash(), valid_block.hash()).await?;
|
||||
|
||||
// Verify the receiver node is at the expected block
|
||||
let receiver_head = receiver.block_hash(1);
|
||||
let producer_head = producer.block_hash(1);
|
||||
assert_eq!(
|
||||
receiver_head, producer_head,
|
||||
"Receiver should have synced to the same chain as producer"
|
||||
);
|
||||
|
||||
println!(
|
||||
"Test passed: Receiver successfully handled invalid payloads and synced to valid chain"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Tests that a node can handle multiple consecutive invalid payloads
|
||||
/// before receiving a valid one.
|
||||
#[tokio::test]
|
||||
async fn can_handle_multiple_invalid_payloads() -> eyre::Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let seed: [u8; 32] = rand::rng().random();
|
||||
let mut rng = StdRng::from_seed(seed);
|
||||
println!("Seed: {seed:?}");
|
||||
|
||||
let chain_spec = Arc::new(
|
||||
ChainSpecBuilder::default()
|
||||
.chain(MAINNET.chain)
|
||||
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
|
||||
.cancun_activated()
|
||||
.build(),
|
||||
);
|
||||
|
||||
let (mut nodes, _tasks, wallet) = setup_engine::<EthereumNode>(
|
||||
2,
|
||||
chain_spec.clone(),
|
||||
false,
|
||||
Default::default(),
|
||||
eth_payload_attributes,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut producer = nodes.pop().unwrap();
|
||||
let receiver = nodes.pop().unwrap();
|
||||
|
||||
let receiver_engine = receiver.auth_server_handle().http_client();
|
||||
|
||||
// Inject a transaction to allow block building
|
||||
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
|
||||
producer.rpc.inject_tx(raw_tx).await?;
|
||||
|
||||
// Produce a valid block
|
||||
let payload = producer.advance_block().await?;
|
||||
let valid_block = payload.block().clone();
|
||||
|
||||
// Send multiple invalid payloads with different corruptions
|
||||
for i in 0..3 {
|
||||
// Create valid payload first, then corrupt the state root
|
||||
let mut invalid_payload = ExecutionPayloadV3::from_block_unchecked(
|
||||
valid_block.hash(),
|
||||
&valid_block.clone().into_block(),
|
||||
);
|
||||
invalid_payload.payload_inner.payload_inner.state_root = B256::random_with(&mut rng);
|
||||
|
||||
let result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
|
||||
&receiver_engine,
|
||||
invalid_payload,
|
||||
vec![],
|
||||
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!("Invalid payload {i}: status = {:?}", result.status);
|
||||
|
||||
assert!(
|
||||
matches!(result.status, PayloadStatusEnum::Invalid { .. } | PayloadStatusEnum::Syncing),
|
||||
"Expected INVALID or SYNCING for invalid payload {i}, got {:?}",
|
||||
result.status
|
||||
);
|
||||
}
|
||||
|
||||
// Now send the valid payload
|
||||
let valid_payload = ExecutionPayloadV3::from_block_unchecked(
|
||||
valid_block.hash(),
|
||||
&valid_block.clone().into_block(),
|
||||
);
|
||||
|
||||
let valid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
|
||||
&receiver_engine,
|
||||
valid_payload,
|
||||
vec![],
|
||||
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!("Valid payload: status = {:?}", valid_result.status);
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
valid_result.status,
|
||||
PayloadStatusEnum::Valid | PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted
|
||||
),
|
||||
"Expected valid status for correct payload, got {:?}",
|
||||
valid_result.status
|
||||
);
|
||||
|
||||
// Finalize the valid block
|
||||
receiver.update_forkchoice(valid_block.hash(), valid_block.hash()).await?;
|
||||
|
||||
println!("Test passed: Receiver handled multiple invalid payloads and accepted valid one");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Tests invalid payload handling with blocks that contain transactions.
|
||||
///
|
||||
/// This test sends real transactions to node 1, produces blocks with those transactions,
|
||||
/// then sends invalid (corrupted state root) and valid payloads to node 2.
|
||||
#[tokio::test]
|
||||
async fn can_handle_invalid_payload_with_transactions() -> eyre::Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let seed: [u8; 32] = rand::rng().random();
|
||||
let mut rng = StdRng::from_seed(seed);
|
||||
println!("Seed: {seed:?}");
|
||||
|
||||
let chain_spec = Arc::new(
|
||||
ChainSpecBuilder::default()
|
||||
.chain(MAINNET.chain)
|
||||
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
|
||||
.cancun_activated()
|
||||
.build(),
|
||||
);
|
||||
|
||||
let (mut nodes, _tasks, wallet) = setup_engine::<EthereumNode>(
|
||||
2,
|
||||
chain_spec.clone(),
|
||||
false,
|
||||
Default::default(),
|
||||
eth_payload_attributes,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut producer = nodes.pop().unwrap();
|
||||
let receiver = nodes.pop().unwrap();
|
||||
|
||||
let receiver_engine = receiver.auth_server_handle().http_client();
|
||||
|
||||
// Create and send a transaction to the producer node
|
||||
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
|
||||
let tx_hash = producer.rpc.inject_tx(raw_tx).await?;
|
||||
println!("Injected transaction {tx_hash}");
|
||||
|
||||
// Build a block containing the transaction
|
||||
let payload = producer.advance_block().await?;
|
||||
let valid_block = payload.block().clone();
|
||||
|
||||
// Verify the block contains a transaction
|
||||
let tx_count = valid_block.body().transactions().count();
|
||||
println!("Block contains {tx_count} transaction(s)");
|
||||
assert!(tx_count > 0, "Block should contain at least one transaction");
|
||||
|
||||
// Create invalid payload by corrupting the state root
|
||||
let mut invalid_payload = ExecutionPayloadV3::from_block_unchecked(
|
||||
valid_block.hash(),
|
||||
&valid_block.clone().into_block(),
|
||||
);
|
||||
let original_state_root = invalid_payload.payload_inner.payload_inner.state_root;
|
||||
invalid_payload.payload_inner.payload_inner.state_root = B256::random_with(&mut rng);
|
||||
|
||||
// Send invalid payload - should be rejected
|
||||
let invalid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
|
||||
&receiver_engine,
|
||||
invalid_payload.clone(),
|
||||
vec![],
|
||||
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!(
|
||||
"Invalid payload (with tx) response: {:?} (state_root changed from {original_state_root} to {})",
|
||||
invalid_result.status,
|
||||
invalid_payload.payload_inner.payload_inner.state_root
|
||||
);
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
invalid_result.status,
|
||||
PayloadStatusEnum::Invalid { .. } | PayloadStatusEnum::Syncing
|
||||
),
|
||||
"Expected INVALID or SYNCING for invalid payload with transactions, got {:?}",
|
||||
invalid_result.status
|
||||
);
|
||||
|
||||
// Send valid payload - should be accepted
|
||||
let valid_payload = ExecutionPayloadV3::from_block_unchecked(
|
||||
valid_block.hash(),
|
||||
&valid_block.clone().into_block(),
|
||||
);
|
||||
|
||||
let valid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
|
||||
&receiver_engine,
|
||||
valid_payload,
|
||||
vec![],
|
||||
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!("Valid payload (with tx) response: {:?}", valid_result.status);
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
valid_result.status,
|
||||
PayloadStatusEnum::Valid | PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted
|
||||
),
|
||||
"Expected valid status for correct payload with transactions, got {:?}",
|
||||
valid_result.status
|
||||
);
|
||||
|
||||
// Update forkchoice
|
||||
receiver.update_forkchoice(valid_block.hash(), valid_block.hash()).await?;
|
||||
|
||||
// Verify both nodes are at the same head
|
||||
let receiver_head = receiver.block_hash(1);
|
||||
let producer_head = producer.block_hash(1);
|
||||
assert_eq!(
|
||||
receiver_head, producer_head,
|
||||
"Receiver should have synced to the same chain as producer"
|
||||
);
|
||||
|
||||
println!("Test passed: Receiver handled invalid payloads with transactions correctly");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -4,7 +4,6 @@ mod blobs;
|
||||
mod custom_genesis;
|
||||
mod dev;
|
||||
mod eth;
|
||||
mod invalid_payload;
|
||||
mod p2p;
|
||||
mod pool;
|
||||
mod prestate;
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
use clap::Args;
|
||||
use reth_config::config::{BlocksPerFileConfig, StaticFilesConfig};
|
||||
use reth_storage_api::StorageSettings;
|
||||
|
||||
/// Blocks per static file when running in `--minimal` node.
|
||||
///
|
||||
@@ -41,10 +40,6 @@ pub struct StaticFilesArgs {
|
||||
#[arg(long = "static-files.blocks-per-file.account-change-sets")]
|
||||
pub blocks_per_file_account_change_sets: Option<u64>,
|
||||
|
||||
/// Number of blocks per file for the storage changesets segment.
|
||||
#[arg(long = "static-files.blocks-per-file.storage-change-sets")]
|
||||
pub blocks_per_file_storage_change_sets: Option<u64>,
|
||||
|
||||
/// Store receipts in static files instead of the database.
|
||||
///
|
||||
/// When enabled, receipts will be written to static files on disk instead of the database.
|
||||
@@ -73,16 +68,6 @@ pub struct StaticFilesArgs {
|
||||
/// the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
#[arg(long = "static-files.account-change-sets", default_value_t = default_static_file_flag(), action = clap::ArgAction::Set)]
|
||||
pub account_changesets: bool,
|
||||
|
||||
/// Store storage changesets in static files.
|
||||
///
|
||||
/// When enabled, storage changesets will be written to static files on disk instead of the
|
||||
/// database.
|
||||
///
|
||||
/// Note: This setting can only be configured at genesis initialization. Once
|
||||
/// the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
#[arg(long = "static-files.storage-change-sets", default_value_t = default_static_file_flag(), action = clap::ArgAction::Set)]
|
||||
pub storage_changesets: bool,
|
||||
}
|
||||
|
||||
impl StaticFilesArgs {
|
||||
@@ -113,25 +98,9 @@ impl StaticFilesArgs {
|
||||
account_change_sets: self
|
||||
.blocks_per_file_account_change_sets
|
||||
.or(config.blocks_per_file.account_change_sets),
|
||||
storage_change_sets: self
|
||||
.blocks_per_file_storage_change_sets
|
||||
.or(config.blocks_per_file.storage_change_sets),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts the static files arguments into [`StorageSettings`].
|
||||
pub const fn to_settings(&self) -> StorageSettings {
|
||||
#[cfg(feature = "edge")]
|
||||
let base = StorageSettings::edge();
|
||||
#[cfg(not(feature = "edge"))]
|
||||
let base = StorageSettings::legacy();
|
||||
|
||||
base.with_receipts_in_static_files(self.receipts)
|
||||
.with_transaction_senders_in_static_files(self.transaction_senders)
|
||||
.with_account_changesets_in_static_files(self.account_changesets)
|
||||
.with_storage_changesets_in_static_files(self.storage_changesets)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for StaticFilesArgs {
|
||||
@@ -142,11 +111,9 @@ impl Default for StaticFilesArgs {
|
||||
blocks_per_file_receipts: None,
|
||||
blocks_per_file_transaction_senders: None,
|
||||
blocks_per_file_account_change_sets: None,
|
||||
blocks_per_file_storage_change_sets: None,
|
||||
receipts: default_static_file_flag(),
|
||||
transaction_senders: default_static_file_flag(),
|
||||
account_changesets: default_static_file_flag(),
|
||||
storage_changesets: default_static_file_flag(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -363,7 +363,6 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
.with_receipts_in_static_files(self.static_files.receipts)
|
||||
.with_transaction_senders_in_static_files(self.static_files.transaction_senders)
|
||||
.with_account_changesets_in_static_files(self.static_files.account_changesets)
|
||||
.with_storage_changesets_in_static_files(self.static_files.storage_changesets)
|
||||
.with_transaction_hash_numbers_in_rocksdb(self.rocksdb.all || self.rocksdb.tx_hash)
|
||||
.with_storages_history_in_rocksdb(self.rocksdb.all || self.rocksdb.storages_history)
|
||||
.with_account_history_in_rocksdb(self.rocksdb.all || self.rocksdb.account_history)
|
||||
|
||||
@@ -9,16 +9,6 @@ use std::{fmt::Debug, ops::RangeBounds};
|
||||
use tracing::debug;
|
||||
|
||||
pub(crate) trait DbTxPruneExt: DbTxMut + DbTx {
|
||||
/// Clear the entire table in a single operation.
|
||||
///
|
||||
/// This is much faster than iterating entry-by-entry for `PruneMode::Full`.
|
||||
/// Returns the number of entries that were in the table.
|
||||
fn clear_table<T: Table>(&self) -> Result<usize, DatabaseError> {
|
||||
let count = self.entries::<T>()?;
|
||||
<Self as DbTxMut>::clear::<T>(self)?;
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
/// Prune the table for the specified pre-sorted key iterator.
|
||||
///
|
||||
/// Returns number of rows pruned.
|
||||
|
||||
@@ -6,7 +6,7 @@ use crate::{
|
||||
use reth_db_api::{tables, transaction::DbTxMut};
|
||||
use reth_provider::{BlockReader, DBProvider, TransactionsProvider};
|
||||
use reth_prune_types::{
|
||||
PruneMode, PruneProgress, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint,
|
||||
PruneMode, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint,
|
||||
};
|
||||
use tracing::{instrument, trace};
|
||||
|
||||
@@ -48,25 +48,6 @@ where
|
||||
};
|
||||
let tx_range_end = *tx_range.end();
|
||||
|
||||
// For PruneMode::Full, clear the entire table in one operation
|
||||
if self.mode.is_full() {
|
||||
let pruned = provider.tx_ref().clear_table::<tables::TransactionSenders>()?;
|
||||
trace!(target: "pruner", %pruned, "Cleared transaction senders table");
|
||||
|
||||
let last_pruned_block = provider
|
||||
.block_by_transaction_id(tx_range_end)?
|
||||
.ok_or(PrunerError::InconsistentData("Block for transaction is not found"))?;
|
||||
|
||||
return Ok(SegmentOutput {
|
||||
progress: PruneProgress::Finished,
|
||||
pruned,
|
||||
checkpoint: Some(SegmentOutputCheckpoint {
|
||||
block_number: Some(last_pruned_block),
|
||||
tx_number: Some(tx_range_end),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
let mut limiter = input.limiter;
|
||||
|
||||
let mut last_pruned_transaction = tx_range_end;
|
||||
|
||||
@@ -8,7 +8,7 @@ use rayon::prelude::*;
|
||||
use reth_db_api::{tables, transaction::DbTxMut};
|
||||
use reth_provider::{BlockReader, DBProvider, PruneCheckpointReader, StaticFileProviderFactory};
|
||||
use reth_prune_types::{
|
||||
PruneCheckpoint, PruneMode, PruneProgress, PrunePurpose, PruneSegment, SegmentOutputCheckpoint,
|
||||
PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment, SegmentOutputCheckpoint,
|
||||
};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use tracing::{debug, instrument, trace};
|
||||
@@ -82,26 +82,6 @@ where
|
||||
}
|
||||
}
|
||||
.into_inner();
|
||||
|
||||
// For PruneMode::Full, clear the entire table in one operation
|
||||
if self.mode.is_full() {
|
||||
let pruned = provider.tx_ref().clear_table::<tables::TransactionHashNumbers>()?;
|
||||
trace!(target: "pruner", %pruned, "Cleared transaction lookup table");
|
||||
|
||||
let last_pruned_block = provider
|
||||
.block_by_transaction_id(end)?
|
||||
.ok_or(PrunerError::InconsistentData("Block for transaction is not found"))?;
|
||||
|
||||
return Ok(SegmentOutput {
|
||||
progress: PruneProgress::Finished,
|
||||
pruned,
|
||||
checkpoint: Some(SegmentOutputCheckpoint {
|
||||
block_number: Some(last_pruned_block),
|
||||
tx_number: Some(end),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
let tx_range = start..=
|
||||
Some(end)
|
||||
.min(
|
||||
@@ -116,17 +96,12 @@ where
|
||||
let tx_range_end = *tx_range.end();
|
||||
|
||||
// Retrieve transactions in the range and calculate their hashes in parallel
|
||||
let mut hashes = provider
|
||||
let hashes = provider
|
||||
.transactions_by_tx_range(tx_range.clone())?
|
||||
.into_par_iter()
|
||||
.map(|transaction| transaction.trie_hash())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
// Sort hashes to enable efficient cursor traversal through the TransactionHashNumbers
|
||||
// table, which is keyed by hash. Without sorting, each seek would be O(log n) random
|
||||
// access; with sorting, the cursor advances sequentially through the B+tree.
|
||||
hashes.sort_unstable();
|
||||
|
||||
// Number of transactions retrieved from the database should match the tx range count
|
||||
let tx_count = tx_range.count();
|
||||
if hashes.len() != tx_count {
|
||||
|
||||
@@ -483,12 +483,6 @@ where
|
||||
.ok_or_else(|| ProviderError::HeaderNotFound(block_hash.into()))?
|
||||
};
|
||||
|
||||
// Check if the block has been pruned (EIP-4444)
|
||||
let earliest_block = self.provider().earliest_block_number()?;
|
||||
if header.number() < earliest_block {
|
||||
return Err(EthApiError::PrunedHistoryUnavailable.into());
|
||||
}
|
||||
|
||||
let block_num_hash = BlockNumHash::new(header.number(), block_hash);
|
||||
|
||||
let mut all_logs = Vec::new();
|
||||
@@ -571,12 +565,6 @@ where
|
||||
let (from_block_number, to_block_number) =
|
||||
logs_utils::get_filter_block_range(from, to, start_block, info)?;
|
||||
|
||||
// Check if the requested range overlaps with pruned history (EIP-4444)
|
||||
let earliest_block = self.provider().earliest_block_number()?;
|
||||
if from_block_number < earliest_block {
|
||||
return Err(EthApiError::PrunedHistoryUnavailable.into());
|
||||
}
|
||||
|
||||
self.get_logs_in_block_range(filter, from_block_number, to_block_number, limits)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -316,7 +316,7 @@ impl<N: ProviderNodeTypes> Pipeline<N> {
|
||||
let _locked_sf_producer = self.static_file_producer.lock();
|
||||
|
||||
let mut provider_rw =
|
||||
self.provider_factory.unwind_provider_rw()?.disable_long_read_transaction_safety();
|
||||
self.provider_factory.database_provider_rw()?.disable_long_read_transaction_safety();
|
||||
|
||||
for stage in unwind_pipeline {
|
||||
let stage_id = stage.id();
|
||||
@@ -396,7 +396,7 @@ impl<N: ProviderNodeTypes> Pipeline<N> {
|
||||
|
||||
stage.post_unwind_commit()?;
|
||||
|
||||
provider_rw = self.provider_factory.unwind_provider_rw()?;
|
||||
provider_rw = self.provider_factory.database_provider_rw()?;
|
||||
}
|
||||
Err(err) => {
|
||||
self.event_sender.notify(PipelineEvent::Error { stage_id });
|
||||
|
||||
@@ -517,95 +517,6 @@ mod tests {
|
||||
assert!(table.is_empty());
|
||||
}
|
||||
|
||||
/// Tests exact shard boundary: exactly k * `NUM_OF_INDICES_IN_SHARD` entries.
|
||||
/// Verifies the final shard correctly uses `u64::MAX` as sentinel key when
|
||||
/// the entry count is an exact multiple of shard size.
|
||||
#[tokio::test]
|
||||
async fn insert_index_exact_shard_boundary() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..NUM_OF_INDICES_IN_SHARD as u64 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 1, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
run(&db, (NUM_OF_INDICES_IN_SHARD - 1) as u64, None);
|
||||
|
||||
let table = cast(db.table::<tables::AccountsHistory>().unwrap());
|
||||
let expected_blocks: Vec<u64> = (0..NUM_OF_INDICES_IN_SHARD as u64).collect();
|
||||
assert_eq!(table.len(), 1, "Should have exactly one shard");
|
||||
assert_eq!(
|
||||
table,
|
||||
BTreeMap::from([(shard(u64::MAX), expected_blocks)]),
|
||||
"Final shard key should be u64::MAX"
|
||||
);
|
||||
|
||||
unwind(&db, (NUM_OF_INDICES_IN_SHARD - 1) as u64, 0);
|
||||
|
||||
let table = cast(db.table::<tables::AccountsHistory>().unwrap());
|
||||
assert_eq!(table, BTreeMap::from([(shard(u64::MAX), vec![0])]));
|
||||
}
|
||||
|
||||
/// Tests incremental merge overflow: existing full shard gets converted
|
||||
/// from `u64::MAX` sentinel to actual highest block, and new entries
|
||||
/// create a new final shard with `u64::MAX`.
|
||||
#[tokio::test]
|
||||
async fn insert_index_incremental_merge_overflow() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
let first_shard_blocks: Vec<u64> = (0..NUM_OF_INDICES_IN_SHARD as u64).collect();
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..(NUM_OF_INDICES_IN_SHARD + 5) as u64 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 1, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
|
||||
tx.put::<tables::AccountsHistory>(shard(u64::MAX), list(&first_shard_blocks))?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let last_block = (NUM_OF_INDICES_IN_SHARD + 4) as u64;
|
||||
run(&db, last_block, Some((NUM_OF_INDICES_IN_SHARD - 1) as u64));
|
||||
|
||||
let table = cast(db.table::<tables::AccountsHistory>().unwrap());
|
||||
assert_eq!(table.len(), 2, "Should have two shards after overflow");
|
||||
|
||||
let new_shard_blocks: Vec<u64> =
|
||||
(NUM_OF_INDICES_IN_SHARD as u64..(NUM_OF_INDICES_IN_SHARD + 5) as u64).collect();
|
||||
|
||||
assert_eq!(
|
||||
table.get(&shard((NUM_OF_INDICES_IN_SHARD - 1) as u64)),
|
||||
Some(&first_shard_blocks),
|
||||
"First shard should have highest_block = last entry"
|
||||
);
|
||||
assert_eq!(
|
||||
table.get(&shard(u64::MAX)),
|
||||
Some(&new_shard_blocks),
|
||||
"New final shard should have u64::MAX key"
|
||||
);
|
||||
|
||||
unwind(&db, last_block, (NUM_OF_INDICES_IN_SHARD - 1) as u64);
|
||||
|
||||
let table = cast(db.table::<tables::AccountsHistory>().unwrap());
|
||||
assert_eq!(
|
||||
table,
|
||||
BTreeMap::from([(shard(u64::MAX), first_shard_blocks)]),
|
||||
"After unwind, should revert to single shard with u64::MAX"
|
||||
);
|
||||
}
|
||||
|
||||
stage_test_suite_ext!(IndexAccountHistoryTestRunner, index_account_history);
|
||||
|
||||
struct IndexAccountHistoryTestRunner {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use super::{collect_history_indices, collect_storage_history_indices};
|
||||
use super::collect_history_indices;
|
||||
use crate::{stages::utils::load_storage_history, StageCheckpoint, StageId};
|
||||
use reth_config::config::{EtlConfig, IndexHistoryConfig};
|
||||
use reth_db_api::{
|
||||
@@ -8,8 +8,7 @@ use reth_db_api::{
|
||||
};
|
||||
use reth_provider::{
|
||||
DBProvider, EitherWriter, HistoryWriter, PruneCheckpointReader, PruneCheckpointWriter,
|
||||
RocksDBProviderFactory, StaticFileProviderFactory, StorageChangeSetReader,
|
||||
StorageSettingsCache,
|
||||
RocksDBProviderFactory, StorageSettingsCache,
|
||||
};
|
||||
use reth_prune_types::{PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment};
|
||||
use reth_stages_api::{ExecInput, ExecOutput, Stage, StageError, UnwindInput, UnwindOutput};
|
||||
@@ -55,8 +54,6 @@ where
|
||||
+ PruneCheckpointWriter
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory
|
||||
+ StorageChangeSetReader
|
||||
+ StaticFileProviderFactory
|
||||
+ reth_provider::NodePrimitivesProvider,
|
||||
{
|
||||
/// Return the id of the stage
|
||||
@@ -124,9 +121,7 @@ where
|
||||
}
|
||||
|
||||
info!(target: "sync::stages::index_storage_history::exec", ?first_sync, ?use_rocksdb, "Collecting indices");
|
||||
let collector = if provider.cached_storage_settings().storage_changesets_in_static_files {
|
||||
collect_storage_history_indices(provider, range.clone(), &self.etl_config)?
|
||||
} else {
|
||||
let collector =
|
||||
collect_history_indices::<_, tables::StorageChangeSets, tables::StoragesHistory, _>(
|
||||
provider,
|
||||
BlockNumberAddress::range(range.clone()),
|
||||
@@ -135,8 +130,7 @@ where
|
||||
},
|
||||
|(key, value)| (key.block_number(), AddressStorageKey((key.address(), value.key))),
|
||||
&self.etl_config,
|
||||
)?
|
||||
};
|
||||
)?;
|
||||
|
||||
info!(target: "sync::stages::index_storage_history::exec", "Loading indices into database");
|
||||
|
||||
@@ -537,101 +531,6 @@ mod tests {
|
||||
assert!(table.is_empty());
|
||||
}
|
||||
|
||||
/// Tests exact shard boundary: exactly k * `NUM_OF_INDICES_IN_SHARD` entries.
|
||||
/// Verifies the final shard correctly uses `u64::MAX` as sentinel key when
|
||||
/// the entry count is an exact multiple of shard size.
|
||||
#[tokio::test]
|
||||
async fn insert_index_exact_shard_boundary() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..NUM_OF_INDICES_IN_SHARD as u64 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 1, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
run(&db, (NUM_OF_INDICES_IN_SHARD - 1) as u64, None);
|
||||
|
||||
let table = cast(db.table::<tables::StoragesHistory>().unwrap());
|
||||
let expected_blocks: Vec<u64> = (0..NUM_OF_INDICES_IN_SHARD as u64).collect();
|
||||
assert_eq!(table.len(), 1, "Should have exactly one shard");
|
||||
assert_eq!(
|
||||
table,
|
||||
BTreeMap::from([(shard(u64::MAX), expected_blocks)]),
|
||||
"Final shard key should be u64::MAX"
|
||||
);
|
||||
|
||||
unwind(&db, (NUM_OF_INDICES_IN_SHARD - 1) as u64, 0);
|
||||
|
||||
let table = cast(db.table::<tables::StoragesHistory>().unwrap());
|
||||
assert_eq!(table, BTreeMap::from([(shard(u64::MAX), vec![0])]));
|
||||
}
|
||||
|
||||
/// Tests incremental merge overflow: existing full shard gets converted
|
||||
/// from `u64::MAX` sentinel to actual highest block, and new entries
|
||||
/// create a new final shard with `u64::MAX`.
|
||||
#[tokio::test]
|
||||
async fn insert_index_incremental_merge_overflow() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
let first_shard_blocks: Vec<u64> = (0..NUM_OF_INDICES_IN_SHARD as u64).collect();
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..(NUM_OF_INDICES_IN_SHARD + 5) as u64 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 1, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
|
||||
tx.put::<tables::StoragesHistory>(shard(u64::MAX), list(&first_shard_blocks))?;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let last_block = (NUM_OF_INDICES_IN_SHARD + 4) as u64;
|
||||
run(&db, last_block, Some((NUM_OF_INDICES_IN_SHARD - 1) as u64));
|
||||
|
||||
let table = cast(db.table::<tables::StoragesHistory>().unwrap());
|
||||
assert_eq!(table.len(), 2, "Should have two shards after overflow");
|
||||
|
||||
let new_shard_blocks: Vec<u64> =
|
||||
(NUM_OF_INDICES_IN_SHARD as u64..(NUM_OF_INDICES_IN_SHARD + 5) as u64).collect();
|
||||
|
||||
assert_eq!(
|
||||
table.get(&shard((NUM_OF_INDICES_IN_SHARD - 1) as u64)),
|
||||
Some(&first_shard_blocks),
|
||||
"First shard should have highest_block = last entry"
|
||||
);
|
||||
assert_eq!(
|
||||
table.get(&shard(u64::MAX)),
|
||||
Some(&new_shard_blocks),
|
||||
"New final shard should have u64::MAX key"
|
||||
);
|
||||
|
||||
unwind(&db, last_block, (NUM_OF_INDICES_IN_SHARD - 1) as u64);
|
||||
|
||||
let table = cast(db.table::<tables::StoragesHistory>().unwrap());
|
||||
assert_eq!(
|
||||
table,
|
||||
BTreeMap::from([(shard(u64::MAX), first_shard_blocks)]),
|
||||
"After unwind, should revert to single shard with u64::MAX"
|
||||
);
|
||||
}
|
||||
|
||||
stage_test_suite_ext!(IndexStorageHistoryTestRunner, index_storage_history);
|
||||
|
||||
struct IndexStorageHistoryTestRunner {
|
||||
|
||||
@@ -9,7 +9,7 @@ use reth_db_api::{
|
||||
use reth_primitives_traits::{GotExpected, SealedHeader};
|
||||
use reth_provider::{
|
||||
ChangeSetReader, DBProvider, HeaderProvider, ProviderError, StageCheckpointReader,
|
||||
StageCheckpointWriter, StatsReader, StorageChangeSetReader, TrieWriter,
|
||||
StageCheckpointWriter, StatsReader, TrieWriter,
|
||||
};
|
||||
use reth_stages_api::{
|
||||
BlockErrorKind, EntitiesCheckpoint, ExecInput, ExecOutput, MerkleCheckpoint, Stage,
|
||||
@@ -159,7 +159,6 @@ where
|
||||
+ StatsReader
|
||||
+ HeaderProvider
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ StageCheckpointReader
|
||||
+ StageCheckpointWriter,
|
||||
{
|
||||
|
||||
@@ -6,7 +6,7 @@ use reth_primitives_traits::{GotExpected, SealedHeader};
|
||||
use reth_provider::{
|
||||
BlockNumReader, ChainStateBlockReader, ChangeSetReader, DBProvider, HeaderProvider,
|
||||
ProviderError, PruneCheckpointReader, PruneCheckpointWriter, StageCheckpointReader,
|
||||
StageCheckpointWriter, StorageChangeSetReader, TrieWriter,
|
||||
StageCheckpointWriter, TrieWriter,
|
||||
};
|
||||
use reth_prune_types::{
|
||||
PruneCheckpoint, PruneMode, PruneSegment, MERKLE_CHANGESETS_RETENTION_BLOCKS,
|
||||
@@ -167,8 +167,7 @@ impl MerkleChangeSets {
|
||||
+ HeaderProvider
|
||||
+ ChainStateBlockReader
|
||||
+ BlockNumReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader,
|
||||
+ ChangeSetReader,
|
||||
{
|
||||
let target_start = target_range.start;
|
||||
let target_end = target_range.end;
|
||||
@@ -309,7 +308,6 @@ where
|
||||
+ PruneCheckpointReader
|
||||
+ PruneCheckpointWriter
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader,
|
||||
{
|
||||
fn id(&self) -> StageId {
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
//! Utils for `stages`.
|
||||
use alloy_primitives::{Address, BlockNumber, TxNumber};
|
||||
use alloy_primitives::{Address, BlockNumber, TxNumber, B256};
|
||||
use reth_config::config::EtlConfig;
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbCursorRW},
|
||||
models::{
|
||||
sharded_key::NUM_OF_INDICES_IN_SHARD, storage_sharded_key::StorageShardedKey,
|
||||
AccountBeforeTx, AddressStorageKey, BlockNumberAddress, ShardedHistoryKey, ShardedKey,
|
||||
AccountBeforeTx, ShardedKey,
|
||||
},
|
||||
table::{Decode, Decompress, Key, Table},
|
||||
table::{Decode, Decompress, Table},
|
||||
transaction::DbTx,
|
||||
BlockNumberList,
|
||||
};
|
||||
@@ -15,177 +15,14 @@ use reth_etl::Collector;
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_provider::{
|
||||
providers::StaticFileProvider, to_range, BlockReader, DBProvider, EitherWriter, ProviderError,
|
||||
ProviderResult, StaticFileProviderFactory,
|
||||
StaticFileProviderFactory,
|
||||
};
|
||||
use reth_stages_api::StageError;
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader};
|
||||
use reth_storage_api::ChangeSetReader;
|
||||
use std::{collections::HashMap, hash::Hash, ops::RangeBounds};
|
||||
use tracing::info;
|
||||
|
||||
/// Trait for writing sharded history indices to the database.
|
||||
pub(crate) trait HistoryShardWriter {
|
||||
/// The full sharded key type for the table.
|
||||
type TableKey: Key + ShardedHistoryKey;
|
||||
|
||||
/// Gets the last shard for a prefix (for incremental sync merging).
|
||||
fn get_last_shard(
|
||||
&mut self,
|
||||
prefix: <Self::TableKey as ShardedHistoryKey>::Prefix,
|
||||
) -> ProviderResult<Option<BlockNumberList>>;
|
||||
|
||||
/// Writes a shard to the database (append or upsert based on flag).
|
||||
fn write_shard(
|
||||
&mut self,
|
||||
key: Self::TableKey,
|
||||
value: &BlockNumberList,
|
||||
append: bool,
|
||||
) -> ProviderResult<()>;
|
||||
}
|
||||
|
||||
/// Loads sharded history indices from a collector into the database.
|
||||
///
|
||||
/// ## Why sharding?
|
||||
/// History indices track "which blocks modified this address/storage slot". A popular contract
|
||||
/// may have millions of changes, too large for a single DB value. We split into shards of
|
||||
/// `NUM_OF_INDICES_IN_SHARD` (2000) block numbers each.
|
||||
///
|
||||
/// ## Key structure
|
||||
/// Each shard is keyed by `(prefix, highest_block_in_shard)`. Example for an address:
|
||||
/// - `(0xABC..., 5000)` → blocks 3001-5000
|
||||
/// - `(0xABC..., u64::MAX)` → blocks 5001-6234 (final shard)
|
||||
///
|
||||
/// The `u64::MAX` sentinel on the last shard enables `seek_exact(prefix, u64::MAX)` to find
|
||||
/// it for incremental sync merging.
|
||||
///
|
||||
/// When `append_only=true`, collector must yield keys in ascending order (MDBX requirement).
|
||||
fn load_sharded_history<H: HistoryShardWriter>(
|
||||
collector: &mut Collector<H::TableKey, BlockNumberList>,
|
||||
append_only: bool,
|
||||
writer: &mut H,
|
||||
) -> Result<(), StageError> {
|
||||
type Prefix<H> = <<H as HistoryShardWriter>::TableKey as ShardedHistoryKey>::Prefix;
|
||||
|
||||
// Option needed to distinguish "no prefix yet" from "processing Address::ZERO"
|
||||
let mut current_prefix: Option<Prefix<H>> = None;
|
||||
// Buffer for block numbers; sized for ~2 shards to minimize reallocations
|
||||
let mut current_list = Vec::<u64>::with_capacity(NUM_OF_INDICES_IN_SHARD * 2);
|
||||
|
||||
// Progress reporting setup
|
||||
let total_entries = collector.len();
|
||||
let interval = (total_entries / 10).max(1);
|
||||
|
||||
for (index, element) in collector.iter()?.enumerate() {
|
||||
let (k, v) = element?;
|
||||
let sharded_key = H::TableKey::decode_owned(k)?;
|
||||
let new_list = BlockNumberList::decompress_owned(v)?;
|
||||
|
||||
if index > 0 && index.is_multiple_of(interval) && total_entries > 10 {
|
||||
info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices");
|
||||
}
|
||||
|
||||
let prefix = sharded_key.prefix();
|
||||
|
||||
// When prefix changes, flush previous prefix's shards and start fresh
|
||||
if current_prefix != Some(prefix) {
|
||||
// Flush remaining shards for the previous prefix (uses u64::MAX for final shard)
|
||||
if let Some(prev_prefix) = current_prefix {
|
||||
flush_shards::<H>(prev_prefix, &mut current_list, append_only, writer)?;
|
||||
}
|
||||
|
||||
current_prefix = Some(prefix);
|
||||
current_list.clear();
|
||||
|
||||
// On incremental sync, merge with existing last shard (stored with u64::MAX key)
|
||||
if !append_only && let Some(last_shard) = writer.get_last_shard(prefix)? {
|
||||
current_list.extend(last_shard.iter());
|
||||
}
|
||||
}
|
||||
|
||||
// Accumulate new block numbers
|
||||
current_list.extend(new_list.iter());
|
||||
// Flush complete shards while keeping one buffered for continued accumulation
|
||||
flush_shards_partial::<H>(prefix, &mut current_list, append_only, writer)?;
|
||||
}
|
||||
|
||||
// Flush final prefix's remaining shard
|
||||
if let Some(prefix) = current_prefix {
|
||||
flush_shards::<H>(prefix, &mut current_list, append_only, writer)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flushes complete shards, keeping at least one shard buffered for continued accumulation.
|
||||
///
|
||||
/// We buffer one shard because `flush_shards` uses `u64::MAX` as the final shard's key.
|
||||
/// If we flushed everything here, we'd write `u64::MAX` keys that get overwritten later.
|
||||
fn flush_shards_partial<H: HistoryShardWriter>(
|
||||
prefix: <H::TableKey as ShardedHistoryKey>::Prefix,
|
||||
list: &mut Vec<u64>,
|
||||
append_only: bool,
|
||||
writer: &mut H,
|
||||
) -> Result<(), StageError> {
|
||||
// Not enough to fill a shard yet
|
||||
if list.len() <= NUM_OF_INDICES_IN_SHARD {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let num_full_shards = list.len() / NUM_OF_INDICES_IN_SHARD;
|
||||
// Keep one shard buffered: if exact multiple, keep last full shard for u64::MAX key later
|
||||
let shards_to_flush = if list.len().is_multiple_of(NUM_OF_INDICES_IN_SHARD) {
|
||||
num_full_shards - 1
|
||||
} else {
|
||||
num_full_shards
|
||||
};
|
||||
|
||||
if shards_to_flush == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let flush_len = shards_to_flush * NUM_OF_INDICES_IN_SHARD;
|
||||
debug_assert!(flush_len <= list.len(), "flush_len exceeds list length");
|
||||
|
||||
// Write complete shards with their actual highest block number as key
|
||||
for chunk in list[..flush_len].chunks(NUM_OF_INDICES_IN_SHARD) {
|
||||
let highest = *chunk.last().expect("chunk is non-empty");
|
||||
let key = H::TableKey::new_sharded(prefix, highest);
|
||||
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
|
||||
writer.write_shard(key, &value, append_only)?;
|
||||
}
|
||||
|
||||
// Shift remaining elements to front (avoids allocation vs split_off)
|
||||
list.copy_within(flush_len.., 0);
|
||||
list.truncate(list.len() - flush_len);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flushes all remaining shards. Uses `u64::MAX` for the final shard's key to enable
|
||||
/// incremental sync lookups via `seek_exact(prefix, u64::MAX)`.
|
||||
fn flush_shards<H: HistoryShardWriter>(
|
||||
prefix: <H::TableKey as ShardedHistoryKey>::Prefix,
|
||||
list: &mut Vec<u64>,
|
||||
append_only: bool,
|
||||
writer: &mut H,
|
||||
) -> Result<(), StageError> {
|
||||
if list.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let num_chunks = list.len().div_ceil(NUM_OF_INDICES_IN_SHARD);
|
||||
|
||||
for (i, chunk) in list.chunks(NUM_OF_INDICES_IN_SHARD).enumerate() {
|
||||
let is_last = i == num_chunks - 1;
|
||||
let highest = if is_last { u64::MAX } else { *chunk.last().expect("chunk is non-empty") };
|
||||
let key = H::TableKey::new_sharded(prefix, highest);
|
||||
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
|
||||
writer.write_shard(key, &value, append_only)?;
|
||||
}
|
||||
|
||||
list.clear();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Number of blocks before pushing indices from cache to [`Collector`]
|
||||
const DEFAULT_CACHE_THRESHOLD: u64 = 100_000;
|
||||
|
||||
@@ -265,15 +102,15 @@ where
|
||||
}
|
||||
|
||||
/// Allows collecting indices from a cache with a custom insert fn
|
||||
fn collect_indices<K, F>(
|
||||
cache: impl Iterator<Item = (K, Vec<u64>)>,
|
||||
fn collect_indices<F>(
|
||||
cache: impl Iterator<Item = (Address, Vec<u64>)>,
|
||||
mut insert_fn: F,
|
||||
) -> Result<(), StageError>
|
||||
where
|
||||
F: FnMut(K, Vec<u64>) -> Result<(), StageError>,
|
||||
F: FnMut(Address, Vec<u64>) -> Result<(), StageError>,
|
||||
{
|
||||
for (key, indices) in cache {
|
||||
insert_fn(key, indices)?
|
||||
for (address, indices) in cache {
|
||||
insert_fn(address, indices)?
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -337,96 +174,17 @@ where
|
||||
Ok(collector)
|
||||
}
|
||||
|
||||
/// Collects storage history indices using a provider that implements `StorageChangeSetReader`.
|
||||
pub(crate) fn collect_storage_history_indices<Provider>(
|
||||
provider: &Provider,
|
||||
range: impl RangeBounds<BlockNumber>,
|
||||
etl_config: &EtlConfig,
|
||||
) -> Result<Collector<StorageShardedKey, BlockNumberList>, StageError>
|
||||
where
|
||||
Provider: DBProvider + StorageChangeSetReader + StaticFileProviderFactory,
|
||||
{
|
||||
let mut collector = Collector::new(etl_config.file_size, etl_config.dir.clone());
|
||||
let mut cache: HashMap<AddressStorageKey, Vec<u64>> = HashMap::default();
|
||||
|
||||
let mut insert_fn = |key: AddressStorageKey, indices: Vec<u64>| {
|
||||
let last = indices.last().expect("qed");
|
||||
collector.insert(
|
||||
StorageShardedKey::new(key.0 .0, key.0 .1, *last),
|
||||
BlockNumberList::new_pre_sorted(indices.into_iter()),
|
||||
)?;
|
||||
Ok::<(), StageError>(())
|
||||
};
|
||||
|
||||
let range = to_range(range);
|
||||
let static_file_provider = provider.static_file_provider();
|
||||
|
||||
let total_changesets = static_file_provider.storage_changeset_count()?;
|
||||
let interval = (total_changesets / 1000).max(1);
|
||||
|
||||
let walker = static_file_provider.walk_storage_changeset_range(range);
|
||||
|
||||
let mut flush_counter = 0;
|
||||
let mut current_block_number = u64::MAX;
|
||||
|
||||
for (idx, changeset_result) in walker.enumerate() {
|
||||
let (BlockNumberAddress((block_number, address)), storage) = changeset_result?;
|
||||
cache.entry(AddressStorageKey((address, storage.key))).or_default().push(block_number);
|
||||
|
||||
if idx > 0 && idx % interval == 0 && total_changesets > 1000 {
|
||||
info!(target: "sync::stages::index_history", progress = %format!("{:.4}%", (idx as f64 / total_changesets as f64) * 100.0), "Collecting indices");
|
||||
}
|
||||
|
||||
if block_number != current_block_number {
|
||||
current_block_number = block_number;
|
||||
flush_counter += 1;
|
||||
}
|
||||
|
||||
if flush_counter > DEFAULT_CACHE_THRESHOLD {
|
||||
collect_indices(cache.drain(), &mut insert_fn)?;
|
||||
flush_counter = 0;
|
||||
}
|
||||
}
|
||||
|
||||
collect_indices(cache.into_iter(), insert_fn)?;
|
||||
|
||||
Ok(collector)
|
||||
}
|
||||
|
||||
/// Adapter for writing account history shards via `EitherWriter`.
|
||||
struct AccountHistoryShardWriter<'a, 'tx, CURSOR, N> {
|
||||
writer: &'a mut EitherWriter<'tx, CURSOR, N>,
|
||||
}
|
||||
|
||||
impl<CURSOR, N: NodePrimitives> HistoryShardWriter for AccountHistoryShardWriter<'_, '_, CURSOR, N>
|
||||
where
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::AccountsHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::AccountsHistory>,
|
||||
{
|
||||
type TableKey = ShardedKey<Address>;
|
||||
|
||||
fn get_last_shard(
|
||||
&mut self,
|
||||
prefix: <Self::TableKey as ShardedHistoryKey>::Prefix,
|
||||
) -> ProviderResult<Option<BlockNumberList>> {
|
||||
self.writer.get_last_account_history_shard(prefix)
|
||||
}
|
||||
|
||||
fn write_shard(
|
||||
&mut self,
|
||||
key: Self::TableKey,
|
||||
value: &BlockNumberList,
|
||||
append: bool,
|
||||
) -> ProviderResult<()> {
|
||||
if append {
|
||||
self.writer.append_account_history(key, value)
|
||||
} else {
|
||||
self.writer.upsert_account_history(key, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Loads account history indices from the collector into the database.
|
||||
/// Loads account history indices into the database via `EitherWriter`.
|
||||
///
|
||||
/// Works with [`EitherWriter`] to support both MDBX and `RocksDB` backends.
|
||||
///
|
||||
/// ## Process
|
||||
/// Iterates over elements, grouping indices by their address. It flushes indices to disk
|
||||
/// when reaching a shard's max length (`NUM_OF_INDICES_IN_SHARD`) or when the address changes,
|
||||
/// ensuring the last previous address shard is stored.
|
||||
///
|
||||
/// Uses `Option<Address>` instead of `Address::default()` as the sentinel to avoid
|
||||
/// incorrectly treating `Address::ZERO` as "no previous address".
|
||||
pub(crate) fn load_account_history<N, CURSOR>(
|
||||
mut collector: Collector<ShardedKey<Address>, BlockNumberList>,
|
||||
append_only: bool,
|
||||
@@ -437,8 +195,155 @@ where
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::AccountsHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::AccountsHistory>,
|
||||
{
|
||||
let mut adapter = AccountHistoryShardWriter { writer };
|
||||
load_sharded_history(&mut collector, append_only, &mut adapter)
|
||||
let mut current_address: Option<Address> = None;
|
||||
// Accumulator for block numbers where the current address changed.
|
||||
let mut current_list = Vec::<u64>::new();
|
||||
|
||||
let total_entries = collector.len();
|
||||
let interval = (total_entries / 10).max(1);
|
||||
|
||||
for (index, element) in collector.iter()?.enumerate() {
|
||||
let (k, v) = element?;
|
||||
let sharded_key = ShardedKey::<Address>::decode_owned(k)?;
|
||||
let new_list = BlockNumberList::decompress_owned(v)?;
|
||||
|
||||
if index > 0 && index.is_multiple_of(interval) && total_entries > 10 {
|
||||
info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices");
|
||||
}
|
||||
|
||||
let address = sharded_key.key;
|
||||
|
||||
// When address changes, flush the previous address's shards and start fresh.
|
||||
if current_address != Some(address) {
|
||||
// Flush all remaining shards for the previous address (uses u64::MAX for last shard).
|
||||
if let Some(prev_addr) = current_address {
|
||||
flush_account_history_shards(prev_addr, &mut current_list, append_only, writer)?;
|
||||
}
|
||||
|
||||
current_address = Some(address);
|
||||
current_list.clear();
|
||||
|
||||
// On incremental sync, merge with the existing last shard from the database.
|
||||
// The last shard is stored with key (address, u64::MAX) so we can find it.
|
||||
if !append_only &&
|
||||
let Some(last_shard) = writer.get_last_account_history_shard(address)?
|
||||
{
|
||||
current_list.extend(last_shard.iter());
|
||||
}
|
||||
}
|
||||
|
||||
// Append new block numbers to the accumulator.
|
||||
current_list.extend(new_list.iter());
|
||||
|
||||
// Flush complete shards, keeping the last (partial) shard buffered.
|
||||
flush_account_history_shards_partial(address, &mut current_list, append_only, writer)?;
|
||||
}
|
||||
|
||||
// Flush the final address's remaining shard.
|
||||
if let Some(addr) = current_address {
|
||||
flush_account_history_shards(addr, &mut current_list, append_only, writer)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flushes complete shards for account history, keeping the trailing partial shard buffered.
|
||||
///
|
||||
/// Only flushes when we have more than one shard's worth of data, keeping the last
|
||||
/// (possibly partial) shard for continued accumulation. This avoids writing a shard
|
||||
/// that may need to be updated when more indices arrive.
|
||||
fn flush_account_history_shards_partial<N, CURSOR>(
|
||||
address: Address,
|
||||
list: &mut Vec<u64>,
|
||||
append_only: bool,
|
||||
writer: &mut EitherWriter<'_, CURSOR, N>,
|
||||
) -> Result<(), StageError>
|
||||
where
|
||||
N: NodePrimitives,
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::AccountsHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::AccountsHistory>,
|
||||
{
|
||||
// Nothing to flush if we haven't filled a complete shard yet.
|
||||
if list.len() <= NUM_OF_INDICES_IN_SHARD {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let num_full_shards = list.len() / NUM_OF_INDICES_IN_SHARD;
|
||||
|
||||
// Always keep at least one shard buffered for continued accumulation.
|
||||
// If len is exact multiple of shard size, keep the last full shard.
|
||||
let shards_to_flush = if list.len().is_multiple_of(NUM_OF_INDICES_IN_SHARD) {
|
||||
num_full_shards - 1
|
||||
} else {
|
||||
num_full_shards
|
||||
};
|
||||
|
||||
if shards_to_flush == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Split: flush the first N shards, keep the remainder buffered.
|
||||
let flush_len = shards_to_flush * NUM_OF_INDICES_IN_SHARD;
|
||||
let remainder = list.split_off(flush_len);
|
||||
|
||||
// Write each complete shard with its highest block number as the key.
|
||||
for chunk in list.chunks(NUM_OF_INDICES_IN_SHARD) {
|
||||
let highest = *chunk.last().expect("chunk is non-empty");
|
||||
let key = ShardedKey::new(address, highest);
|
||||
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
|
||||
|
||||
if append_only {
|
||||
writer.append_account_history(key, &value)?;
|
||||
} else {
|
||||
writer.upsert_account_history(key, &value)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the remaining indices for the next iteration.
|
||||
*list = remainder;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flushes all remaining shards for account history, using `u64::MAX` for the last shard.
|
||||
///
|
||||
/// The `u64::MAX` key for the final shard is an invariant that allows `seek_exact(address,
|
||||
/// u64::MAX)` to find the last shard during incremental sync for merging with new indices.
|
||||
fn flush_account_history_shards<N, CURSOR>(
|
||||
address: Address,
|
||||
list: &mut Vec<u64>,
|
||||
append_only: bool,
|
||||
writer: &mut EitherWriter<'_, CURSOR, N>,
|
||||
) -> Result<(), StageError>
|
||||
where
|
||||
N: NodePrimitives,
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::AccountsHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::AccountsHistory>,
|
||||
{
|
||||
if list.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let num_chunks = list.len().div_ceil(NUM_OF_INDICES_IN_SHARD);
|
||||
|
||||
for (i, chunk) in list.chunks(NUM_OF_INDICES_IN_SHARD).enumerate() {
|
||||
let is_last = i == num_chunks - 1;
|
||||
|
||||
// Use u64::MAX for the final shard's key. This invariant allows incremental sync
|
||||
// to find the last shard via seek_exact(address, u64::MAX) for merging.
|
||||
let highest = if is_last { u64::MAX } else { *chunk.last().expect("chunk is non-empty") };
|
||||
|
||||
let key = ShardedKey::new(address, highest);
|
||||
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
|
||||
|
||||
if append_only {
|
||||
writer.append_account_history(key, &value)?;
|
||||
} else {
|
||||
writer.upsert_account_history(key, &value)?;
|
||||
}
|
||||
}
|
||||
|
||||
list.clear();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Called when database is ahead of static files. Attempts to find the first block we are missing
|
||||
@@ -477,40 +382,17 @@ where
|
||||
})
|
||||
}
|
||||
|
||||
/// Adapter for writing storage history shards via `EitherWriter`.
|
||||
struct StorageHistoryShardWriter<'a, 'tx, CURSOR, N> {
|
||||
writer: &'a mut EitherWriter<'tx, CURSOR, N>,
|
||||
}
|
||||
|
||||
impl<CURSOR, N: NodePrimitives> HistoryShardWriter for StorageHistoryShardWriter<'_, '_, CURSOR, N>
|
||||
where
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::StoragesHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::StoragesHistory>,
|
||||
{
|
||||
type TableKey = StorageShardedKey;
|
||||
|
||||
fn get_last_shard(
|
||||
&mut self,
|
||||
prefix: <Self::TableKey as ShardedHistoryKey>::Prefix,
|
||||
) -> ProviderResult<Option<BlockNumberList>> {
|
||||
self.writer.get_last_storage_history_shard(prefix.0, prefix.1)
|
||||
}
|
||||
|
||||
fn write_shard(
|
||||
&mut self,
|
||||
key: Self::TableKey,
|
||||
value: &BlockNumberList,
|
||||
append: bool,
|
||||
) -> ProviderResult<()> {
|
||||
if append {
|
||||
self.writer.append_storage_history(key, value)
|
||||
} else {
|
||||
self.writer.upsert_storage_history(key, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Loads storage history indices from the collector into the database.
|
||||
/// Loads storage history indices into the database via `EitherWriter`.
|
||||
///
|
||||
/// Works with [`EitherWriter`] to support both MDBX and `RocksDB` backends.
|
||||
///
|
||||
/// ## Process
|
||||
/// Iterates over elements, grouping indices by their (address, `storage_key`) pairs. It flushes
|
||||
/// indices to disk when reaching a shard's max length (`NUM_OF_INDICES_IN_SHARD`) or when the
|
||||
/// (address, `storage_key`) pair changes, ensuring the last previous shard is stored.
|
||||
///
|
||||
/// Uses `Option<(Address, B256)>` instead of default values as the sentinel to avoid
|
||||
/// incorrectly treating `(Address::ZERO, B256::ZERO)` as "no previous key".
|
||||
pub(crate) fn load_storage_history<N, CURSOR>(
|
||||
mut collector: Collector<StorageShardedKey, BlockNumberList>,
|
||||
append_only: bool,
|
||||
@@ -521,6 +403,169 @@ where
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::StoragesHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::StoragesHistory>,
|
||||
{
|
||||
let mut adapter = StorageHistoryShardWriter { writer };
|
||||
load_sharded_history(&mut collector, append_only, &mut adapter)
|
||||
let mut current_key: Option<(Address, B256)> = None;
|
||||
// Accumulator for block numbers where the current (address, storage_key) changed.
|
||||
let mut current_list = Vec::<u64>::new();
|
||||
|
||||
let total_entries = collector.len();
|
||||
let interval = (total_entries / 10).max(1);
|
||||
|
||||
for (index, element) in collector.iter()?.enumerate() {
|
||||
let (k, v) = element?;
|
||||
let sharded_key = StorageShardedKey::decode_owned(k)?;
|
||||
let new_list = BlockNumberList::decompress_owned(v)?;
|
||||
|
||||
if index > 0 && index.is_multiple_of(interval) && total_entries > 10 {
|
||||
info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices");
|
||||
}
|
||||
|
||||
let partial_key = (sharded_key.address, sharded_key.sharded_key.key);
|
||||
|
||||
// When (address, storage_key) changes, flush the previous key's shards and start fresh.
|
||||
if current_key != Some(partial_key) {
|
||||
// Flush all remaining shards for the previous key (uses u64::MAX for last shard).
|
||||
if let Some((prev_addr, prev_storage_key)) = current_key {
|
||||
flush_storage_history_shards(
|
||||
prev_addr,
|
||||
prev_storage_key,
|
||||
&mut current_list,
|
||||
append_only,
|
||||
writer,
|
||||
)?;
|
||||
}
|
||||
|
||||
current_key = Some(partial_key);
|
||||
current_list.clear();
|
||||
|
||||
// On incremental sync, merge with the existing last shard from the database.
|
||||
// The last shard is stored with key (address, storage_key, u64::MAX) so we can find it.
|
||||
if !append_only &&
|
||||
let Some(last_shard) =
|
||||
writer.get_last_storage_history_shard(partial_key.0, partial_key.1)?
|
||||
{
|
||||
current_list.extend(last_shard.iter());
|
||||
}
|
||||
}
|
||||
|
||||
// Append new block numbers to the accumulator.
|
||||
current_list.extend(new_list.iter());
|
||||
|
||||
// Flush complete shards, keeping the last (partial) shard buffered.
|
||||
flush_storage_history_shards_partial(
|
||||
partial_key.0,
|
||||
partial_key.1,
|
||||
&mut current_list,
|
||||
append_only,
|
||||
writer,
|
||||
)?;
|
||||
}
|
||||
|
||||
// Flush the final key's remaining shard.
|
||||
if let Some((addr, storage_key)) = current_key {
|
||||
flush_storage_history_shards(addr, storage_key, &mut current_list, append_only, writer)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flushes complete shards for storage history, keeping the trailing partial shard buffered.
|
||||
///
|
||||
/// Only flushes when we have more than one shard's worth of data, keeping the last
|
||||
/// (possibly partial) shard for continued accumulation. This avoids writing a shard
|
||||
/// that may need to be updated when more indices arrive.
|
||||
fn flush_storage_history_shards_partial<N, CURSOR>(
|
||||
address: Address,
|
||||
storage_key: B256,
|
||||
list: &mut Vec<u64>,
|
||||
append_only: bool,
|
||||
writer: &mut EitherWriter<'_, CURSOR, N>,
|
||||
) -> Result<(), StageError>
|
||||
where
|
||||
N: NodePrimitives,
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::StoragesHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::StoragesHistory>,
|
||||
{
|
||||
// Nothing to flush if we haven't filled a complete shard yet.
|
||||
if list.len() <= NUM_OF_INDICES_IN_SHARD {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let num_full_shards = list.len() / NUM_OF_INDICES_IN_SHARD;
|
||||
|
||||
// Always keep at least one shard buffered for continued accumulation.
|
||||
// If len is exact multiple of shard size, keep the last full shard.
|
||||
let shards_to_flush = if list.len().is_multiple_of(NUM_OF_INDICES_IN_SHARD) {
|
||||
num_full_shards - 1
|
||||
} else {
|
||||
num_full_shards
|
||||
};
|
||||
|
||||
if shards_to_flush == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Split: flush the first N shards, keep the remainder buffered.
|
||||
let flush_len = shards_to_flush * NUM_OF_INDICES_IN_SHARD;
|
||||
let remainder = list.split_off(flush_len);
|
||||
|
||||
// Write each complete shard with its highest block number as the key.
|
||||
for chunk in list.chunks(NUM_OF_INDICES_IN_SHARD) {
|
||||
let highest = *chunk.last().expect("chunk is non-empty");
|
||||
let key = StorageShardedKey::new(address, storage_key, highest);
|
||||
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
|
||||
|
||||
if append_only {
|
||||
writer.append_storage_history(key, &value)?;
|
||||
} else {
|
||||
writer.upsert_storage_history(key, &value)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the remaining indices for the next iteration.
|
||||
*list = remainder;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flushes all remaining shards for storage history, using `u64::MAX` for the last shard.
|
||||
///
|
||||
/// The `u64::MAX` key for the final shard is an invariant that allows
|
||||
/// `seek_exact(address, storage_key, u64::MAX)` to find the last shard during incremental
|
||||
/// sync for merging with new indices.
|
||||
fn flush_storage_history_shards<N, CURSOR>(
|
||||
address: Address,
|
||||
storage_key: B256,
|
||||
list: &mut Vec<u64>,
|
||||
append_only: bool,
|
||||
writer: &mut EitherWriter<'_, CURSOR, N>,
|
||||
) -> Result<(), StageError>
|
||||
where
|
||||
N: NodePrimitives,
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::StoragesHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::StoragesHistory>,
|
||||
{
|
||||
if list.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let num_chunks = list.len().div_ceil(NUM_OF_INDICES_IN_SHARD);
|
||||
|
||||
for (i, chunk) in list.chunks(NUM_OF_INDICES_IN_SHARD).enumerate() {
|
||||
let is_last = i == num_chunks - 1;
|
||||
|
||||
// Use u64::MAX for the final shard's key. This invariant allows incremental sync
|
||||
// to find the last shard via seek_exact(address, storage_key, u64::MAX) for merging.
|
||||
let highest = if is_last { u64::MAX } else { *chunk.last().expect("chunk is non-empty") };
|
||||
|
||||
let key = StorageShardedKey::new(address, storage_key, highest);
|
||||
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
|
||||
|
||||
if append_only {
|
||||
writer.append_storage_history(key, &value)?;
|
||||
} else {
|
||||
writer.upsert_storage_history(key, &value)?;
|
||||
}
|
||||
}
|
||||
|
||||
list.clear();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -55,11 +55,6 @@ pub enum StaticFileSegment {
|
||||
/// * address 0xbb, account info
|
||||
/// * address 0xcc, account info
|
||||
AccountChangeSets,
|
||||
/// Static File segment responsible for the `StorageChangeSets` table.
|
||||
///
|
||||
/// Storage changeset static files append block-by-block changesets sorted by address and
|
||||
/// storage slot.
|
||||
StorageChangeSets,
|
||||
}
|
||||
|
||||
impl StaticFileSegment {
|
||||
@@ -76,7 +71,6 @@ impl StaticFileSegment {
|
||||
Self::Receipts => "receipts",
|
||||
Self::TransactionSenders => "transaction-senders",
|
||||
Self::AccountChangeSets => "account-change-sets",
|
||||
Self::StorageChangeSets => "storage-change-sets",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,7 +83,6 @@ impl StaticFileSegment {
|
||||
Self::Receipts,
|
||||
Self::TransactionSenders,
|
||||
Self::AccountChangeSets,
|
||||
Self::StorageChangeSets,
|
||||
]
|
||||
.into_iter()
|
||||
}
|
||||
@@ -106,8 +99,7 @@ impl StaticFileSegment {
|
||||
Self::Transactions |
|
||||
Self::Receipts |
|
||||
Self::TransactionSenders |
|
||||
Self::AccountChangeSets |
|
||||
Self::StorageChangeSets => 1,
|
||||
Self::AccountChangeSets => 1,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,14 +161,14 @@ impl StaticFileSegment {
|
||||
pub const fn is_tx_based(&self) -> bool {
|
||||
match self {
|
||||
Self::Receipts | Self::Transactions | Self::TransactionSenders => true,
|
||||
Self::Headers | Self::AccountChangeSets | Self::StorageChangeSets => false,
|
||||
Self::Headers | Self::AccountChangeSets => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `true` if the segment is change-based.
|
||||
/// Returns `true` if the segment is [`StaticFileSegment::AccountChangeSets`]
|
||||
pub const fn is_change_based(&self) -> bool {
|
||||
match self {
|
||||
Self::AccountChangeSets | Self::StorageChangeSets => true,
|
||||
Self::AccountChangeSets => true,
|
||||
Self::Receipts | Self::Transactions | Self::Headers | Self::TransactionSenders => false,
|
||||
}
|
||||
}
|
||||
@@ -188,8 +180,7 @@ impl StaticFileSegment {
|
||||
Self::Receipts |
|
||||
Self::Transactions |
|
||||
Self::TransactionSenders |
|
||||
Self::AccountChangeSets |
|
||||
Self::StorageChangeSets => false,
|
||||
Self::AccountChangeSets => false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -268,10 +259,10 @@ impl<'de> Visitor<'de> for SegmentHeaderVisitor {
|
||||
let tx_range =
|
||||
seq.next_element()?.ok_or_else(|| serde::de::Error::invalid_length(2, &self))?;
|
||||
|
||||
let segment: StaticFileSegment =
|
||||
let segment =
|
||||
seq.next_element()?.ok_or_else(|| serde::de::Error::invalid_length(3, &self))?;
|
||||
|
||||
let changeset_offsets = if segment.is_change_based() {
|
||||
let changeset_offsets = if segment == StaticFileSegment::AccountChangeSets {
|
||||
// Try to read the 5th field (changeset_offsets)
|
||||
// If it doesn't exist (old format), this will return None
|
||||
match seq.next_element()? {
|
||||
@@ -318,8 +309,8 @@ impl Serialize for SegmentHeader {
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
// We serialize an extra field, the changeset offsets, for change-based segments
|
||||
let len = if self.segment.is_change_based() { 5 } else { 4 };
|
||||
// We serialize an extra field, the changeset offsets, for account changesets
|
||||
let len = if self.segment.is_account_change_sets() { 5 } else { 4 };
|
||||
|
||||
let mut state = serializer.serialize_struct("SegmentHeader", len)?;
|
||||
state.serialize_field("expected_block_range", &self.expected_block_range)?;
|
||||
@@ -327,7 +318,7 @@ impl Serialize for SegmentHeader {
|
||||
state.serialize_field("tx_range", &self.tx_range)?;
|
||||
state.serialize_field("segment", &self.segment)?;
|
||||
|
||||
if self.segment.is_change_based() {
|
||||
if self.segment.is_account_change_sets() {
|
||||
state.serialize_field("changeset_offsets", &self.changeset_offsets)?;
|
||||
}
|
||||
|
||||
@@ -681,12 +672,6 @@ mod tests {
|
||||
"static_file_account-change-sets_1123233_11223233",
|
||||
None,
|
||||
),
|
||||
(
|
||||
StaticFileSegment::StorageChangeSets,
|
||||
1_123_233..=11_223_233,
|
||||
"static_file_storage-change-sets_1123233_11223233",
|
||||
None,
|
||||
),
|
||||
(
|
||||
StaticFileSegment::Headers,
|
||||
2..=30,
|
||||
@@ -770,13 +755,6 @@ mod tests {
|
||||
segment: StaticFileSegment::AccountChangeSets,
|
||||
changeset_offsets: Some(vec![ChangesetOffset { offset: 1, num_changes: 1 }; 100]),
|
||||
},
|
||||
SegmentHeader {
|
||||
expected_block_range: SegmentRangeInclusive::new(0, 200),
|
||||
block_range: Some(SegmentRangeInclusive::new(0, 100)),
|
||||
tx_range: None,
|
||||
segment: StaticFileSegment::StorageChangeSets,
|
||||
changeset_offsets: Some(vec![ChangesetOffset { offset: 1, num_changes: 1 }; 100]),
|
||||
},
|
||||
];
|
||||
// Check that we test all segments
|
||||
assert_eq!(
|
||||
@@ -810,7 +788,6 @@ mod tests {
|
||||
StaticFileSegment::Receipts => "receipts",
|
||||
StaticFileSegment::TransactionSenders => "transaction-senders",
|
||||
StaticFileSegment::AccountChangeSets => "account-change-sets",
|
||||
StaticFileSegment::StorageChangeSets => "storage-change-sets",
|
||||
};
|
||||
assert_eq!(static_str, expected_str);
|
||||
}
|
||||
@@ -829,7 +806,6 @@ mod tests {
|
||||
StaticFileSegment::Receipts => "Receipts",
|
||||
StaticFileSegment::TransactionSenders => "TransactionSenders",
|
||||
StaticFileSegment::AccountChangeSets => "AccountChangeSets",
|
||||
StaticFileSegment::StorageChangeSets => "StorageChangeSets",
|
||||
};
|
||||
assert_eq!(ser, format!("\"{expected_str}\""));
|
||||
}
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
source: crates/static-file/types/src/segment.rs
|
||||
expression: "Bytes::from(serialized)"
|
||||
---
|
||||
0x01000000000000000000000000000000c800000000000000010000000000000000640000000000000000050000000164000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000000000000000000000000000000000000
|
||||
@@ -31,9 +31,6 @@ pub struct StorageSettings {
|
||||
/// Whether this node should read and write account changesets from static files.
|
||||
#[serde(default)]
|
||||
pub account_changesets_in_static_files: bool,
|
||||
/// Whether this node should read and write storage changesets from static files.
|
||||
#[serde(default)]
|
||||
pub storage_changesets_in_static_files: bool,
|
||||
}
|
||||
|
||||
impl StorageSettings {
|
||||
@@ -62,7 +59,6 @@ impl StorageSettings {
|
||||
receipts_in_static_files: true,
|
||||
transaction_senders_in_static_files: true,
|
||||
account_changesets_in_static_files: true,
|
||||
storage_changesets_in_static_files: true,
|
||||
storages_history_in_rocksdb: false,
|
||||
transaction_hash_numbers_in_rocksdb: true,
|
||||
account_history_in_rocksdb: false,
|
||||
@@ -82,7 +78,6 @@ impl StorageSettings {
|
||||
transaction_hash_numbers_in_rocksdb: false,
|
||||
account_history_in_rocksdb: false,
|
||||
account_changesets_in_static_files: false,
|
||||
storage_changesets_in_static_files: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -122,12 +117,6 @@ impl StorageSettings {
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the `storage_changesets_in_static_files` flag to the provided value.
|
||||
pub const fn with_storage_changesets_in_static_files(mut self, value: bool) -> Self {
|
||||
self.storage_changesets_in_static_files = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns `true` if any tables are configured to be stored in `RocksDB`.
|
||||
pub const fn any_in_rocksdb(&self) -> bool {
|
||||
self.transaction_hash_numbers_in_rocksdb ||
|
||||
|
||||
@@ -29,10 +29,10 @@ pub use blocks::*;
|
||||
pub use integer_list::IntegerList;
|
||||
pub use metadata::*;
|
||||
pub use reth_db_models::{
|
||||
AccountBeforeTx, ClientVersion, StaticFileBlockWithdrawals, StorageBeforeTx,
|
||||
StoredBlockBodyIndices, StoredBlockWithdrawals,
|
||||
AccountBeforeTx, ClientVersion, StaticFileBlockWithdrawals, StoredBlockBodyIndices,
|
||||
StoredBlockWithdrawals,
|
||||
};
|
||||
pub use sharded_key::{ShardedHistoryKey, ShardedKey};
|
||||
pub use sharded_key::ShardedKey;
|
||||
|
||||
/// Macro that implements [`Encode`] and [`Decode`] for uint types.
|
||||
macro_rules! impl_uints {
|
||||
@@ -230,7 +230,6 @@ impl_compression_for_compact!(
|
||||
StaticFileBlockWithdrawals,
|
||||
Bytecode,
|
||||
AccountBeforeTx,
|
||||
StorageBeforeTx,
|
||||
TransactionSigned,
|
||||
CompactU256,
|
||||
StageCheckpoint,
|
||||
|
||||
@@ -10,21 +10,6 @@ use std::hash::Hash;
|
||||
/// Number of indices in one shard.
|
||||
pub const NUM_OF_INDICES_IN_SHARD: usize = 2_000;
|
||||
|
||||
/// Trait for sharded history keys that can be constructed from a prefix + block number.
|
||||
///
|
||||
/// This abstracts over the key construction and prefix extraction for sharded history tables
|
||||
/// like `AccountsHistory` and `StoragesHistory`.
|
||||
pub trait ShardedHistoryKey: Sized {
|
||||
/// The prefix type (e.g., `Address` for accounts, `(Address, B256)` for storage).
|
||||
type Prefix: Copy + Eq;
|
||||
|
||||
/// Creates a new sharded key from prefix and highest block number.
|
||||
fn new_sharded(prefix: Self::Prefix, highest_block_number: u64) -> Self;
|
||||
|
||||
/// Extracts the prefix from this key.
|
||||
fn prefix(&self) -> Self::Prefix;
|
||||
}
|
||||
|
||||
/// Size of `BlockNumber` in bytes (u64 = 8 bytes).
|
||||
const BLOCK_NUMBER_SIZE: usize = std::mem::size_of::<BlockNumber>();
|
||||
|
||||
@@ -92,20 +77,6 @@ impl Decode for ShardedKey<Address> {
|
||||
}
|
||||
}
|
||||
|
||||
impl ShardedHistoryKey for ShardedKey<Address> {
|
||||
type Prefix = Address;
|
||||
|
||||
#[inline]
|
||||
fn new_sharded(prefix: Self::Prefix, highest_block_number: u64) -> Self {
|
||||
Self::new(prefix, highest_block_number)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn prefix(&self) -> Self::Prefix {
|
||||
self.key
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -139,14 +110,4 @@ mod tests {
|
||||
let decoded = ShardedKey::<Address>::decode(&encoded).unwrap();
|
||||
assert_eq!(decoded.highest_block_number, u64::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sharded_history_key_trait_roundtrip() {
|
||||
let addr = address!("0102030405060708091011121314151617181920");
|
||||
let block_num = 0x123456789ABCDEFu64;
|
||||
|
||||
let key = ShardedKey::<Address>::new_sharded(addr, block_num);
|
||||
assert_eq!(key.prefix(), addr);
|
||||
assert_eq!(key.highest_block_number, block_num);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ use alloy_primitives::{Address, BlockNumber, B256};
|
||||
use derive_more::AsRef;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{ShardedHistoryKey, ShardedKey};
|
||||
use super::ShardedKey;
|
||||
|
||||
/// Number of indices in one shard.
|
||||
pub const NUM_OF_INDICES_IN_SHARD: usize = 2_000;
|
||||
@@ -91,20 +91,6 @@ impl Decode for StorageShardedKey {
|
||||
}
|
||||
}
|
||||
|
||||
impl ShardedHistoryKey for StorageShardedKey {
|
||||
type Prefix = (Address, B256);
|
||||
|
||||
#[inline]
|
||||
fn new_sharded(prefix: Self::Prefix, highest_block_number: u64) -> Self {
|
||||
Self::new(prefix.0, prefix.1, highest_block_number)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn prefix(&self) -> Self::Prefix {
|
||||
(self.address, self.sharded_key.key)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -144,15 +130,4 @@ mod tests {
|
||||
let decoded = StorageShardedKey::decode(&encoded).unwrap();
|
||||
assert_eq!(decoded.sharded_key.highest_block_number, u64::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sharded_history_key_trait_roundtrip() {
|
||||
let addr = address!("0102030405060708091011121314151617181920");
|
||||
let storage_key = b256!("0001020304050607080910111213141516171819202122232425262728293031");
|
||||
let block_num = 0x123456789ABCDEFu64;
|
||||
|
||||
let key = StorageShardedKey::new_sharded((addr, storage_key), block_num);
|
||||
assert_eq!(key.prefix(), (addr, storage_key));
|
||||
assert_eq!(key.sharded_key.highest_block_number, block_num);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,10 +19,6 @@ pub use accounts::AccountBeforeTx;
|
||||
pub mod blocks;
|
||||
pub use blocks::{StaticFileBlockWithdrawals, StoredBlockBodyIndices, StoredBlockWithdrawals};
|
||||
|
||||
/// Storage
|
||||
pub mod storage;
|
||||
pub use storage::StorageBeforeTx;
|
||||
|
||||
/// Client Version
|
||||
pub mod client_version;
|
||||
pub use client_version::ClientVersion;
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
use alloy_primitives::{Address, B256, U256};
|
||||
use reth_primitives_traits::ValueWithSubKey;
|
||||
|
||||
/// Storage entry as it is saved in the static files.
|
||||
///
|
||||
/// [`B256`] is the subkey.
|
||||
#[derive(Debug, Default, Copy, Clone, Eq, PartialEq)]
|
||||
#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
#[cfg_attr(any(test, feature = "reth-codec"), reth_codecs::add_arbitrary_tests(compact))]
|
||||
pub struct StorageBeforeTx {
|
||||
/// Address for the storage entry. Acts as `DupSort::SubKey` in static files.
|
||||
pub address: Address,
|
||||
/// Storage key.
|
||||
pub key: B256,
|
||||
/// Value on storage key.
|
||||
pub value: U256,
|
||||
}
|
||||
|
||||
impl ValueWithSubKey for StorageBeforeTx {
|
||||
type SubKey = B256;
|
||||
|
||||
fn get_subkey(&self) -> Self::SubKey {
|
||||
self.key
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE: Removing reth_codec and manually encode subkey
|
||||
// and compress second part of the value. If we have compression
|
||||
// over whole value (Even SubKey) that would mess up fetching of values with seek_by_key_subkey
|
||||
#[cfg(any(test, feature = "reth-codec"))]
|
||||
impl reth_codecs::Compact for StorageBeforeTx {
|
||||
fn to_compact<B>(&self, buf: &mut B) -> usize
|
||||
where
|
||||
B: bytes::BufMut + AsMut<[u8]>,
|
||||
{
|
||||
buf.put_slice(self.address.as_slice());
|
||||
buf.put_slice(&self.key[..]);
|
||||
self.value.to_compact(buf) + 52
|
||||
}
|
||||
|
||||
fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) {
|
||||
let address = Address::from_slice(&buf[..20]);
|
||||
let key = B256::from_slice(&buf[20..52]);
|
||||
let (value, out) = U256::from_compact(&buf[52..], len - 52);
|
||||
(Self { address, key, value }, out)
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,7 @@ use crate::{
|
||||
HeaderTerminalDifficulties,
|
||||
};
|
||||
use alloy_primitives::{Address, BlockHash};
|
||||
use reth_db_api::{models::StorageBeforeTx, table::Table, AccountChangeSets};
|
||||
use reth_db_api::{table::Table, AccountChangeSets};
|
||||
|
||||
// HEADER MASKS
|
||||
add_static_file_mask! {
|
||||
@@ -54,9 +54,3 @@ add_static_file_mask! {
|
||||
#[doc = "Mask for selecting a single changeset from `AccountChangesets` static file segment"]
|
||||
AccountChangesetMask, <AccountChangeSets as Table>::Value, 0b1
|
||||
}
|
||||
|
||||
// STORAGE CHANGESET MASKS
|
||||
add_static_file_mask! {
|
||||
#[doc = "Mask for selecting a single changeset from `StorageChangesets` static file segment"]
|
||||
StorageChangesetMask, StorageBeforeTx, 0b1
|
||||
}
|
||||
|
||||
@@ -412,16 +412,8 @@ impl Transaction<RW> {
|
||||
/// Returns a buffer which can be used to write a value into the item at the
|
||||
/// given key and with the given length. The buffer must be completely
|
||||
/// filled by the caller.
|
||||
///
|
||||
/// This should not be used on dupsort tables.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// The caller must ensure that the returned buffer is not used after the transaction is
|
||||
/// committed or aborted, or if another value is inserted. To be clear: the second call to
|
||||
/// this function is not permitted while the returned slice is reachable.
|
||||
#[allow(clippy::mut_from_ref)]
|
||||
pub unsafe fn reserve(
|
||||
pub fn reserve(
|
||||
&self,
|
||||
dbi: ffi::MDBX_dbi,
|
||||
key: impl AsRef<[u8]>,
|
||||
|
||||
@@ -105,11 +105,8 @@ fn test_reserve() {
|
||||
let txn = env.begin_rw_txn().unwrap();
|
||||
let dbi = txn.open_db(None).unwrap().dbi();
|
||||
{
|
||||
unsafe {
|
||||
// SAFETY: the returned slice is used before the transaction is committed or aborted.
|
||||
let mut writer = txn.reserve(dbi, b"key1", 4, WriteFlags::empty()).unwrap();
|
||||
writer.write_all(b"val1").unwrap();
|
||||
}
|
||||
let mut writer = txn.reserve(dbi, b"key1", 4, WriteFlags::empty()).unwrap();
|
||||
writer.write_all(b"val1").unwrap();
|
||||
}
|
||||
txn.commit().unwrap();
|
||||
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
//! Account/storage changeset iteration support for walking through historical state changes in
|
||||
//! Account changeset iteration support for walking through historical account state changes in
|
||||
//! static files.
|
||||
|
||||
use crate::ProviderResult;
|
||||
use alloy_primitives::BlockNumber;
|
||||
use reth_db::models::AccountBeforeTx;
|
||||
use reth_db_api::models::BlockNumberAddress;
|
||||
use reth_primitives_traits::StorageEntry;
|
||||
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader};
|
||||
use reth_storage_api::ChangeSetReader;
|
||||
use std::ops::{Bound, RangeBounds};
|
||||
|
||||
/// Iterator that walks account changesets from static files in a block range.
|
||||
@@ -99,78 +97,3 @@ where
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterator that walks storage changesets from static files in a block range.
|
||||
#[derive(Debug)]
|
||||
pub struct StaticFileStorageChangesetWalker<P> {
|
||||
/// Static file provider
|
||||
provider: P,
|
||||
/// End block (exclusive). `None` means iterate until exhausted.
|
||||
end_block: Option<BlockNumber>,
|
||||
/// Current block being processed
|
||||
current_block: BlockNumber,
|
||||
/// Changesets for current block
|
||||
current_changesets: Vec<(BlockNumberAddress, StorageEntry)>,
|
||||
/// Index within current block's changesets
|
||||
changeset_index: usize,
|
||||
}
|
||||
|
||||
impl<P> StaticFileStorageChangesetWalker<P> {
|
||||
/// Create a new static file storage changeset walker.
|
||||
pub fn new(provider: P, range: impl RangeBounds<BlockNumber>) -> Self {
|
||||
let start = match range.start_bound() {
|
||||
Bound::Included(&n) => n,
|
||||
Bound::Excluded(&n) => n + 1,
|
||||
Bound::Unbounded => 0,
|
||||
};
|
||||
|
||||
let end_block = match range.end_bound() {
|
||||
Bound::Included(&n) => Some(n + 1),
|
||||
Bound::Excluded(&n) => Some(n),
|
||||
Bound::Unbounded => None,
|
||||
};
|
||||
|
||||
Self {
|
||||
provider,
|
||||
end_block,
|
||||
current_block: start,
|
||||
current_changesets: Vec::new(),
|
||||
changeset_index: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<P> Iterator for StaticFileStorageChangesetWalker<P>
|
||||
where
|
||||
P: StorageChangeSetReader,
|
||||
{
|
||||
type Item = ProviderResult<(BlockNumberAddress, StorageEntry)>;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
if let Some(changeset) = self.current_changesets.get(self.changeset_index).copied() {
|
||||
self.changeset_index += 1;
|
||||
return Some(Ok(changeset));
|
||||
}
|
||||
|
||||
if !self.current_changesets.is_empty() {
|
||||
self.current_block += 1;
|
||||
}
|
||||
|
||||
while self.end_block.is_none_or(|end| self.current_block < end) {
|
||||
match self.provider.storage_changeset(self.current_block) {
|
||||
Ok(changesets) if !changesets.is_empty() => {
|
||||
self.current_changesets = changesets;
|
||||
self.changeset_index = 1;
|
||||
return Some(Ok(self.current_changesets[0]));
|
||||
}
|
||||
Ok(_) => self.current_block += 1,
|
||||
Err(e) => {
|
||||
self.current_block += 1;
|
||||
return Some(Err(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,20 +17,20 @@ use alloy_primitives::{map::HashMap, Address, BlockNumber, TxHash, TxNumber, B25
|
||||
use rayon::slice::ParallelSliceMut;
|
||||
use reth_db::{
|
||||
cursor::{DbCursorRO, DbDupCursorRW},
|
||||
models::{AccountBeforeTx, StorageBeforeTx},
|
||||
models::AccountBeforeTx,
|
||||
static_file::TransactionSenderMask,
|
||||
table::Value,
|
||||
transaction::{CursorMutTy, CursorTy, DbTx, DbTxMut, DupCursorMutTy, DupCursorTy},
|
||||
};
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRW,
|
||||
models::{storage_sharded_key::StorageShardedKey, BlockNumberAddress, ShardedKey},
|
||||
models::{storage_sharded_key::StorageShardedKey, ShardedKey},
|
||||
tables,
|
||||
tables::BlockNumberList,
|
||||
};
|
||||
use reth_errors::ProviderError;
|
||||
use reth_node_types::NodePrimitives;
|
||||
use reth_primitives_traits::{ReceiptTy, StorageEntry};
|
||||
use reth_primitives_traits::ReceiptTy;
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::{ChangeSetReader, DBProvider, NodePrimitivesProvider, StorageSettingsCache};
|
||||
use reth_storage_errors::provider::ProviderResult;
|
||||
@@ -171,27 +171,6 @@ impl<'a> EitherWriter<'a, (), ()> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a new [`EitherWriter`] for storage changesets based on storage settings.
|
||||
pub fn new_storage_changesets<P>(
|
||||
provider: &'a P,
|
||||
block_number: BlockNumber,
|
||||
) -> ProviderResult<DupEitherWriterTy<'a, P, tables::StorageChangeSets>>
|
||||
where
|
||||
P: DBProvider + NodePrimitivesProvider + StorageSettingsCache + StaticFileProviderFactory,
|
||||
P::Tx: DbTxMut,
|
||||
{
|
||||
if provider.cached_storage_settings().storage_changesets_in_static_files {
|
||||
Ok(EitherWriter::StaticFile(
|
||||
provider
|
||||
.get_static_file_writer(block_number, StaticFileSegment::StorageChangeSets)?,
|
||||
))
|
||||
} else {
|
||||
Ok(EitherWriter::Database(
|
||||
provider.tx_ref().cursor_dup_write::<tables::StorageChangeSets>()?,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the destination for writing receipts.
|
||||
///
|
||||
/// The rules are as follows:
|
||||
@@ -229,19 +208,6 @@ impl<'a> EitherWriter<'a, (), ()> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the destination for writing storage changesets.
|
||||
///
|
||||
/// This determines the destination based solely on storage settings.
|
||||
pub fn storage_changesets_destination<P: DBProvider + StorageSettingsCache>(
|
||||
provider: &P,
|
||||
) -> EitherWriterDestination {
|
||||
if provider.cached_storage_settings().storage_changesets_in_static_files {
|
||||
EitherWriterDestination::StaticFile
|
||||
} else {
|
||||
EitherWriterDestination::Database
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a new [`EitherWriter`] for storages history based on storage settings.
|
||||
pub fn new_storages_history<P>(
|
||||
provider: &P,
|
||||
@@ -685,41 +651,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N>
|
||||
where
|
||||
CURSOR: DbDupCursorRW<tables::StorageChangeSets>,
|
||||
{
|
||||
/// Append storage changeset for a block.
|
||||
///
|
||||
/// NOTE: This _sorts_ the changesets by address and storage key before appending.
|
||||
pub fn append_storage_changeset(
|
||||
&mut self,
|
||||
block_number: BlockNumber,
|
||||
mut changeset: Vec<StorageBeforeTx>,
|
||||
) -> ProviderResult<()> {
|
||||
changeset.par_sort_by_key(|change| (change.address, change.key));
|
||||
|
||||
match self {
|
||||
Self::Database(cursor) => {
|
||||
for change in changeset {
|
||||
let storage_id = BlockNumberAddress((block_number, change.address));
|
||||
cursor.append_dup(
|
||||
storage_id,
|
||||
StorageEntry { key: change.key, value: change.value },
|
||||
)?;
|
||||
}
|
||||
}
|
||||
Self::StaticFile(writer) => {
|
||||
writer.append_storage_changeset(changeset, block_number)?;
|
||||
}
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(_) => return Err(ProviderError::UnsupportedProvider),
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Represents a source for reading data, either from database, static files, or `RocksDB`.
|
||||
#[derive(Debug, Display)]
|
||||
pub enum EitherReader<'a, CURSOR, N> {
|
||||
@@ -1056,19 +987,6 @@ impl EitherWriterDestination {
|
||||
Self::Database
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the destination for writing storage changesets based on storage settings.
|
||||
pub fn storage_changesets<P>(provider: &P) -> Self
|
||||
where
|
||||
P: StorageSettingsCache,
|
||||
{
|
||||
// Write storage changesets to static files only if they're explicitly enabled
|
||||
if provider.cached_storage_settings().storage_changesets_in_static_files {
|
||||
Self::StaticFile
|
||||
} else {
|
||||
Self::Database
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -711,26 +711,6 @@ impl<N: ProviderNodeTypes> StorageChangeSetReader for BlockchainProvider<N> {
|
||||
) -> ProviderResult<Vec<(BlockNumberAddress, StorageEntry)>> {
|
||||
self.consistent_provider()?.storage_changeset(block_number)
|
||||
}
|
||||
|
||||
fn get_storage_before_block(
|
||||
&self,
|
||||
block_number: BlockNumber,
|
||||
address: Address,
|
||||
storage_key: B256,
|
||||
) -> ProviderResult<Option<StorageEntry>> {
|
||||
self.consistent_provider()?.get_storage_before_block(block_number, address, storage_key)
|
||||
}
|
||||
|
||||
fn storage_changesets_range(
|
||||
&self,
|
||||
range: RangeInclusive<BlockNumber>,
|
||||
) -> ProviderResult<Vec<(BlockNumberAddress, StorageEntry)>> {
|
||||
self.consistent_provider()?.storage_changesets_range(range)
|
||||
}
|
||||
|
||||
fn storage_changeset_count(&self) -> ProviderResult<usize> {
|
||||
self.consistent_provider()?.storage_changeset_count()
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: ProviderNodeTypes> ChangeSetReader for BlockchainProvider<N> {
|
||||
@@ -1697,11 +1677,14 @@ mod tests {
|
||||
database_state.into_iter().map(|(address, (account, _))| {
|
||||
(address, None, Some(account.into()), Default::default())
|
||||
}),
|
||||
database_changesets.iter().map(|block_changesets| {
|
||||
block_changesets.iter().map(|(address, account, _)| {
|
||||
(*address, Some(Some((*account).into())), [])
|
||||
database_changesets
|
||||
.iter()
|
||||
.map(|block_changesets| {
|
||||
block_changesets.iter().map(|(address, account, _)| {
|
||||
(*address, Some(Some((*account).into())), [])
|
||||
})
|
||||
})
|
||||
}),
|
||||
.collect::<Vec<_>>(),
|
||||
Vec::new(),
|
||||
),
|
||||
first_block: first_database_block,
|
||||
|
||||
@@ -1347,138 +1347,6 @@ impl<N: ProviderNodeTypes> StorageChangeSetReader for ConsistentProvider<N> {
|
||||
self.storage_provider.storage_changeset(block_number)
|
||||
}
|
||||
}
|
||||
|
||||
fn get_storage_before_block(
|
||||
&self,
|
||||
block_number: BlockNumber,
|
||||
address: Address,
|
||||
storage_key: B256,
|
||||
) -> ProviderResult<Option<StorageEntry>> {
|
||||
if let Some(state) =
|
||||
self.head_block.as_ref().and_then(|b| b.block_on_chain(block_number.into()))
|
||||
{
|
||||
let changeset = state
|
||||
.block_ref()
|
||||
.execution_output
|
||||
.state
|
||||
.reverts
|
||||
.clone()
|
||||
.to_plain_state_reverts()
|
||||
.storage
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.find_map(|revert: PlainStorageRevert| {
|
||||
if revert.address != address {
|
||||
return None
|
||||
}
|
||||
revert.storage_revert.into_iter().find_map(|(key, value)| {
|
||||
let key = key.into();
|
||||
(key == storage_key)
|
||||
.then(|| StorageEntry { key, value: value.to_previous_value() })
|
||||
})
|
||||
});
|
||||
Ok(changeset)
|
||||
} else {
|
||||
let storage_history_exists = self
|
||||
.storage_provider
|
||||
.get_prune_checkpoint(PruneSegment::StorageHistory)?
|
||||
.and_then(|checkpoint| {
|
||||
checkpoint.block_number.map(|checkpoint| block_number > checkpoint)
|
||||
})
|
||||
.unwrap_or(true);
|
||||
|
||||
if !storage_history_exists {
|
||||
return Err(ProviderError::StateAtBlockPruned(block_number))
|
||||
}
|
||||
|
||||
self.storage_provider.get_storage_before_block(block_number, address, storage_key)
|
||||
}
|
||||
}
|
||||
|
||||
fn storage_changesets_range(
|
||||
&self,
|
||||
range: RangeInclusive<BlockNumber>,
|
||||
) -> ProviderResult<Vec<(BlockNumberAddress, StorageEntry)>> {
|
||||
let range = to_range(range);
|
||||
let mut changesets = Vec::new();
|
||||
let database_start = range.start;
|
||||
let mut database_end = range.end;
|
||||
|
||||
if let Some(head_block) = &self.head_block {
|
||||
database_end = head_block.anchor().number;
|
||||
|
||||
let chain = head_block.chain().collect::<Vec<_>>();
|
||||
for state in chain {
|
||||
let block_changesets = state
|
||||
.block_ref()
|
||||
.execution_output
|
||||
.state
|
||||
.reverts
|
||||
.clone()
|
||||
.to_plain_state_reverts()
|
||||
.storage
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.flat_map(|revert: PlainStorageRevert| {
|
||||
revert.storage_revert.into_iter().map(move |(key, value)| {
|
||||
(
|
||||
BlockNumberAddress((state.number(), revert.address)),
|
||||
StorageEntry { key: key.into(), value: value.to_previous_value() },
|
||||
)
|
||||
})
|
||||
});
|
||||
|
||||
changesets.extend(block_changesets);
|
||||
}
|
||||
}
|
||||
|
||||
if database_start < database_end {
|
||||
let storage_history_exists = self
|
||||
.storage_provider
|
||||
.get_prune_checkpoint(PruneSegment::StorageHistory)?
|
||||
.and_then(|checkpoint| {
|
||||
checkpoint.block_number.map(|checkpoint| database_start > checkpoint)
|
||||
})
|
||||
.unwrap_or(true);
|
||||
|
||||
if !storage_history_exists {
|
||||
return Err(ProviderError::StateAtBlockPruned(database_start))
|
||||
}
|
||||
|
||||
let db_changesets = self
|
||||
.storage_provider
|
||||
.storage_changesets_range(database_start..=database_end - 1)?;
|
||||
changesets.extend(db_changesets);
|
||||
}
|
||||
|
||||
changesets.sort_by_key(|(block_address, _)| block_address.block_number());
|
||||
|
||||
Ok(changesets)
|
||||
}
|
||||
|
||||
fn storage_changeset_count(&self) -> ProviderResult<usize> {
|
||||
let mut count = 0;
|
||||
if let Some(head_block) = &self.head_block {
|
||||
for state in head_block.chain() {
|
||||
count += state
|
||||
.block_ref()
|
||||
.execution_output
|
||||
.state
|
||||
.reverts
|
||||
.clone()
|
||||
.to_plain_state_reverts()
|
||||
.storage
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|revert: PlainStorageRevert| revert.storage_revert.len())
|
||||
.sum::<usize>();
|
||||
}
|
||||
}
|
||||
|
||||
count += self.storage_provider.storage_changeset_count()?;
|
||||
|
||||
Ok(count)
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: ProviderNodeTypes> ChangeSetReader for ConsistentProvider<N> {
|
||||
|
||||
@@ -44,9 +44,7 @@ use std::{
|
||||
use tracing::trace;
|
||||
|
||||
mod provider;
|
||||
pub use provider::{
|
||||
CommitOrder, DatabaseProvider, DatabaseProviderRO, DatabaseProviderRW, SaveBlocksMode,
|
||||
};
|
||||
pub use provider::{DatabaseProvider, DatabaseProviderRO, DatabaseProviderRW, SaveBlocksMode};
|
||||
|
||||
use super::ProviderNodeTypes;
|
||||
use reth_trie::KeccakKeyHasher;
|
||||
@@ -232,25 +230,6 @@ impl<N: ProviderNodeTypes> ProviderFactory<N> {
|
||||
)))
|
||||
}
|
||||
|
||||
/// Returns a provider with a created `DbTxMut` inside, configured for unwind operations.
|
||||
/// Uses unwind commit order (MDBX first, then `RocksDB`, then static files) to allow
|
||||
/// recovery by truncating static files on restart if interrupted.
|
||||
#[track_caller]
|
||||
pub fn unwind_provider_rw(
|
||||
&self,
|
||||
) -> ProviderResult<DatabaseProvider<<N::DB as Database>::TXMut, N>> {
|
||||
Ok(DatabaseProvider::new_unwind_rw(
|
||||
self.db.tx_mut()?,
|
||||
self.chain_spec.clone(),
|
||||
self.static_file_provider.clone(),
|
||||
self.prune_modes.clone(),
|
||||
self.storage.clone(),
|
||||
self.storage_settings.clone(),
|
||||
self.rocksdb_provider.clone(),
|
||||
self.changeset_cache.clone(),
|
||||
))
|
||||
}
|
||||
|
||||
/// State provider for latest block
|
||||
#[track_caller]
|
||||
pub fn latest(&self) -> ProviderResult<StateProviderBox> {
|
||||
|
||||
@@ -40,8 +40,7 @@ use reth_db_api::{
|
||||
database::Database,
|
||||
models::{
|
||||
sharded_key, storage_sharded_key::StorageShardedKey, AccountBeforeTx, BlockNumberAddress,
|
||||
BlockNumberHashedAddress, ShardedKey, StorageBeforeTx, StorageSettings,
|
||||
StoredBlockBodyIndices,
|
||||
BlockNumberHashedAddress, ShardedKey, StorageSettings, StoredBlockBodyIndices,
|
||||
},
|
||||
table::Table,
|
||||
tables,
|
||||
@@ -85,24 +84,6 @@ use std::{
|
||||
};
|
||||
use tracing::{debug, instrument, trace};
|
||||
|
||||
/// Determines the commit order for database operations.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum CommitOrder {
|
||||
/// Normal commit order: static files first, then `RocksDB`, then MDBX.
|
||||
#[default]
|
||||
Normal,
|
||||
/// Unwind commit order: MDBX first, then `RocksDB`, then static files.
|
||||
/// Used for unwind operations to allow recovery by truncating static files on restart.
|
||||
Unwind,
|
||||
}
|
||||
|
||||
impl CommitOrder {
|
||||
/// Returns true if this is unwind commit order.
|
||||
pub const fn is_unwind(&self) -> bool {
|
||||
matches!(self, Self::Unwind)
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`DatabaseProvider`] that holds a read-only database transaction.
|
||||
pub type DatabaseProviderRO<DB, N> = DatabaseProvider<<DB as Database>::TX, N>;
|
||||
|
||||
@@ -205,8 +186,6 @@ pub struct DatabaseProvider<TX, N: NodeTypes> {
|
||||
/// Pending `RocksDB` batches to be committed at provider commit time.
|
||||
#[cfg_attr(not(all(unix, feature = "rocksdb")), allow(dead_code))]
|
||||
pending_rocksdb_batches: PendingRocksDBBatches,
|
||||
/// Commit order for database operations.
|
||||
commit_order: CommitOrder,
|
||||
/// Minimum distance from tip required for pruning
|
||||
minimum_pruning_distance: u64,
|
||||
/// Database provider metrics
|
||||
@@ -225,7 +204,6 @@ impl<TX: Debug, N: NodeTypes> Debug for DatabaseProvider<TX, N> {
|
||||
.field("rocksdb_provider", &self.rocksdb_provider)
|
||||
.field("changeset_cache", &self.changeset_cache)
|
||||
.field("pending_rocksdb_batches", &"<pending batches>")
|
||||
.field("commit_order", &self.commit_order)
|
||||
.field("minimum_pruning_distance", &self.minimum_pruning_distance)
|
||||
.finish()
|
||||
}
|
||||
@@ -339,7 +317,7 @@ impl<TX: Debug + Send, N: NodeTypes<ChainSpec: EthChainSpec + 'static>> ChainSpe
|
||||
impl<TX: DbTxMut, N: NodeTypes> DatabaseProvider<TX, N> {
|
||||
/// Creates a provider with an inner read-write transaction.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn new_rw_inner(
|
||||
pub fn new_rw(
|
||||
tx: TX,
|
||||
chain_spec: Arc<N::ChainSpec>,
|
||||
static_file_provider: StaticFileProvider<N::Primitives>,
|
||||
@@ -348,7 +326,6 @@ impl<TX: DbTxMut, N: NodeTypes> DatabaseProvider<TX, N> {
|
||||
storage_settings: Arc<RwLock<StorageSettings>>,
|
||||
rocksdb_provider: RocksDBProvider,
|
||||
changeset_cache: ChangesetCache,
|
||||
commit_order: CommitOrder,
|
||||
) -> Self {
|
||||
Self {
|
||||
tx,
|
||||
@@ -360,61 +337,10 @@ impl<TX: DbTxMut, N: NodeTypes> DatabaseProvider<TX, N> {
|
||||
rocksdb_provider,
|
||||
changeset_cache,
|
||||
pending_rocksdb_batches: Default::default(),
|
||||
commit_order,
|
||||
minimum_pruning_distance: MINIMUM_PRUNING_DISTANCE,
|
||||
metrics: metrics::DatabaseProviderMetrics::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a provider with an inner read-write transaction using normal commit order.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new_rw(
|
||||
tx: TX,
|
||||
chain_spec: Arc<N::ChainSpec>,
|
||||
static_file_provider: StaticFileProvider<N::Primitives>,
|
||||
prune_modes: PruneModes,
|
||||
storage: Arc<N::Storage>,
|
||||
storage_settings: Arc<RwLock<StorageSettings>>,
|
||||
rocksdb_provider: RocksDBProvider,
|
||||
changeset_cache: ChangesetCache,
|
||||
) -> Self {
|
||||
Self::new_rw_inner(
|
||||
tx,
|
||||
chain_spec,
|
||||
static_file_provider,
|
||||
prune_modes,
|
||||
storage,
|
||||
storage_settings,
|
||||
rocksdb_provider,
|
||||
changeset_cache,
|
||||
CommitOrder::Normal,
|
||||
)
|
||||
}
|
||||
|
||||
/// Creates a provider with an inner read-write transaction using unwind commit order.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new_unwind_rw(
|
||||
tx: TX,
|
||||
chain_spec: Arc<N::ChainSpec>,
|
||||
static_file_provider: StaticFileProvider<N::Primitives>,
|
||||
prune_modes: PruneModes,
|
||||
storage: Arc<N::Storage>,
|
||||
storage_settings: Arc<RwLock<StorageSettings>>,
|
||||
rocksdb_provider: RocksDBProvider,
|
||||
changeset_cache: ChangesetCache,
|
||||
) -> Self {
|
||||
Self::new_rw_inner(
|
||||
tx,
|
||||
chain_spec,
|
||||
static_file_provider,
|
||||
prune_modes,
|
||||
storage,
|
||||
storage_settings,
|
||||
rocksdb_provider,
|
||||
changeset_cache,
|
||||
CommitOrder::Unwind,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl<TX, N: NodeTypes> AsRef<Self> for DatabaseProvider<TX, N> {
|
||||
@@ -464,8 +390,6 @@ impl<TX: DbTx + DbTxMut + 'static, N: NodeTypesForProvider> DatabaseProvider<TX,
|
||||
EitherWriter::receipts_destination(self).is_static_file(),
|
||||
write_account_changesets: save_mode.with_state() &&
|
||||
EitherWriterDestination::account_changesets(self).is_static_file(),
|
||||
write_storage_changesets: save_mode.with_state() &&
|
||||
EitherWriterDestination::storage_changesets(self).is_static_file(),
|
||||
tip,
|
||||
receipts_prune_mode: self.prune_modes.receipts,
|
||||
// Receipts are prunable if no receipts exist in SF yet and within pruning distance
|
||||
@@ -952,7 +876,6 @@ impl<TX: DbTx + 'static, N: NodeTypesForProvider> DatabaseProvider<TX, N> {
|
||||
rocksdb_provider,
|
||||
changeset_cache,
|
||||
pending_rocksdb_batches: Default::default(),
|
||||
commit_order: CommitOrder::Normal,
|
||||
minimum_pruning_distance: MINIMUM_PRUNING_DISTANCE,
|
||||
metrics: metrics::DatabaseProviderMetrics::default(),
|
||||
}
|
||||
@@ -1347,58 +1270,13 @@ impl<TX: DbTx, N: NodeTypes> StorageChangeSetReader for DatabaseProvider<TX, N>
|
||||
&self,
|
||||
block_number: BlockNumber,
|
||||
) -> ProviderResult<Vec<(BlockNumberAddress, StorageEntry)>> {
|
||||
if self.cached_storage_settings().storage_changesets_in_static_files {
|
||||
self.static_file_provider.storage_changeset(block_number)
|
||||
} else {
|
||||
let range = block_number..=block_number;
|
||||
let storage_range = BlockNumberAddress::range(range);
|
||||
self.tx
|
||||
.cursor_dup_read::<tables::StorageChangeSets>()?
|
||||
.walk_range(storage_range)?
|
||||
.map(|result| -> ProviderResult<_> { Ok(result?) })
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn get_storage_before_block(
|
||||
&self,
|
||||
block_number: BlockNumber,
|
||||
address: Address,
|
||||
storage_key: B256,
|
||||
) -> ProviderResult<Option<StorageEntry>> {
|
||||
if self.cached_storage_settings().storage_changesets_in_static_files {
|
||||
self.static_file_provider.get_storage_before_block(block_number, address, storage_key)
|
||||
} else {
|
||||
self.tx
|
||||
.cursor_dup_read::<tables::StorageChangeSets>()?
|
||||
.seek_by_key_subkey(BlockNumberAddress((block_number, address)), storage_key)?
|
||||
.filter(|entry| entry.key == storage_key)
|
||||
.map(Ok)
|
||||
.transpose()
|
||||
}
|
||||
}
|
||||
|
||||
fn storage_changesets_range(
|
||||
&self,
|
||||
range: RangeInclusive<BlockNumber>,
|
||||
) -> ProviderResult<Vec<(BlockNumberAddress, StorageEntry)>> {
|
||||
if self.cached_storage_settings().storage_changesets_in_static_files {
|
||||
self.static_file_provider.storage_changesets_range(range)
|
||||
} else {
|
||||
self.tx
|
||||
.cursor_dup_read::<tables::StorageChangeSets>()?
|
||||
.walk_range(BlockNumberAddress::range(range))?
|
||||
.map(|result| -> ProviderResult<_> { Ok(result?) })
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn storage_changeset_count(&self) -> ProviderResult<usize> {
|
||||
if self.cached_storage_settings().storage_changesets_in_static_files {
|
||||
self.static_file_provider.storage_changeset_count()
|
||||
} else {
|
||||
Ok(self.tx.entries::<tables::StorageChangeSets>()?)
|
||||
}
|
||||
let range = block_number..=block_number;
|
||||
let storage_range = BlockNumberAddress::range(range);
|
||||
self.tx
|
||||
.cursor_dup_read::<tables::StorageChangeSets>()?
|
||||
.walk_range(storage_range)?
|
||||
.map(|result| -> ProviderResult<_> { Ok(result?) })
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2120,67 +1998,38 @@ impl<TX: DbTx + 'static, N: NodeTypes> StorageReader for DatabaseProvider<TX, N>
|
||||
&self,
|
||||
range: RangeInclusive<BlockNumber>,
|
||||
) -> ProviderResult<BTreeMap<Address, BTreeSet<B256>>> {
|
||||
if self.cached_storage_settings().storage_changesets_in_static_files {
|
||||
self.storage_changesets_range(range)?.into_iter().try_fold(
|
||||
BTreeMap::new(),
|
||||
|mut accounts: BTreeMap<Address, BTreeSet<B256>>, entry| {
|
||||
let (BlockNumberAddress((_, address)), storage_entry) = entry;
|
||||
accounts.entry(address).or_default().insert(storage_entry.key);
|
||||
Ok(accounts)
|
||||
},
|
||||
)
|
||||
} else {
|
||||
self.tx
|
||||
.cursor_read::<tables::StorageChangeSets>()?
|
||||
.walk_range(BlockNumberAddress::range(range))?
|
||||
// fold all storages and save its old state so we can remove it from HashedStorage
|
||||
// it is needed as it is dup table.
|
||||
.try_fold(
|
||||
BTreeMap::new(),
|
||||
|mut accounts: BTreeMap<Address, BTreeSet<B256>>, entry| {
|
||||
let (BlockNumberAddress((_, address)), storage_entry) = entry?;
|
||||
accounts.entry(address).or_default().insert(storage_entry.key);
|
||||
Ok(accounts)
|
||||
},
|
||||
)
|
||||
}
|
||||
self.tx
|
||||
.cursor_read::<tables::StorageChangeSets>()?
|
||||
.walk_range(BlockNumberAddress::range(range))?
|
||||
// fold all storages and save its old state so we can remove it from HashedStorage
|
||||
// it is needed as it is dup table.
|
||||
.try_fold(BTreeMap::new(), |mut accounts: BTreeMap<Address, BTreeSet<B256>>, entry| {
|
||||
let (BlockNumberAddress((_, address)), storage_entry) = entry?;
|
||||
accounts.entry(address).or_default().insert(storage_entry.key);
|
||||
Ok(accounts)
|
||||
})
|
||||
}
|
||||
|
||||
fn changed_storages_and_blocks_with_range(
|
||||
&self,
|
||||
range: RangeInclusive<BlockNumber>,
|
||||
) -> ProviderResult<BTreeMap<(Address, B256), Vec<u64>>> {
|
||||
if self.cached_storage_settings().storage_changesets_in_static_files {
|
||||
self.storage_changesets_range(range)?.into_iter().try_fold(
|
||||
let mut changeset_cursor = self.tx.cursor_read::<tables::StorageChangeSets>()?;
|
||||
|
||||
let storage_changeset_lists =
|
||||
changeset_cursor.walk_range(BlockNumberAddress::range(range))?.try_fold(
|
||||
BTreeMap::new(),
|
||||
|mut storages: BTreeMap<(Address, B256), Vec<u64>>, (index, storage)| {
|
||||
|mut storages: BTreeMap<(Address, B256), Vec<u64>>, entry| -> ProviderResult<_> {
|
||||
let (index, storage) = entry?;
|
||||
storages
|
||||
.entry((index.address(), storage.key))
|
||||
.or_default()
|
||||
.push(index.block_number());
|
||||
Ok(storages)
|
||||
},
|
||||
)
|
||||
} else {
|
||||
let mut changeset_cursor = self.tx.cursor_read::<tables::StorageChangeSets>()?;
|
||||
)?;
|
||||
|
||||
let storage_changeset_lists =
|
||||
changeset_cursor.walk_range(BlockNumberAddress::range(range))?.try_fold(
|
||||
BTreeMap::new(),
|
||||
|mut storages: BTreeMap<(Address, B256), Vec<u64>>,
|
||||
entry|
|
||||
-> ProviderResult<_> {
|
||||
let (index, storage) = entry?;
|
||||
storages
|
||||
.entry((index.address(), storage.key))
|
||||
.or_default()
|
||||
.push(index.block_number());
|
||||
Ok(storages)
|
||||
},
|
||||
)?;
|
||||
|
||||
Ok(storage_changeset_lists)
|
||||
}
|
||||
Ok(storage_changeset_lists)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2303,16 +2152,17 @@ impl<TX: DbTxMut + DbTx + 'static, N: NodeTypesForProvider> StateWriter
|
||||
// Write storage changes
|
||||
tracing::trace!("Writing storage changes");
|
||||
let mut storages_cursor = self.tx_ref().cursor_dup_write::<tables::PlainStorageState>()?;
|
||||
let mut storage_changeset_cursor =
|
||||
self.tx_ref().cursor_dup_write::<tables::StorageChangeSets>()?;
|
||||
for (block_index, mut storage_changes) in reverts.storage.into_iter().enumerate() {
|
||||
let block_number = first_block + block_index as BlockNumber;
|
||||
|
||||
tracing::trace!(block_number, "Writing block change");
|
||||
// sort changes by address.
|
||||
storage_changes.par_sort_unstable_by_key(|a| a.address);
|
||||
let total_changes =
|
||||
storage_changes.iter().map(|change| change.storage_revert.len()).sum();
|
||||
let mut changeset = Vec::with_capacity(total_changes);
|
||||
for PlainStorageRevert { address, wiped, storage_revert } in storage_changes {
|
||||
let storage_id = BlockNumberAddress((block_number, address));
|
||||
|
||||
let mut storage = storage_revert
|
||||
.into_iter()
|
||||
.map(|(k, v)| (B256::new(k.to_be_bytes()), v))
|
||||
@@ -2340,13 +2190,9 @@ impl<TX: DbTxMut + DbTx + 'static, N: NodeTypesForProvider> StateWriter
|
||||
|
||||
tracing::trace!(?address, ?storage, "Writing storage reverts");
|
||||
for (key, value) in StorageRevertsIter::new(storage, wiped_storage) {
|
||||
changeset.push(StorageBeforeTx { address, key, value });
|
||||
storage_changeset_cursor.append_dup(storage_id, StorageEntry { key, value })?;
|
||||
}
|
||||
}
|
||||
|
||||
let mut storage_changesets_writer =
|
||||
EitherWriter::new_storage_changesets(self, block_number)?;
|
||||
storage_changesets_writer.append_storage_changeset(block_number, changeset)?;
|
||||
}
|
||||
|
||||
if !config.write_account_changesets {
|
||||
@@ -2507,19 +2353,8 @@ impl<TX: DbTxMut + DbTx + 'static, N: NodeTypesForProvider> StateWriter
|
||||
block_bodies.first().expect("already checked if there are blocks").first_tx_num();
|
||||
|
||||
let storage_range = BlockNumberAddress::range(range.clone());
|
||||
let storage_changeset = if let Some(_highest_block) = self
|
||||
.static_file_provider
|
||||
.get_highest_static_file_block(StaticFileSegment::StorageChangeSets) &&
|
||||
self.cached_storage_settings().storage_changesets_in_static_files
|
||||
{
|
||||
let changesets = self.storage_changesets_range(range.clone())?;
|
||||
let mut changeset_writer =
|
||||
self.static_file_provider.latest_writer(StaticFileSegment::StorageChangeSets)?;
|
||||
changeset_writer.prune_storage_changesets(block)?;
|
||||
changesets
|
||||
} else {
|
||||
self.take::<tables::StorageChangeSets>(storage_range)?
|
||||
};
|
||||
|
||||
let storage_changeset = self.take::<tables::StorageChangeSets>(storage_range)?;
|
||||
let account_changeset = self.take::<tables::AccountChangeSets>(range)?;
|
||||
|
||||
// This is not working for blocks that are not at tip. as plain state is not the last
|
||||
@@ -2614,19 +2449,8 @@ impl<TX: DbTxMut + DbTx + 'static, N: NodeTypesForProvider> StateWriter
|
||||
block_bodies.last().expect("already checked if there are blocks").last_tx_num();
|
||||
|
||||
let storage_range = BlockNumberAddress::range(range.clone());
|
||||
let storage_changeset = if let Some(highest_block) = self
|
||||
.static_file_provider
|
||||
.get_highest_static_file_block(StaticFileSegment::StorageChangeSets) &&
|
||||
self.cached_storage_settings().storage_changesets_in_static_files
|
||||
{
|
||||
let changesets = self.storage_changesets_range(block + 1..=highest_block)?;
|
||||
let mut changeset_writer =
|
||||
self.static_file_provider.latest_writer(StaticFileSegment::StorageChangeSets)?;
|
||||
changeset_writer.prune_storage_changesets(block)?;
|
||||
changesets
|
||||
} else {
|
||||
self.take::<tables::StorageChangeSets>(storage_range)?
|
||||
};
|
||||
|
||||
let storage_changeset = self.take::<tables::StorageChangeSets>(storage_range)?;
|
||||
|
||||
// This is not working for blocks that are not at tip. as plain state is not the last
|
||||
// state of end range. We should rename the functions or add support to access
|
||||
@@ -3659,7 +3483,7 @@ impl<TX: DbTx + 'static, N: NodeTypes + 'static> DBProvider for DatabaseProvider
|
||||
// it is interrupted before the static files commit, we can just
|
||||
// truncate the static files according to the
|
||||
// checkpoints on the next start-up.
|
||||
if self.static_file_provider.has_unwind_queued() || self.commit_order.is_unwind() {
|
||||
if self.static_file_provider.has_unwind_queued() {
|
||||
self.tx.commit()?;
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
|
||||
@@ -93,6 +93,9 @@ const DEFAULT_MAX_BACKGROUND_JOBS: i32 = 6;
|
||||
/// Default bytes per sync for `RocksDB` WAL writes (1 MB).
|
||||
const DEFAULT_BYTES_PER_SYNC: u64 = 1_048_576;
|
||||
|
||||
/// Default bloom filter bits per key (~1% false positive rate).
|
||||
const DEFAULT_BLOOM_FILTER_BITS: f64 = 10.0;
|
||||
|
||||
/// Default buffer capacity for compression in batches.
|
||||
/// 4 KiB matches common block/page sizes and comfortably holds typical history values,
|
||||
/// reducing the first few reallocations without over-allocating.
|
||||
@@ -142,6 +145,11 @@ impl RocksDBBuilder {
|
||||
table_options.set_pin_l0_filter_and_index_blocks_in_cache(true);
|
||||
// Shared block cache for all column families.
|
||||
table_options.set_block_cache(cache);
|
||||
// Bloom filter: 10 bits/key = ~1% false positive rate, full filter for better read
|
||||
// performance. this setting is good trade off a little bit of memory for better
|
||||
// point lookup performance. see https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#configuration-basics
|
||||
table_options.set_bloom_filter(DEFAULT_BLOOM_FILTER_BITS, false);
|
||||
table_options.set_optimize_filters_for_memory(true);
|
||||
table_options
|
||||
}
|
||||
|
||||
@@ -193,32 +201,6 @@ impl RocksDBBuilder {
|
||||
cf_options
|
||||
}
|
||||
|
||||
/// Creates optimized column family options for `TransactionHashNumbers`.
|
||||
///
|
||||
/// This table stores `B256 -> TxNumber` mappings where:
|
||||
/// - Keys are incompressible 32-byte hashes (compression wastes CPU for zero benefit)
|
||||
/// - Values are varint-encoded `u64` (a few bytes - too small to benefit from compression)
|
||||
/// - Every lookup expects a hit (bloom filters only help when checking non-existent keys)
|
||||
fn tx_hash_numbers_column_family_options(cache: &Cache) -> Options {
|
||||
let mut table_options = BlockBasedOptions::default();
|
||||
table_options.set_block_size(DEFAULT_BLOCK_SIZE);
|
||||
table_options.set_cache_index_and_filter_blocks(true);
|
||||
table_options.set_pin_l0_filter_and_index_blocks_in_cache(true);
|
||||
table_options.set_block_cache(cache);
|
||||
// Disable bloom filter: every lookup expects a hit, so bloom filters provide no benefit
|
||||
// and waste memory
|
||||
|
||||
let mut cf_options = Options::default();
|
||||
cf_options.set_block_based_table_factory(&table_options);
|
||||
cf_options.set_level_compaction_dynamic_level_bytes(true);
|
||||
// Disable compression: B256 keys are incompressible hashes, TxNumber values are
|
||||
// varint-encoded u64 (a few bytes). Compression wastes CPU cycles for zero space savings.
|
||||
cf_options.set_compression_type(DBCompressionType::None);
|
||||
cf_options.set_bottommost_compression_type(DBCompressionType::None);
|
||||
|
||||
cf_options
|
||||
}
|
||||
|
||||
/// Adds a column family for a specific table type.
|
||||
pub fn with_table<T: Table>(mut self) -> Self {
|
||||
self.column_families.push(T::NAME.to_string());
|
||||
@@ -280,12 +262,10 @@ impl RocksDBBuilder {
|
||||
.column_families
|
||||
.iter()
|
||||
.map(|name| {
|
||||
let cf_options = if name == tables::TransactionHashNumbers::NAME {
|
||||
Self::tx_hash_numbers_column_family_options(&self.block_cache)
|
||||
} else {
|
||||
Self::default_column_family_options(&self.block_cache)
|
||||
};
|
||||
ColumnFamilyDescriptor::new(name.clone(), cf_options)
|
||||
ColumnFamilyDescriptor::new(
|
||||
name.clone(),
|
||||
Self::default_column_family_options(&self.block_cache),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -1089,8 +1069,13 @@ impl RocksDBProvider {
|
||||
for (block_idx, block) in blocks.iter().enumerate() {
|
||||
let block_number = ctx.first_block_number + block_idx as u64;
|
||||
let bundle = &block.execution_outcome().state;
|
||||
for &address in bundle.state().keys() {
|
||||
account_history.entry(address).or_default().push(block_number);
|
||||
// Only record accounts where account-info changed OR account was destroyed.
|
||||
// Skip accounts that only had storage changes - this matches MDBX semantics
|
||||
// where AccountsHistory tracks account-level changes, not storage-only touches.
|
||||
for (&address, account) in bundle.state() {
|
||||
if account.is_info_changed() || account.was_destroyed() {
|
||||
account_history.entry(address).or_default().push(block_number);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1115,9 +1100,14 @@ impl RocksDBProvider {
|
||||
let block_number = ctx.first_block_number + block_idx as u64;
|
||||
let bundle = &block.execution_outcome().state;
|
||||
for (&address, account) in bundle.state() {
|
||||
for &slot in account.storage.keys() {
|
||||
let key = B256::new(slot.to_be_bytes());
|
||||
storage_history.entry((address, key)).or_default().push(block_number);
|
||||
// Only record storage slots that actually changed value.
|
||||
// This matches MDBX semantics where StoragesHistory only tracks
|
||||
// slots with value changes, not just touched slots.
|
||||
for (&slot, storage_slot) in &account.storage {
|
||||
if storage_slot.is_changed() {
|
||||
let key = B256::new(slot.to_be_bytes());
|
||||
storage_history.entry((address, key)).or_default().push(block_number);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,13 +81,6 @@ impl RocksDBProvider {
|
||||
pub const fn table_stats(&self) -> Vec<RocksDBTableStats> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// Clears all entries from the specified table (stub implementation).
|
||||
///
|
||||
/// This is a no-op since there is no `RocksDB` when the feature is disabled.
|
||||
pub const fn clear<T>(&self) -> ProviderResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl DatabaseMetrics for RocksDBProvider {
|
||||
|
||||
@@ -14,7 +14,7 @@ use reth_db_api::{
|
||||
use reth_primitives_traits::{Account, Bytecode};
|
||||
use reth_storage_api::{
|
||||
BlockNumReader, BytecodeReader, DBProvider, NodePrimitivesProvider, StateProofProvider,
|
||||
StorageChangeSetReader, StorageRootProvider, StorageSettingsCache,
|
||||
StorageRootProvider, StorageSettingsCache,
|
||||
};
|
||||
use reth_storage_errors::provider::ProviderResult;
|
||||
use reth_trie::{
|
||||
@@ -26,8 +26,8 @@ use reth_trie::{
|
||||
TrieInputSorted,
|
||||
};
|
||||
use reth_trie_db::{
|
||||
hashed_storage_from_reverts_with_provider, DatabaseHashedPostState, DatabaseProof,
|
||||
DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot, DatabaseTrieWitness,
|
||||
DatabaseHashedPostState, DatabaseHashedStorage, DatabaseProof, DatabaseStateRoot,
|
||||
DatabaseStorageProof, DatabaseStorageRoot, DatabaseTrieWitness,
|
||||
};
|
||||
|
||||
use std::fmt::Debug;
|
||||
@@ -109,7 +109,7 @@ pub struct HistoricalStateProviderRef<'b, Provider> {
|
||||
lowest_available_blocks: LowestAvailableBlocks,
|
||||
}
|
||||
|
||||
impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumReader>
|
||||
impl<'b, Provider: DBProvider + ChangeSetReader + BlockNumReader>
|
||||
HistoricalStateProviderRef<'b, Provider>
|
||||
{
|
||||
/// Create new `StateProvider` for historical block number
|
||||
@@ -210,7 +210,7 @@ impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + Block
|
||||
);
|
||||
}
|
||||
|
||||
hashed_storage_from_reverts_with_provider(self.provider, address, self.block_number)
|
||||
Ok(HashedStorage::from_reverts(self.tx(), address, self.block_number)?)
|
||||
}
|
||||
|
||||
/// Set the lowest block number at which the account history is available.
|
||||
@@ -242,7 +242,6 @@ impl<
|
||||
Provider: DBProvider
|
||||
+ BlockNumReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory
|
||||
+ NodePrimitivesProvider,
|
||||
@@ -286,8 +285,8 @@ impl<Provider: DBProvider + BlockNumReader + BlockHashReader> BlockHashReader
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumReader>
|
||||
StateRootProvider for HistoricalStateProviderRef<'_, Provider>
|
||||
impl<Provider: DBProvider + ChangeSetReader + BlockNumReader> StateRootProvider
|
||||
for HistoricalStateProviderRef<'_, Provider>
|
||||
{
|
||||
fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult<B256> {
|
||||
let mut revert_state = self.revert_state()?;
|
||||
@@ -323,8 +322,8 @@ impl<Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumR
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumReader>
|
||||
StorageRootProvider for HistoricalStateProviderRef<'_, Provider>
|
||||
impl<Provider: DBProvider + ChangeSetReader + BlockNumReader> StorageRootProvider
|
||||
for HistoricalStateProviderRef<'_, Provider>
|
||||
{
|
||||
fn storage_root(
|
||||
&self,
|
||||
@@ -362,8 +361,8 @@ impl<Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumR
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumReader>
|
||||
StateProofProvider for HistoricalStateProviderRef<'_, Provider>
|
||||
impl<Provider: DBProvider + ChangeSetReader + BlockNumReader> StateProofProvider
|
||||
for HistoricalStateProviderRef<'_, Provider>
|
||||
{
|
||||
/// Get account and storage proofs.
|
||||
fn proof(
|
||||
@@ -406,7 +405,6 @@ impl<
|
||||
+ BlockNumReader
|
||||
+ BlockHashReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory
|
||||
+ NodePrimitivesProvider,
|
||||
@@ -420,16 +418,18 @@ impl<
|
||||
) -> ProviderResult<Option<StorageValue>> {
|
||||
match self.storage_history_lookup(address, storage_key)? {
|
||||
HistoryInfo::NotYetWritten => Ok(None),
|
||||
HistoryInfo::InChangeset(changeset_block_number) => self
|
||||
.provider
|
||||
.get_storage_before_block(changeset_block_number, address, storage_key)?
|
||||
.ok_or_else(|| ProviderError::StorageChangesetNotFound {
|
||||
block_number: changeset_block_number,
|
||||
address,
|
||||
storage_key: Box::new(storage_key),
|
||||
})
|
||||
.map(|entry| entry.value)
|
||||
.map(Some),
|
||||
HistoryInfo::InChangeset(changeset_block_number) => Ok(Some(
|
||||
self.tx()
|
||||
.cursor_dup_read::<tables::StorageChangeSets>()?
|
||||
.seek_by_key_subkey((changeset_block_number, address).into(), storage_key)?
|
||||
.filter(|entry| entry.key == storage_key)
|
||||
.ok_or_else(|| ProviderError::StorageChangesetNotFound {
|
||||
block_number: changeset_block_number,
|
||||
address,
|
||||
storage_key: Box::new(storage_key),
|
||||
})?
|
||||
.value,
|
||||
)),
|
||||
HistoryInfo::InPlainState | HistoryInfo::MaybeInPlainState => Ok(self
|
||||
.tx()
|
||||
.cursor_dup_read::<tables::PlainStorageState>()?
|
||||
@@ -462,9 +462,7 @@ pub struct HistoricalStateProvider<Provider> {
|
||||
lowest_available_blocks: LowestAvailableBlocks,
|
||||
}
|
||||
|
||||
impl<Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumReader>
|
||||
HistoricalStateProvider<Provider>
|
||||
{
|
||||
impl<Provider: DBProvider + ChangeSetReader + BlockNumReader> HistoricalStateProvider<Provider> {
|
||||
/// Create new `StateProvider` for historical block number
|
||||
pub fn new(provider: Provider, block_number: BlockNumber) -> Self {
|
||||
Self { provider, block_number, lowest_available_blocks: Default::default() }
|
||||
@@ -500,7 +498,7 @@ impl<Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumR
|
||||
}
|
||||
|
||||
// Delegates all provider impls to [HistoricalStateProviderRef]
|
||||
reth_storage_api::macros::delegate_provider_impls!(HistoricalStateProvider<Provider> where [Provider: DBProvider + BlockNumReader + BlockHashReader + ChangeSetReader + StorageChangeSetReader + StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider]);
|
||||
reth_storage_api::macros::delegate_provider_impls!(HistoricalStateProvider<Provider> where [Provider: DBProvider + BlockNumReader + BlockHashReader + ChangeSetReader + StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider]);
|
||||
|
||||
/// Lowest blocks at which different parts of the state are available.
|
||||
/// They may be [Some] if pruning is enabled.
|
||||
@@ -633,7 +631,7 @@ mod tests {
|
||||
use reth_primitives_traits::{Account, StorageEntry};
|
||||
use reth_storage_api::{
|
||||
BlockHashReader, BlockNumReader, ChangeSetReader, DBProvider, DatabaseProviderFactory,
|
||||
NodePrimitivesProvider, StorageChangeSetReader, StorageSettingsCache,
|
||||
NodePrimitivesProvider, StorageSettingsCache,
|
||||
};
|
||||
use reth_storage_errors::provider::ProviderError;
|
||||
|
||||
@@ -649,7 +647,6 @@ mod tests {
|
||||
+ BlockNumReader
|
||||
+ BlockHashReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory
|
||||
+ NodePrimitivesProvider,
|
||||
|
||||
@@ -10,7 +10,6 @@ use reth_stages_types::StageId;
|
||||
use reth_storage_api::{
|
||||
BlockNumReader, ChangeSetReader, DBProvider, DatabaseProviderFactory,
|
||||
DatabaseProviderROFactory, PruneCheckpointReader, StageCheckpointReader,
|
||||
StorageChangeSetReader,
|
||||
};
|
||||
use reth_trie::{
|
||||
hashed_cursor::{HashedCursorFactory, HashedPostStateCursorFactory},
|
||||
@@ -197,7 +196,6 @@ where
|
||||
F::Provider: StageCheckpointReader
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ DBProvider
|
||||
+ BlockNumReader,
|
||||
{
|
||||
@@ -448,11 +446,7 @@ where
|
||||
impl<F> DatabaseProviderROFactory for OverlayStateProviderFactory<F>
|
||||
where
|
||||
F: DatabaseProviderFactory,
|
||||
F::Provider: StageCheckpointReader
|
||||
+ PruneCheckpointReader
|
||||
+ BlockNumReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader,
|
||||
F::Provider: StageCheckpointReader + PruneCheckpointReader + BlockNumReader + ChangeSetReader,
|
||||
{
|
||||
type Provider = OverlayStateProvider<F::Provider>;
|
||||
|
||||
|
||||
@@ -3,10 +3,10 @@ use super::{
|
||||
StaticFileJarProvider, StaticFileProviderRW, StaticFileProviderRWRefMut,
|
||||
};
|
||||
use crate::{
|
||||
changeset_walker::{StaticFileAccountChangesetWalker, StaticFileStorageChangesetWalker},
|
||||
to_range, BlockHashReader, BlockNumReader, BlockReader, BlockSource, EitherWriter,
|
||||
EitherWriterDestination, HeaderProvider, ReceiptProvider, StageCheckpointReader, StatsReader,
|
||||
TransactionVariant, TransactionsProvider, TransactionsProviderExt,
|
||||
changeset_walker::StaticFileAccountChangesetWalker, to_range, BlockHashReader, BlockNumReader,
|
||||
BlockReader, BlockSource, EitherWriter, EitherWriterDestination, HeaderProvider,
|
||||
ReceiptProvider, StageCheckpointReader, StatsReader, TransactionVariant, TransactionsProvider,
|
||||
TransactionsProviderExt,
|
||||
};
|
||||
use alloy_consensus::{transaction::TransactionMeta, Header};
|
||||
use alloy_eips::{eip2718::Encodable2718, BlockHashOrNumber};
|
||||
@@ -20,12 +20,12 @@ use reth_db::{
|
||||
lockfile::StorageLock,
|
||||
static_file::{
|
||||
iter_static_files, BlockHashMask, HeaderMask, HeaderWithHashMask, ReceiptMask,
|
||||
StaticFileCursor, StorageChangesetMask, TransactionMask, TransactionSenderMask,
|
||||
StaticFileCursor, TransactionMask, TransactionSenderMask,
|
||||
},
|
||||
};
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO,
|
||||
models::{AccountBeforeTx, BlockNumberAddress, StorageBeforeTx, StoredBlockBodyIndices},
|
||||
models::{AccountBeforeTx, StoredBlockBodyIndices},
|
||||
table::{Decompress, Table, Value},
|
||||
tables,
|
||||
transaction::DbTx,
|
||||
@@ -35,7 +35,6 @@ use reth_nippy_jar::{NippyJar, NippyJarChecker, CONFIG_FILE_EXTENSION};
|
||||
use reth_node_types::NodePrimitives;
|
||||
use reth_primitives_traits::{
|
||||
AlloyBlockHeader as _, BlockBody as _, RecoveredBlock, SealedHeader, SignedTransaction,
|
||||
StorageEntry,
|
||||
};
|
||||
use reth_stages_types::{PipelineTarget, StageId};
|
||||
use reth_static_file_types::{
|
||||
@@ -43,8 +42,7 @@ use reth_static_file_types::{
|
||||
StaticFileSegment, DEFAULT_BLOCKS_PER_STATIC_FILE,
|
||||
};
|
||||
use reth_storage_api::{
|
||||
BlockBodyIndicesProvider, ChangeSetReader, DBProvider, StorageChangeSetReader,
|
||||
StorageSettingsCache,
|
||||
BlockBodyIndicesProvider, ChangeSetReader, DBProvider, StorageSettingsCache,
|
||||
};
|
||||
use reth_storage_errors::provider::{ProviderError, ProviderResult, StaticFileWriterError};
|
||||
use std::{
|
||||
@@ -94,8 +92,6 @@ pub struct StaticFileWriteCtx {
|
||||
pub write_receipts: bool,
|
||||
/// Whether account changesets should be written to static files.
|
||||
pub write_account_changesets: bool,
|
||||
/// Whether storage changesets should be written to static files.
|
||||
pub write_storage_changesets: bool,
|
||||
/// The current chain tip block number (for pruning).
|
||||
pub tip: BlockNumber,
|
||||
/// The prune mode for receipts, if any.
|
||||
@@ -626,35 +622,6 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Writes storage changesets for all blocks to the static file segment.
|
||||
#[instrument(level = "debug", target = "providers::db", skip_all)]
|
||||
fn write_storage_changesets(
|
||||
w: &mut StaticFileProviderRWRefMut<'_, N>,
|
||||
blocks: &[ExecutedBlock<N>],
|
||||
) -> ProviderResult<()> {
|
||||
for block in blocks {
|
||||
let block_number = block.recovered_block().number();
|
||||
let reverts = block.execution_outcome().state.reverts.to_plain_state_reverts();
|
||||
|
||||
for storage_block_reverts in reverts.storage {
|
||||
let changeset = storage_block_reverts
|
||||
.into_iter()
|
||||
.flat_map(|revert| {
|
||||
revert.storage_revert.into_iter().map(move |(key, revert_to_slot)| {
|
||||
StorageBeforeTx {
|
||||
address: revert.address,
|
||||
key: B256::new(key.to_be_bytes()),
|
||||
value: revert_to_slot.to_previous_value(),
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
w.append_storage_changeset(changeset, block_number)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Spawns a scoped thread that writes to a static file segment using the provided closure.
|
||||
///
|
||||
/// The closure receives a mutable reference to the segment writer. After the closure completes,
|
||||
@@ -730,15 +697,6 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
)
|
||||
});
|
||||
|
||||
let h_storage_changesets = ctx.write_storage_changesets.then(|| {
|
||||
self.spawn_segment_writer(
|
||||
s,
|
||||
StaticFileSegment::StorageChangeSets,
|
||||
first_block_number,
|
||||
|w| Self::write_storage_changesets(w, blocks),
|
||||
)
|
||||
});
|
||||
|
||||
h_headers.join().map_err(|_| StaticFileWriterError::ThreadPanic("headers"))??;
|
||||
h_txs.join().map_err(|_| StaticFileWriterError::ThreadPanic("transactions"))??;
|
||||
if let Some(h) = h_senders {
|
||||
@@ -751,10 +709,6 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
h.join()
|
||||
.map_err(|_| StaticFileWriterError::ThreadPanic("account_changesets"))??;
|
||||
}
|
||||
if let Some(h) = h_storage_changesets {
|
||||
h.join()
|
||||
.map_err(|_| StaticFileWriterError::ThreadPanic("storage_changesets"))??;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
@@ -1427,13 +1381,6 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
highest_tx,
|
||||
highest_block,
|
||||
)?,
|
||||
StaticFileSegment::StorageChangeSets => self
|
||||
.ensure_changeset_invariants_by_block::<_, tables::StorageChangeSets, _>(
|
||||
provider,
|
||||
segment,
|
||||
highest_block,
|
||||
|key| key.block_number(),
|
||||
)?,
|
||||
} {
|
||||
debug!(target: "reth::providers::static_file", ?segment, unwind_target=unwind, "Invariants check returned unwind target");
|
||||
update_unwind_target(unwind);
|
||||
@@ -1515,13 +1462,6 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
}
|
||||
true
|
||||
}
|
||||
StaticFileSegment::StorageChangeSets => {
|
||||
if EitherWriter::storage_changesets_destination(provider).is_database() {
|
||||
debug!(target: "reth::providers::static_file", ?segment, "Skipping storage changesets segment: changesets stored in database");
|
||||
return false
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1654,9 +1594,9 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
let stage_id = match segment {
|
||||
StaticFileSegment::Headers => StageId::Headers,
|
||||
StaticFileSegment::Transactions => StageId::Bodies,
|
||||
StaticFileSegment::Receipts |
|
||||
StaticFileSegment::AccountChangeSets |
|
||||
StaticFileSegment::StorageChangeSets => StageId::Execution,
|
||||
StaticFileSegment::Receipts | StaticFileSegment::AccountChangeSets => {
|
||||
StageId::Execution
|
||||
}
|
||||
StaticFileSegment::TransactionSenders => StageId::SenderRecovery,
|
||||
};
|
||||
let checkpoint_block_number =
|
||||
@@ -1711,9 +1651,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
StaticFileSegment::TransactionSenders => {
|
||||
writer.prune_transaction_senders(number, checkpoint_block_number)?
|
||||
}
|
||||
StaticFileSegment::Headers |
|
||||
StaticFileSegment::AccountChangeSets |
|
||||
StaticFileSegment::StorageChangeSets => {
|
||||
StaticFileSegment::Headers | StaticFileSegment::AccountChangeSets => {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
@@ -1724,9 +1662,6 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
StaticFileSegment::AccountChangeSets => {
|
||||
writer.prune_account_changesets(checkpoint_block_number)?;
|
||||
}
|
||||
StaticFileSegment::StorageChangeSets => {
|
||||
writer.prune_storage_changesets(checkpoint_block_number)?;
|
||||
}
|
||||
}
|
||||
debug!(target: "reth::providers::static_file", ?segment, "Committing writer after pruning");
|
||||
writer.commit()?;
|
||||
@@ -1737,105 +1672,6 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn ensure_changeset_invariants_by_block<Provider, T, F>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
segment: StaticFileSegment,
|
||||
highest_static_file_block: Option<BlockNumber>,
|
||||
block_from_key: F,
|
||||
) -> ProviderResult<Option<BlockNumber>>
|
||||
where
|
||||
Provider: DBProvider + BlockReader + StageCheckpointReader,
|
||||
T: Table,
|
||||
F: Fn(&T::Key) -> BlockNumber,
|
||||
{
|
||||
debug!(
|
||||
target: "reth::providers::static_file",
|
||||
?segment,
|
||||
?highest_static_file_block,
|
||||
"Ensuring changeset invariants"
|
||||
);
|
||||
let mut db_cursor = provider.tx_ref().cursor_read::<T>()?;
|
||||
|
||||
if let Some((db_first_key, _)) = db_cursor.first()? {
|
||||
let db_first_block = block_from_key(&db_first_key);
|
||||
if let Some(highest_block) = highest_static_file_block &&
|
||||
!(db_first_block <= highest_block || highest_block + 1 == db_first_block)
|
||||
{
|
||||
info!(
|
||||
target: "reth::providers::static_file",
|
||||
?db_first_block,
|
||||
?highest_block,
|
||||
unwind_target = highest_block,
|
||||
?segment,
|
||||
"Setting unwind target."
|
||||
);
|
||||
return Ok(Some(highest_block))
|
||||
}
|
||||
|
||||
if let Some((db_last_key, _)) = db_cursor.last()? &&
|
||||
highest_static_file_block
|
||||
.is_none_or(|highest_block| block_from_key(&db_last_key) > highest_block)
|
||||
{
|
||||
debug!(
|
||||
target: "reth::providers::static_file",
|
||||
?segment,
|
||||
"Database has entries beyond static files, no unwind needed"
|
||||
);
|
||||
return Ok(None)
|
||||
}
|
||||
} else {
|
||||
debug!(target: "reth::providers::static_file", ?segment, "No database entries found");
|
||||
}
|
||||
|
||||
let highest_static_file_block = highest_static_file_block.unwrap_or_default();
|
||||
|
||||
let stage_id = match segment {
|
||||
StaticFileSegment::Headers => StageId::Headers,
|
||||
StaticFileSegment::Transactions => StageId::Bodies,
|
||||
StaticFileSegment::Receipts |
|
||||
StaticFileSegment::AccountChangeSets |
|
||||
StaticFileSegment::StorageChangeSets => StageId::Execution,
|
||||
StaticFileSegment::TransactionSenders => StageId::SenderRecovery,
|
||||
};
|
||||
let checkpoint_block_number =
|
||||
provider.get_stage_checkpoint(stage_id)?.unwrap_or_default().block_number;
|
||||
|
||||
if checkpoint_block_number > highest_static_file_block {
|
||||
info!(
|
||||
target: "reth::providers::static_file",
|
||||
checkpoint_block_number,
|
||||
unwind_target = highest_static_file_block,
|
||||
?segment,
|
||||
"Setting unwind target."
|
||||
);
|
||||
return Ok(Some(highest_static_file_block))
|
||||
}
|
||||
|
||||
if checkpoint_block_number < highest_static_file_block {
|
||||
info!(
|
||||
target: "reth::providers",
|
||||
?segment,
|
||||
from = highest_static_file_block,
|
||||
to = checkpoint_block_number,
|
||||
"Unwinding static file segment."
|
||||
);
|
||||
let mut writer = self.latest_writer(segment)?;
|
||||
match segment {
|
||||
StaticFileSegment::AccountChangeSets => {
|
||||
writer.prune_account_changesets(checkpoint_block_number)?;
|
||||
}
|
||||
StaticFileSegment::StorageChangeSets => {
|
||||
writer.prune_storage_changesets(checkpoint_block_number)?;
|
||||
}
|
||||
_ => unreachable!("invalid segment for changeset invariants"),
|
||||
}
|
||||
writer.commit()?;
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Returns the earliest available block number that has not been expired and is still
|
||||
/// available.
|
||||
///
|
||||
@@ -2376,124 +2212,6 @@ impl<N: NodePrimitives> ChangeSetReader for StaticFileProvider<N> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives> StorageChangeSetReader for StaticFileProvider<N> {
|
||||
fn storage_changeset(
|
||||
&self,
|
||||
block_number: BlockNumber,
|
||||
) -> ProviderResult<Vec<(BlockNumberAddress, StorageEntry)>> {
|
||||
let provider = match self.get_segment_provider_for_block(
|
||||
StaticFileSegment::StorageChangeSets,
|
||||
block_number,
|
||||
None,
|
||||
) {
|
||||
Ok(provider) => provider,
|
||||
Err(ProviderError::MissingStaticFileBlock(_, _)) => return Ok(Vec::new()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
if let Some(offset) = provider.user_header().changeset_offset(block_number) {
|
||||
let mut cursor = provider.cursor()?;
|
||||
let mut changeset = Vec::with_capacity(offset.num_changes() as usize);
|
||||
|
||||
for i in offset.changeset_range() {
|
||||
if let Some(change) = cursor.get_one::<StorageChangesetMask>(i.into())? {
|
||||
let block_address = BlockNumberAddress((block_number, change.address));
|
||||
let entry = StorageEntry { key: change.key, value: change.value };
|
||||
changeset.push((block_address, entry));
|
||||
}
|
||||
}
|
||||
Ok(changeset)
|
||||
} else {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
}
|
||||
|
||||
fn get_storage_before_block(
|
||||
&self,
|
||||
block_number: BlockNumber,
|
||||
address: Address,
|
||||
storage_key: B256,
|
||||
) -> ProviderResult<Option<StorageEntry>> {
|
||||
let provider = match self.get_segment_provider_for_block(
|
||||
StaticFileSegment::StorageChangeSets,
|
||||
block_number,
|
||||
None,
|
||||
) {
|
||||
Ok(provider) => provider,
|
||||
Err(ProviderError::MissingStaticFileBlock(_, _)) => return Ok(None),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
let user_header = provider.user_header();
|
||||
let Some(offset) = user_header.changeset_offset(block_number) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let mut cursor = provider.cursor()?;
|
||||
let range = offset.changeset_range();
|
||||
let mut low = range.start;
|
||||
let mut high = range.end;
|
||||
|
||||
while low < high {
|
||||
let mid = low + (high - low) / 2;
|
||||
if let Some(change) = cursor.get_one::<StorageChangesetMask>(mid.into())? {
|
||||
match (change.address, change.key).cmp(&(address, storage_key)) {
|
||||
std::cmp::Ordering::Less => low = mid + 1,
|
||||
_ => high = mid,
|
||||
}
|
||||
} else {
|
||||
debug!(
|
||||
target: "provider::static_file",
|
||||
?low,
|
||||
?mid,
|
||||
?high,
|
||||
?range,
|
||||
?block_number,
|
||||
?address,
|
||||
?storage_key,
|
||||
"Cannot continue binary search for storage changeset fetch"
|
||||
);
|
||||
low = range.end;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if low < range.end &&
|
||||
let Some(change) = cursor
|
||||
.get_one::<StorageChangesetMask>(low.into())?
|
||||
.filter(|change| change.address == address && change.key == storage_key)
|
||||
{
|
||||
return Ok(Some(StorageEntry { key: change.key, value: change.value }));
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn storage_changesets_range(
|
||||
&self,
|
||||
range: RangeInclusive<BlockNumber>,
|
||||
) -> ProviderResult<Vec<(BlockNumberAddress, StorageEntry)>> {
|
||||
self.walk_storage_changeset_range(range).collect()
|
||||
}
|
||||
|
||||
fn storage_changeset_count(&self) -> ProviderResult<usize> {
|
||||
let mut count = 0;
|
||||
|
||||
let static_files = iter_static_files(&self.path).map_err(ProviderError::other)?;
|
||||
if let Some(changeset_segments) = static_files.get(StaticFileSegment::StorageChangeSets) {
|
||||
for (_, header) in changeset_segments {
|
||||
if let Some(changeset_offsets) = header.changeset_offsets() {
|
||||
for offset in changeset_offsets {
|
||||
count += offset.num_changes() as usize;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(count)
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
/// Creates an iterator for walking through account changesets in the specified block range.
|
||||
///
|
||||
@@ -2510,14 +2228,6 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
) -> StaticFileAccountChangesetWalker<Self> {
|
||||
StaticFileAccountChangesetWalker::new(self.clone(), range)
|
||||
}
|
||||
|
||||
/// Creates an iterator for walking through storage changesets in the specified block range.
|
||||
pub fn walk_storage_changeset_range(
|
||||
&self,
|
||||
range: impl RangeBounds<BlockNumber>,
|
||||
) -> StaticFileStorageChangesetWalker<Self> {
|
||||
StaticFileStorageChangesetWalker::new(self.clone(), range)
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives<BlockHeader: Value>> HeaderProvider for StaticFileProvider<N> {
|
||||
|
||||
@@ -69,19 +69,14 @@ mod tests {
|
||||
use alloy_consensus::{Header, SignableTransaction, Transaction, TxLegacy};
|
||||
use alloy_primitives::{Address, BlockHash, Signature, TxNumber, B256, U160, U256};
|
||||
use rand::seq::SliceRandom;
|
||||
use reth_db::{
|
||||
models::{AccountBeforeTx, StorageBeforeTx},
|
||||
test_utils::create_test_static_files_dir,
|
||||
};
|
||||
use reth_db::{models::AccountBeforeTx, test_utils::create_test_static_files_dir};
|
||||
use reth_db_api::{transaction::DbTxMut, CanonicalHeaders, HeaderNumbers, Headers};
|
||||
use reth_ethereum_primitives::{EthPrimitives, Receipt, TransactionSigned};
|
||||
use reth_primitives_traits::Account;
|
||||
use reth_static_file_types::{
|
||||
find_fixed_range, SegmentRangeInclusive, DEFAULT_BLOCKS_PER_STATIC_FILE,
|
||||
};
|
||||
use reth_storage_api::{
|
||||
ChangeSetReader, ReceiptProvider, StorageChangeSetReader, TransactionsProvider,
|
||||
};
|
||||
use reth_storage_api::{ChangeSetReader, ReceiptProvider, TransactionsProvider};
|
||||
use reth_testing_utils::generators::{self, random_header_range};
|
||||
use std::{collections::BTreeMap, fmt::Debug, fs, ops::Range, path::Path};
|
||||
|
||||
@@ -326,9 +321,7 @@ mod tests {
|
||||
// Append transaction/receipt if there's still a transaction count to append
|
||||
if tx_count > 0 {
|
||||
match segment {
|
||||
StaticFileSegment::Headers |
|
||||
StaticFileSegment::AccountChangeSets |
|
||||
StaticFileSegment::StorageChangeSets => {
|
||||
StaticFileSegment::Headers | StaticFileSegment::AccountChangeSets => {
|
||||
panic!("non tx based segment")
|
||||
}
|
||||
StaticFileSegment::Transactions => {
|
||||
@@ -445,9 +438,7 @@ mod tests {
|
||||
|
||||
// Prune transactions or receipts based on the segment type
|
||||
match segment {
|
||||
StaticFileSegment::Headers |
|
||||
StaticFileSegment::AccountChangeSets |
|
||||
StaticFileSegment::StorageChangeSets => {
|
||||
StaticFileSegment::Headers | StaticFileSegment::AccountChangeSets => {
|
||||
panic!("non tx based segment")
|
||||
}
|
||||
StaticFileSegment::Transactions => {
|
||||
@@ -472,9 +463,7 @@ mod tests {
|
||||
// cumulative_gas_used & nonce as ids.
|
||||
if let Some(id) = expected_tx_tip {
|
||||
match segment {
|
||||
StaticFileSegment::Headers |
|
||||
StaticFileSegment::AccountChangeSets |
|
||||
StaticFileSegment::StorageChangeSets => {
|
||||
StaticFileSegment::Headers | StaticFileSegment::AccountChangeSets => {
|
||||
panic!("non tx based segment")
|
||||
}
|
||||
StaticFileSegment::Transactions => assert_eyre(
|
||||
@@ -1044,311 +1033,4 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_storage_changeset_static_files() {
|
||||
let (static_dir, _) = create_test_static_files_dir();
|
||||
|
||||
let sf_rw = StaticFileProvider::<EthPrimitives>::read_write(&static_dir)
|
||||
.expect("Failed to create static file provider");
|
||||
|
||||
// Test writing and reading storage changesets
|
||||
{
|
||||
let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
|
||||
|
||||
// Create test data for multiple blocks
|
||||
let test_blocks = 10u64;
|
||||
let entries_per_block = 5;
|
||||
|
||||
for block_num in 0..test_blocks {
|
||||
let changeset = (0..entries_per_block)
|
||||
.map(|i| {
|
||||
let mut addr = Address::ZERO;
|
||||
addr.0[0] = block_num as u8;
|
||||
addr.0[1] = i as u8;
|
||||
StorageBeforeTx {
|
||||
address: addr,
|
||||
key: B256::with_last_byte(i as u8),
|
||||
value: U256::from(block_num * 1000 + i as u64),
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
writer.append_storage_changeset(changeset, block_num).unwrap();
|
||||
}
|
||||
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
// Verify data can be read back correctly
|
||||
{
|
||||
let provider = sf_rw
|
||||
.get_segment_provider_for_block(StaticFileSegment::StorageChangeSets, 5, None)
|
||||
.unwrap();
|
||||
|
||||
// Check that the segment header has changeset offsets
|
||||
assert!(provider.user_header().changeset_offsets().is_some());
|
||||
let offsets = provider.user_header().changeset_offsets().unwrap();
|
||||
assert_eq!(offsets.len(), 10); // Should have 10 blocks worth of offsets
|
||||
|
||||
// Verify each block has the expected number of changes
|
||||
for (i, offset) in offsets.iter().enumerate() {
|
||||
assert_eq!(offset.num_changes(), 5, "Block {} should have 5 changes", i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_storage_before_block() {
|
||||
let (static_dir, _) = create_test_static_files_dir();
|
||||
|
||||
let sf_rw = StaticFileProvider::<EthPrimitives>::read_write(&static_dir)
|
||||
.expect("Failed to create static file provider");
|
||||
|
||||
let test_address = Address::from([1u8; 20]);
|
||||
let other_address = Address::from([2u8; 20]);
|
||||
let missing_address = Address::from([3u8; 20]);
|
||||
let test_key = B256::with_last_byte(1);
|
||||
let other_key = B256::with_last_byte(2);
|
||||
|
||||
// Write changesets for multiple blocks
|
||||
{
|
||||
let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
|
||||
|
||||
// Block 0: test_address and other_address change
|
||||
writer
|
||||
.append_storage_changeset(
|
||||
vec![
|
||||
StorageBeforeTx { address: test_address, key: test_key, value: U256::ZERO },
|
||||
StorageBeforeTx {
|
||||
address: other_address,
|
||||
key: other_key,
|
||||
value: U256::from(5),
|
||||
},
|
||||
],
|
||||
0,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Block 1: only other_address changes
|
||||
writer
|
||||
.append_storage_changeset(
|
||||
vec![StorageBeforeTx {
|
||||
address: other_address,
|
||||
key: other_key,
|
||||
value: U256::from(7),
|
||||
}],
|
||||
1,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Block 2: test_address changes again
|
||||
writer
|
||||
.append_storage_changeset(
|
||||
vec![StorageBeforeTx {
|
||||
address: test_address,
|
||||
key: test_key,
|
||||
value: U256::from(9),
|
||||
}],
|
||||
2,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
// Test get_storage_before_block
|
||||
{
|
||||
let result = sf_rw.get_storage_before_block(0, test_address, test_key).unwrap();
|
||||
assert!(result.is_some());
|
||||
let entry = result.unwrap();
|
||||
assert_eq!(entry.key, test_key);
|
||||
assert_eq!(entry.value, U256::ZERO);
|
||||
|
||||
let result = sf_rw.get_storage_before_block(2, test_address, test_key).unwrap();
|
||||
assert!(result.is_some());
|
||||
let entry = result.unwrap();
|
||||
assert_eq!(entry.key, test_key);
|
||||
assert_eq!(entry.value, U256::from(9));
|
||||
|
||||
let result = sf_rw.get_storage_before_block(1, test_address, test_key).unwrap();
|
||||
assert!(result.is_none());
|
||||
|
||||
let result = sf_rw.get_storage_before_block(2, missing_address, test_key).unwrap();
|
||||
assert!(result.is_none());
|
||||
|
||||
let result = sf_rw.get_storage_before_block(1, other_address, other_key).unwrap();
|
||||
assert!(result.is_some());
|
||||
let entry = result.unwrap();
|
||||
assert_eq!(entry.key, other_key);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_storage_changeset_truncation() {
|
||||
let (static_dir, _) = create_test_static_files_dir();
|
||||
|
||||
let blocks_per_file = 10;
|
||||
let files_per_range = 3;
|
||||
let file_set_count = 3;
|
||||
let initial_file_count = files_per_range * file_set_count;
|
||||
let tip = blocks_per_file * file_set_count - 1;
|
||||
|
||||
// Setup: Create storage changesets for multiple blocks
|
||||
{
|
||||
let sf_rw: StaticFileProvider<EthPrimitives> =
|
||||
StaticFileProviderBuilder::read_write(&static_dir)
|
||||
.with_blocks_per_file(blocks_per_file)
|
||||
.build()
|
||||
.expect("failed to create static file provider");
|
||||
|
||||
let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
|
||||
|
||||
for block_num in 0..=tip {
|
||||
let num_changes = ((block_num % 5) + 1) as usize;
|
||||
let mut changeset = Vec::with_capacity(num_changes);
|
||||
|
||||
for i in 0..num_changes {
|
||||
let mut address = Address::ZERO;
|
||||
address.0[0] = block_num as u8;
|
||||
address.0[1] = i as u8;
|
||||
|
||||
changeset.push(StorageBeforeTx {
|
||||
address,
|
||||
key: B256::with_last_byte(i as u8),
|
||||
value: U256::from(block_num * 1000 + i as u64),
|
||||
});
|
||||
}
|
||||
|
||||
writer.append_storage_changeset(changeset, block_num).unwrap();
|
||||
}
|
||||
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
fn validate_truncation(
|
||||
sf_rw: &StaticFileProvider<EthPrimitives>,
|
||||
static_dir: impl AsRef<Path>,
|
||||
expected_tip: Option<u64>,
|
||||
expected_file_count: u64,
|
||||
) -> eyre::Result<()> {
|
||||
let highest_block =
|
||||
sf_rw.get_highest_static_file_block(StaticFileSegment::StorageChangeSets);
|
||||
assert_eyre(highest_block, expected_tip, "block tip mismatch")?;
|
||||
|
||||
assert_eyre(
|
||||
count_files_without_lockfile(static_dir)?,
|
||||
expected_file_count as usize,
|
||||
"file count mismatch",
|
||||
)?;
|
||||
|
||||
if let Some(tip) = expected_tip {
|
||||
let provider = sf_rw.get_segment_provider_for_block(
|
||||
StaticFileSegment::StorageChangeSets,
|
||||
tip,
|
||||
None,
|
||||
)?;
|
||||
let offsets = provider.user_header().changeset_offsets();
|
||||
assert!(offsets.is_some(), "Should have changeset offsets");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
let sf_rw = StaticFileProviderBuilder::read_write(&static_dir)
|
||||
.with_blocks_per_file(blocks_per_file)
|
||||
.build()
|
||||
.expect("failed to create static file provider");
|
||||
|
||||
sf_rw.initialize_index().expect("Failed to initialize index");
|
||||
|
||||
// Case 1: Truncate to block 20
|
||||
{
|
||||
let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
|
||||
writer.prune_storage_changesets(20).unwrap();
|
||||
writer.commit().unwrap();
|
||||
|
||||
validate_truncation(&sf_rw, &static_dir, Some(20), initial_file_count)
|
||||
.expect("Truncation validation failed");
|
||||
}
|
||||
|
||||
// Case 2: Truncate to block 9
|
||||
{
|
||||
let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
|
||||
writer.prune_storage_changesets(9).unwrap();
|
||||
writer.commit().unwrap();
|
||||
|
||||
validate_truncation(&sf_rw, &static_dir, Some(9), files_per_range)
|
||||
.expect("Truncation validation failed");
|
||||
}
|
||||
|
||||
// Case 3: Truncate all (should keep block 0)
|
||||
{
|
||||
let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
|
||||
writer.prune_storage_changesets(0).unwrap();
|
||||
writer.commit().unwrap();
|
||||
|
||||
validate_truncation(&sf_rw, &static_dir, Some(0), files_per_range)
|
||||
.expect("Truncation validation failed");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_storage_changeset_binary_search() {
|
||||
let (static_dir, _) = create_test_static_files_dir();
|
||||
|
||||
let sf_rw = StaticFileProvider::<EthPrimitives>::read_write(&static_dir)
|
||||
.expect("Failed to create static file provider");
|
||||
|
||||
let block_num = 0u64;
|
||||
let num_slots = 100;
|
||||
let address = Address::from([4u8; 20]);
|
||||
|
||||
let mut keys: Vec<B256> = Vec::with_capacity(num_slots);
|
||||
for i in 0..num_slots {
|
||||
keys.push(B256::with_last_byte(i as u8));
|
||||
}
|
||||
|
||||
{
|
||||
let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
|
||||
let changeset = keys
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, key)| StorageBeforeTx { address, key: *key, value: U256::from(i as u64) })
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
writer.append_storage_changeset(changeset, block_num).unwrap();
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
{
|
||||
let result = sf_rw.get_storage_before_block(block_num, address, keys[0]).unwrap();
|
||||
assert!(result.is_some());
|
||||
let entry = result.unwrap();
|
||||
assert_eq!(entry.key, keys[0]);
|
||||
assert_eq!(entry.value, U256::from(0));
|
||||
|
||||
let result =
|
||||
sf_rw.get_storage_before_block(block_num, address, keys[num_slots - 1]).unwrap();
|
||||
assert!(result.is_some());
|
||||
let entry = result.unwrap();
|
||||
assert_eq!(entry.key, keys[num_slots - 1]);
|
||||
|
||||
let mid = num_slots / 2;
|
||||
let result = sf_rw.get_storage_before_block(block_num, address, keys[mid]).unwrap();
|
||||
assert!(result.is_some());
|
||||
let entry = result.unwrap();
|
||||
assert_eq!(entry.key, keys[mid]);
|
||||
|
||||
let missing_key = B256::with_last_byte(255);
|
||||
let result = sf_rw.get_storage_before_block(block_num, address, missing_key).unwrap();
|
||||
assert!(result.is_none());
|
||||
|
||||
for i in (0..num_slots).step_by(10) {
|
||||
let result = sf_rw.get_storage_before_block(block_num, address, keys[i]).unwrap();
|
||||
assert!(result.is_some());
|
||||
assert_eq!(result.unwrap().key, keys[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ use alloy_consensus::BlockHeader;
|
||||
use alloy_primitives::{BlockHash, BlockNumber, TxNumber, U256};
|
||||
use parking_lot::{lock_api::RwLockWriteGuard, RawRwLock, RwLock};
|
||||
use reth_codecs::Compact;
|
||||
use reth_db::models::{AccountBeforeTx, StorageBeforeTx};
|
||||
use reth_db::models::AccountBeforeTx;
|
||||
use reth_db_api::models::CompactU256;
|
||||
use reth_nippy_jar::{NippyJar, NippyJarError, NippyJarWriter};
|
||||
use reth_node_types::NodePrimitives;
|
||||
@@ -56,11 +56,6 @@ enum PruneStrategy {
|
||||
/// The target block number to prune to.
|
||||
last_block: BlockNumber,
|
||||
},
|
||||
/// Prune storage changesets to a target block number.
|
||||
StorageChangeSets {
|
||||
/// The target block number to prune to.
|
||||
last_block: BlockNumber,
|
||||
},
|
||||
}
|
||||
|
||||
/// Static file writers for every known [`StaticFileSegment`].
|
||||
@@ -74,7 +69,6 @@ pub(crate) struct StaticFileWriters<N> {
|
||||
receipts: RwLock<Option<StaticFileProviderRW<N>>>,
|
||||
transaction_senders: RwLock<Option<StaticFileProviderRW<N>>>,
|
||||
account_change_sets: RwLock<Option<StaticFileProviderRW<N>>>,
|
||||
storage_change_sets: RwLock<Option<StaticFileProviderRW<N>>>,
|
||||
}
|
||||
|
||||
impl<N> Default for StaticFileWriters<N> {
|
||||
@@ -85,7 +79,6 @@ impl<N> Default for StaticFileWriters<N> {
|
||||
receipts: Default::default(),
|
||||
transaction_senders: Default::default(),
|
||||
account_change_sets: Default::default(),
|
||||
storage_change_sets: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -102,7 +95,6 @@ impl<N: NodePrimitives> StaticFileWriters<N> {
|
||||
StaticFileSegment::Receipts => self.receipts.write(),
|
||||
StaticFileSegment::TransactionSenders => self.transaction_senders.write(),
|
||||
StaticFileSegment::AccountChangeSets => self.account_change_sets.write(),
|
||||
StaticFileSegment::StorageChangeSets => self.storage_change_sets.write(),
|
||||
};
|
||||
|
||||
if write_guard.is_none() {
|
||||
@@ -121,7 +113,6 @@ impl<N: NodePrimitives> StaticFileWriters<N> {
|
||||
&self.receipts,
|
||||
&self.transaction_senders,
|
||||
&self.account_change_sets,
|
||||
&self.storage_change_sets,
|
||||
] {
|
||||
let mut writer = writer_lock.write();
|
||||
if let Some(writer) = writer.as_mut() {
|
||||
@@ -140,7 +131,6 @@ impl<N: NodePrimitives> StaticFileWriters<N> {
|
||||
&self.receipts,
|
||||
&self.transaction_senders,
|
||||
&self.account_change_sets,
|
||||
&self.storage_change_sets,
|
||||
] {
|
||||
let writer = writer_lock.read();
|
||||
if let Some(writer) = writer.as_ref() &&
|
||||
@@ -165,7 +155,6 @@ impl<N: NodePrimitives> StaticFileWriters<N> {
|
||||
&self.receipts,
|
||||
&self.transaction_senders,
|
||||
&self.account_change_sets,
|
||||
&self.storage_change_sets,
|
||||
] {
|
||||
let mut writer = writer_lock.write();
|
||||
if let Some(writer) = writer.as_mut() {
|
||||
@@ -399,9 +388,6 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
|
||||
PruneStrategy::AccountChangeSets { last_block } => {
|
||||
self.prune_account_changeset_data(last_block)?
|
||||
}
|
||||
PruneStrategy::StorageChangeSets { last_block } => {
|
||||
self.prune_storage_changeset_data(last_block)?
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -610,7 +596,7 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
|
||||
/// Commits to the configuration file at the end
|
||||
fn truncate_changesets(&mut self, last_block: u64) -> ProviderResult<()> {
|
||||
let segment = self.writer.user_header().segment();
|
||||
debug_assert!(segment.is_change_based());
|
||||
debug_assert_eq!(segment, StaticFileSegment::AccountChangeSets);
|
||||
|
||||
// Get the current block range
|
||||
let current_block_end = self
|
||||
@@ -1090,41 +1076,6 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Appends a block storage changeset to the static file.
|
||||
///
|
||||
/// It **CALLS** `increment_block()`.
|
||||
pub fn append_storage_changeset(
|
||||
&mut self,
|
||||
mut changeset: Vec<StorageBeforeTx>,
|
||||
block_number: u64,
|
||||
) -> ProviderResult<()> {
|
||||
debug_assert!(self.writer.user_header().segment() == StaticFileSegment::StorageChangeSets);
|
||||
let start = Instant::now();
|
||||
|
||||
self.increment_block(block_number)?;
|
||||
self.ensure_no_queued_prune()?;
|
||||
|
||||
// sort by address + storage key
|
||||
changeset.sort_by_key(|change| (change.address, change.key));
|
||||
|
||||
let mut count: u64 = 0;
|
||||
for change in changeset {
|
||||
self.append_change(&change)?;
|
||||
count += 1;
|
||||
}
|
||||
|
||||
if let Some(metrics) = &self.metrics {
|
||||
metrics.record_segment_operations(
|
||||
StaticFileSegment::StorageChangeSets,
|
||||
StaticFileProviderOperation::Append,
|
||||
count,
|
||||
Some(start.elapsed()),
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Adds an instruction to prune `to_delete` transactions during commit.
|
||||
///
|
||||
/// Note: `last_block` refers to the block the unwinds ends at.
|
||||
@@ -1176,12 +1127,6 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
|
||||
self.queue_prune(PruneStrategy::AccountChangeSets { last_block })
|
||||
}
|
||||
|
||||
/// Adds an instruction to prune storage changesets until the given block.
|
||||
pub fn prune_storage_changesets(&mut self, last_block: u64) -> ProviderResult<()> {
|
||||
debug_assert_eq!(self.writer.user_header().segment(), StaticFileSegment::StorageChangeSets);
|
||||
self.queue_prune(PruneStrategy::StorageChangeSets { last_block })
|
||||
}
|
||||
|
||||
/// Adds an instruction to prune elements during commit using the specified strategy.
|
||||
fn queue_prune(&mut self, strategy: PruneStrategy) -> ProviderResult<()> {
|
||||
self.ensure_no_queued_prune()?;
|
||||
@@ -1241,25 +1186,6 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Prunes the last storage changesets from the data file.
|
||||
fn prune_storage_changeset_data(&mut self, last_block: BlockNumber) -> ProviderResult<()> {
|
||||
let start = Instant::now();
|
||||
|
||||
debug_assert!(self.writer.user_header().segment() == StaticFileSegment::StorageChangeSets);
|
||||
|
||||
self.truncate_changesets(last_block)?;
|
||||
|
||||
if let Some(metrics) = &self.metrics {
|
||||
metrics.record_segment_operation(
|
||||
StaticFileSegment::StorageChangeSets,
|
||||
StaticFileProviderOperation::Prune,
|
||||
Some(start.elapsed()),
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Prunes the last `to_delete` receipts from the data file.
|
||||
fn prune_receipt_data(
|
||||
&mut self,
|
||||
|
||||
@@ -27,14 +27,14 @@ use reth_ethereum_primitives::EthPrimitives;
|
||||
use reth_execution_types::ExecutionOutcome;
|
||||
use reth_primitives_traits::{
|
||||
Account, Block, BlockBody, Bytecode, GotExpected, NodePrimitives, RecoveredBlock, SealedHeader,
|
||||
SignerRecoverable, StorageEntry,
|
||||
SignerRecoverable,
|
||||
};
|
||||
use reth_prune_types::{PruneCheckpoint, PruneModes, PruneSegment};
|
||||
use reth_stages_types::{StageCheckpoint, StageId};
|
||||
use reth_storage_api::{
|
||||
BlockBodyIndicesProvider, BytecodeReader, DBProvider, DatabaseProviderFactory,
|
||||
HashedPostStateProvider, NodePrimitivesProvider, StageCheckpointReader, StateProofProvider,
|
||||
StorageChangeSetReader, StorageRootProvider,
|
||||
StorageRootProvider,
|
||||
};
|
||||
use reth_storage_errors::provider::{ConsistentViewError, ProviderError, ProviderResult};
|
||||
use reth_trie::{
|
||||
@@ -989,37 +989,6 @@ impl<T: NodePrimitives, ChainSpec: Send + Sync> ChangeSetReader for MockEthProvi
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: NodePrimitives, ChainSpec: Send + Sync> StorageChangeSetReader
|
||||
for MockEthProvider<T, ChainSpec>
|
||||
{
|
||||
fn storage_changeset(
|
||||
&self,
|
||||
_block_number: BlockNumber,
|
||||
) -> ProviderResult<Vec<(reth_db_api::models::BlockNumberAddress, StorageEntry)>> {
|
||||
Ok(Vec::default())
|
||||
}
|
||||
|
||||
fn get_storage_before_block(
|
||||
&self,
|
||||
_block_number: BlockNumber,
|
||||
_address: Address,
|
||||
_storage_key: B256,
|
||||
) -> ProviderResult<Option<StorageEntry>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn storage_changesets_range(
|
||||
&self,
|
||||
_range: RangeInclusive<BlockNumber>,
|
||||
) -> ProviderResult<Vec<(reth_db_api::models::BlockNumberAddress, StorageEntry)>> {
|
||||
Ok(Vec::default())
|
||||
}
|
||||
|
||||
fn storage_changeset_count(&self) -> ProviderResult<usize> {
|
||||
Ok(0)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: NodePrimitives, ChainSpec: Send + Sync> StateReader for MockEthProvider<T, ChainSpec> {
|
||||
type Receipt = T::Receipt;
|
||||
|
||||
|
||||
@@ -10,18 +10,14 @@ use reth_chain_state::{
|
||||
CanonStateSubscriptions, ForkChoiceSubscriptions, PersistedBlockSubscriptions,
|
||||
};
|
||||
use reth_node_types::{BlockTy, HeaderTy, NodeTypesWithDB, ReceiptTy, TxTy};
|
||||
use reth_storage_api::{NodePrimitivesProvider, StorageChangeSetReader};
|
||||
use reth_storage_api::NodePrimitivesProvider;
|
||||
use std::fmt::Debug;
|
||||
|
||||
/// Helper trait to unify all provider traits for simplicity.
|
||||
pub trait FullProvider<N: NodeTypesWithDB>:
|
||||
DatabaseProviderFactory<
|
||||
DB = N::DB,
|
||||
Provider: BlockReader
|
||||
+ StageCheckpointReader
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader,
|
||||
Provider: BlockReader + StageCheckpointReader + PruneCheckpointReader + ChangeSetReader,
|
||||
> + NodePrimitivesProvider<Primitives = N::Primitives>
|
||||
+ StaticFileProviderFactory<Primitives = N::Primitives>
|
||||
+ RocksDBProviderFactory
|
||||
@@ -36,7 +32,6 @@ pub trait FullProvider<N: NodeTypesWithDB>:
|
||||
+ HashedPostStateProvider
|
||||
+ ChainSpecProvider<ChainSpec = N::ChainSpec>
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ CanonStateSubscriptions
|
||||
+ ForkChoiceSubscriptions<Header = HeaderTy<N>>
|
||||
+ PersistedBlockSubscriptions
|
||||
@@ -51,11 +46,7 @@ pub trait FullProvider<N: NodeTypesWithDB>:
|
||||
impl<T, N: NodeTypesWithDB> FullProvider<N> for T where
|
||||
T: DatabaseProviderFactory<
|
||||
DB = N::DB,
|
||||
Provider: BlockReader
|
||||
+ StageCheckpointReader
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader,
|
||||
Provider: BlockReader + StageCheckpointReader + PruneCheckpointReader + ChangeSetReader,
|
||||
> + NodePrimitivesProvider<Primitives = N::Primitives>
|
||||
+ StaticFileProviderFactory<Primitives = N::Primitives>
|
||||
+ RocksDBProviderFactory
|
||||
@@ -70,7 +61,6 @@ impl<T, N: NodeTypesWithDB> FullProvider<N> for T where
|
||||
+ HashedPostStateProvider
|
||||
+ ChainSpecProvider<ChainSpec = N::ChainSpec>
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ CanonStateSubscriptions
|
||||
+ ForkChoiceSubscriptions<Header = HeaderTy<N>>
|
||||
+ PersistedBlockSubscriptions
|
||||
|
||||
@@ -10,7 +10,7 @@ use crate::{
|
||||
};
|
||||
|
||||
#[cfg(feature = "db-api")]
|
||||
use crate::{DBProvider, DatabaseProviderFactory, StorageChangeSetReader};
|
||||
use crate::{DBProvider, DatabaseProviderFactory};
|
||||
use alloc::{boxed::Box, string::String, sync::Arc, vec::Vec};
|
||||
use alloy_consensus::transaction::TransactionMeta;
|
||||
use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumberOrTag};
|
||||
@@ -28,9 +28,7 @@ use reth_db_api::mock::{DatabaseMock, TxMock};
|
||||
use reth_db_models::{AccountBeforeTx, StoredBlockBodyIndices};
|
||||
use reth_ethereum_primitives::EthPrimitives;
|
||||
use reth_execution_types::ExecutionOutcome;
|
||||
use reth_primitives_traits::{
|
||||
Account, Bytecode, NodePrimitives, RecoveredBlock, SealedHeader, StorageEntry,
|
||||
};
|
||||
use reth_primitives_traits::{Account, Bytecode, NodePrimitives, RecoveredBlock, SealedHeader};
|
||||
#[cfg(feature = "db-api")]
|
||||
use reth_prune_types::PruneModes;
|
||||
use reth_prune_types::{PruneCheckpoint, PruneSegment};
|
||||
@@ -410,36 +408,6 @@ impl<C: Send + Sync, N: NodePrimitives> ChangeSetReader for NoopProvider<C, N> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "db-api")]
|
||||
impl<C: Send + Sync, N: NodePrimitives> StorageChangeSetReader for NoopProvider<C, N> {
|
||||
fn storage_changeset(
|
||||
&self,
|
||||
_block_number: BlockNumber,
|
||||
) -> ProviderResult<Vec<(reth_db_api::models::BlockNumberAddress, StorageEntry)>> {
|
||||
Ok(Vec::default())
|
||||
}
|
||||
|
||||
fn get_storage_before_block(
|
||||
&self,
|
||||
_block_number: BlockNumber,
|
||||
_address: Address,
|
||||
_storage_key: B256,
|
||||
) -> ProviderResult<Option<StorageEntry>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn storage_changesets_range(
|
||||
&self,
|
||||
_range: RangeInclusive<BlockNumber>,
|
||||
) -> ProviderResult<Vec<(reth_db_api::models::BlockNumberAddress, StorageEntry)>> {
|
||||
Ok(Vec::default())
|
||||
}
|
||||
|
||||
fn storage_changeset_count(&self) -> ProviderResult<usize> {
|
||||
Ok(0)
|
||||
}
|
||||
}
|
||||
|
||||
impl<C: Send + Sync, N: NodePrimitives> StateRootProvider for NoopProvider<C, N> {
|
||||
fn state_root(&self, _state: HashedPostState) -> ProviderResult<B256> {
|
||||
Ok(B256::default())
|
||||
|
||||
@@ -4,7 +4,6 @@ use alloc::{
|
||||
};
|
||||
use alloy_primitives::{Address, BlockNumber, B256};
|
||||
use core::ops::RangeInclusive;
|
||||
use reth_db_models::StorageBeforeTx;
|
||||
use reth_primitives_traits::StorageEntry;
|
||||
use reth_storage_errors::provider::ProviderResult;
|
||||
|
||||
@@ -42,44 +41,4 @@ pub trait StorageChangeSetReader: Send {
|
||||
&self,
|
||||
block_number: BlockNumber,
|
||||
) -> ProviderResult<Vec<(reth_db_api::models::BlockNumberAddress, StorageEntry)>>;
|
||||
|
||||
/// Search the block's changesets for the given address and storage key, and return the result.
|
||||
///
|
||||
/// Returns `None` if the storage slot was not changed in this block.
|
||||
fn get_storage_before_block(
|
||||
&self,
|
||||
block_number: BlockNumber,
|
||||
address: Address,
|
||||
storage_key: B256,
|
||||
) -> ProviderResult<Option<StorageEntry>>;
|
||||
|
||||
/// Get all storage changesets in a range of blocks.
|
||||
///
|
||||
/// NOTE: Get inclusive range of blocks.
|
||||
fn storage_changesets_range(
|
||||
&self,
|
||||
range: RangeInclusive<BlockNumber>,
|
||||
) -> ProviderResult<Vec<(reth_db_api::models::BlockNumberAddress, StorageEntry)>>;
|
||||
|
||||
/// Get the total count of all storage changes.
|
||||
fn storage_changeset_count(&self) -> ProviderResult<usize>;
|
||||
|
||||
/// Get storage changesets for a block as static-file rows.
|
||||
///
|
||||
/// Default implementation uses `storage_changeset` and maps to `StorageBeforeTx`.
|
||||
fn storage_block_changeset(
|
||||
&self,
|
||||
block_number: BlockNumber,
|
||||
) -> ProviderResult<Vec<StorageBeforeTx>> {
|
||||
self.storage_changeset(block_number).map(|changesets| {
|
||||
changesets
|
||||
.into_iter()
|
||||
.map(|(block_address, entry)| StorageBeforeTx {
|
||||
address: block_address.address(),
|
||||
key: entry.key,
|
||||
value: entry.value,
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,9 +10,7 @@
|
||||
use crate::{DatabaseHashedPostState, DatabaseStateRoot, DatabaseTrieCursorFactory};
|
||||
use alloy_primitives::{map::B256Map, BlockNumber, B256};
|
||||
use parking_lot::RwLock;
|
||||
use reth_storage_api::{
|
||||
BlockNumReader, ChangeSetReader, DBProvider, StageCheckpointReader, StorageChangeSetReader,
|
||||
};
|
||||
use reth_storage_api::{BlockNumReader, ChangeSetReader, DBProvider, StageCheckpointReader};
|
||||
use reth_storage_errors::provider::{ProviderError, ProviderResult};
|
||||
use reth_trie::{
|
||||
changesets::compute_trie_changesets,
|
||||
@@ -67,11 +65,7 @@ pub fn compute_block_trie_changesets<Provider>(
|
||||
block_number: BlockNumber,
|
||||
) -> Result<TrieUpdatesSorted, ProviderError>
|
||||
where
|
||||
Provider: DBProvider
|
||||
+ StageCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader,
|
||||
Provider: DBProvider + StageCheckpointReader + ChangeSetReader + BlockNumReader,
|
||||
{
|
||||
debug!(
|
||||
target: "trie::changeset_cache",
|
||||
@@ -181,11 +175,7 @@ pub fn compute_block_trie_updates<Provider>(
|
||||
block_number: BlockNumber,
|
||||
) -> ProviderResult<TrieUpdatesSorted>
|
||||
where
|
||||
Provider: DBProvider
|
||||
+ StageCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader,
|
||||
Provider: DBProvider + StageCheckpointReader + ChangeSetReader + BlockNumReader,
|
||||
{
|
||||
let tx = provider.tx_ref();
|
||||
|
||||
@@ -333,11 +323,7 @@ impl ChangesetCache {
|
||||
provider: &P,
|
||||
) -> ProviderResult<Arc<TrieUpdatesSorted>>
|
||||
where
|
||||
P: DBProvider
|
||||
+ StageCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader,
|
||||
P: DBProvider + StageCheckpointReader + ChangeSetReader + BlockNumReader,
|
||||
{
|
||||
// Try cache first (with read lock)
|
||||
{
|
||||
@@ -422,11 +408,7 @@ impl ChangesetCache {
|
||||
range: RangeInclusive<BlockNumber>,
|
||||
) -> ProviderResult<TrieUpdatesSorted>
|
||||
where
|
||||
P: DBProvider
|
||||
+ StageCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader,
|
||||
P: DBProvider + StageCheckpointReader + ChangeSetReader + BlockNumReader,
|
||||
{
|
||||
// Get the database tip block number
|
||||
let db_tip_block = provider
|
||||
|
||||
@@ -18,9 +18,7 @@ pub use hashed_cursor::{
|
||||
pub use prefix_set::{load_prefix_sets_with_provider, PrefixSetLoader};
|
||||
pub use proof::{DatabaseProof, DatabaseStorageProof};
|
||||
pub use state::{DatabaseHashedPostState, DatabaseStateRoot};
|
||||
pub use storage::{
|
||||
hashed_storage_from_reverts_with_provider, DatabaseHashedStorage, DatabaseStorageRoot,
|
||||
};
|
||||
pub use storage::{DatabaseHashedStorage, DatabaseStorageRoot};
|
||||
pub use trie_cursor::{
|
||||
DatabaseAccountTrieCursor, DatabaseStorageTrieCursor, DatabaseTrieCursorFactory,
|
||||
};
|
||||
|
||||
@@ -14,7 +14,7 @@ use reth_db_api::{
|
||||
DatabaseError,
|
||||
};
|
||||
use reth_primitives_traits::StorageEntry;
|
||||
use reth_storage_api::{ChangeSetReader, DBProvider, StorageChangeSetReader};
|
||||
use reth_storage_api::{ChangeSetReader, DBProvider};
|
||||
use reth_storage_errors::provider::ProviderError;
|
||||
use reth_trie::{
|
||||
prefix_set::{PrefixSetMut, TriePrefixSets},
|
||||
@@ -93,7 +93,7 @@ pub fn load_prefix_sets_with_provider<Provider, KH>(
|
||||
range: RangeInclusive<BlockNumber>,
|
||||
) -> Result<TriePrefixSets, ProviderError>
|
||||
where
|
||||
Provider: ChangeSetReader + StorageChangeSetReader + DBProvider,
|
||||
Provider: ChangeSetReader + DBProvider,
|
||||
KH: KeyHasher,
|
||||
{
|
||||
let tx = provider.tx_ref();
|
||||
@@ -118,9 +118,12 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// Walk storage changesets using the provider (handles static files + database)
|
||||
let storage_changesets = provider.storage_changesets_range(range)?;
|
||||
for (BlockNumberAddress((_, address)), StorageEntry { key, .. }) in storage_changesets {
|
||||
// Walk storage changeset and insert storage prefixes
|
||||
// Note: Storage changesets don't have static files yet, so we still use direct cursor
|
||||
let mut storage_cursor = tx.cursor_dup_read::<tables::StorageChangeSets>()?;
|
||||
let storage_range = BlockNumberAddress::range(range);
|
||||
for storage_entry in storage_cursor.walk_range(storage_range)? {
|
||||
let (BlockNumberAddress((_, address)), StorageEntry { key, .. }) = storage_entry?;
|
||||
let hashed_address = KH::hash_key(address);
|
||||
account_prefix_set.insert(Nibbles::unpack(hashed_address));
|
||||
storage_prefix_sets
|
||||
|
||||
@@ -3,11 +3,13 @@ use crate::{
|
||||
};
|
||||
use alloy_primitives::{map::B256Map, BlockNumber, B256};
|
||||
use reth_db_api::{
|
||||
models::{AccountBeforeTx, BlockNumberAddress},
|
||||
cursor::DbCursorRO,
|
||||
models::{AccountBeforeTx, BlockNumberAddress, BlockNumberAddressRange},
|
||||
tables,
|
||||
transaction::DbTx,
|
||||
};
|
||||
use reth_execution_errors::StateRootError;
|
||||
use reth_storage_api::{BlockNumReader, ChangeSetReader, DBProvider, StorageChangeSetReader};
|
||||
use reth_storage_api::{BlockNumReader, ChangeSetReader, DBProvider};
|
||||
use reth_storage_errors::provider::ProviderError;
|
||||
use reth_trie::{
|
||||
hashed_cursor::HashedPostStateCursorFactory, trie_cursor::InMemoryTrieCursorFactory,
|
||||
@@ -32,7 +34,7 @@ pub trait DatabaseStateRoot<'a, TX>: Sized {
|
||||
///
|
||||
/// An instance of state root calculator with account and storage prefixes loaded.
|
||||
fn incremental_root_calculator(
|
||||
provider: &'a (impl ChangeSetReader + StorageChangeSetReader + DBProvider<Tx = TX>),
|
||||
provider: &'a (impl ChangeSetReader + DBProvider<Tx = TX>),
|
||||
range: RangeInclusive<BlockNumber>,
|
||||
) -> Result<Self, StateRootError>;
|
||||
|
||||
@@ -43,7 +45,7 @@ pub trait DatabaseStateRoot<'a, TX>: Sized {
|
||||
///
|
||||
/// The updated state root.
|
||||
fn incremental_root(
|
||||
provider: &'a (impl ChangeSetReader + StorageChangeSetReader + DBProvider<Tx = TX>),
|
||||
provider: &'a (impl ChangeSetReader + DBProvider<Tx = TX>),
|
||||
range: RangeInclusive<BlockNumber>,
|
||||
) -> Result<B256, StateRootError>;
|
||||
|
||||
@@ -56,7 +58,7 @@ pub trait DatabaseStateRoot<'a, TX>: Sized {
|
||||
///
|
||||
/// The updated state root and the trie updates.
|
||||
fn incremental_root_with_updates(
|
||||
provider: &'a (impl ChangeSetReader + StorageChangeSetReader + DBProvider<Tx = TX>),
|
||||
provider: &'a (impl ChangeSetReader + DBProvider<Tx = TX>),
|
||||
range: RangeInclusive<BlockNumber>,
|
||||
) -> Result<(B256, TrieUpdates), StateRootError>;
|
||||
|
||||
@@ -67,7 +69,7 @@ pub trait DatabaseStateRoot<'a, TX>: Sized {
|
||||
///
|
||||
/// The intermediate progress of state root computation.
|
||||
fn incremental_root_with_progress(
|
||||
provider: &'a (impl ChangeSetReader + StorageChangeSetReader + DBProvider<Tx = TX>),
|
||||
provider: &'a (impl ChangeSetReader + DBProvider<Tx = TX>),
|
||||
range: RangeInclusive<BlockNumber>,
|
||||
) -> Result<StateRootProgress, StateRootError>;
|
||||
|
||||
@@ -131,7 +133,7 @@ pub trait DatabaseHashedPostState: Sized {
|
||||
/// Initializes [`HashedPostStateSorted`] from reverts. Iterates over state reverts in the
|
||||
/// specified range and aggregates them into sorted hashed state.
|
||||
fn from_reverts<KH: KeyHasher>(
|
||||
provider: &(impl ChangeSetReader + StorageChangeSetReader + BlockNumReader + DBProvider),
|
||||
provider: &(impl ChangeSetReader + BlockNumReader + DBProvider),
|
||||
range: impl RangeBounds<BlockNumber>,
|
||||
) -> Result<HashedPostStateSorted, ProviderError>;
|
||||
}
|
||||
@@ -144,7 +146,7 @@ impl<'a, TX: DbTx> DatabaseStateRoot<'a, TX>
|
||||
}
|
||||
|
||||
fn incremental_root_calculator(
|
||||
provider: &'a (impl ChangeSetReader + StorageChangeSetReader + DBProvider<Tx = TX>),
|
||||
provider: &'a (impl ChangeSetReader + DBProvider<Tx = TX>),
|
||||
range: RangeInclusive<BlockNumber>,
|
||||
) -> Result<Self, StateRootError> {
|
||||
let loaded_prefix_sets =
|
||||
@@ -153,7 +155,7 @@ impl<'a, TX: DbTx> DatabaseStateRoot<'a, TX>
|
||||
}
|
||||
|
||||
fn incremental_root(
|
||||
provider: &'a (impl ChangeSetReader + StorageChangeSetReader + DBProvider<Tx = TX>),
|
||||
provider: &'a (impl ChangeSetReader + DBProvider<Tx = TX>),
|
||||
range: RangeInclusive<BlockNumber>,
|
||||
) -> Result<B256, StateRootError> {
|
||||
debug!(target: "trie::loader", ?range, "incremental state root");
|
||||
@@ -161,7 +163,7 @@ impl<'a, TX: DbTx> DatabaseStateRoot<'a, TX>
|
||||
}
|
||||
|
||||
fn incremental_root_with_updates(
|
||||
provider: &'a (impl ChangeSetReader + StorageChangeSetReader + DBProvider<Tx = TX>),
|
||||
provider: &'a (impl ChangeSetReader + DBProvider<Tx = TX>),
|
||||
range: RangeInclusive<BlockNumber>,
|
||||
) -> Result<(B256, TrieUpdates), StateRootError> {
|
||||
debug!(target: "trie::loader", ?range, "incremental state root");
|
||||
@@ -169,7 +171,7 @@ impl<'a, TX: DbTx> DatabaseStateRoot<'a, TX>
|
||||
}
|
||||
|
||||
fn incremental_root_with_progress(
|
||||
provider: &'a (impl ChangeSetReader + StorageChangeSetReader + DBProvider<Tx = TX>),
|
||||
provider: &'a (impl ChangeSetReader + DBProvider<Tx = TX>),
|
||||
range: RangeInclusive<BlockNumber>,
|
||||
) -> Result<StateRootProgress, StateRootError> {
|
||||
debug!(target: "trie::loader", ?range, "incremental state root with progress");
|
||||
@@ -246,9 +248,11 @@ impl DatabaseHashedPostState for HashedPostStateSorted {
|
||||
/// - Hashes keys and returns them already ordered for trie iteration.
|
||||
#[instrument(target = "trie::db", skip(provider), fields(range))]
|
||||
fn from_reverts<KH: KeyHasher>(
|
||||
provider: &(impl ChangeSetReader + StorageChangeSetReader + BlockNumReader + DBProvider),
|
||||
provider: &(impl ChangeSetReader + BlockNumReader + DBProvider),
|
||||
range: impl RangeBounds<BlockNumber>,
|
||||
) -> Result<Self, ProviderError> {
|
||||
let tx = provider.tx_ref();
|
||||
|
||||
// Extract concrete start/end values to use for both account and storage changesets.
|
||||
let start = match range.start_bound() {
|
||||
Bound::Included(&n) => n,
|
||||
@@ -262,6 +266,9 @@ impl DatabaseHashedPostState for HashedPostStateSorted {
|
||||
Bound::Unbounded => BlockNumber::MAX,
|
||||
};
|
||||
|
||||
// Convert to BlockNumberAddressRange for storage changesets.
|
||||
let storage_range: BlockNumberAddressRange = (start..end).into();
|
||||
|
||||
// Iterate over account changesets and record value before first occurring account change
|
||||
let mut accounts = Vec::new();
|
||||
let mut seen_accounts = HashSet::new();
|
||||
@@ -273,23 +280,20 @@ impl DatabaseHashedPostState for HashedPostStateSorted {
|
||||
}
|
||||
accounts.sort_unstable_by_key(|(hash, _)| *hash);
|
||||
|
||||
// Read storages into B256Map<Vec<_>> with HashSet to track seen keys.
|
||||
// Read storages directly into B256Map<Vec<_>> with HashSet to track seen keys.
|
||||
// Only keep the first (oldest) occurrence of each (address, slot) pair.
|
||||
let mut storages = B256Map::<Vec<_>>::default();
|
||||
let mut seen_storage_keys = HashSet::new();
|
||||
let mut storage_changesets_cursor = tx.cursor_read::<tables::StorageChangeSets>()?;
|
||||
|
||||
if start < end {
|
||||
let end_inclusive = end.saturating_sub(1);
|
||||
for (BlockNumberAddress((_, address)), storage) in
|
||||
provider.storage_changesets_range(start..=end_inclusive)?
|
||||
{
|
||||
if seen_storage_keys.insert((address, storage.key)) {
|
||||
let hashed_address = KH::hash_key(address);
|
||||
storages
|
||||
.entry(hashed_address)
|
||||
.or_default()
|
||||
.push((KH::hash_key(storage.key), storage.value));
|
||||
}
|
||||
for entry in storage_changesets_cursor.walk_range(storage_range)? {
|
||||
let (BlockNumberAddress((_, address)), storage) = entry?;
|
||||
if seen_storage_keys.insert((address, storage.key)) {
|
||||
let hashed_address = KH::hash_key(address);
|
||||
storages
|
||||
.entry(hashed_address)
|
||||
.or_default()
|
||||
.push((KH::hash_key(storage.key), storage.value));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,8 +4,6 @@ use reth_db_api::{
|
||||
cursor::DbCursorRO, models::BlockNumberAddress, tables, transaction::DbTx, DatabaseError,
|
||||
};
|
||||
use reth_execution_errors::StorageRootError;
|
||||
use reth_storage_api::{BlockNumReader, StorageChangeSetReader};
|
||||
use reth_storage_errors::provider::ProviderResult;
|
||||
use reth_trie::{
|
||||
hashed_cursor::HashedPostStateCursorFactory, HashedPostState, HashedStorage, StorageRoot,
|
||||
};
|
||||
@@ -36,36 +34,6 @@ pub trait DatabaseHashedStorage<TX>: Sized {
|
||||
fn from_reverts(tx: &TX, address: Address, from: BlockNumber) -> Result<Self, DatabaseError>;
|
||||
}
|
||||
|
||||
/// Initializes [`HashedStorage`] from reverts using a provider.
|
||||
pub fn hashed_storage_from_reverts_with_provider<P>(
|
||||
provider: &P,
|
||||
address: Address,
|
||||
from: BlockNumber,
|
||||
) -> ProviderResult<HashedStorage>
|
||||
where
|
||||
P: StorageChangeSetReader + BlockNumReader,
|
||||
{
|
||||
let mut storage = HashedStorage::new(false);
|
||||
let tip = provider.last_block_number()?;
|
||||
|
||||
if from > tip {
|
||||
return Ok(storage)
|
||||
}
|
||||
|
||||
for (BlockNumberAddress((_, storage_address)), storage_change) in
|
||||
provider.storage_changesets_range(from..=tip)?
|
||||
{
|
||||
if storage_address == address {
|
||||
let hashed_slot = keccak256(storage_change.key);
|
||||
if let hash_map::Entry::Vacant(entry) = storage.storage.entry(hashed_slot) {
|
||||
entry.insert(storage_change.value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(storage)
|
||||
}
|
||||
|
||||
impl<'a, TX: DbTx> DatabaseStorageRoot<'a, TX>
|
||||
for StorageRoot<DatabaseTrieCursorFactory<&'a TX>, DatabaseHashedCursorFactory<&'a TX>>
|
||||
{
|
||||
|
||||
@@ -13,8 +13,8 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
# reth
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-execution-errors.workspace = true
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-storage-errors.workspace = true
|
||||
reth-trie-common.workspace = true
|
||||
|
||||
@@ -197,7 +197,7 @@ impl ParallelProof {
|
||||
let (result_tx, result_rx) = crossbeam_unbounded();
|
||||
let account_multiproof_start_time = Instant::now();
|
||||
|
||||
let input = AccountMultiproofInput::Legacy {
|
||||
let input = AccountMultiproofInput {
|
||||
targets,
|
||||
prefix_sets,
|
||||
collect_branch_node_masks: self.collect_branch_node_masks,
|
||||
@@ -208,6 +208,7 @@ impl ParallelProof {
|
||||
HashedPostState::default(),
|
||||
account_multiproof_start_time,
|
||||
),
|
||||
v2_proofs_enabled: self.v2_proofs_enabled,
|
||||
};
|
||||
|
||||
self.proof_worker_handle
|
||||
@@ -221,9 +222,7 @@ impl ParallelProof {
|
||||
)
|
||||
})?;
|
||||
|
||||
let ProofResult::Legacy(multiproof, stats) = proof_result_msg.result? else {
|
||||
panic!("AccountMultiproofInput::Legacy was submitted, expected legacy result")
|
||||
};
|
||||
let ProofResult { proof: multiproof, stats } = proof_result_msg.result?;
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
self.metrics.record(stats);
|
||||
@@ -236,7 +235,7 @@ impl ParallelProof {
|
||||
leaves_added = stats.leaves_added(),
|
||||
missed_leaves = stats.missed_leaves(),
|
||||
precomputed_storage_roots = stats.precomputed_storage_roots(),
|
||||
"Calculated decoded proof",
|
||||
"Calculated decoded proof"
|
||||
);
|
||||
|
||||
Ok(multiproof)
|
||||
|
||||
@@ -32,8 +32,6 @@
|
||||
use crate::{
|
||||
root::ParallelStateRootError,
|
||||
stats::{ParallelTrieStats, ParallelTrieTracker},
|
||||
targets_v2::MultiProofTargetsV2,
|
||||
value_encoder::AsyncAccountValueEncoder,
|
||||
StorageRootTargets,
|
||||
};
|
||||
use alloy_primitives::{
|
||||
@@ -51,11 +49,11 @@ use reth_trie::{
|
||||
node_iter::{TrieElement, TrieNodeIter},
|
||||
prefix_set::TriePrefixSets,
|
||||
proof::{ProofBlindedAccountProvider, ProofBlindedStorageProvider, StorageProof},
|
||||
proof_v2,
|
||||
proof_v2::{self, StorageProofCalculator},
|
||||
trie_cursor::{InstrumentedTrieCursor, TrieCursorFactory, TrieCursorMetricsCache},
|
||||
walker::TrieWalker,
|
||||
DecodedMultiProof, DecodedMultiProofV2, DecodedStorageMultiProof, HashBuilder, HashedPostState,
|
||||
MultiProofTargets, Nibbles, ProofTrieNode, TRIE_ACCOUNT_RLP_MAX_SIZE,
|
||||
DecodedMultiProof, DecodedStorageMultiProof, HashBuilder, HashedPostState, MultiProofTargets,
|
||||
Nibbles, ProofTrieNode, TRIE_ACCOUNT_RLP_MAX_SIZE,
|
||||
};
|
||||
use reth_trie_common::{
|
||||
added_removed_keys::MultiAddedRemovedKeys,
|
||||
@@ -222,8 +220,7 @@ impl ProofWorkerHandle {
|
||||
metrics,
|
||||
#[cfg(feature = "metrics")]
|
||||
cursor_metrics,
|
||||
)
|
||||
.with_v2_proofs(v2_proofs_enabled);
|
||||
);
|
||||
if let Err(error) = worker.run() {
|
||||
error!(
|
||||
target: "trie::proof_task",
|
||||
@@ -336,12 +333,16 @@ impl ProofWorkerHandle {
|
||||
ProviderError::other(std::io::Error::other("account workers unavailable"));
|
||||
|
||||
if let AccountWorkerJob::AccountMultiproof { input } = err.0 {
|
||||
let ProofResultContext {
|
||||
sender: result_tx,
|
||||
sequence_number: seq,
|
||||
state,
|
||||
start_time: start,
|
||||
} = input.into_proof_result_sender();
|
||||
let AccountMultiproofInput {
|
||||
proof_result_sender:
|
||||
ProofResultContext {
|
||||
sender: result_tx,
|
||||
sequence_number: seq,
|
||||
state,
|
||||
start_time: start,
|
||||
},
|
||||
..
|
||||
} = *input;
|
||||
|
||||
let _ = result_tx.send(ProofResultMessage {
|
||||
sequence_number: seq,
|
||||
@@ -604,65 +605,11 @@ impl TrieNodeProvider for ProofTaskTrieNodeProvider {
|
||||
|
||||
/// Result of a multiproof calculation.
|
||||
#[derive(Debug)]
|
||||
pub enum ProofResult {
|
||||
/// Legacy multiproof calculation result.
|
||||
Legacy(DecodedMultiProof, ParallelTrieStats),
|
||||
/// V2 multiproof calculation result.
|
||||
V2(DecodedMultiProofV2),
|
||||
}
|
||||
|
||||
impl ProofResult {
|
||||
/// Creates an empty [`ProofResult`] of the appropriate variant based on `v2_enabled`.
|
||||
///
|
||||
/// Use this when constructing empty proofs (e.g., for state updates where all targets
|
||||
/// were already fetched) to ensure consistency with the proof version being used.
|
||||
pub fn empty(v2_enabled: bool) -> Self {
|
||||
if v2_enabled {
|
||||
Self::V2(DecodedMultiProofV2::default())
|
||||
} else {
|
||||
let stats = ParallelTrieTracker::default().finish();
|
||||
Self::Legacy(DecodedMultiProof::default(), stats)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if the result contains no proofs
|
||||
pub fn is_empty(&self) -> bool {
|
||||
match self {
|
||||
Self::Legacy(proof, _) => proof.is_empty(),
|
||||
Self::V2(proof) => proof.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Extends the receiver with the value of the given results.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// This method panics if the two [`ProofResult`]s are not the same variant.
|
||||
pub fn extend(&mut self, other: Self) {
|
||||
match (self, other) {
|
||||
(Self::Legacy(proof, _), Self::Legacy(other, _)) => proof.extend(other),
|
||||
(Self::V2(proof), Self::V2(other)) => proof.extend(other),
|
||||
_ => panic!("mismatched ProofResults, cannot extend one with the other"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of account proofs.
|
||||
pub fn account_proofs_len(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(proof, _) => proof.account_subtree.len(),
|
||||
Self::V2(proof) => proof.account_proofs.len(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the total number of storage proofs
|
||||
pub fn storage_proofs_len(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(proof, _) => {
|
||||
proof.storages.values().map(|p| p.subtree.len()).sum::<usize>()
|
||||
}
|
||||
Self::V2(proof) => proof.storage_proofs.values().map(|p| p.len()).sum::<usize>(),
|
||||
}
|
||||
}
|
||||
pub struct ProofResult {
|
||||
/// The account multiproof
|
||||
pub proof: DecodedMultiProof,
|
||||
/// Statistics collected during proof computation
|
||||
pub stats: ParallelTrieStats,
|
||||
}
|
||||
|
||||
/// Channel used by worker threads to deliver `ProofResultMessage` items back to
|
||||
@@ -942,7 +889,7 @@ where
|
||||
&self,
|
||||
proof_tx: &ProofTaskTx<Provider>,
|
||||
v2_calculator: Option<
|
||||
&mut proof_v2::StorageProofCalculator<
|
||||
&mut StorageProofCalculator<
|
||||
<Provider as TrieCursorFactory>::StorageTrieCursor<'_>,
|
||||
<Provider as HashedCursorFactory>::StorageCursor<'_>,
|
||||
>,
|
||||
@@ -1106,8 +1053,6 @@ struct AccountProofWorker<Factory> {
|
||||
/// Cursor metrics for this worker
|
||||
#[cfg(feature = "metrics")]
|
||||
cursor_metrics: ProofTaskCursorMetrics,
|
||||
/// Set to true if V2 proofs are enabled.
|
||||
v2_enabled: bool,
|
||||
}
|
||||
|
||||
impl<Factory> AccountProofWorker<Factory>
|
||||
@@ -1137,16 +1082,9 @@ where
|
||||
metrics,
|
||||
#[cfg(feature = "metrics")]
|
||||
cursor_metrics,
|
||||
v2_enabled: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Changes whether or not V2 proofs are enabled.
|
||||
const fn with_v2_proofs(mut self, v2_enabled: bool) -> Self {
|
||||
self.v2_enabled = v2_enabled;
|
||||
self
|
||||
}
|
||||
|
||||
/// Runs the worker loop, processing jobs until the channel closes.
|
||||
///
|
||||
/// # Lifecycle
|
||||
@@ -1179,17 +1117,6 @@ where
|
||||
let mut account_nodes_processed = 0u64;
|
||||
let mut cursor_metrics_cache = ProofTaskCursorMetricsCache::default();
|
||||
|
||||
let mut v2_calculator = if self.v2_enabled {
|
||||
let trie_cursor = proof_tx.provider.account_trie_cursor()?;
|
||||
let hashed_cursor = proof_tx.provider.hashed_account_cursor()?;
|
||||
Some(proof_v2::ProofCalculator::<_, _, AsyncAccountValueEncoder>::new(
|
||||
trie_cursor,
|
||||
hashed_cursor,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Count this worker as available only after successful initialization.
|
||||
self.available_workers.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
@@ -1201,7 +1128,6 @@ where
|
||||
AccountWorkerJob::AccountMultiproof { input } => {
|
||||
self.process_account_multiproof(
|
||||
&proof_tx,
|
||||
v2_calculator.as_mut(),
|
||||
*input,
|
||||
&mut account_proofs_processed,
|
||||
&mut cursor_metrics_cache,
|
||||
@@ -1240,18 +1166,26 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn compute_legacy_account_multiproof<Provider>(
|
||||
/// Processes an account multiproof request.
|
||||
fn process_account_multiproof<Provider>(
|
||||
&self,
|
||||
proof_tx: &ProofTaskTx<Provider>,
|
||||
targets: MultiProofTargets,
|
||||
mut prefix_sets: TriePrefixSets,
|
||||
collect_branch_node_masks: bool,
|
||||
multi_added_removed_keys: Option<Arc<MultiAddedRemovedKeys>>,
|
||||
proof_cursor_metrics: &mut ProofTaskCursorMetricsCache,
|
||||
) -> Result<ProofResult, ParallelStateRootError>
|
||||
where
|
||||
input: AccountMultiproofInput,
|
||||
account_proofs_processed: &mut u64,
|
||||
cursor_metrics_cache: &mut ProofTaskCursorMetricsCache,
|
||||
) where
|
||||
Provider: TrieCursorFactory + HashedCursorFactory,
|
||||
{
|
||||
let AccountMultiproofInput {
|
||||
targets,
|
||||
mut prefix_sets,
|
||||
collect_branch_node_masks,
|
||||
multi_added_removed_keys,
|
||||
proof_result_sender:
|
||||
ProofResultContext { sender: result_tx, sequence_number: seq, state, start_time: start },
|
||||
v2_proofs_enabled,
|
||||
} = input;
|
||||
|
||||
let span = debug_span!(
|
||||
target: "trie::proof_task",
|
||||
"Account multiproof calculation",
|
||||
@@ -1265,6 +1199,8 @@ where
|
||||
"Processing account multiproof"
|
||||
);
|
||||
|
||||
let proof_start = Instant::now();
|
||||
|
||||
let mut tracker = ParallelTrieTracker::default();
|
||||
|
||||
let mut storage_prefix_sets = std::mem::take(&mut prefix_sets.storage_prefix_sets);
|
||||
@@ -1274,14 +1210,29 @@ where
|
||||
|
||||
tracker.set_precomputed_storage_roots(storage_root_targets_len as u64);
|
||||
|
||||
let storage_proof_receivers = dispatch_storage_proofs(
|
||||
let storage_proof_receivers = match dispatch_storage_proofs(
|
||||
&self.storage_work_tx,
|
||||
&targets,
|
||||
&mut storage_prefix_sets,
|
||||
collect_branch_node_masks,
|
||||
multi_added_removed_keys.as_ref(),
|
||||
)?;
|
||||
v2_proofs_enabled,
|
||||
) {
|
||||
Ok(receivers) => receivers,
|
||||
Err(error) => {
|
||||
// Send error through result channel
|
||||
error!(target: "trie::proof_task", "Failed to dispatch storage proofs: {error}");
|
||||
let _ = result_tx.send(ProofResultMessage {
|
||||
sequence_number: seq,
|
||||
result: Err(error),
|
||||
elapsed: start.elapsed(),
|
||||
state,
|
||||
});
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Use the missed leaves cache passed from the multiproof manager
|
||||
let account_prefix_set = std::mem::take(&mut prefix_sets.account_prefix_set);
|
||||
|
||||
let ctx = AccountMultiproofParams {
|
||||
@@ -1293,115 +1244,17 @@ where
|
||||
cached_storage_roots: &self.cached_storage_roots,
|
||||
};
|
||||
|
||||
let result = build_account_multiproof_with_storage_roots(
|
||||
&proof_tx.provider,
|
||||
ctx,
|
||||
&mut tracker,
|
||||
proof_cursor_metrics,
|
||||
);
|
||||
let result =
|
||||
build_account_multiproof_with_storage_roots(&proof_tx.provider, ctx, &mut tracker);
|
||||
|
||||
let now = Instant::now();
|
||||
let proof_elapsed = now.duration_since(proof_start);
|
||||
let total_elapsed = now.duration_since(start);
|
||||
let proof_cursor_metrics = tracker.cursor_metrics;
|
||||
proof_cursor_metrics.record_spans();
|
||||
|
||||
let stats = tracker.finish();
|
||||
result.map(|proof| ProofResult::Legacy(proof, stats))
|
||||
}
|
||||
|
||||
fn compute_v2_account_multiproof<Provider>(
|
||||
&self,
|
||||
v2_calculator: &mut proof_v2::ProofCalculator<
|
||||
<Provider as TrieCursorFactory>::AccountTrieCursor<'_>,
|
||||
<Provider as HashedCursorFactory>::AccountCursor<'_>,
|
||||
AsyncAccountValueEncoder,
|
||||
>,
|
||||
targets: MultiProofTargetsV2,
|
||||
) -> Result<ProofResult, ParallelStateRootError>
|
||||
where
|
||||
Provider: TrieCursorFactory + HashedCursorFactory,
|
||||
{
|
||||
let MultiProofTargetsV2 { mut account_targets, storage_targets } = targets;
|
||||
|
||||
let span = debug_span!(
|
||||
target: "trie::proof_task",
|
||||
"Account V2 multiproof calculation",
|
||||
account_targets = account_targets.len(),
|
||||
storage_targets = storage_targets.values().map(|t| t.len()).sum::<usize>(),
|
||||
worker_id = self.worker_id,
|
||||
);
|
||||
let _span_guard = span.enter();
|
||||
|
||||
trace!(target: "trie::proof_task", "Processing V2 account multiproof");
|
||||
|
||||
let storage_proof_receivers =
|
||||
dispatch_v2_storage_proofs(&self.storage_work_tx, &account_targets, storage_targets)?;
|
||||
|
||||
let mut value_encoder = AsyncAccountValueEncoder::new(
|
||||
self.storage_work_tx.clone(),
|
||||
storage_proof_receivers,
|
||||
self.cached_storage_roots.clone(),
|
||||
);
|
||||
|
||||
let proof = DecodedMultiProofV2 {
|
||||
account_proofs: v2_calculator.proof(&mut value_encoder, &mut account_targets)?,
|
||||
storage_proofs: value_encoder.into_storage_proofs()?,
|
||||
};
|
||||
|
||||
Ok(ProofResult::V2(proof))
|
||||
}
|
||||
|
||||
/// Processes an account multiproof request.
|
||||
fn process_account_multiproof<Provider>(
|
||||
&self,
|
||||
proof_tx: &ProofTaskTx<Provider>,
|
||||
v2_calculator: Option<
|
||||
&mut proof_v2::ProofCalculator<
|
||||
<Provider as TrieCursorFactory>::AccountTrieCursor<'_>,
|
||||
<Provider as HashedCursorFactory>::AccountCursor<'_>,
|
||||
AsyncAccountValueEncoder,
|
||||
>,
|
||||
>,
|
||||
input: AccountMultiproofInput,
|
||||
account_proofs_processed: &mut u64,
|
||||
cursor_metrics_cache: &mut ProofTaskCursorMetricsCache,
|
||||
) where
|
||||
Provider: TrieCursorFactory + HashedCursorFactory,
|
||||
{
|
||||
let mut proof_cursor_metrics = ProofTaskCursorMetricsCache::default();
|
||||
let proof_start = Instant::now();
|
||||
|
||||
let (proof_result_sender, result) = match input {
|
||||
AccountMultiproofInput::Legacy {
|
||||
targets,
|
||||
prefix_sets,
|
||||
collect_branch_node_masks,
|
||||
multi_added_removed_keys,
|
||||
proof_result_sender,
|
||||
} => (
|
||||
proof_result_sender,
|
||||
self.compute_legacy_account_multiproof(
|
||||
proof_tx,
|
||||
targets,
|
||||
prefix_sets,
|
||||
collect_branch_node_masks,
|
||||
multi_added_removed_keys,
|
||||
&mut proof_cursor_metrics,
|
||||
),
|
||||
),
|
||||
AccountMultiproofInput::V2 { targets, proof_result_sender } => (
|
||||
proof_result_sender,
|
||||
self.compute_v2_account_multiproof::<Provider>(
|
||||
v2_calculator.expect("v2 calculator provided"),
|
||||
targets,
|
||||
),
|
||||
),
|
||||
};
|
||||
|
||||
let ProofResultContext {
|
||||
sender: result_tx,
|
||||
sequence_number: seq,
|
||||
state,
|
||||
start_time: start,
|
||||
} = proof_result_sender;
|
||||
|
||||
let proof_elapsed = proof_start.elapsed();
|
||||
let total_elapsed = start.elapsed();
|
||||
let result = result.map(|proof| ProofResult { proof, stats });
|
||||
*account_proofs_processed += 1;
|
||||
|
||||
// Send result to MultiProofTask
|
||||
@@ -1422,8 +1275,6 @@ where
|
||||
);
|
||||
}
|
||||
|
||||
proof_cursor_metrics.record_spans();
|
||||
|
||||
trace!(
|
||||
target: "trie::proof_task",
|
||||
proof_time_us = proof_elapsed.as_micros(),
|
||||
@@ -1504,7 +1355,6 @@ fn build_account_multiproof_with_storage_roots<P>(
|
||||
provider: &P,
|
||||
ctx: AccountMultiproofParams<'_>,
|
||||
tracker: &mut ParallelTrieTracker,
|
||||
proof_cursor_metrics: &mut ProofTaskCursorMetricsCache,
|
||||
) -> Result<DecodedMultiProof, ParallelStateRootError>
|
||||
where
|
||||
P: TrieCursorFactory + HashedCursorFactory,
|
||||
@@ -1512,12 +1362,15 @@ where
|
||||
let accounts_added_removed_keys =
|
||||
ctx.multi_added_removed_keys.as_ref().map(|keys| keys.get_accounts());
|
||||
|
||||
// Create local metrics caches for account cursors. We can't directly use the metrics caches in
|
||||
// the tracker due to the call to `inc_missed_leaves` which occurs on it.
|
||||
let mut account_trie_cursor_metrics = TrieCursorMetricsCache::default();
|
||||
let mut account_hashed_cursor_metrics = HashedCursorMetricsCache::default();
|
||||
|
||||
// Wrap account trie cursor with instrumented cursor
|
||||
let account_trie_cursor = provider.account_trie_cursor().map_err(ProviderError::Database)?;
|
||||
let account_trie_cursor = InstrumentedTrieCursor::new(
|
||||
account_trie_cursor,
|
||||
&mut proof_cursor_metrics.account_trie_cursor,
|
||||
);
|
||||
let account_trie_cursor =
|
||||
InstrumentedTrieCursor::new(account_trie_cursor, &mut account_trie_cursor_metrics);
|
||||
|
||||
// Create the walker.
|
||||
let walker = TrieWalker::<_>::state_trie(account_trie_cursor, ctx.prefix_set)
|
||||
@@ -1544,10 +1397,8 @@ where
|
||||
// Wrap account hashed cursor with instrumented cursor
|
||||
let account_hashed_cursor =
|
||||
provider.hashed_account_cursor().map_err(ProviderError::Database)?;
|
||||
let account_hashed_cursor = InstrumentedHashedCursor::new(
|
||||
account_hashed_cursor,
|
||||
&mut proof_cursor_metrics.account_hashed_cursor,
|
||||
);
|
||||
let account_hashed_cursor =
|
||||
InstrumentedHashedCursor::new(account_hashed_cursor, &mut account_hashed_cursor_metrics);
|
||||
|
||||
let mut account_node_iter = TrieNodeIter::state_trie(walker, account_hashed_cursor);
|
||||
|
||||
@@ -1611,10 +1462,10 @@ where
|
||||
StorageProof::new_hashed(provider, provider, hashed_address)
|
||||
.with_prefix_set_mut(Default::default())
|
||||
.with_trie_cursor_metrics(
|
||||
&mut proof_cursor_metrics.storage_trie_cursor,
|
||||
&mut tracker.cursor_metrics.storage_trie_cursor,
|
||||
)
|
||||
.with_hashed_cursor_metrics(
|
||||
&mut proof_cursor_metrics.storage_hashed_cursor,
|
||||
&mut tracker.cursor_metrics.storage_hashed_cursor,
|
||||
)
|
||||
.storage_multiproof(
|
||||
ctx.targets
|
||||
@@ -1665,6 +1516,10 @@ where
|
||||
BranchNodeMasksMap::default()
|
||||
};
|
||||
|
||||
// Extend tracker with accumulated metrics from account cursors
|
||||
tracker.cursor_metrics.account_trie_cursor.extend(&account_trie_cursor_metrics);
|
||||
tracker.cursor_metrics.account_hashed_cursor.extend(&account_hashed_cursor_metrics);
|
||||
|
||||
// Consume remaining storage proof receivers for accounts not encountered during trie walk.
|
||||
// Done last to allow storage workers more time to complete while we finalized the account trie.
|
||||
for (hashed_address, receiver) in storage_proof_receivers {
|
||||
@@ -1695,6 +1550,7 @@ fn dispatch_storage_proofs(
|
||||
storage_prefix_sets: &mut B256Map<PrefixSet>,
|
||||
with_branch_node_masks: bool,
|
||||
multi_added_removed_keys: Option<&Arc<MultiAddedRemovedKeys>>,
|
||||
use_v2_proofs: bool,
|
||||
) -> Result<B256Map<CrossbeamReceiver<StorageProofResultMessage>>, ParallelStateRootError> {
|
||||
let mut storage_proof_receivers =
|
||||
B256Map::with_capacity_and_hasher(targets.len(), Default::default());
|
||||
@@ -1708,14 +1564,20 @@ fn dispatch_storage_proofs(
|
||||
let (result_tx, result_rx) = crossbeam_channel::unbounded();
|
||||
|
||||
// Create computation input based on V2 flag
|
||||
let prefix_set = storage_prefix_sets.remove(hashed_address).unwrap_or_default();
|
||||
let input = StorageProofInput::legacy(
|
||||
*hashed_address,
|
||||
prefix_set,
|
||||
target_slots.clone(),
|
||||
with_branch_node_masks,
|
||||
multi_added_removed_keys.cloned(),
|
||||
);
|
||||
let input = if use_v2_proofs {
|
||||
// Convert target slots to V2 targets
|
||||
let v2_targets = target_slots.iter().copied().map(Into::into).collect();
|
||||
StorageProofInput::new(*hashed_address, v2_targets)
|
||||
} else {
|
||||
let prefix_set = storage_prefix_sets.remove(hashed_address).unwrap_or_default();
|
||||
StorageProofInput::legacy(
|
||||
*hashed_address,
|
||||
prefix_set,
|
||||
target_slots.clone(),
|
||||
with_branch_node_masks,
|
||||
multi_added_removed_keys.cloned(),
|
||||
)
|
||||
};
|
||||
|
||||
// Always dispatch a storage proof so we obtain the storage root even when no slots are
|
||||
// requested.
|
||||
@@ -1733,64 +1595,6 @@ fn dispatch_storage_proofs(
|
||||
|
||||
Ok(storage_proof_receivers)
|
||||
}
|
||||
|
||||
/// Queues V2 storage proofs for all accounts in the targets and returns receivers.
|
||||
///
|
||||
/// This function queues all storage proof tasks to the worker pool but returns immediately
|
||||
/// with receivers, allowing the account trie walk to proceed in parallel with storage proof
|
||||
/// computation. This enables interleaved parallelism for better performance.
|
||||
///
|
||||
/// Propagates errors up if queuing fails. Receivers must be consumed by the caller.
|
||||
fn dispatch_v2_storage_proofs(
|
||||
storage_work_tx: &CrossbeamSender<StorageWorkerJob>,
|
||||
account_targets: &Vec<proof_v2::Target>,
|
||||
storage_targets: B256Map<Vec<proof_v2::Target>>,
|
||||
) -> Result<B256Map<CrossbeamReceiver<StorageProofResultMessage>>, ParallelStateRootError> {
|
||||
let mut storage_proof_receivers =
|
||||
B256Map::with_capacity_and_hasher(account_targets.len(), Default::default());
|
||||
|
||||
// Dispatch all proofs for targeted storage slots
|
||||
for (hashed_address, targets) in storage_targets {
|
||||
// Create channel for receiving StorageProofResultMessage
|
||||
let (result_tx, result_rx) = crossbeam_channel::unbounded();
|
||||
let input = StorageProofInput::new(hashed_address, targets);
|
||||
|
||||
storage_work_tx
|
||||
.send(StorageWorkerJob::StorageProof { input, proof_result_sender: result_tx })
|
||||
.map_err(|_| {
|
||||
ParallelStateRootError::Other(format!(
|
||||
"Failed to queue storage proof for {hashed_address:?}: storage worker pool unavailable",
|
||||
))
|
||||
})?;
|
||||
|
||||
storage_proof_receivers.insert(hashed_address, result_rx);
|
||||
}
|
||||
|
||||
// If there are any targeted accounts which did not have storage targets then we generate a
|
||||
// single proof target for them so that we get their root.
|
||||
for target in account_targets {
|
||||
let hashed_address = target.key();
|
||||
if storage_proof_receivers.contains_key(&hashed_address) {
|
||||
continue
|
||||
}
|
||||
|
||||
let (result_tx, result_rx) = crossbeam_channel::unbounded();
|
||||
let input = StorageProofInput::new(hashed_address, vec![proof_v2::Target::new(B256::ZERO)]);
|
||||
|
||||
storage_work_tx
|
||||
.send(StorageWorkerJob::StorageProof { input, proof_result_sender: result_tx })
|
||||
.map_err(|_| {
|
||||
ParallelStateRootError::Other(format!(
|
||||
"Failed to queue storage proof for {hashed_address:?}: storage worker pool unavailable",
|
||||
))
|
||||
})?;
|
||||
|
||||
storage_proof_receivers.insert(hashed_address, result_rx);
|
||||
}
|
||||
|
||||
Ok(storage_proof_receivers)
|
||||
}
|
||||
|
||||
/// Input parameters for storage proof computation.
|
||||
#[derive(Debug)]
|
||||
pub enum StorageProofInput {
|
||||
@@ -1835,7 +1639,7 @@ impl StorageProofInput {
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a new [`StorageProofInput`] with the given hashed address and target slots.
|
||||
/// Creates a new [`StorageProofInput`] with the given hashed address and target slots.
|
||||
pub const fn new(hashed_address: B256, targets: Vec<proof_v2::Target>) -> Self {
|
||||
Self::V2 { hashed_address, targets }
|
||||
}
|
||||
@@ -1851,39 +1655,20 @@ impl StorageProofInput {
|
||||
}
|
||||
|
||||
/// Input parameters for account multiproof computation.
|
||||
#[derive(Debug)]
|
||||
pub enum AccountMultiproofInput {
|
||||
/// Legacy account multiproof proof variant
|
||||
Legacy {
|
||||
/// The targets for which to compute the multiproof.
|
||||
targets: MultiProofTargets,
|
||||
/// The prefix sets for the proof calculation.
|
||||
prefix_sets: TriePrefixSets,
|
||||
/// Whether or not to collect branch node masks.
|
||||
collect_branch_node_masks: bool,
|
||||
/// Provided by the user to give the necessary context to retain extra proofs.
|
||||
multi_added_removed_keys: Option<Arc<MultiAddedRemovedKeys>>,
|
||||
/// Context for sending the proof result.
|
||||
proof_result_sender: ProofResultContext,
|
||||
},
|
||||
/// V2 account multiproof variant
|
||||
V2 {
|
||||
/// The targets for which to compute the multiproof.
|
||||
targets: MultiProofTargetsV2,
|
||||
/// Context for sending the proof result.
|
||||
proof_result_sender: ProofResultContext,
|
||||
},
|
||||
}
|
||||
|
||||
impl AccountMultiproofInput {
|
||||
/// Returns the [`ProofResultContext`] for this input, consuming the input.
|
||||
fn into_proof_result_sender(self) -> ProofResultContext {
|
||||
match self {
|
||||
Self::Legacy { proof_result_sender, .. } | Self::V2 { proof_result_sender, .. } => {
|
||||
proof_result_sender
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AccountMultiproofInput {
|
||||
/// The targets for which to compute the multiproof.
|
||||
pub targets: MultiProofTargets,
|
||||
/// The prefix sets for the proof calculation.
|
||||
pub prefix_sets: TriePrefixSets,
|
||||
/// Whether or not to collect branch node masks.
|
||||
pub collect_branch_node_masks: bool,
|
||||
/// Provided by the user to give the necessary context to retain extra proofs.
|
||||
pub multi_added_removed_keys: Option<Arc<MultiAddedRemovedKeys>>,
|
||||
/// Context for sending the proof result.
|
||||
pub proof_result_sender: ProofResultContext,
|
||||
/// Whether to use V2 storage proofs.
|
||||
pub v2_proofs_enabled: bool,
|
||||
}
|
||||
|
||||
/// Parameters for building an account multiproof with pre-computed storage roots.
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
#[cfg(feature = "metrics")]
|
||||
use crate::proof_task_metrics::ProofTaskCursorMetricsCache;
|
||||
use derive_more::Deref;
|
||||
use reth_trie::stats::{TrieStats, TrieTracker};
|
||||
|
||||
@@ -34,6 +36,9 @@ pub struct ParallelTrieTracker {
|
||||
trie: TrieTracker,
|
||||
precomputed_storage_roots: u64,
|
||||
missed_leaves: u64,
|
||||
#[cfg(feature = "metrics")]
|
||||
/// Local tracking of cursor-related metrics
|
||||
pub cursor_metrics: ProofTaskCursorMetricsCache,
|
||||
}
|
||||
|
||||
impl ParallelTrieTracker {
|
||||
|
||||
@@ -86,6 +86,7 @@ pub(crate) struct AsyncAccountValueEncoder {
|
||||
impl AsyncAccountValueEncoder {
|
||||
/// Initializes a [`Self`] using a `ProofWorkerHandle` which will be used to calculate storage
|
||||
/// roots asynchronously.
|
||||
#[expect(dead_code)]
|
||||
pub(crate) fn new(
|
||||
storage_work_tx: CrossbeamSender<StorageWorkerJob>,
|
||||
dispatched: B256Map<CrossbeamReceiver<StorageProofResultMessage>>,
|
||||
@@ -105,6 +106,7 @@ impl AsyncAccountValueEncoder {
|
||||
///
|
||||
/// This method panics if any deferred encoders produced by [`Self::deferred_encoder`] have not
|
||||
/// been dropped.
|
||||
#[expect(dead_code)]
|
||||
pub(crate) fn into_storage_proofs(
|
||||
self,
|
||||
) -> Result<B256Map<Vec<ProofTrieNode>>, StateProofError> {
|
||||
|
||||
@@ -768,17 +768,7 @@ impl SparseTrieInterface for ParallelSparseTrie {
|
||||
}
|
||||
|
||||
fn get_leaf_value(&self, full_path: &Nibbles) -> Option<&Vec<u8>> {
|
||||
// `subtrie_for_path` is intended for a node path, but here we are using a full key path. So
|
||||
// we need to check if the subtrie that the key might belong to has any nodes; if not then
|
||||
// the key's portion of the trie doesn't have enough depth to reach into the subtrie, and
|
||||
// the key will be in the upper subtrie
|
||||
if let Some(subtrie) = self.subtrie_for_path(full_path) &&
|
||||
!subtrie.is_empty()
|
||||
{
|
||||
return subtrie.inner.values.get(full_path);
|
||||
}
|
||||
|
||||
self.upper_subtrie.inner.values.get(full_path)
|
||||
self.subtrie_for_path(full_path).and_then(|subtrie| subtrie.inner.values.get(full_path))
|
||||
}
|
||||
|
||||
fn updates_ref(&self) -> Cow<'_, SparseTrieUpdates> {
|
||||
@@ -2286,24 +2276,14 @@ impl SparseSubtrieInner {
|
||||
if let Some((hash, store_in_db_trie)) =
|
||||
hash.zip(*store_in_db_trie).filter(|_| !prefix_set_contains(&path))
|
||||
{
|
||||
let rlp_node = RlpNode::word_rlp(&hash);
|
||||
let node_type =
|
||||
SparseNodeType::Branch { store_in_db_trie: Some(store_in_db_trie) };
|
||||
|
||||
trace!(
|
||||
target: "trie::parallel_sparse",
|
||||
?path,
|
||||
?node_type,
|
||||
?rlp_node,
|
||||
"Adding node to RLP node stack (cached branch)"
|
||||
);
|
||||
|
||||
// If the node hash is already computed, and the node path is not in
|
||||
// the prefix set, return the pre-computed hash
|
||||
self.buffers.rlp_node_stack.push(RlpNodeStackItem {
|
||||
path,
|
||||
rlp_node,
|
||||
node_type,
|
||||
rlp_node: RlpNode::word_rlp(&hash),
|
||||
node_type: SparseNodeType::Branch {
|
||||
store_in_db_trie: Some(store_in_db_trie),
|
||||
},
|
||||
});
|
||||
return
|
||||
}
|
||||
@@ -2467,14 +2447,13 @@ impl SparseSubtrieInner {
|
||||
}
|
||||
};
|
||||
|
||||
self.buffers.rlp_node_stack.push(RlpNodeStackItem { path, rlp_node, node_type });
|
||||
trace!(
|
||||
target: "trie::parallel_sparse",
|
||||
?path,
|
||||
?node_type,
|
||||
?rlp_node,
|
||||
"Adding node to RLP node stack"
|
||||
"Added node to RLP node stack"
|
||||
);
|
||||
self.buffers.rlp_node_stack.push(RlpNodeStackItem { path, rlp_node, node_type });
|
||||
}
|
||||
|
||||
/// Clears the subtrie, keeping the data structures allocated.
|
||||
@@ -6989,122 +6968,4 @@ mod tests {
|
||||
|
||||
assert_eq!(branch_0x3_update, &expected_branch);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_leaf_value_lower_subtrie() {
|
||||
// This test demonstrates that get_leaf_value must look in the correct subtrie,
|
||||
// not always in upper_subtrie.
|
||||
|
||||
let mut trie = ParallelSparseTrie::default();
|
||||
|
||||
// Create a leaf node with path >= 2 nibbles (will go to lower subtrie)
|
||||
let leaf_path = Nibbles::from_nibbles([0x1, 0x2]);
|
||||
let leaf_key = Nibbles::from_nibbles([0x3, 0x4]);
|
||||
let leaf_node = create_leaf_node(leaf_key.to_vec(), 42);
|
||||
|
||||
// Reveal the leaf node
|
||||
trie.reveal_nodes(vec![ProofTrieNode { path: leaf_path, node: leaf_node, masks: None }])
|
||||
.unwrap();
|
||||
|
||||
// The full path is leaf_path + leaf_key
|
||||
let full_path = Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4]);
|
||||
|
||||
// Verify the value is stored in the lower subtrie, not upper
|
||||
let idx = path_subtrie_index_unchecked(&leaf_path);
|
||||
let lower_subtrie = trie.lower_subtries[idx].as_revealed_ref().unwrap();
|
||||
assert!(
|
||||
lower_subtrie.inner.values.contains_key(&full_path),
|
||||
"value should be in lower subtrie"
|
||||
);
|
||||
assert!(
|
||||
!trie.upper_subtrie.inner.values.contains_key(&full_path),
|
||||
"value should NOT be in upper subtrie"
|
||||
);
|
||||
|
||||
// get_leaf_value should find the value
|
||||
assert!(
|
||||
trie.get_leaf_value(&full_path).is_some(),
|
||||
"get_leaf_value should find the value in lower subtrie"
|
||||
);
|
||||
}
|
||||
|
||||
/// Test that `get_leaf_value` correctly returns values stored via `update_leaf`
|
||||
/// when the leaf node ends up in the upper subtrie (depth < 2).
|
||||
///
|
||||
/// This can happen when the trie is sparse and the leaf is inserted at the root level.
|
||||
/// Previously, `get_leaf_value` only checked the lower subtrie based on the full path,
|
||||
/// missing values stored in `upper_subtrie.inner.values`.
|
||||
#[test]
|
||||
fn test_get_leaf_value_upper_subtrie_via_update_leaf() {
|
||||
let provider = MockTrieNodeProvider::new();
|
||||
|
||||
// Create an empty trie with an empty root
|
||||
let mut trie = ParallelSparseTrie::default()
|
||||
.with_root(TrieNode::EmptyRoot, None, false)
|
||||
.expect("root revealed");
|
||||
|
||||
// Create a full 64-nibble path (like a real account hash)
|
||||
let full_path = pad_nibbles_right(Nibbles::from_nibbles([0x0, 0xA, 0xB, 0xC]));
|
||||
let value = encode_account_value(42);
|
||||
|
||||
// Insert the leaf - since the trie is empty, the leaf node will be created
|
||||
// at the root level (depth 0), which is in the upper subtrie
|
||||
trie.update_leaf(full_path, value.clone(), &provider).unwrap();
|
||||
|
||||
// Verify the value is stored in upper_subtrie (where update_leaf puts it)
|
||||
assert!(
|
||||
trie.upper_subtrie.inner.values.contains_key(&full_path),
|
||||
"value should be in upper subtrie after update_leaf"
|
||||
);
|
||||
|
||||
// Verify the value can be retrieved via get_leaf_value
|
||||
// Before the fix, this would return None because get_leaf_value only
|
||||
// checked the lower subtrie based on the path length
|
||||
let retrieved = trie.get_leaf_value(&full_path);
|
||||
assert_eq!(retrieved, Some(&value));
|
||||
}
|
||||
|
||||
/// Test that `get_leaf_value` works for values in both upper and lower subtries.
|
||||
#[test]
|
||||
fn test_get_leaf_value_upper_and_lower_subtries() {
|
||||
let provider = MockTrieNodeProvider::new();
|
||||
|
||||
// Create an empty trie
|
||||
let mut trie = ParallelSparseTrie::default()
|
||||
.with_root(TrieNode::EmptyRoot, None, false)
|
||||
.expect("root revealed");
|
||||
|
||||
// Insert first leaf - will be at root level (upper subtrie)
|
||||
let path1 = pad_nibbles_right(Nibbles::from_nibbles([0x0, 0xA]));
|
||||
let value1 = encode_account_value(1);
|
||||
trie.update_leaf(path1, value1.clone(), &provider).unwrap();
|
||||
|
||||
// Insert second leaf with different prefix - creates a branch
|
||||
let path2 = pad_nibbles_right(Nibbles::from_nibbles([0x1, 0xB]));
|
||||
let value2 = encode_account_value(2);
|
||||
trie.update_leaf(path2, value2.clone(), &provider).unwrap();
|
||||
|
||||
// Both values should be retrievable
|
||||
assert_eq!(trie.get_leaf_value(&path1), Some(&value1));
|
||||
assert_eq!(trie.get_leaf_value(&path2), Some(&value2));
|
||||
}
|
||||
|
||||
/// Test that `get_leaf_value` works for storage tries which are often very sparse.
|
||||
#[test]
|
||||
fn test_get_leaf_value_sparse_storage_trie() {
|
||||
let provider = MockTrieNodeProvider::new();
|
||||
|
||||
// Simulate a storage trie with a single slot
|
||||
let mut trie = ParallelSparseTrie::default()
|
||||
.with_root(TrieNode::EmptyRoot, None, false)
|
||||
.expect("root revealed");
|
||||
|
||||
// Single storage slot - leaf will be at root (depth 0)
|
||||
let slot_path = pad_nibbles_right(Nibbles::from_nibbles([0x2, 0x9]));
|
||||
let slot_value = alloy_rlp::encode(U256::from(12345));
|
||||
trie.update_leaf(slot_path, slot_value.clone(), &provider).unwrap();
|
||||
|
||||
// Value should be retrievable
|
||||
assert_eq!(trie.get_leaf_value(&slot_path), Some(&slot_value));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,7 +35,6 @@
|
||||
- [`reth db settings set storages_history`](./reth/db/settings/set/storages_history.mdx)
|
||||
- [`reth db settings set transaction_hash_numbers`](./reth/db/settings/set/transaction_hash_numbers.mdx)
|
||||
- [`reth db settings set account_history`](./reth/db/settings/set/account_history.mdx)
|
||||
- [`reth db settings set storage_changesets`](./reth/db/settings/set/storage_changesets.mdx)
|
||||
- [`reth db account-storage`](./reth/db/account-storage.mdx)
|
||||
- [`reth download`](./reth/download.mdx)
|
||||
- [`reth stage`](./reth/stage.mdx)
|
||||
@@ -94,7 +93,6 @@
|
||||
- [`op-reth db settings set storages_history`](./op-reth/db/settings/set/storages_history.mdx)
|
||||
- [`op-reth db settings set transaction_hash_numbers`](./op-reth/db/settings/set/transaction_hash_numbers.mdx)
|
||||
- [`op-reth db settings set account_history`](./op-reth/db/settings/set/account_history.mdx)
|
||||
- [`op-reth db settings set storage_changesets`](./op-reth/db/settings/set/storage_changesets.mdx)
|
||||
- [`op-reth db account-storage`](./op-reth/db/account-storage.mdx)
|
||||
- [`op-reth stage`](./op-reth/stage.mdx)
|
||||
- [`op-reth stage run`](./op-reth/stage/run.mdx)
|
||||
|
||||
@@ -124,9 +124,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -157,16 +154,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
@@ -18,7 +18,6 @@ Arguments:
|
||||
- receipts: Static File segment responsible for the `Receipts` table
|
||||
- transaction-senders: Static File segment responsible for the `TransactionSenders` table
|
||||
- account-change-sets: Static File segment responsible for the `AccountChangeSets` table
|
||||
- storage-change-sets: Static File segment responsible for the `StorageChangeSets` table
|
||||
|
||||
Options:
|
||||
--start-block <START_BLOCK>
|
||||
|
||||
@@ -16,7 +16,6 @@ Arguments:
|
||||
- receipts: Static File segment responsible for the `Receipts` table
|
||||
- transaction-senders: Static File segment responsible for the `TransactionSenders` table
|
||||
- account-change-sets: Static File segment responsible for the `AccountChangeSets` table
|
||||
- storage-change-sets: Static File segment responsible for the `StorageChangeSets` table
|
||||
|
||||
Options:
|
||||
-h, --help
|
||||
|
||||
@@ -16,7 +16,6 @@ Arguments:
|
||||
- receipts: Static File segment responsible for the `Receipts` table
|
||||
- transaction-senders: Static File segment responsible for the `TransactionSenders` table
|
||||
- account-change-sets: Static File segment responsible for the `AccountChangeSets` table
|
||||
- storage-change-sets: Static File segment responsible for the `StorageChangeSets` table
|
||||
|
||||
<KEY>
|
||||
The key to get content for
|
||||
|
||||
@@ -15,7 +15,6 @@ Commands:
|
||||
storages_history Store storage history in rocksdb instead of MDBX
|
||||
transaction_hash_numbers Store transaction hash to number mapping in rocksdb instead of MDBX
|
||||
account_history Store account history in rocksdb instead of MDBX
|
||||
storage_changesets Store storage changesets in static files instead of the database
|
||||
help Print this message or the help of the given subcommand(s)
|
||||
|
||||
Options:
|
||||
|
||||
@@ -1,170 +0,0 @@
|
||||
# op-reth db settings set storage_changesets
|
||||
|
||||
Store storage changesets in static files instead of the database
|
||||
|
||||
```bash
|
||||
$ op-reth db settings set storage_changesets --help
|
||||
```
|
||||
```txt
|
||||
Usage: op-reth db settings set storage_changesets [OPTIONS] <VALUE>
|
||||
|
||||
Arguments:
|
||||
<VALUE>
|
||||
[possible values: true, false]
|
||||
|
||||
Options:
|
||||
-h, --help
|
||||
Print help (see a summary with '-h')
|
||||
|
||||
Datadir:
|
||||
--chain <CHAIN_OR_PATH>
|
||||
The chain this node is running.
|
||||
Possible values are either a built-in chain or the path to a chain specification file.
|
||||
|
||||
Built-in chains:
|
||||
optimism, optimism_sepolia, optimism-sepolia, base, base_sepolia, base-sepolia, arena-z, arena-z-sepolia, automata, base-devnet-0-sepolia-dev-0, bob, boba-sepolia, boba, camp-sepolia, celo, creator-chain-testnet-sepolia, cyber, cyber-sepolia, ethernity, ethernity-sepolia, fraxtal, funki, funki-sepolia, hashkeychain, ink, ink-sepolia, lisk, lisk-sepolia, lyra, metal, metal-sepolia, mint, mode, mode-sepolia, oplabs-devnet-0-sepolia-dev-0, orderly, ozean-sepolia, pivotal-sepolia, polynomial, race, race-sepolia, radius_testnet-sepolia, redstone, rehearsal-0-bn-0-rehearsal-0-bn, rehearsal-0-bn-1-rehearsal-0-bn, settlus-mainnet, settlus-sepolia-sepolia, shape, shape-sepolia, silent-data-mainnet, snax, soneium, soneium-minato-sepolia, sseed, swan, swell, tbn, tbn-sepolia, unichain, unichain-sepolia, worldchain, worldchain-sepolia, xterio-eth, zora, zora-sepolia, dev
|
||||
|
||||
[default: optimism]
|
||||
|
||||
Logging:
|
||||
--log.stdout.format <FORMAT>
|
||||
The format to use for logs written to stdout
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.stdout.filter <FILTER>
|
||||
The filter to use for logs written to stdout
|
||||
|
||||
[default: ]
|
||||
|
||||
--log.file.format <FORMAT>
|
||||
The format to use for logs written to the log file
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.file.filter <FILTER>
|
||||
The filter to use for logs written to the log file
|
||||
|
||||
[default: debug]
|
||||
|
||||
--log.file.directory <PATH>
|
||||
The path to put log files in
|
||||
|
||||
[default: <CACHE_DIR>/logs]
|
||||
|
||||
--log.file.name <NAME>
|
||||
The prefix name of the log files
|
||||
|
||||
[default: reth.log]
|
||||
|
||||
--log.file.max-size <SIZE>
|
||||
The maximum size (in MB) of one log file
|
||||
|
||||
[default: 200]
|
||||
|
||||
--log.file.max-files <COUNT>
|
||||
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
|
||||
|
||||
[default: 5]
|
||||
|
||||
--log.journald
|
||||
Write logs to journald
|
||||
|
||||
--log.journald.filter <FILTER>
|
||||
The filter to use for logs written to journald
|
||||
|
||||
[default: error]
|
||||
|
||||
--color <COLOR>
|
||||
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
|
||||
|
||||
Possible values:
|
||||
- always: Colors on
|
||||
- auto: Auto-detect
|
||||
- never: Colors off
|
||||
|
||||
[default: always]
|
||||
|
||||
--logs-otlp[=<URL>]
|
||||
Enable `Opentelemetry` logs export to an OTLP endpoint.
|
||||
|
||||
If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/logs` - gRPC: `http://localhost:4317`
|
||||
|
||||
Example: --logs-otlp=http://collector:4318/v1/logs
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=]
|
||||
|
||||
--logs-otlp.filter <FILTER>
|
||||
Set a filter directive for the OTLP logs exporter. This controls the verbosity of logs sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable.
|
||||
|
||||
Example: --logs-otlp.filter=info,reth=debug
|
||||
|
||||
Defaults to INFO if not specified.
|
||||
|
||||
[default: info]
|
||||
|
||||
Display:
|
||||
-v, --verbosity...
|
||||
Set the minimum log level.
|
||||
|
||||
-v Errors
|
||||
-vv Warnings
|
||||
-vvv Info
|
||||
-vvvv Debug
|
||||
-vvvvv Traces (warning: very verbose!)
|
||||
|
||||
-q, --quiet
|
||||
Silence all log output
|
||||
|
||||
Tracing:
|
||||
--tracing-otlp[=<URL>]
|
||||
Enable `Opentelemetry` tracing export to an OTLP endpoint.
|
||||
|
||||
If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317`
|
||||
|
||||
Example: --tracing-otlp=http://collector:4318/v1/traces
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=]
|
||||
|
||||
--tracing-otlp-protocol <PROTOCOL>
|
||||
OTLP transport protocol to use for exporting traces and logs.
|
||||
|
||||
- `http`: expects endpoint path to end with `/v1/traces` or `/v1/logs` - `grpc`: expects endpoint without a path
|
||||
|
||||
Defaults to HTTP if not specified.
|
||||
|
||||
Possible values:
|
||||
- http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path
|
||||
- grpc: gRPC transport, port 4317
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_PROTOCOL=]
|
||||
[default: http]
|
||||
|
||||
--tracing-otlp.filter <FILTER>
|
||||
Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable.
|
||||
|
||||
Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off
|
||||
|
||||
Defaults to TRACE if not specified.
|
||||
|
||||
[default: debug]
|
||||
|
||||
--tracing-otlp.sample-ratio <RATIO>
|
||||
Trace sampling ratio to control the percentage of traces to export.
|
||||
|
||||
Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
|
||||
|
||||
Example: --tracing-otlp.sample-ratio=0.0.
|
||||
|
||||
[env: OTEL_TRACES_SAMPLER_ARG=]
|
||||
```
|
||||
@@ -18,7 +18,6 @@ Arguments:
|
||||
- receipts: Static File segment responsible for the `Receipts` table
|
||||
- transaction-senders: Static File segment responsible for the `TransactionSenders` table
|
||||
- account-change-sets: Static File segment responsible for the `AccountChangeSets` table
|
||||
- storage-change-sets: Static File segment responsible for the `StorageChangeSets` table
|
||||
|
||||
<BLOCK>
|
||||
Block number to query
|
||||
|
||||
@@ -108,9 +108,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -141,16 +138,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
@@ -108,9 +108,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -141,16 +138,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
@@ -108,9 +108,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -141,16 +138,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
@@ -108,9 +108,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -141,16 +138,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
@@ -1036,9 +1036,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -1069,16 +1066,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
Rollup:
|
||||
--rollup.sequencer <SEQUENCER>
|
||||
Endpoint for the sequencer mempool (can be both HTTP and WS)
|
||||
|
||||
@@ -108,9 +108,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -141,16 +138,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
@@ -108,9 +108,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -141,16 +138,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
@@ -108,9 +108,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -141,16 +138,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
@@ -115,9 +115,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -148,16 +145,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
@@ -108,9 +108,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -141,16 +138,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
@@ -113,9 +113,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -146,16 +143,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
@@ -124,9 +124,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -157,16 +154,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
@@ -18,7 +18,6 @@ Arguments:
|
||||
- receipts: Static File segment responsible for the `Receipts` table
|
||||
- transaction-senders: Static File segment responsible for the `TransactionSenders` table
|
||||
- account-change-sets: Static File segment responsible for the `AccountChangeSets` table
|
||||
- storage-change-sets: Static File segment responsible for the `StorageChangeSets` table
|
||||
|
||||
Options:
|
||||
--start-block <START_BLOCK>
|
||||
|
||||
@@ -16,7 +16,6 @@ Arguments:
|
||||
- receipts: Static File segment responsible for the `Receipts` table
|
||||
- transaction-senders: Static File segment responsible for the `TransactionSenders` table
|
||||
- account-change-sets: Static File segment responsible for the `AccountChangeSets` table
|
||||
- storage-change-sets: Static File segment responsible for the `StorageChangeSets` table
|
||||
|
||||
Options:
|
||||
-h, --help
|
||||
|
||||
@@ -16,7 +16,6 @@ Arguments:
|
||||
- receipts: Static File segment responsible for the `Receipts` table
|
||||
- transaction-senders: Static File segment responsible for the `TransactionSenders` table
|
||||
- account-change-sets: Static File segment responsible for the `AccountChangeSets` table
|
||||
- storage-change-sets: Static File segment responsible for the `StorageChangeSets` table
|
||||
|
||||
<KEY>
|
||||
The key to get content for
|
||||
|
||||
@@ -15,7 +15,6 @@ Commands:
|
||||
storages_history Store storage history in rocksdb instead of MDBX
|
||||
transaction_hash_numbers Store transaction hash to number mapping in rocksdb instead of MDBX
|
||||
account_history Store account history in rocksdb instead of MDBX
|
||||
storage_changesets Store storage changesets in static files instead of the database
|
||||
help Print this message or the help of the given subcommand(s)
|
||||
|
||||
Options:
|
||||
|
||||
@@ -1,170 +0,0 @@
|
||||
# reth db settings set storage_changesets
|
||||
|
||||
Store storage changesets in static files instead of the database
|
||||
|
||||
```bash
|
||||
$ reth db settings set storage_changesets --help
|
||||
```
|
||||
```txt
|
||||
Usage: reth db settings set storage_changesets [OPTIONS] <VALUE>
|
||||
|
||||
Arguments:
|
||||
<VALUE>
|
||||
[possible values: true, false]
|
||||
|
||||
Options:
|
||||
-h, --help
|
||||
Print help (see a summary with '-h')
|
||||
|
||||
Datadir:
|
||||
--chain <CHAIN_OR_PATH>
|
||||
The chain this node is running.
|
||||
Possible values are either a built-in chain or the path to a chain specification file.
|
||||
|
||||
Built-in chains:
|
||||
mainnet, sepolia, holesky, hoodi, dev
|
||||
|
||||
[default: mainnet]
|
||||
|
||||
Logging:
|
||||
--log.stdout.format <FORMAT>
|
||||
The format to use for logs written to stdout
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.stdout.filter <FILTER>
|
||||
The filter to use for logs written to stdout
|
||||
|
||||
[default: ]
|
||||
|
||||
--log.file.format <FORMAT>
|
||||
The format to use for logs written to the log file
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.file.filter <FILTER>
|
||||
The filter to use for logs written to the log file
|
||||
|
||||
[default: debug]
|
||||
|
||||
--log.file.directory <PATH>
|
||||
The path to put log files in
|
||||
|
||||
[default: <CACHE_DIR>/logs]
|
||||
|
||||
--log.file.name <NAME>
|
||||
The prefix name of the log files
|
||||
|
||||
[default: reth.log]
|
||||
|
||||
--log.file.max-size <SIZE>
|
||||
The maximum size (in MB) of one log file
|
||||
|
||||
[default: 200]
|
||||
|
||||
--log.file.max-files <COUNT>
|
||||
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
|
||||
|
||||
[default: 5]
|
||||
|
||||
--log.journald
|
||||
Write logs to journald
|
||||
|
||||
--log.journald.filter <FILTER>
|
||||
The filter to use for logs written to journald
|
||||
|
||||
[default: error]
|
||||
|
||||
--color <COLOR>
|
||||
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
|
||||
|
||||
Possible values:
|
||||
- always: Colors on
|
||||
- auto: Auto-detect
|
||||
- never: Colors off
|
||||
|
||||
[default: always]
|
||||
|
||||
--logs-otlp[=<URL>]
|
||||
Enable `Opentelemetry` logs export to an OTLP endpoint.
|
||||
|
||||
If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/logs` - gRPC: `http://localhost:4317`
|
||||
|
||||
Example: --logs-otlp=http://collector:4318/v1/logs
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=]
|
||||
|
||||
--logs-otlp.filter <FILTER>
|
||||
Set a filter directive for the OTLP logs exporter. This controls the verbosity of logs sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable.
|
||||
|
||||
Example: --logs-otlp.filter=info,reth=debug
|
||||
|
||||
Defaults to INFO if not specified.
|
||||
|
||||
[default: info]
|
||||
|
||||
Display:
|
||||
-v, --verbosity...
|
||||
Set the minimum log level.
|
||||
|
||||
-v Errors
|
||||
-vv Warnings
|
||||
-vvv Info
|
||||
-vvvv Debug
|
||||
-vvvvv Traces (warning: very verbose!)
|
||||
|
||||
-q, --quiet
|
||||
Silence all log output
|
||||
|
||||
Tracing:
|
||||
--tracing-otlp[=<URL>]
|
||||
Enable `Opentelemetry` tracing export to an OTLP endpoint.
|
||||
|
||||
If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317`
|
||||
|
||||
Example: --tracing-otlp=http://collector:4318/v1/traces
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=]
|
||||
|
||||
--tracing-otlp-protocol <PROTOCOL>
|
||||
OTLP transport protocol to use for exporting traces and logs.
|
||||
|
||||
- `http`: expects endpoint path to end with `/v1/traces` or `/v1/logs` - `grpc`: expects endpoint without a path
|
||||
|
||||
Defaults to HTTP if not specified.
|
||||
|
||||
Possible values:
|
||||
- http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path
|
||||
- grpc: gRPC transport, port 4317
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_PROTOCOL=]
|
||||
[default: http]
|
||||
|
||||
--tracing-otlp.filter <FILTER>
|
||||
Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable.
|
||||
|
||||
Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off
|
||||
|
||||
Defaults to TRACE if not specified.
|
||||
|
||||
[default: debug]
|
||||
|
||||
--tracing-otlp.sample-ratio <RATIO>
|
||||
Trace sampling ratio to control the percentage of traces to export.
|
||||
|
||||
Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
|
||||
|
||||
Example: --tracing-otlp.sample-ratio=0.0.
|
||||
|
||||
[env: OTEL_TRACES_SAMPLER_ARG=]
|
||||
```
|
||||
@@ -18,7 +18,6 @@ Arguments:
|
||||
- receipts: Static File segment responsible for the `Receipts` table
|
||||
- transaction-senders: Static File segment responsible for the `TransactionSenders` table
|
||||
- account-change-sets: Static File segment responsible for the `AccountChangeSets` table
|
||||
- storage-change-sets: Static File segment responsible for the `StorageChangeSets` table
|
||||
|
||||
<BLOCK>
|
||||
Block number to query
|
||||
|
||||
@@ -108,9 +108,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -141,16 +138,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
@@ -108,9 +108,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -141,16 +138,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
@@ -108,9 +108,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -141,16 +138,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
@@ -108,9 +108,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -141,16 +138,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
@@ -108,9 +108,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -141,16 +138,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
@@ -108,9 +108,6 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.blocks-per-file.storage-change-sets <BLOCKS_PER_FILE_STORAGE_CHANGE_SETS>
|
||||
Number of blocks per file for the storage changesets segment
|
||||
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
@@ -141,16 +138,6 @@ Static Files:
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.storage-change-sets <STORAGE_CHANGESETS>
|
||||
Store storage changesets in static files.
|
||||
|
||||
When enabled, storage changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user