diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ffbd600db7..eed64b157f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,12 +1,12 @@ * @gakonst -crates/blockchain-tree-api/ @rakita @rkrasiuk @mattsse @Rjected -crates/blockchain-tree/ @rakita @rkrasiuk @mattsse @Rjected -crates/chain-state/ @fgimenez @mattsse @rkrasiuk +crates/blockchain-tree-api/ @rakita @mattsse @Rjected +crates/blockchain-tree/ @rakita @mattsse @Rjected +crates/chain-state/ @fgimenez @mattsse crates/chainspec/ @Rjected @joshieDo @mattsse crates/cli/ @mattsse -crates/consensus/ @rkrasiuk @mattsse @Rjected +crates/consensus/ @mattsse @Rjected crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez -crates/engine/ @rkrasiuk @mattsse @Rjected @fgimenez @mediocregopher @yongkangc +crates/engine/ @mattsse @Rjected @fgimenez @mediocregopher @yongkangc crates/era/ @mattsse @RomanHodulak crates/errors/ @mattsse crates/ethereum-forks/ @mattsse @Rjected @@ -15,17 +15,17 @@ crates/etl/ @joshieDo @shekhirin crates/evm/ @rakita @mattsse @Rjected crates/exex/ @shekhirin crates/net/ @mattsse @Rjected -crates/net/downloaders/ @rkrasiuk +crates/net/downloaders/ @Rjected crates/node/ @mattsse @Rjected @klkvr crates/optimism/ @mattsse @Rjected @fgimenez crates/payload/ @mattsse @Rjected crates/primitives-traits/ @Rjected @RomanHodulak @mattsse @klkvr crates/primitives/ @Rjected @mattsse @klkvr crates/prune/ @shekhirin @joshieDo -crates/ress @rkrasiuk +crates/ress @shekhirin @Rjected crates/revm/ @mattsse @rakita crates/rpc/ @mattsse @Rjected @RomanHodulak -crates/stages/ @rkrasiuk @shekhirin @mediocregopher +crates/stages/ @shekhirin @mediocregopher crates/static-file/ @joshieDo @shekhirin crates/storage/codecs/ @joshieDo crates/storage/db-api/ @joshieDo @rakita @@ -35,10 +35,10 @@ crates/storage/errors/ @rakita crates/storage/libmdbx-rs/ @rakita @shekhirin crates/storage/nippy-jar/ @joshieDo @shekhirin crates/storage/provider/ @rakita @joshieDo @shekhirin -crates/storage/storage-api/ @joshieDo @rkrasiuk +crates/storage/storage-api/ @joshieDo crates/tasks/ @mattsse crates/tokio-util/ @fgimenez crates/transaction-pool/ @mattsse @yongkangc -crates/trie/ @rkrasiuk @Rjected @shekhirin @mediocregopher +crates/trie/ @Rjected @shekhirin @mediocregopher etc/ @Rjected @shekhirin .github/ @gakonst @DaniPopes diff --git a/Cargo.lock b/Cargo.lock index fec80d44a6..089ebbc43e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3993,7 +3993,6 @@ version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ - "serde", "typenum", "version_check", "zeroize", @@ -7723,7 +7722,6 @@ dependencies = [ "assert_matches", "discv5", "enr", - "generic-array", "itertools 0.14.0", "parking_lot", "rand 0.8.5", @@ -7935,7 +7933,7 @@ dependencies = [ "reth-optimism-chainspec", "reth-payload-builder", "reth-payload-primitives", - "reth-provider", + "reth-storage-api", "reth-transaction-pool", "tokio", "tokio-stream", @@ -8995,7 +8993,6 @@ dependencies = [ "serde", "shellexpand", "strum 0.27.2", - "thiserror 2.0.17", "tokio", "toml", "tracing", @@ -9327,20 +9324,20 @@ dependencies = [ "futures-util", "metrics", "reth-chain-state", + "reth-engine-primitives", "reth-errors", "reth-evm", "reth-execution-types", "reth-metrics", - "reth-node-api", "reth-optimism-evm", "reth-optimism-payload-builder", "reth-optimism-primitives", + "reth-payload-primitives", "reth-primitives-traits", "reth-revm", "reth-rpc-eth-types", "reth-storage-api", "reth-tasks", - "reth-trie", "ringbuffer", "serde", "serde_json", @@ -9775,14 +9772,12 @@ dependencies = [ name = "reth-prune" version = "1.8.2" dependencies = [ - "alloy-consensus", "alloy-eips", "alloy-primitives", "assert_matches", "itertools 0.14.0", "metrics", "rayon", - "reth-chainspec", "reth-config", "reth-db", "reth-db-api", @@ -10671,6 +10666,7 @@ dependencies = [ "alloy-serde", "alloy-trie", "arbitrary", + "arrayvec", "bincode 1.3.3", "bytes", "codspeed-criterion-compat", diff --git a/Cargo.toml b/Cargo.toml index 8dcaf811ba..f1369a36b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -534,6 +534,7 @@ op-alloy-flz = { version = "0.13.1", default-features = false } # misc either = { version = "1.15.0", default-features = false } +arrayvec = { version = "0.7.6", default-features = false } aquamarine = "0.6" auto_impl = "1" backon = { version = "1.2", default-features = false, features = ["std-blocking-sleep", "tokio-sleep"] } diff --git a/bin/reth-bench/src/bench/context.rs b/bin/reth-bench/src/bench/context.rs index 75c8592ad3..1d53ce8e1a 100644 --- a/bin/reth-bench/src/bench/context.rs +++ b/bin/reth-bench/src/bench/context.rs @@ -7,6 +7,7 @@ use alloy_primitives::address; use alloy_provider::{network::AnyNetwork, Provider, RootProvider}; use alloy_rpc_client::ClientBuilder; use alloy_rpc_types_engine::JwtSecret; +use alloy_transport::layers::RetryBackoffLayer; use reqwest::Url; use reth_node_core::args::BenchmarkArgs; use tracing::info; @@ -49,7 +50,9 @@ impl BenchContext { } // set up alloy client for blocks - let client = ClientBuilder::default().http(rpc_url.parse()?); + let client = ClientBuilder::default() + .layer(RetryBackoffLayer::new(10, 800, u64::MAX)) + .http(rpc_url.parse()?); let block_provider = RootProvider::::new(client); // Check if this is an OP chain by checking code at a predeploy address. diff --git a/bin/reth-bench/src/bench/new_payload_fcu.rs b/bin/reth-bench/src/bench/new_payload_fcu.rs index 0303c7d014..ce094895ee 100644 --- a/bin/reth-bench/src/bench/new_payload_fcu.rs +++ b/bin/reth-bench/src/bench/new_payload_fcu.rs @@ -33,6 +33,16 @@ pub struct Command { #[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)] wait_time: Option, + /// The size of the block buffer (channel capacity) for prefetching blocks from the RPC + /// endpoint. + #[arg( + long = "rpc-block-buffer-size", + value_name = "RPC_BLOCK_BUFFER_SIZE", + default_value = "20", + verbatim_doc_comment + )] + rpc_block_buffer_size: usize, + #[command(flatten)] benchmark: BenchmarkArgs, } @@ -48,7 +58,12 @@ impl Command { is_optimism, } = BenchContext::new(&self.benchmark, self.rpc_url).await?; - let (sender, mut receiver) = tokio::sync::mpsc::channel(1000); + let buffer_size = self.rpc_block_buffer_size; + + // Use a oneshot channel to propagate errors from the spawned task + let (error_sender, mut error_receiver) = tokio::sync::oneshot::channel(); + let (sender, mut receiver) = tokio::sync::mpsc::channel(buffer_size); + tokio::task::spawn(async move { while benchmark_mode.contains(next_block) { let block_res = block_provider @@ -60,6 +75,7 @@ impl Command { Ok(block) => block, Err(e) => { tracing::error!("Failed to fetch block {next_block}: {e}"); + let _ = error_sender.send(e); break; } }; @@ -69,6 +85,7 @@ impl Command { Ok(result) => result, Err(e) => { tracing::error!("Failed to convert block to new payload: {e}"); + let _ = error_sender.send(e); break; } }; @@ -163,6 +180,11 @@ impl Command { results.push((gas_row, combined_result)); } + // Check if the spawned task encountered an error + if let Ok(error) = error_receiver.try_recv() { + return Err(error); + } + let (gas_output_results, combined_results): (_, Vec) = results.into_iter().unzip(); diff --git a/bin/reth-bench/src/bench/new_payload_only.rs b/bin/reth-bench/src/bench/new_payload_only.rs index 34fe378055..3dfa619ec7 100644 --- a/bin/reth-bench/src/bench/new_payload_only.rs +++ b/bin/reth-bench/src/bench/new_payload_only.rs @@ -13,7 +13,7 @@ use crate::{ use alloy_provider::Provider; use clap::Parser; use csv::Writer; -use eyre::Context; +use eyre::{Context, OptionExt}; use reth_cli_runner::CliContext; use reth_node_core::args::BenchmarkArgs; use std::time::{Duration, Instant}; @@ -26,6 +26,16 @@ pub struct Command { #[arg(long, value_name = "RPC_URL", verbatim_doc_comment)] rpc_url: String, + /// The size of the block buffer (channel capacity) for prefetching blocks from the RPC + /// endpoint. + #[arg( + long = "rpc-block-buffer-size", + value_name = "RPC_BLOCK_BUFFER_SIZE", + default_value = "20", + verbatim_doc_comment + )] + rpc_block_buffer_size: usize, + #[command(flatten)] benchmark: BenchmarkArgs, } @@ -41,7 +51,12 @@ impl Command { is_optimism, } = BenchContext::new(&self.benchmark, self.rpc_url).await?; - let (sender, mut receiver) = tokio::sync::mpsc::channel(1000); + let buffer_size = self.rpc_block_buffer_size; + + // Use a oneshot channel to propagate errors from the spawned task + let (error_sender, mut error_receiver) = tokio::sync::oneshot::channel(); + let (sender, mut receiver) = tokio::sync::mpsc::channel(buffer_size); + tokio::task::spawn(async move { while benchmark_mode.contains(next_block) { let block_res = block_provider @@ -49,13 +64,30 @@ impl Command { .full() .await .wrap_err_with(|| format!("Failed to fetch block by number {next_block}")); - let block = block_res.unwrap().unwrap(); + let block = match block_res.and_then(|opt| opt.ok_or_eyre("Block not found")) { + Ok(block) => block, + Err(e) => { + tracing::error!("Failed to fetch block {next_block}: {e}"); + let _ = error_sender.send(e); + break; + } + }; let header = block.header.clone(); - let (version, params) = block_to_new_payload(block, is_optimism).unwrap(); + let (version, params) = match block_to_new_payload(block, is_optimism) { + Ok(result) => result, + Err(e) => { + tracing::error!("Failed to convert block to new payload: {e}"); + let _ = error_sender.send(e); + break; + } + }; next_block += 1; - sender.send((header, version, params)).await.unwrap(); + if let Err(e) = sender.send((header, version, params)).await { + tracing::error!("Failed to send block data: {e}"); + break; + } } }); @@ -96,6 +128,11 @@ impl Command { results.push((row, new_payload_result)); } + // Check if the spawned task encountered an error + if let Ok(error) = error_receiver.try_recv() { + return Err(error); + } + let (gas_output_results, new_payload_results): (_, Vec) = results.into_iter().unzip(); diff --git a/crates/cli/commands/src/common.rs b/crates/cli/commands/src/common.rs index 25f32f63a2..5b8cfce771 100644 --- a/crates/cli/commands/src/common.rs +++ b/crates/cli/commands/src/common.rs @@ -24,7 +24,7 @@ use reth_provider::{ providers::{BlockchainProvider, NodeTypesForProvider, StaticFileProvider}, ProviderFactory, StaticFileProviderFactory, }; -use reth_stages::{sets::DefaultStages, Pipeline}; +use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget}; use reth_static_file::StaticFileProducer; use std::{path::PathBuf, sync::Arc}; use tokio::sync::watch; @@ -126,8 +126,8 @@ impl EnvironmentArgs { where C: ChainSpecParser, { - let prune_modes = - config.prune.as_ref().map(|prune| prune.segments.clone()).unwrap_or_default(); + let has_receipt_pruning = config.prune.has_receipts_pruning(); + let prune_modes = config.prune.segments.clone(); let factory = ProviderFactory::>>::new( db, self.chain.clone(), @@ -136,8 +136,9 @@ impl EnvironmentArgs { .with_prune_modes(prune_modes.clone()); // Check for consistency between database and static files. - if let Some(unwind_target) = - factory.static_file_provider().check_consistency(&factory.provider()?)? + if let Some(unwind_target) = factory + .static_file_provider() + .check_consistency(&factory.provider()?, has_receipt_pruning)? { if factory.db_ref().is_read_only()? { warn!(target: "reth::cli", ?unwind_target, "Inconsistent storage. Restart node to heal."); @@ -148,7 +149,7 @@ impl EnvironmentArgs { // instead. assert_ne!( unwind_target, - 0, + PipelineTarget::Unwind(0), "A static file <> database inconsistency was found that would trigger an unwind to block 0" ); @@ -173,7 +174,7 @@ impl EnvironmentArgs { // Move all applicable data from database to static files. pipeline.move_to_static_files()?; - pipeline.unwind(unwind_target, None)?; + pipeline.unwind(unwind_target.unwind_target().expect("should exist"), None)?; } Ok(factory) diff --git a/crates/cli/commands/src/db/repair_trie.rs b/crates/cli/commands/src/db/repair_trie.rs index e5b7db0e2f..f7dea67b76 100644 --- a/crates/cli/commands/src/db/repair_trie.rs +++ b/crates/cli/commands/src/db/repair_trie.rs @@ -52,7 +52,7 @@ fn verify_only(provider_factory: ProviderFactory) -> eyre // Create the verifier let hashed_cursor_factory = DatabaseHashedCursorFactory::new(&tx); let trie_cursor_factory = DatabaseTrieCursorFactory::new(&tx); - let verifier = Verifier::new(trie_cursor_factory, hashed_cursor_factory)?; + let verifier = Verifier::new(&trie_cursor_factory, hashed_cursor_factory)?; let mut inconsistent_nodes = 0; let start_time = Instant::now(); @@ -136,7 +136,7 @@ fn verify_and_repair( let trie_cursor_factory = DatabaseTrieCursorFactory::new(tx); // Create the verifier - let verifier = Verifier::new(trie_cursor_factory, hashed_cursor_factory)?; + let verifier = Verifier::new(&trie_cursor_factory, hashed_cursor_factory)?; let mut inconsistent_nodes = 0; let start_time = Instant::now(); diff --git a/crates/cli/commands/src/init_state/without_evm.rs b/crates/cli/commands/src/init_state/without_evm.rs index de6320fc86..8da0bde068 100644 --- a/crates/cli/commands/src/init_state/without_evm.rs +++ b/crates/cli/commands/src/init_state/without_evm.rs @@ -1,5 +1,5 @@ use alloy_consensus::BlockHeader; -use alloy_primitives::{BlockNumber, B256, U256}; +use alloy_primitives::{BlockNumber, B256}; use alloy_rlp::Decodable; use reth_codecs::Compact; use reth_node_builder::NodePrimitives; @@ -133,7 +133,7 @@ where for block_num in 1..=target_height { // TODO: should we fill with real parent_hash? let header = header_factory(block_num); - writer.append_header(&header, U256::ZERO, &B256::ZERO)?; + writer.append_header(&header, &B256::ZERO)?; } Ok(()) }); diff --git a/crates/cli/commands/src/prune.rs b/crates/cli/commands/src/prune.rs index de60fbfdb3..cae0fa0090 100644 --- a/crates/cli/commands/src/prune.rs +++ b/crates/cli/commands/src/prune.rs @@ -1,5 +1,5 @@ //! Command that runs pruning without any limits. -use crate::common::{AccessRights, CliNodeTypes, Environment, EnvironmentArgs}; +use crate::common::{AccessRights, CliNodeTypes, EnvironmentArgs}; use clap::Parser; use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_cli::chainspec::ChainSpecParser; @@ -18,22 +18,23 @@ pub struct PruneCommand { impl> PruneCommand { /// Execute the `prune` command pub async fn execute>(self) -> eyre::Result<()> { - let Environment { config, provider_factory, .. } = self.env.init::(AccessRights::RW)?; - let prune_config = config.prune.unwrap_or_default(); + let env = self.env.init::(AccessRights::RW)?; + let provider_factory = env.provider_factory; + let config = env.config.prune; // Copy data from database to static files info!(target: "reth::cli", "Copying data from database to static files..."); let static_file_producer = - StaticFileProducer::new(provider_factory.clone(), prune_config.segments.clone()); + StaticFileProducer::new(provider_factory.clone(), config.segments.clone()); let lowest_static_file_height = static_file_producer.lock().copy_to_static_files()?.min_block_num(); info!(target: "reth::cli", ?lowest_static_file_height, "Copied data from database to static files"); // Delete data which has been copied to static files. if let Some(prune_tip) = lowest_static_file_height { - info!(target: "reth::cli", ?prune_tip, ?prune_config, "Pruning data from database..."); + info!(target: "reth::cli", ?prune_tip, ?config, "Pruning data from database..."); // Run the pruner according to the configuration, and don't enforce any limits on it - let mut pruner = PrunerBuilder::new(prune_config) + let mut pruner = PrunerBuilder::new(config) .delete_limit(usize::MAX) .build_with_provider_factory(provider_factory); diff --git a/crates/cli/commands/src/stage/drop.rs b/crates/cli/commands/src/stage/drop.rs index 94e973aa23..c4dce8d350 100644 --- a/crates/cli/commands/src/stage/drop.rs +++ b/crates/cli/commands/src/stage/drop.rs @@ -70,7 +70,6 @@ impl Command { StageEnum::Headers => { tx.clear::()?; tx.clear::>>()?; - tx.clear::()?; tx.clear::()?; reset_stage_checkpoint(tx, StageId::Headers)?; diff --git a/crates/cli/commands/src/stage/dump/execution.rs b/crates/cli/commands/src/stage/dump/execution.rs index 921af75c78..9e8e68e980 100644 --- a/crates/cli/commands/src/stage/dump/execution.rs +++ b/crates/cli/commands/src/stage/dump/execution.rs @@ -69,13 +69,6 @@ fn import_tables_with_range( to, ) })??; - output_db.update(|tx| { - tx.import_table_with_range::( - &db_tool.provider_factory.db_ref().tx()?, - Some(from), - to, - ) - })??; output_db.update(|tx| { tx.import_table_with_range::( &db_tool.provider_factory.db_ref().tx()?, diff --git a/crates/cli/commands/src/stage/run.rs b/crates/cli/commands/src/stage/run.rs index 4e577af06b..010277480f 100644 --- a/crates/cli/commands/src/stage/run.rs +++ b/crates/cli/commands/src/stage/run.rs @@ -151,7 +151,7 @@ impl let batch_size = self.batch_size.unwrap_or(self.to.saturating_sub(self.from) + 1); let etl_config = config.stages.etl.clone(); - let prune_modes = config.prune.clone().map(|prune| prune.segments).unwrap_or_default(); + let prune_modes = config.prune.segments.clone(); let (mut exec_stage, mut unwind_stage): (Box>, Option>>) = match self.stage { diff --git a/crates/cli/commands/src/stage/unwind.rs b/crates/cli/commands/src/stage/unwind.rs index ba9a00b11e..ffd8e33006 100644 --- a/crates/cli/commands/src/stage/unwind.rs +++ b/crates/cli/commands/src/stage/unwind.rs @@ -85,7 +85,7 @@ impl> Command evm_config: impl ConfigureEvm + 'static, ) -> Result, eyre::Error> { let stage_conf = &config.stages; - let prune_modes = config.prune.clone().map(|prune| prune.segments).unwrap_or_default(); + let prune_modes = config.prune.segments.clone(); let (tip_tx, tip_rx) = watch::channel(B256::ZERO); diff --git a/crates/cli/commands/src/test_vectors/tables.rs b/crates/cli/commands/src/test_vectors/tables.rs index ef34e5b5e8..10b9469539 100644 --- a/crates/cli/commands/src/test_vectors/tables.rs +++ b/crates/cli/commands/src/test_vectors/tables.rs @@ -69,7 +69,6 @@ pub fn generate_vectors(mut tables: Vec) -> Result<()> { generate!([ (CanonicalHeaders, PER_TABLE, TABLE), - (HeaderTerminalDifficulties, PER_TABLE, TABLE), (HeaderNumbers, PER_TABLE, TABLE), (Headers
, PER_TABLE, TABLE), (BlockBodyIndices, PER_TABLE, TABLE), diff --git a/crates/config/src/config.rs b/crates/config/src/config.rs index 7ea5569834..dd2e7046b0 100644 --- a/crates/config/src/config.rs +++ b/crates/config/src/config.rs @@ -23,8 +23,8 @@ pub struct Config { // TODO(onbjerg): Can we make this easier to maintain when we add/remove stages? pub stages: StageConfig, /// Configuration for pruning. - #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] - pub prune: Option, + #[cfg_attr(feature = "serde", serde(default))] + pub prune: PruneConfig, /// Configuration for the discovery service. pub peers: PeersConfig, /// Configuration for peer sessions. @@ -33,8 +33,8 @@ pub struct Config { impl Config { /// Sets the pruning configuration. - pub fn update_prune_config(&mut self, prune_config: PruneConfig) { - self.prune = Some(prune_config); + pub const fn set_prune_config(&mut self, prune_config: PruneConfig) { + self.prune = prune_config; } } @@ -445,15 +445,20 @@ impl Default for PruneConfig { } impl PruneConfig { + /// Returns whether this configuration is the default one. + pub fn is_default(&self) -> bool { + self == &Self::default() + } + /// Returns whether there is any kind of receipt pruning configuration. - pub fn has_receipts_pruning(&self) -> bool { - self.segments.receipts.is_some() || !self.segments.receipts_log_filter.is_empty() + pub const fn has_receipts_pruning(&self) -> bool { + self.segments.receipts.is_some() } /// Merges another `PruneConfig` into this one, taking values from the other config if and only /// if the corresponding value in this config is not set. - pub fn merge(&mut self, other: Option) { - let Some(other) = other else { return }; + pub fn merge(&mut self, other: Self) { + #[expect(deprecated)] let Self { block_interval, segments: @@ -465,7 +470,7 @@ impl PruneConfig { storage_history, bodies_history, merkle_changesets, - receipts_log_filter, + receipts_log_filter: (), }, } = other; @@ -483,10 +488,6 @@ impl PruneConfig { self.segments.bodies_history = self.segments.bodies_history.or(bodies_history); // Merkle changesets is not optional, so we just replace it if provided self.segments.merkle_changesets = merkle_changesets; - - if self.segments.receipts_log_filter.0.is_empty() && !receipts_log_filter.0.is_empty() { - self.segments.receipts_log_filter = receipts_log_filter; - } } } @@ -513,10 +514,9 @@ where mod tests { use super::{Config, EXTENSION}; use crate::PruneConfig; - use alloy_primitives::Address; use reth_network_peers::TrustedPeer; - use reth_prune_types::{PruneMode, PruneModes, ReceiptsLogPruneConfig}; - use std::{collections::BTreeMap, path::Path, str::FromStr, time::Duration}; + use reth_prune_types::{PruneMode, PruneModes}; + use std::{path::Path, str::FromStr, time::Duration}; fn with_tempdir(filename: &str, proc: fn(&std::path::Path)) { let temp_dir = tempfile::tempdir().unwrap(); @@ -1005,10 +1005,8 @@ receipts = 'full' storage_history: Some(PruneMode::Before(5000)), bodies_history: None, merkle_changesets: PruneMode::Before(0), - receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([( - Address::random(), - PruneMode::Full, - )])), + #[expect(deprecated)] + receipts_log_filter: (), }, }; @@ -1022,15 +1020,12 @@ receipts = 'full' storage_history: Some(PruneMode::Distance(3000)), bodies_history: None, merkle_changesets: PruneMode::Distance(10000), - receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([ - (Address::random(), PruneMode::Distance(1000)), - (Address::random(), PruneMode::Before(2000)), - ])), + #[expect(deprecated)] + receipts_log_filter: (), }, }; - let original_filter = config1.segments.receipts_log_filter.clone(); - config1.merge(Some(config2)); + config1.merge(config2); // Check that the configuration has been merged. Any configuration present in config1 // should not be overwritten by config2 @@ -1041,7 +1036,6 @@ receipts = 'full' assert_eq!(config1.segments.account_history, Some(PruneMode::Distance(2000))); assert_eq!(config1.segments.storage_history, Some(PruneMode::Before(5000))); assert_eq!(config1.segments.merkle_changesets, PruneMode::Distance(10000)); - assert_eq!(config1.segments.receipts_log_filter, original_filter); } #[test] diff --git a/crates/e2e-test-utils/src/lib.rs b/crates/e2e-test-utils/src/lib.rs index a51b78ae65..e7b83cb3ad 100644 --- a/crates/e2e-test-utils/src/lib.rs +++ b/crates/e2e-test-utils/src/lib.rs @@ -1,23 +1,19 @@ //! Utilities for end-to-end tests. use node::NodeTestContext; -use reth_chainspec::{ChainSpec, EthChainSpec}; +use reth_chainspec::ChainSpec; use reth_db::{test_utils::TempDatabase, DatabaseEnv}; use reth_engine_local::LocalPayloadAttributesBuilder; use reth_network_api::test_utils::PeersHandleProvider; use reth_node_builder::{ components::NodeComponentsBuilder, rpc::{EngineValidatorAddOn, RethRpcAddOns}, - EngineNodeLauncher, FullNodeTypesAdapter, Node, NodeAdapter, NodeBuilder, NodeComponents, - NodeConfig, NodeHandle, NodePrimitives, NodeTypes, NodeTypesWithDBAdapter, - PayloadAttributesBuilder, PayloadTypes, + FullNodeTypesAdapter, Node, NodeAdapter, NodeComponents, NodePrimitives, NodeTypes, + NodeTypesWithDBAdapter, PayloadAttributesBuilder, PayloadTypes, }; -use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs}; use reth_provider::providers::{BlockchainProvider, NodeTypesForProvider}; -use reth_rpc_server_types::RpcModuleSelection; use reth_tasks::TaskManager; use std::sync::Arc; -use tracing::{span, Level}; use wallet::Wallet; /// Wrapper type to create test nodes @@ -45,6 +41,10 @@ mod rpc; /// Utilities for creating and writing RLP test data pub mod test_rlp_utils; +/// Builder for configuring test node setups +mod setup_builder; +pub use setup_builder::E2ETestSetupBuilder; + /// Creates the initial setup with `num_nodes` started and interconnected. pub async fn setup( num_nodes: usize, @@ -53,60 +53,14 @@ pub async fn setup( attributes_generator: impl Fn(u64) -> <::Payload as PayloadTypes>::PayloadBuilderAttributes + Send + Sync + Copy + 'static, ) -> eyre::Result<(Vec>, TaskManager, Wallet)> where - N: Default + Node> + NodeTypesForProvider, - N::ComponentsBuilder: NodeComponentsBuilder< - TmpNodeAdapter, - Components: NodeComponents, Network: PeersHandleProvider>, - >, - N::AddOns: RethRpcAddOns> + EngineValidatorAddOn>, + N: NodeBuilderHelper, LocalPayloadAttributesBuilder: PayloadAttributesBuilder<<::Payload as PayloadTypes>::PayloadAttributes>, { - let tasks = TaskManager::current(); - let exec = tasks.executor(); - - let network_config = NetworkArgs { - discovery: DiscoveryArgs { disable_discovery: true, ..DiscoveryArgs::default() }, - ..NetworkArgs::default() - }; - - // Create nodes and peer them - let mut nodes: Vec> = Vec::with_capacity(num_nodes); - - for idx in 0..num_nodes { - let node_config = NodeConfig::new(chain_spec.clone()) - .with_network(network_config.clone()) - .with_unused_ports() - .with_rpc(RpcServerArgs::default().with_unused_ports().with_http()) - .set_dev(is_dev); - - let span = span!(Level::INFO, "node", idx); - let _enter = span.enter(); - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone()) - .testing_node(exec.clone()) - .node(Default::default()) - .launch() - .await?; - - let mut node = NodeTestContext::new(node, attributes_generator).await?; - - // Connect each node in a chain. - if let Some(previous_node) = nodes.last_mut() { - previous_node.connect(&mut node).await; - } - - // Connect last node with the first if there are more than two - if idx + 1 == num_nodes && - num_nodes > 2 && - let Some(first_node) = nodes.first_mut() - { - node.connect(first_node).await; - } - - nodes.push(node); - } - - Ok((nodes, tasks, Wallet::default().with_chain_id(chain_spec.chain().into()))) + E2ETestSetupBuilder::new(num_nodes, chain_spec, attributes_generator) + .with_node_config_modifier(move |config| config.set_dev(is_dev)) + .build() + .await } /// Creates the initial setup with `num_nodes` started and interconnected. @@ -155,71 +109,12 @@ where LocalPayloadAttributesBuilder: PayloadAttributesBuilder<::PayloadAttributes>, { - let tasks = TaskManager::current(); - let exec = tasks.executor(); - - let network_config = NetworkArgs { - discovery: DiscoveryArgs { disable_discovery: true, ..DiscoveryArgs::default() }, - ..NetworkArgs::default() - }; - - // Create nodes and peer them - let mut nodes: Vec> = Vec::with_capacity(num_nodes); - - for idx in 0..num_nodes { - let node_config = NodeConfig::new(chain_spec.clone()) - .with_network(network_config.clone()) - .with_unused_ports() - .with_rpc( - RpcServerArgs::default() - .with_unused_ports() - .with_http() - .with_http_api(RpcModuleSelection::All), - ) - .set_dev(is_dev); - - let span = span!(Level::INFO, "node", idx); - let _enter = span.enter(); - let node = N::default(); - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone()) - .testing_node(exec.clone()) - .with_types_and_provider::>() - .with_components(node.components_builder()) - .with_add_ons(node.add_ons()) - .launch_with_fn(|builder| { - let launcher = EngineNodeLauncher::new( - builder.task_executor().clone(), - builder.config().datadir(), - tree_config.clone(), - ); - builder.launch_with(launcher) - }) - .await?; - - let mut node = NodeTestContext::new(node, attributes_generator).await?; - - let genesis = node.block_hash(0); - node.update_forkchoice(genesis, genesis).await?; - - // Connect each node in a chain if requested. - if connect_nodes { - if let Some(previous_node) = nodes.last_mut() { - previous_node.connect(&mut node).await; - } - - // Connect last node with the first if there are more than two - if idx + 1 == num_nodes && - num_nodes > 2 && - let Some(first_node) = nodes.first_mut() - { - node.connect(first_node).await; - } - } - - nodes.push(node); - } - - Ok((nodes, tasks, Wallet::default().with_chain_id(chain_spec.chain().into()))) + E2ETestSetupBuilder::new(num_nodes, chain_spec, attributes_generator) + .with_tree_config_modifier(move |_| tree_config.clone()) + .with_node_config_modifier(move |config| config.set_dev(is_dev)) + .with_connect_nodes(connect_nodes) + .build() + .await } // Type aliases diff --git a/crates/e2e-test-utils/src/payload.rs b/crates/e2e-test-utils/src/payload.rs index b3f9b027fb..4e185ce969 100644 --- a/crates/e2e-test-utils/src/payload.rs +++ b/crates/e2e-test-utils/src/payload.rs @@ -57,8 +57,9 @@ impl PayloadTestContext { /// Wait until the best built payload is ready pub async fn wait_for_built_payload(&self, payload_id: PayloadId) { loop { - let payload = self.payload_builder.best_payload(payload_id).await.unwrap().unwrap(); - if payload.block().body().transactions().is_empty() { + let payload = + self.payload_builder.best_payload(payload_id).await.transpose().ok().flatten(); + if payload.is_none_or(|p| p.block().body().transactions().is_empty()) { tokio::time::sleep(std::time::Duration::from_millis(20)).await; continue } diff --git a/crates/e2e-test-utils/src/setup_builder.rs b/crates/e2e-test-utils/src/setup_builder.rs new file mode 100644 index 0000000000..8de2280fe4 --- /dev/null +++ b/crates/e2e-test-utils/src/setup_builder.rs @@ -0,0 +1,210 @@ +//! Builder for configuring and creating test node setups. +//! +//! This module provides a flexible builder API for setting up test nodes with custom +//! configurations through closures that modify `NodeConfig` and `TreeConfig`. + +use crate::{node::NodeTestContext, wallet::Wallet, NodeBuilderHelper, NodeHelperType, TmpDB}; +use reth_chainspec::EthChainSpec; +use reth_engine_local::LocalPayloadAttributesBuilder; +use reth_node_builder::{ + EngineNodeLauncher, NodeBuilder, NodeConfig, NodeHandle, NodeTypes, NodeTypesWithDBAdapter, + PayloadAttributesBuilder, PayloadTypes, +}; +use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs}; +use reth_provider::providers::BlockchainProvider; +use reth_rpc_server_types::RpcModuleSelection; +use reth_tasks::TaskManager; +use std::sync::Arc; +use tracing::{span, Level}; + +/// Type alias for tree config modifier closure +type TreeConfigModifier = + Box reth_node_api::TreeConfig + Send + Sync>; + +/// Type alias for node config modifier closure +type NodeConfigModifier = Box) -> NodeConfig + Send + Sync>; + +/// Builder for configuring and creating test node setups. +/// +/// This builder allows customizing test node configurations through closures that +/// modify `NodeConfig` and `TreeConfig`. It avoids code duplication by centralizing +/// the node creation logic. +pub struct E2ETestSetupBuilder +where + N: NodeBuilderHelper, + F: Fn(u64) -> <::Payload as PayloadTypes>::PayloadBuilderAttributes + + Send + + Sync + + Copy + + 'static, + LocalPayloadAttributesBuilder: + PayloadAttributesBuilder<::PayloadAttributes>, +{ + num_nodes: usize, + chain_spec: Arc, + attributes_generator: F, + connect_nodes: bool, + tree_config_modifier: Option, + node_config_modifier: Option>, +} + +impl E2ETestSetupBuilder +where + N: NodeBuilderHelper, + F: Fn(u64) -> <::Payload as PayloadTypes>::PayloadBuilderAttributes + + Send + + Sync + + Copy + + 'static, + LocalPayloadAttributesBuilder: + PayloadAttributesBuilder<::PayloadAttributes>, +{ + /// Creates a new builder with the required parameters. + pub fn new(num_nodes: usize, chain_spec: Arc, attributes_generator: F) -> Self { + Self { + num_nodes, + chain_spec, + attributes_generator, + connect_nodes: true, + tree_config_modifier: None, + node_config_modifier: None, + } + } + + /// Sets whether nodes should be interconnected (default: true). + pub const fn with_connect_nodes(mut self, connect_nodes: bool) -> Self { + self.connect_nodes = connect_nodes; + self + } + + /// Sets a modifier function for the tree configuration. + /// + /// The closure receives the base tree config and returns a modified version. + pub fn with_tree_config_modifier(mut self, modifier: G) -> Self + where + G: Fn(reth_node_api::TreeConfig) -> reth_node_api::TreeConfig + Send + Sync + 'static, + { + self.tree_config_modifier = Some(Box::new(modifier)); + self + } + + /// Sets a modifier function for the node configuration. + /// + /// The closure receives the base node config and returns a modified version. + pub fn with_node_config_modifier(mut self, modifier: G) -> Self + where + G: Fn(NodeConfig) -> NodeConfig + Send + Sync + 'static, + { + self.node_config_modifier = Some(Box::new(modifier)); + self + } + + /// Builds and launches the test nodes. + pub async fn build( + self, + ) -> eyre::Result<( + Vec>>>, + TaskManager, + Wallet, + )> { + let tasks = TaskManager::current(); + let exec = tasks.executor(); + + let network_config = NetworkArgs { + discovery: DiscoveryArgs { disable_discovery: true, ..DiscoveryArgs::default() }, + ..NetworkArgs::default() + }; + + // Apply tree config modifier if present + let tree_config = if let Some(modifier) = self.tree_config_modifier { + modifier(reth_node_api::TreeConfig::default()) + } else { + reth_node_api::TreeConfig::default() + }; + + let mut nodes: Vec> = Vec::with_capacity(self.num_nodes); + + for idx in 0..self.num_nodes { + // Create base node config + let base_config = NodeConfig::new(self.chain_spec.clone()) + .with_network(network_config.clone()) + .with_unused_ports() + .with_rpc( + RpcServerArgs::default() + .with_unused_ports() + .with_http() + .with_http_api(RpcModuleSelection::All), + ); + + // Apply node config modifier if present + let node_config = if let Some(modifier) = &self.node_config_modifier { + modifier(base_config) + } else { + base_config + }; + + let span = span!(Level::INFO, "node", idx); + let _enter = span.enter(); + let node = N::default(); + let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config) + .testing_node(exec.clone()) + .with_types_and_provider::>() + .with_components(node.components_builder()) + .with_add_ons(node.add_ons()) + .launch_with_fn(|builder| { + let launcher = EngineNodeLauncher::new( + builder.task_executor().clone(), + builder.config().datadir(), + tree_config.clone(), + ); + builder.launch_with(launcher) + }) + .await?; + + let mut node = NodeTestContext::new(node, self.attributes_generator).await?; + + let genesis = node.block_hash(0); + node.update_forkchoice(genesis, genesis).await?; + + // Connect nodes if requested + if self.connect_nodes { + if let Some(previous_node) = nodes.last_mut() { + previous_node.connect(&mut node).await; + } + + // Connect last node with the first if there are more than two + if idx + 1 == self.num_nodes && + self.num_nodes > 2 && + let Some(first_node) = nodes.first_mut() + { + node.connect(first_node).await; + } + } + + nodes.push(node); + } + + Ok((nodes, tasks, Wallet::default().with_chain_id(self.chain_spec.chain().into()))) + } +} + +impl std::fmt::Debug for E2ETestSetupBuilder +where + N: NodeBuilderHelper, + F: Fn(u64) -> <::Payload as PayloadTypes>::PayloadBuilderAttributes + + Send + + Sync + + Copy + + 'static, + LocalPayloadAttributesBuilder: + PayloadAttributesBuilder<::PayloadAttributes>, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("E2ETestSetupBuilder") + .field("num_nodes", &self.num_nodes) + .field("connect_nodes", &self.connect_nodes) + .field("tree_config_modifier", &self.tree_config_modifier.as_ref().map(|_| "")) + .field("node_config_modifier", &self.node_config_modifier.as_ref().map(|_| "")) + .finish_non_exhaustive() + } +} diff --git a/crates/e2e-test-utils/src/test_rlp_utils.rs b/crates/e2e-test-utils/src/test_rlp_utils.rs index 2d48e08d36..5055196946 100644 --- a/crates/e2e-test-utils/src/test_rlp_utils.rs +++ b/crates/e2e-test-utils/src/test_rlp_utils.rs @@ -159,7 +159,6 @@ pub fn write_blocks_to_rlp(blocks: &[SealedBlock], path: &Path) -> std::io::Resu ); // Debug: check what's in the encoded data - debug!(target: "e2e::import", "Block {} encoded to {} bytes", i, buf.len()); if buf.len() < 20 { debug!(target: "e2e::import", " Raw bytes: {:?}", &buf); } else { diff --git a/crates/e2e-test-utils/tests/e2e-testsuite/main.rs b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs index 5cd1bfe8c6..04422ba34a 100644 --- a/crates/e2e-test-utils/tests/e2e-testsuite/main.rs +++ b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs @@ -15,9 +15,11 @@ use reth_e2e_test_utils::{ setup::{NetworkSetup, Setup}, Environment, TestBuilder, }, + E2ETestSetupBuilder, }; use reth_node_api::TreeConfig; use reth_node_ethereum::{EthEngineTypes, EthereumNode}; +use reth_payload_builder::EthPayloadBuilderAttributes; use std::sync::Arc; use tempfile::TempDir; use tracing::debug; @@ -349,3 +351,38 @@ async fn test_testsuite_multinode_block_production() -> Result<()> { Ok(()) } + +#[tokio::test] +async fn test_setup_builder_with_custom_tree_config() -> Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis( + serde_json::from_str(include_str!( + "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" + )) + .unwrap(), + ) + .cancun_activated() + .build(), + ); + + let (nodes, _tasks, _wallet) = + E2ETestSetupBuilder::::new(1, chain_spec, |_| { + EthPayloadBuilderAttributes::default() + }) + .with_tree_config_modifier(|config| { + config.with_persistence_threshold(0).with_memory_block_buffer_target(5) + }) + .build() + .await?; + + assert_eq!(nodes.len(), 1); + + let genesis_hash = nodes[0].block_hash(0); + assert_ne!(genesis_hash, B256::ZERO); + + Ok(()) +} diff --git a/crates/engine/local/Cargo.toml b/crates/engine/local/Cargo.toml index 98793a24b2..dd708dee90 100644 --- a/crates/engine/local/Cargo.toml +++ b/crates/engine/local/Cargo.toml @@ -11,11 +11,11 @@ exclude.workspace = true [dependencies] # reth reth-chainspec.workspace = true -reth-engine-primitives.workspace = true +reth-engine-primitives = { workspace = true, features = ["std"] } reth-ethereum-engine-primitives.workspace = true reth-payload-builder.workspace = true reth-payload-primitives.workspace = true -reth-provider.workspace = true +reth-storage-api.workspace = true reth-transaction-pool.workspace = true # alloy diff --git a/crates/engine/local/src/miner.rs b/crates/engine/local/src/miner.rs index f668606ffe..4713ebb3cd 100644 --- a/crates/engine/local/src/miner.rs +++ b/crates/engine/local/src/miner.rs @@ -10,7 +10,7 @@ use reth_payload_builder::PayloadBuilderHandle; use reth_payload_primitives::{ BuiltPayload, EngineApiMessageVersion, PayloadAttributesBuilder, PayloadKind, PayloadTypes, }; -use reth_provider::BlockReader; +use reth_storage_api::BlockReader; use reth_transaction_pool::TransactionPool; use std::{ collections::VecDeque, diff --git a/crates/engine/primitives/src/config.rs b/crates/engine/primitives/src/config.rs index 6f759036eb..0b9b7d9f82 100644 --- a/crates/engine/primitives/src/config.rs +++ b/crates/engine/primitives/src/config.rs @@ -6,9 +6,6 @@ pub const DEFAULT_PERSISTENCE_THRESHOLD: u64 = 2; /// How close to the canonical head we persist blocks. pub const DEFAULT_MEMORY_BLOCK_BUFFER_TARGET: u64 = 0; -/// Default maximum concurrency for on-demand proof tasks (blinded nodes) -pub const DEFAULT_MAX_PROOF_TASK_CONCURRENCY: u64 = 256; - /// Minimum number of workers we allow configuring explicitly. pub const MIN_WORKER_COUNT: usize = 32; @@ -102,8 +99,6 @@ pub struct TreeConfig { cross_block_cache_size: u64, /// Whether the host has enough parallelism to run state root task. has_enough_parallelism: bool, - /// Maximum number of concurrent proof tasks - max_proof_task_concurrency: u64, /// Whether multiproof task should chunk proof targets. multiproof_chunking_enabled: bool, /// Multiproof task chunk size for proof targets. @@ -153,7 +148,6 @@ impl Default for TreeConfig { state_provider_metrics: false, cross_block_cache_size: DEFAULT_CROSS_BLOCK_CACHE_SIZE, has_enough_parallelism: has_enough_parallelism(), - max_proof_task_concurrency: DEFAULT_MAX_PROOF_TASK_CONCURRENCY, multiproof_chunking_enabled: true, multiproof_chunk_size: DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE, reserved_cpu_cores: DEFAULT_RESERVED_CPU_CORES, @@ -184,7 +178,6 @@ impl TreeConfig { state_provider_metrics: bool, cross_block_cache_size: u64, has_enough_parallelism: bool, - max_proof_task_concurrency: u64, multiproof_chunking_enabled: bool, multiproof_chunk_size: usize, reserved_cpu_cores: usize, @@ -196,7 +189,6 @@ impl TreeConfig { storage_worker_count: usize, account_worker_count: usize, ) -> Self { - assert!(max_proof_task_concurrency > 0, "max_proof_task_concurrency must be at least 1"); Self { persistence_threshold, memory_block_buffer_target, @@ -210,7 +202,6 @@ impl TreeConfig { state_provider_metrics, cross_block_cache_size, has_enough_parallelism, - max_proof_task_concurrency, multiproof_chunking_enabled, multiproof_chunk_size, reserved_cpu_cores, @@ -249,11 +240,6 @@ impl TreeConfig { self.max_execute_block_batch_size } - /// Return the maximum proof task concurrency. - pub const fn max_proof_task_concurrency(&self) -> u64 { - self.max_proof_task_concurrency - } - /// Return whether the multiproof task chunking is enabled. pub const fn multiproof_chunking_enabled(&self) -> bool { self.multiproof_chunking_enabled @@ -420,16 +406,6 @@ impl TreeConfig { self } - /// Setter for maximum number of concurrent proof tasks. - pub const fn with_max_proof_task_concurrency( - mut self, - max_proof_task_concurrency: u64, - ) -> Self { - assert!(max_proof_task_concurrency > 0, "max_proof_task_concurrency must be at least 1"); - self.max_proof_task_concurrency = max_proof_task_concurrency; - self - } - /// Setter for whether multiproof task should chunk proof targets. pub const fn with_multiproof_chunking_enabled( mut self, diff --git a/crates/engine/tree/src/chain.rs b/crates/engine/tree/src/chain.rs index d1e63a6b3d..3e6207c9d4 100644 --- a/crates/engine/tree/src/chain.rs +++ b/crates/engine/tree/src/chain.rs @@ -71,7 +71,7 @@ where /// Internal function used to advance the chain. /// /// Polls the `ChainOrchestrator` for the next event. - #[tracing::instrument(name = "ChainOrchestrator::poll", skip(self, cx))] + #[tracing::instrument(level = "debug", target = "engine::tree::chain_orchestrator", skip_all)] fn poll_next_event(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = self.get_mut(); diff --git a/crates/engine/tree/src/tree/cached_state.rs b/crates/engine/tree/src/tree/cached_state.rs index 3e9cda38f1..c1bb028cab 100644 --- a/crates/engine/tree/src/tree/cached_state.rs +++ b/crates/engine/tree/src/tree/cached_state.rs @@ -354,7 +354,7 @@ impl ExecutionCache { } /// Invalidates the storage for all addresses in the set - #[instrument(level = "debug", target = "engine::tree", skip_all, fields(accounts = addresses.len()))] + #[instrument(level = "debug", target = "engine::caching", skip_all, fields(accounts = addresses.len()))] pub(crate) fn invalidate_storages(&self, addresses: HashSet<&Address>) { // NOTE: this must collect because the invalidate function should not be called while we // hold an iter for it @@ -386,7 +386,7 @@ impl ExecutionCache { /// ## Error Handling /// /// Returns an error if the state updates are inconsistent and should be discarded. - #[instrument(level = "debug", target = "engine::tree", skip_all)] + #[instrument(level = "debug", target = "engine::caching", skip_all)] pub(crate) fn insert_state(&self, state_updates: &BundleState) -> Result<(), ()> { let _enter = debug_span!(target: "engine::tree", "contracts", len = state_updates.contracts.len()) diff --git a/crates/engine/tree/src/tree/payload_processor/mod.rs b/crates/engine/tree/src/tree/payload_processor/mod.rs index 10c0f8f44c..8be64d00e7 100644 --- a/crates/engine/tree/src/tree/payload_processor/mod.rs +++ b/crates/engine/tree/src/tree/payload_processor/mod.rs @@ -40,10 +40,13 @@ use reth_trie_sparse::{ ClearedSparseStateTrie, SparseStateTrie, SparseTrie, }; use reth_trie_sparse_parallel::{ParallelSparseTrie, ParallelismThresholds}; -use std::sync::{ - atomic::AtomicBool, - mpsc::{self, channel, Sender}, - Arc, +use std::{ + sync::{ + atomic::AtomicBool, + mpsc::{self, channel, Sender}, + Arc, + }, + time::Instant, }; use tracing::{debug, debug_span, instrument, warn}; @@ -193,6 +196,7 @@ where + Clone + 'static, { + let span = tracing::Span::current(); let (to_sparse_trie, sparse_trie_rx) = channel(); // spawn multiproof task, save the trie input let (trie_input, state_root_config) = MultiProofConfig::from_input(trie_input); @@ -206,7 +210,6 @@ where ); let storage_worker_count = config.storage_worker_count(); let account_worker_count = config.account_worker_count(); - let max_proof_task_concurrency = config.max_proof_task_concurrency() as usize; let proof_handle = ProofWorkerHandle::new( self.executor.handle().clone(), consistent_view, @@ -215,15 +218,11 @@ where account_worker_count, ); - // We set it to half of the proof task concurrency, because often for each multiproof we - // spawn one Tokio task for the account proof, and one Tokio task for the storage proof. - let max_multi_proof_task_concurrency = max_proof_task_concurrency / 2; let multi_proof_task = MultiProofTask::new( state_root_config, self.executor.clone(), proof_handle.clone(), to_sparse_trie, - max_multi_proof_task_concurrency, config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size()), ); @@ -242,7 +241,6 @@ where ); // spawn multi-proof task - let span = tracing::Span::current(); self.executor.spawn_blocking(move || { let _enter = span.entered(); multi_proof_task.run(); @@ -601,8 +599,16 @@ impl ExecutionCache { /// A cache is considered available when: /// - It exists and matches the requested parent hash /// - No other tasks are currently using it (checked via Arc reference count) + #[instrument(level = "debug", target = "engine::tree::payload_processor", skip(self))] pub(crate) fn get_cache_for(&self, parent_hash: B256) -> Option { + let start = Instant::now(); let cache = self.inner.read(); + + let elapsed = start.elapsed(); + if elapsed.as_millis() > 5 { + warn!(blocked_for=?elapsed, "Blocked waiting for execution cache mutex"); + } + cache .as_ref() .filter(|c| c.executed_block_hash() == parent_hash && c.is_available()) diff --git a/crates/engine/tree/src/tree/payload_processor/multiproof.rs b/crates/engine/tree/src/tree/payload_processor/multiproof.rs index 163714483f..1e5b226f59 100644 --- a/crates/engine/tree/src/tree/payload_processor/multiproof.rs +++ b/crates/engine/tree/src/tree/payload_processor/multiproof.rs @@ -34,6 +34,10 @@ use std::{ }; use tracing::{debug, error, instrument, trace}; +/// Default upper bound for inflight multiproof calculations. These would be sitting in the queue +/// waiting to be processed. +const DEFAULT_MULTIPROOF_INFLIGHT_LIMIT: usize = 128; + /// A trie update that can be applied to sparse trie alongside the proofs for touched parts of the /// state. #[derive(Default, Debug)] @@ -338,8 +342,8 @@ impl MultiproofInput { /// availability has been signaled. #[derive(Debug)] pub struct MultiproofManager { - /// Maximum number of concurrent calculations. - max_concurrent: usize, + /// Maximum number of proof calculations allowed to be inflight at once. + inflight_limit: usize, /// Currently running calculations. inflight: usize, /// Queued calculations. @@ -370,11 +374,10 @@ impl MultiproofManager { executor: WorkloadExecutor, metrics: MultiProofTaskMetrics, proof_worker_handle: ProofWorkerHandle, - max_concurrent: usize, ) -> Self { Self { - pending: VecDeque::with_capacity(max_concurrent), - max_concurrent, + pending: VecDeque::with_capacity(DEFAULT_MULTIPROOF_INFLIGHT_LIMIT), + inflight_limit: DEFAULT_MULTIPROOF_INFLIGHT_LIMIT, executor, inflight: 0, metrics, @@ -384,11 +387,10 @@ impl MultiproofManager { } const fn is_full(&self) -> bool { - self.inflight >= self.max_concurrent + self.inflight >= self.inflight_limit } - /// Spawns a new multiproof calculation or enqueues it for later if - /// `max_concurrent` are already inflight. + /// Spawns a new multiproof calculation or enqueues it if the inflight limit is reached. fn spawn_or_queue(&mut self, input: PendingMultiproofTask) { // If there are no proof targets, we can just send an empty multiproof back immediately if input.proof_targets_is_empty() { @@ -685,7 +687,6 @@ impl MultiProofTask { executor: WorkloadExecutor, proof_worker_handle: ProofWorkerHandle, to_sparse_trie: Sender, - max_concurrency: usize, chunk_size: Option, ) -> Self { let (tx, rx) = channel(); @@ -704,7 +705,6 @@ impl MultiProofTask { executor, metrics.clone(), proof_worker_handle, - max_concurrency, ), metrics, } @@ -780,7 +780,7 @@ impl MultiProofTask { let all_proofs_processed = proofs_processed >= state_update_proofs_requested + prefetch_proofs_requested; let no_pending = !self.proof_sequencer.has_pending(); - debug!( + trace!( target: "engine::root", proofs_processed, state_update_proofs_requested, @@ -975,7 +975,12 @@ impl MultiProofTask { /// currently being calculated, or if there are any pending proofs in the proof sequencer /// left to be revealed by checking the pending tasks. /// 6. This task exits after all pending proofs are processed. - #[instrument(level = "debug", target = "engine::tree::payload_processor::multiproof", skip_all)] + #[instrument( + level = "debug", + name = "MultiProofTask::run", + target = "engine::tree::payload_processor::multiproof", + skip_all + )] pub(crate) fn run(mut self) { // TODO convert those into fields let mut prefetch_proofs_requested = 0; @@ -1011,7 +1016,7 @@ impl MultiProofTask { let storage_targets = targets.values().map(|slots| slots.len()).sum::(); prefetch_proofs_requested += self.on_prefetch_proof(targets); - debug!( + trace!( target: "engine::root", account_targets, storage_targets, @@ -1032,7 +1037,7 @@ impl MultiProofTask { let len = update.len(); state_update_proofs_requested += self.on_state_update(source, update); - debug!( + trace!( target: "engine::root", ?source, len, @@ -1094,7 +1099,7 @@ impl MultiProofTask { .proof_calculation_duration_histogram .record(proof_calculated.elapsed); - debug!( + trace!( target: "engine::root", sequence = proof_calculated.sequence_number, total_proofs = proofs_processed, @@ -1234,7 +1239,7 @@ mod tests { ProofWorkerHandle::new(executor.handle().clone(), consistent_view, task_ctx, 1, 1); let channel = channel(); - MultiProofTask::new(config, executor, proof_handle, channel.0, 1, None) + MultiProofTask::new(config, executor, proof_handle, channel.0, Some(1)) } #[test] diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index de8a88a167..e57a2aeaa8 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -147,9 +147,6 @@ where let (done_tx, done_rx) = mpsc::channel(); let mut executing = 0usize; - // Initialize worker handles container - let mut handles = Vec::with_capacity(max_concurrency); - // When transaction_count_hint is 0, it means the count is unknown. In this case, spawn // max workers to handle potentially many transactions in parallel rather // than bottlenecking on a single worker. @@ -159,6 +156,9 @@ where transaction_count_hint.min(max_concurrency) }; + // Initialize worker handles container + let mut handles = Vec::with_capacity(workers_needed); + // Only spawn initial workers as needed for i in 0..workers_needed { handles.push(ctx.spawn_worker(i, &executor, actions_tx.clone(), done_tx.clone())); @@ -233,8 +233,19 @@ where }); } + /// Returns true if prewarming was terminated and no more transactions should be prewarmed. + fn is_execution_terminated(&self) -> bool { + self.ctx.terminate_execution.load(Ordering::Relaxed) + } + /// If configured and the tx returned proof targets, emit the targets the transaction produced fn send_multi_proof_targets(&self, targets: Option) { + if self.is_execution_terminated() { + // if execution is already terminated then we dont need to send more proof fetch + // messages + return + } + if let Some((proof_targets, to_multi_proof)) = targets.zip(self.to_multi_proof.as_ref()) { let _ = to_multi_proof.send(MultiProofMessage::PrefetchProofs(proof_targets)); } @@ -308,6 +319,7 @@ where match event { PrewarmTaskEvent::TerminateTransactionExecution => { // stop tx processing + debug!(target: "engine::tree::prewarm", "Terminating prewarm execution"); self.ctx.terminate_execution.store(true, Ordering::Relaxed); } PrewarmTaskEvent::Outcome { proof_targets } => { @@ -338,7 +350,7 @@ where } } - trace!(target: "engine::tree::prewarm", "Completed prewarm execution"); + debug!(target: "engine::tree::prewarm", "Completed prewarm execution"); // save caches and finish if let Some(Some(state)) = final_block_output { @@ -460,6 +472,9 @@ where debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm tx", index, tx_hash=%tx.tx().tx_hash()) .entered(); + // create the tx env + let start = Instant::now(); + // If the task was cancelled, stop execution, send an empty result to notify the task, // and exit. if terminate_execution.load(Ordering::Relaxed) { @@ -467,8 +482,6 @@ where break } - // create the tx env - let start = Instant::now(); let res = match evm.transact(&tx) { Ok(res) => res, Err(err) => { @@ -489,6 +502,13 @@ where drop(_enter); + // If the task was cancelled, stop execution, send an empty result to notify the task, + // and exit. + if terminate_execution.load(Ordering::Relaxed) { + let _ = sender.send(PrewarmTaskEvent::Outcome { proof_targets: None }); + break + } + // Only send outcome for transactions after the first txn // as the main execution will be just as fast if index > 0 { diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index cacc546e44..424f08c261 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -329,7 +329,7 @@ where skip_all, fields( parent = ?input.parent_hash(), - block_num_hash = ?input.num_hash() + type_name = ?input.type_name(), ) )] pub fn validate_block_with_state>>( @@ -1259,4 +1259,12 @@ impl BlockOrPayload { Self::Block(block) => block.block_with_parent(), } } + + /// Returns a string showing whether or not this is a block or payload. + pub const fn type_name(&self) -> &'static str { + match self { + Self::Payload(_) => "payload", + Self::Block(_) => "block", + } + } } diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index 49ce5ab9cf..7c40680c80 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -1391,13 +1391,8 @@ fn test_validate_block_synchronous_strategy_during_persistence() { let genesis_hash = MAINNET.genesis_hash(); let valid_block = block_factory.create_valid_block(genesis_hash); - // Call validate_block_with_state directly - // This should execute the Synchronous strategy logic during active persistence - let result = test_harness.validate_block_direct(valid_block); - - // Verify validation was attempted (may fail due to test environment limitations) - // The key test is that the Synchronous strategy path is executed during persistence - assert!(result.is_ok() || result.is_err(), "Validation should complete") + // Test that Synchronous strategy executes during active persistence without panicking + let _result = test_harness.validate_block_direct(valid_block); } /// Test multiple validation scenarios including valid, consensus-invalid, and execution-invalid @@ -1411,15 +1406,9 @@ fn test_validate_block_multiple_scenarios() { let mut block_factory = TestBlockFactory::new(MAINNET.as_ref().clone()); let genesis_hash = MAINNET.genesis_hash(); - // Scenario 1: Valid block validation (may fail due to test environment limitations) + // Scenario 1: Valid block validation (test execution, not result) let valid_block = block_factory.create_valid_block(genesis_hash); - let result1 = test_harness.validate_block_direct(valid_block); - // Note: Valid blocks might fail in test environment due to missing provider data, - // but the important thing is that the validation logic executes without panicking - assert!( - result1.is_ok() || result1.is_err(), - "Valid block validation should complete (may fail due to test environment)" - ); + let _result1 = test_harness.validate_block_direct(valid_block); // Scenario 2: Block with consensus issues should be rejected let consensus_invalid = block_factory.create_invalid_consensus_block(genesis_hash); diff --git a/crates/era-utils/src/export.rs b/crates/era-utils/src/export.rs index 670a534ba0..6ccdba2426 100644 --- a/crates/era-utils/src/export.rs +++ b/crates/era-utils/src/export.rs @@ -1,6 +1,7 @@ //! Logic to export from database era1 block history //! and injecting them into era1 files with `Era1Writer`. +use crate::calculate_td_by_number; use alloy_consensus::BlockHeader; use alloy_primitives::{BlockNumber, B256, U256}; use eyre::{eyre, Result}; @@ -114,9 +115,7 @@ where let mut total_difficulty = if config.first_block_number > 0 { let prev_block_number = config.first_block_number - 1; - provider - .header_td_by_number(prev_block_number)? - .ok_or_else(|| eyre!("Total difficulty not found for block {prev_block_number}"))? + calculate_td_by_number(provider, prev_block_number)? } else { U256::ZERO }; diff --git a/crates/era-utils/src/history.rs b/crates/era-utils/src/history.rs index b1c3cd309c..58d5e383c3 100644 --- a/crates/era-utils/src/history.rs +++ b/crates/era-utils/src/history.rs @@ -1,3 +1,4 @@ +use alloy_consensus::BlockHeader; use alloy_primitives::{BlockHash, BlockNumber, U256}; use futures_util::{Stream, StreamExt}; use reth_db_api::{ @@ -19,15 +20,15 @@ use reth_etl::Collector; use reth_fs_util as fs; use reth_primitives_traits::{Block, FullBlockBody, FullBlockHeader, NodePrimitives}; use reth_provider::{ - providers::StaticFileProviderRWRefMut, BlockWriter, ProviderError, StaticFileProviderFactory, + providers::StaticFileProviderRWRefMut, BlockReader, BlockWriter, StaticFileProviderFactory, StaticFileSegment, StaticFileWriter, }; use reth_stages_types::{ CheckpointBlockRange, EntitiesCheckpoint, HeadersCheckpoint, StageCheckpoint, StageId, }; use reth_storage_api::{ - errors::ProviderResult, DBProvider, DatabaseProviderFactory, HeaderProvider, - NodePrimitivesProvider, StageCheckpointWriter, + errors::ProviderResult, DBProvider, DatabaseProviderFactory, NodePrimitivesProvider, + StageCheckpointWriter, }; use std::{ collections::Bound, @@ -82,11 +83,6 @@ where .get_highest_static_file_block(StaticFileSegment::Headers) .unwrap_or_default(); - // Find the latest total difficulty - let mut td = static_file_provider - .header_td_by_number(height)? - .ok_or(ProviderError::TotalDifficultyNotFound(height))?; - while let Some(meta) = rx.recv()? { let from = height; let provider = provider_factory.database_provider_rw()?; @@ -96,7 +92,6 @@ where &mut static_file_provider.latest_writer(StaticFileSegment::Headers)?, &provider, hash_collector, - &mut td, height.., )?; @@ -146,7 +141,7 @@ where /// Extracts block headers and bodies from `meta` and appends them using `writer` and `provider`. /// -/// Adds on to `total_difficulty` and collects hash to height using `hash_collector`. +/// Collects hash to height using `hash_collector`. /// /// Skips all blocks below the [`start_bound`] of `block_numbers` and stops when reaching past the /// [`end_bound`] or the end of the file. @@ -160,7 +155,6 @@ pub fn process( writer: &mut StaticFileProviderRWRefMut<'_,

::Primitives>, provider: &P, hash_collector: &mut Collector, - total_difficulty: &mut U256, block_numbers: impl RangeBounds, ) -> eyre::Result where @@ -182,7 +176,7 @@ where as Box) -> eyre::Result<(BH, BB)>>); let iter = ProcessIter { iter, era: meta }; - process_iter(iter, writer, provider, hash_collector, total_difficulty, block_numbers) + process_iter(iter, writer, provider, hash_collector, block_numbers) } type ProcessInnerIter = @@ -271,7 +265,6 @@ pub fn process_iter( writer: &mut StaticFileProviderRWRefMut<'_,

::Primitives>, provider: &P, hash_collector: &mut Collector, - total_difficulty: &mut U256, block_numbers: impl RangeBounds, ) -> eyre::Result where @@ -311,11 +304,8 @@ where let hash = header.hash_slow(); last_header_number = number; - // Increase total difficulty - *total_difficulty += header.difficulty(); - // Append to Headers segment - writer.append_header(&header, *total_difficulty, &hash)?; + writer.append_header(&header, &hash)?; // Write bodies to database. provider.append_block_bodies(vec![(header.number(), Some(body))])?; @@ -382,3 +372,28 @@ where Ok(()) } + +/// Calculates the total difficulty for a given block number by summing the difficulty +/// of all blocks from genesis to the given block. +/// +/// Very expensive - iterates through all blocks in batches of 1000. +/// +/// Returns an error if any block is missing. +pub fn calculate_td_by_number

(provider: &P, num: BlockNumber) -> eyre::Result +where + P: BlockReader, +{ + let mut total_difficulty = U256::ZERO; + let mut start = 0; + + while start <= num { + let end = (start + 1000 - 1).min(num); + + total_difficulty += + provider.headers_range(start..=end)?.iter().map(|h| h.difficulty()).sum::(); + + start = end + 1; + } + + Ok(total_difficulty) +} diff --git a/crates/era-utils/src/lib.rs b/crates/era-utils/src/lib.rs index 966709d2f2..13a5ceefe9 100644 --- a/crates/era-utils/src/lib.rs +++ b/crates/era-utils/src/lib.rs @@ -14,5 +14,6 @@ pub use export::{export, ExportConfig}; /// Imports history from ERA files. pub use history::{ - build_index, decode, import, open, process, process_iter, save_stage_checkpoints, ProcessIter, + build_index, calculate_td_by_number, decode, import, open, process, process_iter, + save_stage_checkpoints, ProcessIter, }; diff --git a/crates/ethereum/node/tests/e2e/dev.rs b/crates/ethereum/node/tests/e2e/dev.rs index 5ccd74ecb2..bf022a514e 100644 --- a/crates/ethereum/node/tests/e2e/dev.rs +++ b/crates/ethereum/node/tests/e2e/dev.rs @@ -3,7 +3,7 @@ use alloy_genesis::Genesis; use alloy_primitives::{b256, hex, Address}; use futures::StreamExt; use reth_chainspec::ChainSpec; -use reth_node_api::{BlockBody, FullNodeComponents, FullNodePrimitives, NodeTypes}; +use reth_node_api::{BlockBody, FullNodeComponents}; use reth_node_builder::{rpc::RethRpcAddOns, FullNode, NodeBuilder, NodeConfig, NodeHandle}; use reth_node_core::args::DevArgs; use reth_node_ethereum::{node::EthereumAddOns, EthereumNode}; @@ -81,7 +81,6 @@ async fn assert_chain_advances(node: &FullNode) where N: FullNodeComponents, AddOns: RethRpcAddOns, - N::Types: NodeTypes, { let mut notifications = node.provider.canonical_state_stream(); diff --git a/crates/ethereum/payload/src/lib.rs b/crates/ethereum/payload/src/lib.rs index 30d5c4053b..5a0f394608 100644 --- a/crates/ethereum/payload/src/lib.rs +++ b/crates/ethereum/payload/src/lib.rs @@ -236,10 +236,10 @@ where if is_osaka && estimated_block_size_with_tx > MAX_RLP_BLOCK_SIZE { best_txs.mark_invalid( &pool_tx, - InvalidPoolTransactionError::OversizedData( - estimated_block_size_with_tx, - MAX_RLP_BLOCK_SIZE, - ), + InvalidPoolTransactionError::OversizedData { + size: estimated_block_size_with_tx, + limit: MAX_RLP_BLOCK_SIZE, + }, ); continue; } diff --git a/crates/ethereum/primitives/Cargo.toml b/crates/ethereum/primitives/Cargo.toml index efa8b945f9..3bf9e8f3a4 100644 --- a/crates/ethereum/primitives/Cargo.toml +++ b/crates/ethereum/primitives/Cargo.toml @@ -73,6 +73,7 @@ reth-codec = [ "dep:reth-zstd-compressors", ] arbitrary = [ + "std", "dep:arbitrary", "alloy-consensus/arbitrary", "alloy-consensus/k256", diff --git a/crates/exex/exex/src/backfill/factory.rs b/crates/exex/exex/src/backfill/factory.rs index d9a51bc47a..29734b905e 100644 --- a/crates/exex/exex/src/backfill/factory.rs +++ b/crates/exex/exex/src/backfill/factory.rs @@ -39,7 +39,7 @@ impl BackfillJobFactory { } /// Sets the prune modes - pub fn with_prune_modes(mut self, prune_modes: PruneModes) -> Self { + pub const fn with_prune_modes(mut self, prune_modes: PruneModes) -> Self { self.prune_modes = prune_modes; self } diff --git a/crates/exex/exex/src/backfill/stream.rs b/crates/exex/exex/src/backfill/stream.rs index aa7cacdba4..9d50737f5a 100644 --- a/crates/exex/exex/src/backfill/stream.rs +++ b/crates/exex/exex/src/backfill/stream.rs @@ -256,7 +256,7 @@ mod tests { use reth_ethereum_primitives::{Block, BlockBody, Transaction}; use reth_evm_ethereum::EthEvmConfig; use reth_primitives_traits::{ - crypto::secp256k1::public_key_to_address, Block as _, FullNodePrimitives, + crypto::secp256k1::public_key_to_address, Block as _, NodePrimitives, }; use reth_provider::{ providers::{BlockchainProvider, ProviderNodeTypes}, @@ -395,7 +395,7 @@ mod tests { ) -> Result<()> where N: ProviderNodeTypes< - Primitives: FullNodePrimitives< + Primitives: NodePrimitives< Block = reth_ethereum_primitives::Block, BlockBody = reth_ethereum_primitives::BlockBody, Receipt = reth_ethereum_primitives::Receipt, diff --git a/crates/exex/exex/src/backfill/test_utils.rs b/crates/exex/exex/src/backfill/test_utils.rs index a3d8242882..e489a98abf 100644 --- a/crates/exex/exex/src/backfill/test_utils.rs +++ b/crates/exex/exex/src/backfill/test_utils.rs @@ -10,7 +10,7 @@ use reth_evm::{ ConfigureEvm, }; use reth_evm_ethereum::EthEvmConfig; -use reth_node_api::FullNodePrimitives; +use reth_node_api::NodePrimitives; use reth_primitives_traits::{Block as _, RecoveredBlock}; use reth_provider::{ providers::ProviderNodeTypes, BlockWriter as _, ExecutionOutcome, LatestStateProviderRef, @@ -58,7 +58,7 @@ pub(crate) fn execute_block_and_commit_to_database( ) -> eyre::Result> where N: ProviderNodeTypes< - Primitives: FullNodePrimitives< + Primitives: NodePrimitives< Block = reth_ethereum_primitives::Block, BlockBody = reth_ethereum_primitives::BlockBody, Receipt = reth_ethereum_primitives::Receipt, @@ -169,7 +169,7 @@ pub(crate) fn blocks_and_execution_outputs( > where N: ProviderNodeTypes< - Primitives: FullNodePrimitives< + Primitives: NodePrimitives< Block = reth_ethereum_primitives::Block, BlockBody = reth_ethereum_primitives::BlockBody, Receipt = reth_ethereum_primitives::Receipt, @@ -193,7 +193,7 @@ pub(crate) fn blocks_and_execution_outcome( ) -> eyre::Result<(Vec>, ExecutionOutcome)> where N: ProviderNodeTypes, - N::Primitives: FullNodePrimitives< + N::Primitives: NodePrimitives< Block = reth_ethereum_primitives::Block, Receipt = reth_ethereum_primitives::Receipt, >, diff --git a/crates/net/discv4/Cargo.toml b/crates/net/discv4/Cargo.toml index 20691a6d92..fadda2b634 100644 --- a/crates/net/discv4/Cargo.toml +++ b/crates/net/discv4/Cargo.toml @@ -35,7 +35,6 @@ tracing.workspace = true thiserror.workspace = true parking_lot.workspace = true rand_08 = { workspace = true, optional = true } -generic-array.workspace = true serde = { workspace = true, optional = true } itertools.workspace = true @@ -53,7 +52,6 @@ serde = [ "alloy-primitives/serde", "discv5/serde", "enr/serde", - "generic-array/serde", "parking_lot/serde", "rand_08?/serde", "secp256k1/serde", diff --git a/crates/net/discv4/src/node.rs b/crates/net/discv4/src/node.rs index 242c388322..7e993ff833 100644 --- a/crates/net/discv4/src/node.rs +++ b/crates/net/discv4/src/node.rs @@ -1,5 +1,4 @@ use alloy_primitives::keccak256; -use generic_array::GenericArray; use reth_network_peers::{NodeRecord, PeerId}; /// The key type for the table. @@ -15,8 +14,7 @@ impl From for NodeKey { impl From for discv5::Key { fn from(value: NodeKey) -> Self { let hash = keccak256(value.0.as_slice()); - let hash = *GenericArray::from_slice(hash.as_slice()); - Self::new_raw(value, hash) + Self::new_raw(value, hash.0.into()) } } diff --git a/crates/net/downloaders/src/bodies/test_utils.rs b/crates/net/downloaders/src/bodies/test_utils.rs index a7172ec1a0..513226a2c9 100644 --- a/crates/net/downloaders/src/bodies/test_utils.rs +++ b/crates/net/downloaders/src/bodies/test_utils.rs @@ -3,7 +3,7 @@ #![allow(dead_code)] use alloy_consensus::BlockHeader; -use alloy_primitives::{B256, U256}; +use alloy_primitives::B256; use reth_ethereum_primitives::BlockBody; use reth_network_p2p::bodies::response::BlockResponse; use reth_primitives_traits::{Block, SealedBlock, SealedHeader}; @@ -55,9 +55,7 @@ pub(crate) fn insert_headers( .expect("failed to create writer"); for header in headers { - writer - .append_header(header.header(), U256::ZERO, &header.hash()) - .expect("failed to append header"); + writer.append_header(header.header(), &header.hash()).expect("failed to append header"); } drop(writer); provider_rw.commit().expect("failed to commit"); diff --git a/crates/net/downloaders/src/file_client.rs b/crates/net/downloaders/src/file_client.rs index 34c2f56b75..de3d8f8f1f 100644 --- a/crates/net/downloaders/src/file_client.rs +++ b/crates/net/downloaders/src/file_client.rs @@ -815,7 +815,7 @@ mod tests { // construct headers downloader and use first header let mut header_downloader = ReverseHeadersDownloaderBuilder::default() - .build(Arc::clone(&Arc::new(client)), Arc::new(TestConsensus::default())); + .build(Arc::new(client), Arc::new(TestConsensus::default())); header_downloader.update_local_head(local_header.clone()); header_downloader.update_sync_target(SyncTarget::Tip(sync_target_hash)); @@ -890,7 +890,7 @@ mod tests { // construct headers downloader and use first header let mut header_downloader = ReverseHeadersDownloaderBuilder::default() - .build(Arc::clone(&Arc::new(client)), Arc::new(TestConsensus::default())); + .build(Arc::new(client), Arc::new(TestConsensus::default())); header_downloader.update_local_head(local_header.clone()); header_downloader.update_sync_target(SyncTarget::Tip(sync_target_hash)); diff --git a/crates/net/ecies/src/codec.rs b/crates/net/ecies/src/codec.rs index c4c45366c6..938e44d938 100644 --- a/crates/net/ecies/src/codec.rs +++ b/crates/net/ecies/src/codec.rs @@ -58,7 +58,7 @@ impl Decoder for ECIESCodec { type Item = IngressECIESValue; type Error = ECIESError; - #[instrument(skip_all, fields(peer=?self.ecies.remote_id, state=?self.state))] + #[instrument(level = "trace", skip_all, fields(peer=?self.ecies.remote_id, state=?self.state))] fn decode(&mut self, buf: &mut BytesMut) -> Result, Self::Error> { loop { match self.state { @@ -110,7 +110,7 @@ impl Decoder for ECIESCodec { self.ecies.read_header(&mut buf.split_to(ECIES::header_len()))?; if body_size > MAX_INITIAL_HANDSHAKE_SIZE { - trace!(?body_size, max=?MAX_INITIAL_HANDSHAKE_SIZE, "Header exceeds max initial handshake size"); + trace!(?body_size, max=?MAX_INITIAL_HANDSHAKE_SIZE, "Body exceeds max initial handshake size"); return Err(ECIESErrorImpl::InitialHeaderBodyTooLarge { body_size, max_body_size: MAX_INITIAL_HANDSHAKE_SIZE, @@ -150,7 +150,7 @@ impl Decoder for ECIESCodec { impl Encoder for ECIESCodec { type Error = io::Error; - #[instrument(skip(self, buf), fields(peer=?self.ecies.remote_id, state=?self.state))] + #[instrument(level = "trace", skip(self, buf), fields(peer=?self.ecies.remote_id, state=?self.state))] fn encode(&mut self, item: EgressECIESValue, buf: &mut BytesMut) -> Result<(), Self::Error> { match item { EgressECIESValue::Auth => { diff --git a/crates/net/eth-wire/src/multiplex.rs b/crates/net/eth-wire/src/multiplex.rs index 9eb4f15f0b..058dfe311e 100644 --- a/crates/net/eth-wire/src/multiplex.rs +++ b/crates/net/eth-wire/src/multiplex.rs @@ -332,9 +332,9 @@ impl ProtocolProxy { return Ok(msg); } - let mut masked = Vec::from(msg); + let mut masked: BytesMut = msg.into(); masked[0] = masked[0].checked_add(offset).ok_or(io::ErrorKind::InvalidInput)?; - Ok(masked.into()) + Ok(masked.freeze()) } /// Unmasks the message ID of a message received from the wire. diff --git a/crates/net/peers/src/node_record.rs b/crates/net/peers/src/node_record.rs index 0b1ef38b3d..641f2d274d 100644 --- a/crates/net/peers/src/node_record.rs +++ b/crates/net/peers/src/node_record.rs @@ -309,6 +309,18 @@ mod tests { } } + #[test] + fn test_node_record() { + let url = "enode://fc8a2ff614e848c0af4c99372a81b8655edb8e11b617cffd0aab1a0691bcca66ca533626a528ee567f05f70c8cb529bda2c0a864cc0aec638a367fd2bb8e49fb@127.0.0.1:35481?discport=0"; + let node: NodeRecord = url.parse().unwrap(); + assert_eq!(node, NodeRecord { + address: IpAddr::V4([127,0,0, 1].into()), + tcp_port: 35481, + udp_port: 0, + id: "0xfc8a2ff614e848c0af4c99372a81b8655edb8e11b617cffd0aab1a0691bcca66ca533626a528ee567f05f70c8cb529bda2c0a864cc0aec638a367fd2bb8e49fb".parse().unwrap(), + }) + } + #[test] fn test_url_parse() { let url = "enode://6f8a80d14311c39f35f516fa664deaaaa13e85b2f7493f37f6144d86991ec012937307647bd3b9a82abe2974e1407241d54947bbb39763a4cac9f77166ad92a0@10.3.58.6:30303?discport=30301"; diff --git a/crates/node/builder/src/builder/mod.rs b/crates/node/builder/src/builder/mod.rs index fb22a82795..8f01f251b5 100644 --- a/crates/node/builder/src/builder/mod.rs +++ b/crates/node/builder/src/builder/mod.rs @@ -21,8 +21,7 @@ use reth_network::{ NetworkPrimitives, }; use reth_node_api::{ - FullNodePrimitives, FullNodeTypes, FullNodeTypesAdapter, NodeAddOns, NodeTypes, - NodeTypesWithDBAdapter, + FullNodeTypes, FullNodeTypesAdapter, NodeAddOns, NodeTypes, NodeTypesWithDBAdapter, }; use reth_node_core::{ cli::config::{PayloadBuilderConfig, RethTransactionPoolConfig}, @@ -397,7 +396,6 @@ where >>::Components, >, >, - N::Primitives: FullNodePrimitives, EngineNodeLauncher: LaunchNode< NodeBuilderWithComponents, N::ComponentsBuilder, N::AddOns>, >, diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 2d1fb6924d..190cfdc881 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -41,10 +41,12 @@ use eyre::Context; use rayon::ThreadPoolBuilder; use reth_chainspec::{Chain, EthChainSpec, EthereumHardfork, EthereumHardforks}; use reth_config::{config::EtlConfig, PruneConfig}; +use reth_consensus::noop::NoopConsensus; use reth_db_api::{database::Database, database_metrics::DatabaseMetrics}; use reth_db_common::init::{init_genesis, InitStorageError}; +use reth_downloaders::{bodies::noop::NoopBodiesDownloader, headers::noop::NoopHeaderDownloader}; use reth_engine_local::MiningMode; -use reth_evm::ConfigureEvm; +use reth_evm::{noop::NoopEvmConfig, ConfigureEvm}; use reth_exex::ExExManagerHandle; use reth_fs_util as fs; use reth_network_p2p::headers::client::HeadersClient; @@ -65,19 +67,25 @@ use reth_node_metrics::{ }; use reth_provider::{ providers::{NodeTypesForProvider, ProviderNodeTypes, StaticFileProvider}, - BlockNumReader, BlockReaderIdExt, ProviderError, ProviderFactory, ProviderResult, - StaticFileProviderFactory, + BlockHashReader, BlockNumReader, BlockReaderIdExt, ProviderError, ProviderFactory, + ProviderResult, StageCheckpointReader, StaticFileProviderFactory, }; use reth_prune::{PruneModes, PrunerBuilder}; use reth_rpc_builder::config::RethRpcServerConfig; use reth_rpc_layer::JwtSecret; -use reth_stages::{stages::EraImportSource, MetricEvent}; +use reth_stages::{ + sets::DefaultStages, stages::EraImportSource, MetricEvent, PipelineBuilder, PipelineTarget, + StageId, +}; use reth_static_file::StaticFileProducer; use reth_tasks::TaskExecutor; use reth_tracing::tracing::{debug, error, info, warn}; use reth_transaction_pool::TransactionPool; use std::{sync::Arc, thread::available_parallelism}; -use tokio::sync::mpsc::{unbounded_channel, UnboundedSender}; +use tokio::sync::{ + mpsc::{unbounded_channel, UnboundedSender}, + oneshot, watch, +}; use futures::{future::Either, stream, Stream, StreamExt}; use reth_node_ethstats::EthStatsService; @@ -151,7 +159,7 @@ impl LaunchContext { let mut toml_config = reth_config::Config::from_path(&config_path) .wrap_err_with(|| format!("Could not load config file {config_path:?}"))?; - Self::save_pruning_config_if_full_node(&mut toml_config, config, &config_path)?; + Self::save_pruning_config(&mut toml_config, config, &config_path)?; info!(target: "reth::cli", path = ?config_path, "Configuration loaded"); @@ -161,8 +169,9 @@ impl LaunchContext { Ok(toml_config) } - /// Save prune config to the toml file if node is a full node. - fn save_pruning_config_if_full_node( + /// Save prune config to the toml file if node is a full node or has custom pruning CLI + /// arguments. + fn save_pruning_config( reth_config: &mut reth_config::Config, config: &NodeConfig, config_path: impl AsRef, @@ -170,14 +179,14 @@ impl LaunchContext { where ChainSpec: EthChainSpec + reth_chainspec::EthereumHardforks, { - if reth_config.prune.is_none() { - if let Some(prune_config) = config.prune_config() { - reth_config.update_prune_config(prune_config); + if let Some(prune_config) = config.prune_config() { + if reth_config.prune != prune_config { + reth_config.set_prune_config(prune_config); info!(target: "reth::cli", "Saving prune config to toml file"); reth_config.save(config_path.as_ref())?; } - } else if config.prune_config().is_none() { - warn!(target: "reth::cli", "Prune configs present in config file but --full not provided. Running as a Full node"); + } else if !reth_config.prune.is_default() { + warn!(target: "reth::cli", "Pruning configuration is present in the config file, but no CLI arguments are provided. Using config from file."); } Ok(()) } @@ -393,18 +402,19 @@ impl LaunchContextWith Option + pub fn prune_config(&self) -> PruneConfig where ChainSpec: reth_chainspec::EthereumHardforks, { + let toml_config = self.toml_config().prune.clone(); let Some(mut node_prune_config) = self.node_config().prune_config() else { // No CLI config is set, use the toml config. - return self.toml_config().prune.clone(); + return toml_config; }; // Otherwise, use the CLI configuration and merge with toml config. - node_prune_config.merge(self.toml_config().prune.clone()); - Some(node_prune_config) + node_prune_config.merge(toml_config); + node_prune_config } /// Returns the configured [`PruneModes`], returning the default if no config was available. @@ -412,7 +422,7 @@ impl LaunchContextWith LaunchContextWith, Evm: ConfigureEvm + 'static, { - Ok(ProviderFactory::new( + let factory = ProviderFactory::new( self.right().clone(), self.chain_spec(), StaticFileProvider::read_write(self.data_dir().static_files())?, ) .with_prune_modes(self.prune_modes()) - .with_static_files_metrics()) + .with_static_files_metrics(); + + let has_receipt_pruning = self.toml_config().prune.has_receipts_pruning(); + + // Check for consistency between database and static files. If it fails, it unwinds to + // the first block that's consistent between database and static files. + if let Some(unwind_target) = factory + .static_file_provider() + .check_consistency(&factory.provider()?, has_receipt_pruning)? + { + // Highly unlikely to happen, and given its destructive nature, it's better to panic + // instead. + assert_ne!( + unwind_target, + PipelineTarget::Unwind(0), + "A static file <> database inconsistency was found that would trigger an unwind to block 0" + ); + + info!(target: "reth::cli", unwind_target = %unwind_target, "Executing an unwind after a failed storage consistency check."); + + let (_tip_tx, tip_rx) = watch::channel(B256::ZERO); + + // Builds an unwind-only pipeline + let pipeline = PipelineBuilder::default() + .add_stages(DefaultStages::new( + factory.clone(), + tip_rx, + Arc::new(NoopConsensus::default()), + NoopHeaderDownloader::default(), + NoopBodiesDownloader::default(), + NoopEvmConfig::::default(), + self.toml_config().stages.clone(), + self.prune_modes(), + None, + )) + .build( + factory.clone(), + StaticFileProducer::new(factory.clone(), self.prune_modes()), + ); + + // Unwinds to block + let (tx, rx) = oneshot::channel(); + + // Pipeline should be run as blocking and panic if it fails. + self.task_executor().spawn_critical_blocking( + "pipeline task", + Box::pin(async move { + let (_, result) = pipeline.run_as_fut(Some(unwind_target)).await; + let _ = tx.send(result); + }), + ); + rx.await?.inspect_err(|err| { + error!(target: "reth::cli", unwind_target = %unwind_target, %err, "failed to run unwind") + })?; + } + + Ok(factory) } /// Creates a new [`ProviderFactory`] and attaches it to the launch context. @@ -787,6 +851,21 @@ where &self.node_adapter().provider } + /// Returns the initial backfill to sync to at launch. + /// + /// This returns the configured `debug.tip` if set, otherwise it will check if backfill was + /// previously interrupted and returns the block hash of the last checkpoint, see also + /// [`Self::check_pipeline_consistency`] + pub fn initial_backfill_target(&self) -> ProviderResult> { + let mut initial_target = self.node_config().debug.tip; + + if initial_target.is_none() { + initial_target = self.check_pipeline_consistency()?; + } + + Ok(initial_target) + } + /// Returns true if the node should terminate after the initial backfill run. /// /// This is the case if any of these configs are set: @@ -800,7 +879,7 @@ where /// /// This checks for OP-Mainnet and ensures we have all the necessary data to progress (past /// bedrock height) - pub fn ensure_chain_specific_db_checks(&self) -> ProviderResult<()> { + fn ensure_chain_specific_db_checks(&self) -> ProviderResult<()> { if self.chain_spec().is_optimism() && !self.is_dev() && self.chain_id() == Chain::optimism_mainnet() @@ -818,6 +897,54 @@ where Ok(()) } + /// Check if the pipeline is consistent (all stages have the checkpoint block numbers no less + /// than the checkpoint of the first stage). + /// + /// This will return the pipeline target if: + /// * the pipeline was interrupted during its previous run + /// * a new stage was added + /// * stage data was dropped manually through `reth stage drop ...` + /// + /// # Returns + /// + /// A target block hash if the pipeline is inconsistent, otherwise `None`. + pub fn check_pipeline_consistency(&self) -> ProviderResult> { + // If no target was provided, check if the stages are congruent - check if the + // checkpoint of the last stage matches the checkpoint of the first. + let first_stage_checkpoint = self + .blockchain_db() + .get_stage_checkpoint(*StageId::ALL.first().unwrap())? + .unwrap_or_default() + .block_number; + + // Skip the first stage as we've already retrieved it and comparing all other checkpoints + // against it. + for stage_id in StageId::ALL.iter().skip(1) { + let stage_checkpoint = self + .blockchain_db() + .get_stage_checkpoint(*stage_id)? + .unwrap_or_default() + .block_number; + + // If the checkpoint of any stage is less than the checkpoint of the first stage, + // retrieve and return the block hash of the latest header and use it as the target. + if stage_checkpoint < first_stage_checkpoint { + debug!( + target: "consensus::engine", + first_stage_checkpoint, + inconsistent_stage_id = %stage_id, + inconsistent_stage_checkpoint = stage_checkpoint, + "Pipeline sync progress is inconsistent" + ); + return self.blockchain_db().block_hash(first_stage_checkpoint); + } + } + + self.ensure_chain_specific_db_checks()?; + + Ok(None) + } + /// Expire the pre-merge transactions if the node is configured to do so and the chain has a /// merge block. /// @@ -1080,17 +1207,14 @@ mod tests { storage_history_before: None, bodies_pre_merge: false, bodies_distance: None, + #[expect(deprecated)] receipts_log_filter: None, bodies_before: None, }, ..NodeConfig::test() }; - LaunchContext::save_pruning_config_if_full_node( - &mut reth_config, - &node_config, - config_path, - ) - .unwrap(); + LaunchContext::save_pruning_config(&mut reth_config, &node_config, config_path) + .unwrap(); let loaded_config = Config::from_path(config_path).unwrap(); diff --git a/crates/node/builder/src/launch/engine.rs b/crates/node/builder/src/launch/engine.rs index 88ce2ed9cf..1d39572531 100644 --- a/crates/node/builder/src/launch/engine.rs +++ b/crates/node/builder/src/launch/engine.rs @@ -117,6 +117,9 @@ impl EngineNodeLauncher { })? .with_components(components_builder, on_component_initialized).await?; + // Try to expire pre-merge transaction history if configured + ctx.expire_pre_merge_transactions()?; + // spawn exexs if any let maybe_exex_manager_handle = ctx.launch_exex(installed_exex).await?; @@ -138,7 +141,7 @@ impl EngineNodeLauncher { let consensus = Arc::new(ctx.components().consensus().clone()); - let mut pipeline = build_networked_pipeline( + let pipeline = build_networked_pipeline( &ctx.toml_config().stages, network_client.clone(), consensus.clone(), @@ -154,18 +157,7 @@ impl EngineNodeLauncher { )?; // The new engine writes directly to static files. This ensures that they're up to the tip. - pipeline.ensure_static_files_consistency().await?; - - // Try to expire pre-merge transaction history if configured - ctx.expire_pre_merge_transactions()?; - - let initial_target = if let Some(tip) = ctx.node_config().debug.tip { - Some(tip) - } else { - pipeline.initial_backfill_target()? - }; - - ctx.ensure_chain_specific_db_checks()?; + pipeline.move_to_static_files()?; let pipeline_events = pipeline.events(); @@ -176,7 +168,7 @@ impl EngineNodeLauncher { } let pruner = pruner_builder.build_with_provider_factory(ctx.provider_factory().clone()); let pruner_events = pruner.events(); - info!(target: "reth::cli", prune_config=?ctx.prune_config().unwrap_or_default(), "Pruner initialized"); + info!(target: "reth::cli", prune_config=?ctx.prune_config(), "Pruner initialized"); let event_sender = EventSender::default(); @@ -258,6 +250,7 @@ impl EngineNodeLauncher { add_ons.launch_add_ons(add_ons_ctx).await?; // Run consensus engine to completion + let initial_target = ctx.initial_backfill_target()?; let mut built_payloads = ctx .components() .payload_builder_handle() diff --git a/crates/node/builder/src/node.rs b/crates/node/builder/src/node.rs index ca44ad9523..1cc50c4ba6 100644 --- a/crates/node/builder/src/node.rs +++ b/crates/node/builder/src/node.rs @@ -179,14 +179,16 @@ where /// Returns the [`EngineApiClient`] interface for the authenticated engine API. /// /// This will send authenticated http requests to the node's auth server. - pub fn engine_http_client(&self) -> impl EngineApiClient { + pub fn engine_http_client(&self) -> impl EngineApiClient + use { self.auth_server_handle().http_client() } /// Returns the [`EngineApiClient`] interface for the authenticated engine API. /// /// This will send authenticated ws requests to the node's auth server. - pub async fn engine_ws_client(&self) -> impl EngineApiClient { + pub async fn engine_ws_client( + &self, + ) -> impl EngineApiClient + use { self.auth_server_handle().ws_client().await } @@ -194,7 +196,9 @@ where /// /// This will send not authenticated IPC requests to the node's auth server. #[cfg(unix)] - pub async fn engine_ipc_client(&self) -> Option> { + pub async fn engine_ipc_client( + &self, + ) -> Option + use> { self.auth_server_handle().ipc_client().await } } diff --git a/crates/node/builder/src/setup.rs b/crates/node/builder/src/setup.rs index a409969119..ad78ffb59a 100644 --- a/crates/node/builder/src/setup.rs +++ b/crates/node/builder/src/setup.rs @@ -36,7 +36,7 @@ pub fn build_networked_pipeline( provider_factory: ProviderFactory, task_executor: &TaskExecutor, metrics_tx: reth_stages::MetricEventsSender, - prune_config: Option, + prune_config: PruneConfig, max_block: Option, static_file_producer: StaticFileProducer>, evm_config: Evm, @@ -85,7 +85,7 @@ pub fn build_pipeline( consensus: Arc>, max_block: Option, metrics_tx: reth_stages::MetricEventsSender, - prune_config: Option, + prune_config: PruneConfig, static_file_producer: StaticFileProducer>, evm_config: Evm, exex_manager_handle: ExExManagerHandle, @@ -106,8 +106,6 @@ where let (tip_tx, tip_rx) = watch::channel(B256::ZERO); - let prune_modes = prune_config.map(|prune| prune.segments).unwrap_or_default(); - let pipeline = builder .with_tip_sender(tip_tx) .with_metrics_tx(metrics_tx) @@ -120,7 +118,7 @@ where body_downloader, evm_config.clone(), stage_config.clone(), - prune_modes, + prune_config.segments, era_import_source, ) .set(ExecutionStage::new( diff --git a/crates/node/core/Cargo.toml b/crates/node/core/Cargo.toml index 4d4fd475ac..1a4f85b619 100644 --- a/crates/node/core/Cargo.toml +++ b/crates/node/core/Cargo.toml @@ -52,7 +52,6 @@ derive_more.workspace = true toml.workspace = true serde.workspace = true strum = { workspace = true, features = ["derive"] } -thiserror.workspace = true url.workspace = true # io diff --git a/crates/node/core/src/args/engine.rs b/crates/node/core/src/args/engine.rs index c82b1b03a1..29535f2c1d 100644 --- a/crates/node/core/src/args/engine.rs +++ b/crates/node/core/src/args/engine.rs @@ -4,8 +4,8 @@ use clap::Args; use reth_engine_primitives::{TreeConfig, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE}; use crate::node_config::{ - DEFAULT_CROSS_BLOCK_CACHE_SIZE_MB, DEFAULT_MAX_PROOF_TASK_CONCURRENCY, - DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, DEFAULT_PERSISTENCE_THRESHOLD, DEFAULT_RESERVED_CPU_CORES, + DEFAULT_CROSS_BLOCK_CACHE_SIZE_MB, DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, + DEFAULT_PERSISTENCE_THRESHOLD, DEFAULT_RESERVED_CPU_CORES, }; /// Parameters for configuring the engine driver. @@ -63,10 +63,6 @@ pub struct EngineArgs { #[arg(long = "engine.accept-execution-requests-hash")] pub accept_execution_requests_hash: bool, - /// Configure the maximum number of concurrent proof tasks - #[arg(long = "engine.max-proof-task-concurrency", default_value_t = DEFAULT_MAX_PROOF_TASK_CONCURRENCY)] - pub max_proof_task_concurrency: u64, - /// Whether multiproof task should chunk proof targets. #[arg(long = "engine.multiproof-chunking", default_value = "true")] pub multiproof_chunking_enabled: bool, @@ -135,7 +131,6 @@ impl Default for EngineArgs { state_provider_metrics: false, cross_block_cache_size: DEFAULT_CROSS_BLOCK_CACHE_SIZE_MB, accept_execution_requests_hash: false, - max_proof_task_concurrency: DEFAULT_MAX_PROOF_TASK_CONCURRENCY, multiproof_chunking_enabled: true, multiproof_chunk_size: DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE, reserved_cpu_cores: DEFAULT_RESERVED_CPU_CORES, @@ -162,7 +157,6 @@ impl EngineArgs { .with_state_provider_metrics(self.state_provider_metrics) .with_always_compare_trie_updates(self.state_root_task_compare_updates) .with_cross_block_cache_size(self.cross_block_cache_size * 1024 * 1024) - .with_max_proof_task_concurrency(self.max_proof_task_concurrency) .with_multiproof_chunking_enabled(self.multiproof_chunking_enabled) .with_multiproof_chunk_size(self.multiproof_chunk_size) .with_reserved_cpu_cores(self.reserved_cpu_cores) diff --git a/crates/node/core/src/args/error.rs b/crates/node/core/src/args/error.rs deleted file mode 100644 index 163c063cd7..0000000000 --- a/crates/node/core/src/args/error.rs +++ /dev/null @@ -1,22 +0,0 @@ -use std::num::ParseIntError; - -/// Error while parsing a `ReceiptsLogPruneConfig` -#[derive(thiserror::Error, Debug)] -#[expect(clippy::enum_variant_names)] -pub(crate) enum ReceiptsLogError { - /// The format of the filter is invalid. - #[error("invalid filter format: {0}")] - InvalidFilterFormat(String), - /// Address is invalid. - #[error("address is invalid: {0}")] - InvalidAddress(String), - /// The prune mode is not one of full, distance, before. - #[error("prune mode is invalid: {0}")] - InvalidPruneMode(String), - /// The distance value supplied is invalid. - #[error("distance is invalid: {0}")] - InvalidDistance(ParseIntError), - /// The block number supplied is invalid. - #[error("block number is invalid: {0}")] - InvalidBlockNumber(ParseIntError), -} diff --git a/crates/node/core/src/args/mod.rs b/crates/node/core/src/args/mod.rs index 54e7774014..17584a913c 100644 --- a/crates/node/core/src/args/mod.rs +++ b/crates/node/core/src/args/mod.rs @@ -76,5 +76,4 @@ pub use ress_args::RessArgs; mod era; pub use era::{DefaultEraHost, EraArgs, EraSourceArgs}; -mod error; pub mod types; diff --git a/crates/node/core/src/args/pruning.rs b/crates/node/core/src/args/pruning.rs index 846e4e6b20..b5c782e62b 100644 --- a/crates/node/core/src/args/pruning.rs +++ b/crates/node/core/src/args/pruning.rs @@ -1,12 +1,13 @@ //! Pruning and full node arguments -use crate::{args::error::ReceiptsLogError, primitives::EthereumHardfork}; -use alloy_primitives::{Address, BlockNumber}; +use std::ops::Not; + +use crate::primitives::EthereumHardfork; +use alloy_primitives::BlockNumber; use clap::{builder::RangedU64ValueParser, Args}; use reth_chainspec::EthereumHardforks; use reth_config::config::PruneConfig; -use reth_prune_types::{PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_PRUNING_DISTANCE}; -use std::collections::BTreeMap; +use reth_prune_types::{PruneMode, PruneModes, MINIMUM_PRUNING_DISTANCE}; /// Parameters for pruning and full node #[derive(Debug, Clone, Args, PartialEq, Eq, Default)] @@ -59,12 +60,10 @@ pub struct PruningArgs { /// Prune receipts before the specified block number. The specified block number is not pruned. #[arg(long = "prune.receipts.before", value_name = "BLOCK_NUMBER", conflicts_with_all = &["receipts_full", "receipts_pre_merge", "receipts_distance"])] pub receipts_before: Option, - // Receipts Log Filter - /// Configure receipts log filter. Format: - /// <`address`>:<`prune_mode`>... where <`prune_mode`> can be 'full', 'distance:<`blocks`>', or - /// 'before:<`block_number`>' - #[arg(long = "prune.receiptslogfilter", value_name = "FILTER_CONFIG", conflicts_with_all = &["receipts_full", "receipts_pre_merge", "receipts_distance", "receipts_before"], value_parser = parse_receipts_log_filter)] - pub receipts_log_filter: Option, + /// Receipts Log Filter + #[arg(long = "prune.receiptslogfilter", value_name = "FILTER_CONFIG", hide = true)] + #[deprecated] + pub receipts_log_filter: Option, // Account History /// Prunes all account history. @@ -107,6 +106,9 @@ pub struct PruningArgs { impl PruningArgs { /// Returns pruning configuration. + /// + /// Returns [`None`] if no parameters are specified and default pruning configuration should be + /// used. pub fn prune_config(&self, chain_spec: &ChainSpec) -> Option where ChainSpec: EthereumHardforks, @@ -127,7 +129,8 @@ impl PruningArgs { // TODO: set default to pre-merge block if available bodies_history: None, merkle_changesets: PruneMode::Distance(MINIMUM_PRUNING_DISTANCE), - receipts_log_filter: Default::default(), + #[expect(deprecated)] + receipts_log_filter: (), }, } } @@ -154,16 +157,17 @@ impl PruningArgs { if let Some(mode) = self.storage_history_prune_mode() { config.segments.storage_history = Some(mode); } - if let Some(receipt_logs) = - self.receipts_log_filter.as_ref().filter(|c| !c.is_empty()).cloned() - { - config.segments.receipts_log_filter = receipt_logs; - // need to remove the receipts segment filter entirely because that takes precedence - // over the logs filter - config.segments.receipts.take(); + + // Log warning if receipts_log_filter is set (deprecated feature) + #[expect(deprecated)] + if self.receipts_log_filter.is_some() { + tracing::warn!( + target: "reth::cli", + "The --prune.receiptslogfilter flag is deprecated and has no effect. It will be removed in a future release." + ); } - Some(config) + config.is_default().not().then_some(config) } fn bodies_prune_mode(&self, chain_spec: &ChainSpec) -> Option @@ -248,141 +252,3 @@ impl PruningArgs { } } } - -/// Parses `,` separated pruning info into [`ReceiptsLogPruneConfig`]. -pub(crate) fn parse_receipts_log_filter( - value: &str, -) -> Result { - let mut config = BTreeMap::new(); - // Split out each of the filters. - let filters = value.split(','); - for filter in filters { - let parts: Vec<&str> = filter.split(':').collect(); - if parts.len() < 2 { - return Err(ReceiptsLogError::InvalidFilterFormat(filter.to_string())); - } - // Parse the address - let address = parts[0] - .parse::

() - .map_err(|_| ReceiptsLogError::InvalidAddress(parts[0].to_string()))?; - - // Parse the prune mode - let prune_mode = match parts[1] { - "full" => PruneMode::Full, - s if s.starts_with("distance") => { - if parts.len() < 3 { - return Err(ReceiptsLogError::InvalidFilterFormat(filter.to_string())); - } - let distance = - parts[2].parse::().map_err(ReceiptsLogError::InvalidDistance)?; - PruneMode::Distance(distance) - } - s if s.starts_with("before") => { - if parts.len() < 3 { - return Err(ReceiptsLogError::InvalidFilterFormat(filter.to_string())); - } - let block_number = - parts[2].parse::().map_err(ReceiptsLogError::InvalidBlockNumber)?; - PruneMode::Before(block_number) - } - _ => return Err(ReceiptsLogError::InvalidPruneMode(parts[1].to_string())), - }; - config.insert(address, prune_mode); - } - Ok(ReceiptsLogPruneConfig(config)) -} - -#[cfg(test)] -mod tests { - use super::*; - use alloy_primitives::address; - use clap::Parser; - - /// A helper type to parse Args more easily - #[derive(Parser)] - struct CommandParser { - #[command(flatten)] - args: T, - } - - #[test] - fn pruning_args_sanity_check() { - let args = CommandParser::::parse_from([ - "reth", - "--prune.receiptslogfilter", - "0x0000000000000000000000000000000000000003:before:5000000", - ]) - .args; - let mut config = ReceiptsLogPruneConfig::default(); - config.0.insert( - address!("0x0000000000000000000000000000000000000003"), - PruneMode::Before(5000000), - ); - assert_eq!(args.receipts_log_filter, Some(config)); - } - - #[test] - fn parse_receiptslogfilter() { - let default_args = PruningArgs::default(); - let args = CommandParser::::parse_from(["reth"]).args; - assert_eq!(args, default_args); - } - - #[test] - fn test_parse_receipts_log_filter() { - let filter1 = "0x0000000000000000000000000000000000000001:full"; - let filter2 = "0x0000000000000000000000000000000000000002:distance:1000"; - let filter3 = "0x0000000000000000000000000000000000000003:before:5000000"; - let filters = [filter1, filter2, filter3].join(","); - - // Args can be parsed. - let result = parse_receipts_log_filter(&filters); - assert!(result.is_ok()); - let config = result.unwrap(); - assert_eq!(config.0.len(), 3); - - // Check that the args were parsed correctly. - let addr1: Address = "0x0000000000000000000000000000000000000001".parse().unwrap(); - let addr2: Address = "0x0000000000000000000000000000000000000002".parse().unwrap(); - let addr3: Address = "0x0000000000000000000000000000000000000003".parse().unwrap(); - - assert_eq!(config.0.get(&addr1), Some(&PruneMode::Full)); - assert_eq!(config.0.get(&addr2), Some(&PruneMode::Distance(1000))); - assert_eq!(config.0.get(&addr3), Some(&PruneMode::Before(5000000))); - } - - #[test] - fn test_parse_receipts_log_filter_invalid_filter_format() { - let result = parse_receipts_log_filter("invalid_format"); - assert!(matches!(result, Err(ReceiptsLogError::InvalidFilterFormat(_)))); - } - - #[test] - fn test_parse_receipts_log_filter_invalid_address() { - let result = parse_receipts_log_filter("invalid_address:full"); - assert!(matches!(result, Err(ReceiptsLogError::InvalidAddress(_)))); - } - - #[test] - fn test_parse_receipts_log_filter_invalid_prune_mode() { - let result = - parse_receipts_log_filter("0x0000000000000000000000000000000000000000:invalid_mode"); - assert!(matches!(result, Err(ReceiptsLogError::InvalidPruneMode(_)))); - } - - #[test] - fn test_parse_receipts_log_filter_invalid_distance() { - let result = parse_receipts_log_filter( - "0x0000000000000000000000000000000000000000:distance:invalid_distance", - ); - assert!(matches!(result, Err(ReceiptsLogError::InvalidDistance(_)))); - } - - #[test] - fn test_parse_receipts_log_filter_invalid_block_number() { - let result = parse_receipts_log_filter( - "0x0000000000000000000000000000000000000000:before:invalid_block", - ); - assert!(matches!(result, Err(ReceiptsLogError::InvalidBlockNumber(_)))); - } -} diff --git a/crates/node/core/src/args/trace.rs b/crates/node/core/src/args/trace.rs index 2e37feb673..45bc9c9029 100644 --- a/crates/node/core/src/args/trace.rs +++ b/crates/node/core/src/args/trace.rs @@ -39,7 +39,7 @@ pub struct TraceArgs { long = "tracing-otlp.filter", global = true, value_name = "FILTER", - default_value = "TRACE", + default_value = "debug", help_heading = "Tracing" )] pub otlp_filter: EnvFilter, diff --git a/crates/node/core/src/cli/config.rs b/crates/node/core/src/cli/config.rs index 657b8cac1f..8c29c4745e 100644 --- a/crates/node/core/src/cli/config.rs +++ b/crates/node/core/src/cli/config.rs @@ -42,10 +42,9 @@ pub trait PayloadBuilderConfig { } match chain.kind() { - ChainKind::Named(NamedChain::Sepolia | NamedChain::Holesky | NamedChain::Hoodi) => { - ETHEREUM_BLOCK_GAS_LIMIT_60M - } - ChainKind::Named(NamedChain::Mainnet) => ETHEREUM_BLOCK_GAS_LIMIT_60M, + ChainKind::Named( + NamedChain::Mainnet | NamedChain::Sepolia | NamedChain::Holesky | NamedChain::Hoodi, + ) => ETHEREUM_BLOCK_GAS_LIMIT_60M, _ => ETHEREUM_BLOCK_GAS_LIMIT_36M, } } diff --git a/crates/node/core/src/node_config.rs b/crates/node/core/src/node_config.rs index 7b487a1fa7..61eb29db38 100644 --- a/crates/node/core/src/node_config.rs +++ b/crates/node/core/src/node_config.rs @@ -10,7 +10,7 @@ use crate::{ }; use alloy_consensus::BlockHeader; use alloy_eips::BlockHashOrNumber; -use alloy_primitives::{BlockNumber, B256}; +use alloy_primitives::{BlockNumber, B256, U256}; use eyre::eyre; use reth_chainspec::{ChainSpec, EthChainSpec, MAINNET}; use reth_config::config::PruneConfig; @@ -34,8 +34,7 @@ use tracing::*; use crate::args::{EraArgs, MetricArgs}; pub use reth_engine_primitives::{ - DEFAULT_MAX_PROOF_TASK_CONCURRENCY, DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, - DEFAULT_PERSISTENCE_THRESHOLD, DEFAULT_RESERVED_CPU_CORES, + DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, DEFAULT_PERSISTENCE_THRESHOLD, DEFAULT_RESERVED_CPU_CORES, }; /// Default size of cross-block cache in megabytes. @@ -191,6 +190,22 @@ impl NodeConfig { self } + /// Apply a function to the config. + pub fn apply(self, f: F) -> Self + where + F: FnOnce(Self) -> Self, + { + f(self) + } + + /// Applies a fallible function to the config. + pub fn try_apply(self, f: F) -> Result + where + F: FnOnce(Self) -> Result, + { + f(self) + } + /// Sets --dev mode for the node [`NodeConfig::dev`], if `dev` is true. pub const fn set_dev(self, dev: bool) -> Self { if dev { @@ -330,12 +345,6 @@ impl NodeConfig { .header_by_number(head)? .expect("the header for the latest block is missing, database is corrupt"); - let total_difficulty = provider - .header_td_by_number(head)? - // total difficulty is effectively deprecated, but still required in some places, e.g. - // p2p - .unwrap_or_default(); - let hash = provider .block_hash(head)? .expect("the hash for the latest block is missing, database is corrupt"); @@ -344,7 +353,7 @@ impl NodeConfig { number: head, hash, difficulty: header.difficulty(), - total_difficulty, + total_difficulty: U256::ZERO, timestamp: header.timestamp(), }) } diff --git a/crates/node/types/src/lib.rs b/crates/node/types/src/lib.rs index daa4d11153..b5b38f48c7 100644 --- a/crates/node/types/src/lib.rs +++ b/crates/node/types/src/lib.rs @@ -11,7 +11,7 @@ use core::{fmt::Debug, marker::PhantomData}; pub use reth_primitives_traits::{ - Block, BlockBody, FullBlock, FullNodePrimitives, FullReceipt, FullSignedTx, NodePrimitives, + Block, BlockBody, FullBlock, FullReceipt, FullSignedTx, NodePrimitives, }; use reth_chainspec::EthChainSpec; diff --git a/crates/optimism/cli/src/commands/init_state.rs b/crates/optimism/cli/src/commands/init_state.rs index 7af17ca352..950f60193f 100644 --- a/crates/optimism/cli/src/commands/init_state.rs +++ b/crates/optimism/cli/src/commands/init_state.rs @@ -12,8 +12,8 @@ use reth_optimism_primitives::{ }; use reth_primitives_traits::SealedHeader; use reth_provider::{ - BlockNumReader, ChainSpecProvider, DBProvider, DatabaseProviderFactory, - StaticFileProviderFactory, StaticFileWriter, + BlockNumReader, DBProvider, DatabaseProviderFactory, StaticFileProviderFactory, + StaticFileWriter, }; use std::{io::BufReader, sync::Arc}; use tracing::info; @@ -24,12 +24,11 @@ pub struct InitStateCommandOp { #[command(flatten)] init_state: reth_cli_commands::init_state::InitStateCommand, - /// **Optimism Mainnet Only** - /// - /// Specifies whether to initialize the state without relying on OVM historical data. + /// Specifies whether to initialize the state without relying on OVM or EVM historical data. /// /// When enabled, and before inserting the state, it creates a dummy chain up to the last OVM - /// block (#105235062) (14GB / 90 seconds). It then, appends the Bedrock block. + /// block (#105235062) (14GB / 90 seconds). It then, appends the Bedrock block. This is + /// hardcoded for OP mainnet, for other OP chains you will need to pass in a header. /// /// - **Note**: **Do not** import receipts and blocks beforehand, or this will fail or be /// ignored. @@ -40,42 +39,59 @@ pub struct InitStateCommandOp { impl> InitStateCommandOp { /// Execute the `init` command pub async fn execute>( + mut self, + ) -> eyre::Result<()> { + // If using --without-ovm for OP mainnet, handle the special case with hardcoded Bedrock + // header. Otherwise delegate to the base InitStateCommand implementation. + if self.without_ovm { + if self.init_state.env.chain.is_optimism_mainnet() { + return self.execute_with_bedrock_header::(); + } + + // For non-mainnet OP chains with --without-ovm, use the base implementation + // by setting the without_evm flag + self.init_state.without_evm = true; + } + + self.init_state.execute::().await + } + + /// Execute init-state with hardcoded Bedrock header for OP mainnet. + fn execute_with_bedrock_header< + N: CliNodeTypes, + >( self, ) -> eyre::Result<()> { - info!(target: "reth::cli", "Reth init-state starting"); - - let Environment { config, provider_factory, .. } = - self.init_state.env.init::(AccessRights::RW)?; + info!(target: "reth::cli", "Reth init-state starting for OP mainnet"); + let env = self.init_state.env.init::(AccessRights::RW)?; + let Environment { config, provider_factory, .. } = env; let static_file_provider = provider_factory.static_file_provider(); let provider_rw = provider_factory.database_provider_rw()?; - // OP-Mainnet may want to bootstrap a chain without OVM historical data - if provider_factory.chain_spec().is_optimism_mainnet() && self.without_ovm { - let last_block_number = provider_rw.last_block_number()?; + let last_block_number = provider_rw.last_block_number()?; - if last_block_number == 0 { - reth_cli_commands::init_state::without_evm::setup_without_evm( - &provider_rw, - SealedHeader::new(BEDROCK_HEADER, BEDROCK_HEADER_HASH), - |number| { - let mut header = Header::default(); - header.set_number(number); - header - }, - )?; + if last_block_number == 0 { + reth_cli_commands::init_state::without_evm::setup_without_evm( + &provider_rw, + SealedHeader::new(BEDROCK_HEADER, BEDROCK_HEADER_HASH), + |number| { + let mut header = Header::default(); + header.set_number(number); + header + }, + )?; - // SAFETY: it's safe to commit static files, since in the event of a crash, they - // will be unwound according to database checkpoints. - // - // Necessary to commit, so the BEDROCK_HEADER is accessible to provider_rw and - // init_state_dump - static_file_provider.commit()?; - } else if last_block_number > 0 && last_block_number < BEDROCK_HEADER.number { - return Err(eyre::eyre!( - "Data directory should be empty when calling init-state with --without-ovm." - )) - } + // SAFETY: it's safe to commit static files, since in the event of a crash, they + // will be unwound according to database checkpoints. + // + // Necessary to commit, so the BEDROCK_HEADER is accessible to provider_rw and + // init_state_dump + static_file_provider.commit()?; + } else if last_block_number > 0 && last_block_number < BEDROCK_HEADER.number { + return Err(eyre::eyre!( + "Data directory should be empty when calling init-state with --without-ovm." + )) } info!(target: "reth::cli", "Initiating state dump"); diff --git a/crates/optimism/flashblocks/Cargo.toml b/crates/optimism/flashblocks/Cargo.toml index 532cd4d696..977e28d37e 100644 --- a/crates/optimism/flashblocks/Cargo.toml +++ b/crates/optimism/flashblocks/Cargo.toml @@ -16,17 +16,17 @@ reth-optimism-primitives = { workspace = true, features = ["serde"] } reth-optimism-evm.workspace = true reth-chain-state = { workspace = true, features = ["serde"] } reth-primitives-traits = { workspace = true, features = ["serde"] } +reth-engine-primitives = { workspace = true, features = ["std"] } reth-execution-types = { workspace = true, features = ["serde"] } reth-evm.workspace = true reth-revm.workspace = true reth-optimism-payload-builder.workspace = true reth-rpc-eth-types.workspace = true reth-errors.workspace = true +reth-payload-primitives.workspace = true reth-storage-api.workspace = true -reth-node-api.workspace = true reth-tasks.workspace = true reth-metrics.workspace = true -reth-trie.workspace = true # alloy alloy-eips = { workspace = true, features = ["serde"] } diff --git a/crates/optimism/flashblocks/src/consensus.rs b/crates/optimism/flashblocks/src/consensus.rs index 353eddbf4c..60314d2f6c 100644 --- a/crates/optimism/flashblocks/src/consensus.rs +++ b/crates/optimism/flashblocks/src/consensus.rs @@ -1,7 +1,8 @@ use crate::FlashBlockCompleteSequenceRx; use alloy_primitives::B256; -use reth_node_api::{ConsensusEngineHandle, EngineApiMessageVersion}; +use reth_engine_primitives::ConsensusEngineHandle; use reth_optimism_payload_builder::OpPayloadTypes; +use reth_payload_primitives::EngineApiMessageVersion; use ringbuffer::{AllocRingBuffer, RingBuffer}; use tracing::warn; diff --git a/crates/optimism/flashblocks/src/lib.rs b/crates/optimism/flashblocks/src/lib.rs index 1164703993..d36ddb21fc 100644 --- a/crates/optimism/flashblocks/src/lib.rs +++ b/crates/optimism/flashblocks/src/lib.rs @@ -1,5 +1,13 @@ //! A downstream integration of Flashblocks. +#![doc( + html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", + html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", + issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" +)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![cfg_attr(not(test), warn(unused_crate_dependencies))] + pub use payload::{ ExecutionPayloadBaseV1, ExecutionPayloadFlashblockDeltaV1, FlashBlock, FlashBlockDecoder, Metadata, diff --git a/crates/optimism/flashblocks/src/payload.rs b/crates/optimism/flashblocks/src/payload.rs index f7d8a38c96..da81ada016 100644 --- a/crates/optimism/flashblocks/src/payload.rs +++ b/crates/optimism/flashblocks/src/payload.rs @@ -3,9 +3,9 @@ use alloy_eips::eip4895::Withdrawal; use alloy_primitives::{bytes, Address, Bloom, Bytes, B256, U256}; use alloy_rpc_types_engine::PayloadId; use derive_more::Deref; -use reth_node_api::NodePrimitives; use reth_optimism_evm::OpNextBlockEnvAttributes; use reth_optimism_primitives::OpReceipt; +use reth_primitives_traits::NodePrimitives; use reth_rpc_eth_types::PendingBlock; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; diff --git a/crates/primitives-traits/src/block/error.rs b/crates/primitives-traits/src/block/error.rs index f61d352bba..ccb727ce88 100644 --- a/crates/primitives-traits/src/block/error.rs +++ b/crates/primitives-traits/src/block/error.rs @@ -3,6 +3,37 @@ use crate::transaction::signed::RecoveryError; /// Type alias for [`BlockRecoveryError`] with a [`SealedBlock`](crate::SealedBlock) value. +/// +/// This error type is specifically used when recovering a sealed block fails. +/// It contains the original sealed block that could not be recovered, allowing +/// callers to inspect the problematic block or attempt recovery with different +/// parameters. +/// +/// # Example +/// +/// ```rust +/// use alloy_consensus::{Block, BlockBody, Header, Signed, TxEnvelope, TxLegacy}; +/// use alloy_primitives::{Signature, B256}; +/// use reth_primitives_traits::{block::error::SealedBlockRecoveryError, SealedBlock}; +/// +/// // Create a simple block for demonstration +/// let header = Header::default(); +/// let tx = TxLegacy::default(); +/// let signed_tx = Signed::new_unchecked(tx, Signature::test_signature(), B256::ZERO); +/// let envelope = TxEnvelope::Legacy(signed_tx); +/// let body = BlockBody { transactions: vec![envelope], ommers: vec![], withdrawals: None }; +/// let block = Block::new(header, body); +/// let sealed_block = SealedBlock::new_unchecked(block, B256::ZERO); +/// +/// // Simulate a block recovery operation that fails +/// let block_recovery_result: Result<(), SealedBlockRecoveryError<_>> = +/// Err(SealedBlockRecoveryError::new(sealed_block)); +/// +/// // When block recovery fails, you get the error with the original block +/// let error = block_recovery_result.unwrap_err(); +/// let failed_block = error.into_inner(); +/// // Now you can inspect the failed block or try recovery again +/// ``` pub type SealedBlockRecoveryError = BlockRecoveryError>; /// Error when recovering a block from [`SealedBlock`](crate::SealedBlock) to diff --git a/crates/primitives-traits/src/block/mod.rs b/crates/primitives-traits/src/block/mod.rs index 2aeade9bc1..7705512d63 100644 --- a/crates/primitives-traits/src/block/mod.rs +++ b/crates/primitives-traits/src/block/mod.rs @@ -50,17 +50,9 @@ pub mod serde_bincode_compat { } /// Helper trait that unifies all behaviour required by block to support full node operations. -pub trait FullBlock: - Block + alloy_rlp::Encodable + alloy_rlp::Decodable -{ -} +pub trait FullBlock: Block {} -impl FullBlock for T where - T: Block - + alloy_rlp::Encodable - + alloy_rlp::Decodable -{ -} +impl FullBlock for T where T: Block {} /// Helper trait to access [`BlockBody::Transaction`] given a [`Block`]. pub type BlockTx = <::Body as BlockBody>::Transaction; diff --git a/crates/primitives-traits/src/lib.rs b/crates/primitives-traits/src/lib.rs index 1cc56ce2cb..67df9637fa 100644 --- a/crates/primitives-traits/src/lib.rs +++ b/crates/primitives-traits/src/lib.rs @@ -188,7 +188,7 @@ pub use size::InMemorySize; /// Node traits pub mod node; -pub use node::{BlockTy, BodyTy, FullNodePrimitives, HeaderTy, NodePrimitives, ReceiptTy, TxTy}; +pub use node::{BlockTy, BodyTy, HeaderTy, NodePrimitives, ReceiptTy, TxTy}; /// Helper trait that requires de-/serialize implementation since `serde` feature is enabled. #[cfg(feature = "serde")] diff --git a/crates/primitives-traits/src/node.rs b/crates/primitives-traits/src/node.rs index 1f5bfed139..f23ff222ab 100644 --- a/crates/primitives-traits/src/node.rs +++ b/crates/primitives-traits/src/node.rs @@ -1,6 +1,5 @@ use crate::{ - Block, FullBlock, FullBlockBody, FullBlockHeader, FullReceipt, FullSignedTx, - MaybeSerdeBincodeCompat, Receipt, + FullBlock, FullBlockBody, FullBlockHeader, FullReceipt, FullSignedTx, MaybeSerdeBincodeCompat, }; use core::fmt; @@ -13,7 +12,8 @@ pub trait NodePrimitives: Send + Sync + Unpin + Clone + Default + fmt::Debug + PartialEq + Eq + 'static { /// Block primitive. - type Block: Block
+ MaybeSerdeBincodeCompat; + type Block: FullBlock
+ + MaybeSerdeBincodeCompat; /// Block header primitive. type BlockHeader: FullBlockHeader; /// Block body primitive. @@ -24,30 +24,7 @@ pub trait NodePrimitives: /// format that includes the signature and can be included in a block. type SignedTx: FullSignedTx; /// A receipt. - type Receipt: Receipt; -} -/// Helper trait that sets trait bounds on [`NodePrimitives`]. -pub trait FullNodePrimitives -where - Self: NodePrimitives< - Block: FullBlock
, - BlockHeader: FullBlockHeader, - BlockBody: FullBlockBody, - SignedTx: FullSignedTx, - Receipt: FullReceipt, - >, -{ -} - -impl FullNodePrimitives for T where - T: NodePrimitives< - Block: FullBlock
, - BlockHeader: FullBlockHeader, - BlockBody: FullBlockBody, - SignedTx: FullSignedTx, - Receipt: FullReceipt, - > -{ + type Receipt: FullReceipt; } /// Helper adapter type for accessing [`NodePrimitives`] block header types. diff --git a/crates/prune/prune/Cargo.toml b/crates/prune/prune/Cargo.toml index 094570b873..615a793bb8 100644 --- a/crates/prune/prune/Cargo.toml +++ b/crates/prune/prune/Cargo.toml @@ -13,7 +13,6 @@ workspace = true [dependencies] # reth -reth-chainspec.workspace = true reth-exex-types.workspace = true reth-db-api.workspace = true reth-errors.workspace = true @@ -25,7 +24,6 @@ reth-primitives-traits.workspace = true reth-static-file-types.workspace = true # ethereum -alloy-consensus.workspace = true alloy-eips.workspace = true # metrics diff --git a/crates/prune/prune/src/builder.rs b/crates/prune/prune/src/builder.rs index f21319bb45..f61aa6bd46 100644 --- a/crates/prune/prune/src/builder.rs +++ b/crates/prune/prune/src/builder.rs @@ -1,6 +1,5 @@ use crate::{segments::SegmentSet, Pruner}; use alloy_eips::eip2718::Encodable2718; -use reth_chainspec::MAINNET_PRUNE_DELETE_LIMIT; use reth_config::PruneConfig; use reth_db_api::{table::Value, transaction::DbTxMut}; use reth_exex_types::FinishedExExHeight; @@ -30,9 +29,6 @@ pub struct PrunerBuilder { } impl PrunerBuilder { - /// Default timeout for a prune run. - pub const DEFAULT_TIMEOUT: Duration = Duration::from_millis(100); - /// Creates a new [`PrunerBuilder`] from the given [`PruneConfig`]. pub fn new(pruner_config: PruneConfig) -> Self { Self::default() @@ -47,7 +43,7 @@ impl PrunerBuilder { } /// Sets the configuration for every part of the data that can be pruned. - pub fn segments(mut self, segments: PruneModes) -> Self { + pub const fn segments(mut self, segments: PruneModes) -> Self { self.segments = segments; self } @@ -135,7 +131,7 @@ impl Default for PrunerBuilder { Self { block_interval: 5, segments: PruneModes::default(), - delete_limit: MAINNET_PRUNE_DELETE_LIMIT, + delete_limit: usize::MAX, timeout: None, finished_exex_height: watch::channel(FinishedExExHeight::NoExExs).1, } diff --git a/crates/prune/prune/src/segments/mod.rs b/crates/prune/prune/src/segments/mod.rs index dc17525445..f0f688a7c8 100644 --- a/crates/prune/prune/src/segments/mod.rs +++ b/crates/prune/prune/src/segments/mod.rs @@ -15,8 +15,8 @@ pub use static_file::{ use std::{fmt::Debug, ops::RangeInclusive}; use tracing::error; pub use user::{ - AccountHistory, MerkleChangeSets, Receipts as UserReceipts, ReceiptsByLogs, SenderRecovery, - StorageHistory, TransactionLookup, + AccountHistory, MerkleChangeSets, Receipts as UserReceipts, SenderRecovery, StorageHistory, + TransactionLookup, }; /// A segment represents a pruning of some portion of the data. diff --git a/crates/prune/prune/src/segments/set.rs b/crates/prune/prune/src/segments/set.rs index 72847219b0..e551a8de9a 100644 --- a/crates/prune/prune/src/segments/set.rs +++ b/crates/prune/prune/src/segments/set.rs @@ -1,6 +1,6 @@ use crate::segments::{ - AccountHistory, MerkleChangeSets, ReceiptsByLogs, Segment, SenderRecovery, StorageHistory, - TransactionLookup, UserReceipts, + AccountHistory, MerkleChangeSets, Segment, SenderRecovery, StorageHistory, TransactionLookup, + UserReceipts, }; use alloy_eips::eip2718::Encodable2718; use reth_db_api::{table::Value, transaction::DbTxMut}; @@ -61,6 +61,7 @@ where static_file_provider: StaticFileProvider, prune_modes: PruneModes, ) -> Self { + #[expect(deprecated)] let PruneModes { sender_recovery, transaction_lookup, @@ -69,7 +70,7 @@ where storage_history, bodies_history: _, merkle_changesets, - receipts_log_filter, + receipts_log_filter: (), } = prune_modes; Self::default() @@ -87,11 +88,6 @@ where .segment_opt(storage_history.map(StorageHistory::new)) // User receipts .segment_opt(receipts.map(UserReceipts::new)) - // Receipts by logs - .segment_opt( - (!receipts_log_filter.is_empty()) - .then(|| ReceiptsByLogs::new(receipts_log_filter.clone())), - ) // Transaction lookup .segment_opt(transaction_lookup.map(TransactionLookup::new)) // Sender recovery diff --git a/crates/prune/prune/src/segments/static_file/headers.rs b/crates/prune/prune/src/segments/static_file/headers.rs index 9f3c291bf4..19b255ed3d 100644 --- a/crates/prune/prune/src/segments/static_file/headers.rs +++ b/crates/prune/prune/src/segments/static_file/headers.rs @@ -21,7 +21,9 @@ use std::num::NonZeroUsize; use tracing::trace; /// Number of header tables to prune in one step -const HEADER_TABLES_TO_PRUNE: usize = 3; +/// +/// Note: `HeaderTerminalDifficulties` is no longer pruned after Paris/Merge as it's read-only +const HEADER_TABLES_TO_PRUNE: usize = 2; #[derive(Debug)] pub struct Headers { @@ -72,9 +74,6 @@ where .tx_ref() .cursor_write::::BlockHeader>>( )?; - - let mut header_tds_cursor = - provider.tx_ref().cursor_write::()?; let mut canonical_headers_cursor = provider.tx_ref().cursor_write::()?; @@ -86,7 +85,6 @@ where provider, &mut limiter, headers_cursor.walk_range(range.clone())?, - header_tds_cursor.walk_range(range.clone())?, canonical_headers_cursor.walk_range(range)?, ); @@ -111,6 +109,7 @@ where }) } } + type Walker<'a, Provider, T> = RangeWalker<'a, T, <::Tx as DbTxMut>::CursorMut>; @@ -127,7 +126,6 @@ where Provider, tables::Headers<::BlockHeader>, >, - header_tds_walker: Walker<'a, Provider, tables::HeaderTerminalDifficulties>, canonical_headers_walker: Walker<'a, Provider, tables::CanonicalHeaders>, } @@ -149,10 +147,9 @@ where Provider, tables::Headers<::BlockHeader>, >, - header_tds_walker: Walker<'a, Provider, tables::HeaderTerminalDifficulties>, canonical_headers_walker: Walker<'a, Provider, tables::CanonicalHeaders>, ) -> Self { - Self { provider, limiter, headers_walker, header_tds_walker, canonical_headers_walker } + Self { provider, limiter, headers_walker, canonical_headers_walker } } } @@ -168,7 +165,6 @@ where } let mut pruned_block_headers = None; - let mut pruned_block_td = None; let mut pruned_block_canonical = None; if let Err(err) = self.provider.tx_ref().prune_table_with_range_step( @@ -180,15 +176,6 @@ where return Some(Err(err.into())) } - if let Err(err) = self.provider.tx_ref().prune_table_with_range_step( - &mut self.header_tds_walker, - self.limiter, - &mut |_| false, - &mut |row| pruned_block_td = Some(row.0), - ) { - return Some(Err(err.into())) - } - if let Err(err) = self.provider.tx_ref().prune_table_with_range_step( &mut self.canonical_headers_walker, self.limiter, @@ -198,7 +185,7 @@ where return Some(Err(err.into())) } - if ![pruned_block_headers, pruned_block_td, pruned_block_canonical].iter().all_equal() { + if ![pruned_block_headers, pruned_block_canonical].iter().all_equal() { return Some(Err(PrunerError::InconsistentData( "All headers-related tables should be pruned up to the same height", ))) @@ -216,7 +203,7 @@ mod tests { static_file::headers::HEADER_TABLES_TO_PRUNE, PruneInput, PruneLimiter, Segment, SegmentOutput, }; - use alloy_primitives::{BlockNumber, B256, U256}; + use alloy_primitives::{BlockNumber, B256}; use assert_matches::assert_matches; use reth_db_api::{tables, transaction::DbTx}; use reth_provider::{ @@ -241,18 +228,17 @@ mod tests { let headers = random_header_range(&mut rng, 0..100, B256::ZERO); let tx = db.factory.provider_rw().unwrap().into_tx(); for header in &headers { - TestStageDB::insert_header(None, &tx, header, U256::ZERO).unwrap(); + TestStageDB::insert_header(None, &tx, header).unwrap(); } tx.commit().unwrap(); assert_eq!(db.table::().unwrap().len(), headers.len()); assert_eq!(db.table::().unwrap().len(), headers.len()); - assert_eq!(db.table::().unwrap().len(), headers.len()); let test_prune = |to_block: BlockNumber, expected_result: (PruneProgress, usize)| { let segment = super::Headers::new(db.factory.static_file_provider()); let prune_mode = PruneMode::Before(to_block); - let mut limiter = PruneLimiter::default().set_deleted_entries_limit(10); + let mut limiter = PruneLimiter::default().set_deleted_entries_limit(6); let input = PruneInput { previous_checkpoint: db .factory @@ -311,10 +297,6 @@ mod tests { db.table::().unwrap().len(), headers.len() - (last_pruned_block_number + 1) as usize ); - assert_eq!( - db.table::().unwrap().len(), - headers.len() - (last_pruned_block_number + 1) as usize - ); assert_eq!( db.factory.provider().unwrap().get_prune_checkpoint(PruneSegment::Headers).unwrap(), Some(PruneCheckpoint { @@ -325,11 +307,16 @@ mod tests { ); }; + // First test: Prune with limit of 6 entries + // This will prune blocks 0-2 (3 blocks × 2 tables = 6 entries) test_prune( 3, - (PruneProgress::HasMoreData(PruneInterruptReason::DeletedEntriesLimitReached), 9), + (PruneProgress::HasMoreData(PruneInterruptReason::DeletedEntriesLimitReached), 6), ); - test_prune(3, (PruneProgress::Finished, 3)); + + // Second test: Prune remaining blocks + // This will prune block 3 (1 block × 2 tables = 2 entries) + test_prune(3, (PruneProgress::Finished, 2)); } #[test] diff --git a/crates/prune/prune/src/segments/user/mod.rs b/crates/prune/prune/src/segments/user/mod.rs index c25bc6bc76..bdbc27f22f 100644 --- a/crates/prune/prune/src/segments/user/mod.rs +++ b/crates/prune/prune/src/segments/user/mod.rs @@ -2,7 +2,6 @@ mod account_history; mod history; mod merkle_change_sets; mod receipts; -mod receipts_by_logs; mod sender_recovery; mod storage_history; mod transaction_lookup; @@ -10,7 +9,6 @@ mod transaction_lookup; pub use account_history::AccountHistory; pub use merkle_change_sets::MerkleChangeSets; pub use receipts::Receipts; -pub use receipts_by_logs::ReceiptsByLogs; pub use sender_recovery::SenderRecovery; pub use storage_history::StorageHistory; pub use transaction_lookup::TransactionLookup; diff --git a/crates/prune/prune/src/segments/user/receipts_by_logs.rs b/crates/prune/prune/src/segments/user/receipts_by_logs.rs deleted file mode 100644 index 8fd6d1e73a..0000000000 --- a/crates/prune/prune/src/segments/user/receipts_by_logs.rs +++ /dev/null @@ -1,364 +0,0 @@ -use crate::{ - db_ext::DbTxPruneExt, - segments::{PruneInput, Segment}, - PrunerError, -}; -use alloy_consensus::TxReceipt; -use reth_db_api::{table::Value, tables, transaction::DbTxMut}; -use reth_primitives_traits::NodePrimitives; -use reth_provider::{ - BlockReader, DBProvider, NodePrimitivesProvider, PruneCheckpointWriter, TransactionsProvider, -}; -use reth_prune_types::{ - PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment, ReceiptsLogPruneConfig, SegmentOutput, - MINIMUM_PRUNING_DISTANCE, -}; -use tracing::{instrument, trace}; -#[derive(Debug)] -pub struct ReceiptsByLogs { - config: ReceiptsLogPruneConfig, -} - -impl ReceiptsByLogs { - pub const fn new(config: ReceiptsLogPruneConfig) -> Self { - Self { config } - } -} - -impl Segment for ReceiptsByLogs -where - Provider: DBProvider - + PruneCheckpointWriter - + TransactionsProvider - + BlockReader - + NodePrimitivesProvider>, -{ - fn segment(&self) -> PruneSegment { - PruneSegment::ContractLogs - } - - fn mode(&self) -> Option { - None - } - - fn purpose(&self) -> PrunePurpose { - PrunePurpose::User - } - - #[instrument(target = "pruner", skip(self, provider), ret(level = "trace"))] - fn prune(&self, provider: &Provider, input: PruneInput) -> Result { - // Contract log filtering removes every receipt possible except the ones in the list. So, - // for the other receipts it's as if they had a `PruneMode::Distance()` of - // `MINIMUM_PRUNING_DISTANCE`. - let to_block = PruneMode::Distance(MINIMUM_PRUNING_DISTANCE) - .prune_target_block(input.to_block, PruneSegment::ContractLogs, PrunePurpose::User)? - .map(|(bn, _)| bn) - .unwrap_or_default(); - - // Get status checkpoint from latest run - let mut last_pruned_block = - input.previous_checkpoint.and_then(|checkpoint| checkpoint.block_number); - - let initial_last_pruned_block = last_pruned_block; - - let mut from_tx_number = match initial_last_pruned_block { - Some(block) => provider - .block_body_indices(block)? - .map(|block| block.last_tx_num() + 1) - .unwrap_or(0), - None => 0, - }; - - // Figure out what receipts have already been pruned, so we can have an accurate - // `address_filter` - let address_filter = self.config.group_by_block(input.to_block, last_pruned_block)?; - - // Splits all transactions in different block ranges. Each block range will have its own - // filter address list and will check it while going through the table - // - // Example: - // For an `address_filter` such as: - // { block9: [a1, a2], block20: [a3, a4, a5] } - // - // The following structures will be created in the exact order as showed: - // `block_ranges`: [ - // (block0, block8, 0 addresses), - // (block9, block19, 2 addresses), - // (block20, to_block, 5 addresses) - // ] - // `filtered_addresses`: [a1, a2, a3, a4, a5] - // - // The first range will delete all receipts between block0 - block8 - // The second range will delete all receipts between block9 - 19, except the ones with - // emitter logs from these addresses: [a1, a2]. - // The third range will delete all receipts between block20 - to_block, except the ones with - // emitter logs from these addresses: [a1, a2, a3, a4, a5] - let mut block_ranges = vec![]; - let mut blocks_iter = address_filter.iter().peekable(); - let mut filtered_addresses = vec![]; - - while let Some((start_block, addresses)) = blocks_iter.next() { - filtered_addresses.extend_from_slice(addresses); - - // This will clear all receipts before the first appearance of a contract log or since - // the block after the last pruned one. - if block_ranges.is_empty() { - let init = last_pruned_block.map(|b| b + 1).unwrap_or_default(); - if init < *start_block { - block_ranges.push((init, *start_block - 1, 0)); - } - } - - let end_block = - blocks_iter.peek().map(|(next_block, _)| *next_block - 1).unwrap_or(to_block); - - // Addresses in lower block ranges, are still included in the inclusion list for future - // ranges. - block_ranges.push((*start_block, end_block, filtered_addresses.len())); - } - - trace!( - target: "pruner", - ?block_ranges, - ?filtered_addresses, - "Calculated block ranges and filtered addresses", - ); - - let mut limiter = input.limiter; - - let mut done = true; - let mut pruned = 0; - let mut last_pruned_transaction = None; - for (start_block, end_block, num_addresses) in block_ranges { - let block_range = start_block..=end_block; - - // Calculate the transaction range from this block range - let tx_range_end = match provider.block_body_indices(end_block)? { - Some(body) => body.last_tx_num(), - None => { - trace!( - target: "pruner", - ?block_range, - "No receipts to prune." - ); - continue - } - }; - let tx_range = from_tx_number..=tx_range_end; - - // Delete receipts, except the ones in the inclusion list - let mut last_skipped_transaction = 0; - let deleted; - (deleted, done) = provider.tx_ref().prune_table_with_range::::Receipt, - >>( - tx_range, - &mut limiter, - |(tx_num, receipt)| { - let skip = num_addresses > 0 && - receipt.logs().iter().any(|log| { - filtered_addresses[..num_addresses].contains(&&log.address) - }); - - if skip { - last_skipped_transaction = *tx_num; - } - skip - }, - |row| last_pruned_transaction = Some(row.0), - )?; - - trace!(target: "pruner", %deleted, %done, ?block_range, "Pruned receipts"); - - pruned += deleted; - - // For accurate checkpoints we need to know that we have checked every transaction. - // Example: we reached the end of the range, and the last receipt is supposed to skip - // its deletion. - let last_pruned_transaction = *last_pruned_transaction - .insert(last_pruned_transaction.unwrap_or_default().max(last_skipped_transaction)); - - last_pruned_block = Some( - provider - .transaction_block(last_pruned_transaction)? - .ok_or(PrunerError::InconsistentData("Block for transaction is not found"))? - // If there's more receipts to prune, set the checkpoint block number to - // previous, so we could finish pruning its receipts on the - // next run. - .saturating_sub(if done { 0 } else { 1 }), - ); - - if limiter.is_limit_reached() { - done &= end_block == to_block; - break - } - - from_tx_number = last_pruned_transaction + 1; - } - - // If there are contracts using `PruneMode::Distance(_)` there will be receipts before - // `to_block` that become eligible to be pruned in future runs. Therefore, our checkpoint is - // not actually `to_block`, but the `lowest_block_with_distance` from any contract. - // This ensures that in future pruner runs we can prune all these receipts between the - // previous `lowest_block_with_distance` and the new one using - // `get_next_tx_num_range_from_checkpoint`. - // - // Only applies if we were able to prune everything intended for this run, otherwise the - // checkpoint is the `last_pruned_block`. - let prune_mode_block = self - .config - .lowest_block_with_distance(input.to_block, initial_last_pruned_block)? - .unwrap_or(to_block); - - provider.save_prune_checkpoint( - PruneSegment::ContractLogs, - PruneCheckpoint { - block_number: Some(prune_mode_block.min(last_pruned_block.unwrap_or(u64::MAX))), - tx_number: last_pruned_transaction, - prune_mode: PruneMode::Before(prune_mode_block), - }, - )?; - - let progress = limiter.progress(done); - - Ok(SegmentOutput { progress, pruned, checkpoint: None }) - } -} - -#[cfg(test)] -mod tests { - use crate::segments::{PruneInput, PruneLimiter, ReceiptsByLogs, Segment}; - use alloy_primitives::B256; - use assert_matches::assert_matches; - use reth_db_api::{cursor::DbCursorRO, tables, transaction::DbTx}; - use reth_primitives_traits::InMemorySize; - use reth_provider::{ - DBProvider, DatabaseProviderFactory, PruneCheckpointReader, TransactionsProvider, - }; - use reth_prune_types::{PruneMode, PruneSegment, ReceiptsLogPruneConfig}; - use reth_stages::test_utils::{StorageKind, TestStageDB}; - use reth_testing_utils::generators::{ - self, random_block_range, random_eoa_account, random_log, random_receipt, BlockRangeParams, - }; - use std::collections::BTreeMap; - - #[test] - fn prune_receipts_by_logs() { - reth_tracing::init_test_tracing(); - - let db = TestStageDB::default(); - let mut rng = generators::rng(); - - let tip = 20000; - let blocks = [ - random_block_range( - &mut rng, - 0..=100, - BlockRangeParams { parent: Some(B256::ZERO), tx_count: 1..5, ..Default::default() }, - ), - random_block_range( - &mut rng, - (100 + 1)..=(tip - 100), - BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() }, - ), - random_block_range( - &mut rng, - (tip - 100 + 1)..=tip, - BlockRangeParams { parent: Some(B256::ZERO), tx_count: 1..5, ..Default::default() }, - ), - ] - .concat(); - db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); - - let mut receipts = Vec::new(); - - let (deposit_contract_addr, _) = random_eoa_account(&mut rng); - for block in &blocks { - receipts.reserve_exact(block.body().size()); - for (txi, transaction) in block.body().transactions.iter().enumerate() { - let mut receipt = random_receipt(&mut rng, transaction, Some(1), None); - receipt.logs.push(random_log( - &mut rng, - (txi == (block.transaction_count() - 1)).then_some(deposit_contract_addr), - Some(1), - )); - receipts.push((receipts.len() as u64, receipt)); - } - } - db.insert_receipts(receipts).expect("insert receipts"); - - assert_eq!( - db.table::().unwrap().len(), - blocks.iter().map(|block| block.transaction_count()).sum::() - ); - assert_eq!( - db.table::().unwrap().len(), - db.table::().unwrap().len() - ); - - let run_prune = || { - let provider = db.factory.database_provider_rw().unwrap(); - - let prune_before_block: usize = 20; - let prune_mode = PruneMode::Before(prune_before_block as u64); - let receipts_log_filter = - ReceiptsLogPruneConfig(BTreeMap::from([(deposit_contract_addr, prune_mode)])); - - let limiter = PruneLimiter::default().set_deleted_entries_limit(10); - - let result = ReceiptsByLogs::new(receipts_log_filter).prune( - &provider, - PruneInput { - previous_checkpoint: db - .factory - .provider() - .unwrap() - .get_prune_checkpoint(PruneSegment::ContractLogs) - .unwrap(), - to_block: tip, - limiter, - }, - ); - provider.commit().expect("commit"); - - assert_matches!(result, Ok(_)); - let output = result.unwrap(); - - let (pruned_block, pruned_tx) = db - .factory - .provider() - .unwrap() - .get_prune_checkpoint(PruneSegment::ContractLogs) - .unwrap() - .map(|checkpoint| (checkpoint.block_number.unwrap(), checkpoint.tx_number.unwrap())) - .unwrap_or_default(); - - // All receipts are in the end of the block - let unprunable = pruned_block.saturating_sub(prune_before_block as u64 - 1); - - assert_eq!( - db.table::().unwrap().len(), - blocks.iter().map(|block| block.transaction_count()).sum::() - - ((pruned_tx + 1) - unprunable) as usize - ); - - output.progress.is_finished() - }; - - while !run_prune() {} - - let provider = db.factory.provider().unwrap(); - let mut cursor = provider.tx_ref().cursor_read::().unwrap(); - let walker = cursor.walk(None).unwrap(); - for receipt in walker { - let (tx_num, receipt) = receipt.unwrap(); - - // Either we only find our contract, or the receipt is part of the unprunable receipts - // set by tip - 128 - assert!( - receipt.logs.iter().any(|l| l.address == deposit_contract_addr) || - provider.transaction_block(tx_num).unwrap().unwrap() > tip - 128, - ); - } - } -} diff --git a/crates/prune/types/src/lib.rs b/crates/prune/types/src/lib.rs index 315063278b..a588693892 100644 --- a/crates/prune/types/src/lib.rs +++ b/crates/prune/types/src/lib.rs @@ -18,10 +18,6 @@ mod pruner; mod segment; mod target; -use alloc::{collections::BTreeMap, vec::Vec}; -use alloy_primitives::{Address, BlockNumber}; -use core::ops::Deref; - pub use checkpoint::PruneCheckpoint; pub use event::PrunerEvent; pub use mode::PruneMode; @@ -31,300 +27,3 @@ pub use pruner::{ }; pub use segment::{PrunePurpose, PruneSegment, PruneSegmentError}; pub use target::{PruneModes, UnwindTargetPrunedError, MINIMUM_PRUNING_DISTANCE}; - -/// Configuration for pruning receipts not associated with logs emitted by the specified contracts. -#[derive(Debug, Clone, PartialEq, Eq, Default)] -#[cfg_attr(any(test, feature = "serde"), derive(serde::Serialize, serde::Deserialize))] -pub struct ReceiptsLogPruneConfig(pub BTreeMap); - -impl ReceiptsLogPruneConfig { - /// Checks if the configuration is empty - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - /// Given the `tip` block number, consolidates the structure so it can easily be queried for - /// filtering across a range of blocks. - /// - /// Example: - /// - /// `{ addrA: Before(872), addrB: Before(500), addrC: Distance(128) }` - /// - /// for `tip: 1000`, gets transformed to a map such as: - /// - /// `{ 500: [addrB], 872: [addrA, addrC] }` - /// - /// The [`BlockNumber`] key of the new map should be viewed as `PruneMode::Before(block)`, which - /// makes the previous result equivalent to - /// - /// `{ Before(500): [addrB], Before(872): [addrA, addrC] }` - pub fn group_by_block( - &self, - tip: BlockNumber, - pruned_block: Option, - ) -> Result>, PruneSegmentError> { - let mut map = BTreeMap::new(); - let base_block = pruned_block.unwrap_or_default() + 1; - - for (address, mode) in &self.0 { - // Getting `None`, means that there is nothing to prune yet, so we need it to include in - // the BTreeMap (block = 0), otherwise it will be excluded. - // Reminder that this BTreeMap works as an inclusion list that excludes (prunes) all - // other receipts. - // - // Reminder, that we increment because the [`BlockNumber`] key of the new map should be - // viewed as `PruneMode::Before(block)` - let block = base_block.max( - mode.prune_target_block(tip, PruneSegment::ContractLogs, PrunePurpose::User)? - .map(|(block, _)| block) - .unwrap_or_default() + - 1, - ); - - map.entry(block).or_insert_with(Vec::new).push(address) - } - Ok(map) - } - - /// Returns the lowest block where we start filtering logs which use `PruneMode::Distance(_)`. - pub fn lowest_block_with_distance( - &self, - tip: BlockNumber, - pruned_block: Option, - ) -> Result, PruneSegmentError> { - let pruned_block = pruned_block.unwrap_or_default(); - let mut lowest = None; - - for mode in self.values() { - if mode.is_distance() && - let Some((block, _)) = - mode.prune_target_block(tip, PruneSegment::ContractLogs, PrunePurpose::User)? - { - lowest = Some(lowest.unwrap_or(u64::MAX).min(block)); - } - } - - Ok(lowest.map(|lowest| lowest.max(pruned_block))) - } -} - -impl Deref for ReceiptsLogPruneConfig { - type Target = BTreeMap; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_group_by_block_empty_config() { - let config = ReceiptsLogPruneConfig(BTreeMap::new()); - let tip = 1000; - let pruned_block = None; - - let result = config.group_by_block(tip, pruned_block).unwrap(); - assert!(result.is_empty(), "The result should be empty when the config is empty"); - } - - #[test] - fn test_group_by_block_single_entry() { - let mut config_map = BTreeMap::new(); - let address = Address::new([1; 20]); - let prune_mode = PruneMode::Before(500); - config_map.insert(address, prune_mode); - - let config = ReceiptsLogPruneConfig(config_map); - // Big tip to have something to prune for the target block - let tip = 3000000; - let pruned_block = Some(400); - - let result = config.group_by_block(tip, pruned_block).unwrap(); - - // Expect one entry with block 500 and the corresponding address - assert_eq!(result.len(), 1); - assert_eq!(result[&500], vec![&address], "Address should be grouped under block 500"); - - // Tip smaller than the target block, so that we have nothing to prune for the block - let tip = 300; - let pruned_block = Some(400); - - let result = config.group_by_block(tip, pruned_block).unwrap(); - - // Expect one entry with block 400 and the corresponding address - assert_eq!(result.len(), 1); - assert_eq!(result[&401], vec![&address], "Address should be grouped under block 400"); - } - - #[test] - fn test_group_by_block_multiple_entries() { - let mut config_map = BTreeMap::new(); - let address1 = Address::new([1; 20]); - let address2 = Address::new([2; 20]); - let prune_mode1 = PruneMode::Before(600); - let prune_mode2 = PruneMode::Before(800); - config_map.insert(address1, prune_mode1); - config_map.insert(address2, prune_mode2); - - let config = ReceiptsLogPruneConfig(config_map); - let tip = 900000; - let pruned_block = Some(400); - - let result = config.group_by_block(tip, pruned_block).unwrap(); - - // Expect two entries: one for block 600 and another for block 800 - assert_eq!(result.len(), 2); - assert_eq!(result[&600], vec![&address1], "Address1 should be grouped under block 600"); - assert_eq!(result[&800], vec![&address2], "Address2 should be grouped under block 800"); - } - - #[test] - fn test_group_by_block_with_distance_prune_mode() { - let mut config_map = BTreeMap::new(); - let address = Address::new([1; 20]); - let prune_mode = PruneMode::Distance(100000); - config_map.insert(address, prune_mode); - - let config = ReceiptsLogPruneConfig(config_map); - let tip = 100100; - // Pruned block is smaller than the target block - let pruned_block = Some(50); - - let result = config.group_by_block(tip, pruned_block).unwrap(); - - // Expect the entry to be grouped under block 100 (tip - distance) - assert_eq!(result.len(), 1); - assert_eq!(result[&101], vec![&address], "Address should be grouped under block 100"); - - let tip = 100100; - // Pruned block is larger than the target block - let pruned_block = Some(800); - - let result = config.group_by_block(tip, pruned_block).unwrap(); - - // Expect the entry to be grouped under block 800 which is larger than tip - distance - assert_eq!(result.len(), 1); - assert_eq!(result[&801], vec![&address], "Address should be grouped under block 800"); - } - - #[test] - fn test_lowest_block_with_distance_empty_config() { - let config = ReceiptsLogPruneConfig(BTreeMap::new()); - let tip = 1000; - let pruned_block = None; - - let result = config.lowest_block_with_distance(tip, pruned_block).unwrap(); - assert_eq!(result, None, "The result should be None when the config is empty"); - } - - #[test] - fn test_lowest_block_with_distance_no_distance_mode() { - let mut config_map = BTreeMap::new(); - let address = Address::new([1; 20]); - let prune_mode = PruneMode::Before(500); - config_map.insert(address, prune_mode); - - let config = ReceiptsLogPruneConfig(config_map); - let tip = 1000; - let pruned_block = None; - - let result = config.lowest_block_with_distance(tip, pruned_block).unwrap(); - assert_eq!(result, None, "The result should be None when there are no Distance modes"); - } - - #[test] - fn test_lowest_block_with_distance_single_entry() { - let mut config_map = BTreeMap::new(); - let address = Address::new([1; 20]); - let prune_mode = PruneMode::Distance(100000); - config_map.insert(address, prune_mode); - - let config = ReceiptsLogPruneConfig(config_map); - - let tip = 100100; - let pruned_block = Some(400); - - // Expect the lowest block to be 400 as 400 > 100100 - 100000 (tip - distance) - assert_eq!( - config.lowest_block_with_distance(tip, pruned_block).unwrap(), - Some(400), - "The lowest block should be 400" - ); - - let tip = 100100; - let pruned_block = Some(50); - - // Expect the lowest block to be 100 as 100 > 50 (pruned block) - assert_eq!( - config.lowest_block_with_distance(tip, pruned_block).unwrap(), - Some(100), - "The lowest block should be 100" - ); - } - - #[test] - fn test_lowest_block_with_distance_multiple_entries_last() { - let mut config_map = BTreeMap::new(); - let address1 = Address::new([1; 20]); - let address2 = Address::new([2; 20]); - let prune_mode1 = PruneMode::Distance(100100); - let prune_mode2 = PruneMode::Distance(100300); - config_map.insert(address1, prune_mode1); - config_map.insert(address2, prune_mode2); - - let config = ReceiptsLogPruneConfig(config_map); - let tip = 200300; - let pruned_block = Some(100); - - // The lowest block should be 200300 - 100300 = 100000: - // - First iteration will return 100200 => 200300 - 100100 = 100200 - // - Second iteration will return 100000 => 200300 - 100300 = 100000 < 100200 - // - Final result is 100000 - assert_eq!(config.lowest_block_with_distance(tip, pruned_block).unwrap(), Some(100000)); - } - - #[test] - fn test_lowest_block_with_distance_multiple_entries_first() { - let mut config_map = BTreeMap::new(); - let address1 = Address::new([1; 20]); - let address2 = Address::new([2; 20]); - let prune_mode1 = PruneMode::Distance(100400); - let prune_mode2 = PruneMode::Distance(100300); - config_map.insert(address1, prune_mode1); - config_map.insert(address2, prune_mode2); - - let config = ReceiptsLogPruneConfig(config_map); - let tip = 200300; - let pruned_block = Some(100); - - // The lowest block should be 200300 - 100400 = 99900: - // - First iteration, lowest block is 200300 - 100400 = 99900 - // - Second iteration, lowest block is still 99900 < 200300 - 100300 = 100000 - // - Final result is 99900 - assert_eq!(config.lowest_block_with_distance(tip, pruned_block).unwrap(), Some(99900)); - } - - #[test] - fn test_lowest_block_with_distance_multiple_entries_pruned_block() { - let mut config_map = BTreeMap::new(); - let address1 = Address::new([1; 20]); - let address2 = Address::new([2; 20]); - let prune_mode1 = PruneMode::Distance(100400); - let prune_mode2 = PruneMode::Distance(100300); - config_map.insert(address1, prune_mode1); - config_map.insert(address2, prune_mode2); - - let config = ReceiptsLogPruneConfig(config_map); - let tip = 200300; - let pruned_block = Some(100000); - - // The lowest block should be 100000 because: - // - Lowest is 200300 - 100400 = 99900 < 200300 - 100300 = 100000 - // - Lowest is compared to the pruned block 100000: 100000 > 99900 - // - Finally the lowest block is 100000 - assert_eq!(config.lowest_block_with_distance(tip, pruned_block).unwrap(), Some(100000)); - } -} diff --git a/crates/prune/types/src/segment.rs b/crates/prune/types/src/segment.rs index 0d60d90013..c5cbecd4cc 100644 --- a/crates/prune/types/src/segment.rs +++ b/crates/prune/types/src/segment.rs @@ -24,8 +24,7 @@ pub enum PruneSegment { AccountHistory, /// Prune segment responsible for the `StorageChangeSets` and `StoragesHistory` tables. StorageHistory, - /// Prune segment responsible for the `CanonicalHeaders`, `Headers` and - /// `HeaderTerminalDifficulties` tables. + /// Prune segment responsible for the `CanonicalHeaders`, `Headers` tables. Headers, /// Prune segment responsible for the `Transactions` table. Transactions, diff --git a/crates/prune/types/src/target.rs b/crates/prune/types/src/target.rs index 657cf6a37c..bb61c006cd 100644 --- a/crates/prune/types/src/target.rs +++ b/crates/prune/types/src/target.rs @@ -2,7 +2,7 @@ use alloy_primitives::BlockNumber; use derive_more::Display; use thiserror::Error; -use crate::{PruneCheckpoint, PruneMode, PruneSegment, ReceiptsLogPruneConfig}; +use crate::{PruneCheckpoint, PruneMode, PruneSegment}; /// Minimum distance from the tip necessary for the node to work correctly: /// 1. Minimum 2 epochs (32 blocks per epoch) required to handle any reorg according to the @@ -99,12 +99,10 @@ pub struct PruneModes { ) )] pub merkle_changesets: PruneMode, - /// Receipts pruning configuration by retaining only those receipts that contain logs emitted - /// by the specified addresses, discarding others. This setting is overridden by `receipts`. - /// - /// The [`BlockNumber`](`crate::BlockNumber`) represents the starting block from which point - /// onwards the receipts are preserved. - pub receipts_log_filter: ReceiptsLogPruneConfig, + /// Receipts log filtering has been deprecated and will be removed in a future release. + #[deprecated] + #[cfg_attr(any(test, feature = "serde"), serde(skip))] + pub receipts_log_filter: (), } impl Default for PruneModes { @@ -117,14 +115,15 @@ impl Default for PruneModes { storage_history: None, bodies_history: None, merkle_changesets: default_merkle_changesets_mode(), - receipts_log_filter: ReceiptsLogPruneConfig::default(), + #[expect(deprecated)] + receipts_log_filter: (), } } } impl PruneModes { /// Sets pruning to all targets. - pub fn all() -> Self { + pub const fn all() -> Self { Self { sender_recovery: Some(PruneMode::Full), transaction_lookup: Some(PruneMode::Full), @@ -133,13 +132,14 @@ impl PruneModes { storage_history: Some(PruneMode::Full), bodies_history: Some(PruneMode::Full), merkle_changesets: PruneMode::Full, - receipts_log_filter: Default::default(), + #[expect(deprecated)] + receipts_log_filter: (), } } /// Returns whether there is any kind of receipt pruning configuration. - pub fn has_receipts_pruning(&self) -> bool { - self.receipts.is_some() || !self.receipts_log_filter.is_empty() + pub const fn has_receipts_pruning(&self) -> bool { + self.receipts.is_some() } /// Returns an error if we can't unwind to the targeted block because the target block is diff --git a/crates/rpc/ipc/src/server/mod.rs b/crates/rpc/ipc/src/server/mod.rs index 6e6b092c40..75431b915a 100644 --- a/crates/rpc/ipc/src/server/mod.rs +++ b/crates/rpc/ipc/src/server/mod.rs @@ -391,7 +391,7 @@ where fn call(&mut self, request: String) -> Self::Future { trace!("{:?}", request); - let cfg = RpcServiceCfg::CallsAndSubscriptions { + let cfg = RpcServiceCfg { bounded_subscriptions: BoundedSubscriptions::new( self.inner.server_cfg.max_subscriptions_per_connection, ), diff --git a/crates/rpc/ipc/src/server/rpc_service.rs b/crates/rpc/ipc/src/server/rpc_service.rs index 75bd53ad6d..f7fcdace4c 100644 --- a/crates/rpc/ipc/src/server/rpc_service.rs +++ b/crates/rpc/ipc/src/server/rpc_service.rs @@ -25,17 +25,11 @@ pub struct RpcService { } /// Configuration of the `RpcService`. -#[allow(dead_code)] #[derive(Clone, Debug)] -pub(crate) enum RpcServiceCfg { - /// The server supports only calls. - OnlyCalls, - /// The server supports both method calls and subscriptions. - CallsAndSubscriptions { - bounded_subscriptions: BoundedSubscriptions, - sink: MethodSink, - id_provider: Arc, - }, +pub(crate) struct RpcServiceCfg { + pub(crate) bounded_subscriptions: BoundedSubscriptions, + pub(crate) sink: MethodSink, + pub(crate) id_provider: Arc, } impl RpcService { @@ -82,30 +76,20 @@ impl RpcServiceT for RpcService { ResponseFuture::future(fut) } MethodCallback::Subscription(callback) => { - let RpcServiceCfg::CallsAndSubscriptions { - bounded_subscriptions, - sink, - id_provider, - } = &self.cfg - else { - tracing::warn!(id = ?id, method = %name, "Attempted subscription on a service not configured for subscriptions."); - let rp = - MethodResponse::error(id, ErrorObject::from(ErrorCode::InternalError)); - return ResponseFuture::ready(rp); - }; + let cfg = &self.cfg; - if let Some(p) = bounded_subscriptions.acquire() { + if let Some(p) = cfg.bounded_subscriptions.acquire() { let conn_state = SubscriptionState { conn_id, - id_provider: &**id_provider, + id_provider: &*cfg.id_provider, subscription_permit: p, }; let fut = - callback(id.clone(), params, sink.clone(), conn_state, extensions); + callback(id.clone(), params, cfg.sink.clone(), conn_state, extensions); ResponseFuture::future(fut) } else { - let max = bounded_subscriptions.max(); + let max = cfg.bounded_subscriptions.max(); let rp = MethodResponse::error(id, reject_too_many_subscriptions(max)); ResponseFuture::ready(rp) } @@ -114,13 +98,6 @@ impl RpcServiceT for RpcService { // Don't adhere to any resource or subscription limits; always let unsubscribing // happen! - let RpcServiceCfg::CallsAndSubscriptions { .. } = self.cfg else { - tracing::warn!(id = ?id, method = %name, "Attempted unsubscription on a service not configured for subscriptions."); - let rp = - MethodResponse::error(id, ErrorObject::from(ErrorCode::InternalError)); - return ResponseFuture::ready(rp); - }; - let rp = callback(id, params, conn_id, max_response_body_size, extensions); ResponseFuture::ready(rp) } diff --git a/crates/rpc/rpc-builder/src/auth.rs b/crates/rpc/rpc-builder/src/auth.rs index 777081a7e6..0d0a6165ff 100644 --- a/crates/rpc/rpc-builder/src/auth.rs +++ b/crates/rpc/rpc-builder/src/auth.rs @@ -354,7 +354,9 @@ impl AuthServerHandle { /// Returns a http client connected to the server. /// /// This client uses the JWT token to authenticate requests. - pub fn http_client(&self) -> impl SubscriptionClientT + Clone + Send + Sync + Unpin + 'static { + pub fn http_client( + &self, + ) -> impl SubscriptionClientT + use<> + Clone + Send + Sync + Unpin + 'static { // Create a middleware that adds a new JWT token to every request. let secret_layer = AuthClientLayer::new(self.secret); let middleware = tower::ServiceBuilder::default().layer(secret_layer); diff --git a/crates/rpc/rpc-engine-api/src/error.rs b/crates/rpc/rpc-engine-api/src/error.rs index 2578b2f44e..6155c004c3 100644 --- a/crates/rpc/rpc-engine-api/src/error.rs +++ b/crates/rpc/rpc-engine-api/src/error.rs @@ -1,4 +1,4 @@ -use alloy_primitives::{B256, U256}; +use alloy_primitives::B256; use alloy_rpc_types_engine::{ ForkchoiceUpdateError, INVALID_FORK_CHOICE_STATE_ERROR, INVALID_FORK_CHOICE_STATE_ERROR_MSG, INVALID_PAYLOAD_ATTRIBUTES_ERROR, INVALID_PAYLOAD_ATTRIBUTES_ERROR_MSG, @@ -59,17 +59,6 @@ pub enum EngineApiError { /// Requested number of items count: u64, }, - /// Terminal total difficulty mismatch during transition configuration exchange. - #[error( - "invalid transition terminal total difficulty: \ - execution: {execution}, consensus: {consensus}" - )] - TerminalTD { - /// Execution terminal total difficulty value. - execution: U256, - /// Consensus terminal total difficulty value. - consensus: U256, - }, /// Terminal block hash mismatch during transition configuration exchange. #[error( "invalid transition terminal block hash: \ @@ -202,7 +191,6 @@ impl From for jsonrpsee_types::error::ErrorObject<'static> { } }, // Any other server error - EngineApiError::TerminalTD { .. } | EngineApiError::TerminalBlockHash { .. } | EngineApiError::NewPayload(_) | EngineApiError::Internal(_) | diff --git a/crates/rpc/rpc-eth-types/src/error/mod.rs b/crates/rpc/rpc-eth-types/src/error/mod.rs index fdb5f8f190..ef65e4ccc2 100644 --- a/crates/rpc/rpc-eth-types/src/error/mod.rs +++ b/crates/rpc/rpc-eth-types/src/error/mod.rs @@ -462,7 +462,6 @@ impl From for EthApiError { } ProviderError::BestBlockNotFound => Self::HeaderNotFound(BlockId::latest()), ProviderError::BlockNumberForTransactionIndexNotFound => Self::UnknownBlockOrTxIndex, - ProviderError::TotalDifficultyNotFound(num) => Self::HeaderNotFound(num.into()), ProviderError::FinalizedBlockNotFound => Self::HeaderNotFound(BlockId::finalized()), ProviderError::SafeBlockNotFound => Self::HeaderNotFound(BlockId::safe()), err => Self::Internal(err.into()), @@ -931,8 +930,13 @@ pub enum RpcPoolError { #[error("negative value")] NegativeValue, /// When oversized data is encountered - #[error("oversized data")] - OversizedData, + #[error("oversized data: transaction size {size}, limit {limit}")] + OversizedData { + /// Size of the transaction/input data that exceeded the limit. + size: usize, + /// Configured limit that was exceeded. + limit: usize, + }, /// When the max initcode size is exceeded #[error("max initcode size exceeded")] ExceedsMaxInitCodeSize, @@ -974,7 +978,7 @@ impl From for jsonrpsee_types::error::ErrorObject<'static> { RpcPoolError::MaxTxGasLimitExceeded | RpcPoolError::ExceedsFeeCap { .. } | RpcPoolError::NegativeValue | - RpcPoolError::OversizedData | + RpcPoolError::OversizedData { .. } | RpcPoolError::ExceedsMaxInitCodeSize | RpcPoolError::PoolTransactionError(_) | RpcPoolError::Eip4844(_) | @@ -1018,7 +1022,9 @@ impl From for RpcPoolError { InvalidPoolTransactionError::IntrinsicGasTooLow => { Self::Invalid(RpcInvalidTransactionError::GasTooLow) } - InvalidPoolTransactionError::OversizedData(_, _) => Self::OversizedData, + InvalidPoolTransactionError::OversizedData { size, limit } => { + Self::OversizedData { size, limit } + } InvalidPoolTransactionError::Underpriced => Self::Underpriced, InvalidPoolTransactionError::Eip2681 => { Self::Invalid(RpcInvalidTransactionError::NonceMaxValue) diff --git a/crates/stages/api/src/pipeline/mod.rs b/crates/stages/api/src/pipeline/mod.rs index ac35a48903..e8542c36da 100644 --- a/crates/stages/api/src/pipeline/mod.rs +++ b/crates/stages/api/src/pipeline/mod.rs @@ -9,7 +9,7 @@ use reth_primitives_traits::constants::BEACON_CONSENSUS_REORG_UNWIND_DEPTH; use reth_provider::{ providers::ProviderNodeTypes, BlockHashReader, BlockNumReader, ChainStateBlockReader, ChainStateBlockWriter, DBProvider, DatabaseProviderFactory, ProviderFactory, - PruneCheckpointReader, StageCheckpointReader, StageCheckpointWriter, StaticFileProviderFactory, + PruneCheckpointReader, StageCheckpointReader, StageCheckpointWriter, }; use reth_prune::PrunerBuilder; use reth_static_file::StaticFileProducer; @@ -31,7 +31,7 @@ use crate::{ }; pub use builder::*; use progress::*; -use reth_errors::{ProviderResult, RethResult}; +use reth_errors::RethResult; pub use set::*; /// A container for a queued stage. @@ -101,6 +101,12 @@ impl Pipeline { PipelineBuilder::default() } + /// Return the minimum block number achieved by + /// any stage during the execution of the pipeline. + pub const fn minimum_block_number(&self) -> Option { + self.progress.minimum_block_number + } + /// Set tip for reverse sync. #[track_caller] pub fn set_tip(&self, tip: B256) { @@ -121,7 +127,9 @@ impl Pipeline { ) -> &mut dyn Stage< as DatabaseProviderFactory>::ProviderRW> { &mut self.stages[idx] } +} +impl Pipeline { /// Registers progress metrics for each registered stage pub fn register_metrics(&mut self) -> Result<(), PipelineError> { let Some(metrics_tx) = &mut self.metrics_tx else { return Ok(()) }; @@ -282,81 +290,6 @@ impl Pipeline { Ok(()) } - /// Check if the pipeline is consistent (all stages have the checkpoint block numbers no less - /// than the checkpoint of the first stage). - /// - /// This will return the pipeline target if: - /// * the pipeline was interrupted during its previous run - /// * a new stage was added - /// * stage data was dropped manually through `reth stage drop ...` - /// - /// # Returns - /// - /// A target block hash if the pipeline is inconsistent, otherwise `None`. - pub fn initial_backfill_target(&self) -> ProviderResult> { - let provider = self.provider_factory.provider()?; - - // If no target was provided, check if the stages are congruent - check if the - // checkpoint of the last stage matches the checkpoint of the first. - let first_stage_checkpoint = provider - .get_stage_checkpoint(self.stages.first().unwrap().id())? - .unwrap_or_default() - .block_number; - - // Skip the first stage as we've already retrieved it and comparing all other checkpoints - // against it. - for stage in self.stages.iter().skip(1) { - let stage_id = stage.id(); - - let stage_checkpoint = - provider.get_stage_checkpoint(stage_id)?.unwrap_or_default().block_number; - - // If the checkpoint of any stage is less than the checkpoint of the first stage, - // retrieve and return the block hash of the latest header and use it as the target. - if stage_checkpoint < first_stage_checkpoint { - debug!( - target: "consensus::engine", - first_stage_checkpoint, - inconsistent_stage_id = %stage_id, - inconsistent_stage_checkpoint = stage_checkpoint, - "Pipeline sync progress is inconsistent" - ); - return provider.block_hash(first_stage_checkpoint); - } - } - - Ok(None) - } - - /// Checks for consistency between database and static files. If it fails, it unwinds to - /// the first block that's consistent between database and static files. - pub async fn ensure_static_files_consistency(&mut self) -> Result<(), PipelineError> { - let maybe_unwind_target = self - .provider_factory - .static_file_provider() - .check_consistency(&self.provider_factory.provider()?)?; - - self.move_to_static_files()?; - - if let Some(unwind_target) = maybe_unwind_target { - // Highly unlikely to happen, and given its destructive nature, it's better to panic - // instead. - assert_ne!( - unwind_target, - 0, - "A static file <> database inconsistency was found that would trigger an unwind to block 0" - ); - - info!(target: "reth::cli", unwind_target = %unwind_target, "Executing an unwind after a failed storage consistency check."); - - self.unwind(unwind_target, None).inspect_err(|err| { - error!(target: "reth::cli", unwind_target = %unwind_target, %err, "failed to run unwind") - })?; - } - - Ok(()) - } - /// Unwind the stages to the target block (exclusive). /// /// If the unwind is due to a bad block the number of that block should be specified. diff --git a/crates/stages/stages/benches/setup/mod.rs b/crates/stages/stages/benches/setup/mod.rs index 01d7571e0d..b6010dd6f3 100644 --- a/crates/stages/stages/benches/setup/mod.rs +++ b/crates/stages/stages/benches/setup/mod.rs @@ -1,12 +1,7 @@ #![expect(unreachable_pub)] -use alloy_primitives::{Address, B256, U256}; +use alloy_primitives::{Address, B256}; use itertools::concat; use reth_db::{test_utils::TempDatabase, Database, DatabaseEnv}; -use reth_db_api::{ - cursor::DbCursorRO, - tables, - transaction::{DbTx, DbTxMut}, -}; use reth_primitives_traits::{Account, SealedBlock, SealedHeader}; use reth_provider::{ test_utils::MockNodeTypesWithDB, DBProvider, DatabaseProvider, DatabaseProviderFactory, @@ -198,13 +193,6 @@ pub(crate) fn txs_testdata(num_blocks: u64) -> TestStageDB { ); db.insert_blocks(blocks.iter(), StorageKind::Static).unwrap(); - - // initialize TD - db.commit(|tx| { - let (head, _) = tx.cursor_read::()?.first()?.unwrap_or_default(); - Ok(tx.put::(head, U256::from(0).into())?) - }) - .unwrap(); } db diff --git a/crates/stages/stages/src/stages/bodies.rs b/crates/stages/stages/src/stages/bodies.rs index d1386dded4..7b6090ca86 100644 --- a/crates/stages/stages/src/stages/bodies.rs +++ b/crates/stages/stages/src/stages/bodies.rs @@ -580,7 +580,7 @@ mod tests { ..Default::default() }, ); - self.db.insert_headers_with_td(blocks.iter().map(|block| block.sealed_header()))?; + self.db.insert_headers(blocks.iter().map(|block| block.sealed_header()))?; if let Some(progress) = blocks.get(start as usize) { // Insert last progress data { diff --git a/crates/stages/stages/src/stages/era.rs b/crates/stages/stages/src/stages/era.rs index 971bc11f89..10598f9011 100644 --- a/crates/stages/stages/src/stages/era.rs +++ b/crates/stages/stages/src/stages/era.rs @@ -10,12 +10,11 @@ use reth_era_utils as era; use reth_etl::Collector; use reth_primitives_traits::{FullBlockBody, FullBlockHeader, NodePrimitives}; use reth_provider::{ - BlockReader, BlockWriter, DBProvider, HeaderProvider, StageCheckpointWriter, - StaticFileProviderFactory, StaticFileWriter, + BlockReader, BlockWriter, DBProvider, StageCheckpointWriter, StaticFileProviderFactory, + StaticFileWriter, }; use reth_stages_api::{ExecInput, ExecOutput, Stage, StageError, UnwindInput, UnwindOutput}; use reth_static_file_types::StaticFileSegment; -use reth_storage_errors::ProviderError; use std::{ fmt::{Debug, Formatter}, iter, @@ -176,11 +175,6 @@ where .get_highest_static_file_block(StaticFileSegment::Headers) .unwrap_or_default(); - // Find the latest total difficulty - let mut td = static_file_provider - .header_td_by_number(last_header_number)? - .ok_or(ProviderError::TotalDifficultyNotFound(last_header_number))?; - // Although headers were downloaded in reverse order, the collector iterates it in // ascending order let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?; @@ -190,7 +184,6 @@ where &mut writer, provider, &mut self.hash_collector, - &mut td, last_header_number..=input.target(), ) .map_err(|e| StageError::Fatal(e.into()))?; @@ -336,7 +329,7 @@ mod tests { }; use reth_ethereum_primitives::TransactionSigned; use reth_primitives_traits::{SealedBlock, SealedHeader}; - use reth_provider::{BlockNumReader, TransactionsProvider}; + use reth_provider::{BlockNumReader, HeaderProvider, TransactionsProvider}; use reth_testing_utils::generators::{ random_block_range, random_signed_tx, BlockRangeParams, }; @@ -391,7 +384,7 @@ mod tests { ..Default::default() }, ); - self.db.insert_headers_with_td(blocks.iter().map(|block| block.sealed_header()))?; + self.db.insert_headers(blocks.iter().map(|block| block.sealed_header()))?; if let Some(progress) = blocks.get(start as usize) { // Insert last progress data { @@ -447,9 +440,6 @@ mod tests { match output { Some(output) if output.checkpoint.block_number > initial_checkpoint => { let provider = self.db.factory.provider()?; - let mut td = provider - .header_td_by_number(initial_checkpoint.saturating_sub(1))? - .unwrap_or_default(); for block_num in initial_checkpoint.. output @@ -469,10 +459,6 @@ mod tests { assert!(header.is_some()); let header = SealedHeader::seal_slow(header.unwrap()); assert_eq!(header.hash(), hash); - - // validate the header total difficulty - td += header.difficulty; - assert_eq!(provider.header_td_by_number(block_num)?, Some(td)); } self.validate_db_blocks( @@ -513,10 +499,6 @@ mod tests { .ensure_no_entry_above_by_value::(block, |val| val)?; self.db.ensure_no_entry_above::(block, |key| key)?; self.db.ensure_no_entry_above::(block, |key| key)?; - self.db.ensure_no_entry_above::( - block, - |num| num, - )?; Ok(()) } diff --git a/crates/stages/stages/src/stages/execution.rs b/crates/stages/stages/src/stages/execution.rs index ed50572d58..adfc87c5cc 100644 --- a/crates/stages/stages/src/stages/execution.rs +++ b/crates/stages/stages/src/stages/execution.rs @@ -39,7 +39,6 @@ use super::missing_static_data_error; /// Input tables: /// - [`tables::CanonicalHeaders`] get next block to execute. /// - [`tables::Headers`] get for revm environment variables. -/// - [`tables::HeaderTerminalDifficulties`] /// - [`tables::BlockBodyIndices`] to get tx number /// - [`tables::Transactions`] to execute /// @@ -661,7 +660,7 @@ where mod tests { use super::*; use crate::{stages::MERKLE_STAGE_DEFAULT_REBUILD_THRESHOLD, test_utils::TestStageDB}; - use alloy_primitives::{address, hex_literal::hex, keccak256, Address, B256, U256}; + use alloy_primitives::{address, hex_literal::hex, keccak256, B256, U256}; use alloy_rlp::Decodable; use assert_matches::assert_matches; use reth_chainspec::ChainSpecBuilder; @@ -678,9 +677,7 @@ mod tests { DatabaseProviderFactory, ReceiptProvider, StaticFileProviderFactory, }; use reth_prune::PruneModes; - use reth_prune_types::{PruneMode, ReceiptsLogPruneConfig}; use reth_stages_api::StageUnitCheckpoint; - use std::collections::BTreeMap; fn stage() -> ExecutionStage { let evm_config = @@ -897,20 +894,11 @@ mod tests { // If there is a pruning configuration, then it's forced to use the database. // This way we test both cases. let modes = [None, Some(PruneModes::default())]; - let random_filter = ReceiptsLogPruneConfig(BTreeMap::from([( - Address::random(), - PruneMode::Distance(100000), - )])); // Tests node with database and node with static files - for mut mode in modes { + for mode in modes { let mut provider = factory.database_provider_rw().unwrap(); - if let Some(mode) = &mut mode { - // Simulating a full node where we write receipts to database - mode.receipts_log_filter = random_filter.clone(); - } - let mut execution_stage = stage(); provider.set_prune_modes(mode.clone().unwrap_or_default()); @@ -1034,18 +1022,9 @@ mod tests { // If there is a pruning configuration, then it's forced to use the database. // This way we test both cases. let modes = [None, Some(PruneModes::default())]; - let random_filter = ReceiptsLogPruneConfig(BTreeMap::from([( - Address::random(), - PruneMode::Before(100000), - )])); // Tests node with database and node with static files - for mut mode in modes { - if let Some(mode) = &mut mode { - // Simulating a full node where we write receipts to database - mode.receipts_log_filter = random_filter.clone(); - } - + for mode in modes { // Test Execution let mut execution_stage = stage(); provider.set_prune_modes(mode.clone().unwrap_or_default()); diff --git a/crates/stages/stages/src/stages/finish.rs b/crates/stages/stages/src/stages/finish.rs index 1b9e624b41..8d676c35b9 100644 --- a/crates/stages/stages/src/stages/finish.rs +++ b/crates/stages/stages/src/stages/finish.rs @@ -72,7 +72,7 @@ mod tests { let start = input.checkpoint().block_number; let mut rng = generators::rng(); let head = random_header(&mut rng, start, None); - self.db.insert_headers_with_td(std::iter::once(&head))?; + self.db.insert_headers(std::iter::once(&head))?; // use previous progress as seed size let end = input.target.unwrap_or_default() + 1; @@ -82,7 +82,7 @@ mod tests { } let mut headers = random_header_range(&mut rng, start + 1..end, head.hash()); - self.db.insert_headers_with_td(headers.iter())?; + self.db.insert_headers(headers.iter())?; headers.insert(0, head); Ok(headers) } diff --git a/crates/stages/stages/src/stages/hashing_account.rs b/crates/stages/stages/src/stages/hashing_account.rs index cc86db14d3..1e48f2d38e 100644 --- a/crates/stages/stages/src/stages/hashing_account.rs +++ b/crates/stages/stages/src/stages/hashing_account.rs @@ -64,7 +64,7 @@ impl AccountHashingStage { opts: SeedOpts, ) -> Result, StageError> where - N::Primitives: reth_primitives_traits::FullNodePrimitives< + N::Primitives: reth_primitives_traits::NodePrimitives< Block = reth_ethereum_primitives::Block, BlockHeader = reth_primitives_traits::Header, >, diff --git a/crates/stages/stages/src/stages/headers.rs b/crates/stages/stages/src/stages/headers.rs index d3e690dc51..8ad39be5eb 100644 --- a/crates/stages/stages/src/stages/headers.rs +++ b/crates/stages/stages/src/stages/headers.rs @@ -16,15 +16,14 @@ use reth_network_p2p::headers::{ }; use reth_primitives_traits::{serde_bincode_compat, FullBlockHeader, NodePrimitives, SealedHeader}; use reth_provider::{ - providers::StaticFileWriter, BlockHashReader, DBProvider, HeaderProvider, - HeaderSyncGapProvider, StaticFileProviderFactory, + providers::StaticFileWriter, BlockHashReader, DBProvider, HeaderSyncGapProvider, + StaticFileProviderFactory, }; use reth_stages_api::{ CheckpointBlockRange, EntitiesCheckpoint, ExecInput, ExecOutput, HeadersCheckpoint, Stage, StageCheckpoint, StageError, StageId, UnwindInput, UnwindOutput, }; use reth_static_file_types::StaticFileSegment; -use reth_storage_errors::provider::ProviderError; use std::task::{ready, Context, Poll}; use tokio::sync::watch; @@ -107,11 +106,6 @@ where .get_highest_static_file_block(StaticFileSegment::Headers) .unwrap_or_default(); - // Find the latest total difficulty - let mut td = static_file_provider - .header_td_by_number(last_header_number)? - .ok_or(ProviderError::TotalDifficultyNotFound(last_header_number))?; - // Although headers were downloaded in reverse order, the collector iterates it in ascending // order let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?; @@ -134,11 +128,8 @@ where } last_header_number = header.number(); - // Increase total difficulty - td += header.difficulty(); - // Append to Headers segment - writer.append_header(header, td, header_hash)?; + writer.append_header(header, header_hash)?; } info!(target: "sync::stages::headers", total = total_headers, "Writing headers hash index"); @@ -342,9 +333,6 @@ where (input.unwind_to + 1).., )?; provider.tx_ref().unwind_table_by_num::(input.unwind_to)?; - provider - .tx_ref() - .unwind_table_by_num::(input.unwind_to)?; let unfinalized_headers_unwound = provider.tx_ref().unwind_table_by_num::(input.unwind_to)?; @@ -415,7 +403,7 @@ mod tests { ReverseHeadersDownloader, ReverseHeadersDownloaderBuilder, }; use reth_network_p2p::test_utils::{TestHeaderDownloader, TestHeadersClient}; - use reth_provider::{test_utils::MockNodeTypesWithDB, BlockNumReader}; + use reth_provider::{test_utils::MockNodeTypesWithDB, BlockNumReader, HeaderProvider}; use tokio::sync::watch; pub(crate) struct HeadersTestRunner { @@ -469,7 +457,7 @@ mod tests { let start = input.checkpoint().block_number; let headers = random_header_range(&mut rng, 0..start + 1, B256::ZERO); let head = headers.last().cloned().unwrap(); - self.db.insert_headers_with_td(headers.iter())?; + self.db.insert_headers(headers.iter())?; // use previous checkpoint as seed size let end = input.target.unwrap_or_default() + 1; @@ -493,9 +481,6 @@ mod tests { match output { Some(output) if output.checkpoint.block_number > initial_checkpoint => { let provider = self.db.factory.provider()?; - let mut td = provider - .header_td_by_number(initial_checkpoint.saturating_sub(1))? - .unwrap_or_default(); for block_num in initial_checkpoint..output.checkpoint.block_number { // look up the header hash @@ -509,10 +494,6 @@ mod tests { assert!(header.is_some()); let header = SealedHeader::seal_slow(header.unwrap()); assert_eq!(header.hash(), hash); - - // validate the header total difficulty - td += header.difficulty; - assert_eq!(provider.header_td_by_number(block_num)?, Some(td)); } } _ => self.check_no_header_entry_above(initial_checkpoint)?, @@ -567,10 +548,6 @@ mod tests { .ensure_no_entry_above_by_value::(block, |val| val)?; self.db.ensure_no_entry_above::(block, |key| key)?; self.db.ensure_no_entry_above::(block, |key| key)?; - self.db.ensure_no_entry_above::( - block, - |num| num, - )?; Ok(()) } @@ -635,16 +612,7 @@ mod tests { let static_file_provider = provider.static_file_provider(); let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); for header in sealed_headers { - let ttd = if header.number() == 0 { - header.difficulty() - } else { - let parent_block_number = header.number() - 1; - let parent_ttd = - provider.header_td_by_number(parent_block_number).unwrap().unwrap_or_default(); - parent_ttd + header.difficulty() - }; - - writer.append_header(header.header(), ttd, &header.hash()).unwrap(); + writer.append_header(header.header(), &header.hash()).unwrap(); } drop(writer); diff --git a/crates/stages/stages/src/stages/merkle.rs b/crates/stages/stages/src/stages/merkle.rs index b4f24db7c5..a3a3ac8848 100644 --- a/crates/stages/stages/src/stages/merkle.rs +++ b/crates/stages/stages/src/stages/merkle.rs @@ -738,7 +738,7 @@ mod tests { let hash = last_header.hash_slow(); writer.prune_headers(1).unwrap(); writer.commit().unwrap(); - writer.append_header(&last_header, U256::ZERO, &hash).unwrap(); + writer.append_header(&last_header, &hash).unwrap(); writer.commit().unwrap(); Ok(blocks) diff --git a/crates/stages/stages/src/stages/mod.rs b/crates/stages/stages/src/stages/mod.rs index 40c4cb9136..58fa7cfb32 100644 --- a/crates/stages/stages/src/stages/mod.rs +++ b/crates/stages/stages/src/stages/mod.rs @@ -75,7 +75,9 @@ mod tests { StaticFileProviderFactory, StorageReader, }; use reth_prune_types::{PruneMode, PruneModes}; - use reth_stages_api::{ExecInput, ExecutionStageThresholds, Stage, StageCheckpoint, StageId}; + use reth_stages_api::{ + ExecInput, ExecutionStageThresholds, PipelineTarget, Stage, StageCheckpoint, StageId, + }; use reth_static_file_types::StaticFileSegment; use reth_testing_utils::generators::{ self, random_block, random_block_range, random_receipt, BlockRangeParams, @@ -302,7 +304,7 @@ mod tests { prune_count: usize, segment: StaticFileSegment, is_full_node: bool, - expected: Option, + expected: Option, ) { // We recreate the static file provider, since consistency heals are done on fetching the // writer for the first time. @@ -324,18 +326,11 @@ mod tests { // We recreate the static file provider, since consistency heals are done on fetching the // writer for the first time. - let mut provider = db.factory.database_provider_ro().unwrap(); - if is_full_node { - provider.set_prune_modes(PruneModes { - receipts: Some(PruneMode::Full), - ..Default::default() - }); - } let mut static_file_provider = db.factory.static_file_provider(); static_file_provider = StaticFileProvider::read_write(static_file_provider.path()).unwrap(); assert!(matches!( static_file_provider - .check_consistency(&provider), + .check_consistency(&db.factory.database_provider_ro().unwrap(), is_full_node,), Ok(e) if e == expected )); } @@ -346,7 +341,7 @@ mod tests { db: &TestStageDB, stage_id: StageId, checkpoint_block_number: BlockNumber, - expected: Option, + expected: Option, ) { let provider_rw = db.factory.provider_rw().unwrap(); provider_rw @@ -357,15 +352,18 @@ mod tests { assert!(matches!( db.factory .static_file_provider() - .check_consistency(&db.factory.database_provider_ro().unwrap()), + .check_consistency(&db.factory.database_provider_ro().unwrap(), false,), Ok(e) if e == expected )); } /// Inserts a dummy value at key and compare the check consistency result against the expected /// one. - fn update_db_and_check>(db: &TestStageDB, key: u64, expected: Option) - where + fn update_db_and_check>( + db: &TestStageDB, + key: u64, + expected: Option, + ) where ::Value: Default, { update_db_with_and_check::(db, key, expected, &Default::default()); @@ -376,7 +374,7 @@ mod tests { fn update_db_with_and_check>( db: &TestStageDB, key: u64, - expected: Option, + expected: Option, value: &T::Value, ) { let provider_rw = db.factory.provider_rw().unwrap(); @@ -387,7 +385,7 @@ mod tests { assert!(matches!( db.factory .static_file_provider() - .check_consistency(&db.factory.database_provider_ro().unwrap()), + .check_consistency(&db.factory.database_provider_ro().unwrap(), false), Ok(e) if e == expected )); } @@ -398,7 +396,7 @@ mod tests { let db_provider = db.factory.database_provider_ro().unwrap(); assert!(matches!( - db.factory.static_file_provider().check_consistency(&db_provider), + db.factory.static_file_provider().check_consistency(&db_provider, false), Ok(None) )); } @@ -420,7 +418,7 @@ mod tests { 1, StaticFileSegment::Receipts, archive_node, - Some(88), + Some(PipelineTarget::Unwind(88)), ); simulate_behind_checkpoint_corruption( @@ -428,7 +426,7 @@ mod tests { 3, StaticFileSegment::Headers, archive_node, - Some(86), + Some(PipelineTarget::Unwind(86)), ); } @@ -477,7 +475,7 @@ mod tests { ); // When a checkpoint is ahead, we request a pipeline unwind. - save_checkpoint_and_check(&db, StageId::Headers, 91, Some(block)); + save_checkpoint_and_check(&db, StageId::Headers, 91, Some(PipelineTarget::Unwind(block))); } #[test] @@ -490,7 +488,7 @@ mod tests { .unwrap(); // Creates a gap of one header: static_file db - update_db_and_check::(&db, current + 2, Some(89)); + update_db_and_check::(&db, current + 2, Some(PipelineTarget::Unwind(89))); // Fill the gap, and ensure no unwind is necessary. update_db_and_check::(&db, current + 1, None); @@ -509,7 +507,7 @@ mod tests { update_db_with_and_check::( &db, current + 2, - Some(89), + Some(PipelineTarget::Unwind(89)), &TxLegacy::default().into_signed(Signature::test_signature()).into(), ); @@ -532,7 +530,7 @@ mod tests { .unwrap(); // Creates a gap of one receipt: static_file db - update_db_and_check::(&db, current + 2, Some(89)); + update_db_and_check::(&db, current + 2, Some(PipelineTarget::Unwind(89))); // Fill the gap, and ensure no unwind is necessary. update_db_and_check::(&db, current + 1, None); diff --git a/crates/stages/stages/src/test_utils/test_db.rs b/crates/stages/stages/src/test_utils/test_db.rs index f38f77b224..3fe1c7f1f9 100644 --- a/crates/stages/stages/src/test_utils/test_db.rs +++ b/crates/stages/stages/src/test_utils/test_db.rs @@ -1,4 +1,4 @@ -use alloy_primitives::{keccak256, Address, BlockNumber, TxHash, TxNumber, B256, U256}; +use alloy_primitives::{keccak256, Address, BlockNumber, TxHash, TxNumber, B256}; use reth_chainspec::MAINNET; use reth_db::{ test_utils::{create_test_rw_db, create_test_rw_db_with_path, create_test_static_files_dir}, @@ -150,7 +150,6 @@ impl TestStageDB { writer: Option<&mut StaticFileProviderRWRefMut<'_, EthPrimitives>>, tx: &TX, header: &SealedHeader, - td: U256, ) -> ProviderResult<()> { if let Some(writer) = writer { // Backfill: some tests start at a forward block number, but static files require no @@ -160,14 +159,13 @@ impl TestStageDB { for block_number in 0..header.number { let mut prev = header.clone_header(); prev.number = block_number; - writer.append_header(&prev, U256::ZERO, &B256::ZERO)?; + writer.append_header(&prev, &B256::ZERO)?; } } - writer.append_header(header.header(), td, &header.hash())?; + writer.append_header(header.header(), &header.hash())?; } else { tx.put::(header.number, header.hash())?; - tx.put::(header.number, td.into())?; tx.put::(header.number, header.header().clone())?; } @@ -175,20 +173,16 @@ impl TestStageDB { Ok(()) } - fn insert_headers_inner<'a, I, const TD: bool>(&self, headers: I) -> ProviderResult<()> + fn insert_headers_inner<'a, I>(&self, headers: I) -> ProviderResult<()> where I: IntoIterator, { let provider = self.factory.static_file_provider(); let mut writer = provider.latest_writer(StaticFileSegment::Headers)?; let tx = self.factory.provider_rw()?.into_tx(); - let mut td = U256::ZERO; for header in headers { - if TD { - td += header.difficulty; - } - Self::insert_header(Some(&mut writer), &tx, header, td)?; + Self::insert_header(Some(&mut writer), &tx, header)?; } writer.commit()?; @@ -203,17 +197,7 @@ impl TestStageDB { where I: IntoIterator, { - self.insert_headers_inner::(headers) - } - - /// Inserts total difficulty of headers into the corresponding static file and tables. - /// - /// Superset functionality of [`TestStageDB::insert_headers`]. - pub fn insert_headers_with_td<'a, I>(&self, headers: I) -> ProviderResult<()> - where - I: IntoIterator, - { - self.insert_headers_inner::(headers) + self.insert_headers_inner::(headers) } /// Insert ordered collection of [`SealedBlock`] into corresponding tables. @@ -240,7 +224,7 @@ impl TestStageDB { .then(|| provider.latest_writer(StaticFileSegment::Headers).unwrap()); blocks.iter().try_for_each(|block| { - Self::insert_header(headers_writer.as_mut(), &tx, block.sealed_header(), U256::ZERO) + Self::insert_header(headers_writer.as_mut(), &tx, block.sealed_header()) })?; if let Some(mut writer) = headers_writer { diff --git a/crates/static-file/static-file/src/segments/headers.rs b/crates/static-file/static-file/src/segments/headers.rs index 5232061caa..990e33ee52 100644 --- a/crates/static-file/static-file/src/segments/headers.rs +++ b/crates/static-file/static-file/src/segments/headers.rs @@ -36,25 +36,17 @@ where )?; let headers_walker = headers_cursor.walk_range(block_range.clone())?; - let mut header_td_cursor = - provider.tx_ref().cursor_read::()?; - let header_td_walker = header_td_cursor.walk_range(block_range.clone())?; - let mut canonical_headers_cursor = provider.tx_ref().cursor_read::()?; let canonical_headers_walker = canonical_headers_cursor.walk_range(block_range)?; - for ((header_entry, header_td_entry), canonical_header_entry) in - headers_walker.zip(header_td_walker).zip(canonical_headers_walker) - { + for (header_entry, canonical_header_entry) in headers_walker.zip(canonical_headers_walker) { let (header_block, header) = header_entry?; - let (header_td_block, header_td) = header_td_entry?; let (canonical_header_block, canonical_header) = canonical_header_entry?; - debug_assert_eq!(header_block, header_td_block); - debug_assert_eq!(header_td_block, canonical_header_block); + debug_assert_eq!(header_block, canonical_header_block); - static_file_writer.append_header(&header, header_td.0, &canonical_header)?; + static_file_writer.append_header(&header, &canonical_header)?; } Ok(()) diff --git a/crates/static-file/static-file/src/static_file_producer.rs b/crates/static-file/static-file/src/static_file_producer.rs index b6d205a42e..185fbf7c49 100644 --- a/crates/static-file/static-file/src/static_file_producer.rs +++ b/crates/static-file/static-file/src/static_file_producer.rs @@ -207,19 +207,17 @@ where headers: finalized_block_numbers.headers.and_then(|finalized_block_number| { self.get_static_file_target(highest_static_files.headers, finalized_block_number) }), - // StaticFile receipts only if they're not pruned according to the user configuration - receipts: if self.prune_modes.receipts.is_none() && - self.prune_modes.receipts_log_filter.is_empty() - { - finalized_block_numbers.receipts.and_then(|finalized_block_number| { + receipts: finalized_block_numbers + .receipts + // StaticFile receipts only if they're not pruned according to the user + // configuration + .filter(|_| !self.prune_modes.has_receipts_pruning()) + .and_then(|finalized_block_number| { self.get_static_file_target( highest_static_files.receipts, finalized_block_number, ) - }) - } else { - None - }, + }), transactions: finalized_block_numbers.transactions.and_then(|finalized_block_number| { self.get_static_file_target( highest_static_files.transactions, diff --git a/crates/storage/codecs/derive/src/compact/structs.rs b/crates/storage/codecs/derive/src/compact/structs.rs index f8ebda3349..4bafe73062 100644 --- a/crates/storage/codecs/derive/src/compact/structs.rs +++ b/crates/storage/codecs/derive/src/compact/structs.rs @@ -155,7 +155,7 @@ impl<'a> StructHandler<'a> { let (#name, new_buf) = #ident_type::#from_compact_ident(buf, flags.#len() as usize); }); } else { - todo!() + unreachable!("flag-type fields are always compact in Compact derive") } self.lines.push(quote! { buf = new_buf; diff --git a/crates/storage/db-api/src/tables/mod.rs b/crates/storage/db-api/src/tables/mod.rs index 906f8b78ac..dfa81f644b 100644 --- a/crates/storage/db-api/src/tables/mod.rs +++ b/crates/storage/db-api/src/tables/mod.rs @@ -309,7 +309,8 @@ tables! { type Value = HeaderHash; } - /// Stores the total difficulty from a block header. + /// Stores the total difficulty from block headers. + /// Note: Deprecated. table HeaderTerminalDifficulties { type Key = BlockNumber; type Value = CompactU256; diff --git a/crates/storage/db-common/src/init.rs b/crates/storage/db-common/src/init.rs index 8b24f0f8d1..de55cea3c9 100644 --- a/crates/storage/db-common/src/init.rs +++ b/crates/storage/db-common/src/init.rs @@ -347,9 +347,8 @@ where match static_file_provider.block_hash(0) { Ok(None) | Err(ProviderError::MissingStaticFileBlock(StaticFileSegment::Headers, 0)) => { - let (difficulty, hash) = (header.difficulty(), block_hash); let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?; - writer.append_header(header, difficulty, &hash)?; + writer.append_header(header, &block_hash)?; } Ok(Some(_)) => {} Err(e) => return Err(e), diff --git a/crates/storage/db/benches/criterion.rs b/crates/storage/db/benches/criterion.rs index 64d6fbdbfd..7d62384c16 100644 --- a/crates/storage/db/benches/criterion.rs +++ b/crates/storage/db/benches/criterion.rs @@ -31,7 +31,6 @@ pub fn db(c: &mut Criterion) { group.warm_up_time(std::time::Duration::from_millis(200)); measure_table_db::(&mut group); - measure_table_db::(&mut group); measure_table_db::(&mut group); measure_table_db::(&mut group); measure_table_db::(&mut group); @@ -48,7 +47,6 @@ pub fn serialization(c: &mut Criterion) { group.warm_up_time(std::time::Duration::from_millis(200)); measure_table_serialization::(&mut group); - measure_table_serialization::(&mut group); measure_table_serialization::(&mut group); measure_table_serialization::(&mut group); measure_table_serialization::(&mut group); diff --git a/crates/storage/errors/src/provider.rs b/crates/storage/errors/src/provider.rs index 47cc630bcb..9630a1b2a6 100644 --- a/crates/storage/errors/src/provider.rs +++ b/crates/storage/errors/src/provider.rs @@ -58,9 +58,6 @@ pub enum ProviderError { /// The account address. address: Address, }, - /// The total difficulty for a block is missing. - #[error("total difficulty not found for block #{_0}")] - TotalDifficultyNotFound(BlockNumber), /// When required header related data was not found but was required. #[error("no header found for {_0:?}")] HeaderNotFound(BlockHashOrNumber), diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 512b8569de..9dbbed9e88 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -10,7 +10,7 @@ use crate::{ }; use alloy_consensus::transaction::TransactionMeta; use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumHash, BlockNumberOrTag}; -use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, U256}; +use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256}; use alloy_rpc_types_engine::ForkchoiceState; use reth_chain_state::{ BlockState, CanonicalInMemoryState, ForkChoiceNotifications, ForkChoiceSubscriptions, @@ -176,14 +176,6 @@ impl HeaderProvider for BlockchainProvider { self.consistent_provider()?.header_by_number(num) } - fn header_td(&self, hash: BlockHash) -> ProviderResult> { - self.consistent_provider()?.header_td(hash) - } - - fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult> { - self.consistent_provider()?.header_td_by_number(number) - } - fn headers_range( &self, range: impl RangeBounds, @@ -1280,24 +1272,12 @@ mod tests { BlockRangeParams::default(), )?; - let database_block = database_blocks.first().unwrap().clone(); - let in_memory_block = in_memory_blocks.last().unwrap().clone(); // make sure that the finalized block is on db let finalized_block = database_blocks.get(database_blocks.len() - 3).unwrap(); provider.set_finalized(finalized_block.clone_sealed_header()); let blocks = [database_blocks, in_memory_blocks].concat(); - assert_eq!( - provider.header_td_by_number(database_block.number)?, - Some(database_block.difficulty) - ); - - assert_eq!( - provider.header_td_by_number(in_memory_block.number)?, - Some(in_memory_block.difficulty) - ); - assert_eq!( provider.sealed_headers_while(0..=10, |header| header.number <= 8)?, blocks diff --git a/crates/storage/provider/src/providers/consistent.rs b/crates/storage/provider/src/providers/consistent.rs index 66a35e5e9b..67113fc5c0 100644 --- a/crates/storage/provider/src/providers/consistent.rs +++ b/crates/storage/provider/src/providers/consistent.rs @@ -13,7 +13,7 @@ use alloy_eips::{ }; use alloy_primitives::{ map::{hash_map, HashMap}, - Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, U256, + Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, }; use reth_chain_state::{BlockState, CanonicalInMemoryState, MemoryOverlayStateProviderRef}; use reth_chainspec::ChainInfo; @@ -663,37 +663,6 @@ impl HeaderProvider for ConsistentProvider { ) } - fn header_td(&self, hash: BlockHash) -> ProviderResult> { - if let Some(num) = self.block_number(hash)? { - self.header_td_by_number(num) - } else { - Ok(None) - } - } - - fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult> { - let number = if self.head_block.as_ref().map(|b| b.block_on_chain(number.into())).is_some() - { - // If the block exists in memory, we should return a TD for it. - // - // The canonical in memory state should only store post-merge blocks. Post-merge blocks - // have zero difficulty. This means we can use the total difficulty for the last - // finalized block number if present (so that we are not affected by reorgs), if not the - // last number in the database will be used. - if let Some(last_finalized_num_hash) = - self.canonical_in_memory_state.get_finalized_num_hash() - { - last_finalized_num_hash.number - } else { - self.last_block_number()? - } - } else { - // Otherwise, return what we have on disk for the input block - number - }; - self.storage_provider.header_td_by_number(number) - } - fn headers_range( &self, range: impl RangeBounds, diff --git a/crates/storage/provider/src/providers/database/chain.rs b/crates/storage/provider/src/providers/database/chain.rs index 2da32d9a05..9ce3861eb3 100644 --- a/crates/storage/provider/src/providers/database/chain.rs +++ b/crates/storage/provider/src/providers/database/chain.rs @@ -1,12 +1,12 @@ use crate::{providers::NodeTypesForProvider, DatabaseProvider}; use reth_db_api::transaction::{DbTx, DbTxMut}; -use reth_node_types::FullNodePrimitives; +use reth_node_types::NodePrimitives; use reth_primitives_traits::{FullBlockHeader, FullSignedTx}; use reth_storage_api::{ChainStorageReader, ChainStorageWriter, EmptyBodyStorage, EthStorage}; /// Trait that provides access to implementations of [`ChainStorage`] -pub trait ChainStorage: Send + Sync { +pub trait ChainStorage: Send + Sync { /// Provides access to the chain reader. fn reader(&self) -> impl ChainStorageReader, Primitives> where @@ -24,7 +24,7 @@ impl ChainStorage for EthStorage where T: FullSignedTx, H: FullBlockHeader, - N: FullNodePrimitives< + N: NodePrimitives< Block = alloy_consensus::Block, BlockHeader = H, BlockBody = alloy_consensus::BlockBody, @@ -52,7 +52,7 @@ impl ChainStorage for EmptyBodyStorage where T: FullSignedTx, H: FullBlockHeader, - N: FullNodePrimitives< + N: NodePrimitives< Block = alloy_consensus::Block, BlockHeader = H, BlockBody = alloy_consensus::BlockBody, diff --git a/crates/storage/provider/src/providers/database/metrics.rs b/crates/storage/provider/src/providers/database/metrics.rs index 4923b51db3..4daac3dfdd 100644 --- a/crates/storage/provider/src/providers/database/metrics.rs +++ b/crates/storage/provider/src/providers/database/metrics.rs @@ -45,7 +45,6 @@ pub(crate) enum Action { InsertBlockBodyIndices, InsertTransactionBlocks, GetNextTxNum, - GetParentTD, } /// Database provider metrics @@ -71,8 +70,6 @@ struct DatabaseProviderMetrics { insert_tx_blocks: Histogram, /// Duration of get next tx num get_next_tx_num: Histogram, - /// Duration of get parent TD - get_parent_td: Histogram, } impl DatabaseProviderMetrics { @@ -88,7 +85,6 @@ impl DatabaseProviderMetrics { Action::InsertBlockBodyIndices => self.insert_block_body_indices.record(duration), Action::InsertTransactionBlocks => self.insert_tx_blocks.record(duration), Action::GetNextTxNum => self.get_next_tx_num.record(duration), - Action::GetParentTD => self.get_parent_td.record(duration), } } } diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index bd6b1e0f47..5d3b5280cd 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -9,7 +9,7 @@ use crate::{ }; use alloy_consensus::transaction::TransactionMeta; use alloy_eips::BlockHashOrNumber; -use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, U256}; +use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256}; use core::fmt; use reth_chainspec::ChainInfo; use reth_db::{init_db, mdbx::DatabaseArguments, DatabaseEnv}; @@ -96,7 +96,7 @@ impl ProviderFactory { } /// Sets the pruning configuration for an existing [`ProviderFactory`]. - pub fn with_prune_modes(mut self, prune_modes: PruneModes) -> Self { + pub const fn with_prune_modes(mut self, prune_modes: PruneModes) -> Self { self.prune_modes = prune_modes; self } @@ -111,11 +111,6 @@ impl ProviderFactory { pub fn into_db(self) -> N::DB { self.db } - - /// Returns reference to the prune modes. - pub const fn prune_modes_ref(&self) -> &PruneModes { - &self.prune_modes - } } impl>> ProviderFactory { @@ -247,14 +242,6 @@ impl HeaderProvider for ProviderFactory { self.static_file_provider.header_by_number(num) } - fn header_td(&self, hash: BlockHash) -> ProviderResult> { - self.provider()?.header_td(hash) - } - - fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult> { - self.static_file_provider.header_td_by_number(number) - } - fn headers_range( &self, range: impl RangeBounds, @@ -590,7 +577,7 @@ mod tests { BlockHashReader, BlockNumReader, BlockWriter, DBProvider, HeaderSyncGapProvider, TransactionsProvider, }; - use alloy_primitives::{TxNumber, B256, U256}; + use alloy_primitives::{TxNumber, B256}; use assert_matches::assert_matches; use reth_chainspec::ChainSpecBuilder; use reth_db::{ @@ -735,7 +722,7 @@ mod tests { let static_file_provider = provider.static_file_provider(); let mut static_file_writer = static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); - static_file_writer.append_header(head.header(), U256::ZERO, &head.hash()).unwrap(); + static_file_writer.append_header(head.header(), &head.hash()).unwrap(); static_file_writer.commit().unwrap(); drop(static_file_writer); diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 235bf57a4a..d5e49d822b 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -22,18 +22,18 @@ use crate::{ }; use alloy_consensus::{ transaction::{SignerRecoverable, TransactionMeta, TxHashRef}, - BlockHeader, TxReceipt, + BlockHeader, }; use alloy_eips::BlockHashOrNumber; use alloy_primitives::{ keccak256, map::{hash_map, B256Map, HashMap, HashSet}, - Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, U256, + Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, }; use itertools::Itertools; use rayon::slice::ParallelSliceMut; use reth_chain_state::ExecutedBlock; -use reth_chainspec::{ChainInfo, ChainSpecProvider, EthChainSpec, EthereumHardforks}; +use reth_chainspec::{ChainInfo, ChainSpecProvider, EthChainSpec}; use reth_db_api::{ cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW}, database::Database, @@ -214,7 +214,7 @@ impl DatabaseProvider { #[cfg(feature = "test-utils")] /// Sets the prune modes for provider. - pub fn set_prune_modes(&mut self, prune_modes: PruneModes) { + pub const fn set_prune_modes(&mut self, prune_modes: PruneModes) { self.prune_modes = prune_modes; } } @@ -971,26 +971,6 @@ impl HeaderProvider for DatabasePro self.static_file_provider.header_by_number(num) } - fn header_td(&self, block_hash: BlockHash) -> ProviderResult> { - if let Some(num) = self.block_number(block_hash)? { - self.header_td_by_number(num) - } else { - Ok(None) - } - } - - fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult> { - if self.chain_spec.is_paris_active_at_block(number) && - let Some(td) = self.chain_spec.final_paris_total_difficulty() - { - // if this block is higher than the final paris(merge) block, return the final paris - // difficulty - return Ok(Some(td)) - } - - self.static_file_provider.header_td_by_number(number) - } - fn headers_range( &self, range: impl RangeBounds, @@ -1641,20 +1621,11 @@ impl StateWriter .then(|| self.static_file_provider.get_writer(first_block, StaticFileSegment::Receipts)) .transpose()?; - let has_contract_log_filter = !self.prune_modes.receipts_log_filter.is_empty(); - let contract_log_pruner = self.prune_modes.receipts_log_filter.group_by_block(tip, None)?; - // All receipts from the last 128 blocks are required for blockchain tree, even with // [`PruneSegment::ContractLogs`]. let prunable_receipts = PruneMode::Distance(MINIMUM_PRUNING_DISTANCE).should_prune(first_block, tip); - // Prepare set of addresses which logs should not be pruned. - let mut allowed_addresses: HashSet = HashSet::new(); - for (_, addresses) in contract_log_pruner.range(..first_block) { - allowed_addresses.extend(addresses.iter().copied()); - } - for (idx, (receipts, first_tx_index)) in execution_outcome.receipts.iter().zip(block_indices).enumerate() { @@ -1674,21 +1645,8 @@ impl StateWriter continue } - // If there are new addresses to retain after this block number, track them - if let Some(new_addresses) = contract_log_pruner.get(&block_number) { - allowed_addresses.extend(new_addresses.iter().copied()); - } - for (idx, receipt) in receipts.iter().enumerate() { let receipt_idx = first_tx_index + idx as u64; - // Skip writing receipt if log filter is active and it does not have any logs to - // retain - if prunable_receipts && - has_contract_log_filter && - !receipt.logs().iter().any(|log| allowed_addresses.contains(&log.address)) - { - continue - } if let Some(writer) = &mut receipts_static_writer { writer.append_receipt(receipt_idx, receipt)?; @@ -2808,7 +2766,6 @@ impl BlockWrite /// tables: /// * [`StaticFileSegment::Headers`] /// * [`tables::HeaderNumbers`] - /// * [`tables::HeaderTerminalDifficulties`] /// * [`tables::BlockBodyIndices`] /// /// If there are transactions in the block, the following static file segments and tables will @@ -2833,19 +2790,9 @@ impl BlockWrite let mut durations_recorder = metrics::DurationsRecorder::default(); - // total difficulty - let ttd = if block_number == 0 { - block.header().difficulty() - } else { - let parent_block_number = block_number - 1; - let parent_ttd = self.header_td_by_number(parent_block_number)?.unwrap_or_default(); - durations_recorder.record_relative(metrics::Action::GetParentTD); - parent_ttd + block.header().difficulty() - }; - self.static_file_provider .get_writer(block_number, StaticFileSegment::Headers)? - .append_header(block.header(), ttd, &block.hash())?; + .append_header(block.header(), &block.hash())?; self.tx.put::(block.hash(), block_number)?; durations_recorder.record_relative(metrics::Action::InsertHeaderNumbers); diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index 5a950bbd7d..41e8121991 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -2,7 +2,7 @@ use reth_chainspec::EthereumHardforks; use reth_db_api::table::Value; -use reth_node_types::{FullNodePrimitives, NodeTypes, NodeTypesWithDB}; +use reth_node_types::{NodePrimitives, NodeTypes, NodeTypesWithDB}; mod database; pub use database::*; @@ -36,7 +36,7 @@ where Self: NodeTypes< ChainSpec: EthereumHardforks, Storage: ChainStorage, - Primitives: FullNodePrimitives, + Primitives: NodePrimitives, >, { } @@ -45,7 +45,7 @@ impl NodeTypesForProvider for T where T: NodeTypes< ChainSpec: EthereumHardforks, Storage: ChainStorage, - Primitives: FullNodePrimitives, + Primitives: NodePrimitives, > { } diff --git a/crates/storage/provider/src/providers/state/overlay.rs b/crates/storage/provider/src/providers/state/overlay.rs index 71c1a69319..98bd17aa4f 100644 --- a/crates/storage/provider/src/providers/state/overlay.rs +++ b/crates/storage/provider/src/providers/state/overlay.rs @@ -1,8 +1,11 @@ use alloy_primitives::{BlockNumber, B256}; use reth_db_api::DatabaseError; use reth_errors::ProviderError; +use reth_prune_types::PruneSegment; use reth_stages_types::StageId; -use reth_storage_api::{DBProvider, DatabaseProviderFactory, StageCheckpointReader, TrieReader}; +use reth_storage_api::{ + DBProvider, DatabaseProviderFactory, PruneCheckpointReader, StageCheckpointReader, TrieReader, +}; use reth_trie::{ hashed_cursor::{HashedCursorFactory, HashedPostStateCursorFactory}, trie_cursor::{InMemoryTrieCursorFactory, TrieCursorFactory}, @@ -13,6 +16,7 @@ use reth_trie_db::{ DatabaseHashedCursorFactory, DatabaseHashedPostState, DatabaseTrieCursorFactory, }; use std::sync::Arc; +use tracing::debug; /// Factory for creating overlay state providers with optional reverts and overlays. /// @@ -33,26 +37,31 @@ pub struct OverlayStateProviderFactory { impl OverlayStateProviderFactory where F: DatabaseProviderFactory, - F::Provider: Clone + TrieReader + StageCheckpointReader, + F::Provider: Clone + TrieReader + StageCheckpointReader + PruneCheckpointReader, { /// Create a new overlay state provider factory pub const fn new(factory: F) -> Self { Self { factory, block_number: None, trie_overlay: None, hashed_state_overlay: None } } - /// Set the block number for collecting reverts + /// Set the block number for collecting reverts. All state will be reverted to the point + /// _after_ this block has been processed. pub const fn with_block_number(mut self, block_number: Option) -> Self { self.block_number = block_number; self } - /// Set the trie overlay + /// Set the trie overlay. + /// + /// This overlay will be applied on top of any reverts applied via `with_block_number`. pub fn with_trie_overlay(mut self, trie_overlay: Option>) -> Self { self.trie_overlay = trie_overlay; self } /// Set the hashed state overlay + /// + /// This overlay will be applied on top of any reverts applied via `with_block_number`. pub fn with_hashed_state_overlay( mut self, hashed_state_overlay: Option>, @@ -64,25 +73,47 @@ where /// Validates that there are sufficient changesets to revert to the requested block number. /// /// Returns an error if the `MerkleChangeSets` checkpoint doesn't cover the requested block. + /// Takes into account both the stage checkpoint and the prune checkpoint to determine the + /// available data range. fn validate_changesets_availability( &self, provider: &F::Provider, requested_block: BlockNumber, ) -> Result<(), ProviderError> { - // Get the MerkleChangeSets stage checkpoint - let errors propagate as-is - let checkpoint = provider.get_stage_checkpoint(StageId::MerkleChangeSets)?; + // Get the MerkleChangeSets stage and prune checkpoints. + let stage_checkpoint = provider.get_stage_checkpoint(StageId::MerkleChangeSets)?; + let prune_checkpoint = provider.get_prune_checkpoint(PruneSegment::MerkleChangeSets)?; - // If there's no checkpoint at all or block range details are missing, we can't revert - let available_range = checkpoint - .and_then(|chk| { - chk.merkle_changesets_stage_checkpoint() - .map(|stage_chk| stage_chk.block_range.from..=chk.block_number) - }) - .ok_or_else(|| ProviderError::InsufficientChangesets { - requested: requested_block, - available: 0..=0, + // Get the upper bound from stage checkpoint + let upper_bound = + stage_checkpoint.as_ref().map(|chk| chk.block_number).ok_or_else(|| { + ProviderError::InsufficientChangesets { + requested: requested_block, + available: 0..=0, + } })?; + // Extract a possible lower bound from stage checkpoint if available + let stage_lower_bound = stage_checkpoint.as_ref().and_then(|chk| { + chk.merkle_changesets_stage_checkpoint().map(|stage_chk| stage_chk.block_range.from) + }); + + // Extract a possible lower bound from prune checkpoint if available + // The prune checkpoint's block_number is the highest pruned block, so data is available + // starting from the next block + let prune_lower_bound = + prune_checkpoint.and_then(|chk| chk.block_number.map(|block| block + 1)); + + // Use the higher of the two lower bounds (or error if neither is available) + let Some(lower_bound) = stage_lower_bound.max(prune_lower_bound) else { + return Err(ProviderError::InsufficientChangesets { + requested: requested_block, + available: 0..=upper_bound, + }) + }; + + let available_range = lower_bound..=upper_bound; + // Check if the requested block is within the available range if !available_range.contains(&requested_block) { return Err(ProviderError::InsufficientChangesets { @@ -105,11 +136,13 @@ where self.validate_changesets_availability(&provider, from_block)?; // Collect trie reverts - let mut trie_updates_mut = provider.trie_reverts(from_block)?; + let mut trie_updates_mut = provider.trie_reverts(from_block + 1)?; // Collect state reverts using HashedPostState::from_reverts - let reverted_state = - HashedPostState::from_reverts::(provider.tx_ref(), from_block..)?; + let reverted_state = HashedPostState::from_reverts::( + provider.tx_ref(), + from_block + 1.., + )?; let mut hashed_state_mut = reverted_state.into_sorted(); // Extend with overlays if provided @@ -121,6 +154,14 @@ where hashed_state_mut.extend_ref(hashed_state_overlay); } + debug!( + target: "providers::state::overlay", + ?from_block, + num_trie_updates = ?trie_updates_mut.total_len(), + num_state_updates = ?hashed_state_mut.total_len(), + "Reverted to target block", + ); + (Arc::new(trie_updates_mut), Arc::new(hashed_state_mut)) } else { // If no block_number, use overlays directly or defaults @@ -145,95 +186,98 @@ where /// using the in-memory overlay factories. #[derive(Debug, Clone)] pub struct OverlayStateProvider { - /// The in-memory trie cursor factory that wraps the database cursor factory. - trie_cursor_factory: - InMemoryTrieCursorFactory, Arc>, - /// The hashed cursor factory that wraps the database cursor factory. - hashed_cursor_factory: HashedPostStateCursorFactory< - DatabaseHashedCursorFactory, - Arc, - >, + provider: Provider, + trie_updates: Arc, + hashed_post_state: Arc, } impl OverlayStateProvider where - Provider: DBProvider + Clone, + Provider: DBProvider, { /// Create new overlay state provider. The `Provider` must be cloneable, which generally means /// it should be wrapped in an `Arc`. - pub fn new( + pub const fn new( provider: Provider, trie_updates: Arc, hashed_post_state: Arc, ) -> Self { - // Create the trie cursor factory - let db_trie_cursor_factory = DatabaseTrieCursorFactory::new(provider.clone().into_tx()); - let trie_cursor_factory = - InMemoryTrieCursorFactory::new(db_trie_cursor_factory, trie_updates); - - // Create the hashed cursor factory - let db_hashed_cursor_factory = DatabaseHashedCursorFactory::new(provider.into_tx()); - let hashed_cursor_factory = - HashedPostStateCursorFactory::new(db_hashed_cursor_factory, hashed_post_state); - - Self { trie_cursor_factory, hashed_cursor_factory } + Self { provider, trie_updates, hashed_post_state } } } impl TrieCursorFactory for OverlayStateProvider where - Provider: DBProvider + Clone, - InMemoryTrieCursorFactory, Arc>: - TrieCursorFactory, + Provider: DBProvider, { - type AccountTrieCursor = , - Arc, - > as TrieCursorFactory>::AccountTrieCursor; + type AccountTrieCursor<'a> + = , + &'a TrieUpdatesSorted, + > as TrieCursorFactory>::AccountTrieCursor<'a> + where + Self: 'a; - type StorageTrieCursor = , - Arc, - > as TrieCursorFactory>::StorageTrieCursor; + type StorageTrieCursor<'a> + = , + &'a TrieUpdatesSorted, + > as TrieCursorFactory>::StorageTrieCursor<'a> + where + Self: 'a; - fn account_trie_cursor(&self) -> Result { - self.trie_cursor_factory.account_trie_cursor() + fn account_trie_cursor(&self) -> Result, DatabaseError> { + let db_trie_cursor_factory = DatabaseTrieCursorFactory::new(self.provider.tx_ref()); + let trie_cursor_factory = + InMemoryTrieCursorFactory::new(db_trie_cursor_factory, self.trie_updates.as_ref()); + trie_cursor_factory.account_trie_cursor() } fn storage_trie_cursor( &self, hashed_address: B256, - ) -> Result { - self.trie_cursor_factory.storage_trie_cursor(hashed_address) + ) -> Result, DatabaseError> { + let db_trie_cursor_factory = DatabaseTrieCursorFactory::new(self.provider.tx_ref()); + let trie_cursor_factory = + InMemoryTrieCursorFactory::new(db_trie_cursor_factory, self.trie_updates.as_ref()); + trie_cursor_factory.storage_trie_cursor(hashed_address) } } impl HashedCursorFactory for OverlayStateProvider where - Provider: DBProvider + Clone, - HashedPostStateCursorFactory< - DatabaseHashedCursorFactory, - Arc, - >: HashedCursorFactory, + Provider: DBProvider, { - type AccountCursor = , - Arc, - > as HashedCursorFactory>::AccountCursor; + type AccountCursor<'a> + = , + &'a Arc, + > as HashedCursorFactory>::AccountCursor<'a> + where + Self: 'a; - type StorageCursor = , - Arc, - > as HashedCursorFactory>::StorageCursor; + type StorageCursor<'a> + = , + &'a Arc, + > as HashedCursorFactory>::StorageCursor<'a> + where + Self: 'a; - fn hashed_account_cursor(&self) -> Result { - self.hashed_cursor_factory.hashed_account_cursor() + fn hashed_account_cursor(&self) -> Result, DatabaseError> { + let db_hashed_cursor_factory = DatabaseHashedCursorFactory::new(self.provider.tx_ref()); + let hashed_cursor_factory = + HashedPostStateCursorFactory::new(db_hashed_cursor_factory, &self.hashed_post_state); + hashed_cursor_factory.hashed_account_cursor() } fn hashed_storage_cursor( &self, hashed_address: B256, - ) -> Result { - self.hashed_cursor_factory.hashed_storage_cursor(hashed_address) + ) -> Result, DatabaseError> { + let db_hashed_cursor_factory = DatabaseHashedCursorFactory::new(self.provider.tx_ref()); + let hashed_cursor_factory = + HashedPostStateCursorFactory::new(db_hashed_cursor_factory, &self.hashed_post_state); + hashed_cursor_factory.hashed_storage_cursor(hashed_address) } } diff --git a/crates/storage/provider/src/providers/static_file/jar.rs b/crates/storage/provider/src/providers/static_file/jar.rs index 9906583f90..2cd7ec98ae 100644 --- a/crates/storage/provider/src/providers/static_file/jar.rs +++ b/crates/storage/provider/src/providers/static_file/jar.rs @@ -8,11 +8,10 @@ use crate::{ }; use alloy_consensus::transaction::{SignerRecoverable, TransactionMeta}; use alloy_eips::{eip2718::Encodable2718, BlockHashOrNumber}; -use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, U256}; +use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256}; use reth_chainspec::ChainInfo; use reth_db::static_file::{ - BlockHashMask, HeaderMask, HeaderWithHashMask, ReceiptMask, StaticFileCursor, TDWithHashMask, - TotalDifficultyMask, TransactionMask, + BlockHashMask, HeaderMask, HeaderWithHashMask, ReceiptMask, StaticFileCursor, TransactionMask, }; use reth_db_api::table::{Decompress, Value}; use reth_node_types::NodePrimitives; @@ -101,18 +100,6 @@ impl> HeaderProvider for StaticFileJarProv self.cursor()?.get_one::>(num.into()) } - fn header_td(&self, block_hash: BlockHash) -> ProviderResult> { - Ok(self - .cursor()? - .get_two::((&block_hash).into())? - .filter(|(_, hash)| hash == &block_hash) - .map(|(td, _)| td.into())) - } - - fn header_td_by_number(&self, num: BlockNumber) -> ProviderResult> { - Ok(self.cursor()?.get_one::(num.into())?.map(Into::into)) - } - fn headers_range( &self, range: impl RangeBounds, diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index 800c761718..76fa45f5a5 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -12,9 +12,7 @@ use alloy_consensus::{ Header, }; use alloy_eips::{eip2718::Encodable2718, BlockHashOrNumber}; -use alloy_primitives::{ - b256, keccak256, Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, U256, -}; +use alloy_primitives::{b256, keccak256, Address, BlockHash, BlockNumber, TxHash, TxNumber, B256}; use dashmap::DashMap; use notify::{RecommendedWatcher, RecursiveMode, Watcher}; use parking_lot::RwLock; @@ -23,7 +21,7 @@ use reth_db::{ lockfile::StorageLock, static_file::{ iter_static_files, BlockHashMask, HeaderMask, HeaderWithHashMask, ReceiptMask, - StaticFileCursor, TDWithHashMask, TransactionMask, + StaticFileCursor, TransactionMask, }, }; use reth_db_api::{ @@ -35,9 +33,9 @@ use reth_db_api::{ }; use reth_ethereum_primitives::{Receipt, TransactionSigned}; use reth_nippy_jar::{NippyJar, NippyJarChecker, CONFIG_FILE_EXTENSION}; -use reth_node_types::{FullNodePrimitives, NodePrimitives}; +use reth_node_types::NodePrimitives; use reth_primitives_traits::{RecoveredBlock, SealedHeader, SignedTransaction}; -use reth_stages_types::StageId; +use reth_stages_types::{PipelineTarget, StageId}; use reth_static_file_types::{ find_fixed_range, HighestStaticFiles, SegmentHeader, SegmentRangeInclusive, StaticFileSegment, DEFAULT_BLOCKS_PER_STATIC_FILE, @@ -731,14 +729,15 @@ impl StaticFileProvider { /// * its highest block should match the stage checkpoint block number if it's equal or higher /// than the corresponding database table last entry. /// - /// Returns a [`Option`] with block number to unwind to if any healing is further required. + /// Returns a [`Option`] of [`PipelineTarget::Unwind`] if any healing is further required. /// /// WARNING: No static file writer should be held before calling this function, otherwise it /// will deadlock. pub fn check_consistency( &self, provider: &Provider, - ) -> ProviderResult> + has_receipt_pruning: bool, + ) -> ProviderResult> where Provider: DBProvider + BlockReader + StageCheckpointReader + ChainSpecProvider, N: NodePrimitives, @@ -775,7 +774,7 @@ impl StaticFileProvider { }; for segment in StaticFileSegment::iter() { - if provider.prune_modes_ref().has_receipts_pruning() && segment.is_receipts() { + if has_receipt_pruning && segment.is_receipts() { // Pruned nodes (including full node) do not store receipts as static files. continue } @@ -886,7 +885,7 @@ impl StaticFileProvider { } } - Ok(unwind_target) + Ok(unwind_target.map(PipelineTarget::Unwind)) } /// Checks consistency of the latest static file segment and throws an error if at fault. @@ -1402,27 +1401,6 @@ impl> HeaderProvider for StaticFileProvide }) } - fn header_td(&self, block_hash: BlockHash) -> ProviderResult> { - self.find_static_file(StaticFileSegment::Headers, |jar_provider| { - Ok(jar_provider - .cursor()? - .get_two::((&block_hash).into())? - .and_then(|(td, hash)| (hash == block_hash).then_some(td.0))) - }) - } - - fn header_td_by_number(&self, num: BlockNumber) -> ProviderResult> { - self.get_segment_provider_from_block(StaticFileSegment::Headers, num, None) - .and_then(|provider| provider.header_td_by_number(num)) - .or_else(|err| { - if let ProviderError::MissingStaticFileBlock(_, _) = err { - Ok(None) - } else { - Err(err) - } - }) - } - fn headers_range( &self, range: impl RangeBounds, @@ -1546,8 +1524,8 @@ impl> Rec } } -impl> - TransactionsProviderExt for StaticFileProvider +impl> TransactionsProviderExt + for StaticFileProvider { fn transaction_hashes_by_range( &self, @@ -1745,7 +1723,7 @@ impl BlockNumReader for StaticFileProvider { /* Cannot be successfully implemented but must exist for trait requirements */ -impl> BlockReader +impl> BlockReader for StaticFileProvider { type Block = N::Block; diff --git a/crates/storage/provider/src/providers/static_file/mod.rs b/crates/storage/provider/src/providers/static_file/mod.rs index 1c3bfd58a7..3c25f157bb 100644 --- a/crates/storage/provider/src/providers/static_file/mod.rs +++ b/crates/storage/provider/src/providers/static_file/mod.rs @@ -58,12 +58,10 @@ mod tests { test_utils::create_test_provider_factory, HeaderProvider, StaticFileProviderFactory, }; use alloy_consensus::{Header, SignableTransaction, Transaction, TxLegacy}; - use alloy_primitives::{BlockHash, Signature, TxNumber, B256, U256}; + use alloy_primitives::{BlockHash, Signature, TxNumber, B256}; use rand::seq::SliceRandom; use reth_db::test_utils::create_test_static_files_dir; - use reth_db_api::{ - transaction::DbTxMut, CanonicalHeaders, HeaderNumbers, HeaderTerminalDifficulties, Headers, - }; + use reth_db_api::{transaction::DbTxMut, CanonicalHeaders, HeaderNumbers, Headers}; use reth_ethereum_primitives::{EthPrimitives, Receipt, TransactionSigned}; use reth_static_file_types::{ find_fixed_range, SegmentRangeInclusive, DEFAULT_BLOCKS_PER_STATIC_FILE, @@ -102,14 +100,11 @@ mod tests { let mut provider_rw = factory.provider_rw().unwrap(); let tx = provider_rw.tx_mut(); - let mut td = U256::ZERO; for header in headers.clone() { - td += header.header().difficulty; let hash = header.hash(); tx.put::(header.number, hash).unwrap(); tx.put::(header.number, header.clone_header()).unwrap(); - tx.put::(header.number, td.into()).unwrap(); tx.put::(hash, header.number).unwrap(); } provider_rw.commit().unwrap(); @@ -118,12 +113,10 @@ mod tests { { let manager = factory.static_file_provider(); let mut writer = manager.latest_writer(StaticFileSegment::Headers).unwrap(); - let mut td = U256::ZERO; for header in headers.clone() { - td += header.header().difficulty; let hash = header.hash(); - writer.append_header(&header.unseal(), td, &hash).unwrap(); + writer.append_header(&header.unseal(), &hash).unwrap(); } writer.commit().unwrap(); } @@ -148,12 +141,6 @@ mod tests { // Compare Header assert_eq!(header, db_provider.header(header_hash).unwrap().unwrap()); assert_eq!(header, jar_provider.header_by_number(header.number).unwrap().unwrap()); - - // Compare HeaderTerminalDifficulties - assert_eq!( - db_provider.header_td(header_hash).unwrap().unwrap(), - jar_provider.header_td_by_number(header.number).unwrap().unwrap() - ); } } } @@ -180,9 +167,7 @@ mod tests { let mut header = Header::default(); for num in 0..=tip { header.number = num; - header_writer - .append_header(&header, U256::default(), &BlockHash::default()) - .unwrap(); + header_writer.append_header(&header, &BlockHash::default()).unwrap(); } header_writer.commit().unwrap(); } diff --git a/crates/storage/provider/src/providers/static_file/writer.rs b/crates/storage/provider/src/providers/static_file/writer.rs index b9c17f8292..7b0ae9ce11 100644 --- a/crates/storage/provider/src/providers/static_file/writer.rs +++ b/crates/storage/provider/src/providers/static_file/writer.rs @@ -531,7 +531,20 @@ impl StaticFileProviderRW { /// blocks. /// /// Returns the current [`BlockNumber`] as seen in the static file. - pub fn append_header( + pub fn append_header(&mut self, header: &N::BlockHeader, hash: &BlockHash) -> ProviderResult<()> + where + N::BlockHeader: Compact, + { + self.append_header_with_td(header, U256::ZERO, hash) + } + + /// Appends header to static file with a specified total difficulty. + /// + /// It **CALLS** `increment_block()` since the number of headers is equal to the number of + /// blocks. + /// + /// Returns the current [`BlockNumber`] as seen in the static file. + pub fn append_header_with_td( &mut self, header: &N::BlockHeader, total_difficulty: U256, diff --git a/crates/storage/provider/src/test_utils/blocks.rs b/crates/storage/provider/src/test_utils/blocks.rs index 818b97e0c1..0b27c5dc99 100644 --- a/crates/storage/provider/src/test_utils/blocks.rs +++ b/crates/storage/provider/src/test_utils/blocks.rs @@ -31,10 +31,6 @@ pub fn assert_genesis_block( assert_eq!(tx.table::().unwrap(), vec![(h, n)]); assert_eq!(tx.table::().unwrap(), vec![(n, h)]); - assert_eq!( - tx.table::().unwrap(), - vec![(n, g.difficulty.into())] - ); assert_eq!( tx.table::().unwrap(), vec![(0, StoredBlockBodyIndices::default())] diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 3e33e2b050..4b3829cf8e 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -292,24 +292,6 @@ impl HeaderP Ok(lock.values().find(|h| h.number() == num).cloned()) } - fn header_td(&self, hash: BlockHash) -> ProviderResult> { - let lock = self.headers.lock(); - Ok(lock.get(&hash).map(|target| { - lock.values() - .filter(|h| h.number() < target.number()) - .fold(target.difficulty(), |td, h| td + h.difficulty()) - })) - } - - fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult> { - let lock = self.headers.lock(); - let sum = lock - .values() - .filter(|h| h.number() <= number) - .fold(U256::ZERO, |td, h| td + h.difficulty()); - Ok(Some(sum)) - } - fn headers_range( &self, range: impl RangeBounds, diff --git a/crates/storage/provider/src/traits/full.rs b/crates/storage/provider/src/traits/full.rs index 710ca9400e..6fe88a6640 100644 --- a/crates/storage/provider/src/traits/full.rs +++ b/crates/storage/provider/src/traits/full.rs @@ -2,8 +2,8 @@ use crate::{ AccountReader, BlockReader, BlockReaderIdExt, ChainSpecProvider, ChangeSetReader, - DatabaseProviderFactory, HashedPostStateProvider, StageCheckpointReader, StateProviderFactory, - StateReader, StaticFileProviderFactory, TrieReader, + DatabaseProviderFactory, HashedPostStateProvider, PruneCheckpointReader, StageCheckpointReader, + StateProviderFactory, StateReader, StaticFileProviderFactory, TrieReader, }; use reth_chain_state::{CanonStateSubscriptions, ForkChoiceSubscriptions}; use reth_node_types::{BlockTy, HeaderTy, NodeTypesWithDB, ReceiptTy, TxTy}; @@ -12,8 +12,10 @@ use std::fmt::Debug; /// Helper trait to unify all provider traits for simplicity. pub trait FullProvider: - DatabaseProviderFactory - + NodePrimitivesProvider + DatabaseProviderFactory< + DB = N::DB, + Provider: BlockReader + TrieReader + StageCheckpointReader + PruneCheckpointReader, + > + NodePrimitivesProvider + StaticFileProviderFactory + BlockReaderIdExt< Transaction = TxTy, @@ -37,8 +39,10 @@ pub trait FullProvider: } impl FullProvider for T where - T: DatabaseProviderFactory - + NodePrimitivesProvider + T: DatabaseProviderFactory< + DB = N::DB, + Provider: BlockReader + TrieReader + StageCheckpointReader + PruneCheckpointReader, + > + NodePrimitivesProvider + StaticFileProviderFactory + BlockReaderIdExt< Transaction = TxTy, diff --git a/crates/storage/rpc-provider/src/lib.rs b/crates/storage/rpc-provider/src/lib.rs index ed6e49eefb..6e5bd17218 100644 --- a/crates/storage/rpc-provider/src/lib.rs +++ b/crates/storage/rpc-provider/src/lib.rs @@ -364,18 +364,6 @@ where Ok(Some(sealed_header.into_header())) } - fn header_td(&self, hash: BlockHash) -> ProviderResult> { - let header = self.header(hash).map_err(ProviderError::other)?; - - Ok(header.map(|b| b.difficulty())) - } - - fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult> { - let header = self.header_by_number(number).map_err(ProviderError::other)?; - - Ok(header.map(|b| b.difficulty())) - } - fn headers_range( &self, _range: impl RangeBounds, @@ -1674,14 +1662,6 @@ where Err(ProviderError::UnsupportedProvider) } - fn header_td(&self, _hash: BlockHash) -> Result, ProviderError> { - Err(ProviderError::UnsupportedProvider) - } - - fn header_td_by_number(&self, _number: BlockNumber) -> Result, ProviderError> { - Err(ProviderError::UnsupportedProvider) - } - fn headers_range( &self, _range: impl RangeBounds, diff --git a/crates/storage/storage-api/src/chain.rs b/crates/storage/storage-api/src/chain.rs index 50b0722d6c..5705f2be99 100644 --- a/crates/storage/storage-api/src/chain.rs +++ b/crates/storage/storage-api/src/chain.rs @@ -14,7 +14,7 @@ use reth_db_api::{ use reth_db_models::{blocks::StoredBlockAccessList, StoredBlockWithdrawals}; use reth_ethereum_primitives::TransactionSigned; use reth_primitives_traits::{ - Block, BlockBody, FullBlockHeader, FullNodePrimitives, SignedTransaction, + Block, BlockBody, FullBlockHeader, NodePrimitives, SignedTransaction, }; use reth_storage_errors::provider::ProviderResult; @@ -40,11 +40,11 @@ pub trait BlockBodyWriter { } /// Trait that implements how chain-specific types are written to the storage. -pub trait ChainStorageWriter: +pub trait ChainStorageWriter: BlockBodyWriter::Body> { } -impl ChainStorageWriter for T where +impl ChainStorageWriter for T where T: BlockBodyWriter::Body> { } @@ -73,11 +73,11 @@ pub trait BlockBodyReader { } /// Trait that implements how chain-specific types are read from storage. -pub trait ChainStorageReader: +pub trait ChainStorageReader: BlockBodyReader { } -impl ChainStorageReader for T where +impl ChainStorageReader for T where T: BlockBodyReader { } diff --git a/crates/storage/storage-api/src/header.rs b/crates/storage/storage-api/src/header.rs index 7e3133ec71..39b2eef903 100644 --- a/crates/storage/storage-api/src/header.rs +++ b/crates/storage/storage-api/src/header.rs @@ -1,6 +1,6 @@ use alloc::vec::Vec; use alloy_eips::BlockHashOrNumber; -use alloy_primitives::{BlockHash, BlockNumber, U256}; +use alloy_primitives::{BlockHash, BlockNumber}; use core::ops::RangeBounds; use reth_primitives_traits::{BlockHeader, SealedHeader}; use reth_storage_errors::provider::ProviderResult; @@ -44,12 +44,6 @@ pub trait HeaderProvider: Send + Sync { } } - /// Get total difficulty by block hash. - fn header_td(&self, hash: BlockHash) -> ProviderResult>; - - /// Get total difficulty by block number. - fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult>; - /// Get headers in range of block numbers fn headers_range( &self, diff --git a/crates/storage/storage-api/src/noop.rs b/crates/storage/storage-api/src/noop.rs index 6b70a5260a..e538e1216e 100644 --- a/crates/storage/storage-api/src/noop.rs +++ b/crates/storage/storage-api/src/noop.rs @@ -15,7 +15,7 @@ use alloc::{boxed::Box, string::String, sync::Arc, vec::Vec}; use alloy_consensus::transaction::TransactionMeta; use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumberOrTag}; use alloy_primitives::{ - Address, BlockHash, BlockNumber, Bytes, StorageKey, StorageValue, TxHash, TxNumber, B256, U256, + Address, BlockHash, BlockNumber, Bytes, StorageKey, StorageValue, TxHash, TxNumber, B256, }; use core::{ fmt::Debug, @@ -356,14 +356,6 @@ impl HeaderProvider for NoopProvider { Ok(None) } - fn header_td(&self, _hash: BlockHash) -> ProviderResult> { - Ok(None) - } - - fn header_td_by_number(&self, _number: BlockNumber) -> ProviderResult> { - Ok(None) - } - fn headers_range( &self, _range: impl RangeBounds, diff --git a/crates/tracing/src/layers.rs b/crates/tracing/src/layers.rs index d27bbc96b6..156bd8c825 100644 --- a/crates/tracing/src/layers.rs +++ b/crates/tracing/src/layers.rs @@ -20,12 +20,16 @@ pub(crate) type BoxedLayer = Box + Send + Sync>; /// Default [directives](Directive) for [`EnvFilter`] which disables high-frequency debug logs from /// `hyper`, `hickory-resolver`, `jsonrpsee-server`, and `discv5`. -const DEFAULT_ENV_FILTER_DIRECTIVES: [&str; 5] = [ +const DEFAULT_ENV_FILTER_DIRECTIVES: [&str; 9] = [ "hyper::proto::h1=off", "hickory_resolver=off", "hickory_proto=off", "discv5=off", "jsonrpsee-server=off", + "opentelemetry-otlp=off", + "opentelemetry_sdk=off", + "opentelemetry-http=off", + "hyper_util::client::legacy::pool=off", ]; /// Manages the collection of layers for a tracing subscriber. diff --git a/crates/transaction-pool/src/error.rs b/crates/transaction-pool/src/error.rs index 6360817caa..3bcbb4cd0a 100644 --- a/crates/transaction-pool/src/error.rs +++ b/crates/transaction-pool/src/error.rs @@ -237,8 +237,13 @@ pub enum InvalidPoolTransactionError { /// Thrown if the input data of a transaction is greater /// than some meaningful limit a user might use. This is not a consensus error /// making the transaction invalid, rather a DOS protection. - #[error("input data too large")] - OversizedData(usize, usize), + #[error("oversized data: transaction size {size}, limit {limit}")] + OversizedData { + /// Size of the transaction/input data that exceeded the limit. + size: usize, + /// Configured limit that was exceeded. + limit: usize, + }, /// Thrown if the transaction's fee is below the minimum fee #[error("transaction underpriced")] Underpriced, @@ -335,7 +340,7 @@ impl InvalidPoolTransactionError { } Self::ExceedsFeeCap { max_tx_fee_wei: _, tx_fee_cap_wei: _ } => true, Self::ExceedsMaxInitCodeSize(_, _) => true, - Self::OversizedData(_, _) => true, + Self::OversizedData { .. } => true, Self::Underpriced => { // local setting false @@ -393,7 +398,7 @@ impl InvalidPoolTransactionError { /// Returns `true` if an import failed due to an oversized transaction pub const fn is_oversized(&self) -> bool { - matches!(self, Self::OversizedData(_, _)) + matches!(self, Self::OversizedData { .. }) } /// Returns `true` if an import failed due to nonce gap. diff --git a/crates/transaction-pool/src/validate/eth.rs b/crates/transaction-pool/src/validate/eth.rs index e368ce85e6..4bc87ca9b1 100644 --- a/crates/transaction-pool/src/validate/eth.rs +++ b/crates/transaction-pool/src/validate/eth.rs @@ -326,10 +326,10 @@ where if tx_input_len > self.max_tx_input_bytes { return Err(TransactionValidationOutcome::Invalid( transaction, - InvalidPoolTransactionError::OversizedData( - tx_input_len, - self.max_tx_input_bytes, - ), + InvalidPoolTransactionError::OversizedData { + size: tx_input_len, + limit: self.max_tx_input_bytes, + }, )) } } else { @@ -338,7 +338,10 @@ where if tx_size > self.max_tx_input_bytes { return Err(TransactionValidationOutcome::Invalid( transaction, - InvalidPoolTransactionError::OversizedData(tx_size, self.max_tx_input_bytes), + InvalidPoolTransactionError::OversizedData { + size: tx_size, + limit: self.max_tx_input_bytes, + }, )) } } diff --git a/crates/trie/common/Cargo.toml b/crates/trie/common/Cargo.toml index 0aa93adb59..2fcc23ab53 100644 --- a/crates/trie/common/Cargo.toml +++ b/crates/trie/common/Cargo.toml @@ -23,6 +23,7 @@ reth-codecs = { workspace = true, optional = true } alloy-rpc-types-eth = { workspace = true, optional = true } alloy-serde = { workspace = true, optional = true } +arrayvec = { workspace = true, optional = true } bytes = { workspace = true, optional = true } derive_more.workspace = true itertools = { workspace = true, features = ["use_alloc"] } @@ -52,6 +53,7 @@ alloy-genesis.workspace = true alloy-primitives = { workspace = true, features = ["getrandom"] } alloy-trie = { workspace = true, features = ["arbitrary", "serde"] } bytes.workspace = true +arrayvec.workspace = true hash-db.workspace = true plain_hasher.workspace = true arbitrary = { workspace = true, features = ["derive"] } @@ -74,6 +76,7 @@ std = [ "alloy-rpc-types-eth?/std", "alloy-serde?/std", "alloy-trie/std", + "arrayvec?/std", "bytes?/std", "derive_more/std", "nybbles/std", @@ -87,6 +90,7 @@ std = [ eip1186 = ["alloy-rpc-types-eth/serde", "dep:alloy-serde"] serde = [ "dep:serde", + "arrayvec?/serde", "bytes?/serde", "nybbles/serde", "alloy-primitives/serde", @@ -98,7 +102,7 @@ serde = [ "revm-database/serde", "revm-state/serde", ] -reth-codec = ["dep:reth-codecs", "dep:bytes"] +reth-codec = ["dep:reth-codecs", "dep:bytes", "dep:arrayvec"] serde-bincode-compat = [ "serde", "reth-primitives-traits/serde-bincode-compat", diff --git a/crates/trie/common/src/hashed_state.rs b/crates/trie/common/src/hashed_state.rs index 27c2807ad2..8fb994dadd 100644 --- a/crates/trie/common/src/hashed_state.rs +++ b/crates/trie/common/src/hashed_state.rs @@ -486,6 +486,13 @@ impl HashedPostStateSorted { &self.storages } + /// Returns the total number of updates including all accounts and storage updates. + pub fn total_len(&self) -> usize { + self.accounts.accounts.len() + + self.accounts.destroyed_accounts.len() + + self.storages.values().map(|storage| storage.len()).sum::() + } + /// Extends this state with contents of another sorted state. /// Entries in `other` take precedence for duplicate keys. pub fn extend_ref(&mut self, other: &Self) { @@ -568,6 +575,16 @@ impl HashedStorageSorted { .sorted_by_key(|entry| *entry.0) } + /// Returns the total number of storage slot updates. + pub fn len(&self) -> usize { + self.non_zero_valued_slots.len() + self.zero_valued_slots.len() + } + + /// Returns `true` if there are no storage slot updates. + pub fn is_empty(&self) -> bool { + self.non_zero_valued_slots.is_empty() && self.zero_valued_slots.is_empty() + } + /// Extends this storage with contents of another sorted storage. /// Entries in `other` take precedence for duplicate keys. pub fn extend_ref(&mut self, other: &Self) { diff --git a/crates/trie/common/src/nibbles.rs b/crates/trie/common/src/nibbles.rs index 7d9e6670be..82d710395f 100644 --- a/crates/trie/common/src/nibbles.rs +++ b/crates/trie/common/src/nibbles.rs @@ -28,10 +28,9 @@ impl reth_codecs::Compact for StoredNibbles { where B: bytes::BufMut + AsMut<[u8]>, { - for i in self.0.iter() { - buf.put_u8(i); - } - self.0.len() + let bytes = self.0.iter().collect::>(); + buf.put_slice(&bytes); + bytes.len() } fn from_compact(mut buf: &[u8], len: usize) -> (Self, &[u8]) { @@ -78,14 +77,14 @@ impl reth_codecs::Compact for StoredNibblesSubKey { { assert!(self.0.len() <= 64); - // right-pad with zeros - for i in self.0.iter() { - buf.put_u8(i); - } - static ZERO: &[u8; 64] = &[0; 64]; - buf.put_slice(&ZERO[self.0.len()..]); + let bytes = self.0.iter().collect::>(); + buf.put_slice(&bytes); - buf.put_u8(self.0.len() as u8); + // Right-pad with zeros + static ZERO: &[u8; 64] = &[0; 64]; + buf.put_slice(&ZERO[bytes.len()..]); + + buf.put_u8(bytes.len() as u8); 64 + 1 } diff --git a/crates/trie/common/src/storage.rs b/crates/trie/common/src/storage.rs index 557b9e4a60..1e56739386 100644 --- a/crates/trie/common/src/storage.rs +++ b/crates/trie/common/src/storage.rs @@ -1,4 +1,4 @@ -use super::{BranchNodeCompact, Nibbles, StoredNibblesSubKey}; +use super::{BranchNodeCompact, StoredNibblesSubKey}; /// Account storage trie node. /// @@ -61,7 +61,7 @@ impl reth_codecs::Compact for TrieChangeSetsEntry { if len == 0 { // Return an empty entry without trying to parse anything return ( - Self { nibbles: StoredNibblesSubKey::from(Nibbles::default()), node: None }, + Self { nibbles: StoredNibblesSubKey::from(super::Nibbles::default()), node: None }, buf, ) } diff --git a/crates/trie/common/src/updates.rs b/crates/trie/common/src/updates.rs index 00a160c4f9..e3e098ac8e 100644 --- a/crates/trie/common/src/updates.rs +++ b/crates/trie/common/src/updates.rs @@ -453,6 +453,12 @@ impl TrieUpdatesSorted { &self.storage_tries } + /// Returns the total number of updates including account nodes and all storage updates. + pub fn total_len(&self) -> usize { + self.account_nodes.len() + + self.storage_tries.values().map(|storage| storage.len()).sum::() + } + /// Extends the trie updates with another set of sorted updates. /// /// This merges the account nodes and storage tries from `other` into `self`. @@ -535,6 +541,16 @@ impl StorageTrieUpdatesSorted { &self.storage_nodes } + /// Returns the total number of storage node updates. + pub const fn len(&self) -> usize { + self.storage_nodes.len() + } + + /// Returns `true` if there are no storage node updates. + pub const fn is_empty(&self) -> bool { + self.storage_nodes.is_empty() + } + /// Extends the storage trie updates with another set of sorted updates. /// /// If `other` is marked as deleted, this will be marked as deleted and all nodes cleared. diff --git a/crates/trie/common/src/utils.rs b/crates/trie/common/src/utils.rs index e5d16d3ef5..5a2234fe26 100644 --- a/crates/trie/common/src/utils.rs +++ b/crates/trie/common/src/utils.rs @@ -11,7 +11,7 @@ use alloc::vec::Vec; /// 5. Appending and re-sorting only if new items were added pub(crate) fn extend_sorted_vec(target: &mut Vec<(K, V)>, other: &[(K, V)]) where - K: Clone + Ord + core::hash::Hash + Eq, + K: Clone + Ord, V: Clone, { if other.is_empty() { diff --git a/crates/trie/db/src/hashed_cursor.rs b/crates/trie/db/src/hashed_cursor.rs index 06e6914275..4fe3d57429 100644 --- a/crates/trie/db/src/hashed_cursor.rs +++ b/crates/trie/db/src/hashed_cursor.rs @@ -20,18 +20,23 @@ impl DatabaseHashedCursorFactory { } impl HashedCursorFactory for DatabaseHashedCursorFactory<&TX> { - type AccountCursor = DatabaseHashedAccountCursor<::Cursor>; - type StorageCursor = - DatabaseHashedStorageCursor<::DupCursor>; + type AccountCursor<'a> + = DatabaseHashedAccountCursor<::Cursor> + where + Self: 'a; + type StorageCursor<'a> + = DatabaseHashedStorageCursor<::DupCursor> + where + Self: 'a; - fn hashed_account_cursor(&self) -> Result { + fn hashed_account_cursor(&self) -> Result, DatabaseError> { Ok(DatabaseHashedAccountCursor(self.0.cursor_read::()?)) } fn hashed_storage_cursor( &self, hashed_address: B256, - ) -> Result { + ) -> Result, DatabaseError> { Ok(DatabaseHashedStorageCursor::new( self.0.cursor_dup_read::()?, hashed_address, diff --git a/crates/trie/db/src/trie_cursor.rs b/crates/trie/db/src/trie_cursor.rs index b1e9032fc0..d05c3fd92d 100644 --- a/crates/trie/db/src/trie_cursor.rs +++ b/crates/trie/db/src/trie_cursor.rs @@ -26,18 +26,24 @@ impl TrieCursorFactory for DatabaseTrieCursorFactory<&TX> where TX: DbTx, { - type AccountTrieCursor = DatabaseAccountTrieCursor<::Cursor>; - type StorageTrieCursor = - DatabaseStorageTrieCursor<::DupCursor>; + type AccountTrieCursor<'a> + = DatabaseAccountTrieCursor<::Cursor> + where + Self: 'a; - fn account_trie_cursor(&self) -> Result { + type StorageTrieCursor<'a> + = DatabaseStorageTrieCursor<::DupCursor> + where + Self: 'a; + + fn account_trie_cursor(&self) -> Result, DatabaseError> { Ok(DatabaseAccountTrieCursor::new(self.0.cursor_read::()?)) } fn storage_trie_cursor( &self, hashed_address: B256, - ) -> Result { + ) -> Result, DatabaseError> { Ok(DatabaseStorageTrieCursor::new( self.0.cursor_dup_read::()?, hashed_address, diff --git a/crates/trie/parallel/src/proof_task.rs b/crates/trie/parallel/src/proof_task.rs index b3269f21fb..6525500a2a 100644 --- a/crates/trie/parallel/src/proof_task.rs +++ b/crates/trie/parallel/src/proof_task.rs @@ -57,7 +57,7 @@ use std::{ time::Instant, }; use tokio::runtime::Handle; -use tracing::trace; +use tracing::{debug_span, trace}; #[cfg(feature = "metrics")] use crate::proof_task_metrics::ProofTaskTrieMetrics; @@ -300,10 +300,16 @@ fn account_worker_loop( while let Ok(job) = work_rx.recv() { match job { AccountWorkerJob::AccountMultiproof { mut input, result_sender } => { + let span = tracing::debug_span!( + target: "trie::proof_task", + "Account multiproof calculation", + targets = input.targets.len(), + worker_id, + ); + let _span_guard = span.enter(); + trace!( target: "trie::proof_task", - worker_id, - targets = input.targets.len(), "Processing account multiproof" ); @@ -370,18 +376,24 @@ fn account_worker_loop( trace!( target: "trie::proof_task", - worker_id, proof_time_us = proof_elapsed.as_micros(), total_processed = account_proofs_processed, "Account multiproof completed" ); + drop(_span_guard); } AccountWorkerJob::BlindedAccountNode { path, result_sender } => { + let span = tracing::debug_span!( + target: "trie::proof_task", + "Blinded account node calculation", + ?path, + worker_id, + ); + let _span_guard = span.enter(); + trace!( target: "trie::proof_task", - worker_id, - ?path, "Processing blinded account node" ); @@ -403,12 +415,11 @@ fn account_worker_loop( trace!( target: "trie::proof_task", - worker_id, - ?path, node_time_us = elapsed.as_micros(), total_processed = account_nodes_processed, "Blinded account node completed" ); + drop(_span_guard); } } } @@ -693,7 +704,7 @@ where multi_added_removed_keys.unwrap_or_else(|| Arc::new(MultiAddedRemovedKeys::new())); let added_removed_keys = multi_added_removed_keys.get_storage(&hashed_address); - let span = tracing::info_span!( + let span = tracing::debug_span!( target: "trie::proof_task", "Storage proof calculation", hashed_address = ?hashed_address, @@ -889,8 +900,13 @@ impl ProofWorkerHandle { "Spawning proof worker pools" ); + let storage_worker_parent = + debug_span!(target: "trie::proof_task", "Storage worker tasks", ?storage_worker_count); + let _guard = storage_worker_parent.enter(); + // Spawn storage workers for worker_id in 0..storage_worker_count { + let parent_span = debug_span!(target: "trie::proof_task", "Storage worker", ?worker_id); let view_clone = view.clone(); let task_ctx_clone = task_ctx.clone(); let work_rx_clone = storage_work_rx.clone(); @@ -899,6 +915,7 @@ impl ProofWorkerHandle { #[cfg(feature = "metrics")] let metrics = ProofTaskTrieMetrics::default(); + let _guard = parent_span.enter(); storage_worker_loop( view_clone, task_ctx_clone, @@ -916,8 +933,15 @@ impl ProofWorkerHandle { ); } + drop(_guard); + + let account_worker_parent = + debug_span!(target: "trie::proof_task", "Account worker tasks", ?account_worker_count); + let _guard = account_worker_parent.enter(); + // Spawn account workers for worker_id in 0..account_worker_count { + let parent_span = debug_span!(target: "trie::proof_task", "Account worker", ?worker_id); let view_clone = view.clone(); let task_ctx_clone = task_ctx.clone(); let work_rx_clone = account_work_rx.clone(); @@ -927,6 +951,7 @@ impl ProofWorkerHandle { #[cfg(feature = "metrics")] let metrics = ProofTaskTrieMetrics::default(); + let _guard = parent_span.enter(); account_worker_loop( view_clone, task_ctx_clone, @@ -945,6 +970,8 @@ impl ProofWorkerHandle { ); } + drop(_guard); + Self::new_handle(storage_work_tx, account_work_tx) } diff --git a/crates/trie/sparse-parallel/src/trie.rs b/crates/trie/sparse-parallel/src/trie.rs index b15eb7f4ed..5e5a838f41 100644 --- a/crates/trie/sparse-parallel/src/trie.rs +++ b/crates/trie/sparse-parallel/src/trie.rs @@ -741,7 +741,7 @@ impl SparseTrieInterface for ParallelSparseTrie { // Update subtrie hashes in parallel { use rayon::iter::{IntoParallelIterator, ParallelIterator}; - use tracing::info_span; + use tracing::debug_span; let (tx, rx) = mpsc::channel(); @@ -751,8 +751,8 @@ impl SparseTrieInterface for ParallelSparseTrie { changed_subtries .into_par_iter() .map(|mut changed_subtrie| { - let _enter = info_span!( - target: "trie::sparse::parallel", + let _enter = debug_span!( + target: "trie::parallel_sparse", parent: span.clone(), "subtrie", index = changed_subtrie.index @@ -1303,7 +1303,7 @@ impl ParallelSparseTrie { /// Drains any [`SparseTrieUpdatesAction`]s from the given subtrie, and applies each action to /// the given `updates` set. If the given set is None then this is a no-op. - #[instrument(target = "trie::sparse::parallel", skip_all)] + #[instrument(target = "trie::parallel_sparse", skip_all)] fn apply_subtrie_update_actions( &mut self, update_actions: impl Iterator, @@ -1405,7 +1405,7 @@ impl ParallelSparseTrie { /// /// IMPORTANT: The method removes the subtries from `lower_subtries`, and the caller is /// responsible for returning them back into the array. - #[instrument(target = "trie::sparse::parallel", skip_all, fields(prefix_set_len = prefix_set.len()))] + #[instrument(target = "trie::parallel_sparse", skip_all, fields(prefix_set_len = prefix_set.len()))] fn take_changed_lower_subtries( &mut self, prefix_set: &mut PrefixSet, @@ -1562,7 +1562,7 @@ impl ParallelSparseTrie { /// Return updated subtries back to the trie after executing any actions required on the /// top-level `SparseTrieUpdates`. - #[instrument(target = "trie::sparse::parallel", skip_all)] + #[instrument(target = "trie::parallel_sparse", skip_all)] fn insert_changed_subtries( &mut self, changed_subtries: impl IntoIterator, diff --git a/crates/trie/trie/src/hashed_cursor/mock.rs b/crates/trie/trie/src/hashed_cursor/mock.rs index 308f05e4c8..f091ae6ffe 100644 --- a/crates/trie/trie/src/hashed_cursor/mock.rs +++ b/crates/trie/trie/src/hashed_cursor/mock.rs @@ -55,17 +55,23 @@ impl MockHashedCursorFactory { } impl HashedCursorFactory for MockHashedCursorFactory { - type AccountCursor = MockHashedCursor; - type StorageCursor = MockHashedCursor; + type AccountCursor<'a> + = MockHashedCursor + where + Self: 'a; + type StorageCursor<'a> + = MockHashedCursor + where + Self: 'a; - fn hashed_account_cursor(&self) -> Result { + fn hashed_account_cursor(&self) -> Result, DatabaseError> { Ok(MockHashedCursor::new(self.hashed_accounts.clone(), self.visited_account_keys.clone())) } fn hashed_storage_cursor( &self, hashed_address: B256, - ) -> Result { + ) -> Result, DatabaseError> { Ok(MockHashedCursor::new( self.hashed_storage_tries .get(&hashed_address) diff --git a/crates/trie/trie/src/hashed_cursor/mod.rs b/crates/trie/trie/src/hashed_cursor/mod.rs index 7917f67545..6c4788a336 100644 --- a/crates/trie/trie/src/hashed_cursor/mod.rs +++ b/crates/trie/trie/src/hashed_cursor/mod.rs @@ -14,23 +14,29 @@ pub mod noop; pub mod mock; /// The factory trait for creating cursors over the hashed state. +#[auto_impl::auto_impl(&)] pub trait HashedCursorFactory { /// The hashed account cursor type. - type AccountCursor: HashedCursor; + type AccountCursor<'a>: HashedCursor + where + Self: 'a; /// The hashed storage cursor type. - type StorageCursor: HashedStorageCursor; + type StorageCursor<'a>: HashedStorageCursor + where + Self: 'a; /// Returns a cursor for iterating over all hashed accounts in the state. - fn hashed_account_cursor(&self) -> Result; + fn hashed_account_cursor(&self) -> Result, DatabaseError>; /// Returns a cursor for iterating over all hashed storage entries in the state. fn hashed_storage_cursor( &self, hashed_address: B256, - ) -> Result; + ) -> Result, DatabaseError>; } /// The cursor for iterating over hashed entries. +#[auto_impl::auto_impl(&mut)] pub trait HashedCursor { /// Value returned by the cursor. type Value: std::fmt::Debug; @@ -44,6 +50,7 @@ pub trait HashedCursor { } /// The cursor for iterating over hashed storage entries. +#[auto_impl::auto_impl(&mut)] pub trait HashedStorageCursor: HashedCursor { /// Returns `true` if there are no entries for a given key. fn is_storage_empty(&mut self) -> Result; diff --git a/crates/trie/trie/src/hashed_cursor/noop.rs b/crates/trie/trie/src/hashed_cursor/noop.rs index 58b78dc245..e5bc44f0f5 100644 --- a/crates/trie/trie/src/hashed_cursor/noop.rs +++ b/crates/trie/trie/src/hashed_cursor/noop.rs @@ -9,17 +9,23 @@ use reth_storage_errors::db::DatabaseError; pub struct NoopHashedCursorFactory; impl HashedCursorFactory for NoopHashedCursorFactory { - type AccountCursor = NoopHashedAccountCursor; - type StorageCursor = NoopHashedStorageCursor; + type AccountCursor<'a> + = NoopHashedAccountCursor + where + Self: 'a; + type StorageCursor<'a> + = NoopHashedStorageCursor + where + Self: 'a; - fn hashed_account_cursor(&self) -> Result { + fn hashed_account_cursor(&self) -> Result, DatabaseError> { Ok(NoopHashedAccountCursor::default()) } fn hashed_storage_cursor( &self, _hashed_address: B256, - ) -> Result { + ) -> Result, DatabaseError> { Ok(NoopHashedStorageCursor::default()) } } diff --git a/crates/trie/trie/src/hashed_cursor/post_state.rs b/crates/trie/trie/src/hashed_cursor/post_state.rs index e81aa4af22..896251f363 100644 --- a/crates/trie/trie/src/hashed_cursor/post_state.rs +++ b/crates/trie/trie/src/hashed_cursor/post_state.rs @@ -19,15 +19,21 @@ impl HashedPostStateCursorFactory { } } -impl<'a, CF, T> HashedCursorFactory for HashedPostStateCursorFactory +impl<'overlay, CF, T> HashedCursorFactory for HashedPostStateCursorFactory where CF: HashedCursorFactory, T: AsRef, { - type AccountCursor = HashedPostStateAccountCursor<'a, CF::AccountCursor>; - type StorageCursor = HashedPostStateStorageCursor<'a, CF::StorageCursor>; + type AccountCursor<'cursor> + = HashedPostStateAccountCursor<'overlay, CF::AccountCursor<'cursor>> + where + Self: 'cursor; + type StorageCursor<'cursor> + = HashedPostStateStorageCursor<'overlay, CF::StorageCursor<'cursor>> + where + Self: 'cursor; - fn hashed_account_cursor(&self) -> Result { + fn hashed_account_cursor(&self) -> Result, DatabaseError> { let cursor = self.cursor_factory.hashed_account_cursor()?; Ok(HashedPostStateAccountCursor::new(cursor, &self.post_state.as_ref().accounts)) } @@ -35,7 +41,7 @@ where fn hashed_storage_cursor( &self, hashed_address: B256, - ) -> Result { + ) -> Result, DatabaseError> { let cursor = self.cursor_factory.hashed_storage_cursor(hashed_address)?; Ok(HashedPostStateStorageCursor::new( cursor, diff --git a/crates/trie/trie/src/node_iter.rs b/crates/trie/trie/src/node_iter.rs index e11cd51f79..862176c803 100644 --- a/crates/trie/trie/src/node_iter.rs +++ b/crates/trie/trie/src/node_iter.rs @@ -191,10 +191,11 @@ where /// /// NOTE: The iteration will start from the key of the previous hashed entry if it was supplied. #[instrument( + level = "trace", target = "trie::node_iter", skip_all, fields(trie_type = ?self.trie_type), - ret(level = "trace") + ret )] pub fn try_next( &mut self, diff --git a/crates/trie/trie/src/trie_cursor/in_memory.rs b/crates/trie/trie/src/trie_cursor/in_memory.rs index 7f1b933e20..1c7f179ad0 100644 --- a/crates/trie/trie/src/trie_cursor/in_memory.rs +++ b/crates/trie/trie/src/trie_cursor/in_memory.rs @@ -20,15 +20,22 @@ impl InMemoryTrieCursorFactory { } } -impl<'a, CF, T> TrieCursorFactory for InMemoryTrieCursorFactory +impl<'overlay, CF, T> TrieCursorFactory for InMemoryTrieCursorFactory where - CF: TrieCursorFactory, + CF: TrieCursorFactory + 'overlay, T: AsRef, { - type AccountTrieCursor = InMemoryTrieCursor<'a, CF::AccountTrieCursor>; - type StorageTrieCursor = InMemoryTrieCursor<'a, CF::StorageTrieCursor>; + type AccountTrieCursor<'cursor> + = InMemoryTrieCursor<'overlay, CF::AccountTrieCursor<'cursor>> + where + Self: 'cursor; - fn account_trie_cursor(&self) -> Result { + type StorageTrieCursor<'cursor> + = InMemoryTrieCursor<'overlay, CF::StorageTrieCursor<'cursor>> + where + Self: 'cursor; + + fn account_trie_cursor(&self) -> Result, DatabaseError> { let cursor = self.cursor_factory.account_trie_cursor()?; Ok(InMemoryTrieCursor::new(Some(cursor), self.trie_updates.as_ref().account_nodes_ref())) } @@ -36,7 +43,7 @@ where fn storage_trie_cursor( &self, hashed_address: B256, - ) -> Result { + ) -> Result, DatabaseError> { // if the storage trie has no updates then we use this as the in-memory overlay. static EMPTY_UPDATES: Vec<(Nibbles, Option)> = Vec::new(); diff --git a/crates/trie/trie/src/trie_cursor/mock.rs b/crates/trie/trie/src/trie_cursor/mock.rs index add2d7ddef..313df0443e 100644 --- a/crates/trie/trie/src/trie_cursor/mock.rs +++ b/crates/trie/trie/src/trie_cursor/mock.rs @@ -52,11 +52,17 @@ impl MockTrieCursorFactory { } impl TrieCursorFactory for MockTrieCursorFactory { - type AccountTrieCursor = MockTrieCursor; - type StorageTrieCursor = MockTrieCursor; + type AccountTrieCursor<'a> + = MockTrieCursor + where + Self: 'a; + type StorageTrieCursor<'a> + = MockTrieCursor + where + Self: 'a; /// Generates a mock account trie cursor. - fn account_trie_cursor(&self) -> Result { + fn account_trie_cursor(&self) -> Result, DatabaseError> { Ok(MockTrieCursor::new(self.account_trie_nodes.clone(), self.visited_account_keys.clone())) } @@ -64,7 +70,7 @@ impl TrieCursorFactory for MockTrieCursorFactory { fn storage_trie_cursor( &self, hashed_address: B256, - ) -> Result { + ) -> Result, DatabaseError> { Ok(MockTrieCursor::new( self.storage_tries .get(&hashed_address) diff --git a/crates/trie/trie/src/trie_cursor/mod.rs b/crates/trie/trie/src/trie_cursor/mod.rs index 269611150d..05a6c09e94 100644 --- a/crates/trie/trie/src/trie_cursor/mod.rs +++ b/crates/trie/trie/src/trie_cursor/mod.rs @@ -24,24 +24,29 @@ pub use self::{depth_first::DepthFirstTrieIterator, in_memory::*, subnode::Curso #[auto_impl::auto_impl(&)] pub trait TrieCursorFactory { /// The account trie cursor type. - type AccountTrieCursor: TrieCursor; + type AccountTrieCursor<'a>: TrieCursor + where + Self: 'a; + /// The storage trie cursor type. - type StorageTrieCursor: TrieCursor; + type StorageTrieCursor<'a>: TrieCursor + where + Self: 'a; /// Create an account trie cursor. - fn account_trie_cursor(&self) -> Result; + fn account_trie_cursor(&self) -> Result, DatabaseError>; /// Create a storage tries cursor. fn storage_trie_cursor( &self, hashed_address: B256, - ) -> Result; + ) -> Result, DatabaseError>; } /// A cursor for traversing stored trie nodes. The cursor must iterate over keys in /// lexicographical order. -#[auto_impl::auto_impl(&mut, Box)] -pub trait TrieCursor: Send + Sync { +#[auto_impl::auto_impl(&mut)] +pub trait TrieCursor { /// Move the cursor to the key and return if it is an exact match. fn seek_exact( &mut self, diff --git a/crates/trie/trie/src/trie_cursor/noop.rs b/crates/trie/trie/src/trie_cursor/noop.rs index de409c59fe..a00a18e4f0 100644 --- a/crates/trie/trie/src/trie_cursor/noop.rs +++ b/crates/trie/trie/src/trie_cursor/noop.rs @@ -9,11 +9,18 @@ use reth_storage_errors::db::DatabaseError; pub struct NoopTrieCursorFactory; impl TrieCursorFactory for NoopTrieCursorFactory { - type AccountTrieCursor = NoopAccountTrieCursor; - type StorageTrieCursor = NoopStorageTrieCursor; + type AccountTrieCursor<'a> + = NoopAccountTrieCursor + where + Self: 'a; + + type StorageTrieCursor<'a> + = NoopStorageTrieCursor + where + Self: 'a; /// Generates a noop account trie cursor. - fn account_trie_cursor(&self) -> Result { + fn account_trie_cursor(&self) -> Result, DatabaseError> { Ok(NoopAccountTrieCursor::default()) } @@ -21,7 +28,7 @@ impl TrieCursorFactory for NoopTrieCursorFactory { fn storage_trie_cursor( &self, _hashed_address: B256, - ) -> Result { + ) -> Result, DatabaseError> { Ok(NoopStorageTrieCursor::default()) } } diff --git a/crates/trie/trie/src/verify.rs b/crates/trie/trie/src/verify.rs index 9605921145..4299a66916 100644 --- a/crates/trie/trie/src/verify.rs +++ b/crates/trie/trie/src/verify.rs @@ -301,21 +301,24 @@ impl SingleVerifier> { /// database tables as the source of truth. This will iteratively recompute the entire trie based /// on the hashed state, and produce any discovered [`Output`]s via the `next` method. #[derive(Debug)] -pub struct Verifier { - trie_cursor_factory: T, +pub struct Verifier<'a, T: TrieCursorFactory, H> { + trie_cursor_factory: &'a T, hashed_cursor_factory: H, branch_node_iter: StateRootBranchNodesIter, outputs: Vec, - account: SingleVerifier>, - storage: Option<(B256, SingleVerifier>)>, + account: SingleVerifier>>, + storage: Option<(B256, SingleVerifier>>)>, complete: bool, } -impl Verifier { +impl<'a, T: TrieCursorFactory, H: HashedCursorFactory + Clone> Verifier<'a, T, H> { /// Creates a new verifier instance. - pub fn new(trie_cursor_factory: T, hashed_cursor_factory: H) -> Result { + pub fn new( + trie_cursor_factory: &'a T, + hashed_cursor_factory: H, + ) -> Result { Ok(Self { - trie_cursor_factory: trie_cursor_factory.clone(), + trie_cursor_factory, hashed_cursor_factory: hashed_cursor_factory.clone(), branch_node_iter: StateRootBranchNodesIter::new(hashed_cursor_factory), outputs: Default::default(), @@ -326,7 +329,7 @@ impl Verifier Verifier { +impl<'a, T: TrieCursorFactory, H: HashedCursorFactory + Clone> Verifier<'a, T, H> { fn new_storage( &mut self, account: B256, @@ -444,7 +447,7 @@ impl Verifier { } } -impl Iterator for Verifier { +impl<'a, T: TrieCursorFactory, H: HashedCursorFactory + Clone> Iterator for Verifier<'a, T, H> { type Item = Result; fn next(&mut self) -> Option { diff --git a/crates/trie/trie/src/walker.rs b/crates/trie/trie/src/walker.rs index 0ea466437f..f12bf46f74 100644 --- a/crates/trie/trie/src/walker.rs +++ b/crates/trie/trie/src/walker.rs @@ -157,7 +157,7 @@ impl> TrieWalker { } /// Returns the next unprocessed key in the trie along with its raw [`Nibbles`] representation. - #[instrument(skip(self), ret(level = "trace"))] + #[instrument(level = "trace", skip(self), ret)] pub fn next_unprocessed_key(&self) -> Option<(B256, Nibbles)> { self.key() .and_then(|key| if self.can_skip_current_node { key.increment() } else { Some(*key) }) @@ -297,7 +297,7 @@ impl> TrieWalker { } /// Consumes the next node in the trie, updating the stack. - #[instrument(skip(self), ret(level = "trace"))] + #[instrument(level = "trace", skip(self), ret)] fn consume_node(&mut self) -> Result<(), DatabaseError> { let Some((key, node)) = self.node(false)? else { // If no next node is found, clear the stack. @@ -343,7 +343,7 @@ impl> TrieWalker { } /// Moves to the next sibling node in the trie, updating the stack. - #[instrument(skip(self), ret(level = "trace"))] + #[instrument(level = "trace", skip(self), ret)] fn move_to_next_sibling( &mut self, allow_root_to_child_nibble: bool, diff --git a/docs/cli/help.rs b/docs/cli/help.rs index 05e61eef74..0474d00e72 100755 --- a/docs/cli/help.rs +++ b/docs/cli/help.rs @@ -269,11 +269,6 @@ fn preprocess_help(s: &str) -> Cow<'_, str> { r"(rpc.max-tracing-requests \n.*\n.*\n.*\n.*\n.*)\[default: \d+\]", r"$1[default: ]", ), - // Handle engine.max-proof-task-concurrency dynamic default - ( - r"(engine\.max-proof-task-concurrency.*)\[default: \d+\]", - r"$1[default: ]", - ), // Handle engine.reserved-cpu-cores dynamic default ( r"(engine\.reserved-cpu-cores.*)\[default: \d+\]", diff --git a/docs/vocs/docs/pages/cli/reth.mdx b/docs/vocs/docs/pages/cli/reth.mdx index 0344c23bf2..041d494523 100644 --- a/docs/vocs/docs/pages/cli/reth.mdx +++ b/docs/vocs/docs/pages/cli/reth.mdx @@ -131,5 +131,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/config.mdx b/docs/vocs/docs/pages/cli/reth/config.mdx index adc08cd96e..96bdcf7a98 100644 --- a/docs/vocs/docs/pages/cli/reth/config.mdx +++ b/docs/vocs/docs/pages/cli/reth/config.mdx @@ -117,5 +117,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db.mdx b/docs/vocs/docs/pages/cli/reth/db.mdx index 91397e0f7e..f2a4942083 100644 --- a/docs/vocs/docs/pages/cli/reth/db.mdx +++ b/docs/vocs/docs/pages/cli/reth/db.mdx @@ -182,5 +182,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/checksum.mdx b/docs/vocs/docs/pages/cli/reth/db/checksum.mdx index 834fd42e44..c86273aacf 100644 --- a/docs/vocs/docs/pages/cli/reth/db/checksum.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/checksum.mdx @@ -134,5 +134,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/clear.mdx b/docs/vocs/docs/pages/cli/reth/db/clear.mdx index 0b64cefb71..88fd92763f 100644 --- a/docs/vocs/docs/pages/cli/reth/db/clear.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/clear.mdx @@ -126,5 +126,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/clear/mdbx.mdx b/docs/vocs/docs/pages/cli/reth/db/clear/mdbx.mdx index eb4120a34c..c467fe9d3d 100644 --- a/docs/vocs/docs/pages/cli/reth/db/clear/mdbx.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/clear/mdbx.mdx @@ -125,5 +125,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx b/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx index 913c6fcc5e..d4b59a0522 100644 --- a/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx @@ -128,5 +128,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/diff.mdx b/docs/vocs/docs/pages/cli/reth/db/diff.mdx index b5120d7409..4bb81ac07c 100644 --- a/docs/vocs/docs/pages/cli/reth/db/diff.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/diff.mdx @@ -161,5 +161,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/drop.mdx b/docs/vocs/docs/pages/cli/reth/db/drop.mdx index e0a54dcac3..c75a889458 100644 --- a/docs/vocs/docs/pages/cli/reth/db/drop.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/drop.mdx @@ -124,5 +124,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/get.mdx b/docs/vocs/docs/pages/cli/reth/db/get.mdx index 0d027754d5..8c20c7e311 100644 --- a/docs/vocs/docs/pages/cli/reth/db/get.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/get.mdx @@ -126,5 +126,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/get/mdbx.mdx b/docs/vocs/docs/pages/cli/reth/db/get/mdbx.mdx index 2ea1ea48f2..3b8df2f3a4 100644 --- a/docs/vocs/docs/pages/cli/reth/db/get/mdbx.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/get/mdbx.mdx @@ -134,5 +134,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx b/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx index 21e0849345..3980903c65 100644 --- a/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx @@ -134,5 +134,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/list.mdx b/docs/vocs/docs/pages/cli/reth/db/list.mdx index 55e14d822c..16131a95a1 100644 --- a/docs/vocs/docs/pages/cli/reth/db/list.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/list.mdx @@ -167,5 +167,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/path.mdx b/docs/vocs/docs/pages/cli/reth/db/path.mdx index 3f95c5761d..0c09f5be69 100644 --- a/docs/vocs/docs/pages/cli/reth/db/path.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/path.mdx @@ -121,5 +121,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/repair-trie.mdx b/docs/vocs/docs/pages/cli/reth/db/repair-trie.mdx index d972bcccd5..9c08ff331e 100644 --- a/docs/vocs/docs/pages/cli/reth/db/repair-trie.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/repair-trie.mdx @@ -124,5 +124,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/stats.mdx b/docs/vocs/docs/pages/cli/reth/db/stats.mdx index 1fd305c4e6..47695e1b22 100644 --- a/docs/vocs/docs/pages/cli/reth/db/stats.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/stats.mdx @@ -134,5 +134,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/version.mdx b/docs/vocs/docs/pages/cli/reth/db/version.mdx index c2b50b8944..7611b69946 100644 --- a/docs/vocs/docs/pages/cli/reth/db/version.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/version.mdx @@ -121,5 +121,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/download.mdx b/docs/vocs/docs/pages/cli/reth/download.mdx index 1890b95821..b18faa9320 100644 --- a/docs/vocs/docs/pages/cli/reth/download.mdx +++ b/docs/vocs/docs/pages/cli/reth/download.mdx @@ -179,5 +179,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/dump-genesis.mdx b/docs/vocs/docs/pages/cli/reth/dump-genesis.mdx index 4791d56198..bf5b0ac534 100644 --- a/docs/vocs/docs/pages/cli/reth/dump-genesis.mdx +++ b/docs/vocs/docs/pages/cli/reth/dump-genesis.mdx @@ -120,5 +120,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/export-era.mdx b/docs/vocs/docs/pages/cli/reth/export-era.mdx index 430e0948a9..cd413c1284 100644 --- a/docs/vocs/docs/pages/cli/reth/export-era.mdx +++ b/docs/vocs/docs/pages/cli/reth/export-era.mdx @@ -185,5 +185,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/import-era.mdx b/docs/vocs/docs/pages/cli/reth/import-era.mdx index c0d03852de..7d62409a63 100644 --- a/docs/vocs/docs/pages/cli/reth/import-era.mdx +++ b/docs/vocs/docs/pages/cli/reth/import-era.mdx @@ -180,5 +180,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/import.mdx b/docs/vocs/docs/pages/cli/reth/import.mdx index b5795a6e1d..8e3e1cdb0a 100644 --- a/docs/vocs/docs/pages/cli/reth/import.mdx +++ b/docs/vocs/docs/pages/cli/reth/import.mdx @@ -181,5 +181,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/init-state.mdx b/docs/vocs/docs/pages/cli/reth/init-state.mdx index 1ba1affc51..49c0e09809 100644 --- a/docs/vocs/docs/pages/cli/reth/init-state.mdx +++ b/docs/vocs/docs/pages/cli/reth/init-state.mdx @@ -201,5 +201,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/init.mdx b/docs/vocs/docs/pages/cli/reth/init.mdx index 11777b1f6e..ac1c7ff254 100644 --- a/docs/vocs/docs/pages/cli/reth/init.mdx +++ b/docs/vocs/docs/pages/cli/reth/init.mdx @@ -169,5 +169,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index a752f76b01..3fc6988dc6 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -776,9 +776,6 @@ Pruning: --prune.receipts.before Prune receipts before the specified block number. The specified block number is not pruned - --prune.receiptslogfilter - Configure receipts log filter. Format: <`address`>:<`prune_mode`>... where <`prune_mode`> can be 'full', 'distance:<`blocks`>', or 'before:<`block_number`>' - --prune.accounthistory.full Prunes all account history @@ -840,11 +837,6 @@ Engine: --engine.accept-execution-requests-hash Enables accepting requests hash instead of an array of requests in `engine_newPayloadV4` - --engine.max-proof-task-concurrency - Configure the maximum number of concurrent proof tasks - - [default: 256] - --engine.multiproof-chunking Whether multiproof task should chunk proof targets @@ -1016,5 +1008,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/p2p.mdx b/docs/vocs/docs/pages/cli/reth/p2p.mdx index 4138656604..b81c00a038 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p.mdx @@ -118,5 +118,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/p2p/body.mdx b/docs/vocs/docs/pages/cli/reth/p2p/body.mdx index 63f77913f9..fd28a37ebb 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/body.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/body.mdx @@ -338,5 +338,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/p2p/bootnode.mdx b/docs/vocs/docs/pages/cli/reth/p2p/bootnode.mdx index 578932411f..63baa86d36 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/bootnode.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/bootnode.mdx @@ -129,5 +129,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/p2p/header.mdx b/docs/vocs/docs/pages/cli/reth/p2p/header.mdx index f9b3276ced..f9f9449754 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/header.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/header.mdx @@ -338,5 +338,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/p2p/rlpx.mdx b/docs/vocs/docs/pages/cli/reth/p2p/rlpx.mdx index 8bf19d3eca..78d6dd8d3b 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/rlpx.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/rlpx.mdx @@ -115,5 +115,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/p2p/rlpx/ping.mdx b/docs/vocs/docs/pages/cli/reth/p2p/rlpx/ping.mdx index de13e93b56..2089c92461 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/rlpx/ping.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/rlpx/ping.mdx @@ -115,5 +115,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/prune.mdx b/docs/vocs/docs/pages/cli/reth/prune.mdx index bc5d038569..8f5828e8a6 100644 --- a/docs/vocs/docs/pages/cli/reth/prune.mdx +++ b/docs/vocs/docs/pages/cli/reth/prune.mdx @@ -169,5 +169,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/re-execute.mdx b/docs/vocs/docs/pages/cli/reth/re-execute.mdx index dc3bcbe462..56a7e3558c 100644 --- a/docs/vocs/docs/pages/cli/reth/re-execute.mdx +++ b/docs/vocs/docs/pages/cli/reth/re-execute.mdx @@ -182,5 +182,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage.mdx b/docs/vocs/docs/pages/cli/reth/stage.mdx index 85f2559de4..822f0f0c2d 100644 --- a/docs/vocs/docs/pages/cli/reth/stage.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage.mdx @@ -118,5 +118,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/drop.mdx b/docs/vocs/docs/pages/cli/reth/stage/drop.mdx index 923fd5ff95..037495979a 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/drop.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/drop.mdx @@ -184,5 +184,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/dump.mdx b/docs/vocs/docs/pages/cli/reth/stage/dump.mdx index 2466edcb96..8484379fe3 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/dump.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/dump.mdx @@ -176,5 +176,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/dump/account-hashing.mdx b/docs/vocs/docs/pages/cli/reth/stage/dump/account-hashing.mdx index c79571b31c..079804ff08 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/dump/account-hashing.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/dump/account-hashing.mdx @@ -133,5 +133,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/dump/execution.mdx b/docs/vocs/docs/pages/cli/reth/stage/dump/execution.mdx index c2480bae00..7aee318e1a 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/dump/execution.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/dump/execution.mdx @@ -133,5 +133,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/dump/merkle.mdx b/docs/vocs/docs/pages/cli/reth/stage/dump/merkle.mdx index 423771b183..17b2b7c951 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/dump/merkle.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/dump/merkle.mdx @@ -133,5 +133,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/dump/storage-hashing.mdx b/docs/vocs/docs/pages/cli/reth/stage/dump/storage-hashing.mdx index 211f4e5997..de64aa51c3 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/dump/storage-hashing.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/dump/storage-hashing.mdx @@ -133,5 +133,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/run.mdx b/docs/vocs/docs/pages/cli/reth/stage/run.mdx index 9eae5963a1..5407938072 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/run.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/run.mdx @@ -405,5 +405,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx b/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx index ab5776e2e5..2d2f94d680 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx @@ -177,5 +177,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/unwind/num-blocks.mdx b/docs/vocs/docs/pages/cli/reth/stage/unwind/num-blocks.mdx index 500cb3197f..a376af8401 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/unwind/num-blocks.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/unwind/num-blocks.mdx @@ -125,5 +125,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/unwind/to-block.mdx b/docs/vocs/docs/pages/cli/reth/stage/unwind/to-block.mdx index 4ec68dbb1e..ce62c64360 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/unwind/to-block.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/unwind/to-block.mdx @@ -125,5 +125,5 @@ Tracing: Defaults to TRACE if not specified. - [default: TRACE] + [default: debug] ``` \ No newline at end of file diff --git a/examples/custom-node/src/primitives/tx.rs b/examples/custom-node/src/primitives/tx.rs index 7c282922f4..fe763e079e 100644 --- a/examples/custom-node/src/primitives/tx.rs +++ b/examples/custom-node/src/primitives/tx.rs @@ -23,7 +23,7 @@ pub enum CustomTransaction { /// A regular Optimism transaction as defined by [`OpTxEnvelope`]. #[envelope(flatten)] Op(OpTxEnvelope), - /// A [`TxPayment`] tagged with type 0x7E. + /// A [`TxPayment`] tagged with type 0x2A (decimal 42). #[envelope(ty = 42)] Payment(Signed), } diff --git a/examples/db-access/src/main.rs b/examples/db-access/src/main.rs index 93896accbb..339aa1ae3d 100644 --- a/examples/db-access/src/main.rs +++ b/examples/db-access/src/main.rs @@ -66,11 +66,6 @@ fn header_provider_example(provider: T, number: u64) -> eyre: provider.header(sealed_header.hash())?.ok_or(eyre::eyre!("header by hash not found"))?; assert_eq!(sealed_header.header(), &header_by_hash); - // The header's total difficulty is stored in a separate table, so we have a separate call for - // it. This is not needed for post PoS transition chains. - let td = provider.header_td_by_number(number)?.ok_or(eyre::eyre!("header td not found"))?; - assert!(!td.is_zero()); - // Can query headers by range as well, already sealed! let headers = provider.sealed_headers_range(100..200)?; assert_eq!(headers.len(), 100); diff --git a/examples/node-custom-rpc/src/main.rs b/examples/node-custom-rpc/src/main.rs index 2af789a989..7ab271b4cc 100644 --- a/examples/node-custom-rpc/src/main.rs +++ b/examples/node-custom-rpc/src/main.rs @@ -91,7 +91,7 @@ pub trait TxpoolExtApi { ) -> SubscriptionResult; } -/// The type that implements the `txpool` rpc namespace trait +/// The type that implements the `txpoolExt` rpc namespace trait pub struct TxpoolExt { pool: Pool, } diff --git a/flake.lock b/flake.lock index 704d14161e..fd2bf9ac61 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "crane": { "locked": { - "lastModified": 1754269165, - "narHash": "sha256-0tcS8FHd4QjbCVoxN9jI+PjHgA4vc/IjkUSp+N3zy0U=", + "lastModified": 1760924934, + "narHash": "sha256-tuuqY5aU7cUkR71sO2TraVKK2boYrdW3gCSXUkF4i44=", "owner": "ipetkov", "repo": "crane", - "rev": "444e81206df3f7d92780680e45858e31d2f07a08", + "rev": "c6b4d5308293d0d04fcfeee92705017537cad02f", "type": "github" }, "original": { @@ -23,11 +23,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1755499663, - "narHash": "sha256-OxHGov+A4qR4kpO3e1I3LFR78IAKvDFnWoWsDWvFhKU=", + "lastModified": 1760942671, + "narHash": "sha256-LyO+TwzM7C8TJJkgbqC+BMnPiJX8XHQJmssTWS2Ze9k=", "owner": "nix-community", "repo": "fenix", - "rev": "d1ff4457857ad551e8d6c7c79324b44fac518b8b", + "rev": "b5e669194d67dbd4c659c40bb67476d9285b9a13", "type": "github" }, "original": { @@ -63,11 +63,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1755004716, - "narHash": "sha256-TbhPR5Fqw5LjAeI3/FOPhNNFQCF3cieKCJWWupeZmiA=", + "lastModified": 1760898410, + "narHash": "sha256-bTMk3D0V+6t3qOjXUfWSwjztEuLoAsgtAtqp6/wwfOc=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "b2a58b8c6eff3c3a2c8b5c70dbf69ead78284194", + "rev": "c7e7eb9dc42df01016d795b0fd3a9ae87b7ada1c", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 7550edc31e..512b69e366 100644 --- a/flake.nix +++ b/flake.nix @@ -120,6 +120,11 @@ rustNightly.rustfmt pkgs.cargo-nextest ]; + + # Remove the hardening added by nix to fix jmalloc compilation error. + # More info: https://github.com/tikv/jemallocator/issues/108 + hardeningDisable = [ "fortify" ]; + } overrides); } );