github/reth
1
1
Fork 0
mirror of https://github.com/paradigmxyz/reth.git synced 2026-04-30 03:01:58 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'paradigmxyz:main' into new-approach

Browse Source
This commit is contained in:
Soubhik Singha Mahapatra
2025-10-22 20:35:44 +05:30
committed by GitHub
parent cf8ba329c7 b9f6068f59
commit 2eb460d74a
196 changed files with 1484 additions and 2225 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
.github/CODEOWNERS vendored
View File

@@ -1,12 +1,12 @@
* @gakonst
crates/blockchain-tree-api/ @rakita @rkrasiuk @mattsse @Rjected
crates/blockchain-tree/ @rakita @rkrasiuk @mattsse @Rjected
crates/chain-state/ @fgimenez @mattsse @rkrasiuk
crates/blockchain-tree-api/ @rakita @mattsse @Rjected
crates/blockchain-tree/ @rakita @mattsse @Rjected
crates/chain-state/ @fgimenez @mattsse
crates/chainspec/ @Rjected @joshieDo @mattsse
crates/cli/ @mattsse
crates/consensus/ @rkrasiuk @mattsse @Rjected
crates/consensus/ @mattsse @Rjected
crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez
crates/engine/ @rkrasiuk @mattsse @Rjected @fgimenez @mediocregopher @yongkangc
crates/engine/ @mattsse @Rjected @fgimenez @mediocregopher @yongkangc
crates/era/ @mattsse @RomanHodulak
crates/errors/ @mattsse
crates/ethereum-forks/ @mattsse @Rjected
@@ -15,17 +15,17 @@ crates/etl/ @joshieDo @shekhirin
crates/evm/ @rakita @mattsse @Rjected
crates/exex/ @shekhirin
crates/net/ @mattsse @Rjected
crates/net/downloaders/ @rkrasiuk
crates/net/downloaders/ @Rjected
crates/node/ @mattsse @Rjected @klkvr
crates/optimism/ @mattsse @Rjected @fgimenez
crates/payload/ @mattsse @Rjected
crates/primitives-traits/ @Rjected @RomanHodulak @mattsse @klkvr
crates/primitives/ @Rjected @mattsse @klkvr
crates/prune/ @shekhirin @joshieDo
crates/ress @rkrasiuk
crates/ress @shekhirin @Rjected
crates/revm/ @mattsse @rakita
crates/rpc/ @mattsse @Rjected @RomanHodulak
crates/stages/ @rkrasiuk @shekhirin @mediocregopher
crates/stages/ @shekhirin @mediocregopher
crates/static-file/ @joshieDo @shekhirin
crates/storage/codecs/ @joshieDo
crates/storage/db-api/ @joshieDo @rakita
@@ -35,10 +35,10 @@ crates/storage/errors/ @rakita
crates/storage/libmdbx-rs/ @rakita @shekhirin
crates/storage/nippy-jar/ @joshieDo @shekhirin
crates/storage/provider/ @rakita @joshieDo @shekhirin
crates/storage/storage-api/ @joshieDo @rkrasiuk
crates/storage/storage-api/ @joshieDo
crates/tasks/ @mattsse
crates/tokio-util/ @fgimenez
crates/transaction-pool/ @mattsse @yongkangc
crates/trie/ @rkrasiuk @Rjected @shekhirin @mediocregopher
crates/trie/ @Rjected @shekhirin @mediocregopher
etc/ @Rjected @shekhirin
.github/ @gakonst @DaniPopes

12
Cargo.lock generated
View File

@@ -3993,7 +3993,6 @@ version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"serde",
"typenum",
"version_check",
"zeroize",
@@ -7723,7 +7722,6 @@ dependencies = [
"assert_matches",
"discv5",
"enr",
"generic-array",
"itertools 0.14.0",
"parking_lot",
"rand 0.8.5",
@@ -7935,7 +7933,7 @@ dependencies = [
"reth-optimism-chainspec",
"reth-payload-builder",
"reth-payload-primitives",
"reth-provider",
"reth-storage-api",
"reth-transaction-pool",
"tokio",
"tokio-stream",
@@ -8995,7 +8993,6 @@ dependencies = [
"serde",
"shellexpand",
"strum 0.27.2",
"thiserror 2.0.17",
"tokio",
"toml",
"tracing",
@@ -9327,20 +9324,20 @@ dependencies = [
"futures-util",
"metrics",
"reth-chain-state",
"reth-engine-primitives",
"reth-errors",
"reth-evm",
"reth-execution-types",
"reth-metrics",
"reth-node-api",
"reth-optimism-evm",
"reth-optimism-payload-builder",
"reth-optimism-primitives",
"reth-payload-primitives",
"reth-primitives-traits",
"reth-revm",
"reth-rpc-eth-types",
"reth-storage-api",
"reth-tasks",
"reth-trie",
"ringbuffer",
"serde",
"serde_json",
@@ -9775,14 +9772,12 @@ dependencies = [
name = "reth-prune"
version = "1.8.2"
dependencies = [
"alloy-consensus",
"alloy-eips",
"alloy-primitives",
"assert_matches",
"itertools 0.14.0",
"metrics",
"rayon",
"reth-chainspec",
"reth-config",
"reth-db",
"reth-db-api",
@@ -10671,6 +10666,7 @@ dependencies = [
"alloy-serde",
"alloy-trie",
"arbitrary",
"arrayvec",
"bincode 1.3.3",
"bytes",
"codspeed-criterion-compat",

1
Cargo.toml
View File

@@ -534,6 +534,7 @@ op-alloy-flz = { version = "0.13.1", default-features = false }
# misc
either = { version = "1.15.0", default-features = false }
arrayvec = { version = "0.7.6", default-features = false }
aquamarine = "0.6"
auto_impl = "1"
backon = { version = "1.2", default-features = false, features = ["std-blocking-sleep", "tokio-sleep"] }

5
bin/reth-bench/src/bench/context.rs
View File

@@ -7,6 +7,7 @@ use alloy_primitives::address;
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::JwtSecret;
use alloy_transport::layers::RetryBackoffLayer;
use reqwest::Url;
use reth_node_core::args::BenchmarkArgs;
use tracing::info;
@@ -49,7 +50,9 @@ impl BenchContext {
}
// set up alloy client for blocks
let client = ClientBuilder::default().http(rpc_url.parse()?);
let client = ClientBuilder::default()
.layer(RetryBackoffLayer::new(10, 800, u64::MAX))
.http(rpc_url.parse()?);
let block_provider = RootProvider::<AnyNetwork>::new(client);
// Check if this is an OP chain by checking code at a predeploy address.

24
bin/reth-bench/src/bench/new_payload_fcu.rs
View File

@@ -33,6 +33,16 @@ pub struct Command {
#[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)]
wait_time: Option<Duration>,
/// The size of the block buffer (channel capacity) for prefetching blocks from the RPC
/// endpoint.
#[arg(
long = "rpc-block-buffer-size",
value_name = "RPC_BLOCK_BUFFER_SIZE",
default_value = "20",
verbatim_doc_comment
)]
rpc_block_buffer_size: usize,
#[command(flatten)]
benchmark: BenchmarkArgs,
}
@@ -48,7 +58,12 @@ impl Command {
is_optimism,
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
let (sender, mut receiver) = tokio::sync::mpsc::channel(1000);
let buffer_size = self.rpc_block_buffer_size;
// Use a oneshot channel to propagate errors from the spawned task
let (error_sender, mut error_receiver) = tokio::sync::oneshot::channel();
let (sender, mut receiver) = tokio::sync::mpsc::channel(buffer_size);
tokio::task::spawn(async move {
while benchmark_mode.contains(next_block) {
let block_res = block_provider
@@ -60,6 +75,7 @@ impl Command {
Ok(block) => block,
Err(e) => {
tracing::error!("Failed to fetch block {next_block}: {e}");
let _ = error_sender.send(e);
break;
}
};
@@ -69,6 +85,7 @@ impl Command {
Ok(result) => result,
Err(e) => {
tracing::error!("Failed to convert block to new payload: {e}");
let _ = error_sender.send(e);
break;
}
};
@@ -163,6 +180,11 @@ impl Command {
results.push((gas_row, combined_result));
}
// Check if the spawned task encountered an error
if let Ok(error) = error_receiver.try_recv() {
return Err(error);
}
let (gas_output_results, combined_results): (_, Vec<CombinedResult>) =
results.into_iter().unzip();

47
bin/reth-bench/src/bench/new_payload_only.rs
View File

@@ -13,7 +13,7 @@ use crate::{
use alloy_provider::Provider;
use clap::Parser;
use csv::Writer;
use eyre::Context;
use eyre::{Context, OptionExt};
use reth_cli_runner::CliContext;
use reth_node_core::args::BenchmarkArgs;
use std::time::{Duration, Instant};
@@ -26,6 +26,16 @@ pub struct Command {
#[arg(long, value_name = "RPC_URL", verbatim_doc_comment)]
rpc_url: String,
/// The size of the block buffer (channel capacity) for prefetching blocks from the RPC
/// endpoint.
#[arg(
long = "rpc-block-buffer-size",
value_name = "RPC_BLOCK_BUFFER_SIZE",
default_value = "20",
verbatim_doc_comment
)]
rpc_block_buffer_size: usize,
#[command(flatten)]
benchmark: BenchmarkArgs,
}
@@ -41,7 +51,12 @@ impl Command {
is_optimism,
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
let (sender, mut receiver) = tokio::sync::mpsc::channel(1000);
let buffer_size = self.rpc_block_buffer_size;
// Use a oneshot channel to propagate errors from the spawned task
let (error_sender, mut error_receiver) = tokio::sync::oneshot::channel();
let (sender, mut receiver) = tokio::sync::mpsc::channel(buffer_size);
tokio::task::spawn(async move {
while benchmark_mode.contains(next_block) {
let block_res = block_provider
@@ -49,13 +64,30 @@ impl Command {
.full()
.await
.wrap_err_with(|| format!("Failed to fetch block by number {next_block}"));
let block = block_res.unwrap().unwrap();
let block = match block_res.and_then(|opt| opt.ok_or_eyre("Block not found")) {
Ok(block) => block,
Err(e) => {
tracing::error!("Failed to fetch block {next_block}: {e}");
let _ = error_sender.send(e);
break;
}
};
let header = block.header.clone();
let (version, params) = block_to_new_payload(block, is_optimism).unwrap();
let (version, params) = match block_to_new_payload(block, is_optimism) {
Ok(result) => result,
Err(e) => {
tracing::error!("Failed to convert block to new payload: {e}");
let _ = error_sender.send(e);
break;
}
};
next_block += 1;
sender.send((header, version, params)).await.unwrap();
if let Err(e) = sender.send((header, version, params)).await {
tracing::error!("Failed to send block data: {e}");
break;
}
}
});
@@ -96,6 +128,11 @@ impl Command {
results.push((row, new_payload_result));
}
// Check if the spawned task encountered an error
if let Ok(error) = error_receiver.try_recv() {
return Err(error);
}
let (gas_output_results, new_payload_results): (_, Vec<NewPayloadResult>) =
results.into_iter().unzip();

15
crates/cli/commands/src/common.rs
View File

@@ -24,7 +24,7 @@ use reth_provider::{
providers::{BlockchainProvider, NodeTypesForProvider, StaticFileProvider},
ProviderFactory, StaticFileProviderFactory,
};
use reth_stages::{sets::DefaultStages, Pipeline};
use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget};
use reth_static_file::StaticFileProducer;
use std::{path::PathBuf, sync::Arc};
use tokio::sync::watch;
@@ -126,8 +126,8 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
where
C: ChainSpecParser<ChainSpec = N::ChainSpec>,
{
let prune_modes =
config.prune.as_ref().map(|prune| prune.segments.clone()).unwrap_or_default();
let has_receipt_pruning = config.prune.has_receipts_pruning();
let prune_modes = config.prune.segments.clone();
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>::new(
db,
self.chain.clone(),
@@ -136,8 +136,9 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
.with_prune_modes(prune_modes.clone());
// Check for consistency between database and static files.
if let Some(unwind_target) =
factory.static_file_provider().check_consistency(&factory.provider()?)?
if let Some(unwind_target) = factory
.static_file_provider()
.check_consistency(&factory.provider()?, has_receipt_pruning)?
{
if factory.db_ref().is_read_only()? {
warn!(target: "reth::cli", ?unwind_target, "Inconsistent storage. Restart node to heal.");
@@ -148,7 +149,7 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
// instead.
assert_ne!(
unwind_target,
0,
PipelineTarget::Unwind(0),
"A static file <> database inconsistency was found that would trigger an unwind to block 0"
);
@@ -173,7 +174,7 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
// Move all applicable data from database to static files.
pipeline.move_to_static_files()?;
pipeline.unwind(unwind_target, None)?;
pipeline.unwind(unwind_target.unwind_target().expect("should exist"), None)?;
}
Ok(factory)

4
crates/cli/commands/src/db/repair_trie.rs
View File

@@ -52,7 +52,7 @@ fn verify_only<N: NodeTypesWithDB>(provider_factory: ProviderFactory<N>) -> eyre
// Create the verifier
let hashed_cursor_factory = DatabaseHashedCursorFactory::new(&tx);
let trie_cursor_factory = DatabaseTrieCursorFactory::new(&tx);
let verifier = Verifier::new(trie_cursor_factory, hashed_cursor_factory)?;
let verifier = Verifier::new(&trie_cursor_factory, hashed_cursor_factory)?;
let mut inconsistent_nodes = 0;
let start_time = Instant::now();
@@ -136,7 +136,7 @@ fn verify_and_repair<N: ProviderNodeTypes>(
let trie_cursor_factory = DatabaseTrieCursorFactory::new(tx);
// Create the verifier
let verifier = Verifier::new(trie_cursor_factory, hashed_cursor_factory)?;
let verifier = Verifier::new(&trie_cursor_factory, hashed_cursor_factory)?;
let mut inconsistent_nodes = 0;
let start_time = Instant::now();

4
crates/cli/commands/src/init_state/without_evm.rs
View File

@@ -1,5 +1,5 @@
use alloy_consensus::BlockHeader;
use alloy_primitives::{BlockNumber, B256, U256};
use alloy_primitives::{BlockNumber, B256};
use alloy_rlp::Decodable;
use reth_codecs::Compact;
use reth_node_builder::NodePrimitives;
@@ -133,7 +133,7 @@ where
for block_num in 1..=target_height {
// TODO: should we fill with real parent_hash?
let header = header_factory(block_num);
writer.append_header(&header, U256::ZERO, &B256::ZERO)?;
writer.append_header(&header, &B256::ZERO)?;
}
Ok(())
});

13
crates/cli/commands/src/prune.rs
View File

@@ -1,5 +1,5 @@
//! Command that runs pruning without any limits.
use crate::common::{AccessRights, CliNodeTypes, Environment, EnvironmentArgs};
use crate::common::{AccessRights, CliNodeTypes, EnvironmentArgs};
use clap::Parser;
use reth_chainspec::{EthChainSpec, EthereumHardforks};
use reth_cli::chainspec::ChainSpecParser;
@@ -18,22 +18,23 @@ pub struct PruneCommand<C: ChainSpecParser> {
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneCommand<C> {
/// Execute the `prune` command
pub async fn execute<N: CliNodeTypes<ChainSpec = C::ChainSpec>>(self) -> eyre::Result<()> {
let Environment { config, provider_factory, .. } = self.env.init::<N>(AccessRights::RW)?;
let prune_config = config.prune.unwrap_or_default();
let env = self.env.init::<N>(AccessRights::RW)?;
let provider_factory = env.provider_factory;
let config = env.config.prune;
// Copy data from database to static files
info!(target: "reth::cli", "Copying data from database to static files...");
let static_file_producer =
StaticFileProducer::new(provider_factory.clone(), prune_config.segments.clone());
StaticFileProducer::new(provider_factory.clone(), config.segments.clone());
let lowest_static_file_height =
static_file_producer.lock().copy_to_static_files()?.min_block_num();
info!(target: "reth::cli", ?lowest_static_file_height, "Copied data from database to static files");
// Delete data which has been copied to static files.
if let Some(prune_tip) = lowest_static_file_height {
info!(target: "reth::cli", ?prune_tip, ?prune_config, "Pruning data from database...");
info!(target: "reth::cli", ?prune_tip, ?config, "Pruning data from database...");
// Run the pruner according to the configuration, and don't enforce any limits on it
let mut pruner = PrunerBuilder::new(prune_config)
let mut pruner = PrunerBuilder::new(config)
.delete_limit(usize::MAX)
.build_with_provider_factory(provider_factory);

1
crates/cli/commands/src/stage/drop.rs
View File

@@ -70,7 +70,6 @@ impl<C: ChainSpecParser> Command<C> {
StageEnum::Headers => {
tx.clear::<tables::CanonicalHeaders>()?;
tx.clear::<tables::Headers<HeaderTy<N>>>()?;
tx.clear::<tables::HeaderTerminalDifficulties>()?;
tx.clear::<tables::HeaderNumbers>()?;
reset_stage_checkpoint(tx, StageId::Headers)?;

7
crates/cli/commands/src/stage/dump/execution.rs
View File

@@ -69,13 +69,6 @@ fn import_tables_with_range<N: NodeTypesWithDB>(
to,
)
})??;
output_db.update(|tx| {
tx.import_table_with_range::<tables::HeaderTerminalDifficulties, _>(
&db_tool.provider_factory.db_ref().tx()?,
Some(from),
to,
)
})??;
output_db.update(|tx| {
tx.import_table_with_range::<tables::Headers, _>(
&db_tool.provider_factory.db_ref().tx()?,

2
crates/cli/commands/src/stage/run.rs
View File

@@ -151,7 +151,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
let batch_size = self.batch_size.unwrap_or(self.to.saturating_sub(self.from) + 1);
let etl_config = config.stages.etl.clone();
let prune_modes = config.prune.clone().map(|prune| prune.segments).unwrap_or_default();
let prune_modes = config.prune.segments.clone();
let (mut exec_stage, mut unwind_stage): (Box<dyn Stage<_>>, Option<Box<dyn Stage<_>>>) =
match self.stage {

2
crates/cli/commands/src/stage/unwind.rs
View File

@@ -85,7 +85,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
evm_config: impl ConfigureEvm<Primitives = N::Primitives> + 'static,
) -> Result<Pipeline<N>, eyre::Error> {
let stage_conf = &config.stages;
let prune_modes = config.prune.clone().map(|prune| prune.segments).unwrap_or_default();
let prune_modes = config.prune.segments.clone();
let (tip_tx, tip_rx) = watch::channel(B256::ZERO);

1
crates/cli/commands/src/test_vectors/tables.rs
View File

@@ -69,7 +69,6 @@ pub fn generate_vectors(mut tables: Vec<String>) -> Result<()> {
generate!([
(CanonicalHeaders, PER_TABLE, TABLE),
(HeaderTerminalDifficulties, PER_TABLE, TABLE),
(HeaderNumbers, PER_TABLE, TABLE),
(Headers<Header>, PER_TABLE, TABLE),
(BlockBodyIndices, PER_TABLE, TABLE),

48
crates/config/src/config.rs
View File

@@ -23,8 +23,8 @@ pub struct Config {
// TODO(onbjerg): Can we make this easier to maintain when we add/remove stages?
pub stages: StageConfig,
/// Configuration for pruning.
#[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))]
pub prune: Option<PruneConfig>,
#[cfg_attr(feature = "serde", serde(default))]
pub prune: PruneConfig,
/// Configuration for the discovery service.
pub peers: PeersConfig,
/// Configuration for peer sessions.
@@ -33,8 +33,8 @@ pub struct Config {
impl Config {
/// Sets the pruning configuration.
pub fn update_prune_config(&mut self, prune_config: PruneConfig) {
self.prune = Some(prune_config);
pub const fn set_prune_config(&mut self, prune_config: PruneConfig) {
self.prune = prune_config;
}
}
@@ -445,15 +445,20 @@ impl Default for PruneConfig {
}
impl PruneConfig {
/// Returns whether this configuration is the default one.
pub fn is_default(&self) -> bool {
self == &Self::default()
}
/// Returns whether there is any kind of receipt pruning configuration.
pub fn has_receipts_pruning(&self) -> bool {
self.segments.receipts.is_some() || !self.segments.receipts_log_filter.is_empty()
pub const fn has_receipts_pruning(&self) -> bool {
self.segments.receipts.is_some()
}
/// Merges another `PruneConfig` into this one, taking values from the other config if and only
/// if the corresponding value in this config is not set.
pub fn merge(&mut self, other: Option<Self>) {
let Some(other) = other else { return };
pub fn merge(&mut self, other: Self) {
#[expect(deprecated)]
let Self {
block_interval,
segments:
@@ -465,7 +470,7 @@ impl PruneConfig {
storage_history,
bodies_history,
merkle_changesets,
receipts_log_filter,
receipts_log_filter: (),
},
} = other;
@@ -483,10 +488,6 @@ impl PruneConfig {
self.segments.bodies_history = self.segments.bodies_history.or(bodies_history);
// Merkle changesets is not optional, so we just replace it if provided
self.segments.merkle_changesets = merkle_changesets;
if self.segments.receipts_log_filter.0.is_empty() && !receipts_log_filter.0.is_empty() {
self.segments.receipts_log_filter = receipts_log_filter;
}
}
}
@@ -513,10 +514,9 @@ where
mod tests {
use super::{Config, EXTENSION};
use crate::PruneConfig;
use alloy_primitives::Address;
use reth_network_peers::TrustedPeer;
use reth_prune_types::{PruneMode, PruneModes, ReceiptsLogPruneConfig};
use std::{collections::BTreeMap, path::Path, str::FromStr, time::Duration};
use reth_prune_types::{PruneMode, PruneModes};
use std::{path::Path, str::FromStr, time::Duration};
fn with_tempdir(filename: &str, proc: fn(&std::path::Path)) {
let temp_dir = tempfile::tempdir().unwrap();
@@ -1005,10 +1005,8 @@ receipts = 'full'
storage_history: Some(PruneMode::Before(5000)),
bodies_history: None,
merkle_changesets: PruneMode::Before(0),
receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([(
Address::random(),
PruneMode::Full,
)])),
#[expect(deprecated)]
receipts_log_filter: (),
},
};
@@ -1022,15 +1020,12 @@ receipts = 'full'
storage_history: Some(PruneMode::Distance(3000)),
bodies_history: None,
merkle_changesets: PruneMode::Distance(10000),
receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([
(Address::random(), PruneMode::Distance(1000)),
(Address::random(), PruneMode::Before(2000)),
])),
#[expect(deprecated)]
receipts_log_filter: (),
},
};
let original_filter = config1.segments.receipts_log_filter.clone();
config1.merge(Some(config2));
config1.merge(config2);
// Check that the configuration has been merged. Any configuration present in config1
// should not be overwritten by config2
@@ -1041,7 +1036,6 @@ receipts = 'full'
assert_eq!(config1.segments.account_history, Some(PruneMode::Distance(2000)));
assert_eq!(config1.segments.storage_history, Some(PruneMode::Before(5000)));
assert_eq!(config1.segments.merkle_changesets, PruneMode::Distance(10000));
assert_eq!(config1.segments.receipts_log_filter, original_filter);
}
#[test]

141
crates/e2e-test-utils/src/lib.rs
View File

@@ -1,23 +1,19 @@
//! Utilities for end-to-end tests.
use node::NodeTestContext;
use reth_chainspec::{ChainSpec, EthChainSpec};
use reth_chainspec::ChainSpec;
use reth_db::{test_utils::TempDatabase, DatabaseEnv};
use reth_engine_local::LocalPayloadAttributesBuilder;
use reth_network_api::test_utils::PeersHandleProvider;
use reth_node_builder::{
components::NodeComponentsBuilder,
rpc::{EngineValidatorAddOn, RethRpcAddOns},
EngineNodeLauncher, FullNodeTypesAdapter, Node, NodeAdapter, NodeBuilder, NodeComponents,
NodeConfig, NodeHandle, NodePrimitives, NodeTypes, NodeTypesWithDBAdapter,
PayloadAttributesBuilder, PayloadTypes,
FullNodeTypesAdapter, Node, NodeAdapter, NodeComponents, NodePrimitives, NodeTypes,
NodeTypesWithDBAdapter, PayloadAttributesBuilder, PayloadTypes,
};
use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs};
use reth_provider::providers::{BlockchainProvider, NodeTypesForProvider};
use reth_rpc_server_types::RpcModuleSelection;
use reth_tasks::TaskManager;
use std::sync::Arc;
use tracing::{span, Level};
use wallet::Wallet;
/// Wrapper type to create test nodes
@@ -45,6 +41,10 @@ mod rpc;
/// Utilities for creating and writing RLP test data
pub mod test_rlp_utils;
/// Builder for configuring test node setups
mod setup_builder;
pub use setup_builder::E2ETestSetupBuilder;
/// Creates the initial setup with `num_nodes` started and interconnected.
pub async fn setup<N>(
num_nodes: usize,
@@ -53,60 +53,14 @@ pub async fn setup<N>(
attributes_generator: impl Fn(u64) -> <<N as NodeTypes>::Payload as PayloadTypes>::PayloadBuilderAttributes + Send + Sync + Copy + 'static,
) -> eyre::Result<(Vec<NodeHelperType<N>>, TaskManager, Wallet)>
where
N: Default + Node<TmpNodeAdapter<N>> + NodeTypesForProvider,
N::ComponentsBuilder: NodeComponentsBuilder<
TmpNodeAdapter<N>,
Components: NodeComponents<TmpNodeAdapter<N>, Network: PeersHandleProvider>,
>,
N::AddOns: RethRpcAddOns<Adapter<N>> + EngineValidatorAddOn<Adapter<N>>,
N: NodeBuilderHelper,
LocalPayloadAttributesBuilder<N::ChainSpec>:
PayloadAttributesBuilder<<<N as NodeTypes>::Payload as PayloadTypes>::PayloadAttributes>,
{
let tasks = TaskManager::current();
let exec = tasks.executor();
let network_config = NetworkArgs {
discovery: DiscoveryArgs { disable_discovery: true, ..DiscoveryArgs::default() },
..NetworkArgs::default()
};
// Create nodes and peer them
let mut nodes: Vec<NodeTestContext<_, _>> = Vec::with_capacity(num_nodes);
for idx in 0..num_nodes {
let node_config = NodeConfig::new(chain_spec.clone())
.with_network(network_config.clone())
.with_unused_ports()
.with_rpc(RpcServerArgs::default().with_unused_ports().with_http())
.set_dev(is_dev);
let span = span!(Level::INFO, "node", idx);
let _enter = span.enter();
let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone())
.testing_node(exec.clone())
.node(Default::default())
.launch()
.await?;
let mut node = NodeTestContext::new(node, attributes_generator).await?;
// Connect each node in a chain.
if let Some(previous_node) = nodes.last_mut() {
previous_node.connect(&mut node).await;
}
// Connect last node with the first if there are more than two
if idx + 1 == num_nodes &&
num_nodes > 2 &&
let Some(first_node) = nodes.first_mut()
{
node.connect(first_node).await;
}
nodes.push(node);
}
Ok((nodes, tasks, Wallet::default().with_chain_id(chain_spec.chain().into())))
E2ETestSetupBuilder::new(num_nodes, chain_spec, attributes_generator)
.with_node_config_modifier(move |config| config.set_dev(is_dev))
.build()
.await
}
/// Creates the initial setup with `num_nodes` started and interconnected.
@@ -155,71 +109,12 @@ where
LocalPayloadAttributesBuilder<N::ChainSpec>:
PayloadAttributesBuilder<<N::Payload as PayloadTypes>::PayloadAttributes>,
{
let tasks = TaskManager::current();
let exec = tasks.executor();
let network_config = NetworkArgs {
discovery: DiscoveryArgs { disable_discovery: true, ..DiscoveryArgs::default() },
..NetworkArgs::default()
};
// Create nodes and peer them
let mut nodes: Vec<NodeTestContext<_, _>> = Vec::with_capacity(num_nodes);
for idx in 0..num_nodes {
let node_config = NodeConfig::new(chain_spec.clone())
.with_network(network_config.clone())
.with_unused_ports()
.with_rpc(
RpcServerArgs::default()
.with_unused_ports()
.with_http()
.with_http_api(RpcModuleSelection::All),
)
.set_dev(is_dev);
let span = span!(Level::INFO, "node", idx);
let _enter = span.enter();
let node = N::default();
let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone())
.testing_node(exec.clone())
.with_types_and_provider::<N, BlockchainProvider<_>>()
.with_components(node.components_builder())
.with_add_ons(node.add_ons())
.launch_with_fn(|builder| {
let launcher = EngineNodeLauncher::new(
builder.task_executor().clone(),
builder.config().datadir(),
tree_config.clone(),
);
builder.launch_with(launcher)
})
.await?;
let mut node = NodeTestContext::new(node, attributes_generator).await?;
let genesis = node.block_hash(0);
node.update_forkchoice(genesis, genesis).await?;
// Connect each node in a chain if requested.
if connect_nodes {
if let Some(previous_node) = nodes.last_mut() {
previous_node.connect(&mut node).await;
}
// Connect last node with the first if there are more than two
if idx + 1 == num_nodes &&
num_nodes > 2 &&
let Some(first_node) = nodes.first_mut()
{
node.connect(first_node).await;
}
}
nodes.push(node);
}
Ok((nodes, tasks, Wallet::default().with_chain_id(chain_spec.chain().into())))
E2ETestSetupBuilder::new(num_nodes, chain_spec, attributes_generator)
.with_tree_config_modifier(move |_| tree_config.clone())
.with_node_config_modifier(move |config| config.set_dev(is_dev))
.with_connect_nodes(connect_nodes)
.build()
.await
}
// Type aliases

5
crates/e2e-test-utils/src/payload.rs
View File

@@ -57,8 +57,9 @@ impl<T: PayloadTypes> PayloadTestContext<T> {
/// Wait until the best built payload is ready
pub async fn wait_for_built_payload(&self, payload_id: PayloadId) {
loop {
let payload = self.payload_builder.best_payload(payload_id).await.unwrap().unwrap();
if payload.block().body().transactions().is_empty() {
let payload =
self.payload_builder.best_payload(payload_id).await.transpose().ok().flatten();
if payload.is_none_or(|p| p.block().body().transactions().is_empty()) {
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
continue
}

210
crates/e2e-test-utils/src/setup_builder.rs Normal file
View File

@@ -0,0 +1,210 @@
//! Builder for configuring and creating test node setups.
//!
//! This module provides a flexible builder API for setting up test nodes with custom
//! configurations through closures that modify `NodeConfig` and `TreeConfig`.
use crate::{node::NodeTestContext, wallet::Wallet, NodeBuilderHelper, NodeHelperType, TmpDB};
use reth_chainspec::EthChainSpec;
use reth_engine_local::LocalPayloadAttributesBuilder;
use reth_node_builder::{
EngineNodeLauncher, NodeBuilder, NodeConfig, NodeHandle, NodeTypes, NodeTypesWithDBAdapter,
PayloadAttributesBuilder, PayloadTypes,
};
use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs};
use reth_provider::providers::BlockchainProvider;
use reth_rpc_server_types::RpcModuleSelection;
use reth_tasks::TaskManager;
use std::sync::Arc;
use tracing::{span, Level};
/// Type alias for tree config modifier closure
type TreeConfigModifier =
Box<dyn Fn(reth_node_api::TreeConfig) -> reth_node_api::TreeConfig + Send + Sync>;
/// Type alias for node config modifier closure
type NodeConfigModifier<C> = Box<dyn Fn(NodeConfig<C>) -> NodeConfig<C> + Send + Sync>;
/// Builder for configuring and creating test node setups.
///
/// This builder allows customizing test node configurations through closures that
/// modify `NodeConfig` and `TreeConfig`. It avoids code duplication by centralizing
/// the node creation logic.
pub struct E2ETestSetupBuilder<N, F>
where
N: NodeBuilderHelper,
F: Fn(u64) -> <<N as NodeTypes>::Payload as PayloadTypes>::PayloadBuilderAttributes
+ Send
+ Sync
+ Copy
+ 'static,
LocalPayloadAttributesBuilder<N::ChainSpec>:
PayloadAttributesBuilder<<N::Payload as PayloadTypes>::PayloadAttributes>,
{
num_nodes: usize,
chain_spec: Arc<N::ChainSpec>,
attributes_generator: F,
connect_nodes: bool,
tree_config_modifier: Option<TreeConfigModifier>,
node_config_modifier: Option<NodeConfigModifier<N::ChainSpec>>,
}
impl<N, F> E2ETestSetupBuilder<N, F>
where
N: NodeBuilderHelper,
F: Fn(u64) -> <<N as NodeTypes>::Payload as PayloadTypes>::PayloadBuilderAttributes
+ Send
+ Sync
+ Copy
+ 'static,
LocalPayloadAttributesBuilder<N::ChainSpec>:
PayloadAttributesBuilder<<N::Payload as PayloadTypes>::PayloadAttributes>,
{
/// Creates a new builder with the required parameters.
pub fn new(num_nodes: usize, chain_spec: Arc<N::ChainSpec>, attributes_generator: F) -> Self {
Self {
num_nodes,
chain_spec,
attributes_generator,
connect_nodes: true,
tree_config_modifier: None,
node_config_modifier: None,
}
}
/// Sets whether nodes should be interconnected (default: true).
pub const fn with_connect_nodes(mut self, connect_nodes: bool) -> Self {
self.connect_nodes = connect_nodes;
self
}
/// Sets a modifier function for the tree configuration.
///
/// The closure receives the base tree config and returns a modified version.
pub fn with_tree_config_modifier<G>(mut self, modifier: G) -> Self
where
G: Fn(reth_node_api::TreeConfig) -> reth_node_api::TreeConfig + Send + Sync + 'static,
{
self.tree_config_modifier = Some(Box::new(modifier));
self
}
/// Sets a modifier function for the node configuration.
///
/// The closure receives the base node config and returns a modified version.
pub fn with_node_config_modifier<G>(mut self, modifier: G) -> Self
where
G: Fn(NodeConfig<N::ChainSpec>) -> NodeConfig<N::ChainSpec> + Send + Sync + 'static,
{
self.node_config_modifier = Some(Box::new(modifier));
self
}
/// Builds and launches the test nodes.
pub async fn build(
self,
) -> eyre::Result<(
Vec<NodeHelperType<N, BlockchainProvider<NodeTypesWithDBAdapter<N, TmpDB>>>>,
TaskManager,
Wallet,
)> {
let tasks = TaskManager::current();
let exec = tasks.executor();
let network_config = NetworkArgs {
discovery: DiscoveryArgs { disable_discovery: true, ..DiscoveryArgs::default() },
..NetworkArgs::default()
};
// Apply tree config modifier if present
let tree_config = if let Some(modifier) = self.tree_config_modifier {
modifier(reth_node_api::TreeConfig::default())
} else {
reth_node_api::TreeConfig::default()
};
let mut nodes: Vec<NodeTestContext<_, _>> = Vec::with_capacity(self.num_nodes);
for idx in 0..self.num_nodes {
// Create base node config
let base_config = NodeConfig::new(self.chain_spec.clone())
.with_network(network_config.clone())
.with_unused_ports()
.with_rpc(
RpcServerArgs::default()
.with_unused_ports()
.with_http()
.with_http_api(RpcModuleSelection::All),
);
// Apply node config modifier if present
let node_config = if let Some(modifier) = &self.node_config_modifier {
modifier(base_config)
} else {
base_config
};
let span = span!(Level::INFO, "node", idx);
let _enter = span.enter();
let node = N::default();
let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config)
.testing_node(exec.clone())
.with_types_and_provider::<N, BlockchainProvider<_>>()
.with_components(node.components_builder())
.with_add_ons(node.add_ons())
.launch_with_fn(|builder| {
let launcher = EngineNodeLauncher::new(
builder.task_executor().clone(),
builder.config().datadir(),
tree_config.clone(),
);
builder.launch_with(launcher)
})
.await?;
let mut node = NodeTestContext::new(node, self.attributes_generator).await?;
let genesis = node.block_hash(0);
node.update_forkchoice(genesis, genesis).await?;
// Connect nodes if requested
if self.connect_nodes {
if let Some(previous_node) = nodes.last_mut() {
previous_node.connect(&mut node).await;
}
// Connect last node with the first if there are more than two
if idx + 1 == self.num_nodes &&
self.num_nodes > 2 &&
let Some(first_node) = nodes.first_mut()
{
node.connect(first_node).await;
}
}
nodes.push(node);
}
Ok((nodes, tasks, Wallet::default().with_chain_id(self.chain_spec.chain().into())))
}
}
impl<N, F> std::fmt::Debug for E2ETestSetupBuilder<N, F>
where
N: NodeBuilderHelper,
F: Fn(u64) -> <<N as NodeTypes>::Payload as PayloadTypes>::PayloadBuilderAttributes
+ Send
+ Sync
+ Copy
+ 'static,
LocalPayloadAttributesBuilder<N::ChainSpec>:
PayloadAttributesBuilder<<N::Payload as PayloadTypes>::PayloadAttributes>,
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("E2ETestSetupBuilder")
.field("num_nodes", &self.num_nodes)
.field("connect_nodes", &self.connect_nodes)
.field("tree_config_modifier", &self.tree_config_modifier.as_ref().map(|_| "<closure>"))
.field("node_config_modifier", &self.node_config_modifier.as_ref().map(|_| "<closure>"))
.finish_non_exhaustive()
}
}

1
crates/e2e-test-utils/src/test_rlp_utils.rs
View File

@@ -159,7 +159,6 @@ pub fn write_blocks_to_rlp(blocks: &[SealedBlock], path: &Path) -> std::io::Resu
);
// Debug: check what's in the encoded data
debug!(target: "e2e::import", "Block {} encoded to {} bytes", i, buf.len());
if buf.len() < 20 {
debug!(target: "e2e::import", " Raw bytes: {:?}", &buf);
} else {

37
crates/e2e-test-utils/tests/e2e-testsuite/main.rs
View File

@@ -15,9 +15,11 @@ use reth_e2e_test_utils::{
setup::{NetworkSetup, Setup},
Environment, TestBuilder,
},
E2ETestSetupBuilder,
};
use reth_node_api::TreeConfig;
use reth_node_ethereum::{EthEngineTypes, EthereumNode};
use reth_payload_builder::EthPayloadBuilderAttributes;
use std::sync::Arc;
use tempfile::TempDir;
use tracing::debug;
@@ -349,3 +351,38 @@ async fn test_testsuite_multinode_block_production() -> Result<()> {
Ok(())
}
#[tokio::test]
async fn test_setup_builder_with_custom_tree_config() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(
serde_json::from_str(include_str!(
"../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json"
))
.unwrap(),
)
.cancun_activated()
.build(),
);
let (nodes, _tasks, _wallet) =
E2ETestSetupBuilder::<EthereumNode, _>::new(1, chain_spec, |_| {
EthPayloadBuilderAttributes::default()
})
.with_tree_config_modifier(|config| {
config.with_persistence_threshold(0).with_memory_block_buffer_target(5)
})
.build()
.await?;
assert_eq!(nodes.len(), 1);
let genesis_hash = nodes[0].block_hash(0);
assert_ne!(genesis_hash, B256::ZERO);
Ok(())
}

4
crates/engine/local/Cargo.toml
View File

@@ -11,11 +11,11 @@ exclude.workspace = true
[dependencies]
# reth
reth-chainspec.workspace = true
reth-engine-primitives.workspace = true
reth-engine-primitives = { workspace = true, features = ["std"] }
reth-ethereum-engine-primitives.workspace = true
reth-payload-builder.workspace = true
reth-payload-primitives.workspace = true
reth-provider.workspace = true
reth-storage-api.workspace = true
reth-transaction-pool.workspace = true
# alloy

2
crates/engine/local/src/miner.rs
View File

@@ -10,7 +10,7 @@ use reth_payload_builder::PayloadBuilderHandle;
use reth_payload_primitives::{
BuiltPayload, EngineApiMessageVersion, PayloadAttributesBuilder, PayloadKind, PayloadTypes,
};
use reth_provider::BlockReader;
use reth_storage_api::BlockReader;
use reth_transaction_pool::TransactionPool;
use std::{
collections::VecDeque,

24
crates/engine/primitives/src/config.rs
View File

@@ -6,9 +6,6 @@ pub const DEFAULT_PERSISTENCE_THRESHOLD: u64 = 2;
/// How close to the canonical head we persist blocks.
pub const DEFAULT_MEMORY_BLOCK_BUFFER_TARGET: u64 = 0;
/// Default maximum concurrency for on-demand proof tasks (blinded nodes)
pub const DEFAULT_MAX_PROOF_TASK_CONCURRENCY: u64 = 256;
/// Minimum number of workers we allow configuring explicitly.
pub const MIN_WORKER_COUNT: usize = 32;
@@ -102,8 +99,6 @@ pub struct TreeConfig {
cross_block_cache_size: u64,
/// Whether the host has enough parallelism to run state root task.
has_enough_parallelism: bool,
/// Maximum number of concurrent proof tasks
max_proof_task_concurrency: u64,
/// Whether multiproof task should chunk proof targets.
multiproof_chunking_enabled: bool,
/// Multiproof task chunk size for proof targets.
@@ -153,7 +148,6 @@ impl Default for TreeConfig {
state_provider_metrics: false,
cross_block_cache_size: DEFAULT_CROSS_BLOCK_CACHE_SIZE,
has_enough_parallelism: has_enough_parallelism(),
max_proof_task_concurrency: DEFAULT_MAX_PROOF_TASK_CONCURRENCY,
multiproof_chunking_enabled: true,
multiproof_chunk_size: DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE,
reserved_cpu_cores: DEFAULT_RESERVED_CPU_CORES,
@@ -184,7 +178,6 @@ impl TreeConfig {
state_provider_metrics: bool,
cross_block_cache_size: u64,
has_enough_parallelism: bool,
max_proof_task_concurrency: u64,
multiproof_chunking_enabled: bool,
multiproof_chunk_size: usize,
reserved_cpu_cores: usize,
@@ -196,7 +189,6 @@ impl TreeConfig {
storage_worker_count: usize,
account_worker_count: usize,
) -> Self {
assert!(max_proof_task_concurrency > 0, "max_proof_task_concurrency must be at least 1");
Self {
persistence_threshold,
memory_block_buffer_target,
@@ -210,7 +202,6 @@ impl TreeConfig {
state_provider_metrics,
cross_block_cache_size,
has_enough_parallelism,
max_proof_task_concurrency,
multiproof_chunking_enabled,
multiproof_chunk_size,
reserved_cpu_cores,
@@ -249,11 +240,6 @@ impl TreeConfig {
self.max_execute_block_batch_size
}
/// Return the maximum proof task concurrency.
pub const fn max_proof_task_concurrency(&self) -> u64 {
self.max_proof_task_concurrency
}
/// Return whether the multiproof task chunking is enabled.
pub const fn multiproof_chunking_enabled(&self) -> bool {
self.multiproof_chunking_enabled
@@ -420,16 +406,6 @@ impl TreeConfig {
self
}
/// Setter for maximum number of concurrent proof tasks.
pub const fn with_max_proof_task_concurrency(
mut self,
max_proof_task_concurrency: u64,
) -> Self {
assert!(max_proof_task_concurrency > 0, "max_proof_task_concurrency must be at least 1");
self.max_proof_task_concurrency = max_proof_task_concurrency;
self
}
/// Setter for whether multiproof task should chunk proof targets.
pub const fn with_multiproof_chunking_enabled(
mut self,

2
crates/engine/tree/src/chain.rs
View File

@@ -71,7 +71,7 @@ where
/// Internal function used to advance the chain.
///
/// Polls the `ChainOrchestrator` for the next event.
#[tracing::instrument(name = "ChainOrchestrator::poll", skip(self, cx))]
#[tracing::instrument(level = "debug", target = "engine::tree::chain_orchestrator", skip_all)]
fn poll_next_event(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<ChainEvent<T::Event>> {
let this = self.get_mut();

4
crates/engine/tree/src/tree/cached_state.rs
View File

@@ -354,7 +354,7 @@ impl ExecutionCache {
}
/// Invalidates the storage for all addresses in the set
#[instrument(level = "debug", target = "engine::tree", skip_all, fields(accounts = addresses.len()))]
#[instrument(level = "debug", target = "engine::caching", skip_all, fields(accounts = addresses.len()))]
pub(crate) fn invalidate_storages(&self, addresses: HashSet<&Address>) {
// NOTE: this must collect because the invalidate function should not be called while we
// hold an iter for it
@@ -386,7 +386,7 @@ impl ExecutionCache {
/// ## Error Handling
///
/// Returns an error if the state updates are inconsistent and should be discarded.
#[instrument(level = "debug", target = "engine::tree", skip_all)]
#[instrument(level = "debug", target = "engine::caching", skip_all)]
pub(crate) fn insert_state(&self, state_updates: &BundleState) -> Result<(), ()> {
let _enter =
debug_span!(target: "engine::tree", "contracts", len = state_updates.contracts.len())

26
crates/engine/tree/src/tree/payload_processor/mod.rs
View File

@@ -40,10 +40,13 @@ use reth_trie_sparse::{
ClearedSparseStateTrie, SparseStateTrie, SparseTrie,
};
use reth_trie_sparse_parallel::{ParallelSparseTrie, ParallelismThresholds};
use std::sync::{
atomic::AtomicBool,
mpsc::{self, channel, Sender},
Arc,
use std::{
sync::{
atomic::AtomicBool,
mpsc::{self, channel, Sender},
Arc,
},
time::Instant,
};
use tracing::{debug, debug_span, instrument, warn};
@@ -193,6 +196,7 @@ where
+ Clone
+ 'static,
{
let span = tracing::Span::current();
let (to_sparse_trie, sparse_trie_rx) = channel();
// spawn multiproof task, save the trie input
let (trie_input, state_root_config) = MultiProofConfig::from_input(trie_input);
@@ -206,7 +210,6 @@ where
);
let storage_worker_count = config.storage_worker_count();
let account_worker_count = config.account_worker_count();
let max_proof_task_concurrency = config.max_proof_task_concurrency() as usize;
let proof_handle = ProofWorkerHandle::new(
self.executor.handle().clone(),
consistent_view,
@@ -215,15 +218,11 @@ where
account_worker_count,
);
// We set it to half of the proof task concurrency, because often for each multiproof we
// spawn one Tokio task for the account proof, and one Tokio task for the storage proof.
let max_multi_proof_task_concurrency = max_proof_task_concurrency / 2;
let multi_proof_task = MultiProofTask::new(
state_root_config,
self.executor.clone(),
proof_handle.clone(),
to_sparse_trie,
max_multi_proof_task_concurrency,
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size()),
);
@@ -242,7 +241,6 @@ where
);
// spawn multi-proof task
let span = tracing::Span::current();
self.executor.spawn_blocking(move || {
let _enter = span.entered();
multi_proof_task.run();
@@ -601,8 +599,16 @@ impl ExecutionCache {
/// A cache is considered available when:
/// - It exists and matches the requested parent hash
/// - No other tasks are currently using it (checked via Arc reference count)
#[instrument(level = "debug", target = "engine::tree::payload_processor", skip(self))]
pub(crate) fn get_cache_for(&self, parent_hash: B256) -> Option<SavedCache> {
let start = Instant::now();
let cache = self.inner.read();
let elapsed = start.elapsed();
if elapsed.as_millis() > 5 {
warn!(blocked_for=?elapsed, "Blocked waiting for execution cache mutex");
}
cache
.as_ref()
.filter(|c| c.executed_block_hash() == parent_hash && c.is_available())

37
crates/engine/tree/src/tree/payload_processor/multiproof.rs
View File

@@ -34,6 +34,10 @@ use std::{
};
use tracing::{debug, error, instrument, trace};
/// Default upper bound for inflight multiproof calculations. These would be sitting in the queue
/// waiting to be processed.
const DEFAULT_MULTIPROOF_INFLIGHT_LIMIT: usize = 128;
/// A trie update that can be applied to sparse trie alongside the proofs for touched parts of the
/// state.
#[derive(Default, Debug)]
@@ -338,8 +342,8 @@ impl MultiproofInput {
/// availability has been signaled.
#[derive(Debug)]
pub struct MultiproofManager {
/// Maximum number of concurrent calculations.
max_concurrent: usize,
/// Maximum number of proof calculations allowed to be inflight at once.
inflight_limit: usize,
/// Currently running calculations.
inflight: usize,
/// Queued calculations.
@@ -370,11 +374,10 @@ impl MultiproofManager {
executor: WorkloadExecutor,
metrics: MultiProofTaskMetrics,
proof_worker_handle: ProofWorkerHandle,
max_concurrent: usize,
) -> Self {
Self {
pending: VecDeque::with_capacity(max_concurrent),
max_concurrent,
pending: VecDeque::with_capacity(DEFAULT_MULTIPROOF_INFLIGHT_LIMIT),
inflight_limit: DEFAULT_MULTIPROOF_INFLIGHT_LIMIT,
executor,
inflight: 0,
metrics,
@@ -384,11 +387,10 @@ impl MultiproofManager {
}
const fn is_full(&self) -> bool {
self.inflight >= self.max_concurrent
self.inflight >= self.inflight_limit
}
/// Spawns a new multiproof calculation or enqueues it for later if
/// `max_concurrent` are already inflight.
/// Spawns a new multiproof calculation or enqueues it if the inflight limit is reached.
fn spawn_or_queue(&mut self, input: PendingMultiproofTask) {
// If there are no proof targets, we can just send an empty multiproof back immediately
if input.proof_targets_is_empty() {
@@ -685,7 +687,6 @@ impl MultiProofTask {
executor: WorkloadExecutor,
proof_worker_handle: ProofWorkerHandle,
to_sparse_trie: Sender<SparseTrieUpdate>,
max_concurrency: usize,
chunk_size: Option<usize>,
) -> Self {
let (tx, rx) = channel();
@@ -704,7 +705,6 @@ impl MultiProofTask {
executor,
metrics.clone(),
proof_worker_handle,
max_concurrency,
),
metrics,
}
@@ -780,7 +780,7 @@ impl MultiProofTask {
let all_proofs_processed =
proofs_processed >= state_update_proofs_requested + prefetch_proofs_requested;
let no_pending = !self.proof_sequencer.has_pending();
debug!(
trace!(
target: "engine::root",
proofs_processed,
state_update_proofs_requested,
@@ -975,7 +975,12 @@ impl MultiProofTask {
/// currently being calculated, or if there are any pending proofs in the proof sequencer
/// left to be revealed by checking the pending tasks.
/// 6. This task exits after all pending proofs are processed.
#[instrument(level = "debug", target = "engine::tree::payload_processor::multiproof", skip_all)]
#[instrument(
level = "debug",
name = "MultiProofTask::run",
target = "engine::tree::payload_processor::multiproof",
skip_all
)]
pub(crate) fn run(mut self) {
// TODO convert those into fields
let mut prefetch_proofs_requested = 0;
@@ -1011,7 +1016,7 @@ impl MultiProofTask {
let storage_targets =
targets.values().map(|slots| slots.len()).sum::<usize>();
prefetch_proofs_requested += self.on_prefetch_proof(targets);
debug!(
trace!(
target: "engine::root",
account_targets,
storage_targets,
@@ -1032,7 +1037,7 @@ impl MultiProofTask {
let len = update.len();
state_update_proofs_requested += self.on_state_update(source, update);
debug!(
trace!(
target: "engine::root",
?source,
len,
@@ -1094,7 +1099,7 @@ impl MultiProofTask {
.proof_calculation_duration_histogram
.record(proof_calculated.elapsed);
debug!(
trace!(
target: "engine::root",
sequence = proof_calculated.sequence_number,
total_proofs = proofs_processed,
@@ -1234,7 +1239,7 @@ mod tests {
ProofWorkerHandle::new(executor.handle().clone(), consistent_view, task_ctx, 1, 1);
let channel = channel();
MultiProofTask::new(config, executor, proof_handle, channel.0, 1, None)
MultiProofTask::new(config, executor, proof_handle, channel.0, Some(1))
}
#[test]

32
crates/engine/tree/src/tree/payload_processor/prewarm.rs
View File

@@ -147,9 +147,6 @@ where
let (done_tx, done_rx) = mpsc::channel();
let mut executing = 0usize;
// Initialize worker handles container
let mut handles = Vec::with_capacity(max_concurrency);
// When transaction_count_hint is 0, it means the count is unknown. In this case, spawn
// max workers to handle potentially many transactions in parallel rather
// than bottlenecking on a single worker.
@@ -159,6 +156,9 @@ where
transaction_count_hint.min(max_concurrency)
};
// Initialize worker handles container
let mut handles = Vec::with_capacity(workers_needed);
// Only spawn initial workers as needed
for i in 0..workers_needed {
handles.push(ctx.spawn_worker(i, &executor, actions_tx.clone(), done_tx.clone()));
@@ -233,8 +233,19 @@ where
});
}
/// Returns true if prewarming was terminated and no more transactions should be prewarmed.
fn is_execution_terminated(&self) -> bool {
self.ctx.terminate_execution.load(Ordering::Relaxed)
}
/// If configured and the tx returned proof targets, emit the targets the transaction produced
fn send_multi_proof_targets(&self, targets: Option<MultiProofTargets>) {
if self.is_execution_terminated() {
// if execution is already terminated then we dont need to send more proof fetch
// messages
return
}
if let Some((proof_targets, to_multi_proof)) = targets.zip(self.to_multi_proof.as_ref()) {
let _ = to_multi_proof.send(MultiProofMessage::PrefetchProofs(proof_targets));
}
@@ -308,6 +319,7 @@ where
match event {
PrewarmTaskEvent::TerminateTransactionExecution => {
// stop tx processing
debug!(target: "engine::tree::prewarm", "Terminating prewarm execution");
self.ctx.terminate_execution.store(true, Ordering::Relaxed);
}
PrewarmTaskEvent::Outcome { proof_targets } => {
@@ -338,7 +350,7 @@ where
}
}
trace!(target: "engine::tree::prewarm", "Completed prewarm execution");
debug!(target: "engine::tree::prewarm", "Completed prewarm execution");
// save caches and finish
if let Some(Some(state)) = final_block_output {
@@ -460,6 +472,9 @@ where
debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm tx", index, tx_hash=%tx.tx().tx_hash())
.entered();
// create the tx env
let start = Instant::now();
// If the task was cancelled, stop execution, send an empty result to notify the task,
// and exit.
if terminate_execution.load(Ordering::Relaxed) {
@@ -467,8 +482,6 @@ where
break
}
// create the tx env
let start = Instant::now();
let res = match evm.transact(&tx) {
Ok(res) => res,
Err(err) => {
@@ -489,6 +502,13 @@ where
drop(_enter);
// If the task was cancelled, stop execution, send an empty result to notify the task,
// and exit.
if terminate_execution.load(Ordering::Relaxed) {
let _ = sender.send(PrewarmTaskEvent::Outcome { proof_targets: None });
break
}
// Only send outcome for transactions after the first txn
// as the main execution will be just as fast
if index > 0 {

10
crates/engine/tree/src/tree/payload_validator.rs
View File

@@ -329,7 +329,7 @@ where
skip_all,
fields(
parent = ?input.parent_hash(),
block_num_hash = ?input.num_hash()
type_name = ?input.type_name(),
)
)]
pub fn validate_block_with_state<T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>>(
@@ -1259,4 +1259,12 @@ impl<T: PayloadTypes> BlockOrPayload<T> {
Self::Block(block) => block.block_with_parent(),
}
}
/// Returns a string showing whether or not this is a block or payload.
pub const fn type_name(&self) -> &'static str {
match self {
Self::Payload(_) => "payload",
Self::Block(_) => "block",
}
}
}

19
crates/engine/tree/src/tree/tests.rs
View File

@@ -1391,13 +1391,8 @@ fn test_validate_block_synchronous_strategy_during_persistence() {
let genesis_hash = MAINNET.genesis_hash();
let valid_block = block_factory.create_valid_block(genesis_hash);
// Call validate_block_with_state directly
// This should execute the Synchronous strategy logic during active persistence
let result = test_harness.validate_block_direct(valid_block);
// Verify validation was attempted (may fail due to test environment limitations)
// The key test is that the Synchronous strategy path is executed during persistence
assert!(result.is_ok() || result.is_err(), "Validation should complete")
// Test that Synchronous strategy executes during active persistence without panicking
let _result = test_harness.validate_block_direct(valid_block);
}
/// Test multiple validation scenarios including valid, consensus-invalid, and execution-invalid
@@ -1411,15 +1406,9 @@ fn test_validate_block_multiple_scenarios() {
let mut block_factory = TestBlockFactory::new(MAINNET.as_ref().clone());
let genesis_hash = MAINNET.genesis_hash();
// Scenario 1: Valid block validation (may fail due to test environment limitations)
// Scenario 1: Valid block validation (test execution, not result)
let valid_block = block_factory.create_valid_block(genesis_hash);
let result1 = test_harness.validate_block_direct(valid_block);
// Note: Valid blocks might fail in test environment due to missing provider data,
// but the important thing is that the validation logic executes without panicking
assert!(
result1.is_ok() || result1.is_err(),
"Valid block validation should complete (may fail due to test environment)"
);
let _result1 = test_harness.validate_block_direct(valid_block);
// Scenario 2: Block with consensus issues should be rejected
let consensus_invalid = block_factory.create_invalid_consensus_block(genesis_hash);

5
crates/era-utils/src/export.rs
View File

@@ -1,6 +1,7 @@
//! Logic to export from database era1 block history
//! and injecting them into era1 files with `Era1Writer`.
use crate::calculate_td_by_number;
use alloy_consensus::BlockHeader;
use alloy_primitives::{BlockNumber, B256, U256};
use eyre::{eyre, Result};
@@ -114,9 +115,7 @@ where
let mut total_difficulty = if config.first_block_number > 0 {
let prev_block_number = config.first_block_number - 1;
provider
.header_td_by_number(prev_block_number)?
.ok_or_else(|| eyre!("Total difficulty not found for block {prev_block_number}"))?
calculate_td_by_number(provider, prev_block_number)?
} else {
U256::ZERO
};

49
crates/era-utils/src/history.rs
View File

@@ -1,3 +1,4 @@
use alloy_consensus::BlockHeader;
use alloy_primitives::{BlockHash, BlockNumber, U256};
use futures_util::{Stream, StreamExt};
use reth_db_api::{
@@ -19,15 +20,15 @@ use reth_etl::Collector;
use reth_fs_util as fs;
use reth_primitives_traits::{Block, FullBlockBody, FullBlockHeader, NodePrimitives};
use reth_provider::{
providers::StaticFileProviderRWRefMut, BlockWriter, ProviderError, StaticFileProviderFactory,
providers::StaticFileProviderRWRefMut, BlockReader, BlockWriter, StaticFileProviderFactory,
StaticFileSegment, StaticFileWriter,
};
use reth_stages_types::{
CheckpointBlockRange, EntitiesCheckpoint, HeadersCheckpoint, StageCheckpoint, StageId,
};
use reth_storage_api::{
errors::ProviderResult, DBProvider, DatabaseProviderFactory, HeaderProvider,
NodePrimitivesProvider, StageCheckpointWriter,
errors::ProviderResult, DBProvider, DatabaseProviderFactory, NodePrimitivesProvider,
StageCheckpointWriter,
};
use std::{
collections::Bound,
@@ -82,11 +83,6 @@ where
.get_highest_static_file_block(StaticFileSegment::Headers)
.unwrap_or_default();
// Find the latest total difficulty
let mut td = static_file_provider
.header_td_by_number(height)?
.ok_or(ProviderError::TotalDifficultyNotFound(height))?;
while let Some(meta) = rx.recv()? {
let from = height;
let provider = provider_factory.database_provider_rw()?;
@@ -96,7 +92,6 @@ where
&mut static_file_provider.latest_writer(StaticFileSegment::Headers)?,
&provider,
hash_collector,
&mut td,
height..,
)?;
@@ -146,7 +141,7 @@ where
/// Extracts block headers and bodies from `meta` and appends them using `writer` and `provider`.
///
/// Adds on to `total_difficulty` and collects hash to height using `hash_collector`.
/// Collects hash to height using `hash_collector`.
///
/// Skips all blocks below the [`start_bound`] of `block_numbers` and stops when reaching past the
/// [`end_bound`] or the end of the file.
@@ -160,7 +155,6 @@ pub fn process<Era, P, B, BB, BH>(
writer: &mut StaticFileProviderRWRefMut<'_, <P as NodePrimitivesProvider>::Primitives>,
provider: &P,
hash_collector: &mut Collector<BlockHash, BlockNumber>,
total_difficulty: &mut U256,
block_numbers: impl RangeBounds<BlockNumber>,
) -> eyre::Result<BlockNumber>
where
@@ -182,7 +176,7 @@ where
as Box<dyn Fn(Result<BlockTuple, E2sError>) -> eyre::Result<(BH, BB)>>);
let iter = ProcessIter { iter, era: meta };
process_iter(iter, writer, provider, hash_collector, total_difficulty, block_numbers)
process_iter(iter, writer, provider, hash_collector, block_numbers)
}
type ProcessInnerIter<R, BH, BB> =
@@ -271,7 +265,6 @@ pub fn process_iter<P, B, BB, BH>(
writer: &mut StaticFileProviderRWRefMut<'_, <P as NodePrimitivesProvider>::Primitives>,
provider: &P,
hash_collector: &mut Collector<BlockHash, BlockNumber>,
total_difficulty: &mut U256,
block_numbers: impl RangeBounds<BlockNumber>,
) -> eyre::Result<BlockNumber>
where
@@ -311,11 +304,8 @@ where
let hash = header.hash_slow();
last_header_number = number;
// Increase total difficulty
*total_difficulty += header.difficulty();
// Append to Headers segment
writer.append_header(&header, *total_difficulty, &hash)?;
writer.append_header(&header, &hash)?;
// Write bodies to database.
provider.append_block_bodies(vec![(header.number(), Some(body))])?;
@@ -382,3 +372,28 @@ where
Ok(())
}
/// Calculates the total difficulty for a given block number by summing the difficulty
/// of all blocks from genesis to the given block.
///
/// Very expensive - iterates through all blocks in batches of 1000.
///
/// Returns an error if any block is missing.
pub fn calculate_td_by_number<P>(provider: &P, num: BlockNumber) -> eyre::Result<U256>
where
P: BlockReader,
{
let mut total_difficulty = U256::ZERO;
let mut start = 0;
while start <= num {
let end = (start + 1000 - 1).min(num);
total_difficulty +=
provider.headers_range(start..=end)?.iter().map(|h| h.difficulty()).sum::<U256>();
start = end + 1;
}
Ok(total_difficulty)
}

3
crates/era-utils/src/lib.rs
View File

@@ -14,5 +14,6 @@ pub use export::{export, ExportConfig};
/// Imports history from ERA files.
pub use history::{
build_index, decode, import, open, process, process_iter, save_stage_checkpoints, ProcessIter,
build_index, calculate_td_by_number, decode, import, open, process, process_iter,
save_stage_checkpoints, ProcessIter,
};

3
crates/ethereum/node/tests/e2e/dev.rs
View File

@@ -3,7 +3,7 @@ use alloy_genesis::Genesis;
use alloy_primitives::{b256, hex, Address};
use futures::StreamExt;
use reth_chainspec::ChainSpec;
use reth_node_api::{BlockBody, FullNodeComponents, FullNodePrimitives, NodeTypes};
use reth_node_api::{BlockBody, FullNodeComponents};
use reth_node_builder::{rpc::RethRpcAddOns, FullNode, NodeBuilder, NodeConfig, NodeHandle};
use reth_node_core::args::DevArgs;
use reth_node_ethereum::{node::EthereumAddOns, EthereumNode};
@@ -81,7 +81,6 @@ async fn assert_chain_advances<N, AddOns>(node: &FullNode<N, AddOns>)
where
N: FullNodeComponents<Provider: CanonStateSubscriptions>,
AddOns: RethRpcAddOns<N, EthApi: EthTransactions>,
N::Types: NodeTypes<Primitives: FullNodePrimitives>,
{
let mut notifications = node.provider.canonical_state_stream();

8
crates/ethereum/payload/src/lib.rs
View File

@@ -236,10 +236,10 @@ where
if is_osaka && estimated_block_size_with_tx > MAX_RLP_BLOCK_SIZE {
best_txs.mark_invalid(
&pool_tx,
InvalidPoolTransactionError::OversizedData(
estimated_block_size_with_tx,
MAX_RLP_BLOCK_SIZE,
),
InvalidPoolTransactionError::OversizedData {
size: estimated_block_size_with_tx,
limit: MAX_RLP_BLOCK_SIZE,
},
);
continue;
}

1
crates/ethereum/primitives/Cargo.toml
View File

@@ -73,6 +73,7 @@ reth-codec = [
"dep:reth-zstd-compressors",
]
arbitrary = [
"std",
"dep:arbitrary",
"alloy-consensus/arbitrary",
"alloy-consensus/k256",

2
crates/exex/exex/src/backfill/factory.rs
View File

@@ -39,7 +39,7 @@ impl<E, P> BackfillJobFactory<E, P> {
}
/// Sets the prune modes
pub fn with_prune_modes(mut self, prune_modes: PruneModes) -> Self {
pub const fn with_prune_modes(mut self, prune_modes: PruneModes) -> Self {
self.prune_modes = prune_modes;
self
}

4
crates/exex/exex/src/backfill/stream.rs
View File

@@ -256,7 +256,7 @@ mod tests {
use reth_ethereum_primitives::{Block, BlockBody, Transaction};
use reth_evm_ethereum::EthEvmConfig;
use reth_primitives_traits::{
crypto::secp256k1::public_key_to_address, Block as _, FullNodePrimitives,
crypto::secp256k1::public_key_to_address, Block as _, NodePrimitives,
};
use reth_provider::{
providers::{BlockchainProvider, ProviderNodeTypes},
@@ -395,7 +395,7 @@ mod tests {
) -> Result<()>
where
N: ProviderNodeTypes<
Primitives: FullNodePrimitives<
Primitives: NodePrimitives<
Block = reth_ethereum_primitives::Block,
BlockBody = reth_ethereum_primitives::BlockBody,
Receipt = reth_ethereum_primitives::Receipt,

8
crates/exex/exex/src/backfill/test_utils.rs
View File

@@ -10,7 +10,7 @@ use reth_evm::{
ConfigureEvm,
};
use reth_evm_ethereum::EthEvmConfig;
use reth_node_api::FullNodePrimitives;
use reth_node_api::NodePrimitives;
use reth_primitives_traits::{Block as _, RecoveredBlock};
use reth_provider::{
providers::ProviderNodeTypes, BlockWriter as _, ExecutionOutcome, LatestStateProviderRef,
@@ -58,7 +58,7 @@ pub(crate) fn execute_block_and_commit_to_database<N>(
) -> eyre::Result<BlockExecutionOutput<Receipt>>
where
N: ProviderNodeTypes<
Primitives: FullNodePrimitives<
Primitives: NodePrimitives<
Block = reth_ethereum_primitives::Block,
BlockBody = reth_ethereum_primitives::BlockBody,
Receipt = reth_ethereum_primitives::Receipt,
@@ -169,7 +169,7 @@ pub(crate) fn blocks_and_execution_outputs<N>(
>
where
N: ProviderNodeTypes<
Primitives: FullNodePrimitives<
Primitives: NodePrimitives<
Block = reth_ethereum_primitives::Block,
BlockBody = reth_ethereum_primitives::BlockBody,
Receipt = reth_ethereum_primitives::Receipt,
@@ -193,7 +193,7 @@ pub(crate) fn blocks_and_execution_outcome<N>(
) -> eyre::Result<(Vec<RecoveredBlock<reth_ethereum_primitives::Block>>, ExecutionOutcome)>
where
N: ProviderNodeTypes,
N::Primitives: FullNodePrimitives<
N::Primitives: NodePrimitives<
Block = reth_ethereum_primitives::Block,
Receipt = reth_ethereum_primitives::Receipt,
>,

2
crates/net/discv4/Cargo.toml
View File

@@ -35,7 +35,6 @@ tracing.workspace = true
thiserror.workspace = true
parking_lot.workspace = true
rand_08 = { workspace = true, optional = true }
generic-array.workspace = true
serde = { workspace = true, optional = true }
itertools.workspace = true
@@ -53,7 +52,6 @@ serde = [
"alloy-primitives/serde",
"discv5/serde",
"enr/serde",
"generic-array/serde",
"parking_lot/serde",
"rand_08?/serde",
"secp256k1/serde",

4
crates/net/discv4/src/node.rs
View File

@@ -1,5 +1,4 @@
use alloy_primitives::keccak256;
use generic_array::GenericArray;
use reth_network_peers::{NodeRecord, PeerId};
/// The key type for the table.
@@ -15,8 +14,7 @@ impl From<PeerId> for NodeKey {
impl From<NodeKey> for discv5::Key<NodeKey> {
fn from(value: NodeKey) -> Self {
let hash = keccak256(value.0.as_slice());
let hash = *GenericArray::from_slice(hash.as_slice());
Self::new_raw(value, hash)
Self::new_raw(value, hash.0.into())
}
}

6
crates/net/downloaders/src/bodies/test_utils.rs
View File

@@ -3,7 +3,7 @@
#![allow(dead_code)]
use alloy_consensus::BlockHeader;
use alloy_primitives::{B256, U256};
use alloy_primitives::B256;
use reth_ethereum_primitives::BlockBody;
use reth_network_p2p::bodies::response::BlockResponse;
use reth_primitives_traits::{Block, SealedBlock, SealedHeader};
@@ -55,9 +55,7 @@ pub(crate) fn insert_headers(
.expect("failed to create writer");
for header in headers {
writer
.append_header(header.header(), U256::ZERO, &header.hash())
.expect("failed to append header");
writer.append_header(header.header(), &header.hash()).expect("failed to append header");
}
drop(writer);
provider_rw.commit().expect("failed to commit");

4
crates/net/downloaders/src/file_client.rs
View File

@@ -815,7 +815,7 @@ mod tests {
// construct headers downloader and use first header
let mut header_downloader = ReverseHeadersDownloaderBuilder::default()
.build(Arc::clone(&Arc::new(client)), Arc::new(TestConsensus::default()));
.build(Arc::new(client), Arc::new(TestConsensus::default()));
header_downloader.update_local_head(local_header.clone());
header_downloader.update_sync_target(SyncTarget::Tip(sync_target_hash));
@@ -890,7 +890,7 @@ mod tests {
// construct headers downloader and use first header
let mut header_downloader = ReverseHeadersDownloaderBuilder::default()
.build(Arc::clone(&Arc::new(client)), Arc::new(TestConsensus::default()));
.build(Arc::new(client), Arc::new(TestConsensus::default()));
header_downloader.update_local_head(local_header.clone());
header_downloader.update_sync_target(SyncTarget::Tip(sync_target_hash));

6
crates/net/ecies/src/codec.rs
View File

@@ -58,7 +58,7 @@ impl Decoder for ECIESCodec {
type Item = IngressECIESValue;
type Error = ECIESError;
#[instrument(skip_all, fields(peer=?self.ecies.remote_id, state=?self.state))]
#[instrument(level = "trace", skip_all, fields(peer=?self.ecies.remote_id, state=?self.state))]
fn decode(&mut self, buf: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
loop {
match self.state {
@@ -110,7 +110,7 @@ impl Decoder for ECIESCodec {
self.ecies.read_header(&mut buf.split_to(ECIES::header_len()))?;
if body_size > MAX_INITIAL_HANDSHAKE_SIZE {
trace!(?body_size, max=?MAX_INITIAL_HANDSHAKE_SIZE, "Header exceeds max initial handshake size");
trace!(?body_size, max=?MAX_INITIAL_HANDSHAKE_SIZE, "Body exceeds max initial handshake size");
return Err(ECIESErrorImpl::InitialHeaderBodyTooLarge {
body_size,
max_body_size: MAX_INITIAL_HANDSHAKE_SIZE,
@@ -150,7 +150,7 @@ impl Decoder for ECIESCodec {
impl Encoder<EgressECIESValue> for ECIESCodec {
type Error = io::Error;
#[instrument(skip(self, buf), fields(peer=?self.ecies.remote_id, state=?self.state))]
#[instrument(level = "trace", skip(self, buf), fields(peer=?self.ecies.remote_id, state=?self.state))]
fn encode(&mut self, item: EgressECIESValue, buf: &mut BytesMut) -> Result<(), Self::Error> {
match item {
EgressECIESValue::Auth => {

4
crates/net/eth-wire/src/multiplex.rs
View File

@@ -332,9 +332,9 @@ impl ProtocolProxy {
return Ok(msg);
}
let mut masked = Vec::from(msg);
let mut masked: BytesMut = msg.into();
masked[0] = masked[0].checked_add(offset).ok_or(io::ErrorKind::InvalidInput)?;
Ok(masked.into())
Ok(masked.freeze())
}
/// Unmasks the message ID of a message received from the wire.

12
crates/net/peers/src/node_record.rs
View File

@@ -309,6 +309,18 @@ mod tests {
}
}
#[test]
fn test_node_record() {
let url = "enode://fc8a2ff614e848c0af4c99372a81b8655edb8e11b617cffd0aab1a0691bcca66ca533626a528ee567f05f70c8cb529bda2c0a864cc0aec638a367fd2bb8e49fb@127.0.0.1:35481?discport=0";
let node: NodeRecord = url.parse().unwrap();
assert_eq!(node, NodeRecord {
address: IpAddr::V4([127,0,0, 1].into()),
tcp_port: 35481,
udp_port: 0,
id: "0xfc8a2ff614e848c0af4c99372a81b8655edb8e11b617cffd0aab1a0691bcca66ca533626a528ee567f05f70c8cb529bda2c0a864cc0aec638a367fd2bb8e49fb".parse().unwrap(),
})
}
#[test]
fn test_url_parse() {
let url = "enode://6f8a80d14311c39f35f516fa664deaaaa13e85b2f7493f37f6144d86991ec012937307647bd3b9a82abe2974e1407241d54947bbb39763a4cac9f77166ad92a0@10.3.58.6:30303?discport=30301";

4
crates/node/builder/src/builder/mod.rs
View File

@@ -21,8 +21,7 @@ use reth_network::{
NetworkPrimitives,
};
use reth_node_api::{
FullNodePrimitives, FullNodeTypes, FullNodeTypesAdapter, NodeAddOns, NodeTypes,
NodeTypesWithDBAdapter,
FullNodeTypes, FullNodeTypesAdapter, NodeAddOns, NodeTypes, NodeTypesWithDBAdapter,
};
use reth_node_core::{
cli::config::{PayloadBuilderConfig, RethTransactionPoolConfig},
@@ -397,7 +396,6 @@ where
<N::ComponentsBuilder as NodeComponentsBuilder<RethFullAdapter<DB, N>>>::Components,
>,
>,
N::Primitives: FullNodePrimitives,
EngineNodeLauncher: LaunchNode<
NodeBuilderWithComponents<RethFullAdapter<DB, N>, N::ComponentsBuilder, N::AddOns>,
>,

184
crates/node/builder/src/launch/common.rs
View File

@@ -41,10 +41,12 @@ use eyre::Context;
use rayon::ThreadPoolBuilder;
use reth_chainspec::{Chain, EthChainSpec, EthereumHardfork, EthereumHardforks};
use reth_config::{config::EtlConfig, PruneConfig};
use reth_consensus::noop::NoopConsensus;
use reth_db_api::{database::Database, database_metrics::DatabaseMetrics};
use reth_db_common::init::{init_genesis, InitStorageError};
use reth_downloaders::{bodies::noop::NoopBodiesDownloader, headers::noop::NoopHeaderDownloader};
use reth_engine_local::MiningMode;
use reth_evm::ConfigureEvm;
use reth_evm::{noop::NoopEvmConfig, ConfigureEvm};
use reth_exex::ExExManagerHandle;
use reth_fs_util as fs;
use reth_network_p2p::headers::client::HeadersClient;
@@ -65,19 +67,25 @@ use reth_node_metrics::{
};
use reth_provider::{
providers::{NodeTypesForProvider, ProviderNodeTypes, StaticFileProvider},
BlockNumReader, BlockReaderIdExt, ProviderError, ProviderFactory, ProviderResult,
StaticFileProviderFactory,
BlockHashReader, BlockNumReader, BlockReaderIdExt, ProviderError, ProviderFactory,
ProviderResult, StageCheckpointReader, StaticFileProviderFactory,
};
use reth_prune::{PruneModes, PrunerBuilder};
use reth_rpc_builder::config::RethRpcServerConfig;
use reth_rpc_layer::JwtSecret;
use reth_stages::{stages::EraImportSource, MetricEvent};
use reth_stages::{
sets::DefaultStages, stages::EraImportSource, MetricEvent, PipelineBuilder, PipelineTarget,
StageId,
};
use reth_static_file::StaticFileProducer;
use reth_tasks::TaskExecutor;
use reth_tracing::tracing::{debug, error, info, warn};
use reth_transaction_pool::TransactionPool;
use std::{sync::Arc, thread::available_parallelism};
use tokio::sync::mpsc::{unbounded_channel, UnboundedSender};
use tokio::sync::{
mpsc::{unbounded_channel, UnboundedSender},
oneshot, watch,
};
use futures::{future::Either, stream, Stream, StreamExt};
use reth_node_ethstats::EthStatsService;
@@ -151,7 +159,7 @@ impl LaunchContext {
let mut toml_config = reth_config::Config::from_path(&config_path)
.wrap_err_with(|| format!("Could not load config file {config_path:?}"))?;
Self::save_pruning_config_if_full_node(&mut toml_config, config, &config_path)?;
Self::save_pruning_config(&mut toml_config, config, &config_path)?;
info!(target: "reth::cli", path = ?config_path, "Configuration loaded");
@@ -161,8 +169,9 @@ impl LaunchContext {
Ok(toml_config)
}
/// Save prune config to the toml file if node is a full node.
fn save_pruning_config_if_full_node<ChainSpec>(
/// Save prune config to the toml file if node is a full node or has custom pruning CLI
/// arguments.
fn save_pruning_config<ChainSpec>(
reth_config: &mut reth_config::Config,
config: &NodeConfig<ChainSpec>,
config_path: impl AsRef<std::path::Path>,
@@ -170,14 +179,14 @@ impl LaunchContext {
where
ChainSpec: EthChainSpec + reth_chainspec::EthereumHardforks,
{
if reth_config.prune.is_none() {
if let Some(prune_config) = config.prune_config() {
reth_config.update_prune_config(prune_config);
if let Some(prune_config) = config.prune_config() {
if reth_config.prune != prune_config {
reth_config.set_prune_config(prune_config);
info!(target: "reth::cli", "Saving prune config to toml file");
reth_config.save(config_path.as_ref())?;
}
} else if config.prune_config().is_none() {
warn!(target: "reth::cli", "Prune configs present in config file but --full not provided. Running as a Full node");
} else if !reth_config.prune.is_default() {
warn!(target: "reth::cli", "Pruning configuration is present in the config file, but no CLI arguments are provided. Using config from file.");
}
Ok(())
}
@@ -393,18 +402,19 @@ impl<R, ChainSpec: EthChainSpec> LaunchContextWith<Attached<WithConfigs<ChainSpe
/// Returns the configured [`PruneConfig`]
///
/// Any configuration set in CLI will take precedence over those set in toml
pub fn prune_config(&self) -> Option<PruneConfig>
pub fn prune_config(&self) -> PruneConfig
where
ChainSpec: reth_chainspec::EthereumHardforks,
{
let toml_config = self.toml_config().prune.clone();
let Some(mut node_prune_config) = self.node_config().prune_config() else {
// No CLI config is set, use the toml config.
return self.toml_config().prune.clone();
return toml_config;
};
// Otherwise, use the CLI configuration and merge with toml config.
node_prune_config.merge(self.toml_config().prune.clone());
Some(node_prune_config)
node_prune_config.merge(toml_config);
node_prune_config
}
/// Returns the configured [`PruneModes`], returning the default if no config was available.
@@ -412,7 +422,7 @@ impl<R, ChainSpec: EthChainSpec> LaunchContextWith<Attached<WithConfigs<ChainSpe
where
ChainSpec: reth_chainspec::EthereumHardforks,
{
self.prune_config().map(|config| config.segments).unwrap_or_default()
self.prune_config().segments
}
/// Returns an initialized [`PrunerBuilder`] based on the configured [`PruneConfig`]
@@ -420,9 +430,7 @@ impl<R, ChainSpec: EthChainSpec> LaunchContextWith<Attached<WithConfigs<ChainSpe
where
ChainSpec: reth_chainspec::EthereumHardforks,
{
PrunerBuilder::new(self.prune_config().unwrap_or_default())
.delete_limit(self.chain_spec().prune_delete_limit())
.timeout(PrunerBuilder::DEFAULT_TIMEOUT)
PrunerBuilder::new(self.prune_config())
}
/// Loads the JWT secret for the engine API
@@ -458,13 +466,69 @@ where
N: ProviderNodeTypes<DB = DB, ChainSpec = ChainSpec>,
Evm: ConfigureEvm<Primitives = N::Primitives> + 'static,
{
Ok(ProviderFactory::new(
let factory = ProviderFactory::new(
self.right().clone(),
self.chain_spec(),
StaticFileProvider::read_write(self.data_dir().static_files())?,
)
.with_prune_modes(self.prune_modes())
.with_static_files_metrics())
.with_static_files_metrics();
let has_receipt_pruning = self.toml_config().prune.has_receipts_pruning();
// Check for consistency between database and static files. If it fails, it unwinds to
// the first block that's consistent between database and static files.
if let Some(unwind_target) = factory
.static_file_provider()
.check_consistency(&factory.provider()?, has_receipt_pruning)?
{
// Highly unlikely to happen, and given its destructive nature, it's better to panic
// instead.
assert_ne!(
unwind_target,
PipelineTarget::Unwind(0),
"A static file <> database inconsistency was found that would trigger an unwind to block 0"
);
info!(target: "reth::cli", unwind_target = %unwind_target, "Executing an unwind after a failed storage consistency check.");
let (_tip_tx, tip_rx) = watch::channel(B256::ZERO);
// Builds an unwind-only pipeline
let pipeline = PipelineBuilder::default()
.add_stages(DefaultStages::new(
factory.clone(),
tip_rx,
Arc::new(NoopConsensus::default()),
NoopHeaderDownloader::default(),
NoopBodiesDownloader::default(),
NoopEvmConfig::<Evm>::default(),
self.toml_config().stages.clone(),
self.prune_modes(),
None,
))
.build(
factory.clone(),
StaticFileProducer::new(factory.clone(), self.prune_modes()),
);
// Unwinds to block
let (tx, rx) = oneshot::channel();
// Pipeline should be run as blocking and panic if it fails.
self.task_executor().spawn_critical_blocking(
"pipeline task",
Box::pin(async move {
let (_, result) = pipeline.run_as_fut(Some(unwind_target)).await;
let _ = tx.send(result);
}),
);
rx.await?.inspect_err(|err| {
error!(target: "reth::cli", unwind_target = %unwind_target, %err, "failed to run unwind")
})?;
}
Ok(factory)
}
/// Creates a new [`ProviderFactory`] and attaches it to the launch context.
@@ -787,6 +851,21 @@ where
&self.node_adapter().provider
}
/// Returns the initial backfill to sync to at launch.
///
/// This returns the configured `debug.tip` if set, otherwise it will check if backfill was
/// previously interrupted and returns the block hash of the last checkpoint, see also
/// [`Self::check_pipeline_consistency`]
pub fn initial_backfill_target(&self) -> ProviderResult<Option<B256>> {
let mut initial_target = self.node_config().debug.tip;
if initial_target.is_none() {
initial_target = self.check_pipeline_consistency()?;
}
Ok(initial_target)
}
/// Returns true if the node should terminate after the initial backfill run.
///
/// This is the case if any of these configs are set:
@@ -800,7 +879,7 @@ where
///
/// This checks for OP-Mainnet and ensures we have all the necessary data to progress (past
/// bedrock height)
pub fn ensure_chain_specific_db_checks(&self) -> ProviderResult<()> {
fn ensure_chain_specific_db_checks(&self) -> ProviderResult<()> {
if self.chain_spec().is_optimism() &&
!self.is_dev() &&
self.chain_id() == Chain::optimism_mainnet()
@@ -818,6 +897,54 @@ where
Ok(())
}
/// Check if the pipeline is consistent (all stages have the checkpoint block numbers no less
/// than the checkpoint of the first stage).
///
/// This will return the pipeline target if:
/// * the pipeline was interrupted during its previous run
/// * a new stage was added
/// * stage data was dropped manually through `reth stage drop ...`
///
/// # Returns
///
/// A target block hash if the pipeline is inconsistent, otherwise `None`.
pub fn check_pipeline_consistency(&self) -> ProviderResult<Option<B256>> {
// If no target was provided, check if the stages are congruent - check if the
// checkpoint of the last stage matches the checkpoint of the first.
let first_stage_checkpoint = self
.blockchain_db()
.get_stage_checkpoint(*StageId::ALL.first().unwrap())?
.unwrap_or_default()
.block_number;
// Skip the first stage as we've already retrieved it and comparing all other checkpoints
// against it.
for stage_id in StageId::ALL.iter().skip(1) {
let stage_checkpoint = self
.blockchain_db()
.get_stage_checkpoint(*stage_id)?
.unwrap_or_default()
.block_number;
// If the checkpoint of any stage is less than the checkpoint of the first stage,
// retrieve and return the block hash of the latest header and use it as the target.
if stage_checkpoint < first_stage_checkpoint {
debug!(
target: "consensus::engine",
first_stage_checkpoint,
inconsistent_stage_id = %stage_id,
inconsistent_stage_checkpoint = stage_checkpoint,
"Pipeline sync progress is inconsistent"
);
return self.blockchain_db().block_hash(first_stage_checkpoint);
}
}
self.ensure_chain_specific_db_checks()?;
Ok(None)
}
/// Expire the pre-merge transactions if the node is configured to do so and the chain has a
/// merge block.
///
@@ -1080,17 +1207,14 @@ mod tests {
storage_history_before: None,
bodies_pre_merge: false,
bodies_distance: None,
#[expect(deprecated)]
receipts_log_filter: None,
bodies_before: None,
},
..NodeConfig::test()
};
LaunchContext::save_pruning_config_if_full_node(
&mut reth_config,
&node_config,
config_path,
)
.unwrap();
LaunchContext::save_pruning_config(&mut reth_config, &node_config, config_path)
.unwrap();
let loaded_config = Config::from_path(config_path).unwrap();

21
crates/node/builder/src/launch/engine.rs
View File

@@ -117,6 +117,9 @@ impl EngineNodeLauncher {
})?
.with_components(components_builder, on_component_initialized).await?;
// Try to expire pre-merge transaction history if configured
ctx.expire_pre_merge_transactions()?;
// spawn exexs if any
let maybe_exex_manager_handle = ctx.launch_exex(installed_exex).await?;
@@ -138,7 +141,7 @@ impl EngineNodeLauncher {
let consensus = Arc::new(ctx.components().consensus().clone());
let mut pipeline = build_networked_pipeline(
let pipeline = build_networked_pipeline(
&ctx.toml_config().stages,
network_client.clone(),
consensus.clone(),
@@ -154,18 +157,7 @@ impl EngineNodeLauncher {
)?;
// The new engine writes directly to static files. This ensures that they're up to the tip.
pipeline.ensure_static_files_consistency().await?;
// Try to expire pre-merge transaction history if configured
ctx.expire_pre_merge_transactions()?;
let initial_target = if let Some(tip) = ctx.node_config().debug.tip {
Some(tip)
} else {
pipeline.initial_backfill_target()?
};
ctx.ensure_chain_specific_db_checks()?;
pipeline.move_to_static_files()?;
let pipeline_events = pipeline.events();
@@ -176,7 +168,7 @@ impl EngineNodeLauncher {
}
let pruner = pruner_builder.build_with_provider_factory(ctx.provider_factory().clone());
let pruner_events = pruner.events();
info!(target: "reth::cli", prune_config=?ctx.prune_config().unwrap_or_default(), "Pruner initialized");
info!(target: "reth::cli", prune_config=?ctx.prune_config(), "Pruner initialized");
let event_sender = EventSender::default();
@@ -258,6 +250,7 @@ impl EngineNodeLauncher {
add_ons.launch_add_ons(add_ons_ctx).await?;
// Run consensus engine to completion
let initial_target = ctx.initial_backfill_target()?;
let mut built_payloads = ctx
.components()
.payload_builder_handle()

10
crates/node/builder/src/node.rs
View File

@@ -179,14 +179,16 @@ where
/// Returns the [`EngineApiClient`] interface for the authenticated engine API.
///
/// This will send authenticated http requests to the node's auth server.
pub fn engine_http_client(&self) -> impl EngineApiClient<Engine> {
pub fn engine_http_client(&self) -> impl EngineApiClient<Engine> + use<Engine, Node, AddOns> {
self.auth_server_handle().http_client()
}
/// Returns the [`EngineApiClient`] interface for the authenticated engine API.
///
/// This will send authenticated ws requests to the node's auth server.
pub async fn engine_ws_client(&self) -> impl EngineApiClient<Engine> {
pub async fn engine_ws_client(
&self,
) -> impl EngineApiClient<Engine> + use<Engine, Node, AddOns> {
self.auth_server_handle().ws_client().await
}
@@ -194,7 +196,9 @@ where
///
/// This will send not authenticated IPC requests to the node's auth server.
#[cfg(unix)]
pub async fn engine_ipc_client(&self) -> Option<impl EngineApiClient<Engine>> {
pub async fn engine_ipc_client(
&self,
) -> Option<impl EngineApiClient<Engine> + use<Engine, Node, AddOns>> {
self.auth_server_handle().ipc_client().await
}
}

8
crates/node/builder/src/setup.rs
View File

@@ -36,7 +36,7 @@ pub fn build_networked_pipeline<N, Client, Evm>(
provider_factory: ProviderFactory<N>,
task_executor: &TaskExecutor,
metrics_tx: reth_stages::MetricEventsSender,
prune_config: Option<PruneConfig>,
prune_config: PruneConfig,
max_block: Option<BlockNumber>,
static_file_producer: StaticFileProducer<ProviderFactory<N>>,
evm_config: Evm,
@@ -85,7 +85,7 @@ pub fn build_pipeline<N, H, B, Evm>(
consensus: Arc<dyn FullConsensus<N::Primitives, Error = ConsensusError>>,
max_block: Option<u64>,
metrics_tx: reth_stages::MetricEventsSender,
prune_config: Option<PruneConfig>,
prune_config: PruneConfig,
static_file_producer: StaticFileProducer<ProviderFactory<N>>,
evm_config: Evm,
exex_manager_handle: ExExManagerHandle<N::Primitives>,
@@ -106,8 +106,6 @@ where
let (tip_tx, tip_rx) = watch::channel(B256::ZERO);
let prune_modes = prune_config.map(|prune| prune.segments).unwrap_or_default();
let pipeline = builder
.with_tip_sender(tip_tx)
.with_metrics_tx(metrics_tx)
@@ -120,7 +118,7 @@ where
body_downloader,
evm_config.clone(),
stage_config.clone(),
prune_modes,
prune_config.segments,
era_import_source,
)
.set(ExecutionStage::new(

1
crates/node/core/Cargo.toml
View File

@@ -52,7 +52,6 @@ derive_more.workspace = true
toml.workspace = true
serde.workspace = true
strum = { workspace = true, features = ["derive"] }
thiserror.workspace = true
url.workspace = true
# io

10
crates/node/core/src/args/engine.rs
View File

@@ -4,8 +4,8 @@ use clap::Args;
use reth_engine_primitives::{TreeConfig, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE};
use crate::node_config::{
DEFAULT_CROSS_BLOCK_CACHE_SIZE_MB, DEFAULT_MAX_PROOF_TASK_CONCURRENCY,
DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, DEFAULT_PERSISTENCE_THRESHOLD, DEFAULT_RESERVED_CPU_CORES,
DEFAULT_CROSS_BLOCK_CACHE_SIZE_MB, DEFAULT_MEMORY_BLOCK_BUFFER_TARGET,
DEFAULT_PERSISTENCE_THRESHOLD, DEFAULT_RESERVED_CPU_CORES,
};
/// Parameters for configuring the engine driver.
@@ -63,10 +63,6 @@ pub struct EngineArgs {
#[arg(long = "engine.accept-execution-requests-hash")]
pub accept_execution_requests_hash: bool,
/// Configure the maximum number of concurrent proof tasks
#[arg(long = "engine.max-proof-task-concurrency", default_value_t = DEFAULT_MAX_PROOF_TASK_CONCURRENCY)]
pub max_proof_task_concurrency: u64,
/// Whether multiproof task should chunk proof targets.
#[arg(long = "engine.multiproof-chunking", default_value = "true")]
pub multiproof_chunking_enabled: bool,
@@ -135,7 +131,6 @@ impl Default for EngineArgs {
state_provider_metrics: false,
cross_block_cache_size: DEFAULT_CROSS_BLOCK_CACHE_SIZE_MB,
accept_execution_requests_hash: false,
max_proof_task_concurrency: DEFAULT_MAX_PROOF_TASK_CONCURRENCY,
multiproof_chunking_enabled: true,
multiproof_chunk_size: DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE,
reserved_cpu_cores: DEFAULT_RESERVED_CPU_CORES,
@@ -162,7 +157,6 @@ impl EngineArgs {
.with_state_provider_metrics(self.state_provider_metrics)
.with_always_compare_trie_updates(self.state_root_task_compare_updates)
.with_cross_block_cache_size(self.cross_block_cache_size * 1024 * 1024)
.with_max_proof_task_concurrency(self.max_proof_task_concurrency)
.with_multiproof_chunking_enabled(self.multiproof_chunking_enabled)
.with_multiproof_chunk_size(self.multiproof_chunk_size)
.with_reserved_cpu_cores(self.reserved_cpu_cores)

22
crates/node/core/src/args/error.rs
View File

@@ -1,22 +0,0 @@
use std::num::ParseIntError;
/// Error while parsing a `ReceiptsLogPruneConfig`
#[derive(thiserror::Error, Debug)]
#[expect(clippy::enum_variant_names)]
pub(crate) enum ReceiptsLogError {
/// The format of the filter is invalid.
#[error("invalid filter format: {0}")]
InvalidFilterFormat(String),
/// Address is invalid.
#[error("address is invalid: {0}")]
InvalidAddress(String),
/// The prune mode is not one of full, distance, before.
#[error("prune mode is invalid: {0}")]
InvalidPruneMode(String),
/// The distance value supplied is invalid.
#[error("distance is invalid: {0}")]
InvalidDistance(ParseIntError),
/// The block number supplied is invalid.
#[error("block number is invalid: {0}")]
InvalidBlockNumber(ParseIntError),
}

1
crates/node/core/src/args/mod.rs
View File

@@ -76,5 +76,4 @@ pub use ress_args::RessArgs;
mod era;
pub use era::{DefaultEraHost, EraArgs, EraSourceArgs};
mod error;
pub mod types;

180
crates/node/core/src/args/pruning.rs
View File

@@ -1,12 +1,13 @@
//! Pruning and full node arguments
use crate::{args::error::ReceiptsLogError, primitives::EthereumHardfork};
use alloy_primitives::{Address, BlockNumber};
use std::ops::Not;
use crate::primitives::EthereumHardfork;
use alloy_primitives::BlockNumber;
use clap::{builder::RangedU64ValueParser, Args};
use reth_chainspec::EthereumHardforks;
use reth_config::config::PruneConfig;
use reth_prune_types::{PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_PRUNING_DISTANCE};
use std::collections::BTreeMap;
use reth_prune_types::{PruneMode, PruneModes, MINIMUM_PRUNING_DISTANCE};
/// Parameters for pruning and full node
#[derive(Debug, Clone, Args, PartialEq, Eq, Default)]
@@ -59,12 +60,10 @@ pub struct PruningArgs {
/// Prune receipts before the specified block number. The specified block number is not pruned.
#[arg(long = "prune.receipts.before", value_name = "BLOCK_NUMBER", conflicts_with_all = &["receipts_full", "receipts_pre_merge", "receipts_distance"])]
pub receipts_before: Option<BlockNumber>,
// Receipts Log Filter
/// Configure receipts log filter. Format:
/// <`address`>:<`prune_mode`>... where <`prune_mode`> can be 'full', 'distance:<`blocks`>', or
/// 'before:<`block_number`>'
#[arg(long = "prune.receiptslogfilter", value_name = "FILTER_CONFIG", conflicts_with_all = &["receipts_full", "receipts_pre_merge", "receipts_distance", "receipts_before"], value_parser = parse_receipts_log_filter)]
pub receipts_log_filter: Option<ReceiptsLogPruneConfig>,
/// Receipts Log Filter
#[arg(long = "prune.receiptslogfilter", value_name = "FILTER_CONFIG", hide = true)]
#[deprecated]
pub receipts_log_filter: Option<String>,
// Account History
/// Prunes all account history.
@@ -107,6 +106,9 @@ pub struct PruningArgs {
impl PruningArgs {
/// Returns pruning configuration.
///
/// Returns [`None`] if no parameters are specified and default pruning configuration should be
/// used.
pub fn prune_config<ChainSpec>(&self, chain_spec: &ChainSpec) -> Option<PruneConfig>
where
ChainSpec: EthereumHardforks,
@@ -127,7 +129,8 @@ impl PruningArgs {
// TODO: set default to pre-merge block if available
bodies_history: None,
merkle_changesets: PruneMode::Distance(MINIMUM_PRUNING_DISTANCE),
receipts_log_filter: Default::default(),
#[expect(deprecated)]
receipts_log_filter: (),
},
}
}
@@ -154,16 +157,17 @@ impl PruningArgs {
if let Some(mode) = self.storage_history_prune_mode() {
config.segments.storage_history = Some(mode);
}
if let Some(receipt_logs) =
self.receipts_log_filter.as_ref().filter(|c| !c.is_empty()).cloned()
{
config.segments.receipts_log_filter = receipt_logs;
// need to remove the receipts segment filter entirely because that takes precedence
// over the logs filter
config.segments.receipts.take();
// Log warning if receipts_log_filter is set (deprecated feature)
#[expect(deprecated)]
if self.receipts_log_filter.is_some() {
tracing::warn!(
target: "reth::cli",
"The --prune.receiptslogfilter flag is deprecated and has no effect. It will be removed in a future release."
);
}
Some(config)
config.is_default().not().then_some(config)
}
fn bodies_prune_mode<ChainSpec>(&self, chain_spec: &ChainSpec) -> Option<PruneMode>
@@ -248,141 +252,3 @@ impl PruningArgs {
}
}
}
/// Parses `,` separated pruning info into [`ReceiptsLogPruneConfig`].
pub(crate) fn parse_receipts_log_filter(
value: &str,
) -> Result<ReceiptsLogPruneConfig, ReceiptsLogError> {
let mut config = BTreeMap::new();
// Split out each of the filters.
let filters = value.split(',');
for filter in filters {
let parts: Vec<&str> = filter.split(':').collect();
if parts.len() < 2 {
return Err(ReceiptsLogError::InvalidFilterFormat(filter.to_string()));
}
// Parse the address
let address = parts[0]
.parse::<Address>()
.map_err(|_| ReceiptsLogError::InvalidAddress(parts[0].to_string()))?;
// Parse the prune mode
let prune_mode = match parts[1] {
"full" => PruneMode::Full,
s if s.starts_with("distance") => {
if parts.len() < 3 {
return Err(ReceiptsLogError::InvalidFilterFormat(filter.to_string()));
}
let distance =
parts[2].parse::<u64>().map_err(ReceiptsLogError::InvalidDistance)?;
PruneMode::Distance(distance)
}
s if s.starts_with("before") => {
if parts.len() < 3 {
return Err(ReceiptsLogError::InvalidFilterFormat(filter.to_string()));
}
let block_number =
parts[2].parse::<u64>().map_err(ReceiptsLogError::InvalidBlockNumber)?;
PruneMode::Before(block_number)
}
_ => return Err(ReceiptsLogError::InvalidPruneMode(parts[1].to_string())),
};
config.insert(address, prune_mode);
}
Ok(ReceiptsLogPruneConfig(config))
}
#[cfg(test)]
mod tests {
use super::*;
use alloy_primitives::address;
use clap::Parser;
/// A helper type to parse Args more easily
#[derive(Parser)]
struct CommandParser<T: Args> {
#[command(flatten)]
args: T,
}
#[test]
fn pruning_args_sanity_check() {
let args = CommandParser::<PruningArgs>::parse_from([
"reth",
"--prune.receiptslogfilter",
"0x0000000000000000000000000000000000000003:before:5000000",
])
.args;
let mut config = ReceiptsLogPruneConfig::default();
config.0.insert(
address!("0x0000000000000000000000000000000000000003"),
PruneMode::Before(5000000),
);
assert_eq!(args.receipts_log_filter, Some(config));
}
#[test]
fn parse_receiptslogfilter() {
let default_args = PruningArgs::default();
let args = CommandParser::<PruningArgs>::parse_from(["reth"]).args;
assert_eq!(args, default_args);
}
#[test]
fn test_parse_receipts_log_filter() {
let filter1 = "0x0000000000000000000000000000000000000001:full";
let filter2 = "0x0000000000000000000000000000000000000002:distance:1000";
let filter3 = "0x0000000000000000000000000000000000000003:before:5000000";
let filters = [filter1, filter2, filter3].join(",");
// Args can be parsed.
let result = parse_receipts_log_filter(&filters);
assert!(result.is_ok());
let config = result.unwrap();
assert_eq!(config.0.len(), 3);
// Check that the args were parsed correctly.
let addr1: Address = "0x0000000000000000000000000000000000000001".parse().unwrap();
let addr2: Address = "0x0000000000000000000000000000000000000002".parse().unwrap();
let addr3: Address = "0x0000000000000000000000000000000000000003".parse().unwrap();
assert_eq!(config.0.get(&addr1), Some(&PruneMode::Full));
assert_eq!(config.0.get(&addr2), Some(&PruneMode::Distance(1000)));
assert_eq!(config.0.get(&addr3), Some(&PruneMode::Before(5000000)));
}
#[test]
fn test_parse_receipts_log_filter_invalid_filter_format() {
let result = parse_receipts_log_filter("invalid_format");
assert!(matches!(result, Err(ReceiptsLogError::InvalidFilterFormat(_))));
}
#[test]
fn test_parse_receipts_log_filter_invalid_address() {
let result = parse_receipts_log_filter("invalid_address:full");
assert!(matches!(result, Err(ReceiptsLogError::InvalidAddress(_))));
}
#[test]
fn test_parse_receipts_log_filter_invalid_prune_mode() {
let result =
parse_receipts_log_filter("0x0000000000000000000000000000000000000000:invalid_mode");
assert!(matches!(result, Err(ReceiptsLogError::InvalidPruneMode(_))));
}
#[test]
fn test_parse_receipts_log_filter_invalid_distance() {
let result = parse_receipts_log_filter(
"0x0000000000000000000000000000000000000000:distance:invalid_distance",
);
assert!(matches!(result, Err(ReceiptsLogError::InvalidDistance(_))));
}
#[test]
fn test_parse_receipts_log_filter_invalid_block_number() {
let result = parse_receipts_log_filter(
"0x0000000000000000000000000000000000000000:before:invalid_block",
);
assert!(matches!(result, Err(ReceiptsLogError::InvalidBlockNumber(_))));
}
}

2
crates/node/core/src/args/trace.rs
View File

@@ -39,7 +39,7 @@ pub struct TraceArgs {
long = "tracing-otlp.filter",
global = true,
value_name = "FILTER",
default_value = "TRACE",
default_value = "debug",
help_heading = "Tracing"
)]
pub otlp_filter: EnvFilter,

7
crates/node/core/src/cli/config.rs
View File

@@ -42,10 +42,9 @@ pub trait PayloadBuilderConfig {
}
match chain.kind() {
ChainKind::Named(NamedChain::Sepolia | NamedChain::Holesky | NamedChain::Hoodi) => {
ETHEREUM_BLOCK_GAS_LIMIT_60M
}
ChainKind::Named(NamedChain::Mainnet) => ETHEREUM_BLOCK_GAS_LIMIT_60M,
ChainKind::Named(
NamedChain::Mainnet | NamedChain::Sepolia | NamedChain::Holesky | NamedChain::Hoodi,
) => ETHEREUM_BLOCK_GAS_LIMIT_60M,
_ => ETHEREUM_BLOCK_GAS_LIMIT_36M,
}
}

29
crates/node/core/src/node_config.rs
View File

@@ -10,7 +10,7 @@ use crate::{
};
use alloy_consensus::BlockHeader;
use alloy_eips::BlockHashOrNumber;
use alloy_primitives::{BlockNumber, B256};
use alloy_primitives::{BlockNumber, B256, U256};
use eyre::eyre;
use reth_chainspec::{ChainSpec, EthChainSpec, MAINNET};
use reth_config::config::PruneConfig;
@@ -34,8 +34,7 @@ use tracing::*;
use crate::args::{EraArgs, MetricArgs};
pub use reth_engine_primitives::{
DEFAULT_MAX_PROOF_TASK_CONCURRENCY, DEFAULT_MEMORY_BLOCK_BUFFER_TARGET,
DEFAULT_PERSISTENCE_THRESHOLD, DEFAULT_RESERVED_CPU_CORES,
DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, DEFAULT_PERSISTENCE_THRESHOLD, DEFAULT_RESERVED_CPU_CORES,
};
/// Default size of cross-block cache in megabytes.
@@ -191,6 +190,22 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
self
}
/// Apply a function to the config.
pub fn apply<F>(self, f: F) -> Self
where
F: FnOnce(Self) -> Self,
{
f(self)
}
/// Applies a fallible function to the config.
pub fn try_apply<F, R>(self, f: F) -> Result<Self, R>
where
F: FnOnce(Self) -> Result<Self, R>,
{
f(self)
}
/// Sets --dev mode for the node [`NodeConfig::dev`], if `dev` is true.
pub const fn set_dev(self, dev: bool) -> Self {
if dev {
@@ -330,12 +345,6 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
.header_by_number(head)?
.expect("the header for the latest block is missing, database is corrupt");
let total_difficulty = provider
.header_td_by_number(head)?
// total difficulty is effectively deprecated, but still required in some places, e.g.
// p2p
.unwrap_or_default();
let hash = provider
.block_hash(head)?
.expect("the hash for the latest block is missing, database is corrupt");
@@ -344,7 +353,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
number: head,
hash,
difficulty: header.difficulty(),
total_difficulty,
total_difficulty: U256::ZERO,
timestamp: header.timestamp(),
})
}

2
crates/node/types/src/lib.rs
View File

@@ -11,7 +11,7 @@
use core::{fmt::Debug, marker::PhantomData};
pub use reth_primitives_traits::{
Block, BlockBody, FullBlock, FullNodePrimitives, FullReceipt, FullSignedTx, NodePrimitives,
Block, BlockBody, FullBlock, FullReceipt, FullSignedTx, NodePrimitives,
};
use reth_chainspec::EthChainSpec;

84
crates/optimism/cli/src/commands/init_state.rs
View File

@@ -12,8 +12,8 @@ use reth_optimism_primitives::{
};
use reth_primitives_traits::SealedHeader;
use reth_provider::{
BlockNumReader, ChainSpecProvider, DBProvider, DatabaseProviderFactory,
StaticFileProviderFactory, StaticFileWriter,
BlockNumReader, DBProvider, DatabaseProviderFactory, StaticFileProviderFactory,
StaticFileWriter,
};
use std::{io::BufReader, sync::Arc};
use tracing::info;
@@ -24,12 +24,11 @@ pub struct InitStateCommandOp<C: ChainSpecParser> {
#[command(flatten)]
init_state: reth_cli_commands::init_state::InitStateCommand<C>,
/// **Optimism Mainnet Only**
///
/// Specifies whether to initialize the state without relying on OVM historical data.
/// Specifies whether to initialize the state without relying on OVM or EVM historical data.
///
/// When enabled, and before inserting the state, it creates a dummy chain up to the last OVM
/// block (#105235062) (14GB / 90 seconds). It then, appends the Bedrock block.
/// block (#105235062) (14GB / 90 seconds). It then, appends the Bedrock block. This is
/// hardcoded for OP mainnet, for other OP chains you will need to pass in a header.
///
/// - **Note**: **Do not** import receipts and blocks beforehand, or this will fail or be
/// ignored.
@@ -40,42 +39,59 @@ pub struct InitStateCommandOp<C: ChainSpecParser> {
impl<C: ChainSpecParser<ChainSpec = OpChainSpec>> InitStateCommandOp<C> {
/// Execute the `init` command
pub async fn execute<N: CliNodeTypes<ChainSpec = C::ChainSpec, Primitives = OpPrimitives>>(
mut self,
) -> eyre::Result<()> {
// If using --without-ovm for OP mainnet, handle the special case with hardcoded Bedrock
// header. Otherwise delegate to the base InitStateCommand implementation.
if self.without_ovm {
if self.init_state.env.chain.is_optimism_mainnet() {
return self.execute_with_bedrock_header::<N>();
}
// For non-mainnet OP chains with --without-ovm, use the base implementation
// by setting the without_evm flag
self.init_state.without_evm = true;
}
self.init_state.execute::<N>().await
}
/// Execute init-state with hardcoded Bedrock header for OP mainnet.
fn execute_with_bedrock_header<
N: CliNodeTypes<ChainSpec = C::ChainSpec, Primitives = OpPrimitives>,
>(
self,
) -> eyre::Result<()> {
info!(target: "reth::cli", "Reth init-state starting");
let Environment { config, provider_factory, .. } =
self.init_state.env.init::<N>(AccessRights::RW)?;
info!(target: "reth::cli", "Reth init-state starting for OP mainnet");
let env = self.init_state.env.init::<N>(AccessRights::RW)?;
let Environment { config, provider_factory, .. } = env;
let static_file_provider = provider_factory.static_file_provider();
let provider_rw = provider_factory.database_provider_rw()?;
// OP-Mainnet may want to bootstrap a chain without OVM historical data
if provider_factory.chain_spec().is_optimism_mainnet() && self.without_ovm {
let last_block_number = provider_rw.last_block_number()?;
let last_block_number = provider_rw.last_block_number()?;
if last_block_number == 0 {
reth_cli_commands::init_state::without_evm::setup_without_evm(
&provider_rw,
SealedHeader::new(BEDROCK_HEADER, BEDROCK_HEADER_HASH),
|number| {
let mut header = Header::default();
header.set_number(number);
header
},
)?;
if last_block_number == 0 {
reth_cli_commands::init_state::without_evm::setup_without_evm(
&provider_rw,
SealedHeader::new(BEDROCK_HEADER, BEDROCK_HEADER_HASH),
|number| {
let mut header = Header::default();
header.set_number(number);
header
},
)?;
// SAFETY: it's safe to commit static files, since in the event of a crash, they
// will be unwound according to database checkpoints.
//
// Necessary to commit, so the BEDROCK_HEADER is accessible to provider_rw and
// init_state_dump
static_file_provider.commit()?;
} else if last_block_number > 0 && last_block_number < BEDROCK_HEADER.number {
return Err(eyre::eyre!(
"Data directory should be empty when calling init-state with --without-ovm."
))
}
// SAFETY: it's safe to commit static files, since in the event of a crash, they
// will be unwound according to database checkpoints.
//
// Necessary to commit, so the BEDROCK_HEADER is accessible to provider_rw and
// init_state_dump
static_file_provider.commit()?;
} else if last_block_number > 0 && last_block_number < BEDROCK_HEADER.number {
return Err(eyre::eyre!(
"Data directory should be empty when calling init-state with --without-ovm."
))
}
info!(target: "reth::cli", "Initiating state dump");

4
crates/optimism/flashblocks/Cargo.toml
View File

@@ -16,17 +16,17 @@ reth-optimism-primitives = { workspace = true, features = ["serde"] }
reth-optimism-evm.workspace = true
reth-chain-state = { workspace = true, features = ["serde"] }
reth-primitives-traits = { workspace = true, features = ["serde"] }
reth-engine-primitives = { workspace = true, features = ["std"] }
reth-execution-types = { workspace = true, features = ["serde"] }
reth-evm.workspace = true
reth-revm.workspace = true
reth-optimism-payload-builder.workspace = true
reth-rpc-eth-types.workspace = true
reth-errors.workspace = true
reth-payload-primitives.workspace = true
reth-storage-api.workspace = true
reth-node-api.workspace = true
reth-tasks.workspace = true
reth-metrics.workspace = true
reth-trie.workspace = true
# alloy
alloy-eips = { workspace = true, features = ["serde"] }

3
crates/optimism/flashblocks/src/consensus.rs
View File

@@ -1,7 +1,8 @@
use crate::FlashBlockCompleteSequenceRx;
use alloy_primitives::B256;
use reth_node_api::{ConsensusEngineHandle, EngineApiMessageVersion};
use reth_engine_primitives::ConsensusEngineHandle;
use reth_optimism_payload_builder::OpPayloadTypes;
use reth_payload_primitives::EngineApiMessageVersion;
use ringbuffer::{AllocRingBuffer, RingBuffer};
use tracing::warn;

8
crates/optimism/flashblocks/src/lib.rs
View File

@@ -1,5 +1,13 @@
//! A downstream integration of Flashblocks.
#![doc(
html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png",
html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256",
issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/"
)]
#![cfg_attr(docsrs, feature(doc_cfg))]
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
pub use payload::{
ExecutionPayloadBaseV1, ExecutionPayloadFlashblockDeltaV1, FlashBlock, FlashBlockDecoder,
Metadata,

2
crates/optimism/flashblocks/src/payload.rs
View File

@@ -3,9 +3,9 @@ use alloy_eips::eip4895::Withdrawal;
use alloy_primitives::{bytes, Address, Bloom, Bytes, B256, U256};
use alloy_rpc_types_engine::PayloadId;
use derive_more::Deref;
use reth_node_api::NodePrimitives;
use reth_optimism_evm::OpNextBlockEnvAttributes;
use reth_optimism_primitives::OpReceipt;
use reth_primitives_traits::NodePrimitives;
use reth_rpc_eth_types::PendingBlock;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;

31
crates/primitives-traits/src/block/error.rs
View File

@@ -3,6 +3,37 @@
use crate::transaction::signed::RecoveryError;
/// Type alias for [`BlockRecoveryError`] with a [`SealedBlock`](crate::SealedBlock) value.
///
/// This error type is specifically used when recovering a sealed block fails.
/// It contains the original sealed block that could not be recovered, allowing
/// callers to inspect the problematic block or attempt recovery with different
/// parameters.
///
/// # Example
///
/// ```rust
/// use alloy_consensus::{Block, BlockBody, Header, Signed, TxEnvelope, TxLegacy};
/// use alloy_primitives::{Signature, B256};
/// use reth_primitives_traits::{block::error::SealedBlockRecoveryError, SealedBlock};
///
/// // Create a simple block for demonstration
/// let header = Header::default();
/// let tx = TxLegacy::default();
/// let signed_tx = Signed::new_unchecked(tx, Signature::test_signature(), B256::ZERO);
/// let envelope = TxEnvelope::Legacy(signed_tx);
/// let body = BlockBody { transactions: vec![envelope], ommers: vec![], withdrawals: None };
/// let block = Block::new(header, body);
/// let sealed_block = SealedBlock::new_unchecked(block, B256::ZERO);
///
/// // Simulate a block recovery operation that fails
/// let block_recovery_result: Result<(), SealedBlockRecoveryError<_>> =
/// Err(SealedBlockRecoveryError::new(sealed_block));
///
/// // When block recovery fails, you get the error with the original block
/// let error = block_recovery_result.unwrap_err();
/// let failed_block = error.into_inner();
/// // Now you can inspect the failed block or try recovery again
/// ```
pub type SealedBlockRecoveryError<B> = BlockRecoveryError<crate::SealedBlock<B>>;
/// Error when recovering a block from [`SealedBlock`](crate::SealedBlock) to

12
crates/primitives-traits/src/block/mod.rs
View File

@@ -50,17 +50,9 @@ pub mod serde_bincode_compat {
}
/// Helper trait that unifies all behaviour required by block to support full node operations.
pub trait FullBlock:
Block<Header: FullBlockHeader, Body: FullBlockBody> + alloy_rlp::Encodable + alloy_rlp::Decodable
{
}
pub trait FullBlock: Block<Header: FullBlockHeader, Body: FullBlockBody> {}
impl<T> FullBlock for T where
T: Block<Header: FullBlockHeader, Body: FullBlockBody>
+ alloy_rlp::Encodable
+ alloy_rlp::Decodable
{
}
impl<T> FullBlock for T where T: Block<Header: FullBlockHeader, Body: FullBlockBody> {}
/// Helper trait to access [`BlockBody::Transaction`] given a [`Block`].
pub type BlockTx<B> = <<B as Block>::Body as BlockBody>::Transaction;

2
crates/primitives-traits/src/lib.rs
View File

@@ -188,7 +188,7 @@ pub use size::InMemorySize;
/// Node traits
pub mod node;
pub use node::{BlockTy, BodyTy, FullNodePrimitives, HeaderTy, NodePrimitives, ReceiptTy, TxTy};
pub use node::{BlockTy, BodyTy, HeaderTy, NodePrimitives, ReceiptTy, TxTy};
/// Helper trait that requires de-/serialize implementation since `serde` feature is enabled.
#[cfg(feature = "serde")]

31
crates/primitives-traits/src/node.rs
View File

@@ -1,6 +1,5 @@
use crate::{
Block, FullBlock, FullBlockBody, FullBlockHeader, FullReceipt, FullSignedTx,
MaybeSerdeBincodeCompat, Receipt,
FullBlock, FullBlockBody, FullBlockHeader, FullReceipt, FullSignedTx, MaybeSerdeBincodeCompat,
};
use core::fmt;
@@ -13,7 +12,8 @@ pub trait NodePrimitives:
Send + Sync + Unpin + Clone + Default + fmt::Debug + PartialEq + Eq + 'static
{
/// Block primitive.
type Block: Block<Header = Self::BlockHeader, Body = Self::BlockBody> + MaybeSerdeBincodeCompat;
type Block: FullBlock<Header = Self::BlockHeader, Body = Self::BlockBody>
+ MaybeSerdeBincodeCompat;
/// Block header primitive.
type BlockHeader: FullBlockHeader;
/// Block body primitive.
@@ -24,30 +24,7 @@ pub trait NodePrimitives:
/// format that includes the signature and can be included in a block.
type SignedTx: FullSignedTx;
/// A receipt.
type Receipt: Receipt;
}
/// Helper trait that sets trait bounds on [`NodePrimitives`].
pub trait FullNodePrimitives
where
Self: NodePrimitives<
Block: FullBlock<Header = Self::BlockHeader, Body = Self::BlockBody>,
BlockHeader: FullBlockHeader,
BlockBody: FullBlockBody<Transaction = Self::SignedTx>,
SignedTx: FullSignedTx,
Receipt: FullReceipt,
>,
{
}
impl<T> FullNodePrimitives for T where
T: NodePrimitives<
Block: FullBlock<Header = Self::BlockHeader, Body = Self::BlockBody>,
BlockHeader: FullBlockHeader,
BlockBody: FullBlockBody<Transaction = Self::SignedTx>,
SignedTx: FullSignedTx,
Receipt: FullReceipt,
>
{
type Receipt: FullReceipt;
}
/// Helper adapter type for accessing [`NodePrimitives`] block header types.

2
crates/prune/prune/Cargo.toml
View File

@@ -13,7 +13,6 @@ workspace = true
[dependencies]
# reth
reth-chainspec.workspace = true
reth-exex-types.workspace = true
reth-db-api.workspace = true
reth-errors.workspace = true
@@ -25,7 +24,6 @@ reth-primitives-traits.workspace = true
reth-static-file-types.workspace = true
# ethereum
alloy-consensus.workspace = true
alloy-eips.workspace = true
# metrics

8
crates/prune/prune/src/builder.rs
View File

@@ -1,6 +1,5 @@
use crate::{segments::SegmentSet, Pruner};
use alloy_eips::eip2718::Encodable2718;
use reth_chainspec::MAINNET_PRUNE_DELETE_LIMIT;
use reth_config::PruneConfig;
use reth_db_api::{table::Value, transaction::DbTxMut};
use reth_exex_types::FinishedExExHeight;
@@ -30,9 +29,6 @@ pub struct PrunerBuilder {
}
impl PrunerBuilder {
/// Default timeout for a prune run.
pub const DEFAULT_TIMEOUT: Duration = Duration::from_millis(100);
/// Creates a new [`PrunerBuilder`] from the given [`PruneConfig`].
pub fn new(pruner_config: PruneConfig) -> Self {
Self::default()
@@ -47,7 +43,7 @@ impl PrunerBuilder {
}
/// Sets the configuration for every part of the data that can be pruned.
pub fn segments(mut self, segments: PruneModes) -> Self {
pub const fn segments(mut self, segments: PruneModes) -> Self {
self.segments = segments;
self
}
@@ -135,7 +131,7 @@ impl Default for PrunerBuilder {
Self {
block_interval: 5,
segments: PruneModes::default(),
delete_limit: MAINNET_PRUNE_DELETE_LIMIT,
delete_limit: usize::MAX,
timeout: None,
finished_exex_height: watch::channel(FinishedExExHeight::NoExExs).1,
}

4
crates/prune/prune/src/segments/mod.rs
View File

@@ -15,8 +15,8 @@ pub use static_file::{
use std::{fmt::Debug, ops::RangeInclusive};
use tracing::error;
pub use user::{
AccountHistory, MerkleChangeSets, Receipts as UserReceipts, ReceiptsByLogs, SenderRecovery,
StorageHistory, TransactionLookup,
AccountHistory, MerkleChangeSets, Receipts as UserReceipts, SenderRecovery, StorageHistory,
TransactionLookup,
};
/// A segment represents a pruning of some portion of the data.

12
crates/prune/prune/src/segments/set.rs
View File

@@ -1,6 +1,6 @@
use crate::segments::{
AccountHistory, MerkleChangeSets, ReceiptsByLogs, Segment, SenderRecovery, StorageHistory,
TransactionLookup, UserReceipts,
AccountHistory, MerkleChangeSets, Segment, SenderRecovery, StorageHistory, TransactionLookup,
UserReceipts,
};
use alloy_eips::eip2718::Encodable2718;
use reth_db_api::{table::Value, transaction::DbTxMut};
@@ -61,6 +61,7 @@ where
static_file_provider: StaticFileProvider<Provider::Primitives>,
prune_modes: PruneModes,
) -> Self {
#[expect(deprecated)]
let PruneModes {
sender_recovery,
transaction_lookup,
@@ -69,7 +70,7 @@ where
storage_history,
bodies_history: _,
merkle_changesets,
receipts_log_filter,
receipts_log_filter: (),
} = prune_modes;
Self::default()
@@ -87,11 +88,6 @@ where
.segment_opt(storage_history.map(StorageHistory::new))
// User receipts
.segment_opt(receipts.map(UserReceipts::new))
// Receipts by logs
.segment_opt(
(!receipts_log_filter.is_empty())
.then(|| ReceiptsByLogs::new(receipts_log_filter.clone())),
)
// Transaction lookup
.segment_opt(transaction_lookup.map(TransactionLookup::new))
// Sender recovery

45
crates/prune/prune/src/segments/static_file/headers.rs
View File

@@ -21,7 +21,9 @@ use std::num::NonZeroUsize;
use tracing::trace;
/// Number of header tables to prune in one step
const HEADER_TABLES_TO_PRUNE: usize = 3;
///
/// Note: `HeaderTerminalDifficulties` is no longer pruned after Paris/Merge as it's read-only
const HEADER_TABLES_TO_PRUNE: usize = 2;
#[derive(Debug)]
pub struct Headers<N> {
@@ -72,9 +74,6 @@ where
.tx_ref()
.cursor_write::<tables::Headers<<Provider::Primitives as NodePrimitives>::BlockHeader>>(
)?;
let mut header_tds_cursor =
provider.tx_ref().cursor_write::<tables::HeaderTerminalDifficulties>()?;
let mut canonical_headers_cursor =
provider.tx_ref().cursor_write::<tables::CanonicalHeaders>()?;
@@ -86,7 +85,6 @@ where
provider,
&mut limiter,
headers_cursor.walk_range(range.clone())?,
header_tds_cursor.walk_range(range.clone())?,
canonical_headers_cursor.walk_range(range)?,
);
@@ -111,6 +109,7 @@ where
})
}
}
type Walker<'a, Provider, T> =
RangeWalker<'a, T, <<Provider as DBProvider>::Tx as DbTxMut>::CursorMut<T>>;
@@ -127,7 +126,6 @@ where
Provider,
tables::Headers<<Provider::Primitives as NodePrimitives>::BlockHeader>,
>,
header_tds_walker: Walker<'a, Provider, tables::HeaderTerminalDifficulties>,
canonical_headers_walker: Walker<'a, Provider, tables::CanonicalHeaders>,
}
@@ -149,10 +147,9 @@ where
Provider,
tables::Headers<<Provider::Primitives as NodePrimitives>::BlockHeader>,
>,
header_tds_walker: Walker<'a, Provider, tables::HeaderTerminalDifficulties>,
canonical_headers_walker: Walker<'a, Provider, tables::CanonicalHeaders>,
) -> Self {
Self { provider, limiter, headers_walker, header_tds_walker, canonical_headers_walker }
Self { provider, limiter, headers_walker, canonical_headers_walker }
}
}
@@ -168,7 +165,6 @@ where
}
let mut pruned_block_headers = None;
let mut pruned_block_td = None;
let mut pruned_block_canonical = None;
if let Err(err) = self.provider.tx_ref().prune_table_with_range_step(
@@ -180,15 +176,6 @@ where
return Some(Err(err.into()))
}
if let Err(err) = self.provider.tx_ref().prune_table_with_range_step(
&mut self.header_tds_walker,
self.limiter,
&mut |_| false,
&mut |row| pruned_block_td = Some(row.0),
) {
return Some(Err(err.into()))
}
if let Err(err) = self.provider.tx_ref().prune_table_with_range_step(
&mut self.canonical_headers_walker,
self.limiter,
@@ -198,7 +185,7 @@ where
return Some(Err(err.into()))
}
if ![pruned_block_headers, pruned_block_td, pruned_block_canonical].iter().all_equal() {
if ![pruned_block_headers, pruned_block_canonical].iter().all_equal() {
return Some(Err(PrunerError::InconsistentData(
"All headers-related tables should be pruned up to the same height",
)))
@@ -216,7 +203,7 @@ mod tests {
static_file::headers::HEADER_TABLES_TO_PRUNE, PruneInput, PruneLimiter, Segment,
SegmentOutput,
};
use alloy_primitives::{BlockNumber, B256, U256};
use alloy_primitives::{BlockNumber, B256};
use assert_matches::assert_matches;
use reth_db_api::{tables, transaction::DbTx};
use reth_provider::{
@@ -241,18 +228,17 @@ mod tests {
let headers = random_header_range(&mut rng, 0..100, B256::ZERO);
let tx = db.factory.provider_rw().unwrap().into_tx();
for header in &headers {
TestStageDB::insert_header(None, &tx, header, U256::ZERO).unwrap();
TestStageDB::insert_header(None, &tx, header).unwrap();
}
tx.commit().unwrap();
assert_eq!(db.table::<tables::CanonicalHeaders>().unwrap().len(), headers.len());
assert_eq!(db.table::<tables::Headers>().unwrap().len(), headers.len());
assert_eq!(db.table::<tables::HeaderTerminalDifficulties>().unwrap().len(), headers.len());
let test_prune = |to_block: BlockNumber, expected_result: (PruneProgress, usize)| {
let segment = super::Headers::new(db.factory.static_file_provider());
let prune_mode = PruneMode::Before(to_block);
let mut limiter = PruneLimiter::default().set_deleted_entries_limit(10);
let mut limiter = PruneLimiter::default().set_deleted_entries_limit(6);
let input = PruneInput {
previous_checkpoint: db
.factory
@@ -311,10 +297,6 @@ mod tests {
db.table::<tables::Headers>().unwrap().len(),
headers.len() - (last_pruned_block_number + 1) as usize
);
assert_eq!(
db.table::<tables::HeaderTerminalDifficulties>().unwrap().len(),
headers.len() - (last_pruned_block_number + 1) as usize
);
assert_eq!(
db.factory.provider().unwrap().get_prune_checkpoint(PruneSegment::Headers).unwrap(),
Some(PruneCheckpoint {
@@ -325,11 +307,16 @@ mod tests {
);
};
// First test: Prune with limit of 6 entries
// This will prune blocks 0-2 (3 blocks × 2 tables = 6 entries)
test_prune(
3,
(PruneProgress::HasMoreData(PruneInterruptReason::DeletedEntriesLimitReached), 9),
(PruneProgress::HasMoreData(PruneInterruptReason::DeletedEntriesLimitReached), 6),
);
test_prune(3, (PruneProgress::Finished, 3));
// Second test: Prune remaining blocks
// This will prune block 3 (1 block × 2 tables = 2 entries)
test_prune(3, (PruneProgress::Finished, 2));
}
#[test]

2
crates/prune/prune/src/segments/user/mod.rs
View File

@@ -2,7 +2,6 @@ mod account_history;
mod history;
mod merkle_change_sets;
mod receipts;
mod receipts_by_logs;
mod sender_recovery;
mod storage_history;
mod transaction_lookup;
@@ -10,7 +9,6 @@ mod transaction_lookup;
pub use account_history::AccountHistory;
pub use merkle_change_sets::MerkleChangeSets;
pub use receipts::Receipts;
pub use receipts_by_logs::ReceiptsByLogs;
pub use sender_recovery::SenderRecovery;
pub use storage_history::StorageHistory;
pub use transaction_lookup::TransactionLookup;

364
crates/prune/prune/src/segments/user/receipts_by_logs.rs
View File

@@ -1,364 +0,0 @@
use crate::{
db_ext::DbTxPruneExt,
segments::{PruneInput, Segment},
PrunerError,
};
use alloy_consensus::TxReceipt;
use reth_db_api::{table::Value, tables, transaction::DbTxMut};
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
BlockReader, DBProvider, NodePrimitivesProvider, PruneCheckpointWriter, TransactionsProvider,
};
use reth_prune_types::{
PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment, ReceiptsLogPruneConfig, SegmentOutput,
MINIMUM_PRUNING_DISTANCE,
};
use tracing::{instrument, trace};
#[derive(Debug)]
pub struct ReceiptsByLogs {
config: ReceiptsLogPruneConfig,
}
impl ReceiptsByLogs {
pub const fn new(config: ReceiptsLogPruneConfig) -> Self {
Self { config }
}
}
impl<Provider> Segment<Provider> for ReceiptsByLogs
where
Provider: DBProvider<Tx: DbTxMut>
+ PruneCheckpointWriter
+ TransactionsProvider
+ BlockReader
+ NodePrimitivesProvider<Primitives: NodePrimitives<Receipt: Value>>,
{
fn segment(&self) -> PruneSegment {
PruneSegment::ContractLogs
}
fn mode(&self) -> Option<PruneMode> {
None
}
fn purpose(&self) -> PrunePurpose {
PrunePurpose::User
}
#[instrument(target = "pruner", skip(self, provider), ret(level = "trace"))]
fn prune(&self, provider: &Provider, input: PruneInput) -> Result<SegmentOutput, PrunerError> {
// Contract log filtering removes every receipt possible except the ones in the list. So,
// for the other receipts it's as if they had a `PruneMode::Distance()` of
// `MINIMUM_PRUNING_DISTANCE`.
let to_block = PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)
.prune_target_block(input.to_block, PruneSegment::ContractLogs, PrunePurpose::User)?
.map(|(bn, _)| bn)
.unwrap_or_default();
// Get status checkpoint from latest run
let mut last_pruned_block =
input.previous_checkpoint.and_then(|checkpoint| checkpoint.block_number);
let initial_last_pruned_block = last_pruned_block;
let mut from_tx_number = match initial_last_pruned_block {
Some(block) => provider
.block_body_indices(block)?
.map(|block| block.last_tx_num() + 1)
.unwrap_or(0),
None => 0,
};
// Figure out what receipts have already been pruned, so we can have an accurate
// `address_filter`
let address_filter = self.config.group_by_block(input.to_block, last_pruned_block)?;
// Splits all transactions in different block ranges. Each block range will have its own
// filter address list and will check it while going through the table
//
// Example:
// For an `address_filter` such as:
// { block9: [a1, a2], block20: [a3, a4, a5] }
//
// The following structures will be created in the exact order as showed:
// `block_ranges`: [
// (block0, block8, 0 addresses),
// (block9, block19, 2 addresses),
// (block20, to_block, 5 addresses)
// ]
// `filtered_addresses`: [a1, a2, a3, a4, a5]
//
// The first range will delete all receipts between block0 - block8
// The second range will delete all receipts between block9 - 19, except the ones with
// emitter logs from these addresses: [a1, a2].
// The third range will delete all receipts between block20 - to_block, except the ones with
// emitter logs from these addresses: [a1, a2, a3, a4, a5]
let mut block_ranges = vec![];
let mut blocks_iter = address_filter.iter().peekable();
let mut filtered_addresses = vec![];
while let Some((start_block, addresses)) = blocks_iter.next() {
filtered_addresses.extend_from_slice(addresses);
// This will clear all receipts before the first appearance of a contract log or since
// the block after the last pruned one.
if block_ranges.is_empty() {
let init = last_pruned_block.map(|b| b + 1).unwrap_or_default();
if init < *start_block {
block_ranges.push((init, *start_block - 1, 0));
}
}
let end_block =
blocks_iter.peek().map(|(next_block, _)| *next_block - 1).unwrap_or(to_block);
// Addresses in lower block ranges, are still included in the inclusion list for future
// ranges.
block_ranges.push((*start_block, end_block, filtered_addresses.len()));
}
trace!(
target: "pruner",
?block_ranges,
?filtered_addresses,
"Calculated block ranges and filtered addresses",
);
let mut limiter = input.limiter;
let mut done = true;
let mut pruned = 0;
let mut last_pruned_transaction = None;
for (start_block, end_block, num_addresses) in block_ranges {
let block_range = start_block..=end_block;
// Calculate the transaction range from this block range
let tx_range_end = match provider.block_body_indices(end_block)? {
Some(body) => body.last_tx_num(),
None => {
trace!(
target: "pruner",
?block_range,
"No receipts to prune."
);
continue
}
};
let tx_range = from_tx_number..=tx_range_end;
// Delete receipts, except the ones in the inclusion list
let mut last_skipped_transaction = 0;
let deleted;
(deleted, done) = provider.tx_ref().prune_table_with_range::<tables::Receipts<
<Provider::Primitives as NodePrimitives>::Receipt,
>>(
tx_range,
&mut limiter,
|(tx_num, receipt)| {
let skip = num_addresses > 0 &&
receipt.logs().iter().any(|log| {
filtered_addresses[..num_addresses].contains(&&log.address)
});
if skip {
last_skipped_transaction = *tx_num;
}
skip
},
|row| last_pruned_transaction = Some(row.0),
)?;
trace!(target: "pruner", %deleted, %done, ?block_range, "Pruned receipts");
pruned += deleted;
// For accurate checkpoints we need to know that we have checked every transaction.
// Example: we reached the end of the range, and the last receipt is supposed to skip
// its deletion.
let last_pruned_transaction = *last_pruned_transaction
.insert(last_pruned_transaction.unwrap_or_default().max(last_skipped_transaction));
last_pruned_block = Some(
provider
.transaction_block(last_pruned_transaction)?
.ok_or(PrunerError::InconsistentData("Block for transaction is not found"))?
// If there's more receipts to prune, set the checkpoint block number to
// previous, so we could finish pruning its receipts on the
// next run.
.saturating_sub(if done { 0 } else { 1 }),
);
if limiter.is_limit_reached() {
done &= end_block == to_block;
break
}
from_tx_number = last_pruned_transaction + 1;
}
// If there are contracts using `PruneMode::Distance(_)` there will be receipts before
// `to_block` that become eligible to be pruned in future runs. Therefore, our checkpoint is
// not actually `to_block`, but the `lowest_block_with_distance` from any contract.
// This ensures that in future pruner runs we can prune all these receipts between the
// previous `lowest_block_with_distance` and the new one using
// `get_next_tx_num_range_from_checkpoint`.
//
// Only applies if we were able to prune everything intended for this run, otherwise the
// checkpoint is the `last_pruned_block`.
let prune_mode_block = self
.config
.lowest_block_with_distance(input.to_block, initial_last_pruned_block)?
.unwrap_or(to_block);
provider.save_prune_checkpoint(
PruneSegment::ContractLogs,
PruneCheckpoint {
block_number: Some(prune_mode_block.min(last_pruned_block.unwrap_or(u64::MAX))),
tx_number: last_pruned_transaction,
prune_mode: PruneMode::Before(prune_mode_block),
},
)?;
let progress = limiter.progress(done);
Ok(SegmentOutput { progress, pruned, checkpoint: None })
}
}
#[cfg(test)]
mod tests {
use crate::segments::{PruneInput, PruneLimiter, ReceiptsByLogs, Segment};
use alloy_primitives::B256;
use assert_matches::assert_matches;
use reth_db_api::{cursor::DbCursorRO, tables, transaction::DbTx};
use reth_primitives_traits::InMemorySize;
use reth_provider::{
DBProvider, DatabaseProviderFactory, PruneCheckpointReader, TransactionsProvider,
};
use reth_prune_types::{PruneMode, PruneSegment, ReceiptsLogPruneConfig};
use reth_stages::test_utils::{StorageKind, TestStageDB};
use reth_testing_utils::generators::{
self, random_block_range, random_eoa_account, random_log, random_receipt, BlockRangeParams,
};
use std::collections::BTreeMap;
#[test]
fn prune_receipts_by_logs() {
reth_tracing::init_test_tracing();
let db = TestStageDB::default();
let mut rng = generators::rng();
let tip = 20000;
let blocks = [
random_block_range(
&mut rng,
0..=100,
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 1..5, ..Default::default() },
),
random_block_range(
&mut rng,
(100 + 1)..=(tip - 100),
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() },
),
random_block_range(
&mut rng,
(tip - 100 + 1)..=tip,
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 1..5, ..Default::default() },
),
]
.concat();
db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
let mut receipts = Vec::new();
let (deposit_contract_addr, _) = random_eoa_account(&mut rng);
for block in &blocks {
receipts.reserve_exact(block.body().size());
for (txi, transaction) in block.body().transactions.iter().enumerate() {
let mut receipt = random_receipt(&mut rng, transaction, Some(1), None);
receipt.logs.push(random_log(
&mut rng,
(txi == (block.transaction_count() - 1)).then_some(deposit_contract_addr),
Some(1),
));
receipts.push((receipts.len() as u64, receipt));
}
}
db.insert_receipts(receipts).expect("insert receipts");
assert_eq!(
db.table::<tables::Transactions>().unwrap().len(),
blocks.iter().map(|block| block.transaction_count()).sum::<usize>()
);
assert_eq!(
db.table::<tables::Transactions>().unwrap().len(),
db.table::<tables::Receipts>().unwrap().len()
);
let run_prune = || {
let provider = db.factory.database_provider_rw().unwrap();
let prune_before_block: usize = 20;
let prune_mode = PruneMode::Before(prune_before_block as u64);
let receipts_log_filter =
ReceiptsLogPruneConfig(BTreeMap::from([(deposit_contract_addr, prune_mode)]));
let limiter = PruneLimiter::default().set_deleted_entries_limit(10);
let result = ReceiptsByLogs::new(receipts_log_filter).prune(
&provider,
PruneInput {
previous_checkpoint: db
.factory
.provider()
.unwrap()
.get_prune_checkpoint(PruneSegment::ContractLogs)
.unwrap(),
to_block: tip,
limiter,
},
);
provider.commit().expect("commit");
assert_matches!(result, Ok(_));
let output = result.unwrap();
let (pruned_block, pruned_tx) = db
.factory
.provider()
.unwrap()
.get_prune_checkpoint(PruneSegment::ContractLogs)
.unwrap()
.map(|checkpoint| (checkpoint.block_number.unwrap(), checkpoint.tx_number.unwrap()))
.unwrap_or_default();
// All receipts are in the end of the block
let unprunable = pruned_block.saturating_sub(prune_before_block as u64 - 1);
assert_eq!(
db.table::<tables::Receipts>().unwrap().len(),
blocks.iter().map(|block| block.transaction_count()).sum::<usize>() -
((pruned_tx + 1) - unprunable) as usize
);
output.progress.is_finished()
};
while !run_prune() {}
let provider = db.factory.provider().unwrap();
let mut cursor = provider.tx_ref().cursor_read::<tables::Receipts>().unwrap();
let walker = cursor.walk(None).unwrap();
for receipt in walker {
let (tx_num, receipt) = receipt.unwrap();
// Either we only find our contract, or the receipt is part of the unprunable receipts
// set by tip - 128
assert!(
receipt.logs.iter().any(|l| l.address == deposit_contract_addr) ||
provider.transaction_block(tx_num).unwrap().unwrap() > tip - 128,
);
}
}
}

301
crates/prune/types/src/lib.rs
View File

@@ -18,10 +18,6 @@ mod pruner;
mod segment;
mod target;
use alloc::{collections::BTreeMap, vec::Vec};
use alloy_primitives::{Address, BlockNumber};
use core::ops::Deref;
pub use checkpoint::PruneCheckpoint;
pub use event::PrunerEvent;
pub use mode::PruneMode;
@@ -31,300 +27,3 @@ pub use pruner::{
};
pub use segment::{PrunePurpose, PruneSegment, PruneSegmentError};
pub use target::{PruneModes, UnwindTargetPrunedError, MINIMUM_PRUNING_DISTANCE};
/// Configuration for pruning receipts not associated with logs emitted by the specified contracts.
#[derive(Debug, Clone, PartialEq, Eq, Default)]
#[cfg_attr(any(test, feature = "serde"), derive(serde::Serialize, serde::Deserialize))]
pub struct ReceiptsLogPruneConfig(pub BTreeMap<Address, PruneMode>);
impl ReceiptsLogPruneConfig {
/// Checks if the configuration is empty
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
/// Given the `tip` block number, consolidates the structure so it can easily be queried for
/// filtering across a range of blocks.
///
/// Example:
///
/// `{ addrA: Before(872), addrB: Before(500), addrC: Distance(128) }`
///
/// for `tip: 1000`, gets transformed to a map such as:
///
/// `{ 500: [addrB], 872: [addrA, addrC] }`
///
/// The [`BlockNumber`] key of the new map should be viewed as `PruneMode::Before(block)`, which
/// makes the previous result equivalent to
///
/// `{ Before(500): [addrB], Before(872): [addrA, addrC] }`
pub fn group_by_block(
&self,
tip: BlockNumber,
pruned_block: Option<BlockNumber>,
) -> Result<BTreeMap<BlockNumber, Vec<&Address>>, PruneSegmentError> {
let mut map = BTreeMap::new();
let base_block = pruned_block.unwrap_or_default() + 1;
for (address, mode) in &self.0 {
// Getting `None`, means that there is nothing to prune yet, so we need it to include in
// the BTreeMap (block = 0), otherwise it will be excluded.
// Reminder that this BTreeMap works as an inclusion list that excludes (prunes) all
// other receipts.
//
// Reminder, that we increment because the [`BlockNumber`] key of the new map should be
// viewed as `PruneMode::Before(block)`
let block = base_block.max(
mode.prune_target_block(tip, PruneSegment::ContractLogs, PrunePurpose::User)?
.map(|(block, _)| block)
.unwrap_or_default() +
1,
);
map.entry(block).or_insert_with(Vec::new).push(address)
}
Ok(map)
}
/// Returns the lowest block where we start filtering logs which use `PruneMode::Distance(_)`.
pub fn lowest_block_with_distance(
&self,
tip: BlockNumber,
pruned_block: Option<BlockNumber>,
) -> Result<Option<BlockNumber>, PruneSegmentError> {
let pruned_block = pruned_block.unwrap_or_default();
let mut lowest = None;
for mode in self.values() {
if mode.is_distance() &&
let Some((block, _)) =
mode.prune_target_block(tip, PruneSegment::ContractLogs, PrunePurpose::User)?
{
lowest = Some(lowest.unwrap_or(u64::MAX).min(block));
}
}
Ok(lowest.map(|lowest| lowest.max(pruned_block)))
}
}
impl Deref for ReceiptsLogPruneConfig {
type Target = BTreeMap<Address, PruneMode>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_group_by_block_empty_config() {
let config = ReceiptsLogPruneConfig(BTreeMap::new());
let tip = 1000;
let pruned_block = None;
let result = config.group_by_block(tip, pruned_block).unwrap();
assert!(result.is_empty(), "The result should be empty when the config is empty");
}
#[test]
fn test_group_by_block_single_entry() {
let mut config_map = BTreeMap::new();
let address = Address::new([1; 20]);
let prune_mode = PruneMode::Before(500);
config_map.insert(address, prune_mode);
let config = ReceiptsLogPruneConfig(config_map);
// Big tip to have something to prune for the target block
let tip = 3000000;
let pruned_block = Some(400);
let result = config.group_by_block(tip, pruned_block).unwrap();
// Expect one entry with block 500 and the corresponding address
assert_eq!(result.len(), 1);
assert_eq!(result[&500], vec![&address], "Address should be grouped under block 500");
// Tip smaller than the target block, so that we have nothing to prune for the block
let tip = 300;
let pruned_block = Some(400);
let result = config.group_by_block(tip, pruned_block).unwrap();
// Expect one entry with block 400 and the corresponding address
assert_eq!(result.len(), 1);
assert_eq!(result[&401], vec![&address], "Address should be grouped under block 400");
}
#[test]
fn test_group_by_block_multiple_entries() {
let mut config_map = BTreeMap::new();
let address1 = Address::new([1; 20]);
let address2 = Address::new([2; 20]);
let prune_mode1 = PruneMode::Before(600);
let prune_mode2 = PruneMode::Before(800);
config_map.insert(address1, prune_mode1);
config_map.insert(address2, prune_mode2);
let config = ReceiptsLogPruneConfig(config_map);
let tip = 900000;
let pruned_block = Some(400);
let result = config.group_by_block(tip, pruned_block).unwrap();
// Expect two entries: one for block 600 and another for block 800
assert_eq!(result.len(), 2);
assert_eq!(result[&600], vec![&address1], "Address1 should be grouped under block 600");
assert_eq!(result[&800], vec![&address2], "Address2 should be grouped under block 800");
}
#[test]
fn test_group_by_block_with_distance_prune_mode() {
let mut config_map = BTreeMap::new();
let address = Address::new([1; 20]);
let prune_mode = PruneMode::Distance(100000);
config_map.insert(address, prune_mode);
let config = ReceiptsLogPruneConfig(config_map);
let tip = 100100;
// Pruned block is smaller than the target block
let pruned_block = Some(50);
let result = config.group_by_block(tip, pruned_block).unwrap();
// Expect the entry to be grouped under block 100 (tip - distance)
assert_eq!(result.len(), 1);
assert_eq!(result[&101], vec![&address], "Address should be grouped under block 100");
let tip = 100100;
// Pruned block is larger than the target block
let pruned_block = Some(800);
let result = config.group_by_block(tip, pruned_block).unwrap();
// Expect the entry to be grouped under block 800 which is larger than tip - distance
assert_eq!(result.len(), 1);
assert_eq!(result[&801], vec![&address], "Address should be grouped under block 800");
}
#[test]
fn test_lowest_block_with_distance_empty_config() {
let config = ReceiptsLogPruneConfig(BTreeMap::new());
let tip = 1000;
let pruned_block = None;
let result = config.lowest_block_with_distance(tip, pruned_block).unwrap();
assert_eq!(result, None, "The result should be None when the config is empty");
}
#[test]
fn test_lowest_block_with_distance_no_distance_mode() {
let mut config_map = BTreeMap::new();
let address = Address::new([1; 20]);
let prune_mode = PruneMode::Before(500);
config_map.insert(address, prune_mode);
let config = ReceiptsLogPruneConfig(config_map);
let tip = 1000;
let pruned_block = None;
let result = config.lowest_block_with_distance(tip, pruned_block).unwrap();
assert_eq!(result, None, "The result should be None when there are no Distance modes");
}
#[test]
fn test_lowest_block_with_distance_single_entry() {
let mut config_map = BTreeMap::new();
let address = Address::new([1; 20]);
let prune_mode = PruneMode::Distance(100000);
config_map.insert(address, prune_mode);
let config = ReceiptsLogPruneConfig(config_map);
let tip = 100100;
let pruned_block = Some(400);
// Expect the lowest block to be 400 as 400 > 100100 - 100000 (tip - distance)
assert_eq!(
config.lowest_block_with_distance(tip, pruned_block).unwrap(),
Some(400),
"The lowest block should be 400"
);
let tip = 100100;
let pruned_block = Some(50);
// Expect the lowest block to be 100 as 100 > 50 (pruned block)
assert_eq!(
config.lowest_block_with_distance(tip, pruned_block).unwrap(),
Some(100),
"The lowest block should be 100"
);
}
#[test]
fn test_lowest_block_with_distance_multiple_entries_last() {
let mut config_map = BTreeMap::new();
let address1 = Address::new([1; 20]);
let address2 = Address::new([2; 20]);
let prune_mode1 = PruneMode::Distance(100100);
let prune_mode2 = PruneMode::Distance(100300);
config_map.insert(address1, prune_mode1);
config_map.insert(address2, prune_mode2);
let config = ReceiptsLogPruneConfig(config_map);
let tip = 200300;
let pruned_block = Some(100);
// The lowest block should be 200300 - 100300 = 100000:
// - First iteration will return 100200 => 200300 - 100100 = 100200
// - Second iteration will return 100000 => 200300 - 100300 = 100000 < 100200
// - Final result is 100000
assert_eq!(config.lowest_block_with_distance(tip, pruned_block).unwrap(), Some(100000));
}
#[test]
fn test_lowest_block_with_distance_multiple_entries_first() {
let mut config_map = BTreeMap::new();
let address1 = Address::new([1; 20]);
let address2 = Address::new([2; 20]);
let prune_mode1 = PruneMode::Distance(100400);
let prune_mode2 = PruneMode::Distance(100300);
config_map.insert(address1, prune_mode1);
config_map.insert(address2, prune_mode2);
let config = ReceiptsLogPruneConfig(config_map);
let tip = 200300;
let pruned_block = Some(100);
// The lowest block should be 200300 - 100400 = 99900:
// - First iteration, lowest block is 200300 - 100400 = 99900
// - Second iteration, lowest block is still 99900 < 200300 - 100300 = 100000
// - Final result is 99900
assert_eq!(config.lowest_block_with_distance(tip, pruned_block).unwrap(), Some(99900));
}
#[test]
fn test_lowest_block_with_distance_multiple_entries_pruned_block() {
let mut config_map = BTreeMap::new();
let address1 = Address::new([1; 20]);
let address2 = Address::new([2; 20]);
let prune_mode1 = PruneMode::Distance(100400);
let prune_mode2 = PruneMode::Distance(100300);
config_map.insert(address1, prune_mode1);
config_map.insert(address2, prune_mode2);
let config = ReceiptsLogPruneConfig(config_map);
let tip = 200300;
let pruned_block = Some(100000);
// The lowest block should be 100000 because:
// - Lowest is 200300 - 100400 = 99900 < 200300 - 100300 = 100000
// - Lowest is compared to the pruned block 100000: 100000 > 99900
// - Finally the lowest block is 100000
assert_eq!(config.lowest_block_with_distance(tip, pruned_block).unwrap(), Some(100000));
}
}

3
crates/prune/types/src/segment.rs
View File

@@ -24,8 +24,7 @@ pub enum PruneSegment {
AccountHistory,
/// Prune segment responsible for the `StorageChangeSets` and `StoragesHistory` tables.
StorageHistory,
/// Prune segment responsible for the `CanonicalHeaders`, `Headers` and
/// `HeaderTerminalDifficulties` tables.
/// Prune segment responsible for the `CanonicalHeaders`, `Headers` tables.
Headers,
/// Prune segment responsible for the `Transactions` table.
Transactions,

24
crates/prune/types/src/target.rs
View File

@@ -2,7 +2,7 @@ use alloy_primitives::BlockNumber;
use derive_more::Display;
use thiserror::Error;
use crate::{PruneCheckpoint, PruneMode, PruneSegment, ReceiptsLogPruneConfig};
use crate::{PruneCheckpoint, PruneMode, PruneSegment};
/// Minimum distance from the tip necessary for the node to work correctly:
/// 1. Minimum 2 epochs (32 blocks per epoch) required to handle any reorg according to the
@@ -99,12 +99,10 @@ pub struct PruneModes {
)
)]
pub merkle_changesets: PruneMode,
/// Receipts pruning configuration by retaining only those receipts that contain logs emitted
/// by the specified addresses, discarding others. This setting is overridden by `receipts`.
///
/// The [`BlockNumber`](`crate::BlockNumber`) represents the starting block from which point
/// onwards the receipts are preserved.
pub receipts_log_filter: ReceiptsLogPruneConfig,
/// Receipts log filtering has been deprecated and will be removed in a future release.
#[deprecated]
#[cfg_attr(any(test, feature = "serde"), serde(skip))]
pub receipts_log_filter: (),
}
impl Default for PruneModes {
@@ -117,14 +115,15 @@ impl Default for PruneModes {
storage_history: None,
bodies_history: None,
merkle_changesets: default_merkle_changesets_mode(),
receipts_log_filter: ReceiptsLogPruneConfig::default(),
#[expect(deprecated)]
receipts_log_filter: (),
}
}
}
impl PruneModes {
/// Sets pruning to all targets.
pub fn all() -> Self {
pub const fn all() -> Self {
Self {
sender_recovery: Some(PruneMode::Full),
transaction_lookup: Some(PruneMode::Full),
@@ -133,13 +132,14 @@ impl PruneModes {
storage_history: Some(PruneMode::Full),
bodies_history: Some(PruneMode::Full),
merkle_changesets: PruneMode::Full,
receipts_log_filter: Default::default(),
#[expect(deprecated)]
receipts_log_filter: (),
}
}
/// Returns whether there is any kind of receipt pruning configuration.
pub fn has_receipts_pruning(&self) -> bool {
self.receipts.is_some() || !self.receipts_log_filter.is_empty()
pub const fn has_receipts_pruning(&self) -> bool {
self.receipts.is_some()
}
/// Returns an error if we can't unwind to the targeted block because the target block is

2
crates/rpc/ipc/src/server/mod.rs
View File

@@ -391,7 +391,7 @@ where
fn call(&mut self, request: String) -> Self::Future {
trace!("{:?}", request);
let cfg = RpcServiceCfg::CallsAndSubscriptions {
let cfg = RpcServiceCfg {
bounded_subscriptions: BoundedSubscriptions::new(
self.inner.server_cfg.max_subscriptions_per_connection,
),

41
crates/rpc/ipc/src/server/rpc_service.rs
View File

@@ -25,17 +25,11 @@ pub struct RpcService {
}
/// Configuration of the `RpcService`.
#[allow(dead_code)]
#[derive(Clone, Debug)]
pub(crate) enum RpcServiceCfg {
/// The server supports only calls.
OnlyCalls,
/// The server supports both method calls and subscriptions.
CallsAndSubscriptions {
bounded_subscriptions: BoundedSubscriptions,
sink: MethodSink,
id_provider: Arc<dyn IdProvider>,
},
pub(crate) struct RpcServiceCfg {
pub(crate) bounded_subscriptions: BoundedSubscriptions,
pub(crate) sink: MethodSink,
pub(crate) id_provider: Arc<dyn IdProvider>,
}
impl RpcService {
@@ -82,30 +76,20 @@ impl RpcServiceT for RpcService {
ResponseFuture::future(fut)
}
MethodCallback::Subscription(callback) => {
let RpcServiceCfg::CallsAndSubscriptions {
bounded_subscriptions,
sink,
id_provider,
} = &self.cfg
else {
tracing::warn!(id = ?id, method = %name, "Attempted subscription on a service not configured for subscriptions.");
let rp =
MethodResponse::error(id, ErrorObject::from(ErrorCode::InternalError));
return ResponseFuture::ready(rp);
};
let cfg = &self.cfg;
if let Some(p) = bounded_subscriptions.acquire() {
if let Some(p) = cfg.bounded_subscriptions.acquire() {
let conn_state = SubscriptionState {
conn_id,
id_provider: &**id_provider,
id_provider: &*cfg.id_provider,
subscription_permit: p,
};
let fut =
callback(id.clone(), params, sink.clone(), conn_state, extensions);
callback(id.clone(), params, cfg.sink.clone(), conn_state, extensions);
ResponseFuture::future(fut)
} else {
let max = bounded_subscriptions.max();
let max = cfg.bounded_subscriptions.max();
let rp = MethodResponse::error(id, reject_too_many_subscriptions(max));
ResponseFuture::ready(rp)
}
@@ -114,13 +98,6 @@ impl RpcServiceT for RpcService {
// Don't adhere to any resource or subscription limits; always let unsubscribing
// happen!
let RpcServiceCfg::CallsAndSubscriptions { .. } = self.cfg else {
tracing::warn!(id = ?id, method = %name, "Attempted unsubscription on a service not configured for subscriptions.");
let rp =
MethodResponse::error(id, ErrorObject::from(ErrorCode::InternalError));
return ResponseFuture::ready(rp);
};
let rp = callback(id, params, conn_id, max_response_body_size, extensions);
ResponseFuture::ready(rp)
}

4
crates/rpc/rpc-builder/src/auth.rs
View File

@@ -354,7 +354,9 @@ impl AuthServerHandle {
/// Returns a http client connected to the server.
///
/// This client uses the JWT token to authenticate requests.
pub fn http_client(&self) -> impl SubscriptionClientT + Clone + Send + Sync + Unpin + 'static {
pub fn http_client(
&self,
) -> impl SubscriptionClientT + use<> + Clone + Send + Sync + Unpin + 'static {
// Create a middleware that adds a new JWT token to every request.
let secret_layer = AuthClientLayer::new(self.secret);
let middleware = tower::ServiceBuilder::default().layer(secret_layer);

14
crates/rpc/rpc-engine-api/src/error.rs
View File

@@ -1,4 +1,4 @@
use alloy_primitives::{B256, U256};
use alloy_primitives::B256;
use alloy_rpc_types_engine::{
ForkchoiceUpdateError, INVALID_FORK_CHOICE_STATE_ERROR, INVALID_FORK_CHOICE_STATE_ERROR_MSG,
INVALID_PAYLOAD_ATTRIBUTES_ERROR, INVALID_PAYLOAD_ATTRIBUTES_ERROR_MSG,
@@ -59,17 +59,6 @@ pub enum EngineApiError {
/// Requested number of items
count: u64,
},
/// Terminal total difficulty mismatch during transition configuration exchange.
#[error(
"invalid transition terminal total difficulty: \
execution: {execution}, consensus: {consensus}"
)]
TerminalTD {
/// Execution terminal total difficulty value.
execution: U256,
/// Consensus terminal total difficulty value.
consensus: U256,
},
/// Terminal block hash mismatch during transition configuration exchange.
#[error(
"invalid transition terminal block hash: \
@@ -202,7 +191,6 @@ impl From<EngineApiError> for jsonrpsee_types::error::ErrorObject<'static> {
}
},
// Any other server error
EngineApiError::TerminalTD { .. } |
EngineApiError::TerminalBlockHash { .. } |
EngineApiError::NewPayload(_) |
EngineApiError::Internal(_) |

16
crates/rpc/rpc-eth-types/src/error/mod.rs
View File

@@ -462,7 +462,6 @@ impl From<reth_errors::ProviderError> for EthApiError {
}
ProviderError::BestBlockNotFound => Self::HeaderNotFound(BlockId::latest()),
ProviderError::BlockNumberForTransactionIndexNotFound => Self::UnknownBlockOrTxIndex,
ProviderError::TotalDifficultyNotFound(num) => Self::HeaderNotFound(num.into()),
ProviderError::FinalizedBlockNotFound => Self::HeaderNotFound(BlockId::finalized()),
ProviderError::SafeBlockNotFound => Self::HeaderNotFound(BlockId::safe()),
err => Self::Internal(err.into()),
@@ -931,8 +930,13 @@ pub enum RpcPoolError {
#[error("negative value")]
NegativeValue,
/// When oversized data is encountered
#[error("oversized data")]
OversizedData,
#[error("oversized data: transaction size {size}, limit {limit}")]
OversizedData {
/// Size of the transaction/input data that exceeded the limit.
size: usize,
/// Configured limit that was exceeded.
limit: usize,
},
/// When the max initcode size is exceeded
#[error("max initcode size exceeded")]
ExceedsMaxInitCodeSize,
@@ -974,7 +978,7 @@ impl From<RpcPoolError> for jsonrpsee_types::error::ErrorObject<'static> {
RpcPoolError::MaxTxGasLimitExceeded |
RpcPoolError::ExceedsFeeCap { .. } |
RpcPoolError::NegativeValue |
RpcPoolError::OversizedData |
RpcPoolError::OversizedData { .. } |
RpcPoolError::ExceedsMaxInitCodeSize |
RpcPoolError::PoolTransactionError(_) |
RpcPoolError::Eip4844(_) |
@@ -1018,7 +1022,9 @@ impl From<InvalidPoolTransactionError> for RpcPoolError {
InvalidPoolTransactionError::IntrinsicGasTooLow => {
Self::Invalid(RpcInvalidTransactionError::GasTooLow)
}
InvalidPoolTransactionError::OversizedData(_, _) => Self::OversizedData,
InvalidPoolTransactionError::OversizedData { size, limit } => {
Self::OversizedData { size, limit }
}
InvalidPoolTransactionError::Underpriced => Self::Underpriced,
InvalidPoolTransactionError::Eip2681 => {
Self::Invalid(RpcInvalidTransactionError::NonceMaxValue)

87
crates/stages/api/src/pipeline/mod.rs
View File

@@ -9,7 +9,7 @@ use reth_primitives_traits::constants::BEACON_CONSENSUS_REORG_UNWIND_DEPTH;
use reth_provider::{
providers::ProviderNodeTypes, BlockHashReader, BlockNumReader, ChainStateBlockReader,
ChainStateBlockWriter, DBProvider, DatabaseProviderFactory, ProviderFactory,
PruneCheckpointReader, StageCheckpointReader, StageCheckpointWriter, StaticFileProviderFactory,
PruneCheckpointReader, StageCheckpointReader, StageCheckpointWriter,
};
use reth_prune::PrunerBuilder;
use reth_static_file::StaticFileProducer;
@@ -31,7 +31,7 @@ use crate::{
};
pub use builder::*;
use progress::*;
use reth_errors::{ProviderResult, RethResult};
use reth_errors::RethResult;
pub use set::*;
/// A container for a queued stage.
@@ -101,6 +101,12 @@ impl<N: ProviderNodeTypes> Pipeline<N> {
PipelineBuilder::default()
}
/// Return the minimum block number achieved by
/// any stage during the execution of the pipeline.
pub const fn minimum_block_number(&self) -> Option<u64> {
self.progress.minimum_block_number
}
/// Set tip for reverse sync.
#[track_caller]
pub fn set_tip(&self, tip: B256) {
@@ -121,7 +127,9 @@ impl<N: ProviderNodeTypes> Pipeline<N> {
) -> &mut dyn Stage<<ProviderFactory<N> as DatabaseProviderFactory>::ProviderRW> {
&mut self.stages[idx]
}
}
impl<N: ProviderNodeTypes> Pipeline<N> {
/// Registers progress metrics for each registered stage
pub fn register_metrics(&mut self) -> Result<(), PipelineError> {
let Some(metrics_tx) = &mut self.metrics_tx else { return Ok(()) };
@@ -282,81 +290,6 @@ impl<N: ProviderNodeTypes> Pipeline<N> {
Ok(())
}
/// Check if the pipeline is consistent (all stages have the checkpoint block numbers no less
/// than the checkpoint of the first stage).
///
/// This will return the pipeline target if:
/// * the pipeline was interrupted during its previous run
/// * a new stage was added
/// * stage data was dropped manually through `reth stage drop ...`
///
/// # Returns
///
/// A target block hash if the pipeline is inconsistent, otherwise `None`.
pub fn initial_backfill_target(&self) -> ProviderResult<Option<B256>> {
let provider = self.provider_factory.provider()?;
// If no target was provided, check if the stages are congruent - check if the
// checkpoint of the last stage matches the checkpoint of the first.
let first_stage_checkpoint = provider
.get_stage_checkpoint(self.stages.first().unwrap().id())?
.unwrap_or_default()
.block_number;
// Skip the first stage as we've already retrieved it and comparing all other checkpoints
// against it.
for stage in self.stages.iter().skip(1) {
let stage_id = stage.id();
let stage_checkpoint =
provider.get_stage_checkpoint(stage_id)?.unwrap_or_default().block_number;
// If the checkpoint of any stage is less than the checkpoint of the first stage,
// retrieve and return the block hash of the latest header and use it as the target.
if stage_checkpoint < first_stage_checkpoint {
debug!(
target: "consensus::engine",
first_stage_checkpoint,
inconsistent_stage_id = %stage_id,
inconsistent_stage_checkpoint = stage_checkpoint,
"Pipeline sync progress is inconsistent"
);
return provider.block_hash(first_stage_checkpoint);
}
}
Ok(None)
}
/// Checks for consistency between database and static files. If it fails, it unwinds to
/// the first block that's consistent between database and static files.
pub async fn ensure_static_files_consistency(&mut self) -> Result<(), PipelineError> {
let maybe_unwind_target = self
.provider_factory
.static_file_provider()
.check_consistency(&self.provider_factory.provider()?)?;
self.move_to_static_files()?;
if let Some(unwind_target) = maybe_unwind_target {
// Highly unlikely to happen, and given its destructive nature, it's better to panic
// instead.
assert_ne!(
unwind_target,
0,
"A static file <> database inconsistency was found that would trigger an unwind to block 0"
);
info!(target: "reth::cli", unwind_target = %unwind_target, "Executing an unwind after a failed storage consistency check.");
self.unwind(unwind_target, None).inspect_err(|err| {
error!(target: "reth::cli", unwind_target = %unwind_target, %err, "failed to run unwind")
})?;
}
Ok(())
}
/// Unwind the stages to the target block (exclusive).
///
/// If the unwind is due to a bad block the number of that block should be specified.

14
crates/stages/stages/benches/setup/mod.rs
View File

@@ -1,12 +1,7 @@
#![expect(unreachable_pub)]
use alloy_primitives::{Address, B256, U256};
use alloy_primitives::{Address, B256};
use itertools::concat;
use reth_db::{test_utils::TempDatabase, Database, DatabaseEnv};
use reth_db_api::{
cursor::DbCursorRO,
tables,
transaction::{DbTx, DbTxMut},
};
use reth_primitives_traits::{Account, SealedBlock, SealedHeader};
use reth_provider::{
test_utils::MockNodeTypesWithDB, DBProvider, DatabaseProvider, DatabaseProviderFactory,
@@ -198,13 +193,6 @@ pub(crate) fn txs_testdata(num_blocks: u64) -> TestStageDB {
);
db.insert_blocks(blocks.iter(), StorageKind::Static).unwrap();
// initialize TD
db.commit(|tx| {
let (head, _) = tx.cursor_read::<tables::Headers>()?.first()?.unwrap_or_default();
Ok(tx.put::<tables::HeaderTerminalDifficulties>(head, U256::from(0).into())?)
})
.unwrap();
}
db

2
crates/stages/stages/src/stages/bodies.rs
View File

@@ -580,7 +580,7 @@ mod tests {
..Default::default()
},
);
self.db.insert_headers_with_td(blocks.iter().map(|block| block.sealed_header()))?;
self.db.insert_headers(blocks.iter().map(|block| block.sealed_header()))?;
if let Some(progress) = blocks.get(start as usize) {
// Insert last progress data
{

26
crates/stages/stages/src/stages/era.rs
View File

@@ -10,12 +10,11 @@ use reth_era_utils as era;
use reth_etl::Collector;
use reth_primitives_traits::{FullBlockBody, FullBlockHeader, NodePrimitives};
use reth_provider::{
BlockReader, BlockWriter, DBProvider, HeaderProvider, StageCheckpointWriter,
StaticFileProviderFactory, StaticFileWriter,
BlockReader, BlockWriter, DBProvider, StageCheckpointWriter, StaticFileProviderFactory,
StaticFileWriter,
};
use reth_stages_api::{ExecInput, ExecOutput, Stage, StageError, UnwindInput, UnwindOutput};
use reth_static_file_types::StaticFileSegment;
use reth_storage_errors::ProviderError;
use std::{
fmt::{Debug, Formatter},
iter,
@@ -176,11 +175,6 @@ where
.get_highest_static_file_block(StaticFileSegment::Headers)
.unwrap_or_default();
// Find the latest total difficulty
let mut td = static_file_provider
.header_td_by_number(last_header_number)?
.ok_or(ProviderError::TotalDifficultyNotFound(last_header_number))?;
// Although headers were downloaded in reverse order, the collector iterates it in
// ascending order
let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?;
@@ -190,7 +184,6 @@ where
&mut writer,
provider,
&mut self.hash_collector,
&mut td,
last_header_number..=input.target(),
)
.map_err(|e| StageError::Fatal(e.into()))?;
@@ -336,7 +329,7 @@ mod tests {
};
use reth_ethereum_primitives::TransactionSigned;
use reth_primitives_traits::{SealedBlock, SealedHeader};
use reth_provider::{BlockNumReader, TransactionsProvider};
use reth_provider::{BlockNumReader, HeaderProvider, TransactionsProvider};
use reth_testing_utils::generators::{
random_block_range, random_signed_tx, BlockRangeParams,
};
@@ -391,7 +384,7 @@ mod tests {
..Default::default()
},
);
self.db.insert_headers_with_td(blocks.iter().map(|block| block.sealed_header()))?;
self.db.insert_headers(blocks.iter().map(|block| block.sealed_header()))?;
if let Some(progress) = blocks.get(start as usize) {
// Insert last progress data
{
@@ -447,9 +440,6 @@ mod tests {
match output {
Some(output) if output.checkpoint.block_number > initial_checkpoint => {
let provider = self.db.factory.provider()?;
let mut td = provider
.header_td_by_number(initial_checkpoint.saturating_sub(1))?
.unwrap_or_default();
for block_num in initial_checkpoint..
output
@@ -469,10 +459,6 @@ mod tests {
assert!(header.is_some());
let header = SealedHeader::seal_slow(header.unwrap());
assert_eq!(header.hash(), hash);
// validate the header total difficulty
td += header.difficulty;
assert_eq!(provider.header_td_by_number(block_num)?, Some(td));
}
self.validate_db_blocks(
@@ -513,10 +499,6 @@ mod tests {
.ensure_no_entry_above_by_value::<tables::HeaderNumbers, _>(block, |val| val)?;
self.db.ensure_no_entry_above::<tables::CanonicalHeaders, _>(block, |key| key)?;
self.db.ensure_no_entry_above::<tables::Headers, _>(block, |key| key)?;
self.db.ensure_no_entry_above::<tables::HeaderTerminalDifficulties, _>(
block,
|num| num,
)?;
Ok(())
}

27
crates/stages/stages/src/stages/execution.rs
View File

@@ -39,7 +39,6 @@ use super::missing_static_data_error;
/// Input tables:
/// - [`tables::CanonicalHeaders`] get next block to execute.
/// - [`tables::Headers`] get for revm environment variables.
/// - [`tables::HeaderTerminalDifficulties`]
/// - [`tables::BlockBodyIndices`] to get tx number
/// - [`tables::Transactions`] to execute
///
@@ -661,7 +660,7 @@ where
mod tests {
use super::*;
use crate::{stages::MERKLE_STAGE_DEFAULT_REBUILD_THRESHOLD, test_utils::TestStageDB};
use alloy_primitives::{address, hex_literal::hex, keccak256, Address, B256, U256};
use alloy_primitives::{address, hex_literal::hex, keccak256, B256, U256};
use alloy_rlp::Decodable;
use assert_matches::assert_matches;
use reth_chainspec::ChainSpecBuilder;
@@ -678,9 +677,7 @@ mod tests {
DatabaseProviderFactory, ReceiptProvider, StaticFileProviderFactory,
};
use reth_prune::PruneModes;
use reth_prune_types::{PruneMode, ReceiptsLogPruneConfig};
use reth_stages_api::StageUnitCheckpoint;
use std::collections::BTreeMap;
fn stage() -> ExecutionStage<EthEvmConfig> {
let evm_config =
@@ -897,20 +894,11 @@ mod tests {
// If there is a pruning configuration, then it's forced to use the database.
// This way we test both cases.
let modes = [None, Some(PruneModes::default())];
let random_filter = ReceiptsLogPruneConfig(BTreeMap::from([(
Address::random(),
PruneMode::Distance(100000),
)]));
// Tests node with database and node with static files
for mut mode in modes {
for mode in modes {
let mut provider = factory.database_provider_rw().unwrap();
if let Some(mode) = &mut mode {
// Simulating a full node where we write receipts to database
mode.receipts_log_filter = random_filter.clone();
}
let mut execution_stage = stage();
provider.set_prune_modes(mode.clone().unwrap_or_default());
@@ -1034,18 +1022,9 @@ mod tests {
// If there is a pruning configuration, then it's forced to use the database.
// This way we test both cases.
let modes = [None, Some(PruneModes::default())];
let random_filter = ReceiptsLogPruneConfig(BTreeMap::from([(
Address::random(),
PruneMode::Before(100000),
)]));
// Tests node with database and node with static files
for mut mode in modes {
if let Some(mode) = &mut mode {
// Simulating a full node where we write receipts to database
mode.receipts_log_filter = random_filter.clone();
}
for mode in modes {
// Test Execution
let mut execution_stage = stage();
provider.set_prune_modes(mode.clone().unwrap_or_default());

4
crates/stages/stages/src/stages/finish.rs
View File

@@ -72,7 +72,7 @@ mod tests {
let start = input.checkpoint().block_number;
let mut rng = generators::rng();
let head = random_header(&mut rng, start, None);
self.db.insert_headers_with_td(std::iter::once(&head))?;
self.db.insert_headers(std::iter::once(&head))?;
// use previous progress as seed size
let end = input.target.unwrap_or_default() + 1;
@@ -82,7 +82,7 @@ mod tests {
}
let mut headers = random_header_range(&mut rng, start + 1..end, head.hash());
self.db.insert_headers_with_td(headers.iter())?;
self.db.insert_headers(headers.iter())?;
headers.insert(0, head);
Ok(headers)
}

2
crates/stages/stages/src/stages/hashing_account.rs
View File

@@ -64,7 +64,7 @@ impl AccountHashingStage {
opts: SeedOpts,
) -> Result<Vec<(alloy_primitives::Address, Account)>, StageError>
where
N::Primitives: reth_primitives_traits::FullNodePrimitives<
N::Primitives: reth_primitives_traits::NodePrimitives<
Block = reth_ethereum_primitives::Block,
BlockHeader = reth_primitives_traits::Header,
>,

44
crates/stages/stages/src/stages/headers.rs
View File

@@ -16,15 +16,14 @@ use reth_network_p2p::headers::{
};
use reth_primitives_traits::{serde_bincode_compat, FullBlockHeader, NodePrimitives, SealedHeader};
use reth_provider::{
providers::StaticFileWriter, BlockHashReader, DBProvider, HeaderProvider,
HeaderSyncGapProvider, StaticFileProviderFactory,
providers::StaticFileWriter, BlockHashReader, DBProvider, HeaderSyncGapProvider,
StaticFileProviderFactory,
};
use reth_stages_api::{
CheckpointBlockRange, EntitiesCheckpoint, ExecInput, ExecOutput, HeadersCheckpoint, Stage,
StageCheckpoint, StageError, StageId, UnwindInput, UnwindOutput,
};
use reth_static_file_types::StaticFileSegment;
use reth_storage_errors::provider::ProviderError;
use std::task::{ready, Context, Poll};
use tokio::sync::watch;
@@ -107,11 +106,6 @@ where
.get_highest_static_file_block(StaticFileSegment::Headers)
.unwrap_or_default();
// Find the latest total difficulty
let mut td = static_file_provider
.header_td_by_number(last_header_number)?
.ok_or(ProviderError::TotalDifficultyNotFound(last_header_number))?;
// Although headers were downloaded in reverse order, the collector iterates it in ascending
// order
let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?;
@@ -134,11 +128,8 @@ where
}
last_header_number = header.number();
// Increase total difficulty
td += header.difficulty();
// Append to Headers segment
writer.append_header(header, td, header_hash)?;
writer.append_header(header, header_hash)?;
}
info!(target: "sync::stages::headers", total = total_headers, "Writing headers hash index");
@@ -342,9 +333,6 @@ where
(input.unwind_to + 1)..,
)?;
provider.tx_ref().unwind_table_by_num::<tables::CanonicalHeaders>(input.unwind_to)?;
provider
.tx_ref()
.unwind_table_by_num::<tables::HeaderTerminalDifficulties>(input.unwind_to)?;
let unfinalized_headers_unwound =
provider.tx_ref().unwind_table_by_num::<tables::Headers>(input.unwind_to)?;
@@ -415,7 +403,7 @@ mod tests {
ReverseHeadersDownloader, ReverseHeadersDownloaderBuilder,
};
use reth_network_p2p::test_utils::{TestHeaderDownloader, TestHeadersClient};
use reth_provider::{test_utils::MockNodeTypesWithDB, BlockNumReader};
use reth_provider::{test_utils::MockNodeTypesWithDB, BlockNumReader, HeaderProvider};
use tokio::sync::watch;
pub(crate) struct HeadersTestRunner<D: HeaderDownloader> {
@@ -469,7 +457,7 @@ mod tests {
let start = input.checkpoint().block_number;
let headers = random_header_range(&mut rng, 0..start + 1, B256::ZERO);
let head = headers.last().cloned().unwrap();
self.db.insert_headers_with_td(headers.iter())?;
self.db.insert_headers(headers.iter())?;
// use previous checkpoint as seed size
let end = input.target.unwrap_or_default() + 1;
@@ -493,9 +481,6 @@ mod tests {
match output {
Some(output) if output.checkpoint.block_number > initial_checkpoint => {
let provider = self.db.factory.provider()?;
let mut td = provider
.header_td_by_number(initial_checkpoint.saturating_sub(1))?
.unwrap_or_default();
for block_num in initial_checkpoint..output.checkpoint.block_number {
// look up the header hash
@@ -509,10 +494,6 @@ mod tests {
assert!(header.is_some());
let header = SealedHeader::seal_slow(header.unwrap());
assert_eq!(header.hash(), hash);
// validate the header total difficulty
td += header.difficulty;
assert_eq!(provider.header_td_by_number(block_num)?, Some(td));
}
}
_ => self.check_no_header_entry_above(initial_checkpoint)?,
@@ -567,10 +548,6 @@ mod tests {
.ensure_no_entry_above_by_value::<tables::HeaderNumbers, _>(block, |val| val)?;
self.db.ensure_no_entry_above::<tables::CanonicalHeaders, _>(block, |key| key)?;
self.db.ensure_no_entry_above::<tables::Headers, _>(block, |key| key)?;
self.db.ensure_no_entry_above::<tables::HeaderTerminalDifficulties, _>(
block,
|num| num,
)?;
Ok(())
}
@@ -635,16 +612,7 @@ mod tests {
let static_file_provider = provider.static_file_provider();
let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap();
for header in sealed_headers {
let ttd = if header.number() == 0 {
header.difficulty()
} else {
let parent_block_number = header.number() - 1;
let parent_ttd =
provider.header_td_by_number(parent_block_number).unwrap().unwrap_or_default();
parent_ttd + header.difficulty()
};
writer.append_header(header.header(), ttd, &header.hash()).unwrap();
writer.append_header(header.header(), &header.hash()).unwrap();
}
drop(writer);

2
crates/stages/stages/src/stages/merkle.rs
View File

@@ -738,7 +738,7 @@ mod tests {
let hash = last_header.hash_slow();
writer.prune_headers(1).unwrap();
writer.commit().unwrap();
writer.append_header(&last_header, U256::ZERO, &hash).unwrap();
writer.append_header(&last_header, &hash).unwrap();
writer.commit().unwrap();
Ok(blocks)

44
crates/stages/stages/src/stages/mod.rs
View File

@@ -75,7 +75,9 @@ mod tests {
StaticFileProviderFactory, StorageReader,
};
use reth_prune_types::{PruneMode, PruneModes};
use reth_stages_api::{ExecInput, ExecutionStageThresholds, Stage, StageCheckpoint, StageId};
use reth_stages_api::{
ExecInput, ExecutionStageThresholds, PipelineTarget, Stage, StageCheckpoint, StageId,
};
use reth_static_file_types::StaticFileSegment;
use reth_testing_utils::generators::{
self, random_block, random_block_range, random_receipt, BlockRangeParams,
@@ -302,7 +304,7 @@ mod tests {
prune_count: usize,
segment: StaticFileSegment,
is_full_node: bool,
expected: Option<u64>,
expected: Option<PipelineTarget>,
) {
// We recreate the static file provider, since consistency heals are done on fetching the
// writer for the first time.
@@ -324,18 +326,11 @@ mod tests {
// We recreate the static file provider, since consistency heals are done on fetching the
// writer for the first time.
let mut provider = db.factory.database_provider_ro().unwrap();
if is_full_node {
provider.set_prune_modes(PruneModes {
receipts: Some(PruneMode::Full),
..Default::default()
});
}
let mut static_file_provider = db.factory.static_file_provider();
static_file_provider = StaticFileProvider::read_write(static_file_provider.path()).unwrap();
assert!(matches!(
static_file_provider
.check_consistency(&provider),
.check_consistency(&db.factory.database_provider_ro().unwrap(), is_full_node,),
Ok(e) if e == expected
));
}
@@ -346,7 +341,7 @@ mod tests {
db: &TestStageDB,
stage_id: StageId,
checkpoint_block_number: BlockNumber,
expected: Option<u64>,
expected: Option<PipelineTarget>,
) {
let provider_rw = db.factory.provider_rw().unwrap();
provider_rw
@@ -357,15 +352,18 @@ mod tests {
assert!(matches!(
db.factory
.static_file_provider()
.check_consistency(&db.factory.database_provider_ro().unwrap()),
.check_consistency(&db.factory.database_provider_ro().unwrap(), false,),
Ok(e) if e == expected
));
}
/// Inserts a dummy value at key and compare the check consistency result against the expected
/// one.
fn update_db_and_check<T: Table<Key = u64>>(db: &TestStageDB, key: u64, expected: Option<u64>)
where
fn update_db_and_check<T: Table<Key = u64>>(
db: &TestStageDB,
key: u64,
expected: Option<PipelineTarget>,
) where
<T as Table>::Value: Default,
{
update_db_with_and_check::<T>(db, key, expected, &Default::default());
@@ -376,7 +374,7 @@ mod tests {
fn update_db_with_and_check<T: Table<Key = u64>>(
db: &TestStageDB,
key: u64,
expected: Option<u64>,
expected: Option<PipelineTarget>,
value: &T::Value,
) {
let provider_rw = db.factory.provider_rw().unwrap();
@@ -387,7 +385,7 @@ mod tests {
assert!(matches!(
db.factory
.static_file_provider()
.check_consistency(&db.factory.database_provider_ro().unwrap()),
.check_consistency(&db.factory.database_provider_ro().unwrap(), false),
Ok(e) if e == expected
));
}
@@ -398,7 +396,7 @@ mod tests {
let db_provider = db.factory.database_provider_ro().unwrap();
assert!(matches!(
db.factory.static_file_provider().check_consistency(&db_provider),
db.factory.static_file_provider().check_consistency(&db_provider, false),
Ok(None)
));
}
@@ -420,7 +418,7 @@ mod tests {
1,
StaticFileSegment::Receipts,
archive_node,
Some(88),
Some(PipelineTarget::Unwind(88)),
);
simulate_behind_checkpoint_corruption(
@@ -428,7 +426,7 @@ mod tests {
3,
StaticFileSegment::Headers,
archive_node,
Some(86),
Some(PipelineTarget::Unwind(86)),
);
}
@@ -477,7 +475,7 @@ mod tests {
);
// When a checkpoint is ahead, we request a pipeline unwind.
save_checkpoint_and_check(&db, StageId::Headers, 91, Some(block));
save_checkpoint_and_check(&db, StageId::Headers, 91, Some(PipelineTarget::Unwind(block)));
}
#[test]
@@ -490,7 +488,7 @@ mod tests {
.unwrap();
// Creates a gap of one header: static_file <missing> db
update_db_and_check::<tables::Headers>(&db, current + 2, Some(89));
update_db_and_check::<tables::Headers>(&db, current + 2, Some(PipelineTarget::Unwind(89)));
// Fill the gap, and ensure no unwind is necessary.
update_db_and_check::<tables::Headers>(&db, current + 1, None);
@@ -509,7 +507,7 @@ mod tests {
update_db_with_and_check::<tables::Transactions>(
&db,
current + 2,
Some(89),
Some(PipelineTarget::Unwind(89)),
&TxLegacy::default().into_signed(Signature::test_signature()).into(),
);
@@ -532,7 +530,7 @@ mod tests {
.unwrap();
// Creates a gap of one receipt: static_file <missing> db
update_db_and_check::<tables::Receipts>(&db, current + 2, Some(89));
update_db_and_check::<tables::Receipts>(&db, current + 2, Some(PipelineTarget::Unwind(89)));
// Fill the gap, and ensure no unwind is necessary.
update_db_and_check::<tables::Receipts>(&db, current + 1, None);

30
crates/stages/stages/src/test_utils/test_db.rs
View File

@@ -1,4 +1,4 @@
use alloy_primitives::{keccak256, Address, BlockNumber, TxHash, TxNumber, B256, U256};
use alloy_primitives::{keccak256, Address, BlockNumber, TxHash, TxNumber, B256};
use reth_chainspec::MAINNET;
use reth_db::{
test_utils::{create_test_rw_db, create_test_rw_db_with_path, create_test_static_files_dir},
@@ -150,7 +150,6 @@ impl TestStageDB {
writer: Option<&mut StaticFileProviderRWRefMut<'_, EthPrimitives>>,
tx: &TX,
header: &SealedHeader,
td: U256,
) -> ProviderResult<()> {
if let Some(writer) = writer {
// Backfill: some tests start at a forward block number, but static files require no
@@ -160,14 +159,13 @@ impl TestStageDB {
for block_number in 0..header.number {
let mut prev = header.clone_header();
prev.number = block_number;
writer.append_header(&prev, U256::ZERO, &B256::ZERO)?;
writer.append_header(&prev, &B256::ZERO)?;
}
}
writer.append_header(header.header(), td, &header.hash())?;
writer.append_header(header.header(), &header.hash())?;
} else {
tx.put::<tables::CanonicalHeaders>(header.number, header.hash())?;
tx.put::<tables::HeaderTerminalDifficulties>(header.number, td.into())?;
tx.put::<tables::Headers>(header.number, header.header().clone())?;
}
@@ -175,20 +173,16 @@ impl TestStageDB {
Ok(())
}
fn insert_headers_inner<'a, I, const TD: bool>(&self, headers: I) -> ProviderResult<()>
fn insert_headers_inner<'a, I>(&self, headers: I) -> ProviderResult<()>
where
I: IntoIterator<Item = &'a SealedHeader>,
{
let provider = self.factory.static_file_provider();
let mut writer = provider.latest_writer(StaticFileSegment::Headers)?;
let tx = self.factory.provider_rw()?.into_tx();
let mut td = U256::ZERO;
for header in headers {
if TD {
td += header.difficulty;
}
Self::insert_header(Some(&mut writer), &tx, header, td)?;
Self::insert_header(Some(&mut writer), &tx, header)?;
}
writer.commit()?;
@@ -203,17 +197,7 @@ impl TestStageDB {
where
I: IntoIterator<Item = &'a SealedHeader>,
{
self.insert_headers_inner::<I, false>(headers)
}
/// Inserts total difficulty of headers into the corresponding static file and tables.
///
/// Superset functionality of [`TestStageDB::insert_headers`].
pub fn insert_headers_with_td<'a, I>(&self, headers: I) -> ProviderResult<()>
where
I: IntoIterator<Item = &'a SealedHeader>,
{
self.insert_headers_inner::<I, true>(headers)
self.insert_headers_inner::<I>(headers)
}
/// Insert ordered collection of [`SealedBlock`] into corresponding tables.
@@ -240,7 +224,7 @@ impl TestStageDB {
.then(|| provider.latest_writer(StaticFileSegment::Headers).unwrap());
blocks.iter().try_for_each(|block| {
Self::insert_header(headers_writer.as_mut(), &tx, block.sealed_header(), U256::ZERO)
Self::insert_header(headers_writer.as_mut(), &tx, block.sealed_header())
})?;
if let Some(mut writer) = headers_writer {

14
crates/static-file/static-file/src/segments/headers.rs
View File

@@ -36,25 +36,17 @@ where
)?;
let headers_walker = headers_cursor.walk_range(block_range.clone())?;
let mut header_td_cursor =
provider.tx_ref().cursor_read::<tables::HeaderTerminalDifficulties>()?;
let header_td_walker = header_td_cursor.walk_range(block_range.clone())?;
let mut canonical_headers_cursor =
provider.tx_ref().cursor_read::<tables::CanonicalHeaders>()?;
let canonical_headers_walker = canonical_headers_cursor.walk_range(block_range)?;
for ((header_entry, header_td_entry), canonical_header_entry) in
headers_walker.zip(header_td_walker).zip(canonical_headers_walker)
{
for (header_entry, canonical_header_entry) in headers_walker.zip(canonical_headers_walker) {
let (header_block, header) = header_entry?;
let (header_td_block, header_td) = header_td_entry?;
let (canonical_header_block, canonical_header) = canonical_header_entry?;
debug_assert_eq!(header_block, header_td_block);
debug_assert_eq!(header_td_block, canonical_header_block);
debug_assert_eq!(header_block, canonical_header_block);
static_file_writer.append_header(&header, header_td.0, &canonical_header)?;
static_file_writer.append_header(&header, &canonical_header)?;
}
Ok(())

16
crates/static-file/static-file/src/static_file_producer.rs
View File

@@ -207,19 +207,17 @@ where
headers: finalized_block_numbers.headers.and_then(|finalized_block_number| {
self.get_static_file_target(highest_static_files.headers, finalized_block_number)
}),
// StaticFile receipts only if they're not pruned according to the user configuration
receipts: if self.prune_modes.receipts.is_none() &&
self.prune_modes.receipts_log_filter.is_empty()
{
finalized_block_numbers.receipts.and_then(|finalized_block_number| {
receipts: finalized_block_numbers
.receipts
// StaticFile receipts only if they're not pruned according to the user
// configuration
.filter(|_| !self.prune_modes.has_receipts_pruning())
.and_then(|finalized_block_number| {
self.get_static_file_target(
highest_static_files.receipts,
finalized_block_number,
)
})
} else {
None
},
}),
transactions: finalized_block_numbers.transactions.and_then(|finalized_block_number| {
self.get_static_file_target(
highest_static_files.transactions,

2
crates/storage/codecs/derive/src/compact/structs.rs
View File

@@ -155,7 +155,7 @@ impl<'a> StructHandler<'a> {
let (#name, new_buf) = #ident_type::#from_compact_ident(buf, flags.#len() as usize);
});
} else {
todo!()
unreachable!("flag-type fields are always compact in Compact derive")
}
self.lines.push(quote! {
buf = new_buf;

3
crates/storage/db-api/src/tables/mod.rs
View File

@@ -309,7 +309,8 @@ tables! {
type Value = HeaderHash;
}
/// Stores the total difficulty from a block header.
/// Stores the total difficulty from block headers.
/// Note: Deprecated.
table HeaderTerminalDifficulties {
type Key = BlockNumber;
type Value = CompactU256;

Some files were not shown because too many files have changed in this diff Show More