Compare commits

..

2 Commits

Author SHA1 Message Date
yongkangc
5fb9d04948 fix: add RocksDB clear_table, refactor shard writing
- Add clear_table method to RocksDBProvider for clearing all entries
- Clear RocksDB AccountsHistory table on first_sync (matches MDBX behavior)
- Extract duplicate shard-writing loop into write_full_shards helper
- Remove unused append_only parameter from write_account_history_shards_keep_last
2026-01-18 14:19:18 +00:00
yongkangc
40a72a8720 feat(stages): add RocksDB support for IndexAccountHistoryStage
This implements RocksDB support for the IndexAccountHistoryStage following
the TransactionLookupStage pattern:

- Add RocksDBProviderFactory trait bound to Stage impl
- Use explicit #[cfg(all(unix, feature = "rocksdb"))] blocks for RocksDB batch
  creation instead of macros
- Use EitherWriter::new_accounts_history to route writes to MDBX or RocksDB
  based on storage settings
- Add inline helper functions for loading/flushing shards with proper u64::MAX
  handling for last shard
- Add RocksDB-specific tests: execute_writes_to_rocksdb_when_enabled,
  unwind_deletes_from_rocksdb_when_enabled, execute_incremental_sync

Note: Full unwind support for RocksDB requires updates to the HistoryWriter
trait implementation, which is out of scope for this PR.

Closes #21124
2026-01-17 23:18:37 +00:00
146 changed files with 1979 additions and 8332 deletions

43
.github/CODEOWNERS vendored
View File

@@ -1,52 +1,45 @@
* @gakonst
crates/blockchain-tree-api/ @rakita @mattsse @Rjected
crates/blockchain-tree/ @rakita @mattsse @Rjected
crates/chain-state/ @fgimenez @mattsse
crates/chainspec/ @Rjected @joshieDo @mattsse
crates/cli/ @mattsse
crates/config/ @shekhirin @mattsse @Rjected
crates/consensus/ @mattsse @Rjected
crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez
crates/engine/ @mattsse @Rjected @mediocregopher @yongkangc
crates/era/ @mattsse
crates/era-downloader/ @mattsse
crates/era-utils/ @mattsse
crates/engine/ @mattsse @Rjected @fgimenez @mediocregopher @yongkangc
crates/era/ @mattsse @RomanHodulak
crates/errors/ @mattsse
crates/ethereum-forks/ @mattsse @Rjected
crates/ethereum/ @mattsse @Rjected
crates/etl/ @joshieDo @shekhirin
crates/evm/ @mattsse @Rjected @klkvr
crates/evm/ @rakita @mattsse @Rjected
crates/exex/ @shekhirin
crates/fs-util/ @mattsse
crates/metrics/ @mattsse @Rjected
crates/net/ @mattsse @Rjected
crates/net/downloaders/ @Rjected
crates/node/ @mattsse @Rjected @klkvr
crates/optimism/ @mattsse @Rjected
crates/optimism/ @mattsse @Rjected @fgimenez
crates/payload/ @mattsse @Rjected
crates/primitives-traits/ @Rjected @mattsse @klkvr
crates/primitives-traits/ @Rjected @RomanHodulak @mattsse @klkvr
crates/primitives/ @Rjected @mattsse @klkvr
crates/prune/ @shekhirin @joshieDo
crates/ress/ @shekhirin @Rjected
crates/revm/ @mattsse
crates/rpc/ @mattsse @Rjected
crates/ress @shekhirin @Rjected
crates/revm/ @mattsse @rakita
crates/rpc/ @mattsse @Rjected @RomanHodulak
crates/stages/ @shekhirin @mediocregopher
crates/static-file/ @joshieDo @shekhirin
crates/stateless/ @mattsse
crates/storage/codecs/ @joshieDo
crates/storage/db-api/ @joshieDo
crates/storage/db-api/ @joshieDo @rakita
crates/storage/db-common/ @Rjected
crates/storage/db/ @joshieDo
crates/storage/errors/ @joshieDo
crates/storage/libmdbx-rs/ @shekhirin
crates/storage/db/ @joshieDo @rakita
crates/storage/errors/ @rakita
crates/storage/libmdbx-rs/ @rakita @shekhirin
crates/storage/nippy-jar/ @joshieDo @shekhirin
crates/storage/provider/ @joshieDo @shekhirin
crates/storage/provider/ @rakita @joshieDo @shekhirin
crates/storage/storage-api/ @joshieDo
crates/tasks/ @mattsse
crates/tokio-util/ @mattsse
crates/tracing/ @mattsse @shekhirin
crates/tracing-otlp/ @mattsse @Rjected
crates/tokio-util/ @fgimenez
crates/transaction-pool/ @mattsse @yongkangc
crates/trie/ @Rjected @shekhirin @mediocregopher @yongkangc
bin/reth/ @mattsse @shekhirin @Rjected
bin/reth-bench/ @mattsse @Rjected @shekhirin @yongkangc
crates/trie/ @Rjected @shekhirin @mediocregopher
bin/reth-bench-compare/ @mediocregopher @shekhirin @yongkangc
etc/ @Rjected @shekhirin
.github/ @gakonst @DaniPopes

14
.github/scripts/codspeed-build.sh vendored Executable file
View File

@@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -eo pipefail
# TODO: Benchmarks run WAY too slow due to excessive amount of iterations.
cmd=(cargo codspeed build --profile profiling)
crates=(
-p reth-primitives
-p reth-trie
-p reth-trie-common
-p reth-trie-sparse
)
"${cmd[@]}" --features test-utils "${crates[@]}"

View File

@@ -17,16 +17,6 @@ name: bench
jobs:
codspeed:
runs-on: depot-ubuntu-latest
strategy:
matrix:
partition: [1, 2]
total_partitions: [2]
include:
- partition: 1
crates: "-p reth-primitives -p reth-trie-common -p reth-trie-sparse"
- partition: 2
crates: "-p reth-trie"
name: codspeed (${{ matrix.partition }}/${{ matrix.total_partitions }})
steps:
- uses: actions/checkout@v6
with:
@@ -42,10 +32,10 @@ jobs:
with:
tool: cargo-codspeed
- name: Build the benchmark target(s)
run: cargo codspeed build --profile profiling --features test-utils ${{ matrix.crates }}
run: ./.github/scripts/codspeed-build.sh
- name: Run the benchmarks
uses: CodSpeedHQ/action@v4
with:
run: cargo codspeed run ${{ matrix.crates }}
run: cargo codspeed run --workspace
mode: instrumentation
token: ${{ secrets.CODSPEED_TOKEN }}

View File

@@ -1,66 +0,0 @@
# Checks reth compilation against alloy branches to detect breaking changes.
# Run on-demand via workflow_dispatch.
name: Check Alloy Breaking Changes
on:
workflow_dispatch:
inputs:
alloy_branch:
description: 'Branch/rev for alloy-rs/alloy (leave empty to skip)'
required: false
type: string
alloy_evm_branch:
description: 'Branch/rev for alloy-rs/evm (alloy-evm, alloy-op-evm) (leave empty to skip)'
required: false
type: string
op_alloy_branch:
description: 'Branch/rev for alloy-rs/op-alloy (leave empty to skip)'
required: false
type: string
env:
CARGO_TERM_COLOR: always
jobs:
check:
name: Check compilation with patched alloy
runs-on: depot-ubuntu-latest-16
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Apply alloy patches
run: |
ARGS=""
if [ -n "${{ inputs.alloy_branch }}" ]; then
ARGS="$ARGS --alloy ${{ inputs.alloy_branch }}"
fi
if [ -n "${{ inputs.alloy_evm_branch }}" ]; then
ARGS="$ARGS --evm ${{ inputs.alloy_evm_branch }}"
fi
if [ -n "${{ inputs.op_alloy_branch }}" ]; then
ARGS="$ARGS --op ${{ inputs.op_alloy_branch }}"
fi
if [ -z "$ARGS" ]; then
echo "No branches specified, nothing to patch"
exit 1
fi
./scripts/patch-alloy.sh $ARGS
echo "=== Final patch section ==="
tail -50 Cargo.toml
- name: Check workspace
run: cargo check --workspace --all-features
- name: Check Optimism
run: cargo check -p reth-optimism-node --all-features

View File

@@ -15,6 +15,6 @@ permissions:
jobs:
update:
uses: tempoxyz/ci/.github/workflows/cargo-update-pr.yml@main
uses: ithacaxyz/ci/.github/workflows/cargo-update-pr.yml@main
secrets:
token: ${{ secrets.GITHUB_TOKEN }}

View File

@@ -285,7 +285,7 @@ jobs:
- run: zepter run check
deny:
uses: tempoxyz/ci/.github/workflows/deny.yml@main
uses: ithacaxyz/ci/.github/workflows/deny.yml@main
lint-success:
name: lint success

View File

@@ -249,7 +249,7 @@ Write comments that remain valuable after the PR is merged. Future readers won't
unsafe impl GlobalAlloc for LimitedAllocator { ... }
// Binary search requires sorted input. Panics on unsorted slices.
fn find_index(items: &[Item], target: &Item) -> Option<usize>
fn find_index(items: &[Item], target: &Item) -> Option
// Timeout set to 5s to match EVM block processing limits
const TRACER_TIMEOUT: Duration = Duration::from_secs(5);

706
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,5 @@
[workspace.package]
version = "1.10.2"
version = "1.10.0"
edition = "2024"
rust-version = "1.88"
license = "MIT OR Apache-2.0"
@@ -797,32 +797,3 @@ ipnet = "2.11"
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# Patched by patch-alloy.sh
alloy-consensus = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-contract = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-eips = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-genesis = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-network = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-provider = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-serde = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-signer = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", branch = "main" }

View File

@@ -163,7 +163,6 @@ impl NodeManager {
"eth,reth".to_string(),
"--disable-discovery".to_string(),
"--trusted-only".to_string(),
"--disable-tx-gossip".to_string(),
]);
// Add tracing arguments if OTLP endpoint is configured

View File

@@ -22,42 +22,12 @@ use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIM
use std::{path::PathBuf, time::Instant};
use tracing::info;
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
///
/// Examples: "30000000", "30M", "1G", "2G"
fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
let s = s.trim();
if s.is_empty() {
return Err(eyre::eyre!("empty value"));
}
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
(prefix, 1_000_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
(prefix, 1_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
(prefix, 1_000u64)
} else {
(s, 1u64)
};
let base: u64 = num_str.trim().parse()?;
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
}
/// `reth benchmark gas-limit-ramp` command.
#[derive(Debug, Parser)]
pub struct Command {
/// Number of blocks to generate. Mutually exclusive with --target-gas-limit.
#[arg(long, value_name = "BLOCKS", conflicts_with = "target_gas_limit")]
blocks: Option<u64>,
/// Target gas limit to ramp up to. The benchmark will generate blocks until the gas limit
/// reaches or exceeds this value. Mutually exclusive with --blocks.
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 2G = 2
/// billion).
#[arg(long, value_name = "TARGET_GAS_LIMIT", conflicts_with = "blocks", value_parser = parse_gas_limit)]
target_gas_limit: Option<u64>,
/// Number of blocks to generate.
#[arg(long, value_name = "BLOCKS")]
blocks: u64,
/// The Engine API RPC URL.
#[arg(long = "engine-rpc-url", value_name = "ENGINE_RPC_URL")]
@@ -72,37 +42,12 @@ pub struct Command {
output: PathBuf,
}
/// Mode for determining when to stop ramping.
#[derive(Debug, Clone, Copy)]
enum RampMode {
/// Ramp for a fixed number of blocks.
Blocks(u64),
/// Ramp until reaching or exceeding target gas limit.
TargetGasLimit(u64),
}
impl Command {
/// Execute `benchmark gas-limit-ramp` command.
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
let mode = match (self.blocks, self.target_gas_limit) {
(Some(blocks), None) => {
if blocks == 0 {
return Err(eyre::eyre!("--blocks must be greater than 0"));
}
RampMode::Blocks(blocks)
}
(None, Some(target)) => {
if target == 0 {
return Err(eyre::eyre!("--target-gas-limit must be greater than 0"));
}
RampMode::TargetGasLimit(target)
}
_ => {
return Err(eyre::eyre!(
"Exactly one of --blocks or --target-gas-limit must be specified"
));
}
};
if self.blocks == 0 {
return Err(eyre::eyre!("--blocks must be greater than 0"));
}
// Ensure output directory exists
if self.output.is_file() {
@@ -139,31 +84,14 @@ impl Command {
let canonical_parent = parent_header.number;
let start_block = canonical_parent + 1;
let end_block = start_block + self.blocks - 1;
match mode {
RampMode::Blocks(blocks) => {
info!(
canonical_parent,
start_block,
end_block = start_block + blocks - 1,
"Starting gas limit ramp benchmark (block count mode)"
);
}
RampMode::TargetGasLimit(target) => {
info!(
canonical_parent,
start_block,
current_gas_limit = parent_header.gas_limit,
target_gas_limit = target,
"Starting gas limit ramp benchmark (target gas limit mode)"
);
}
}
info!(canonical_parent, start_block, end_block, "Starting gas limit ramp benchmark");
let mut blocks_processed = 0u64;
let mut next_block_number = start_block;
let total_benchmark_duration = Instant::now();
while !should_stop(mode, blocks_processed, parent_header.gas_limit) {
while next_block_number <= end_block {
let timestamp = parent_header.timestamp.saturating_add(1);
let request = prepare_payload_request(&chain_spec, timestamp, parent_hash);
@@ -212,13 +140,13 @@ impl Command {
parent_header = block.header;
parent_hash = block_hash;
blocks_processed += 1;
next_block_number += 1;
}
let final_gas_limit = parent_header.gas_limit;
info!(
total_duration=?total_benchmark_duration.elapsed(),
blocks_processed,
blocks_processed = self.blocks,
final_gas_limit,
"Benchmark complete"
);
@@ -230,57 +158,3 @@ impl Command {
const fn max_gas_limit_increase(parent_gas_limit: u64) -> u64 {
(parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR).saturating_sub(1)
}
const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u64) -> bool {
match mode {
RampMode::Blocks(target_blocks) => blocks_processed >= target_blocks,
RampMode::TargetGasLimit(target) => current_gas_limit >= target,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_gas_limit_plain_number() {
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("1").unwrap(), 1);
assert_eq!(parse_gas_limit("0").unwrap(), 0);
}
#[test]
fn test_parse_gas_limit_k_suffix() {
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
}
#[test]
fn test_parse_gas_limit_m_suffix() {
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
}
#[test]
fn test_parse_gas_limit_g_suffix() {
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
}
#[test]
fn test_parse_gas_limit_with_whitespace() {
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
}
#[test]
fn test_parse_gas_limit_errors() {
assert!(parse_gas_limit("").is_err());
assert!(parse_gas_limit("abc").is_err());
assert!(parse_gas_limit("G").is_err());
assert!(parse_gas_limit("-1G").is_err());
}
}

View File

@@ -180,7 +180,7 @@ impl Command {
.filter_map(|e| e.ok())
.filter(|e| {
e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
e.file_name().to_string_lossy().starts_with("payload_block_")
e.file_name().to_string_lossy().starts_with("payload_")
})
.collect();
@@ -191,7 +191,7 @@ impl Command {
let name = e.file_name();
let name_str = name.to_string_lossy();
// Extract index from "payload_NNN.json"
let index_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
let index_str = name_str.strip_prefix("payload_")?.strip_suffix(".json")?;
let index: u64 = index_str.parse().ok()?;
Some((index, e.path()))
})

View File

@@ -41,7 +41,6 @@ derive_more.workspace = true
metrics.workspace = true
parking_lot.workspace = true
pin-project.workspace = true
rayon = { workspace = true, optional = true }
serde = { workspace = true, optional = true }
# optional deps for test-utils
@@ -85,7 +84,6 @@ test-utils = [
"reth-trie/test-utils",
"reth-ethereum-primitives/test-utils",
]
rayon = ["dep:rayon"]
[[bench]]
name = "canonical_hashes_range"

View File

@@ -163,29 +163,14 @@ impl DeferredTrieData {
anchor_hash: B256,
ancestors: &[Self],
) -> ComputedTrieData {
#[cfg(feature = "rayon")]
let (sorted_hashed_state, sorted_trie_updates) = rayon::join(
|| match Arc::try_unwrap(hashed_state) {
Ok(state) => state.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
|| match Arc::try_unwrap(trie_updates) {
Ok(updates) => updates.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
);
#[cfg(not(feature = "rayon"))]
let (sorted_hashed_state, sorted_trie_updates) = (
match Arc::try_unwrap(hashed_state) {
Ok(state) => state.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
match Arc::try_unwrap(trie_updates) {
Ok(updates) => updates.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
);
let sorted_hashed_state = match Arc::try_unwrap(hashed_state) {
Ok(state) => state.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
};
let sorted_trie_updates = match Arc::try_unwrap(trie_updates) {
Ok(updates) => updates.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
};
// Reuse parent's overlay if available and anchors match.
// We can only reuse the parent's overlay if it was built on top of the same
@@ -207,10 +192,10 @@ impl DeferredTrieData {
);
// Only trigger COW clone if there's actually data to add.
if !sorted_hashed_state.is_empty() {
Arc::make_mut(&mut overlay.state).extend_ref_and_sort(&sorted_hashed_state);
Arc::make_mut(&mut overlay.state).extend_ref(&sorted_hashed_state);
}
if !sorted_trie_updates.is_empty() {
Arc::make_mut(&mut overlay.nodes).extend_ref_and_sort(&sorted_trie_updates);
Arc::make_mut(&mut overlay.nodes).extend_ref(&sorted_trie_updates);
}
overlay
}
@@ -243,53 +228,8 @@ impl DeferredTrieData {
/// In normal operation, the parent always has a cached overlay and this
/// function is never called.
///
/// When the `rayon` feature is enabled, uses parallel collection and merge:
/// 1. Collects ancestor data in parallel (each `wait_cloned()` may compute)
/// 2. Merges hashed state and trie updates in parallel with each other
/// 3. Uses tree reduction within each merge for O(log n) depth
#[cfg(feature = "rayon")]
fn merge_ancestors_into_overlay(
ancestors: &[Self],
sorted_hashed_state: &HashedPostStateSorted,
sorted_trie_updates: &TrieUpdatesSorted,
) -> TrieInputSorted {
// Early exit: no ancestors means just wrap current block's data
if ancestors.is_empty() {
return TrieInputSorted::new(
Arc::new(sorted_trie_updates.clone()),
Arc::new(sorted_hashed_state.clone()),
Default::default(),
);
}
// Collect ancestor data, unzipping states and updates into Arc slices
let (states, updates): (Vec<_>, Vec<_>) = ancestors
.iter()
.map(|a| {
let data = a.wait_cloned();
(data.hashed_state, data.trie_updates)
})
.unzip();
// Merge state and nodes in parallel with each other using tree reduction
let (state, nodes) = rayon::join(
|| {
let mut merged = HashedPostStateSorted::merge_parallel(&states);
merged.extend_ref_and_sort(sorted_hashed_state);
merged
},
|| {
let mut merged = TrieUpdatesSorted::merge_parallel(&updates);
merged.extend_ref_and_sort(sorted_trie_updates);
merged
},
);
TrieInputSorted::new(Arc::new(nodes), Arc::new(state), Default::default())
}
/// Merge all ancestors and current block's data into a single overlay (sequential fallback).
#[cfg(not(feature = "rayon"))]
/// Iterates ancestors oldest -> newest, then extends with current block's data,
/// so later state takes precedence.
fn merge_ancestors_into_overlay(
ancestors: &[Self],
sorted_hashed_state: &HashedPostStateSorted,
@@ -302,13 +242,13 @@ impl DeferredTrieData {
for ancestor in ancestors {
let ancestor_data = ancestor.wait_cloned();
state_mut.extend_ref_and_sort(ancestor_data.hashed_state.as_ref());
nodes_mut.extend_ref_and_sort(ancestor_data.trie_updates.as_ref());
state_mut.extend_ref(ancestor_data.hashed_state.as_ref());
nodes_mut.extend_ref(ancestor_data.trie_updates.as_ref());
}
// Extend with current block's sorted data last (takes precedence)
state_mut.extend_ref_and_sort(sorted_hashed_state);
nodes_mut.extend_ref_and_sort(sorted_trie_updates);
state_mut.extend_ref(sorted_hashed_state);
nodes_mut.extend_ref(sorted_trie_updates);
overlay
}
@@ -581,7 +521,7 @@ mod tests {
let hashed_state = Arc::new(HashedPostStateSorted::new(accounts, B256Map::default()));
let trie_updates = Arc::default();
let mut overlay = TrieInputSorted::default();
Arc::make_mut(&mut overlay.state).extend_ref_and_sort(hashed_state.as_ref());
Arc::make_mut(&mut overlay.state).extend_ref(hashed_state.as_ref());
DeferredTrieData::ready(ComputedTrieData {
hashed_state,

View File

@@ -123,18 +123,60 @@ impl LazyOverlay {
/// Merge all blocks' trie data into a single [`TrieInputSorted`].
///
/// Blocks are ordered newest to oldest.
/// Blocks are ordered newest to oldest. Uses hybrid merge algorithm that
/// switches between `extend_ref` (small batches) and k-way merge (large batches).
fn merge_blocks(blocks: &[DeferredTrieData]) -> TrieInputSorted {
const MERGE_BATCH_THRESHOLD: usize = 64;
if blocks.is_empty() {
return TrieInputSorted::default();
}
let state =
HashedPostStateSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().hashed_state));
let nodes =
TrieUpdatesSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().trie_updates));
// Single block: use its data directly (no allocation)
if blocks.len() == 1 {
let data = blocks[0].wait_cloned();
return TrieInputSorted {
state: data.hashed_state,
nodes: data.trie_updates,
prefix_sets: Default::default(),
};
}
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
if blocks.len() < MERGE_BATCH_THRESHOLD {
// Small k: extend_ref loop with Arc::make_mut is faster.
// Uses copy-on-write - only clones inner data if Arc has multiple refs.
// Iterate oldest->newest so newer values override older ones.
let mut blocks_iter = blocks.iter().rev();
let first = blocks_iter.next().expect("blocks is non-empty");
let data = first.wait_cloned();
let mut state = data.hashed_state;
let mut nodes = data.trie_updates;
for block in blocks_iter {
let block_data = block.wait_cloned();
Arc::make_mut(&mut state).extend_ref(block_data.hashed_state.as_ref());
Arc::make_mut(&mut nodes).extend_ref(block_data.trie_updates.as_ref());
}
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
} else {
// Large k: k-way merge is faster (O(n log k)).
// Collect is unavoidable here - we need all data materialized for k-way merge.
let trie_data: Vec<_> = blocks.iter().map(|b| b.wait_cloned()).collect();
let merged_state = HashedPostStateSorted::merge_batch(
trie_data.iter().map(|d| d.hashed_state.as_ref()),
);
let merged_nodes =
TrieUpdatesSorted::merge_batch(trie_data.iter().map(|d| d.trie_updates.as_ref()));
TrieInputSorted {
state: Arc::new(merged_state),
nodes: Arc::new(merged_nodes),
prefix_sets: Default::default(),
}
}
}
}

View File

@@ -278,7 +278,6 @@ pub fn create_chain_config(
// Check if DAO fork is supported (it has an activation block)
let dao_fork_support = hardforks.fork(EthereumHardfork::Dao) != ForkCondition::Never;
#[allow(clippy::needless_update)]
ChainConfig {
chain_id: chain.map(|c| c.id()).unwrap_or(0),
homestead_block: block_num(EthereumHardfork::Homestead),
@@ -314,7 +313,6 @@ pub fn create_chain_config(
extra_fields: Default::default(),
deposit_contract_address,
blob_schedule,
..Default::default()
}
}

View File

@@ -131,4 +131,4 @@ arbitrary = [
"reth-ethereum-primitives/arbitrary",
]
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb"]
edge = ["reth-db-common/edge", "reth-stages/rocksdb"]

View File

@@ -19,7 +19,7 @@ use reth_node_builder::{
Node, NodeComponents, NodeComponentsBuilder, NodeTypes, NodeTypesWithDBAdapter,
};
use reth_node_core::{
args::{DatabaseArgs, DatadirArgs, RocksDbArgs, StaticFilesArgs},
args::{DatabaseArgs, DatadirArgs, StaticFilesArgs},
dirs::{ChainPath, DataDirPath},
};
use reth_provider::{
@@ -27,7 +27,7 @@ use reth_provider::{
BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider,
StaticFileProviderBuilder,
},
ProviderFactory, StaticFileProviderFactory, StorageSettings,
ProviderFactory, StaticFileProviderFactory,
};
use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget};
use reth_static_file::StaticFileProducer;
@@ -66,24 +66,9 @@ pub struct EnvironmentArgs<C: ChainSpecParser> {
/// All static files related arguments
#[command(flatten)]
pub static_files: StaticFilesArgs,
/// All `RocksDB` related arguments
#[command(flatten)]
pub rocksdb: RocksDbArgs,
}
impl<C: ChainSpecParser> EnvironmentArgs<C> {
/// Returns the effective storage settings derived from static-file and `RocksDB` CLI args.
pub fn storage_settings(&self) -> StorageSettings {
StorageSettings::base()
.with_receipts_in_static_files(self.static_files.receipts)
.with_transaction_senders_in_static_files(self.static_files.transaction_senders)
.with_account_changesets_in_static_files(self.static_files.account_changesets)
.with_transaction_hash_numbers_in_rocksdb(self.rocksdb.all || self.rocksdb.tx_hash)
.with_storages_history_in_rocksdb(self.rocksdb.all || self.rocksdb.storages_history)
.with_account_history_in_rocksdb(self.rocksdb.all || self.rocksdb.account_history)
}
/// Initializes environment according to [`AccessRights`] and returns an instance of
/// [`Environment`].
pub fn init<N: CliNodeTypes>(&self, access: AccessRights) -> eyre::Result<Environment<N>>
@@ -136,17 +121,17 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
})
}
};
// TransactionDB only support read-write mode
let rocksdb_provider = RocksDBProvider::builder(data_dir.rocksdb())
.with_default_tables()
.with_database_log_level(self.db.log_level)
.with_read_only(!access.is_read_write())
.build()?;
let provider_factory =
self.create_provider_factory(&config, db, sfp, rocksdb_provider, access)?;
if access.is_read_write() {
debug!(target: "reth::cli", chain=%self.chain.chain(), genesis=?self.chain.genesis_hash(), "Initializing genesis");
init_genesis_with_settings(&provider_factory, self.storage_settings())?;
init_genesis_with_settings(&provider_factory, self.static_files.to_settings())?;
}
Ok(Environment { config, provider_factory, data_dir })

View File

@@ -0,0 +1,145 @@
use crate::{
common::CliNodeTypes,
db::get::{maybe_json_value_parser, table_key},
};
use alloy_primitives::map::foldhash::fast::FixedState;
use clap::Parser;
use reth_chainspec::EthereumHardforks;
use reth_db::DatabaseEnv;
use reth_db_api::{
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
Tables,
};
use reth_db_common::DbTool;
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_provider::{providers::ProviderNodeTypes, DBProvider};
use std::{
hash::{BuildHasher, Hasher},
sync::Arc,
time::{Duration, Instant},
};
use tracing::{info, warn};
#[derive(Parser, Debug)]
/// The arguments for the `reth db checksum` command
pub struct Command {
/// The table name
table: Tables,
/// The start of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
start_key: Option<String>,
/// The end of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
end_key: Option<String>,
/// The maximum number of records that are queried and used to compute the
/// checksum.
#[arg(long)]
limit: Option<usize>,
}
impl Command {
/// Execute `db checksum` command
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
) -> eyre::Result<()> {
warn!("This command should be run without the node running!");
self.table.view(&ChecksumViewer {
tool,
start_key: self.start_key,
end_key: self.end_key,
limit: self.limit,
})?;
Ok(())
}
}
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
tool: &'a DbTool<N>,
start_key: Option<String>,
end_key: Option<String>,
limit: Option<usize>,
}
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
}
}
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
type Error = eyre::Report;
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
let provider =
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
let tx = provider.tx_ref();
info!(
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
self.start_key, self.end_key, self.limit
);
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
(Some(start), Some(end)) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(start_key..=end_key)?
}
(None, Some(end)) => {
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(..=end_key)?
}
(Some(start), None) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
cursor.walk_range(start_key..)?
}
(None, None) => cursor.walk_range(..)?,
};
let start_time = Instant::now();
let mut hasher = FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher();
let mut total = 0;
let limit = self.limit.unwrap_or(usize::MAX);
let mut enumerate_start_key = None;
let mut enumerate_end_key = None;
for (index, entry) in walker.enumerate() {
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
if index.is_multiple_of(100_000) {
info!("Hashed {index} entries.");
}
hasher.write(k.raw_key());
hasher.write(v.raw_value());
if enumerate_start_key.is_none() {
enumerate_start_key = Some(k.clone());
}
enumerate_end_key = Some(k);
total = index + 1;
if total >= limit {
break
}
}
info!("Hashed {total} entries.");
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
Ok((checksum, elapsed))
}
}

View File

@@ -1,288 +0,0 @@
use crate::{
common::CliNodeTypes,
db::get::{maybe_json_value_parser, table_key},
};
use alloy_primitives::map::foldhash::fast::FixedState;
use clap::Parser;
use itertools::Itertools;
use reth_chainspec::EthereumHardforks;
use reth_db::{static_file::iter_static_files, DatabaseEnv};
use reth_db_api::{
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
Tables,
};
use reth_db_common::DbTool;
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProviderFactory};
use reth_static_file_types::StaticFileSegment;
use std::{
hash::{BuildHasher, Hasher},
sync::Arc,
time::{Duration, Instant},
};
use tracing::{info, warn};
#[cfg(all(unix, feature = "edge"))]
mod rocksdb;
/// Interval for logging progress during checksum computation.
const PROGRESS_LOG_INTERVAL: usize = 100_000;
#[derive(Parser, Debug)]
/// The arguments for the `reth db checksum` command
pub struct Command {
#[command(subcommand)]
subcommand: Subcommand,
}
#[derive(clap::Subcommand, Debug)]
enum Subcommand {
/// Calculates the checksum of a database table
Mdbx {
/// The table name
table: Tables,
/// The start of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
start_key: Option<String>,
/// The end of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
end_key: Option<String>,
/// The maximum number of records that are queried and used to compute the
/// checksum.
#[arg(long)]
limit: Option<usize>,
},
/// Calculates the checksum of a static file segment
StaticFile {
/// The static file segment
#[arg(value_enum)]
segment: StaticFileSegment,
/// The block number to start from (inclusive).
#[arg(long)]
start_block: Option<u64>,
/// The block number to end at (inclusive).
#[arg(long)]
end_block: Option<u64>,
/// The maximum number of rows to checksum.
#[arg(long)]
limit: Option<usize>,
},
/// Calculates the checksum of a RocksDB table
#[cfg(all(unix, feature = "edge"))]
Rocksdb {
/// The RocksDB table
#[arg(value_enum)]
table: rocksdb::RocksDbTable,
/// The maximum number of records to checksum.
#[arg(long)]
limit: Option<usize>,
},
}
impl Command {
/// Execute `db checksum` command
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
) -> eyre::Result<()> {
warn!("This command should be run without the node running!");
match self.subcommand {
Subcommand::Mdbx { table, start_key, end_key, limit } => {
table.view(&ChecksumViewer { tool, start_key, end_key, limit })?;
}
Subcommand::StaticFile { segment, start_block, end_block, limit } => {
checksum_static_file(tool, segment, start_block, end_block, limit)?;
}
#[cfg(all(unix, feature = "edge"))]
Subcommand::Rocksdb { table, limit } => {
rocksdb::checksum_rocksdb(tool, table, limit)?;
}
}
Ok(())
}
}
/// Creates a new hasher with the standard seed used for checksum computation.
fn checksum_hasher() -> impl Hasher {
FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher()
}
fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
segment: StaticFileSegment,
start_block: Option<u64>,
end_block: Option<u64>,
limit: Option<usize>,
) -> eyre::Result<()> {
let static_file_provider = tool.provider_factory.static_file_provider();
if let Err(err) = static_file_provider.check_consistency(&tool.provider_factory.provider()?) {
warn!("Error checking consistency of static files: {err}");
}
let static_files = iter_static_files(static_file_provider.directory())?;
let ranges = static_files
.get(segment)
.ok_or_else(|| eyre::eyre!("No static files found for segment: {}", segment))?;
let start_time = Instant::now();
let mut hasher = checksum_hasher();
let mut total = 0usize;
let limit = limit.unwrap_or(usize::MAX);
let start_block = start_block.unwrap_or(0);
let end_block = end_block.unwrap_or(u64::MAX);
info!(
"Computing checksum for {} static files, start_block={}, end_block={}, limit={:?}",
segment,
start_block,
end_block,
if limit == usize::MAX { None } else { Some(limit) }
);
'outer: for (block_range, _header) in ranges.iter().sorted_by_key(|(range, _)| range.start()) {
if block_range.end() < start_block || block_range.start() > end_block {
continue;
}
let fixed_block_range = static_file_provider.find_fixed_range(segment, block_range.start());
let jar_provider = static_file_provider
.get_segment_provider_for_range(segment, || Some(fixed_block_range), None)?
.ok_or_else(|| {
eyre::eyre!(
"Failed to get segment provider for segment {} at range {}",
segment,
block_range
)
})?;
let mut cursor = jar_provider.cursor()?;
while let Ok(Some(row)) = cursor.next_row() {
for col_data in row.iter() {
hasher.write(col_data);
}
total += 1;
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {total} entries.");
}
if total >= limit {
break 'outer;
}
}
// Explicitly drop provider before removing from cache to avoid deadlock
drop(jar_provider);
static_file_provider.remove_cached_provider(segment, fixed_block_range.end());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!(
"Checksum for static file segment `{}`: {:#x} ({} entries, elapsed: {:?})",
segment, checksum, total, elapsed
);
Ok(())
}
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
tool: &'a DbTool<N>,
start_key: Option<String>,
end_key: Option<String>,
limit: Option<usize>,
}
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
}
}
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
type Error = eyre::Report;
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
let provider =
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
let tx = provider.tx_ref();
info!(
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
self.start_key, self.end_key, self.limit
);
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
(Some(start), Some(end)) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(start_key..=end_key)?
}
(None, Some(end)) => {
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(..=end_key)?
}
(Some(start), None) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
cursor.walk_range(start_key..)?
}
(None, None) => cursor.walk_range(..)?,
};
let start_time = Instant::now();
let mut hasher = checksum_hasher();
let mut total = 0;
let limit = self.limit.unwrap_or(usize::MAX);
let mut enumerate_start_key = None;
let mut enumerate_end_key = None;
for (index, entry) in walker.enumerate() {
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
if index.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {index} entries.");
}
hasher.write(k.raw_key());
hasher.write(v.raw_value());
if enumerate_start_key.is_none() {
enumerate_start_key = Some(k.clone());
}
enumerate_end_key = Some(k);
total = index + 1;
if total >= limit {
break
}
}
info!("Hashed {total} entries.");
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
Ok((checksum, elapsed))
}
}

View File

@@ -1,106 +0,0 @@
//! RocksDB checksum implementation.
use super::{checksum_hasher, PROGRESS_LOG_INTERVAL};
use crate::common::CliNodeTypes;
use clap::ValueEnum;
use reth_chainspec::EthereumHardforks;
use reth_db::{tables, DatabaseEnv};
use reth_db_api::table::Table;
use reth_db_common::DbTool;
use reth_node_builder::NodeTypesWithDBAdapter;
use reth_provider::RocksDBProviderFactory;
use std::{hash::Hasher, sync::Arc, time::Instant};
use tracing::info;
/// RocksDB tables that can be checksummed.
#[derive(Debug, Clone, Copy, ValueEnum)]
pub enum RocksDbTable {
/// Transaction hash to transaction number mapping
TransactionHashNumbers,
/// Account history indices
AccountsHistory,
/// Storage history indices
StoragesHistory,
}
impl RocksDbTable {
/// Returns the table name as a string
const fn name(&self) -> &'static str {
match self {
Self::TransactionHashNumbers => tables::TransactionHashNumbers::NAME,
Self::AccountsHistory => tables::AccountsHistory::NAME,
Self::StoragesHistory => tables::StoragesHistory::NAME,
}
}
}
/// Computes a checksum for a RocksDB table.
pub fn checksum_rocksdb<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
table: RocksDbTable,
limit: Option<usize>,
) -> eyre::Result<()> {
let rocksdb = tool.provider_factory.rocksdb_provider();
let start_time = Instant::now();
let limit = limit.unwrap_or(usize::MAX);
info!(
"Computing checksum for RocksDB table `{}`, limit={:?}",
table.name(),
if limit == usize::MAX { None } else { Some(limit) }
);
let (checksum, total) = match table {
RocksDbTable::TransactionHashNumbers => {
checksum_rocksdb_table::<tables::TransactionHashNumbers>(&rocksdb, limit)?
}
RocksDbTable::AccountsHistory => {
checksum_rocksdb_table::<tables::AccountsHistory>(&rocksdb, limit)?
}
RocksDbTable::StoragesHistory => {
checksum_rocksdb_table::<tables::StoragesHistory>(&rocksdb, limit)?
}
};
let elapsed = start_time.elapsed();
info!(
"Checksum for RocksDB table `{}`: {:#x} ({} entries, elapsed: {:?})",
table.name(),
checksum,
total,
elapsed
);
Ok(())
}
/// Computes checksum for a specific RocksDB table by iterating over rows.
fn checksum_rocksdb_table<T: Table>(
rocksdb: &reth_provider::providers::RocksDBProvider,
limit: usize,
) -> eyre::Result<(u64, usize)> {
let iter = rocksdb.raw_iter::<T>()?;
let mut hasher = checksum_hasher();
let mut total = 0usize;
for entry in iter {
let (key_bytes, value_bytes) = entry?;
hasher.write(&key_bytes);
hasher.write(&value_bytes);
total += 1;
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {total} entries.");
}
if total >= limit {
break;
}
}
Ok((hasher.finish(), total))
}

View File

@@ -39,7 +39,7 @@ pub enum Subcommands {
Stats(stats::Command),
/// Lists the contents of a table
List(list::Command),
/// Calculates the content checksum of a table or static file segment
/// Calculates the content checksum of a table
Checksum(checksum::Command),
/// Create a diff between two database tables or two entire databases.
Diff(diff::Command),

View File

@@ -11,10 +11,7 @@ use reth_db_common::DbTool;
use reth_fs_util as fs;
use reth_node_builder::{NodePrimitives, NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_provider::{
providers::{ProviderNodeTypes, StaticFileProvider},
RocksDBProviderFactory,
};
use reth_provider::providers::{ProviderNodeTypes, StaticFileProvider};
use reth_static_file_types::SegmentRangeInclusive;
use std::{sync::Arc, time::Duration};
@@ -64,11 +61,6 @@ impl Command {
let db_stats_table = self.db_stats_table(tool)?;
println!("{db_stats_table}");
println!("\n");
let rocksdb_stats_table = self.rocksdb_stats_table(tool);
println!("{rocksdb_stats_table}");
Ok(())
}
@@ -156,60 +148,6 @@ impl Command {
Ok(table)
}
fn rocksdb_stats_table<N: NodeTypesWithDB>(&self, tool: &DbTool<N>) -> ComfyTable {
let mut table = ComfyTable::new();
table.load_preset(comfy_table::presets::ASCII_MARKDOWN);
table.set_header([
"RocksDB Table Name",
"# Entries",
"SST Size",
"Memtable Size",
"Total Size",
"Pending Compaction",
]);
let stats = tool.provider_factory.rocksdb_provider().table_stats();
let mut total_sst: u64 = 0;
let mut total_memtable: u64 = 0;
let mut total_size: u64 = 0;
let mut total_pending: u64 = 0;
for stat in &stats {
total_sst += stat.sst_size_bytes;
total_memtable += stat.memtable_size_bytes;
total_size += stat.estimated_size_bytes;
total_pending += stat.pending_compaction_bytes;
let mut row = Row::new();
row.add_cell(Cell::new(&stat.name))
.add_cell(Cell::new(stat.estimated_num_keys))
.add_cell(Cell::new(human_bytes(stat.sst_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.memtable_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.estimated_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.pending_compaction_bytes as f64)));
table.add_row(row);
}
if !stats.is_empty() {
let max_widths = table.column_max_content_widths();
let mut separator = Row::new();
for width in max_widths {
separator.add_cell(Cell::new("-".repeat(width as usize)));
}
table.add_row(separator);
let mut row = Row::new();
row.add_cell(Cell::new("RocksDB Total"))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(total_sst as f64)))
.add_cell(Cell::new(human_bytes(total_memtable as f64)))
.add_cell(Cell::new(human_bytes(total_size as f64)))
.add_cell(Cell::new(human_bytes(total_pending as f64)));
table.add_row(row);
}
table
}
fn static_files_stats_table<N: NodePrimitives>(
&self,
data_dir: ChainPath<DataDirPath>,

View File

@@ -2,15 +2,14 @@ use crate::common::EnvironmentArgs;
use clap::Parser;
use eyre::Result;
use lz4::Decoder;
use reqwest::{blocking::Client as BlockingClient, header::RANGE, Client, StatusCode};
use reqwest::Client;
use reth_chainspec::{EthChainSpec, EthereumHardforks};
use reth_cli::chainspec::ChainSpecParser;
use reth_fs_util as fs;
use std::{
borrow::Cow,
fs::OpenOptions,
io::{self, BufWriter, Read, Write},
path::{Path, PathBuf},
io::{self, Read, Write},
path::Path,
sync::{Arc, OnceLock},
time::{Duration, Instant},
};
@@ -328,158 +327,18 @@ fn extract_from_file(path: &Path, format: CompressionFormat, target_dir: &Path)
extract_archive(file, total_size, format, target_dir)
}
const MAX_DOWNLOAD_RETRIES: u32 = 10;
const RETRY_BACKOFF_SECS: u64 = 5;
/// Wrapper that tracks download progress while writing data.
/// Used with [`io::copy`] to display progress during downloads.
struct ProgressWriter<W> {
inner: W,
progress: DownloadProgress,
}
impl<W: Write> Write for ProgressWriter<W> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
let n = self.inner.write(buf)?;
let _ = self.progress.update(n as u64);
Ok(n)
}
fn flush(&mut self) -> io::Result<()> {
self.inner.flush()
}
}
/// Downloads a file with resume support using HTTP Range requests.
/// Automatically retries on failure, resuming from where it left off.
/// Returns the path to the downloaded file and its total size.
fn resumable_download(url: &str, target_dir: &Path) -> Result<(PathBuf, u64)> {
let file_name = Url::parse(url)
.ok()
.and_then(|u| u.path_segments()?.next_back().map(|s| s.to_string()))
.unwrap_or_else(|| "snapshot.tar".to_string());
let final_path = target_dir.join(&file_name);
let part_path = target_dir.join(format!("{file_name}.part"));
let client = BlockingClient::builder().timeout(Duration::from_secs(30)).build()?;
let mut total_size: Option<u64> = None;
let mut last_error: Option<eyre::Error> = None;
for attempt in 1..=MAX_DOWNLOAD_RETRIES {
let existing_size = fs::metadata(&part_path).map(|m| m.len()).unwrap_or(0);
if let Some(total) = total_size &&
existing_size >= total
{
fs::rename(&part_path, &final_path)?;
info!(target: "reth::cli", "Download complete: {}", final_path.display());
return Ok((final_path, total));
}
if attempt > 1 {
info!(target: "reth::cli",
"Retry attempt {}/{} - resuming from {} bytes",
attempt, MAX_DOWNLOAD_RETRIES, existing_size
);
}
let mut request = client.get(url);
if existing_size > 0 {
request = request.header(RANGE, format!("bytes={existing_size}-"));
if attempt == 1 {
info!(target: "reth::cli", "Resuming download from {} bytes", existing_size);
}
}
let response = match request.send().and_then(|r| r.error_for_status()) {
Ok(r) => r,
Err(e) => {
last_error = Some(e.into());
if attempt < MAX_DOWNLOAD_RETRIES {
info!(target: "reth::cli",
"Download failed, retrying in {} seconds...", RETRY_BACKOFF_SECS
);
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
}
continue;
}
};
let is_partial = response.status() == StatusCode::PARTIAL_CONTENT;
let size = if is_partial {
response
.headers()
.get("Content-Range")
.and_then(|v| v.to_str().ok())
.and_then(|v| v.split('/').next_back())
.and_then(|v| v.parse().ok())
} else {
response.content_length()
};
if total_size.is_none() {
total_size = size;
}
let current_total = total_size.ok_or_else(|| {
eyre::eyre!("Server did not provide Content-Length or Content-Range header")
})?;
let file = if is_partial && existing_size > 0 {
OpenOptions::new()
.append(true)
.open(&part_path)
.map_err(|e| fs::FsPathError::open(e, &part_path))?
} else {
fs::create_file(&part_path)?
};
let start_offset = if is_partial { existing_size } else { 0 };
let mut progress = DownloadProgress::new(current_total);
progress.downloaded = start_offset;
let mut writer = ProgressWriter { inner: BufWriter::new(file), progress };
let mut reader = response;
let copy_result = io::copy(&mut reader, &mut writer);
let flush_result = writer.inner.flush();
println!();
if let Err(e) = copy_result.and(flush_result) {
last_error = Some(e.into());
if attempt < MAX_DOWNLOAD_RETRIES {
info!(target: "reth::cli",
"Download interrupted, retrying in {} seconds...", RETRY_BACKOFF_SECS
);
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
}
continue;
}
fs::rename(&part_path, &final_path)?;
info!(target: "reth::cli", "Download complete: {}", final_path.display());
return Ok((final_path, current_total));
}
Err(last_error
.unwrap_or_else(|| eyre::eyre!("Download failed after {} attempts", MAX_DOWNLOAD_RETRIES)))
}
/// Fetches the snapshot from a remote URL with resume support, then extracts it.
/// Fetches the snapshot from a remote URL, uncompressing it in a streaming fashion.
fn download_and_extract(url: &str, format: CompressionFormat, target_dir: &Path) -> Result<()> {
let (downloaded_path, total_size) = resumable_download(url, target_dir)?;
let client = reqwest::blocking::Client::builder().build()?;
let response = client.get(url).send()?.error_for_status()?;
info!(target: "reth::cli", "Extracting snapshot...");
let file = fs::open(&downloaded_path)?;
extract_archive(file, total_size, format, target_dir)?;
let total_size = response.content_length().ok_or_else(|| {
eyre::eyre!(
"Server did not provide Content-Length header. This is required for snapshot downloads"
)
})?;
fs::remove_file(&downloaded_path)?;
info!(target: "reth::cli", "Removed downloaded archive");
Ok(())
extract_archive(response, total_size, format, target_dir)
}
/// Downloads and extracts a snapshot, blocking until finished.

View File

@@ -10,8 +10,7 @@ use reth_node_builder::NodeBuilder;
use reth_node_core::{
args::{
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, EraArgs, MetricArgs,
NetworkArgs, PayloadBuilderArgs, PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs,
TxPoolArgs,
NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
},
node_config::NodeConfig,
version,
@@ -103,10 +102,6 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
#[command(flatten)]
pub pruning: PruningArgs,
/// All `RocksDB` table routing arguments
#[command(flatten)]
pub rocksdb: RocksDbArgs,
/// Engine cli arguments
#[command(flatten, next_help_heading = "Engine")]
pub engine: EngineArgs,
@@ -171,16 +166,12 @@ where
db,
dev,
pruning,
rocksdb,
engine,
era,
static_files,
ext,
} = self;
// Validate RocksDB arguments
rocksdb.validate()?;
// set up node config
let mut node_config = NodeConfig {
datadir,
@@ -196,7 +187,6 @@ where
db,
dev,
pruning,
rocksdb,
engine,
era,
static_files,

View File

@@ -121,16 +121,7 @@ pub fn install() {
unsafe {
let alt_stack_size: usize = min_sigstack_size() + 64 * 1024;
let mut alt_stack: libc::stack_t = mem::zeroed();
// Both SysV AMD64 ABI and aarch64 ABI require 16 bytes alignment. We are going to be
// generous here and just use a size of a page.
let raw_page_sz = libc::sysconf(libc::_SC_PAGESIZE);
let page_sz = if raw_page_sz == -1 {
// Fallback alignment in case sysconf fails.
4096_usize
} else {
raw_page_sz as usize
};
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, page_sz).unwrap()).cast();
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, 1).unwrap()).cast();
alt_stack.ss_size = alt_stack_size;
libc::sigaltstack(&raw const alt_stack, ptr::null_mut());

View File

@@ -137,8 +137,6 @@ pub struct TreeConfig {
account_worker_count: usize,
/// Whether to enable V2 storage proofs.
enable_proof_v2: bool,
/// Whether to disable cache metrics recording (can be expensive with large cached state).
disable_cache_metrics: bool,
}
impl Default for TreeConfig {
@@ -168,7 +166,6 @@ impl Default for TreeConfig {
storage_worker_count: default_storage_worker_count(),
account_worker_count: default_account_worker_count(),
enable_proof_v2: false,
disable_cache_metrics: false,
}
}
}
@@ -201,7 +198,6 @@ impl TreeConfig {
storage_worker_count: usize,
account_worker_count: usize,
enable_proof_v2: bool,
disable_cache_metrics: bool,
) -> Self {
Self {
persistence_threshold,
@@ -228,7 +224,6 @@ impl TreeConfig {
storage_worker_count,
account_worker_count,
enable_proof_v2,
disable_cache_metrics,
}
}
@@ -521,15 +516,4 @@ impl TreeConfig {
self.enable_proof_v2 = enable_proof_v2;
self
}
/// Returns whether cache metrics recording is disabled.
pub const fn disable_cache_metrics(&self) -> bool {
self.disable_cache_metrics
}
/// Setter for whether to disable cache metrics recording.
pub const fn without_cache_metrics(mut self, disable_cache_metrics: bool) -> Self {
self.disable_cache_metrics = disable_cache_metrics;
self
}
}

View File

@@ -12,7 +12,7 @@ workspace = true
[dependencies]
# reth
reth-chain-state = { workspace = true, features = ["rayon"] }
reth-chain-state.workspace = true
reth-chainspec = { workspace = true, optional = true }
reth-consensus.workspace = true
reth-db.workspace = true

View File

@@ -606,21 +606,12 @@ pub(crate) struct SavedCache {
/// A guard to track in-flight usage of this cache.
/// The cache is considered available if the strong count is 1.
usage_guard: Arc<()>,
/// Whether to skip cache metrics recording (can be expensive with large cached state).
disable_cache_metrics: bool,
}
impl SavedCache {
/// Creates a new instance with the internals
pub(super) fn new(hash: B256, caches: ExecutionCache, metrics: CachedStateMetrics) -> Self {
Self { hash, caches, metrics, usage_guard: Arc::new(()), disable_cache_metrics: false }
}
/// Sets whether to disable cache metrics recording.
pub(super) const fn with_disable_cache_metrics(mut self, disable: bool) -> Self {
self.disable_cache_metrics = disable;
self
Self { hash, caches, metrics, usage_guard: Arc::new(()) }
}
/// Returns the hash for this cache
@@ -628,9 +619,9 @@ impl SavedCache {
self.hash
}
/// Splits the cache into its caches, metrics, and `disable_cache_metrics` flag, consuming it.
pub(crate) fn split(self) -> (ExecutionCache, CachedStateMetrics, bool) {
(self.caches, self.metrics, self.disable_cache_metrics)
/// Splits the cache into its caches and metrics, consuming it.
pub(crate) fn split(self) -> (ExecutionCache, CachedStateMetrics) {
(self.caches, self.metrics)
}
/// Returns true if the cache is available for use (no other tasks are currently using it).
@@ -654,13 +645,7 @@ impl SavedCache {
}
/// Updates the metrics for the [`ExecutionCache`].
///
/// Note: This can be expensive with large cached state as it iterates over
/// all storage entries. Use `with_disable_cache_metrics(true)` to skip.
pub(crate) fn update_metrics(&self) {
if self.disable_cache_metrics {
return;
}
self.metrics.storage_cache_size.set(self.caches.total_storage_slots() as f64);
self.metrics.account_cache_size.set(self.caches.account_cache.entry_count() as f64);
self.metrics.code_cache_size.set(self.caches.code_cache.entry_count() as f64);

View File

@@ -1,15 +1,25 @@
use crate::tree::{error::InsertBlockFatalError, TreeOutcome};
use crate::tree::{error::InsertBlockFatalError, MeteredStateHook, TreeOutcome};
use alloy_consensus::transaction::TxHashRef;
use alloy_evm::{
block::{BlockExecutor, ExecutableTx},
Evm,
};
use alloy_rpc_types_engine::{PayloadStatus, PayloadStatusEnum};
use core::borrow::BorrowMut;
use reth_engine_primitives::{ForkchoiceStatus, OnForkChoiceUpdated};
use reth_errors::ProviderError;
use reth_evm::metrics::ExecutorMetrics;
use reth_errors::{BlockExecutionError, ProviderError};
use reth_evm::{metrics::ExecutorMetrics, OnStateHook};
use reth_execution_types::BlockExecutionOutput;
use reth_metrics::{
metrics::{Counter, Gauge, Histogram},
Metrics,
};
use reth_primitives_traits::SignedTransaction;
use reth_trie::updates::TrieUpdates;
use std::time::{Duration, Instant};
use revm::database::{states::bundle_state::BundleRetention, State};
use revm_primitives::Address;
use std::time::Instant;
use tracing::{debug_span, trace};
/// Metrics for the `EngineApi`.
#[derive(Debug, Default)]
@@ -25,24 +35,114 @@ pub(crate) struct EngineApiMetrics {
}
impl EngineApiMetrics {
/// Records metrics for block execution.
/// Helper function for metered execution
fn metered<F, R>(&self, f: F) -> R
where
F: FnOnce() -> (u64, R),
{
// Execute the block and record the elapsed time.
let execute_start = Instant::now();
let (gas_used, output) = f();
let execution_duration = execute_start.elapsed().as_secs_f64();
// Update gas metrics.
self.executor.gas_processed_total.increment(gas_used);
self.executor.gas_per_second.set(gas_used as f64 / execution_duration);
self.executor.gas_used_histogram.record(gas_used as f64);
self.executor.execution_histogram.record(execution_duration);
self.executor.execution_duration.set(execution_duration);
output
}
/// Execute the given block using the provided [`BlockExecutor`] and update metrics for the
/// execution.
///
/// This method updates metrics for execution time, gas usage, and the number
/// of accounts, storage slots and bytecodes updated.
pub(crate) fn record_block_execution<R>(
/// of accounts, storage slots and bytecodes loaded and updated.
///
/// The optional `on_receipt` callback is invoked after each transaction with the receipt
/// index and a reference to all receipts collected so far. This allows callers to stream
/// receipts to a background task for incremental receipt root computation.
pub(crate) fn execute_metered<E, DB, F>(
&self,
output: &BlockExecutionOutput<R>,
execution_duration: Duration,
) {
let execution_secs = execution_duration.as_secs_f64();
let gas_used = output.result.gas_used;
executor: E,
mut transactions: impl Iterator<Item = Result<impl ExecutableTx<E>, BlockExecutionError>>,
transaction_count: usize,
state_hook: Box<dyn OnStateHook>,
mut on_receipt: F,
) -> Result<(BlockExecutionOutput<E::Receipt>, Vec<Address>), BlockExecutionError>
where
DB: alloy_evm::Database,
E: BlockExecutor<Evm: Evm<DB: BorrowMut<State<DB>>>, Transaction: SignedTransaction>,
F: FnMut(&[E::Receipt]),
{
// clone here is cheap, all the metrics are Option<Arc<_>>. additionally
// they are globally registered so that the data recorded in the hook will
// be accessible.
let wrapper = MeteredStateHook { metrics: self.executor.clone(), inner_hook: state_hook };
// Update gas metrics
self.executor.gas_processed_total.increment(gas_used);
self.executor.gas_per_second.set(gas_used as f64 / execution_secs);
self.executor.gas_used_histogram.record(gas_used as f64);
self.executor.execution_histogram.record(execution_secs);
self.executor.execution_duration.set(execution_secs);
let mut senders = Vec::with_capacity(transaction_count);
let mut executor = executor.with_state_hook(Some(Box::new(wrapper)));
let f = || {
let start = Instant::now();
debug_span!(target: "engine::tree", "pre execution")
.entered()
.in_scope(|| executor.apply_pre_execution_changes())?;
self.executor.pre_execution_histogram.record(start.elapsed());
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
loop {
let start = Instant::now();
let Some(tx) = transactions.next() else { break };
self.executor.transaction_wait_histogram.record(start.elapsed());
let tx = tx?;
senders.push(*tx.signer());
let span = debug_span!(
target: "engine::tree",
"execute tx",
tx_hash = ?tx.tx().tx_hash(),
gas_used = tracing::field::Empty,
);
let enter = span.entered();
trace!(target: "engine::tree", "Executing transaction");
let start = Instant::now();
let gas_used = executor.execute_transaction(tx)?;
self.executor.transaction_execution_histogram.record(start.elapsed());
// Invoke callback with the latest receipt
on_receipt(executor.receipts());
// record the tx gas used
enter.record("gas_used", gas_used);
}
drop(exec_span);
let start = Instant::now();
let result = debug_span!(target: "engine::tree", "finish")
.entered()
.in_scope(|| executor.finish())
.map(|(evm, result)| (evm.into_db(), result));
self.executor.post_execution_histogram.record(start.elapsed());
result
};
// Use metered to execute and track timing/gas metrics
let (mut db, result) = self.metered(|| {
let res = f();
let gas_used = res.as_ref().map(|r| r.1.gas_used).unwrap_or(0);
(gas_used, res)
})?;
// merge transitions into bundle state
debug_span!(target: "engine::tree", "merge transitions")
.entered()
.in_scope(|| db.borrow_mut().merge_transitions(BundleRetention::Reverts));
let output = BlockExecutionOutput { result, state: db.borrow_mut().take_bundle() };
// Update the metrics for the number of accounts, storage slots and bytecodes updated
let accounts = output.state.state.len();
@@ -53,31 +153,8 @@ impl EngineApiMetrics {
self.executor.accounts_updated_histogram.record(accounts as f64);
self.executor.storage_slots_updated_histogram.record(storage_slots as f64);
self.executor.bytecodes_updated_histogram.record(bytecodes as f64);
}
/// Returns a reference to the executor metrics for use in state hooks.
pub(crate) const fn executor_metrics(&self) -> &ExecutorMetrics {
&self.executor
}
/// Records the duration of block pre-execution changes (e.g., beacon root update).
pub(crate) fn record_pre_execution(&self, elapsed: Duration) {
self.executor.pre_execution_histogram.record(elapsed);
}
/// Records the duration of block post-execution changes (e.g., finalization).
pub(crate) fn record_post_execution(&self, elapsed: Duration) {
self.executor.post_execution_histogram.record(elapsed);
}
/// Records the time spent waiting for the next transaction from the iterator.
pub(crate) fn record_transaction_wait(&self, elapsed: Duration) {
self.executor.transaction_wait_histogram.record(elapsed);
}
/// Records the duration of a single transaction execution.
pub(crate) fn record_transaction_execution(&self, elapsed: Duration) {
self.executor.transaction_execution_histogram.record(elapsed);
Ok((output, senders))
}
}
@@ -133,12 +210,6 @@ pub(crate) struct EngineMetrics {
#[derive(Metrics)]
#[metrics(scope = "consensus.engine.beacon")]
pub(crate) struct ForkchoiceUpdatedMetrics {
/// Finish time of the latest forkchoice updated call.
#[metric(skip)]
pub(crate) latest_finish_at: Option<Instant>,
/// Start time of the latest forkchoice updated call.
#[metric(skip)]
pub(crate) latest_start_at: Option<Instant>,
/// The total count of forkchoice updated messages received.
pub(crate) forkchoice_updated_messages: Counter,
/// The total count of forkchoice updated messages with payload received.
@@ -161,35 +232,18 @@ pub(crate) struct ForkchoiceUpdatedMetrics {
pub(crate) forkchoice_updated_last: Gauge,
/// Time diff between new payload call response and the next forkchoice updated call request.
pub(crate) new_payload_forkchoice_updated_time_diff: Histogram,
/// Time from previous forkchoice updated finish to current forkchoice updated start (idle
/// time).
pub(crate) time_between_forkchoice_updated: Histogram,
/// Time from previous forkchoice updated start to current forkchoice updated start (total
/// interval).
pub(crate) forkchoice_updated_interval: Histogram,
}
impl ForkchoiceUpdatedMetrics {
/// Increment the forkchoiceUpdated counter based on the given result
pub(crate) fn update_response_metrics(
&mut self,
&self,
start: Instant,
latest_new_payload_at: &mut Option<Instant>,
has_attrs: bool,
result: &Result<TreeOutcome<OnForkChoiceUpdated>, ProviderError>,
) {
let finish = Instant::now();
let elapsed = finish - start;
if let Some(prev_finish) = self.latest_finish_at {
self.time_between_forkchoice_updated.record(start - prev_finish);
}
if let Some(prev_start) = self.latest_start_at {
self.forkchoice_updated_interval.record(start - prev_start);
}
self.latest_finish_at = Some(finish);
self.latest_start_at = Some(start);
let elapsed = start.elapsed();
match result {
Ok(outcome) => match outcome.outcome.forkchoice_status() {
ForkchoiceStatus::Valid => self.forkchoice_updated_valid.increment(1),
@@ -356,10 +410,138 @@ pub(crate) struct BlockBufferMetrics {
mod tests {
use super::*;
use alloy_eips::eip7685::Requests;
use alloy_evm::block::StateChangeSource;
use alloy_primitives::{B256, U256};
use metrics_util::debugging::{DebuggingRecorder, Snapshotter};
use reth_ethereum_primitives::Receipt;
use reth_ethereum_primitives::{Receipt, TransactionSigned};
use reth_evm_ethereum::EthEvm;
use reth_execution_types::BlockExecutionResult;
use reth_revm::db::BundleState;
use reth_primitives_traits::RecoveredBlock;
use revm::{
context::result::{ExecutionResult, Output, ResultAndState, SuccessReason},
database::State,
database_interface::EmptyDB,
inspector::NoOpInspector,
state::{Account, AccountInfo, AccountStatus, EvmState, EvmStorage, EvmStorageSlot},
Context, MainBuilder, MainContext,
};
use revm_primitives::Bytes;
use std::sync::mpsc;
/// A simple mock executor for testing that doesn't require complex EVM setup
struct MockExecutor {
state: EvmState,
receipts: Vec<Receipt>,
hook: Option<Box<dyn OnStateHook>>,
}
impl MockExecutor {
fn new(state: EvmState) -> Self {
Self { state, receipts: vec![], hook: None }
}
}
// Mock Evm type for testing
type MockEvm = EthEvm<State<EmptyDB>, NoOpInspector>;
impl BlockExecutor for MockExecutor {
type Transaction = TransactionSigned;
type Receipt = Receipt;
type Evm = MockEvm;
fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> {
Ok(())
}
fn execute_transaction_without_commit(
&mut self,
_tx: impl ExecutableTx<Self>,
) -> Result<ResultAndState<<Self::Evm as Evm>::HaltReason>, BlockExecutionError> {
// Call hook with our mock state for each transaction
if let Some(hook) = self.hook.as_mut() {
hook.on_state(StateChangeSource::Transaction(0), &self.state);
}
Ok(ResultAndState::new(
ExecutionResult::Success {
reason: SuccessReason::Return,
gas_used: 1000, // Mock gas used
gas_refunded: 0,
logs: vec![],
output: Output::Call(Bytes::from(vec![])),
},
Default::default(),
))
}
fn commit_transaction(
&mut self,
_output: ResultAndState<<Self::Evm as Evm>::HaltReason>,
_tx: impl ExecutableTx<Self>,
) -> Result<u64, BlockExecutionError> {
Ok(1000)
}
fn finish(
self,
) -> Result<(Self::Evm, BlockExecutionResult<Self::Receipt>), BlockExecutionError> {
let Self { hook, state, .. } = self;
// Call hook with our mock state
if let Some(mut hook) = hook {
hook.on_state(StateChangeSource::Transaction(0), &state);
}
// Create a mock EVM
let db = State::builder()
.with_database(EmptyDB::default())
.with_bundle_update()
.without_state_clear()
.build();
let evm = EthEvm::new(
Context::mainnet().with_db(db).build_mainnet_with_inspector(NoOpInspector {}),
false,
);
// Return successful result like the original tests
Ok((
evm,
BlockExecutionResult {
receipts: vec![],
requests: Requests::default(),
gas_used: 1000,
blob_gas_used: 0,
},
))
}
fn set_state_hook(&mut self, hook: Option<Box<dyn OnStateHook>>) {
self.hook = hook;
}
fn evm_mut(&mut self) -> &mut Self::Evm {
panic!("Mock executor evm_mut() not implemented")
}
fn evm(&self) -> &Self::Evm {
panic!("Mock executor evm() not implemented")
}
fn receipts(&self) -> &[Self::Receipt] {
&self.receipts
}
}
struct ChannelStateHook {
output: i32,
sender: mpsc::Sender<i32>,
}
impl OnStateHook for ChannelStateHook {
fn on_state(&mut self, _source: StateChangeSource, _state: &EvmState) {
let _ = self.sender.send(self.output);
}
}
fn setup_test_recorder() -> Snapshotter {
let recorder = DebuggingRecorder::new();
@@ -369,7 +551,38 @@ mod tests {
}
#[test]
fn test_record_block_execution_metrics() {
fn test_executor_metrics_hook_called() {
let metrics = EngineApiMetrics::default();
let input = RecoveredBlock::<reth_ethereum_primitives::Block>::default();
let (tx, rx) = mpsc::channel();
let expected_output = 42;
let state_hook = Box::new(ChannelStateHook { sender: tx, output: expected_output });
let state = EvmState::default();
let executor = MockExecutor::new(state);
// This will fail to create the EVM but should still call the hook
let _result = metrics.execute_metered::<_, EmptyDB, _>(
executor,
input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>),
input.transaction_count(),
state_hook,
|_| {},
);
// Check if hook was called (it might not be if finish() fails early)
match rx.try_recv() {
Ok(actual_output) => assert_eq!(actual_output, expected_output),
Err(_) => {
// Hook wasn't called, which is expected if the mock fails early
// The test still validates that the code compiles and runs
}
}
}
#[test]
fn test_executor_metrics_hook_metrics_recorded() {
let snapshotter = setup_test_recorder();
let metrics = EngineApiMetrics::default();
@@ -378,17 +591,45 @@ mod tests {
metrics.executor.gas_per_second.set(0.0);
metrics.executor.gas_used_histogram.record(0.0);
let output = BlockExecutionOutput::<Receipt> {
state: BundleState::default(),
result: BlockExecutionResult {
receipts: vec![],
requests: Requests::default(),
gas_used: 21000,
blob_gas_used: 0,
},
let input = RecoveredBlock::<reth_ethereum_primitives::Block>::default();
let (tx, _rx) = mpsc::channel();
let state_hook = Box::new(ChannelStateHook { sender: tx, output: 42 });
// Create a state with some data
let state = {
let mut state = EvmState::default();
let storage =
EvmStorage::from_iter([(U256::from(1), EvmStorageSlot::new(U256::from(2), 0))]);
state.insert(
Default::default(),
Account {
info: AccountInfo {
balance: U256::from(100),
nonce: 10,
code_hash: B256::random(),
code: Default::default(),
account_id: None,
},
original_info: Box::new(AccountInfo::default()),
storage,
status: AccountStatus::default(),
transaction_id: 0,
},
);
state
};
metrics.record_block_execution(&output, Duration::from_millis(100));
let executor = MockExecutor::new(state);
// Execute (will fail but should still update some metrics)
let _result = metrics.execute_metered::<_, EmptyDB, _>(
executor,
input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>),
input.transaction_count(),
state_hook,
|_| {},
);
let snapshot = snapshotter.snapshot().into_vec();

View File

@@ -19,7 +19,6 @@ use alloy_evm::{block::StateChangeSource, ToTxEnv};
use alloy_primitives::B256;
use crossbeam_channel::Sender as CrossbeamSender;
use executor::WorkloadExecutor;
use metrics::Counter;
use multiproof::{SparseTrieUpdate, *};
use parking_lot::RwLock;
use prewarm::PrewarmMetrics;
@@ -29,7 +28,6 @@ use reth_evm::{
ConfigureEvm, EvmEnvFor, ExecutableTxIterator, ExecutableTxTuple, OnStateHook, SpecFor,
TxEnvFor,
};
use reth_metrics::Metrics;
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
BlockExecutionOutput, BlockReader, DatabaseProviderROFactory, StateProvider,
@@ -141,8 +139,6 @@ where
disable_parallel_sparse_trie: bool,
/// Maximum concurrency for prewarm task.
prewarm_max_concurrency: usize,
/// Whether to disable cache metrics recording.
disable_cache_metrics: bool,
}
impl<N, Evm> PayloadProcessor<Evm>
@@ -175,7 +171,6 @@ where
sparse_state_trie: Arc::default(),
disable_parallel_sparse_trie: config.disable_parallel_sparse_trie(),
prewarm_max_concurrency: config.prewarm_max_concurrency(),
disable_cache_metrics: config.disable_cache_metrics(),
}
}
}
@@ -305,7 +300,7 @@ where
// Build a state provider for the multiproof task
let provider = provider_builder.build().expect("failed to build provider");
let provider = if let Some(saved_cache) = saved_cache {
let (cache, metrics, _) = saved_cache.split();
let (cache, metrics) = saved_cache.split();
Box::new(CachedStateProvider::new(provider, cache, metrics))
as Box<dyn StateProvider>
} else {
@@ -482,7 +477,6 @@ where
debug!("creating new execution cache on cache miss");
let cache = ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size);
SavedCache::new(parent_hash, cache, CachedStateMetrics::zeroed())
.with_disable_cache_metrics(self.disable_cache_metrics)
}
}
@@ -564,7 +558,6 @@ where
block_with_parent: BlockWithParent,
bundle_state: &BundleState,
) {
let disable_cache_metrics = self.disable_cache_metrics;
self.execution_cache.update_with_guard(|cached| {
if cached.as_ref().is_some_and(|c| c.executed_block_hash() != block_with_parent.parent) {
debug!(
@@ -578,8 +571,7 @@ where
// Take existing cache (if any) or create fresh caches
let (caches, cache_metrics) = match cached.take() {
Some(existing) => {
let (c, m, _) = existing.split();
(c, m)
existing.split()
}
None => (
ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size),
@@ -588,8 +580,7 @@ where
};
// Insert the block's bundle state into cache
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics)
.with_disable_cache_metrics(disable_cache_metrics);
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics);
if new_cache.cache().insert_state(bundle_state).is_err() {
*cached = None;
debug!(target: "engine::caching", "cleared execution cache on update error");
@@ -779,8 +770,6 @@ impl<R> Drop for CacheTaskHandle<R> {
struct ExecutionCache {
/// Guarded cloneable cache identified by a block hash.
inner: Arc<RwLock<Option<SavedCache>>>,
/// Metrics for cache operations.
metrics: ExecutionCacheMetrics,
}
impl ExecutionCache {
@@ -822,10 +811,6 @@ impl ExecutionCache {
if hash_matches && available {
return Some(c.clone());
}
if hash_matches && !available {
self.metrics.execution_cache_in_use.increment(1);
}
} else {
debug!(target: "engine::caching", %parent_hash, "No cache found");
}
@@ -861,15 +846,6 @@ impl ExecutionCache {
}
}
/// Metrics for execution cache operations.
#[derive(Metrics, Clone)]
#[metrics(scope = "consensus.engine.beacon")]
pub(crate) struct ExecutionCacheMetrics {
/// Counter for when the execution cache was unavailable because other threads
/// (e.g., prewarming) are still using it.
pub(crate) execution_cache_in_use: Counter,
}
/// EVM context required to execute a block.
#[derive(Debug, Clone)]
pub struct ExecutionEnv<Evm: ConfigureEvm> {

View File

@@ -278,9 +278,8 @@ where
execution_cache.update_with_guard(|cached| {
// consumes the `SavedCache` held by the prewarming task, which releases its usage
// guard
let (caches, cache_metrics, disable_cache_metrics) = saved_cache.split();
let new_cache = SavedCache::new(hash, caches, cache_metrics)
.with_disable_cache_metrics(disable_cache_metrics);
let (caches, cache_metrics) = saved_cache.split();
let new_cache = SavedCache::new(hash, caches, cache_metrics);
// Insert state into cache while holding the lock
// Access the BundleState through the shared ExecutionOutcome
@@ -297,11 +296,6 @@ where
// Replace the shared cache with the new one; the previous cache (if any) is
// dropped.
*cached = Some(new_cache);
} else {
// Block was invalid; caches were already mutated by insert_state above,
// so we must clear to prevent using polluted state
*cached = None;
debug!(target: "engine::caching", "cleared execution cache on invalid block");
}
});

View File

@@ -64,11 +64,14 @@ impl<R: Receipt> ReceiptRootTaskHandle<R> {
///
/// * `receipts_len` - The total number of receipts expected. This is needed to correctly order
/// the trie keys according to RLP encoding rules.
///
/// # Panics
///
/// Panics if the number of receipts received doesn't match `receipts_len`.
pub fn run(self, receipts_len: usize) {
let mut builder = OrderedTrieRootEncodedBuilder::new(receipts_len);
let mut aggregated_bloom = Bloom::ZERO;
let mut encode_buf = Vec::new();
let mut received_count = 0usize;
for indexed_receipt in self.receipt_rx {
let receipt_with_bloom = indexed_receipt.receipt.with_bloom_ref();
@@ -78,21 +81,9 @@ impl<R: Receipt> ReceiptRootTaskHandle<R> {
aggregated_bloom |= *receipt_with_bloom.bloom_ref();
builder.push_unchecked(indexed_receipt.index, &encode_buf);
received_count += 1;
}
let Ok(root) = builder.finalize() else {
// Finalize fails if we didn't receive exactly `receipts_len` receipts. This can
// happen if execution was aborted early (e.g., invalid transaction encountered).
// We return without sending a result, allowing the caller to handle the abort.
tracing::error!(
target: "engine::tree::payload_processor",
expected = receipts_len,
received = received_count,
"Receipt root task received incomplete receipts, execution likely aborted"
);
return;
};
let root = builder.finalize().expect("receipt root builder incomplete");
let _ = self.result_tx.send((root, aggregated_bloom));
}
}

View File

@@ -7,10 +7,10 @@ use crate::tree::{
payload_processor::{executor::WorkloadExecutor, PayloadProcessor},
precompile_cache::{CachedPrecompile, CachedPrecompileMetrics, PrecompileCacheMap},
sparse_trie::StateRootComputeOutcome,
EngineApiMetrics, EngineApiTreeState, ExecutionEnv, MeteredStateHook, PayloadHandle,
StateProviderBuilder, StateProviderDatabase, TreeConfig,
EngineApiMetrics, EngineApiTreeState, ExecutionEnv, PayloadHandle, StateProviderBuilder,
StateProviderDatabase, TreeConfig,
};
use alloy_consensus::transaction::{Either, TxHashRef};
use alloy_consensus::transaction::Either;
use alloy_eip7928::BlockAccessList;
use alloy_eips::{eip1898::BlockWithParent, NumHash};
use alloy_evm::Evm;
@@ -41,7 +41,7 @@ use reth_provider::{
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
StateProviderFactory, StateReader,
};
use reth_revm::db::{states::bundle_state::BundleRetention, State};
use reth_revm::db::State;
use reth_trie::{updates::TrieUpdates, HashedPostState, StateRoot};
use reth_trie_db::ChangesetCache;
use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError};
@@ -479,15 +479,11 @@ where
let block = self.convert_to_block(input)?.with_senders(senders);
// Wait for the receipt root computation to complete.
let receipt_root_bloom = receipt_root_rx
.blocking_recv()
.inspect_err(|_| {
tracing::error!(
target: "engine::tree::payload_validator",
"Receipt root task dropped sender without result, receipt root calculation likely aborted"
);
})
.ok();
let receipt_root_bloom = Some(
receipt_root_rx
.blocking_recv()
.expect("receipt root task dropped sender without result"),
);
let hashed_state = ensure_ok_post_block!(
self.validate_post_execution(
@@ -642,13 +638,7 @@ where
Ok(())
}
/// Executes a block with the given state provider.
///
/// This method orchestrates block execution:
/// 1. Sets up the EVM with state database and precompile caching
/// 2. Spawns a background task for incremental receipt root computation
/// 3. Executes transactions with metrics collection via state hooks
/// 4. Merges state transitions and records execution metrics
/// Executes a block with the given state provider
#[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)]
#[expect(clippy::type_complexity)]
fn execute_block<S, Err, T>(
@@ -711,117 +701,31 @@ where
let task_handle = ReceiptRootTaskHandle::new(receipt_rx, result_tx);
self.payload_processor.executor().spawn_blocking(move || task_handle.run(receipts_len));
// Wrap the state hook with metrics collection
let inner_hook = Box::new(handle.state_hook());
let state_hook =
MeteredStateHook { metrics: self.metrics.executor_metrics().clone(), inner_hook };
let transaction_count = input.transaction_count();
let executor = executor.with_state_hook(Some(Box::new(state_hook)));
let execution_start = Instant::now();
// Execute all transactions and finalize
let (executor, senders) = self.execute_transactions(
let state_hook = Box::new(handle.state_hook());
let (output, senders) = self.metrics.execute_metered(
executor,
transaction_count,
handle.iter_transactions(),
&receipt_tx,
handle.iter_transactions().map(|res| res.map_err(BlockExecutionError::other)),
input.transaction_count(),
state_hook,
|receipts| {
// Send the latest receipt to the background task for incremental root computation.
// The receipt is cloned here; encoding happens in the background thread.
if let Some(receipt) = receipts.last() {
// Infer tx_index from the number of receipts collected so far
let tx_index = receipts.len() - 1;
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
}
},
)?;
drop(receipt_tx);
// Finish execution and get the result
let post_exec_start = Instant::now();
let (_evm, result) = debug_span!(target: "engine::tree", "finish")
.in_scope(|| executor.finish())
.map(|(evm, result)| (evm.into_db(), result))?;
self.metrics.record_post_execution(post_exec_start.elapsed());
// Merge transitions into bundle state
debug_span!(target: "engine::tree", "merge transitions")
.in_scope(|| db.merge_transitions(BundleRetention::Reverts));
let output = BlockExecutionOutput { result, state: db.take_bundle() };
let execution_duration = execution_start.elapsed();
self.metrics.record_block_execution(&output, execution_duration);
debug!(target: "engine::tree::payload_validator", elapsed = ?execution_duration, "Executed block");
let execution_finish = Instant::now();
let execution_time = execution_finish.duration_since(execution_start);
debug!(target: "engine::tree::payload_validator", elapsed = ?execution_time, "Executed block");
Ok((output, senders, result_rx))
}
/// Executes transactions and collects senders, streaming receipts to a background task.
///
/// This method handles:
/// - Applying pre-execution changes (e.g., beacon root updates)
/// - Executing each transaction with timing metrics
/// - Streaming receipts to the receipt root computation task
/// - Collecting transaction senders for later use
///
/// Returns the executor (for finalization) and the collected senders.
fn execute_transactions<E, Tx, InnerTx, Err>(
&self,
mut executor: E,
transaction_count: usize,
transactions: impl Iterator<Item = Result<Tx, Err>>,
receipt_tx: &crossbeam_channel::Sender<IndexedReceipt<N::Receipt>>,
) -> Result<(E, Vec<Address>), BlockExecutionError>
where
E: BlockExecutor<Receipt = N::Receipt>,
Tx: alloy_evm::block::ExecutableTx<E> + alloy_evm::RecoveredTx<InnerTx>,
InnerTx: TxHashRef,
Err: core::error::Error + Send + Sync + 'static,
{
let mut senders = Vec::with_capacity(transaction_count);
// Apply pre-execution changes (e.g., beacon root update)
let pre_exec_start = Instant::now();
debug_span!(target: "engine::tree", "pre execution")
.in_scope(|| executor.apply_pre_execution_changes())?;
self.metrics.record_pre_execution(pre_exec_start.elapsed());
// Execute transactions
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
let mut transactions = transactions.into_iter();
loop {
// Measure time spent waiting for next transaction from iterator
// (e.g., parallel signature recovery)
let wait_start = Instant::now();
let Some(tx_result) = transactions.next() else { break };
self.metrics.record_transaction_wait(wait_start.elapsed());
let tx = tx_result.map_err(BlockExecutionError::other)?;
let tx_signer = *<Tx as alloy_evm::RecoveredTx<InnerTx>>::signer(&tx);
let tx_hash = <Tx as alloy_evm::RecoveredTx<InnerTx>>::tx(&tx).tx_hash();
senders.push(tx_signer);
let span = debug_span!(
target: "engine::tree",
"execute tx",
?tx_hash,
gas_used = tracing::field::Empty,
);
let enter = span.entered();
trace!(target: "engine::tree", "Executing transaction");
let tx_start = Instant::now();
let gas_used = executor.execute_transaction(tx)?;
self.metrics.record_transaction_execution(tx_start.elapsed());
// Send the latest receipt to the background task for incremental root computation
if let Some(receipt) = executor.receipts().last() {
let tx_index = executor.receipts().len() - 1;
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
}
enter.record("gas_used", gas_used);
}
drop(exec_span);
Ok((executor, senders))
}
/// Compute state root for the given hashed post state in parallel.
///
/// Uses an overlay factory which provides the state of the parent block, along with the

View File

@@ -153,7 +153,7 @@ async fn maintain_txpool_reorg() -> eyre::Result<()> {
w1.address(),
);
let pooled_tx1 = EthPooledTransaction::new(tx1.clone(), 200);
let tx_hash1 = *pooled_tx1.hash();
let tx_hash1 = *pooled_tx1.clone().hash();
// build tx2 from wallet2
let envelop2 = TransactionTestContext::transfer_tx(1, w2.clone()).await;
@@ -162,7 +162,7 @@ async fn maintain_txpool_reorg() -> eyre::Result<()> {
w2.address(),
);
let pooled_tx2 = EthPooledTransaction::new(tx2.clone(), 200);
let tx_hash2 = *pooled_tx2.hash();
let tx_hash2 = *pooled_tx2.clone().hash();
let block_info = BlockInfo {
block_gas_limit: ETHEREUM_BLOCK_GAS_LIMIT_30M,

View File

@@ -236,7 +236,7 @@ impl reth_codecs::Compact for Transaction {
// # Panics
//
// A panic will be triggered if an identifier larger than 3 is passed from the database. For
// optimism an identifier with value [`DEPOSIT_TX_TYPE_ID`] is allowed.
// optimism a identifier with value [`DEPOSIT_TX_TYPE_ID`] is allowed.
fn from_compact(buf: &[u8], identifier: usize) -> (Self, &[u8]) {
let (tx_type, buf) = TxType::from_compact(buf, identifier);

View File

@@ -148,6 +148,20 @@ pub trait Executor<DB: Database>: Sized {
fn size_hint(&self) -> usize;
}
/// Helper type for the output of executing a block.
///
/// Deprecated: this type is unused within reth and will be removed in the next
/// major release. Use `reth_execution_types::BlockExecutionResult` or
/// `reth_execution_types::BlockExecutionOutput`.
#[deprecated(note = "Use reth_execution_types::BlockExecutionResult or BlockExecutionOutput")]
#[derive(Debug, Clone)]
pub struct ExecuteOutput<R> {
/// Receipts obtained after executing a block.
pub receipts: Vec<R>,
/// Cumulative gas used in the block execution.
pub gas_used: u64,
}
/// Input for block building. Consumed by [`BlockAssembler`].
///
/// This struct contains all the data needed by the [`BlockAssembler`] to create

View File

@@ -2,9 +2,9 @@
use crate::ExecutionOutcome;
use alloc::{borrow::Cow, collections::BTreeMap, vec::Vec};
use alloy_consensus::{transaction::Recovered, BlockHeader, TxReceipt};
use alloy_consensus::{transaction::Recovered, BlockHeader};
use alloy_eips::{eip1898::ForkBlock, eip2718::Encodable2718, BlockNumHash};
use alloy_primitives::{Address, BlockHash, BlockNumber, Log, TxHash};
use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash};
use core::{fmt, ops::RangeInclusive};
use reth_primitives_traits::{
transaction::signed::SignedTransaction, Block, BlockBody, IndexedTx, NodePrimitives,
@@ -184,19 +184,6 @@ impl<N: NodePrimitives> Chain<N> {
self.execution_outcome.receipts().iter()
}
/// Returns an iterator over all receipts in the chain.
pub fn receipts_iter(&self) -> impl Iterator<Item = &N::Receipt> + '_ {
self.block_receipts_iter().flatten()
}
/// Returns an iterator over all logs in the chain.
pub fn logs_iter(&self) -> impl Iterator<Item = &Log> + '_
where
N::Receipt: TxReceipt<Log = Log>,
{
self.receipts_iter().flat_map(|receipt| receipt.logs())
}
/// Returns an iterator over all blocks in the chain with increasing block number.
pub fn blocks_iter(&self) -> impl Iterator<Item = &RecoveredBlock<N::Block>> + '_ {
self.blocks().iter().map(|block| block.1)

View File

@@ -106,7 +106,7 @@ impl BanList {
self.banned_ips.contains_key(ip)
}
/// checks the ban list to see if it contains the given peer
/// checks the ban list to see if it contains the given ip
#[inline]
pub fn is_banned_peer(&self, peer_id: &PeerId) -> bool {
self.banned_peers.contains_key(peer_id)
@@ -117,7 +117,7 @@ impl BanList {
self.banned_ips.remove(ip);
}
/// Unbans the peer
/// Unbans the ip address
pub fn unban_peer(&mut self, peer_id: &PeerId) {
self.banned_peers.remove(peer_id);
}

View File

@@ -676,13 +676,19 @@ where
/// Convenience function to [`Self::init_genesis`]
pub fn with_genesis(self) -> Result<Self, InitStorageError> {
init_genesis_with_settings(self.provider_factory(), self.node_config().storage_settings())?;
init_genesis_with_settings(
self.provider_factory(),
self.node_config().static_files.to_settings(),
)?;
Ok(self)
}
/// Write the genesis block and state if it has not already been written
pub fn init_genesis(&self) -> Result<B256, InitStorageError> {
init_genesis_with_settings(self.provider_factory(), self.node_config().storage_settings())
init_genesis_with_settings(
self.provider_factory(),
self.node_config().static_files.to_settings(),
)
}
/// Creates a new `WithMeteredProvider` container and attaches it to the
@@ -1277,10 +1283,6 @@ pub fn metrics_hooks<N: NodeTypesWithDB>(provider_factory: &ProviderFactory<N>)
})
}
})
.with_hook({
let rocksdb = provider_factory.rocksdb_provider();
move || throttle!(Duration::from_secs(5 * 60), || rocksdb.report_metrics())
})
.build()
}

View File

@@ -32,7 +32,7 @@ use reth_node_core::{
use reth_node_events::node;
use reth_provider::{
providers::{BlockchainProvider, NodeTypesForProvider},
BlockNumReader, StorageSettingsCache,
BlockNumReader, MetadataProvider,
};
use reth_tasks::TaskExecutor;
use reth_tokio_util::EventSender;
@@ -41,6 +41,7 @@ use reth_trie_db::ChangesetCache;
use std::{future::Future, pin::Pin, sync::Arc};
use tokio::sync::{mpsc::unbounded_channel, oneshot};
use tokio_stream::wrappers::UnboundedReceiverStream;
use tracing::warn;
/// The engine node launcher.
#[derive(Debug)]
@@ -103,8 +104,24 @@ impl EngineNodeLauncher {
.with_adjusted_configs()
// Create the provider factory with changeset cache
.with_provider_factory::<_, <CB::Components as NodeComponents<T>>::Evm>(changeset_cache.clone()).await?
.inspect(|_| {
.inspect(|ctx| {
info!(target: "reth::cli", "Database opened");
match ctx.provider_factory().storage_settings() {
Ok(settings) => {
info!(
target: "reth::cli",
?settings,
"Storage settings"
);
},
Err(err) => {
warn!(
target: "reth::cli",
?err,
"Failed to get storage settings"
);
},
}
})
.with_prometheus_server().await?
.inspect(|this| {
@@ -113,8 +130,6 @@ impl EngineNodeLauncher {
.with_genesis()?
.inspect(|this: &LaunchContextWith<Attached<WithConfigs<<T::Types as NodeTypes>::ChainSpec>, _>>| {
info!(target: "reth::cli", "\n{}", this.chain_spec().display_hardforks());
let settings = this.provider_factory().cached_storage_settings();
info!(target: "reth::cli", ?settings, "Loaded storage settings");
})
.with_metrics_task()
// passing FullNodeTypes as type parameter here so that we can build

View File

@@ -19,6 +19,7 @@ reth-cli-util.workspace = true
reth-db = { workspace = true, features = ["mdbx"] }
reth-storage-errors.workspace = true
reth-storage-api = { workspace = true, features = ["std", "db-api"] }
reth-provider.workspace = true
reth-network = { workspace = true, features = ["serde"] }
reth-network-p2p.workspace = true
reth-rpc-eth-types.workspace = true
@@ -91,7 +92,7 @@ min-debug-logs = ["tracing/release_max_level_debug"]
min-trace-logs = ["tracing/release_max_level_trace"]
# Marker feature for edge/unstable builds - captured by vergen in build.rs
edge = ["reth-storage-api/edge"]
edge = []
[build-dependencies]
vergen = { workspace = true, features = ["build", "cargo", "emit_and_set"] }

View File

@@ -37,7 +37,6 @@ pub struct DefaultEngineValues {
storage_worker_count: Option<usize>,
account_worker_count: Option<usize>,
enable_proof_v2: bool,
cache_metrics_disabled: bool,
}
impl DefaultEngineValues {
@@ -173,12 +172,6 @@ impl DefaultEngineValues {
self.enable_proof_v2 = v;
self
}
/// Set whether to disable cache metrics by default
pub const fn with_cache_metrics_disabled(mut self, v: bool) -> Self {
self.cache_metrics_disabled = v;
self
}
}
impl Default for DefaultEngineValues {
@@ -204,7 +197,6 @@ impl Default for DefaultEngineValues {
storage_worker_count: None,
account_worker_count: None,
enable_proof_v2: false,
cache_metrics_disabled: false,
}
}
}
@@ -328,10 +320,6 @@ pub struct EngineArgs {
/// Enable V2 storage proofs for state root calculations
#[arg(long = "engine.enable-proof-v2", default_value_t = DefaultEngineValues::get_global().enable_proof_v2)]
pub enable_proof_v2: bool,
/// Disable cache metrics recording, which can take up to 50ms with large cached state.
#[arg(long = "engine.disable-cache-metrics", default_value_t = DefaultEngineValues::get_global().cache_metrics_disabled)]
pub cache_metrics_disabled: bool,
}
#[allow(deprecated)]
@@ -358,7 +346,6 @@ impl Default for EngineArgs {
storage_worker_count,
account_worker_count,
enable_proof_v2,
cache_metrics_disabled,
} = DefaultEngineValues::get_global().clone();
Self {
persistence_threshold,
@@ -384,7 +371,6 @@ impl Default for EngineArgs {
storage_worker_count,
account_worker_count,
enable_proof_v2,
cache_metrics_disabled,
}
}
}
@@ -421,7 +407,6 @@ impl EngineArgs {
}
config = config.with_enable_proof_v2(self.enable_proof_v2);
config = config.without_cache_metrics(self.cache_metrics_disabled);
config
}
@@ -473,7 +458,6 @@ mod tests {
storage_worker_count: Some(16),
account_worker_count: Some(8),
enable_proof_v2: false,
cache_metrics_disabled: true,
};
let parsed_args = CommandParser::<EngineArgs>::parse_from([
@@ -504,7 +488,6 @@ mod tests {
"16",
"--engine.account-worker-count",
"8",
"--engine.disable-cache-metrics",
])
.args;

View File

@@ -54,7 +54,7 @@ pub use dev::DevArgs;
/// PruneArgs for configuring the pruning and full node
mod pruning;
pub use pruning::{DefaultPruningValues, PruningArgs};
pub use pruning::PruningArgs;
/// DatadirArgs for configuring data storage paths
mod datadir_args;
@@ -80,9 +80,5 @@ pub use era::{DefaultEraHost, EraArgs, EraSourceArgs};
mod static_files;
pub use static_files::{StaticFilesArgs, MINIMAL_BLOCKS_PER_FILE};
/// `RocksDbArgs` for configuring RocksDB table routing.
mod rocksdb;
pub use rocksdb::{RocksDbArgs, RocksDbArgsError};
mod error;
pub mod types;

View File

@@ -6,88 +6,7 @@ use clap::{builder::RangedU64ValueParser, Args};
use reth_chainspec::EthereumHardforks;
use reth_config::config::PruneConfig;
use reth_prune_types::{PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_PRUNING_DISTANCE};
use std::{collections::BTreeMap, ops::Not, sync::OnceLock};
/// Global static pruning defaults
static PRUNING_DEFAULTS: OnceLock<DefaultPruningValues> = OnceLock::new();
/// Default values for `--full` and `--minimal` pruning modes that can be customized.
///
/// Global defaults can be set via [`DefaultPruningValues::try_init`].
#[derive(Debug, Clone)]
pub struct DefaultPruningValues {
/// Prune modes for `--full` flag.
///
/// Note: `bodies_history` is ignored when `full_bodies_history_use_pre_merge` is `true`.
pub full_prune_modes: PruneModes,
/// If `true`, `--full` will set `bodies_history` to prune everything before the merge block
/// (Paris hardfork). If `false`, uses `full_prune_modes.bodies_history` directly.
pub full_bodies_history_use_pre_merge: bool,
/// Prune modes for `--minimal` flag.
pub minimal_prune_modes: PruneModes,
}
impl DefaultPruningValues {
/// Initialize the global pruning defaults with this configuration.
///
/// Returns `Err(self)` if already initialized.
pub fn try_init(self) -> Result<(), Self> {
PRUNING_DEFAULTS.set(self)
}
/// Get a reference to the global pruning defaults.
pub fn get_global() -> &'static Self {
PRUNING_DEFAULTS.get_or_init(Self::default)
}
/// Set the prune modes for `--full` flag.
pub fn with_full_prune_modes(mut self, modes: PruneModes) -> Self {
self.full_prune_modes = modes;
self
}
/// Set whether `--full` should use pre-merge pruning for bodies history.
///
/// When `true` (default), bodies are pruned before the Paris hardfork block.
/// When `false`, uses `full_prune_modes.bodies_history` directly.
pub const fn with_full_bodies_history_use_pre_merge(mut self, use_pre_merge: bool) -> Self {
self.full_bodies_history_use_pre_merge = use_pre_merge;
self
}
/// Set the prune modes for `--minimal` flag.
pub fn with_minimal_prune_modes(mut self, modes: PruneModes) -> Self {
self.minimal_prune_modes = modes;
self
}
}
impl Default for DefaultPruningValues {
fn default() -> Self {
Self {
full_prune_modes: PruneModes {
sender_recovery: Some(PruneMode::Full),
transaction_lookup: None,
receipts: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
account_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
storage_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
// This field is ignored when full_bodies_history_use_pre_merge is true
bodies_history: None,
receipts_log_filter: Default::default(),
},
full_bodies_history_use_pre_merge: true,
minimal_prune_modes: PruneModes {
sender_recovery: Some(PruneMode::Full),
transaction_lookup: Some(PruneMode::Full),
receipts: Some(PruneMode::Full),
account_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
storage_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
bodies_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
receipts_log_filter: Default::default(),
},
}
}
}
use std::{collections::BTreeMap, ops::Not};
/// Parameters for pruning and full node
#[derive(Debug, Clone, Args, PartialEq, Eq, Default)]
@@ -209,22 +128,36 @@ impl PruningArgs {
// If --full is set, use full node defaults.
if self.full {
let defaults = DefaultPruningValues::get_global();
let mut segments = defaults.full_prune_modes.clone();
if defaults.full_bodies_history_use_pre_merge {
segments.bodies_history = chain_spec
.ethereum_fork_activation(EthereumHardfork::Paris)
.block_number()
.map(PruneMode::Before);
config = PruneConfig {
block_interval: config.block_interval,
segments: PruneModes {
sender_recovery: Some(PruneMode::Full),
transaction_lookup: None,
receipts: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
account_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
storage_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
bodies_history: chain_spec
.ethereum_fork_activation(EthereumHardfork::Paris)
.block_number()
.map(PruneMode::Before),
receipts_log_filter: Default::default(),
},
}
config = PruneConfig { block_interval: config.block_interval, segments }
}
// If --minimal is set, use minimal storage mode with aggressive pruning.
if self.minimal {
config = PruneConfig {
block_interval: config.block_interval,
segments: DefaultPruningValues::get_global().minimal_prune_modes.clone(),
segments: PruneModes {
sender_recovery: Some(PruneMode::Full),
transaction_lookup: Some(PruneMode::Full),
receipts: Some(PruneMode::Full),
account_history: Some(PruneMode::Distance(10064)),
storage_history: Some(PruneMode::Distance(10064)),
bodies_history: Some(PruneMode::Distance(10064)),
receipts_log_filter: Default::default(),
},
}
}

View File

@@ -1,160 +0,0 @@
//! clap [Args](clap::Args) for `RocksDB` table routing configuration
use clap::{ArgAction, Args};
/// Default value for `RocksDB` routing flags.
///
/// When the `edge` feature is enabled, defaults to `true` to enable edge storage features.
/// Otherwise defaults to `false` for legacy behavior.
const fn default_rocksdb_flag() -> bool {
cfg!(feature = "edge")
}
/// Parameters for `RocksDB` table routing configuration.
///
/// These flags control which database tables are stored in `RocksDB` instead of MDBX.
/// All flags are genesis-initialization-only: changing them after genesis requires a re-sync.
#[derive(Debug, Args, PartialEq, Eq, Clone, Copy)]
#[command(next_help_heading = "RocksDB")]
pub struct RocksDbArgs {
/// Route all supported tables to `RocksDB` instead of MDBX.
///
/// This enables `RocksDB` for `tx-hash`, `storages-history`, and `account-history` tables.
/// Cannot be combined with individual flags set to false.
#[arg(long = "rocksdb.all", action = ArgAction::SetTrue)]
pub all: bool,
/// Route tx hash -> number table to `RocksDB` instead of MDBX.
///
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
/// Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
#[arg(long = "rocksdb.tx-hash", default_value_t = default_rocksdb_flag(), action = ArgAction::Set)]
pub tx_hash: bool,
/// Route storages history tables to `RocksDB` instead of MDBX.
///
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
/// Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
#[arg(long = "rocksdb.storages-history", default_value_t = default_rocksdb_flag(), action = ArgAction::Set)]
pub storages_history: bool,
/// Route account history tables to `RocksDB` instead of MDBX.
///
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
/// Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
#[arg(long = "rocksdb.account-history", default_value_t = default_rocksdb_flag(), action = ArgAction::Set)]
pub account_history: bool,
}
impl Default for RocksDbArgs {
fn default() -> Self {
Self {
all: false,
tx_hash: default_rocksdb_flag(),
storages_history: default_rocksdb_flag(),
account_history: default_rocksdb_flag(),
}
}
}
impl RocksDbArgs {
/// Validates the `RocksDB` arguments.
///
/// Returns an error if `--rocksdb.all` is used with any individual flag set to `false`.
pub const fn validate(&self) -> Result<(), RocksDbArgsError> {
if self.all {
if !self.tx_hash {
return Err(RocksDbArgsError::ConflictingFlags("tx-hash"));
}
if !self.storages_history {
return Err(RocksDbArgsError::ConflictingFlags("storages-history"));
}
if !self.account_history {
return Err(RocksDbArgsError::ConflictingFlags("account-history"));
}
}
Ok(())
}
}
/// Error type for `RocksDB` argument validation.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum RocksDbArgsError {
/// `--rocksdb.all` cannot be combined with an individual flag set to false.
#[error("--rocksdb.all cannot be combined with --rocksdb.{0}=false")]
ConflictingFlags(&'static str),
}
#[cfg(test)]
mod tests {
use super::*;
use clap::Parser;
#[derive(Parser)]
struct CommandParser<T: Args> {
#[command(flatten)]
args: T,
}
#[test]
fn test_default_rocksdb_args() {
let args = CommandParser::<RocksDbArgs>::parse_from(["reth"]).args;
assert_eq!(args, RocksDbArgs::default());
}
#[test]
fn test_parse_all_flag() {
let args = CommandParser::<RocksDbArgs>::parse_from(["reth", "--rocksdb.all"]).args;
assert!(args.all);
assert_eq!(args.tx_hash, default_rocksdb_flag());
}
#[test]
fn test_parse_individual_flags() {
let args = CommandParser::<RocksDbArgs>::parse_from([
"reth",
"--rocksdb.tx-hash=true",
"--rocksdb.storages-history=false",
"--rocksdb.account-history=true",
])
.args;
assert!(!args.all);
assert!(args.tx_hash);
assert!(!args.storages_history);
assert!(args.account_history);
}
#[test]
fn test_validate_all_with_true_ok() {
let args =
RocksDbArgs { all: true, tx_hash: true, storages_history: true, account_history: true };
assert!(args.validate().is_ok());
}
#[test]
fn test_validate_all_with_false_errors() {
let args = RocksDbArgs {
all: true,
tx_hash: false,
storages_history: true,
account_history: true,
};
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("tx-hash")));
let args = RocksDbArgs {
all: true,
tx_hash: true,
storages_history: false,
account_history: true,
};
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("storages-history")));
let args = RocksDbArgs {
all: true,
tx_hash: true,
storages_history: true,
account_history: false,
};
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("account-history")));
}
}

View File

@@ -2,22 +2,15 @@
use clap::Args;
use reth_config::config::{BlocksPerFileConfig, StaticFilesConfig};
use reth_provider::StorageSettings;
/// Blocks per static file when running in `--minimal` node.
///
/// 10000 blocks per static file allows us to prune all history every 10k blocks.
pub const MINIMAL_BLOCKS_PER_FILE: u64 = 10000;
/// Default value for static file storage flags.
///
/// When the `edge` feature is enabled, defaults to `true` to enable edge storage features.
/// Otherwise defaults to `false` for legacy behavior.
const fn default_static_file_flag() -> bool {
cfg!(feature = "edge")
}
/// Parameters for static files configuration
#[derive(Debug, Args, PartialEq, Eq, Clone, Copy)]
#[derive(Debug, Args, PartialEq, Eq, Default, Clone, Copy)]
#[command(next_help_heading = "Static Files")]
pub struct StaticFilesArgs {
/// Number of blocks per file for the headers segment.
@@ -46,7 +39,7 @@ pub struct StaticFilesArgs {
///
/// Note: This setting can only be configured at genesis initialization. Once
/// the node has been initialized, changing this flag requires re-syncing from scratch.
#[arg(long = "static-files.receipts", default_value_t = default_static_file_flag(), action = clap::ArgAction::Set)]
#[arg(long = "static-files.receipts")]
pub receipts: bool,
/// Store transaction senders in static files instead of the database.
@@ -56,7 +49,7 @@ pub struct StaticFilesArgs {
///
/// Note: This setting can only be configured at genesis initialization. Once
/// the node has been initialized, changing this flag requires re-syncing from scratch.
#[arg(long = "static-files.transaction-senders", default_value_t = default_static_file_flag(), action = clap::ArgAction::Set)]
#[arg(long = "static-files.transaction-senders")]
pub transaction_senders: bool,
/// Store account changesets in static files.
@@ -66,7 +59,7 @@ pub struct StaticFilesArgs {
///
/// Note: This setting can only be configured at genesis initialization. Once
/// the node has been initialized, changing this flag requires re-syncing from scratch.
#[arg(long = "static-files.account-change-sets", default_value_t = default_static_file_flag(), action = clap::ArgAction::Set)]
#[arg(long = "static-files.account-change-sets")]
pub account_changesets: bool,
}
@@ -101,19 +94,12 @@ impl StaticFilesArgs {
},
}
}
}
impl Default for StaticFilesArgs {
fn default() -> Self {
Self {
blocks_per_file_headers: None,
blocks_per_file_transactions: None,
blocks_per_file_receipts: None,
blocks_per_file_transaction_senders: None,
blocks_per_file_account_change_sets: None,
receipts: default_static_file_flag(),
transaction_senders: default_static_file_flag(),
account_changesets: default_static_file_flag(),
}
/// Converts the static files arguments into [`StorageSettings`].
pub const fn to_settings(&self) -> StorageSettings {
StorageSettings::legacy()
.with_receipts_in_static_files(self.receipts)
.with_transaction_senders_in_static_files(self.transaction_senders)
.with_account_changesets_in_static_files(self.account_changesets)
}
}

View File

@@ -3,7 +3,7 @@
use crate::{
args::{
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, NetworkArgs, PayloadBuilderArgs,
PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
PruningArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
},
dirs::{ChainPath, DataDirPath},
utils::get_single_header,
@@ -21,7 +21,6 @@ use reth_primitives_traits::SealedHeader;
use reth_stages_types::StageId;
use reth_storage_api::{
BlockHashReader, DatabaseProviderFactory, HeaderProvider, StageCheckpointReader,
StorageSettings,
};
use reth_storage_errors::provider::ProviderResult;
use reth_transaction_pool::TransactionPool;
@@ -151,9 +150,6 @@ pub struct NodeConfig<ChainSpec> {
/// All static files related arguments
pub static_files: StaticFilesArgs,
/// All `RocksDB` table routing arguments
pub rocksdb: RocksDbArgs,
}
impl NodeConfig<ChainSpec> {
@@ -185,7 +181,6 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
engine: EngineArgs::default(),
era: EraArgs::default(),
static_files: StaticFilesArgs::default(),
rocksdb: RocksDbArgs::default(),
}
}
@@ -260,7 +255,6 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
engine,
era,
static_files,
rocksdb,
..
} = self;
NodeConfig {
@@ -280,7 +274,6 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
engine,
era,
static_files,
rocksdb,
}
}
@@ -357,17 +350,6 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
self.pruning.prune_config(&self.chain)
}
/// Returns the effective storage settings derived from static-file and `RocksDB` CLI args.
pub const fn storage_settings(&self) -> StorageSettings {
StorageSettings::base()
.with_receipts_in_static_files(self.static_files.receipts)
.with_transaction_senders_in_static_files(self.static_files.transaction_senders)
.with_account_changesets_in_static_files(self.static_files.account_changesets)
.with_transaction_hash_numbers_in_rocksdb(self.rocksdb.all || self.rocksdb.tx_hash)
.with_storages_history_in_rocksdb(self.rocksdb.all || self.rocksdb.storages_history)
.with_account_history_in_rocksdb(self.rocksdb.all || self.rocksdb.account_history)
}
/// Returns the max block that the node should run to, looking it up from the network if
/// necessary
pub async fn max_block<Provider, Client>(
@@ -562,7 +544,6 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
engine: self.engine,
era: self.era,
static_files: self.static_files,
rocksdb: self.rocksdb,
}
}
@@ -604,7 +585,6 @@ impl<ChainSpec> Clone for NodeConfig<ChainSpec> {
engine: self.engine.clone(),
era: self.era.clone(),
static_files: self.static_files,
rocksdb: self.rocksdb,
}
}
}

View File

@@ -106,7 +106,6 @@ impl MetricServer {
// Describe metrics after recorder installation
describe_db_metrics();
describe_static_file_metrics();
describe_rocksdb_metrics();
Collector::default().describe();
describe_memory_stats();
describe_io_stats();
@@ -239,26 +238,6 @@ fn describe_static_file_metrics() {
);
}
fn describe_rocksdb_metrics() {
describe_gauge!(
"rocksdb.table_size",
Unit::Bytes,
"The estimated size of a RocksDB table (SST + memtable)"
);
describe_gauge!("rocksdb.table_entries", "The estimated number of keys in a RocksDB table");
describe_gauge!(
"rocksdb.pending_compaction_bytes",
Unit::Bytes,
"Bytes pending compaction for a RocksDB table"
);
describe_gauge!("rocksdb.sst_size", Unit::Bytes, "The size of SST files for a RocksDB table");
describe_gauge!(
"rocksdb.memtable_size",
Unit::Bytes,
"The size of memtables for a RocksDB table"
);
}
#[cfg(all(feature = "jemalloc", unix))]
fn describe_memory_stats() {
describe_gauge!(

View File

@@ -12,18 +12,10 @@ use reth_primitives_traits::{AlloyBlockHeader, BlockTy, HeaderTy, NodePrimitives
use reth_revm::cached::CachedReads;
use reth_storage_api::{BlockReaderIdExt, StateProviderFactory};
use reth_tasks::TaskExecutor;
use std::{
sync::Arc,
time::{Duration, Instant},
};
use tokio::{
sync::{oneshot, watch},
time::sleep,
};
use std::{sync::Arc, time::Instant};
use tokio::sync::{oneshot, watch};
use tracing::*;
const CONNECTION_BACKOUT_PERIOD: Duration = Duration::from_secs(5);
/// The `FlashBlockService` maintains an in-memory [`PendingFlashBlock`] built out of a sequence of
/// [`FlashBlock`]s.
#[derive(Debug)]
@@ -175,13 +167,7 @@ where
self.try_start_build_job();
}
Some(Err(err)) => {
warn!(
target: "flashblocks",
%err,
retry_period = CONNECTION_BACKOUT_PERIOD.as_secs(),
"Error receiving flashblock"
);
sleep(CONNECTION_BACKOUT_PERIOD).await;
warn!(target: "flashblocks", %err, "Error receiving flashblock");
}
None => {
warn!(target: "flashblocks", "Flashblock stream ended");

View File

@@ -510,12 +510,27 @@ mod tests {
fn test_sealed_block_rlp_roundtrip() {
// Create a sample block using alloy_consensus::Block
let header = alloy_consensus::Header {
parent_hash: B256::ZERO,
ommers_hash: B256::ZERO,
beneficiary: Address::ZERO,
state_root: B256::ZERO,
transactions_root: B256::ZERO,
receipts_root: B256::ZERO,
logs_bloom: Default::default(),
difficulty: Default::default(),
number: 42,
gas_limit: 30_000_000,
gas_used: 21_000,
timestamp: 1_000_000,
extra_data: Default::default(),
mix_hash: B256::ZERO,
nonce: Default::default(),
base_fee_per_gas: Some(1_000_000_000),
..Default::default()
withdrawals_root: None,
blob_gas_used: None,
excess_blob_gas: None,
parent_beacon_block_root: None,
requests_hash: None,
};
// Create a simple transaction
@@ -570,12 +585,27 @@ mod tests {
fn test_decode_sealed_produces_correct_hash() {
// Create a sample block using alloy_consensus::Block
let header = alloy_consensus::Header {
parent_hash: B256::ZERO,
ommers_hash: B256::ZERO,
beneficiary: Address::ZERO,
state_root: B256::ZERO,
transactions_root: B256::ZERO,
receipts_root: B256::ZERO,
logs_bloom: Default::default(),
difficulty: Default::default(),
number: 42,
gas_limit: 30_000_000,
gas_used: 21_000,
timestamp: 1_000_000,
extra_data: Default::default(),
mix_hash: B256::ZERO,
nonce: Default::default(),
base_fee_per_gas: Some(1_000_000_000),
..Default::default()
withdrawals_root: None,
blob_gas_used: None,
excess_blob_gas: None,
parent_beacon_block_root: None,
requests_hash: None,
};
// Create a simple transaction

View File

@@ -84,14 +84,7 @@ where
.into_inner();
let tx_range = start..=
Some(end)
.min(
input
.limiter
.deleted_entries_limit_left()
// Use saturating addition here to avoid panicking on
// `deleted_entries_limit == usize::MAX`
.map(|left| start.saturating_add(left as u64) - 1),
)
.min(input.limiter.deleted_entries_limit_left().map(|left| start + left as u64 - 1))
.unwrap();
let tx_range_end = *tx_range.end();

View File

@@ -252,18 +252,6 @@ pub trait EngineApi<Engine: EngineTypes> {
&self,
versioned_hashes: Vec<B256>,
) -> RpcResult<Option<Vec<Option<BlobAndProofV2>>>>;
/// Returns the Block Access Lists for the given block hashes.
///
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
#[method(name = "getBALsByHashV1")]
async fn get_bals_by_hash_v1(&self, block_hashes: Vec<BlockHash>) -> RpcResult<Vec<Bytes>>;
/// Returns the Block Access Lists for the given block range.
///
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
#[method(name = "getBALsByRangeV1")]
async fn get_bals_by_range_v1(&self, start: U64, count: U64) -> RpcResult<Vec<Bytes>>;
}
/// A subset of the ETH rpc interface: <https://ethereum.github.io/execution-apis/api-documentation>

View File

@@ -1,11 +1,6 @@
//! Engine API capabilities.
use std::collections::HashSet;
use tracing::warn;
/// All Engine API capabilities supported by Reth (Ethereum mainnet).
///
/// See <https://github.com/ethereum/execution-apis/tree/main/src/engine> for updates.
/// The list of all supported Engine capabilities available over the engine endpoint.
pub const CAPABILITIES: &[&str] = &[
"engine_forkchoiceUpdatedV1",
"engine_forkchoiceUpdatedV2",
@@ -27,150 +22,43 @@ pub const CAPABILITIES: &[&str] = &[
"engine_getBlobsV3",
];
/// Engine API capabilities set.
// The list of all supported Engine capabilities available over the engine endpoint.
///
/// Latest spec: Prague
#[derive(Debug, Clone)]
pub struct EngineCapabilities {
inner: HashSet<String>,
}
impl EngineCapabilities {
/// Creates from an iterator of capability strings.
pub fn new(capabilities: impl IntoIterator<Item = impl Into<String>>) -> Self {
/// Creates a new `EngineCapabilities` instance with the given capabilities.
pub fn new(capabilities: impl IntoIterator<Item: Into<String>>) -> Self {
Self { inner: capabilities.into_iter().map(Into::into).collect() }
}
/// Returns the capabilities as a list of strings.
/// Returns the list of all supported Engine capabilities for Prague spec.
fn prague() -> Self {
Self { inner: CAPABILITIES.iter().copied().map(str::to_owned).collect() }
}
/// Returns the list of all supported Engine capabilities.
pub fn list(&self) -> Vec<String> {
self.inner.iter().cloned().collect()
}
/// Returns a reference to the inner set.
pub const fn as_set(&self) -> &HashSet<String> {
&self.inner
/// Inserts a new capability.
pub fn add_capability(&mut self, capability: impl Into<String>) {
self.inner.insert(capability.into());
}
/// Compares CL capabilities with this EL's capabilities and returns any mismatches.
///
/// Called during `engine_exchangeCapabilities` to detect version mismatches
/// between the consensus layer and execution layer.
pub fn get_capability_mismatches(&self, cl_capabilities: &[String]) -> CapabilityMismatches {
let cl_set: HashSet<&str> = cl_capabilities.iter().map(String::as_str).collect();
// CL has methods EL doesn't support
let mut missing_in_el: Vec<_> = cl_capabilities
.iter()
.filter(|cap| !self.inner.contains(cap.as_str()))
.cloned()
.collect();
missing_in_el.sort();
// EL has methods CL doesn't support
let mut missing_in_cl: Vec<_> =
self.inner.iter().filter(|cap| !cl_set.contains(cap.as_str())).cloned().collect();
missing_in_cl.sort();
CapabilityMismatches { missing_in_el, missing_in_cl }
}
/// Logs warnings if CL and EL capabilities don't match.
///
/// Called during `engine_exchangeCapabilities` to warn operators about
/// version mismatches between the consensus layer and execution layer.
pub fn log_capability_mismatches(&self, cl_capabilities: &[String]) {
let mismatches = self.get_capability_mismatches(cl_capabilities);
if !mismatches.missing_in_el.is_empty() {
warn!(
target: "rpc::engine",
missing = ?mismatches.missing_in_el,
"CL supports Engine API methods that Reth doesn't. Consider upgrading Reth."
);
}
if !mismatches.missing_in_cl.is_empty() {
warn!(
target: "rpc::engine",
missing = ?mismatches.missing_in_cl,
"Reth supports Engine API methods that CL doesn't. Consider upgrading your consensus client."
);
}
/// Removes a capability.
pub fn remove_capability(&mut self, capability: &str) -> Option<String> {
self.inner.take(capability)
}
}
impl Default for EngineCapabilities {
fn default() -> Self {
Self::new(CAPABILITIES.iter().copied())
}
}
/// Result of comparing CL and EL capabilities.
#[derive(Debug, Default, PartialEq, Eq)]
pub struct CapabilityMismatches {
/// Methods supported by CL but not by EL (Reth).
/// Operators should consider upgrading Reth.
pub missing_in_el: Vec<String>,
/// Methods supported by EL (Reth) but not by CL.
/// Operators should consider upgrading their consensus client.
pub missing_in_cl: Vec<String>,
}
impl CapabilityMismatches {
/// Returns `true` if there are no mismatches.
pub const fn is_empty(&self) -> bool {
self.missing_in_el.is_empty() && self.missing_in_cl.is_empty()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_no_mismatches() {
let el = EngineCapabilities::new(["method_a", "method_b"]);
let cl = vec!["method_a".to_string(), "method_b".to_string()];
let result = el.get_capability_mismatches(&cl);
assert!(result.is_empty());
}
#[test]
fn test_cl_has_extra_methods() {
let el = EngineCapabilities::new(["method_a"]);
let cl = vec!["method_a".to_string(), "method_b".to_string()];
let result = el.get_capability_mismatches(&cl);
assert_eq!(result.missing_in_el, vec!["method_b"]);
assert!(result.missing_in_cl.is_empty());
}
#[test]
fn test_el_has_extra_methods() {
let el = EngineCapabilities::new(["method_a", "method_b"]);
let cl = vec!["method_a".to_string()];
let result = el.get_capability_mismatches(&cl);
assert!(result.missing_in_el.is_empty());
assert_eq!(result.missing_in_cl, vec!["method_b"]);
}
#[test]
fn test_both_have_extra_methods() {
let el = EngineCapabilities::new(["method_a", "method_c"]);
let cl = vec!["method_a".to_string(), "method_b".to_string()];
let result = el.get_capability_mismatches(&cl);
assert_eq!(result.missing_in_el, vec!["method_b"]);
assert_eq!(result.missing_in_cl, vec!["method_c"]);
}
#[test]
fn test_results_are_sorted() {
let el = EngineCapabilities::new(["z_method", "a_method"]);
let cl = vec!["z_other".to_string(), "a_other".to_string()];
let result = el.get_capability_mismatches(&cl);
assert_eq!(result.missing_in_el, vec!["a_other", "z_other"]);
assert_eq!(result.missing_in_cl, vec!["a_method", "z_method"]);
Self::prague()
}
}

View File

@@ -1134,13 +1134,8 @@ where
/// Handler for `engine_exchangeCapabilitiesV1`
/// See also <https://github.com/ethereum/execution-apis/blob/6452a6b194d7db269bf1dbd087a267251d3cc7f8/src/engine/common.md#capabilities>
async fn exchange_capabilities(&self, capabilities: Vec<String>) -> RpcResult<Vec<String>> {
trace!(target: "rpc::engine", "Serving engine_exchangeCapabilities");
let el_caps = self.capabilities();
el_caps.log_capability_mismatches(&capabilities);
Ok(el_caps.list())
async fn exchange_capabilities(&self, _capabilities: Vec<String>) -> RpcResult<Vec<String>> {
Ok(self.capabilities().list())
}
async fn get_blobs_v1(
@@ -1166,33 +1161,6 @@ where
trace!(target: "rpc::engine", "Serving engine_getBlobsV3");
Ok(self.get_blobs_v3_metered(versioned_hashes)?)
}
/// Handler for `engine_getBALsByHashV1`
///
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
async fn get_bals_by_hash_v1(
&self,
_block_hashes: Vec<BlockHash>,
) -> RpcResult<Vec<alloy_primitives::Bytes>> {
trace!(target: "rpc::engine", "Serving engine_getBALsByHashV1");
Err(EngineApiError::EngineObjectValidationError(
reth_payload_primitives::EngineObjectValidationError::UnsupportedFork,
))?
}
/// Handler for `engine_getBALsByRangeV1`
///
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
async fn get_bals_by_range_v1(
&self,
_start: U64,
_count: U64,
) -> RpcResult<Vec<alloy_primitives::Bytes>> {
trace!(target: "rpc::engine", "Serving engine_getBALsByRangeV1");
Err(EngineApiError::EngineObjectValidationError(
reth_payload_primitives::EngineObjectValidationError::UnsupportedFork,
))?
}
}
impl<Provider, EngineT, Pool, Validator, ChainSpec> IntoEngineApiRpcModule

View File

@@ -507,7 +507,6 @@ impl From<reth_errors::ProviderError> for EthApiError {
ProviderError::BlockNumberForTransactionIndexNotFound => Self::UnknownBlockOrTxIndex,
ProviderError::FinalizedBlockNotFound => Self::HeaderNotFound(BlockId::finalized()),
ProviderError::SafeBlockNotFound => Self::HeaderNotFound(BlockId::safe()),
ProviderError::BlockExpired { .. } => Self::PrunedHistoryUnavailable,
err => Self::Internal(err.into()),
}
}

View File

@@ -86,14 +86,6 @@ where
}
}
/// Clear all ETL state. Called on error paths to prevent buffer pollution on retry.
fn clear_etl_state(&mut self) {
self.sync_gap = None;
self.hash_collector.clear();
self.header_collector.clear();
self.is_etl_ready = false;
}
/// Write downloaded headers to storage from ETL.
///
/// Writes to static files ( `Header | HeaderTD | HeaderHash` ) and [`tables::HeaderNumbers`]
@@ -266,7 +258,7 @@ where
}
Some(Err(HeadersDownloaderError::DetachedHead { local_head, header, error })) => {
error!(target: "sync::stages::headers", %error, "Cannot attach header to head");
self.clear_etl_state();
self.sync_gap = None;
return Poll::Ready(Err(StageError::DetachedHead {
local_head: Box::new(local_head.block_with_parent()),
header: Box::new(header.block_with_parent()),
@@ -274,7 +266,7 @@ where
}))
}
None => {
self.clear_etl_state();
self.sync_gap = None;
return Poll::Ready(Err(StageError::ChannelClosed))
}
}
@@ -332,7 +324,7 @@ where
provider: &Provider,
input: UnwindInput,
) -> Result<UnwindOutput, StageError> {
self.clear_etl_state();
self.sync_gap.take();
// First unwind the db tables, until the unwind_to block number. use the walker to unwind
// HeaderNumbers based on the index in CanonicalHeaders

View File

@@ -1,7 +1,16 @@
use crate::stages::utils::collect_history_indices;
use super::collect_account_history_indices;
use crate::stages::utils::{collect_history_indices, load_account_history};
use alloy_primitives::Address;
use reth_config::config::{EtlConfig, IndexHistoryConfig};
use reth_db_api::{models::ShardedKey, tables, transaction::DbTxMut};
use reth_db_api::{
cursor::DbCursorRO,
models::{sharded_key::NUM_OF_INDICES_IN_SHARD, ShardedKey},
table::{Decode, Decompress},
tables,
transaction::{DbTx, DbTxMut},
BlockNumberList,
};
use reth_provider::{
DBProvider, EitherWriter, HistoryWriter, PruneCheckpointReader, PruneCheckpointWriter,
RocksDBProviderFactory, StorageSettingsCache,
@@ -101,19 +110,21 @@ where
let mut range = input.next_block_range();
let first_sync = input.checkpoint().block_number == 0;
// Check if we're using RocksDB for account history
let use_rocksdb = provider.cached_storage_settings().account_history_in_rocksdb;
// On first sync we might have history coming from genesis. We clear the table since it's
// faster to rebuild from scratch.
if first_sync {
#[cfg(all(unix, feature = "rocksdb"))]
if use_rocksdb {
// Note: RocksDB clear() executes immediately (not deferred to commit like MDBX),
// but this is safe for first_sync because if we crash before commit, the
// checkpoint stays at 0 and we'll just clear and rebuild again on restart. The
// source data (changesets) is intact.
#[cfg(all(unix, feature = "rocksdb"))]
provider.rocksdb_provider().clear::<tables::AccountsHistory>()?;
} else {
provider.rocksdb_provider().clear_table::<tables::AccountsHistory>()?;
}
#[cfg(not(all(unix, feature = "rocksdb")))]
let _ = use_rocksdb;
if !use_rocksdb {
provider.tx_ref().clear::<tables::AccountsHistory>()?;
}
range = 0..=*input.next_block_range().end();
@@ -136,12 +147,25 @@ where
info!(target: "sync::stages::index_account_history::exec", "Loading indices into database");
provider.with_rocksdb_batch(|rocksdb_batch| {
let mut writer = EitherWriter::new_accounts_history(provider, rocksdb_batch)?;
load_account_history(collector, first_sync, &mut writer)
.map_err(|e| reth_provider::ProviderError::other(Box::new(e)))?;
Ok(((), writer.into_raw_rocksdb_batch()))
})?;
// Create RocksDB batch if feature is enabled
#[cfg(all(unix, feature = "rocksdb"))]
let rocksdb = provider.rocksdb_provider();
#[cfg(all(unix, feature = "rocksdb"))]
let rocksdb_batch = rocksdb.batch();
#[cfg(not(all(unix, feature = "rocksdb")))]
let rocksdb_batch = ();
// Create writer that routes to either MDBX or RocksDB based on settings
let mut writer = EitherWriter::new_accounts_history(provider, rocksdb_batch)?;
// Load indices using the writer
load_account_history_indices_with_writer(provider, &mut writer, collector, first_sync)?;
// Extract and register RocksDB batch for commit at provider level
#[cfg(all(unix, feature = "rocksdb"))]
if let Some(batch) = writer.into_raw_rocksdb_batch() {
provider.set_pending_rocksdb_batch(batch);
}
Ok(ExecOutput { checkpoint: StageCheckpoint::new(*range.end()), done: true })
}
@@ -162,6 +186,144 @@ where
}
}
/// Loads account history indices from a collector into the database using an [`EitherWriter`].
///
/// This function processes entries from the collector, grouping indices by address and
/// writing them as shards when they reach the maximum shard size. It handles merging
/// with existing shards during incremental syncs.
fn load_account_history_indices_with_writer<Provider, CURSOR, N>(
provider: &Provider,
writer: &mut EitherWriter<'_, CURSOR, N>,
mut collector: reth_etl::Collector<ShardedKey<Address>, BlockNumberList>,
append_only: bool,
) -> Result<(), StageError>
where
Provider: DBProvider<Tx: DbTxMut> + StorageSettingsCache + RocksDBProviderFactory,
CURSOR: reth_db_api::cursor::DbCursorRW<tables::AccountsHistory>
+ reth_db_api::cursor::DbCursorRO<tables::AccountsHistory>,
N: reth_primitives_traits::NodePrimitives,
{
let mut current_address = Address::ZERO;
let mut current_list = Vec::<u64>::new();
// observability
let total_entries = collector.len();
let interval = (total_entries / 10).max(1);
for (index, element) in collector.iter()?.enumerate() {
let (k, v) = element?;
let sharded_key = ShardedKey::<Address>::decode_owned(k)?;
let new_list = BlockNumberList::decompress_owned(v)?;
if index > 0 && index.is_multiple_of(interval) && total_entries > 10 {
info!(target: "sync::stages::index_account_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices");
}
let address = sharded_key.key;
if current_address != address {
// We have reached the end of this address, flush remaining indices
flush_account_history_shards(writer, current_address, &mut current_list, append_only)?;
current_address = address;
current_list.clear();
// If it's not the first sync, merge with existing last shard
if !append_only &&
let Some(existing_list) =
get_last_account_history_shard(provider, current_address)?
{
current_list.extend(existing_list.iter());
}
}
current_list.extend(new_list.iter());
// Write full shards, keep the partial shard in memory
write_full_shards(writer, current_address, &mut current_list)?;
}
// Flush the last address's remaining shard
flush_account_history_shards(writer, current_address, &mut current_list, append_only)?;
Ok(())
}
/// Retrieves the last shard for an account from `RocksDB` or MDBX based on storage settings.
fn get_last_account_history_shard<Provider>(
provider: &Provider,
address: Address,
) -> Result<Option<BlockNumberList>, StageError>
where
Provider: DBProvider + StorageSettingsCache + RocksDBProviderFactory,
{
#[cfg(all(unix, feature = "rocksdb"))]
if provider.cached_storage_settings().account_history_in_rocksdb {
let rocksdb = provider.rocksdb_provider();
let key = ShardedKey::last(address);
return Ok(rocksdb.get::<tables::AccountsHistory>(key)?);
}
// Read from MDBX
let key = ShardedKey::last(address);
let mut cursor = provider.tx_ref().cursor_read::<tables::AccountsHistory>()?;
Ok(cursor.seek_exact(key)?.map(|(_, v)| v))
}
/// Writes full shards from `list` to the database, draining them from the list.
fn write_full_shards<CURSOR, N>(
writer: &mut EitherWriter<'_, CURSOR, N>,
address: Address,
list: &mut Vec<u64>,
) -> Result<(), StageError>
where
CURSOR: reth_db_api::cursor::DbCursorRW<tables::AccountsHistory>
+ reth_db_api::cursor::DbCursorRO<tables::AccountsHistory>,
N: reth_primitives_traits::NodePrimitives,
{
while list.len() > NUM_OF_INDICES_IN_SHARD {
let chunk: Vec<u64> = list.drain(..NUM_OF_INDICES_IN_SHARD).collect();
let highest = *chunk.last().expect("chunk is not empty");
let key = ShardedKey::new(address, highest);
let value = BlockNumberList::new_pre_sorted(chunk);
writer.put_account_history(key, &value)?;
}
Ok(())
}
/// Flushes all remaining shards to the database. The last shard gets `u64::MAX` as its highest
/// block.
fn flush_account_history_shards<CURSOR, N>(
writer: &mut EitherWriter<'_, CURSOR, N>,
address: Address,
list: &mut Vec<u64>,
append_only: bool,
) -> Result<(), StageError>
where
CURSOR: reth_db_api::cursor::DbCursorRW<tables::AccountsHistory>
+ reth_db_api::cursor::DbCursorRO<tables::AccountsHistory>,
N: reth_primitives_traits::NodePrimitives,
{
if list.is_empty() {
return Ok(());
}
write_full_shards(writer, address, list)?;
// The last shard always uses u64::MAX
if !list.is_empty() {
let key = ShardedKey::last(address);
let value = BlockNumberList::new_pre_sorted(list.drain(..));
// For incremental sync, delete the old last shard first (it will be replaced)
if !append_only {
writer.delete_account_history(key.clone())?;
}
writer.put_account_history(key, &value)?;
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
@@ -169,7 +331,7 @@ mod tests {
stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, TestRunnerError,
TestStageDB, UnwindStageTestRunner,
};
use alloy_primitives::{address, Address, BlockNumber, B256};
use alloy_primitives::{address, BlockNumber, B256};
use itertools::Itertools;
use reth_db_api::{
cursor::DbCursorRO,
@@ -663,7 +825,7 @@ mod tests {
use reth_storage_api::StorageSettings;
/// Test that when `account_history_in_rocksdb` is enabled, the stage
/// writes account history indices to `RocksDB` instead of MDBX.
/// writes account history indices to RocksDB instead of MDBX.
#[tokio::test]
async fn execute_writes_to_rocksdb_when_enabled() {
// init
@@ -674,6 +836,7 @@ mod tests {
StorageSettings::legacy().with_account_history_in_rocksdb(true),
);
// setup - create changesets for blocks 1-10
db.commit(|tx| {
for block in 0..=10 {
tx.put::<tables::BlockBodyIndices>(
@@ -686,6 +849,7 @@ mod tests {
})
.unwrap();
// Run stage
let input = ExecInput { target: Some(10), ..Default::default() };
let mut stage = IndexAccountHistoryStage::default();
let provider = db.factory.database_provider_rw().unwrap();
@@ -710,15 +874,22 @@ mod tests {
assert_eq!(blocks, (0..=10).collect::<Vec<_>>());
}
/// Test that unwind works correctly when `account_history_in_rocksdb` is enabled.
/// Test that when `account_history_in_rocksdb` is enabled, the stage
/// unwind deletes account history indices from RocksDB instead of MDBX.
///
/// Note: Full unwind support for RocksDB requires updates to the HistoryWriter trait
/// implementation. This test verifies the basic execute functionality.
#[tokio::test]
async fn unwind_works_when_rocksdb_enabled() {
async fn unwind_deletes_from_rocksdb_when_enabled() {
// init
let db = TestStageDB::default();
// Enable RocksDB for account history
db.factory.set_storage_settings_cache(
StorageSettings::legacy().with_account_history_in_rocksdb(true),
);
// setup - create changesets for blocks 1-10
db.commit(|tx| {
for block in 0..=10 {
tx.put::<tables::BlockBodyIndices>(
@@ -731,6 +902,7 @@ mod tests {
})
.unwrap();
// Run stage to populate data
let input = ExecInput { target: Some(10), ..Default::default() };
let mut stage = IndexAccountHistoryStage::default();
let provider = db.factory.database_provider_rw().unwrap();
@@ -738,14 +910,14 @@ mod tests {
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
provider.commit().unwrap();
// Verify RocksDB has blocks 0-10 before unwind
// Verify RocksDB has the data before unwind
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should have data before unwind");
let blocks_before: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks_before, (0..=10).collect::<Vec<_>>());
// Unwind to block 5 (remove blocks 6-10)
// Unwind to block 5
// Note: The current HistoryWriter implementation doesn't yet support RocksDB unwind.
// This test verifies the unwind doesn't panic and properly updates checkpoints.
let unwind_input =
UnwindInput { checkpoint: StageCheckpoint::new(10), unwind_to: 5, bad_block: None };
let provider = db.factory.database_provider_rw().unwrap();
@@ -753,23 +925,26 @@ mod tests {
assert_eq!(out, UnwindOutput { checkpoint: StageCheckpoint::new(5) });
provider.commit().unwrap();
// Verify RocksDB now only has blocks 0-5 (blocks 6-10 removed)
// Verify RocksDB data still exists (unwind currently uses MDBX path which doesn't
// affect RocksDB). Once HistoryWriter is updated for RocksDB, this should verify
// that blocks 6-10 are removed.
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should still have data after unwind");
let blocks_after: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks_after, (0..=5).collect::<Vec<_>>(), "Should only have blocks 0-5");
assert!(result.is_some(), "RocksDB should still have data");
}
/// Test incremental sync merges new data with existing shards.
/// Test incremental sync - merging new data with existing data.
#[tokio::test]
async fn execute_incremental_sync() {
// init
let db = TestStageDB::default();
// Enable RocksDB for account history
db.factory.set_storage_settings_cache(
StorageSettings::legacy().with_account_history_in_rocksdb(true),
);
// setup - first batch of changesets
db.commit(|tx| {
for block in 0..=5 {
tx.put::<tables::BlockBodyIndices>(
@@ -782,6 +957,7 @@ mod tests {
})
.unwrap();
// First sync (blocks 0-5)
let input = ExecInput { target: Some(5), ..Default::default() };
let mut stage = IndexAccountHistoryStage::default();
let provider = db.factory.database_provider_rw().unwrap();
@@ -789,12 +965,14 @@ mod tests {
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(5), done: true });
provider.commit().unwrap();
// Verify first sync data in RocksDB
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some());
let blocks: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks, (0..=5).collect::<Vec<_>>());
// Add more changesets for blocks 6-10
db.commit(|tx| {
for block in 6..=10 {
tx.put::<tables::BlockBodyIndices>(
@@ -807,17 +985,19 @@ mod tests {
})
.unwrap();
// Second sync (blocks 6-10)
let input = ExecInput { target: Some(10), checkpoint: Some(StageCheckpoint::new(5)) };
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
provider.commit().unwrap();
// Verify merged data - should have blocks 0-10
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should have merged data");
let blocks: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks, (0..=10).collect::<Vec<_>>());
assert_eq!(blocks, (0..=10).collect::<Vec<_>>(), "Should have merged blocks 0-10");
}
}
}

View File

@@ -1,21 +1,19 @@
use super::collect_history_indices;
use crate::{stages::utils::load_storage_history, StageCheckpoint, StageId};
use super::{collect_history_indices, load_history_indices};
use crate::{StageCheckpoint, StageId};
use reth_config::config::{EtlConfig, IndexHistoryConfig};
use reth_db_api::{
models::{storage_sharded_key::StorageShardedKey, AddressStorageKey, BlockNumberAddress},
table::Decode,
tables,
transaction::DbTxMut,
};
use reth_provider::{
DBProvider, EitherWriter, HistoryWriter, PruneCheckpointReader, PruneCheckpointWriter,
RocksDBProviderFactory, StorageSettingsCache,
};
use reth_provider::{DBProvider, HistoryWriter, PruneCheckpointReader, PruneCheckpointWriter};
use reth_prune_types::{PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment};
use reth_stages_api::{ExecInput, ExecOutput, Stage, StageError, UnwindInput, UnwindOutput};
use std::fmt::Debug;
use tracing::info;
/// Stage is indexing history the storage changesets generated in
/// Stage is indexing history the account changesets generated in
/// [`ExecutionStage`][crate::stages::ExecutionStage]. For more information
/// on index sharding take a look at [`tables::StoragesHistory`].
#[derive(Debug)]
@@ -36,7 +34,7 @@ impl IndexStorageHistoryStage {
etl_config: EtlConfig,
prune_mode: Option<PruneMode>,
) -> Self {
Self { commit_threshold: config.commit_threshold, etl_config, prune_mode }
Self { commit_threshold: config.commit_threshold, prune_mode, etl_config }
}
}
@@ -48,13 +46,8 @@ impl Default for IndexStorageHistoryStage {
impl<Provider> Stage<Provider> for IndexStorageHistoryStage
where
Provider: DBProvider<Tx: DbTxMut>
+ HistoryWriter
+ PruneCheckpointReader
+ PruneCheckpointWriter
+ StorageSettingsCache
+ RocksDBProviderFactory
+ reth_provider::NodePrimitivesProvider,
Provider:
DBProvider<Tx: DbTxMut> + PruneCheckpointWriter + HistoryWriter + PruneCheckpointReader,
{
/// Return the id of the stage
fn id(&self) -> StageId {
@@ -102,25 +95,15 @@ where
let mut range = input.next_block_range();
let first_sync = input.checkpoint().block_number == 0;
let use_rocksdb = provider.cached_storage_settings().storages_history_in_rocksdb;
// On first sync we might have history coming from genesis. We clear the table since it's
// faster to rebuild from scratch.
if first_sync {
if use_rocksdb {
// Note: RocksDB clear() executes immediately (not deferred to commit like MDBX),
// but this is safe for first_sync because if we crash before commit, the
// checkpoint stays at 0 and we'll just clear and rebuild again on restart. The
// source data (changesets) is intact.
#[cfg(all(unix, feature = "rocksdb"))]
provider.rocksdb_provider().clear::<tables::StoragesHistory>()?;
} else {
provider.tx_ref().clear::<tables::StoragesHistory>()?;
}
provider.tx_ref().clear::<tables::StoragesHistory>()?;
range = 0..=*input.next_block_range().end();
}
info!(target: "sync::stages::index_storage_history::exec", ?first_sync, ?use_rocksdb, "Collecting indices");
info!(target: "sync::stages::index_storage_history::exec", ?first_sync, "Collecting indices");
let collector =
collect_history_indices::<_, tables::StorageChangeSets, tables::StoragesHistory, _>(
provider,
@@ -133,13 +116,16 @@ where
)?;
info!(target: "sync::stages::index_storage_history::exec", "Loading indices into database");
provider.with_rocksdb_batch(|rocksdb_batch| {
let mut writer = EitherWriter::new_storages_history(provider, rocksdb_batch)?;
load_storage_history(collector, first_sync, &mut writer)
.map_err(|e| reth_provider::ProviderError::other(Box::new(e)))?;
Ok(((), writer.into_raw_rocksdb_batch()))
})?;
load_history_indices::<_, tables::StoragesHistory, _>(
provider,
collector,
first_sync,
|AddressStorageKey((address, storage_key)), highest_block_number| {
StorageShardedKey::new(address, storage_key, highest_block_number)
},
StorageShardedKey::decode_owned,
|key| AddressStorageKey((key.address, key.sharded_key.key)),
)?;
Ok(ExecOutput { checkpoint: StageCheckpoint::new(*range.end()), done: true })
}
@@ -396,12 +382,12 @@ mod tests {
async fn insert_index_second_half_shard() {
// init
let db = TestStageDB::default();
let mut almost_full_list = (1..=LAST_BLOCK_IN_FULL_SHARD - 1).collect::<Vec<_>>();
let mut close_full_list = (1..=LAST_BLOCK_IN_FULL_SHARD - 1).collect::<Vec<_>>();
// setup
partial_setup(&db);
db.commit(|tx| {
tx.put::<tables::StoragesHistory>(shard(u64::MAX), list(&almost_full_list)).unwrap();
tx.put::<tables::StoragesHistory>(shard(u64::MAX), list(&close_full_list)).unwrap();
Ok(())
})
.unwrap();
@@ -410,12 +396,12 @@ mod tests {
run(&db, LAST_BLOCK_IN_FULL_SHARD + 1, Some(LAST_BLOCK_IN_FULL_SHARD - 1));
// verify
almost_full_list.push(LAST_BLOCK_IN_FULL_SHARD);
close_full_list.push(LAST_BLOCK_IN_FULL_SHARD);
let table = cast(db.table::<tables::StoragesHistory>().unwrap());
assert_eq!(
table,
BTreeMap::from([
(shard(LAST_BLOCK_IN_FULL_SHARD), almost_full_list.clone()),
(shard(LAST_BLOCK_IN_FULL_SHARD), close_full_list.clone()),
(shard(u64::MAX), vec![LAST_BLOCK_IN_FULL_SHARD + 1])
])
);
@@ -424,9 +410,9 @@ mod tests {
unwind(&db, LAST_BLOCK_IN_FULL_SHARD, LAST_BLOCK_IN_FULL_SHARD - 1);
// verify initial state
almost_full_list.pop();
close_full_list.pop();
let table = cast(db.table::<tables::StoragesHistory>().unwrap());
assert_eq!(table, BTreeMap::from([(shard(u64::MAX), almost_full_list)]));
assert_eq!(table, BTreeMap::from([(shard(u64::MAX), close_full_list)]));
}
#[tokio::test]
@@ -677,294 +663,4 @@ mod tests {
Ok(())
}
}
#[cfg(all(unix, feature = "rocksdb"))]
mod rocksdb_tests {
use super::*;
use reth_provider::RocksDBProviderFactory;
use reth_storage_api::StorageSettings;
/// Test that when `storages_history_in_rocksdb` is enabled, the stage
/// writes storage history indices to `RocksDB` instead of MDBX.
#[tokio::test]
async fn execute_writes_to_rocksdb_when_enabled() {
let db = TestStageDB::default();
db.factory.set_storage_settings_cache(
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=10 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(10), ..Default::default() };
let mut stage = IndexStorageHistoryStage::default();
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
provider.commit().unwrap();
let mdbx_table = db.table::<tables::StoragesHistory>().unwrap();
assert!(
mdbx_table.is_empty(),
"MDBX StoragesHistory should be empty when RocksDB is enabled"
);
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should contain storage history");
let block_list = result.unwrap();
let blocks: Vec<u64> = block_list.iter().collect();
assert_eq!(blocks, (0..=10).collect::<Vec<_>>());
}
/// Test that unwind works correctly when `storages_history_in_rocksdb` is enabled.
#[tokio::test]
async fn unwind_works_when_rocksdb_enabled() {
let db = TestStageDB::default();
db.factory.set_storage_settings_cache(
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=10 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(10), ..Default::default() };
let mut stage = IndexStorageHistoryStage::default();
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should have data before unwind");
let blocks_before: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks_before, (0..=10).collect::<Vec<_>>());
let unwind_input =
UnwindInput { checkpoint: StageCheckpoint::new(10), unwind_to: 5, bad_block: None };
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.unwind(&provider, unwind_input).unwrap();
assert_eq!(out, UnwindOutput { checkpoint: StageCheckpoint::new(5) });
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should still have data after partial unwind");
let blocks_after: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(
blocks_after,
(0..=5).collect::<Vec<_>>(),
"Should only have blocks 0-5 after unwind to block 5"
);
}
/// Test that unwind to block 0 keeps only block 0's history.
#[tokio::test]
async fn unwind_to_zero_keeps_block_zero() {
let db = TestStageDB::default();
db.factory.set_storage_settings_cache(
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=5 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(5), ..Default::default() };
let mut stage = IndexStorageHistoryStage::default();
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(5), done: true });
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should have data before unwind");
let unwind_input =
UnwindInput { checkpoint: StageCheckpoint::new(5), unwind_to: 0, bad_block: None };
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.unwind(&provider, unwind_input).unwrap();
assert_eq!(out, UnwindOutput { checkpoint: StageCheckpoint::new(0) });
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should still have block 0 history");
let blocks_after: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks_after, vec![0], "Should only have block 0 after unwinding to 0");
}
/// Test incremental sync merges new data with existing shards.
#[tokio::test]
async fn execute_incremental_sync() {
let db = TestStageDB::default();
db.factory.set_storage_settings_cache(
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=5 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(5), ..Default::default() };
let mut stage = IndexStorageHistoryStage::default();
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(5), done: true });
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some());
let blocks: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks, (0..=5).collect::<Vec<_>>());
db.commit(|tx| {
for block in 6..=10 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(10), checkpoint: Some(StageCheckpoint::new(5)) };
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should have merged data");
let blocks: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks, (0..=10).collect::<Vec<_>>());
}
/// Test multi-shard unwind correctly handles shards that span across unwind boundary.
#[tokio::test]
async fn unwind_multi_shard() {
use reth_db_api::models::sharded_key::NUM_OF_INDICES_IN_SHARD;
let db = TestStageDB::default();
db.factory.set_storage_settings_cache(
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
);
let num_blocks = (NUM_OF_INDICES_IN_SHARD * 2 + 100) as u64;
db.commit(|tx| {
for block in 0..num_blocks {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(num_blocks - 1), ..Default::default() };
let mut stage = IndexStorageHistoryStage::default();
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
assert_eq!(
out,
ExecOutput { checkpoint: StageCheckpoint::new(num_blocks - 1), done: true }
);
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let shards = rocksdb.storage_history_shards(ADDRESS, STORAGE_KEY).unwrap();
assert!(shards.len() >= 2, "Should have at least 2 shards for {} blocks", num_blocks);
let unwind_to = NUM_OF_INDICES_IN_SHARD as u64 + 50;
let unwind_input = UnwindInput {
checkpoint: StageCheckpoint::new(num_blocks - 1),
unwind_to,
bad_block: None,
};
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.unwind(&provider, unwind_input).unwrap();
assert_eq!(out, UnwindOutput { checkpoint: StageCheckpoint::new(unwind_to) });
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let shards_after = rocksdb.storage_history_shards(ADDRESS, STORAGE_KEY).unwrap();
assert!(!shards_after.is_empty(), "Should still have shards after unwind");
let all_blocks: Vec<u64> =
shards_after.iter().flat_map(|(_, list)| list.iter()).collect();
assert_eq!(
all_blocks,
(0..=unwind_to).collect::<Vec<_>>(),
"Should only have blocks 0 to {} after unwind",
unwind_to
);
}
}
}

View File

@@ -217,7 +217,7 @@ impl MerkleChangeSets {
let compute_cumulative_state_revert = |block_number: BlockNumber| -> HashedPostStateSorted {
let mut cumulative_revert = HashedPostStateSorted::default();
for n in (block_number..target_end).rev() {
cumulative_revert.extend_ref_and_sort(get_block_state_revert(n))
cumulative_revert.extend_ref(get_block_state_revert(n))
}
cumulative_revert
};
@@ -270,7 +270,7 @@ impl MerkleChangeSets {
let trie_overlay = Arc::clone(&nodes);
let mut nodes_mut = Arc::unwrap_or_clone(nodes);
nodes_mut.extend_ref_and_sort(&this_trie_updates);
nodes_mut.extend_ref(&this_trie_updates);
nodes = Arc::new(nodes_mut);
// Write the changesets to the DB using the trie updates produced by the block, and the

View File

@@ -1,20 +1,16 @@
//! Utils for `stages`.
use alloy_primitives::{Address, BlockNumber, TxNumber, B256};
use alloy_primitives::{Address, BlockNumber, TxNumber};
use reth_config::config::EtlConfig;
use reth_db_api::{
cursor::{DbCursorRO, DbCursorRW},
models::{
sharded_key::NUM_OF_INDICES_IN_SHARD, storage_sharded_key::StorageShardedKey,
AccountBeforeTx, ShardedKey,
},
table::{Decode, Decompress, Table},
transaction::DbTx,
BlockNumberList,
models::{sharded_key::NUM_OF_INDICES_IN_SHARD, AccountBeforeTx, ShardedKey},
table::{Decompress, Table},
transaction::{DbTx, DbTxMut},
BlockNumberList, DatabaseError,
};
use reth_etl::Collector;
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
providers::StaticFileProvider, to_range, BlockReader, DBProvider, EitherWriter, ProviderError,
providers::StaticFileProvider, to_range, BlockReader, DBProvider, ProviderError,
StaticFileProviderFactory,
};
use reth_stages_api::StageError;
@@ -61,12 +57,12 @@ where
let mut collector = Collector::new(etl_config.file_size, etl_config.dir.clone());
let mut cache: HashMap<P, Vec<u64>> = HashMap::default();
let mut collect = |cache: &mut HashMap<P, Vec<u64>>| {
for (key, indices) in cache.drain() {
let last = *indices.last().expect("qed");
let mut collect = |cache: &HashMap<P, Vec<u64>>| {
for (key, indices) in cache {
let last = indices.last().expect("qed");
collector.insert(
sharded_key_factory(key, last),
BlockNumberList::new_pre_sorted(indices.into_iter()),
sharded_key_factory(*key, *last),
BlockNumberList::new_pre_sorted(indices.iter().copied()),
)?;
}
Ok::<(), StageError>(())
@@ -91,12 +87,13 @@ where
current_block_number = block_number;
flush_counter += 1;
if flush_counter > DEFAULT_CACHE_THRESHOLD {
collect(&mut cache)?;
collect(&cache)?;
cache.clear();
flush_counter = 0;
}
}
}
collect(&mut cache)?;
collect(&cache)?;
Ok(collector)
}
@@ -112,7 +109,7 @@ where
for (address, indices) in cache {
insert_fn(address, indices)?
}
Ok(())
Ok::<(), StageError>(())
}
/// Collects account history indices using a provider that implements `ChangeSetReader`.
@@ -128,12 +125,12 @@ where
let mut cache: HashMap<Address, Vec<u64>> = HashMap::default();
let mut insert_fn = |address: Address, indices: Vec<u64>| {
let last = indices.last().expect("indices is non-empty");
let last = indices.last().expect("qed");
collector.insert(
ShardedKey::new(address, *last),
BlockNumberList::new_pre_sorted(indices.into_iter()),
)?;
Ok(())
Ok::<(), StageError>(())
};
// Convert range bounds to concrete range
@@ -174,176 +171,154 @@ where
Ok(collector)
}
/// Loads account history indices into the database via `EitherWriter`.
/// Given a [`Collector`] created by [`collect_history_indices`] it iterates all entries, loading
/// the indices into the database in shards.
///
/// Works with [`EitherWriter`] to support both MDBX and `RocksDB` backends.
///
/// ## Process
/// Iterates over elements, grouping indices by their address. It flushes indices to disk
/// when reaching a shard's max length (`NUM_OF_INDICES_IN_SHARD`) or when the address changes,
/// ensuring the last previous address shard is stored.
///
/// Uses `Option<Address>` instead of `Address::default()` as the sentinel to avoid
/// incorrectly treating `Address::ZERO` as "no previous address".
pub(crate) fn load_account_history<N, CURSOR>(
mut collector: Collector<ShardedKey<Address>, BlockNumberList>,
/// ## Process
/// Iterates over elements, grouping indices by their partial keys (e.g., `Address` or
/// `Address.StorageKey`). It flushes indices to disk when reaching a shard's max length
/// (`NUM_OF_INDICES_IN_SHARD`) or when the partial key changes, ensuring the last previous partial
/// key shard is stored.
pub(crate) fn load_history_indices<Provider, H, P>(
provider: &Provider,
mut collector: Collector<H::Key, H::Value>,
append_only: bool,
writer: &mut EitherWriter<'_, CURSOR, N>,
sharded_key_factory: impl Clone + Fn(P, u64) -> <H as Table>::Key,
decode_key: impl Fn(Vec<u8>) -> Result<<H as Table>::Key, DatabaseError>,
get_partial: impl Fn(<H as Table>::Key) -> P,
) -> Result<(), StageError>
where
N: NodePrimitives,
CURSOR: DbCursorRW<reth_db_api::tables::AccountsHistory>
+ DbCursorRO<reth_db_api::tables::AccountsHistory>,
Provider: DBProvider<Tx: DbTxMut>,
H: Table<Value = BlockNumberList>,
P: Copy + Default + Eq,
{
let mut current_address: Option<Address> = None;
// Accumulator for block numbers where the current address changed.
let mut write_cursor = provider.tx_ref().cursor_write::<H>()?;
let mut current_partial = P::default();
let mut current_list = Vec::<u64>::new();
// observability
let total_entries = collector.len();
let interval = (total_entries / 10).max(1);
for (index, element) in collector.iter()?.enumerate() {
let (k, v) = element?;
let sharded_key = ShardedKey::<Address>::decode_owned(k)?;
let sharded_key = decode_key(k)?;
let new_list = BlockNumberList::decompress_owned(v)?;
if index > 0 && index.is_multiple_of(interval) && total_entries > 10 {
info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices");
}
let address = sharded_key.key;
// AccountsHistory: `Address`.
// StorageHistory: `Address.StorageKey`.
let partial_key = get_partial(sharded_key);
// When address changes, flush the previous address's shards and start fresh.
if current_address != Some(address) {
// Flush all remaining shards for the previous address (uses u64::MAX for last shard).
if let Some(prev_addr) = current_address {
flush_account_history_shards(prev_addr, &mut current_list, append_only, writer)?;
}
if current_partial != partial_key {
// We have reached the end of this subset of keys so
// we need to flush its last indice shard.
load_indices(
&mut write_cursor,
current_partial,
&mut current_list,
&sharded_key_factory,
append_only,
LoadMode::Flush,
)?;
current_address = Some(address);
current_partial = partial_key;
current_list.clear();
// On incremental sync, merge with the existing last shard from the database.
// The last shard is stored with key (address, u64::MAX) so we can find it.
// If it's not the first sync, there might an existing shard already, so we need to
// merge it with the one coming from the collector
if !append_only &&
let Some(last_shard) = writer.get_last_account_history_shard(address)?
let Some((_, last_database_shard)) =
write_cursor.seek_exact(sharded_key_factory(current_partial, u64::MAX))?
{
current_list.extend(last_shard.iter());
current_list.extend(last_database_shard.iter());
}
}
// Append new block numbers to the accumulator.
current_list.extend(new_list.iter());
// Flush complete shards, keeping the last (partial) shard buffered.
flush_account_history_shards_partial(address, &mut current_list, append_only, writer)?;
load_indices(
&mut write_cursor,
current_partial,
&mut current_list,
&sharded_key_factory,
append_only,
LoadMode::KeepLast,
)?;
}
// Flush the final address's remaining shard.
if let Some(addr) = current_address {
flush_account_history_shards(addr, &mut current_list, append_only, writer)?;
// There will be one remaining shard that needs to be flushed to DB.
load_indices(
&mut write_cursor,
current_partial,
&mut current_list,
&sharded_key_factory,
append_only,
LoadMode::Flush,
)?;
Ok(())
}
/// Shard and insert the indices list according to [`LoadMode`] and its length.
pub(crate) fn load_indices<H, C, P>(
cursor: &mut C,
partial_key: P,
list: &mut Vec<BlockNumber>,
sharded_key_factory: &impl Fn(P, BlockNumber) -> <H as Table>::Key,
append_only: bool,
mode: LoadMode,
) -> Result<(), StageError>
where
C: DbCursorRO<H> + DbCursorRW<H>,
H: Table<Value = BlockNumberList>,
P: Copy,
{
if list.len() > NUM_OF_INDICES_IN_SHARD || mode.is_flush() {
let chunks = list
.chunks(NUM_OF_INDICES_IN_SHARD)
.map(|chunks| chunks.to_vec())
.collect::<Vec<Vec<u64>>>();
let mut iter = chunks.into_iter().peekable();
while let Some(chunk) = iter.next() {
let mut highest = *chunk.last().expect("at least one index");
if !mode.is_flush() && iter.peek().is_none() {
*list = chunk;
} else {
if iter.peek().is_none() {
highest = u64::MAX;
}
let key = sharded_key_factory(partial_key, highest);
let value = BlockNumberList::new_pre_sorted(chunk);
if append_only {
cursor.append(key, &value)?;
} else {
cursor.upsert(key, &value)?;
}
}
}
}
Ok(())
}
/// Flushes complete shards for account history, keeping the trailing partial shard buffered.
///
/// Only flushes when we have more than one shard's worth of data, keeping the last
/// (possibly partial) shard for continued accumulation. This avoids writing a shard
/// that may need to be updated when more indices arrive.
fn flush_account_history_shards_partial<N, CURSOR>(
address: Address,
list: &mut Vec<u64>,
append_only: bool,
writer: &mut EitherWriter<'_, CURSOR, N>,
) -> Result<(), StageError>
where
N: NodePrimitives,
CURSOR: DbCursorRW<reth_db_api::tables::AccountsHistory>
+ DbCursorRO<reth_db_api::tables::AccountsHistory>,
{
// Nothing to flush if we haven't filled a complete shard yet.
if list.len() <= NUM_OF_INDICES_IN_SHARD {
return Ok(());
}
let num_full_shards = list.len() / NUM_OF_INDICES_IN_SHARD;
// Always keep at least one shard buffered for continued accumulation.
// If len is exact multiple of shard size, keep the last full shard.
let shards_to_flush = if list.len().is_multiple_of(NUM_OF_INDICES_IN_SHARD) {
num_full_shards - 1
} else {
num_full_shards
};
if shards_to_flush == 0 {
return Ok(());
}
// Split: flush the first N shards, keep the remainder buffered.
let flush_len = shards_to_flush * NUM_OF_INDICES_IN_SHARD;
let remainder = list.split_off(flush_len);
// Write each complete shard with its highest block number as the key.
for chunk in list.chunks(NUM_OF_INDICES_IN_SHARD) {
let highest = *chunk.last().expect("chunk is non-empty");
let key = ShardedKey::new(address, highest);
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
if append_only {
writer.append_account_history(key, &value)?;
} else {
writer.upsert_account_history(key, &value)?;
}
}
// Keep the remaining indices for the next iteration.
*list = remainder;
Ok(())
/// Mode on how to load index shards into the database.
pub(crate) enum LoadMode {
/// Keep the last shard in memory and don't flush it to the database.
KeepLast,
/// Flush all shards into the database.
Flush,
}
/// Flushes all remaining shards for account history, using `u64::MAX` for the last shard.
///
/// The `u64::MAX` key for the final shard is an invariant that allows `seek_exact(address,
/// u64::MAX)` to find the last shard during incremental sync for merging with new indices.
fn flush_account_history_shards<N, CURSOR>(
address: Address,
list: &mut Vec<u64>,
append_only: bool,
writer: &mut EitherWriter<'_, CURSOR, N>,
) -> Result<(), StageError>
where
N: NodePrimitives,
CURSOR: DbCursorRW<reth_db_api::tables::AccountsHistory>
+ DbCursorRO<reth_db_api::tables::AccountsHistory>,
{
if list.is_empty() {
return Ok(());
impl LoadMode {
const fn is_flush(&self) -> bool {
matches!(self, Self::Flush)
}
let num_chunks = list.len().div_ceil(NUM_OF_INDICES_IN_SHARD);
for (i, chunk) in list.chunks(NUM_OF_INDICES_IN_SHARD).enumerate() {
let is_last = i == num_chunks - 1;
// Use u64::MAX for the final shard's key. This invariant allows incremental sync
// to find the last shard via seek_exact(address, u64::MAX) for merging.
let highest = if is_last { u64::MAX } else { *chunk.last().expect("chunk is non-empty") };
let key = ShardedKey::new(address, highest);
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
if append_only {
writer.append_account_history(key, &value)?;
} else {
writer.upsert_account_history(key, &value)?;
}
}
list.clear();
Ok(())
}
/// Called when database is ahead of static files. Attempts to find the first block we are missing
@@ -381,191 +356,3 @@ where
segment,
})
}
/// Loads storage history indices into the database via `EitherWriter`.
///
/// Works with [`EitherWriter`] to support both MDBX and `RocksDB` backends.
///
/// ## Process
/// Iterates over elements, grouping indices by their (address, `storage_key`) pairs. It flushes
/// indices to disk when reaching a shard's max length (`NUM_OF_INDICES_IN_SHARD`) or when the
/// (address, `storage_key`) pair changes, ensuring the last previous shard is stored.
///
/// Uses `Option<(Address, B256)>` instead of default values as the sentinel to avoid
/// incorrectly treating `(Address::ZERO, B256::ZERO)` as "no previous key".
pub(crate) fn load_storage_history<N, CURSOR>(
mut collector: Collector<StorageShardedKey, BlockNumberList>,
append_only: bool,
writer: &mut EitherWriter<'_, CURSOR, N>,
) -> Result<(), StageError>
where
N: NodePrimitives,
CURSOR: DbCursorRW<reth_db_api::tables::StoragesHistory>
+ DbCursorRO<reth_db_api::tables::StoragesHistory>,
{
let mut current_key: Option<(Address, B256)> = None;
// Accumulator for block numbers where the current (address, storage_key) changed.
let mut current_list = Vec::<u64>::new();
let total_entries = collector.len();
let interval = (total_entries / 10).max(1);
for (index, element) in collector.iter()?.enumerate() {
let (k, v) = element?;
let sharded_key = StorageShardedKey::decode_owned(k)?;
let new_list = BlockNumberList::decompress_owned(v)?;
if index > 0 && index.is_multiple_of(interval) && total_entries > 10 {
info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices");
}
let partial_key = (sharded_key.address, sharded_key.sharded_key.key);
// When (address, storage_key) changes, flush the previous key's shards and start fresh.
if current_key != Some(partial_key) {
// Flush all remaining shards for the previous key (uses u64::MAX for last shard).
if let Some((prev_addr, prev_storage_key)) = current_key {
flush_storage_history_shards(
prev_addr,
prev_storage_key,
&mut current_list,
append_only,
writer,
)?;
}
current_key = Some(partial_key);
current_list.clear();
// On incremental sync, merge with the existing last shard from the database.
// The last shard is stored with key (address, storage_key, u64::MAX) so we can find it.
if !append_only &&
let Some(last_shard) =
writer.get_last_storage_history_shard(partial_key.0, partial_key.1)?
{
current_list.extend(last_shard.iter());
}
}
// Append new block numbers to the accumulator.
current_list.extend(new_list.iter());
// Flush complete shards, keeping the last (partial) shard buffered.
flush_storage_history_shards_partial(
partial_key.0,
partial_key.1,
&mut current_list,
append_only,
writer,
)?;
}
// Flush the final key's remaining shard.
if let Some((addr, storage_key)) = current_key {
flush_storage_history_shards(addr, storage_key, &mut current_list, append_only, writer)?;
}
Ok(())
}
/// Flushes complete shards for storage history, keeping the trailing partial shard buffered.
///
/// Only flushes when we have more than one shard's worth of data, keeping the last
/// (possibly partial) shard for continued accumulation. This avoids writing a shard
/// that may need to be updated when more indices arrive.
fn flush_storage_history_shards_partial<N, CURSOR>(
address: Address,
storage_key: B256,
list: &mut Vec<u64>,
append_only: bool,
writer: &mut EitherWriter<'_, CURSOR, N>,
) -> Result<(), StageError>
where
N: NodePrimitives,
CURSOR: DbCursorRW<reth_db_api::tables::StoragesHistory>
+ DbCursorRO<reth_db_api::tables::StoragesHistory>,
{
// Nothing to flush if we haven't filled a complete shard yet.
if list.len() <= NUM_OF_INDICES_IN_SHARD {
return Ok(());
}
let num_full_shards = list.len() / NUM_OF_INDICES_IN_SHARD;
// Always keep at least one shard buffered for continued accumulation.
// If len is exact multiple of shard size, keep the last full shard.
let shards_to_flush = if list.len().is_multiple_of(NUM_OF_INDICES_IN_SHARD) {
num_full_shards - 1
} else {
num_full_shards
};
if shards_to_flush == 0 {
return Ok(());
}
// Split: flush the first N shards, keep the remainder buffered.
let flush_len = shards_to_flush * NUM_OF_INDICES_IN_SHARD;
let remainder = list.split_off(flush_len);
// Write each complete shard with its highest block number as the key.
for chunk in list.chunks(NUM_OF_INDICES_IN_SHARD) {
let highest = *chunk.last().expect("chunk is non-empty");
let key = StorageShardedKey::new(address, storage_key, highest);
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
if append_only {
writer.append_storage_history(key, &value)?;
} else {
writer.upsert_storage_history(key, &value)?;
}
}
// Keep the remaining indices for the next iteration.
*list = remainder;
Ok(())
}
/// Flushes all remaining shards for storage history, using `u64::MAX` for the last shard.
///
/// The `u64::MAX` key for the final shard is an invariant that allows
/// `seek_exact(address, storage_key, u64::MAX)` to find the last shard during incremental
/// sync for merging with new indices.
fn flush_storage_history_shards<N, CURSOR>(
address: Address,
storage_key: B256,
list: &mut Vec<u64>,
append_only: bool,
writer: &mut EitherWriter<'_, CURSOR, N>,
) -> Result<(), StageError>
where
N: NodePrimitives,
CURSOR: DbCursorRW<reth_db_api::tables::StoragesHistory>
+ DbCursorRO<reth_db_api::tables::StoragesHistory>,
{
if list.is_empty() {
return Ok(());
}
let num_chunks = list.len().div_ceil(NUM_OF_INDICES_IN_SHARD);
for (i, chunk) in list.chunks(NUM_OF_INDICES_IN_SHARD).enumerate() {
let is_last = i == num_chunks - 1;
// Use u64::MAX for the final shard's key. This invariant allows incremental sync
// to find the last shard via seek_exact(address, storage_key, u64::MAX) for merging.
let highest = if is_last { u64::MAX } else { *chunk.last().expect("chunk is non-empty") };
let key = StorageShardedKey::new(address, storage_key, highest);
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
if append_only {
writer.append_storage_history(key, &value)?;
} else {
writer.upsert_storage_history(key, &value)?;
}
}
list.clear();
Ok(())
}

View File

@@ -69,7 +69,7 @@ pub fn generate_from_to(
}
}
/// Generates code to implement the `Compact` trait method `from_compact`.
/// Generates code to implement the `Compact` trait method `to_compact`.
fn generate_from_compact(
fields: &FieldList,
ident: &Ident,
@@ -155,7 +155,7 @@ fn generate_from_compact(
}
}
/// Generates code to implement the `Compact` trait method `to_compact`.
/// Generates code to implement the `Compact` trait method `from_compact`.
fn generate_to_compact(
fields: &FieldList,
ident: &Ident,

View File

@@ -175,7 +175,7 @@ fn should_use_alt_impl(ftype: &str, segment: &syn::PathSegment) -> bool {
let syn::PathArguments::AngleBracketed(ref args) = segment.arguments &&
let Some(syn::GenericArgument::Type(syn::Type::Path(arg_path))) = args.args.last() &&
let (Some(path), 1) = (arg_path.path.segments.first(), arg_path.path.segments.len()) &&
["B256", "Address", "Bloom", "TxHash", "BlockHash", "CompactPlaceholder"]
["B256", "Address", "Address", "Bloom", "TxHash", "BlockHash", "CompactPlaceholder"]
.iter()
.any(|&s| path.ident == s)
{

View File

@@ -40,7 +40,6 @@ serde = { workspace = true, default-features = false }
metrics.workspace = true
# misc
arrayvec.workspace = true
derive_more.workspace = true
bytes.workspace = true

View File

@@ -34,21 +34,6 @@ pub struct StorageSettings {
}
impl StorageSettings {
/// Returns the default base `StorageSettings` for this build.
///
/// When the `edge` feature is enabled, returns [`Self::edge()`].
/// Otherwise, returns [`Self::legacy()`].
pub const fn base() -> Self {
#[cfg(feature = "edge")]
{
Self::edge()
}
#[cfg(not(feature = "edge"))]
{
Self::legacy()
}
}
/// Creates `StorageSettings` for edge nodes with all storage features enabled:
/// - Receipts and transaction senders in static files
/// - History indices in `RocksDB` (storages, accounts, transaction hashes)
@@ -60,7 +45,7 @@ impl StorageSettings {
transaction_senders_in_static_files: true,
account_changesets_in_static_files: true,
storages_history_in_rocksdb: false,
transaction_hash_numbers_in_rocksdb: true,
transaction_hash_numbers_in_rocksdb: false,
account_history_in_rocksdb: false,
}
}

View File

@@ -126,10 +126,13 @@ impl Decode for String {
}
impl Encode for StoredNibbles {
type Encoded = arrayvec::ArrayVec<u8, 64>;
type Encoded = Vec<u8>;
// Delegate to the Compact implementation
fn encode(self) -> Self::Encoded {
self.0.iter().collect()
// NOTE: This used to be `to_compact`, but all it does is append the bytes to the buffer,
// so we can just use the implementation of `Into<Vec<u8>>` to reuse the buffer.
self.0.to_vec()
}
}

View File

@@ -3,16 +3,13 @@ use crate::{
table::{Decode, Encode},
DatabaseError,
};
use alloy_primitives::{Address, BlockNumber};
use alloy_primitives::BlockNumber;
use serde::{Deserialize, Serialize};
use std::hash::Hash;
/// Number of indices in one shard.
pub const NUM_OF_INDICES_IN_SHARD: usize = 2_000;
/// Size of `BlockNumber` in bytes (u64 = 8 bytes).
const BLOCK_NUMBER_SIZE: usize = std::mem::size_of::<BlockNumber>();
/// Sometimes data can be too big to be saved for a single key. This helps out by dividing the data
/// into different shards. Example:
///
@@ -46,68 +43,21 @@ impl<T> ShardedKey<T> {
}
}
/// Stack-allocated encoded key for `ShardedKey<Address>`.
///
/// This avoids heap allocation in hot database paths. The key layout is:
/// - 20 bytes: `Address`
/// - 8 bytes: `BlockNumber` (big-endian)
pub type ShardedKeyAddressEncoded = [u8; 20 + BLOCK_NUMBER_SIZE];
impl<T: Encode> Encode for ShardedKey<T> {
type Encoded = Vec<u8>;
impl Encode for ShardedKey<Address> {
type Encoded = ShardedKeyAddressEncoded;
#[inline]
fn encode(self) -> Self::Encoded {
let mut buf = [0u8; 20 + BLOCK_NUMBER_SIZE];
buf[..20].copy_from_slice(self.key.as_slice());
buf[20..].copy_from_slice(&self.highest_block_number.to_be_bytes());
let mut buf: Vec<u8> = Encode::encode(self.key).into();
buf.extend_from_slice(&self.highest_block_number.to_be_bytes());
buf
}
}
impl Decode for ShardedKey<Address> {
impl<T: Decode> Decode for ShardedKey<T> {
fn decode(value: &[u8]) -> Result<Self, DatabaseError> {
if value.len() != 20 + BLOCK_NUMBER_SIZE {
return Err(DatabaseError::Decode);
}
let key = Address::from_slice(&value[..20]);
let highest_block_number =
u64::from_be_bytes(value[20..].try_into().map_err(|_| DatabaseError::Decode)?);
Ok(Self::new(key, highest_block_number))
}
}
#[cfg(test)]
mod tests {
use super::*;
use alloy_primitives::address;
#[test]
fn sharded_key_address_encode_decode_roundtrip() {
let addr = address!("0102030405060708091011121314151617181920");
let block_num = 0x123456789ABCDEF0u64;
let key = ShardedKey::new(addr, block_num);
let encoded = key.encode();
// Verify it's stack-allocated (28 bytes)
assert_eq!(encoded.len(), 28);
assert_eq!(std::mem::size_of_val(&encoded), 28);
// Verify roundtrip (check against expected values since key was consumed)
let decoded = ShardedKey::<Address>::decode(&encoded).unwrap();
assert_eq!(decoded.key, address!("0102030405060708091011121314151617181920"));
assert_eq!(decoded.highest_block_number, 0x123456789ABCDEF0u64);
}
#[test]
fn sharded_key_last_works() {
let addr = address!("0102030405060708091011121314151617181920");
let key = ShardedKey::<Address>::last(addr);
assert_eq!(key.highest_block_number, u64::MAX);
let encoded = key.encode();
let decoded = ShardedKey::<Address>::decode(&encoded).unwrap();
assert_eq!(decoded.highest_block_number, u64::MAX);
let (key, highest_tx_number) = value.split_last_chunk().ok_or(DatabaseError::Decode)?;
let key = T::decode(key)?;
let highest_tx_number = u64::from_be_bytes(*highest_tx_number);
Ok(Self::new(key, highest_tx_number))
}
}

View File

@@ -16,14 +16,6 @@ pub const NUM_OF_INDICES_IN_SHARD: usize = 2_000;
/// The fields are: 20-byte address, 32-byte key, and 8-byte block number
const STORAGE_SHARD_KEY_BYTES_SIZE: usize = 20 + 32 + 8;
/// Stack-allocated encoded key for `StorageShardedKey`.
///
/// This avoids heap allocation in hot database paths. The key layout is:
/// - 20 bytes: `Address`
/// - 32 bytes: `B256` storage key
/// - 8 bytes: `BlockNumber` (big-endian)
pub type StorageShardedKeyEncoded = [u8; STORAGE_SHARD_KEY_BYTES_SIZE];
/// Sometimes data can be too big to be saved for a single key. This helps out by dividing the data
/// into different shards. Example:
///
@@ -62,14 +54,13 @@ impl StorageShardedKey {
}
impl Encode for StorageShardedKey {
type Encoded = StorageShardedKeyEncoded;
type Encoded = Vec<u8>;
#[inline]
fn encode(self) -> Self::Encoded {
let mut buf = [0u8; STORAGE_SHARD_KEY_BYTES_SIZE];
buf[..20].copy_from_slice(self.address.as_slice());
buf[20..52].copy_from_slice(self.sharded_key.key.as_slice());
buf[52..].copy_from_slice(&self.sharded_key.highest_block_number.to_be_bytes());
let mut buf: Vec<u8> = Vec::with_capacity(STORAGE_SHARD_KEY_BYTES_SIZE);
buf.extend_from_slice(&Encode::encode(self.address));
buf.extend_from_slice(&Encode::encode(self.sharded_key.key));
buf.extend_from_slice(&self.sharded_key.highest_block_number.to_be_bytes());
buf
}
}
@@ -90,44 +81,3 @@ impl Decode for StorageShardedKey {
Ok(Self { address, sharded_key: ShardedKey::new(storage_key, highest_block_number) })
}
}
#[cfg(test)]
mod tests {
use super::*;
use alloy_primitives::{address, b256};
#[test]
fn storage_sharded_key_encode_decode_roundtrip() {
let addr = address!("0102030405060708091011121314151617181920");
let storage_key = b256!("0001020304050607080910111213141516171819202122232425262728293031");
let block_num = 0x123456789ABCDEFu64;
let key = StorageShardedKey::new(addr, storage_key, block_num);
let encoded = key.encode();
// Verify it's stack-allocated (60 bytes)
assert_eq!(encoded.len(), 60);
assert_eq!(std::mem::size_of_val(&encoded), 60);
// Verify roundtrip (check against expected values since key was consumed)
let decoded = StorageShardedKey::decode(&encoded).unwrap();
assert_eq!(decoded.address, address!("0102030405060708091011121314151617181920"));
assert_eq!(
decoded.sharded_key.key,
b256!("0001020304050607080910111213141516171819202122232425262728293031")
);
assert_eq!(decoded.sharded_key.highest_block_number, 0x123456789ABCDEFu64);
}
#[test]
fn storage_sharded_key_last_works() {
let addr = address!("0102030405060708091011121314151617181920");
let storage_key = b256!("0001020304050607080910111213141516171819202122232425262728293031");
let key = StorageShardedKey::last(addr, storage_key);
assert_eq!(key.sharded_key.highest_block_number, u64::MAX);
let encoded = key.encode();
let decoded = StorageShardedKey::decode(&encoded).unwrap();
assert_eq!(decoded.sharded_key.highest_block_number, u64::MAX);
}
}

View File

@@ -46,40 +46,10 @@ pub trait Decompress: Send + Sync + Sized + Debug {
}
}
/// Trait for converting encoded types to `Vec<u8>`.
///
/// This is implemented for all `AsRef<[u8]>` types. For `Vec<u8>` this is a no-op,
/// for other types like `ArrayVec` or fixed arrays it performs a copy.
pub trait IntoVec: AsRef<[u8]> {
/// Convert to a `Vec<u8>`.
fn into_vec(self) -> Vec<u8>;
}
impl IntoVec for Vec<u8> {
#[inline]
fn into_vec(self) -> Vec<u8> {
self
}
}
impl<const N: usize> IntoVec for [u8; N] {
#[inline]
fn into_vec(self) -> Vec<u8> {
self.to_vec()
}
}
impl<const N: usize> IntoVec for arrayvec::ArrayVec<u8, N> {
#[inline]
fn into_vec(self) -> Vec<u8> {
self.to_vec()
}
}
/// Trait that will transform the data to be saved in the DB.
pub trait Encode: Send + Sync + Sized + Debug {
/// Encoded type.
type Encoded: AsRef<[u8]> + IntoVec + Send + Sync + Ord + Debug;
type Encoded: AsRef<[u8]> + Into<Vec<u8>> + Send + Sync + Ord + Debug;
/// Encodes data going into the database.
fn encode(self) -> Self::Encoded;

View File

@@ -1,5 +1,5 @@
use crate::{
table::{Compress, Decode, Decompress, DupSort, Encode, IntoVec, Key, Table, Value},
table::{Compress, Decode, Decompress, DupSort, Encode, Key, Table, Value},
DatabaseError,
};
use serde::{Deserialize, Serialize};
@@ -52,7 +52,7 @@ pub struct RawKey<K: Key> {
impl<K: Key> RawKey<K> {
/// Create new raw key.
pub fn new(key: K) -> Self {
Self { key: K::encode(key).into_vec(), _phantom: std::marker::PhantomData }
Self { key: K::encode(key).into(), _phantom: std::marker::PhantomData }
}
/// Creates a raw key from an existing `Vec`. Useful when we already have the encoded

View File

@@ -11,7 +11,7 @@ use reth_db_api::{
DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW, DupWalker, RangeWalker,
ReverseWalker, Walker,
},
table::{Compress, Decode, Decompress, DupSort, Encode, IntoVec, Table},
table::{Compress, Decode, Decompress, DupSort, Encode, Table},
};
use reth_libmdbx::{Error as MDBXError, TransactionKind, WriteFlags, RO, RW};
use reth_storage_errors::db::{DatabaseErrorInfo, DatabaseWriteError, DatabaseWriteOperation};
@@ -215,26 +215,27 @@ impl<K: TransactionKind, T: DupSort> DbDupCursorRO<T> for Cursor<K, T> {
) -> Result<DupWalker<'_, T, Self>, DatabaseError> {
let start = match (key, subkey) {
(Some(key), Some(subkey)) => {
let encoded_key = key.encode();
// encode key and decode it after.
let key: Vec<u8> = key.encode().into();
self.inner
.get_both_range(encoded_key.as_ref(), subkey.encode().as_ref())
.get_both_range(key.as_ref(), subkey.encode().as_ref())
.map_err(|e| DatabaseError::Read(e.into()))?
.map(|val| decoder::<T>((Cow::Borrowed(encoded_key.as_ref()), val)))
.map(|val| decoder::<T>((Cow::Owned(key), val)))
}
(Some(key), None) => {
let encoded_key = key.encode();
let key: Vec<u8> = key.encode().into();
self.inner
.set(encoded_key.as_ref())
.set(key.as_ref())
.map_err(|e| DatabaseError::Read(e.into()))?
.map(|val| decoder::<T>((Cow::Borrowed(encoded_key.as_ref()), val)))
.map(|val| decoder::<T>((Cow::Owned(key), val)))
}
(None, Some(subkey)) => {
if let Some((key, _)) = self.first()? {
let encoded_key = key.encode();
let key: Vec<u8> = key.encode().into();
self.inner
.get_both_range(encoded_key.as_ref(), subkey.encode().as_ref())
.get_both_range(key.as_ref(), subkey.encode().as_ref())
.map_err(|e| DatabaseError::Read(e.into()))?
.map(|val| decoder::<T>((Cow::Borrowed(encoded_key.as_ref()), val)))
.map(|val| decoder::<T>((Cow::Owned(key), val)))
} else {
Some(Err(DatabaseError::Read(MDBXError::NotFound.into())))
}
@@ -268,7 +269,7 @@ impl<T: Table> DbCursorRW<T> for Cursor<RW, T> {
info: e.into(),
operation: DatabaseWriteOperation::CursorUpsert,
table_name: T::NAME,
key: key.into_vec(),
key: key.into(),
}
.into()
})
@@ -290,7 +291,7 @@ impl<T: Table> DbCursorRW<T> for Cursor<RW, T> {
info: e.into(),
operation: DatabaseWriteOperation::CursorInsert,
table_name: T::NAME,
key: key.into_vec(),
key: key.into(),
}
.into()
})
@@ -314,7 +315,7 @@ impl<T: Table> DbCursorRW<T> for Cursor<RW, T> {
info: e.into(),
operation: DatabaseWriteOperation::CursorAppend,
table_name: T::NAME,
key: key.into_vec(),
key: key.into(),
}
.into()
})
@@ -350,7 +351,7 @@ impl<T: DupSort> DbDupCursorRW<T> for Cursor<RW, T> {
info: e.into(),
operation: DatabaseWriteOperation::CursorAppendDup,
table_name: T::NAME,
key: key.into_vec(),
key: key.into(),
}
.into()
})

View File

@@ -6,7 +6,7 @@ use crate::{
DatabaseError,
};
use reth_db_api::{
table::{Compress, DupSort, Encode, IntoVec, Table, TableImporter},
table::{Compress, DupSort, Encode, Table, TableImporter},
transaction::{DbTx, DbTxMut},
};
use reth_libmdbx::{ffi::MDBX_dbi, CommitLatency, Transaction, TransactionKind, WriteFlags, RW};
@@ -387,7 +387,7 @@ impl Tx<RW> {
info: e.into(),
operation: write_operation,
table_name: T::NAME,
key: key.into_vec(),
key: key.into(),
}
.into()
})

View File

@@ -104,16 +104,6 @@ pub enum ProviderError {
/// State is not available for the given block number because it is pruned.
#[error("state at block #{_0} is pruned")]
StateAtBlockPruned(BlockNumber),
/// Block data is not available because history has expired.
///
/// The requested block number is below the earliest available block.
#[error("block #{requested} is not available, history has expired (earliest available: #{earliest_available})")]
BlockExpired {
/// The block number that was requested.
requested: BlockNumber,
/// The earliest available block number.
earliest_available: BlockNumber,
},
/// Provider does not support this particular request.
#[error("this provider does not support this request")]
UnsupportedProvider,

View File

@@ -312,7 +312,7 @@ where
}
/// Position at first key-value pair greater than or equal to specified, return both key and
/// data, and the return code depends on an exact match.
/// data, and the return code depends on a exact match.
///
/// For non DupSort-ed collections this works the same as [`Self::set_range()`], but returns
/// [false] if key found exactly and [true] if greater key was found.

View File

@@ -223,11 +223,7 @@ where
let mut flags: c_uint = 0;
unsafe {
self.txn_execute(|txn| {
// `mdbx_dbi_flags_ex` requires `status` to be a non-NULL ptr, otherwise it will
// return an EINVAL and panic below, so we just provide a placeholder variable
// which we discard immediately.
let mut _status: c_uint = 0;
mdbx_result(ffi::mdbx_dbi_flags_ex(txn, dbi, &mut flags, &mut _status))
mdbx_result(ffi::mdbx_dbi_flags_ex(txn, dbi, &mut flags, ptr::null_mut()))
})??;
}

View File

@@ -37,12 +37,12 @@ impl<H: NippyJarHeader> NippyJarChecker<H> {
Self { jar, data_file: None, offsets_file: None }
}
/// It will throw an error if the [`NippyJar`] is in an inconsistent state.
/// It will throw an error if the [`NippyJar`] is in a inconsistent state.
pub fn check_consistency(&mut self) -> Result<(), NippyJarError> {
self.handle_consistency(ConsistencyFailStrategy::ThrowError)
}
/// It will attempt to heal if the [`NippyJar`] is in an inconsistent state.
/// It will attempt to heal if the [`NippyJar`] is in a inconsistent state.
///
/// **ATTENTION**: disk commit should be handled externally by consuming `Self`
pub fn ensure_consistency(&mut self) -> Result<(), NippyJarError> {

View File

@@ -85,7 +85,6 @@ rand.workspace = true
tokio = { workspace = true, features = ["sync", "macros", "rt-multi-thread"] }
[features]
edge = ["reth-storage-api/edge", "rocksdb"]
rocksdb = ["dep:rocksdb"]
test-utils = [
"reth-db/test-utils",

View File

@@ -13,7 +13,7 @@ use crate::{
providers::{history_info, HistoryInfo, StaticFileProvider, StaticFileProviderRWRefMut},
StaticFileProviderFactory,
};
use alloy_primitives::{map::HashMap, Address, BlockNumber, TxHash, TxNumber, B256};
use alloy_primitives::{map::HashMap, Address, BlockNumber, TxHash, TxNumber};
use rayon::slice::ParallelSliceMut;
use reth_db::{
cursor::{DbCursorRO, DbDupCursorRW},
@@ -512,71 +512,14 @@ where
Self::RocksDB(batch) => batch.delete::<tables::StoragesHistory>(key),
}
}
/// Appends a storage history entry (for first sync - more efficient).
pub fn append_storage_history(
&mut self,
key: StorageShardedKey,
value: &BlockNumberList,
) -> ProviderResult<()> {
match self {
Self::Database(cursor) => Ok(cursor.append(key, value)?),
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
#[cfg(all(unix, feature = "rocksdb"))]
Self::RocksDB(batch) => batch.put::<tables::StoragesHistory>(key, value),
}
}
/// Upserts a storage history entry (for incremental sync).
pub fn upsert_storage_history(
&mut self,
key: StorageShardedKey,
value: &BlockNumberList,
) -> ProviderResult<()> {
match self {
Self::Database(cursor) => Ok(cursor.upsert(key, value)?),
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
#[cfg(all(unix, feature = "rocksdb"))]
Self::RocksDB(batch) => batch.put::<tables::StoragesHistory>(key, value),
}
}
/// Gets the last shard for an address and storage key (keyed with `u64::MAX`).
pub fn get_last_storage_history_shard(
&mut self,
address: Address,
storage_key: B256,
) -> ProviderResult<Option<BlockNumberList>> {
let key = StorageShardedKey::last(address, storage_key);
match self {
Self::Database(cursor) => Ok(cursor.seek_exact(key)?.map(|(_, v)| v)),
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
#[cfg(all(unix, feature = "rocksdb"))]
Self::RocksDB(batch) => batch.get::<tables::StoragesHistory>(key),
}
}
}
impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N>
where
CURSOR: DbCursorRW<tables::AccountsHistory> + DbCursorRO<tables::AccountsHistory>,
{
/// Appends an account history entry (for first sync - more efficient).
pub fn append_account_history(
&mut self,
key: ShardedKey<Address>,
value: &BlockNumberList,
) -> ProviderResult<()> {
match self {
Self::Database(cursor) => Ok(cursor.append(key, value)?),
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
#[cfg(all(unix, feature = "rocksdb"))]
Self::RocksDB(batch) => batch.put::<tables::AccountsHistory>(key, value),
}
}
/// Upserts an account history entry (for incremental sync).
pub fn upsert_account_history(
/// Puts an account history entry.
pub fn put_account_history(
&mut self,
key: ShardedKey<Address>,
value: &BlockNumberList,
@@ -589,21 +532,6 @@ where
}
}
/// Gets the last shard for an address (keyed with `u64::MAX`).
pub fn get_last_account_history_shard(
&mut self,
address: Address,
) -> ProviderResult<Option<BlockNumberList>> {
match self {
Self::Database(cursor) => {
Ok(cursor.seek_exact(ShardedKey::last(address))?.map(|(_, v)| v))
}
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
#[cfg(all(unix, feature = "rocksdb"))]
Self::RocksDB(batch) => batch.get::<tables::AccountsHistory>(ShardedKey::last(address)),
}
}
/// Deletes an account history entry.
pub fn delete_account_history(&mut self, key: ShardedKey<Address>) -> ProviderResult<()> {
match self {
@@ -1338,8 +1266,8 @@ mod rocksdb_tests {
for (highest_block, blocks) in shards {
let key = ShardedKey::new(address, *highest_block);
let value = IntegerList::new(blocks.clone()).unwrap();
mdbx_writer.upsert_account_history(key.clone(), &value).unwrap();
rocks_writer.upsert_account_history(key, &value).unwrap();
mdbx_writer.put_account_history(key.clone(), &value).unwrap();
rocks_writer.put_account_history(key, &value).unwrap();
}
// Commit both backends
@@ -1351,8 +1279,7 @@ mod rocksdb_tests {
// Run queries against both backends using EitherReader
let mdbx_ro = factory.database_provider_ro().unwrap();
// Use `with_assume_history_complete()` since both backends have identical data
let rocks_tx = rocks_provider.tx().with_assume_history_complete();
let rocks_tx = rocks_provider.tx();
for (i, query) in queries.iter().enumerate() {
// MDBX query via EitherReader
@@ -1444,8 +1371,7 @@ mod rocksdb_tests {
// Run queries against both backends using EitherReader
let mdbx_ro = factory.database_provider_ro().unwrap();
// Use `with_assume_history_complete()` since both backends have identical data
let rocks_tx = rocks_provider.tx().with_assume_history_complete();
let rocks_tx = rocks_provider.tx();
for (i, query) in queries.iter().enumerate() {
// MDBX query via EitherReader

View File

@@ -560,12 +560,42 @@ impl<TX: DbTx + DbTxMut + 'static, N: NodeTypesForProvider> DatabaseProvider<TX,
// Write all trie updates in a single batch.
// This reduces cursor open/close overhead from N calls to 1.
// Uses hybrid algorithm: extend_ref for small batches, k-way merge for large.
if save_mode.with_state() {
let start = Instant::now();
const MERGE_BATCH_THRESHOLD: usize = 30;
// Blocks are oldest-to-newest, merge_batch expects newest-to-oldest.
let merged =
TrieUpdatesSorted::merge_batch(blocks.iter().rev().map(|b| b.trie_updates()));
let start = Instant::now();
let num_blocks = blocks.len();
let merged = if num_blocks == 0 {
TrieUpdatesSorted::default()
} else if num_blocks == 1 {
// Single block: use directly (Arc::try_unwrap avoids clone if refcount is 1)
match Arc::try_unwrap(blocks[0].trie_updates()) {
Ok(owned) => owned,
Err(arc) => (*arc).clone(),
}
} else if num_blocks < MERGE_BATCH_THRESHOLD {
// Small k: extend_ref with Arc::make_mut (copy-on-write).
// Blocks are oldest-to-newest, iterate forward so newest overrides.
let mut blocks_iter = blocks.iter();
let mut result = blocks_iter.next().expect("non-empty").trie_updates();
for block in blocks_iter {
Arc::make_mut(&mut result).extend_ref(block.trie_updates().as_ref());
}
match Arc::try_unwrap(result) {
Ok(owned) => owned,
Err(arc) => (*arc).clone(),
}
} else {
// Large k: k-way merge is faster (O(n log k)).
// Collect Arcs first to extend lifetime, then pass refs.
// Blocks are oldest-to-newest, merge_batch expects newest-to-oldest.
let arcs: Vec<_> = blocks.iter().rev().map(|b| b.trie_updates()).collect();
TrieUpdatesSorted::merge_batch(arcs.iter().map(|arc| arc.as_ref()))
};
if !merged.is_empty() {
self.write_trie_updates_sorted(&merged)?;
@@ -1504,17 +1534,10 @@ impl<TX: DbTx + 'static, N: NodeTypesForProvider> BlockReader for DatabaseProvid
/// If the header for this block is not found, this returns `None`.
/// If the header is found, but the transactions either do not exist, or are not indexed, this
/// will return None.
///
/// Returns an error if the requested block is below the earliest available history.
fn block(&self, id: BlockHashOrNumber) -> ProviderResult<Option<Self::Block>> {
if let Some(number) = self.convert_hash_or_number(id)? {
let earliest_available = self.static_file_provider.earliest_history_height();
if number < earliest_available {
return Err(ProviderError::BlockExpired { requested: number, earliest_available })
}
let Some(header) = self.header_by_number(number)? else { return Ok(None) };
if let Some(number) = self.convert_hash_or_number(id)? &&
let Some(header) = self.header_by_number(number)?
{
// If the body indices are not found, this means that the transactions either do not
// exist in the database yet, or they do exit but are not indexed.
// If they exist but are not indexed, we don't have enough
@@ -2946,33 +2969,25 @@ impl<TX: DbTxMut + DbTx + 'static, N: NodeTypes> HistoryWriter for DatabaseProvi
.into_iter()
.map(|(index, account)| (account.address, *index))
.collect::<Vec<_>>();
last_indices.sort_unstable_by_key(|(a, _)| *a);
last_indices.sort_by_key(|(a, _)| *a);
if self.cached_storage_settings().account_history_in_rocksdb {
#[cfg(all(unix, feature = "rocksdb"))]
{
let batch = self.rocksdb_provider.unwind_account_history_indices(&last_indices)?;
self.pending_rocksdb_batches.lock().push(batch);
}
} else {
// Unwind the account history index in MDBX.
let mut cursor = self.tx.cursor_write::<tables::AccountsHistory>()?;
for &(address, rem_index) in &last_indices {
let partial_shard = unwind_history_shards::<_, tables::AccountsHistory, _>(
&mut cursor,
// Unwind the account history index.
let mut cursor = self.tx.cursor_write::<tables::AccountsHistory>()?;
for &(address, rem_index) in &last_indices {
let partial_shard = unwind_history_shards::<_, tables::AccountsHistory, _>(
&mut cursor,
ShardedKey::last(address),
rem_index,
|sharded_key| sharded_key.key == address,
)?;
// Check the last returned partial shard.
// If it's not empty, the shard needs to be reinserted.
if !partial_shard.is_empty() {
cursor.insert(
ShardedKey::last(address),
rem_index,
|sharded_key| sharded_key.key == address,
&BlockNumberList::new_pre_sorted(partial_shard),
)?;
// Check the last returned partial shard.
// If it's not empty, the shard needs to be reinserted.
if !partial_shard.is_empty() {
cursor.insert(
ShardedKey::last(address),
&BlockNumberList::new_pre_sorted(partial_shard),
)?;
}
}
}
@@ -3012,35 +3027,25 @@ impl<TX: DbTxMut + DbTx + 'static, N: NodeTypes> HistoryWriter for DatabaseProvi
.collect::<Vec<_>>();
storage_changesets.sort_by_key(|(address, key, _)| (*address, *key));
if self.cached_storage_settings().storages_history_in_rocksdb {
#[cfg(all(unix, feature = "rocksdb"))]
{
let batch =
self.rocksdb_provider.unwind_storage_history_indices(&storage_changesets)?;
self.pending_rocksdb_batches.lock().push(batch);
}
} else {
// Unwind the storage history index in MDBX.
let mut cursor = self.tx.cursor_write::<tables::StoragesHistory>()?;
for &(address, storage_key, rem_index) in &storage_changesets {
let partial_shard = unwind_history_shards::<_, tables::StoragesHistory, _>(
&mut cursor,
StorageShardedKey::last(address, storage_key),
rem_index,
|storage_sharded_key| {
storage_sharded_key.address == address &&
storage_sharded_key.sharded_key.key == storage_key
},
)?;
let mut cursor = self.tx.cursor_write::<tables::StoragesHistory>()?;
for &(address, storage_key, rem_index) in &storage_changesets {
let partial_shard = unwind_history_shards::<_, tables::StoragesHistory, _>(
&mut cursor,
StorageShardedKey::last(address, storage_key),
rem_index,
|storage_sharded_key| {
storage_sharded_key.address == address &&
storage_sharded_key.sharded_key.key == storage_key
},
)?;
// Check the last returned partial shard.
// If it's not empty, the shard needs to be reinserted.
if !partial_shard.is_empty() {
cursor.insert(
StorageShardedKey::last(address, storage_key),
&BlockNumberList::new_pre_sorted(partial_shard),
)?;
}
// Check the last returned partial shard.
// If it's not empty, the shard needs to be reinserted.
if !partial_shard.is_empty() {
cursor.insert(
StorageShardedKey::last(address, storage_key),
&BlockNumberList::new_pre_sorted(partial_shard),
)?;
}
}

View File

@@ -38,9 +38,7 @@ pub use consistent::ConsistentProvider;
#[cfg_attr(not(all(unix, feature = "rocksdb")), path = "rocksdb_stub.rs")]
pub(crate) mod rocksdb;
pub use rocksdb::{
RocksDBBatch, RocksDBBuilder, RocksDBProvider, RocksDBRawIter, RocksDBTableStats, RocksTx,
};
pub use rocksdb::{RocksDBBatch, RocksDBBuilder, RocksDBProvider, RocksTx};
/// Helper trait to bound [`NodeTypes`] so that combined with database they satisfy
/// [`ProviderNodeTypes`].

View File

@@ -6,7 +6,7 @@ use reth_db::Tables;
use reth_metrics::Metrics;
use strum::{EnumIter, IntoEnumIterator};
pub(super) const ROCKSDB_TABLES: &[&str] = &[
const ROCKSDB_TABLES: &[&str] = &[
Tables::TransactionHashNumbers.name(),
Tables::StoragesHistory.name(),
Tables::AccountsHistory.name(),

View File

@@ -5,6 +5,4 @@ mod metrics;
mod provider;
pub(crate) use provider::{PendingRocksDBBatches, RocksDBWriteCtx};
pub use provider::{
RocksDBBatch, RocksDBBuilder, RocksDBProvider, RocksDBRawIter, RocksDBTableStats, RocksTx,
};
pub use provider::{RocksDBBatch, RocksDBBuilder, RocksDBProvider, RocksTx};

File diff suppressed because it is too large Load Diff

View File

@@ -5,9 +5,8 @@
//! All method calls are cfg-guarded in the calling code, so only type definitions are needed here.
use alloy_primitives::BlockNumber;
use metrics::Label;
use parking_lot::Mutex;
use reth_db_api::{database_metrics::DatabaseMetrics, models::StorageSettings};
use reth_db_api::models::StorageSettings;
use reth_prune_types::PruneMode;
use reth_storage_errors::{db::LogLevel, provider::ProviderResult};
use std::{path::Path, sync::Arc};
@@ -15,23 +14,6 @@ use std::{path::Path, sync::Arc};
/// Pending `RocksDB` batches type alias (stub - uses unit type).
pub(crate) type PendingRocksDBBatches = Arc<Mutex<Vec<()>>>;
/// Statistics for a single `RocksDB` table (column family) - stub.
#[derive(Debug, Clone)]
pub struct RocksDBTableStats {
/// Size of SST files on disk in bytes.
pub sst_size_bytes: u64,
/// Size of memtables in memory in bytes.
pub memtable_size_bytes: u64,
/// Name of the table/column family.
pub name: String,
/// Estimated number of keys in the table.
pub estimated_num_keys: u64,
/// Estimated size of live data in bytes (SST files + memtables).
pub estimated_size_bytes: u64,
/// Estimated bytes pending compaction (reclaimable space).
pub pending_compaction_bytes: u64,
}
/// Context for `RocksDB` block writes (stub).
#[derive(Debug, Clone)]
#[allow(dead_code)]
@@ -74,19 +56,6 @@ impl RocksDBProvider {
) -> ProviderResult<Option<BlockNumber>> {
Ok(None)
}
/// Returns statistics for all column families in the database (stub implementation).
///
/// Returns an empty vector since there is no `RocksDB` when the feature is disabled.
pub const fn table_stats(&self) -> Vec<RocksDBTableStats> {
Vec::new()
}
}
impl DatabaseMetrics for RocksDBProvider {
fn gauge_metrics(&self) -> Vec<(&'static str, f64, Vec<Label>)> {
vec![]
}
}
/// A stub batch writer for `RocksDB`.
@@ -133,11 +102,6 @@ impl RocksDBBuilder {
self
}
/// Sets read-only mode (stub implementation).
pub const fn with_read_only(self, _read_only: bool) -> Self {
self
}
/// Build the `RocksDB` provider (stub implementation).
pub const fn build(self) -> ProviderResult<RocksDBProvider> {
Ok(RocksDBProvider)
@@ -147,7 +111,3 @@ impl RocksDBBuilder {
/// A stub transaction for `RocksDB`.
#[derive(Debug)]
pub struct RocksTx;
/// A stub raw iterator for `RocksDB`.
#[derive(Debug)]
pub struct RocksDBRawIter;

View File

@@ -291,7 +291,7 @@ impl<Provider: DBProvider + ChangeSetReader + BlockNumReader> StateRootProvider
fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult<B256> {
let mut revert_state = self.revert_state()?;
let hashed_state_sorted = hashed_state.into_sorted();
revert_state.extend_ref_and_sort(&hashed_state_sorted);
revert_state.extend_ref(&hashed_state_sorted);
Ok(StateRoot::overlay_root(self.tx(), &revert_state)?)
}
@@ -306,7 +306,7 @@ impl<Provider: DBProvider + ChangeSetReader + BlockNumReader> StateRootProvider
) -> ProviderResult<(B256, TrieUpdates)> {
let mut revert_state = self.revert_state()?;
let hashed_state_sorted = hashed_state.into_sorted();
revert_state.extend_ref_and_sort(&hashed_state_sorted);
revert_state.extend_ref(&hashed_state_sorted);
Ok(StateRoot::overlay_root_with_updates(self.tx(), &revert_state)?)
}

View File

@@ -129,8 +129,6 @@ impl<F> OverlayStateProviderFactory<F> {
/// This overlay will be applied on top of any reverts applied via `with_block_hash`.
pub fn with_overlay_source(mut self, source: Option<OverlaySource>) -> Self {
self.overlay_source = source;
// Clear the overlay cache since we've updated the source.
self.overlay_cache = Default::default();
self
}
@@ -139,8 +137,6 @@ impl<F> OverlayStateProviderFactory<F> {
/// Convenience method that wraps the lazy overlay in `OverlaySource::Lazy`.
pub fn with_lazy_overlay(mut self, lazy_overlay: Option<LazyOverlay>) -> Self {
self.overlay_source = lazy_overlay.map(OverlaySource::Lazy);
// Clear the overlay cache since we've updated the source.
self.overlay_cache = Default::default();
self
}
@@ -156,8 +152,6 @@ impl<F> OverlayStateProviderFactory<F> {
trie: Arc::new(TrieUpdatesSorted::default()),
state,
});
// Clear the overlay cache since we've updated the source.
self.overlay_cache = Default::default();
}
self
}
@@ -169,12 +163,12 @@ impl<F> OverlayStateProviderFactory<F> {
pub fn with_extended_hashed_state_overlay(mut self, other: HashedPostStateSorted) -> Self {
match &mut self.overlay_source {
Some(OverlaySource::Immediate { state, .. }) => {
Arc::make_mut(state).extend_ref_and_sort(&other);
Arc::make_mut(state).extend_ref(&other);
}
Some(OverlaySource::Lazy(lazy)) => {
// Resolve lazy overlay and convert to immediate with extension
let (trie, mut state) = lazy.as_overlay();
Arc::make_mut(&mut state).extend_ref_and_sort(&other);
Arc::make_mut(&mut state).extend_ref(&other);
self.overlay_source = Some(OverlaySource::Immediate { trie, state });
}
None => {
@@ -184,8 +178,6 @@ impl<F> OverlayStateProviderFactory<F> {
});
}
}
// Clear the overlay cache since we've updated the source.
self.overlay_cache = Default::default();
self
}
}
@@ -350,7 +342,7 @@ where
let trie_updates = if trie_reverts.is_empty() {
overlay_trie
} else if !overlay_trie.is_empty() {
trie_reverts.extend_ref_and_sort(&overlay_trie);
trie_reverts.extend_ref(&overlay_trie);
Arc::new(trie_reverts)
} else {
Arc::new(trie_reverts)
@@ -359,7 +351,7 @@ where
let hashed_state_updates = if hashed_state_reverts.is_empty() {
overlay_state
} else if !overlay_state.is_empty() {
hashed_state_reverts.extend_ref_and_sort(&overlay_state);
hashed_state_reverts.extend_ref(&overlay_state);
Arc::new(hashed_state_reverts)
} else {
Arc::new(hashed_state_reverts)

View File

@@ -528,7 +528,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
}
/// Writes headers for all blocks to the static file segment.
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
#[instrument(level = "debug", target = "providers::db", skip_all)]
fn write_headers(
w: &mut StaticFileProviderRWRefMut<'_, N>,
blocks: &[ExecutedBlock<N>],
@@ -541,7 +541,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
}
/// Writes transactions for all blocks to the static file segment.
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
#[instrument(level = "debug", target = "providers::db", skip_all)]
fn write_transactions(
w: &mut StaticFileProviderRWRefMut<'_, N>,
blocks: &[ExecutedBlock<N>],
@@ -558,7 +558,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
}
/// Writes transaction senders for all blocks to the static file segment.
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
#[instrument(level = "debug", target = "providers::db", skip_all)]
fn write_transaction_senders(
w: &mut StaticFileProviderRWRefMut<'_, N>,
blocks: &[ExecutedBlock<N>],
@@ -575,7 +575,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
}
/// Writes receipts for all blocks to the static file segment.
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
#[instrument(level = "debug", target = "providers::db", skip_all)]
fn write_receipts(
w: &mut StaticFileProviderRWRefMut<'_, N>,
blocks: &[ExecutedBlock<N>],
@@ -602,7 +602,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
}
/// Writes account changesets for all blocks to the static file segment.
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
#[instrument(level = "debug", target = "providers::db", skip_all)]
fn write_account_changesets(
w: &mut StaticFileProviderRWRefMut<'_, N>,
blocks: &[ExecutedBlock<N>],
@@ -647,7 +647,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
///
/// This spawns separate threads for each segment type and each thread calls `sync_all()` on its
/// writer when done.
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
#[instrument(level = "debug", target = "providers::db", skip_all)]
pub fn write_blocks_data(
&self,
blocks: &[ExecutedBlock<N>],

View File

@@ -1,7 +1,4 @@
use crate::{
either_writer::{RawRocksDBBatch, RocksBatchArg, RocksTxRefArg},
providers::RocksDBProvider,
};
use crate::{either_writer::RocksTxRefArg, providers::RocksDBProvider};
use reth_storage_errors::provider::ProviderResult;
/// `RocksDB` provider factory.
@@ -34,28 +31,4 @@ pub trait RocksDBProviderFactory {
#[cfg(not(all(unix, feature = "rocksdb")))]
f(())
}
/// Executes a closure with a `RocksDB` batch, automatically registering it for commit.
///
/// This helper encapsulates all the cfg-gated `RocksDB` batch handling.
fn with_rocksdb_batch<F, R>(&self, f: F) -> ProviderResult<R>
where
F: FnOnce(RocksBatchArg<'_>) -> ProviderResult<(R, Option<RawRocksDBBatch>)>,
{
#[cfg(all(unix, feature = "rocksdb"))]
{
let rocksdb = self.rocksdb_provider();
let batch = rocksdb.batch();
let (result, raw_batch) = f(batch)?;
if let Some(b) = raw_batch {
self.set_pending_rocksdb_batch(b);
}
Ok(result)
}
#[cfg(not(all(unix, feature = "rocksdb")))]
{
let (result, _) = f(())?;
Ok(result)
}
}
}

View File

@@ -36,7 +36,6 @@ serde_json = { workspace = true, optional = true }
[features]
default = ["std"]
edge = ["reth-db-api/edge"]
std = [
"reth-chainspec/std",
"alloy-consensus/std",

View File

@@ -746,18 +746,6 @@ impl<T: PoolTransaction> Default for AllPoolTransactions<T> {
}
}
impl<T: PoolTransaction> IntoIterator for AllPoolTransactions<T> {
type Item = Arc<ValidPoolTransaction<T>>;
type IntoIter = std::iter::Chain<
std::vec::IntoIter<Arc<ValidPoolTransaction<T>>>,
std::vec::IntoIter<Arc<ValidPoolTransaction<T>>>,
>;
fn into_iter(self) -> Self::IntoIter {
self.pending.into_iter().chain(self.queued)
}
}
/// Represents transactions that were propagated over the network.
#[derive(Debug, Clone, Eq, PartialEq, Default)]
pub struct PropagatedTransactions(pub HashMap<TxHash, Vec<PropagateKind>>);

View File

@@ -6,7 +6,7 @@ use crate::{
utils::{extend_sorted_vec, kway_merge_sorted},
KeyHasher, MultiProofTargets, Nibbles,
};
use alloc::{borrow::Cow, sync::Arc, vec::Vec};
use alloc::{borrow::Cow, vec::Vec};
use alloy_primitives::{
keccak256,
map::{hash_map, B256Map, HashMap, HashSet},
@@ -621,9 +621,7 @@ impl HashedPostStateSorted {
/// Extends this state with contents of another sorted state.
/// Entries in `other` take precedence for duplicate keys.
///
/// Sorts the accounts after extending. Sorts the storage after extending, for each account.
pub fn extend_ref_and_sort(&mut self, other: &Self) {
pub fn extend_ref(&mut self, other: &Self) {
// Extend accounts
extend_sorted_vec(&mut self.accounts, &other.accounts);
@@ -638,44 +636,31 @@ impl HashedPostStateSorted {
/// Batch-merge sorted hashed post states. Iterator yields **newest to oldest**.
///
/// For small batches, uses `extend_ref_and_sort` loop.
/// For large batches, uses k-way merge for O(n log k) complexity.
pub fn merge_batch<T: AsRef<Self> + From<Self>>(iter: impl IntoIterator<Item = T>) -> T {
const THRESHOLD: usize = 30;
let items: alloc::vec::Vec<_> = iter.into_iter().collect();
let k = items.len();
if k == 0 {
return Self::default().into();
}
if k == 1 {
return items.into_iter().next().expect("k == 1");
/// Uses k-way merge for O(n log k) complexity and one-pass accumulation for storages.
pub fn merge_batch<'a>(states: impl IntoIterator<Item = &'a Self>) -> Self {
let states: Vec<_> = states.into_iter().collect();
if states.is_empty() {
return Self::default();
}
if k < THRESHOLD {
// Small k: extend loop, oldest-to-newest so newer overrides older.
let mut iter = items.iter().rev();
let mut acc = iter.next().expect("k > 0").as_ref().clone();
for next in iter {
acc.extend_ref_and_sort(next.as_ref());
}
return acc.into();
}
// Large k: k-way merge.
let accounts = kway_merge_sorted(items.iter().map(|i| i.as_ref().accounts.as_slice()));
let accounts = kway_merge_sorted(states.iter().map(|s| s.accounts.as_slice()));
struct StorageAcc<'a> {
/// Account storage was cleared (e.g., SELFDESTRUCT).
wiped: bool,
/// Stop collecting older slices after seeing a wipe.
sealed: bool,
/// Storage slot slices to merge, ordered newest to oldest.
slices: Vec<&'a [(B256, U256)]>,
}
let mut acc: B256Map<StorageAcc<'_>> = B256Map::default();
for item in &items {
for (addr, storage) in &item.as_ref().storages {
// Accumulate storage slices per address from newest to oldest state.
// Once we see a `wiped` flag, the account was cleared at that point,
// so older storage slots are irrelevant - we "seal" and stop collecting.
for state in &states {
for (addr, storage) in &state.storages {
let entry = acc.entry(*addr).or_insert_with(|| StorageAcc {
wiped: false,
sealed: false,
@@ -702,7 +687,7 @@ impl HashedPostStateSorted {
})
.collect();
Self { accounts, storages }.into()
Self { accounts, storages }
}
/// Clears all accounts and storage data.
@@ -710,36 +695,6 @@ impl HashedPostStateSorted {
self.accounts.clear();
self.storages.clear();
}
/// Parallel batch-merge sorted hashed post states. Slice is **oldest to newest**.
///
/// This is more efficient than sequential `extend_ref` calls when merging many states,
/// as it processes all states in parallel with tree reduction using divide-and-conquer.
#[cfg(feature = "rayon")]
pub fn merge_parallel(states: &[Arc<Self>]) -> Self {
fn parallel_merge_tree(states: &[Arc<HashedPostStateSorted>]) -> HashedPostStateSorted {
match states.len() {
0 => HashedPostStateSorted::default(),
1 => states[0].as_ref().clone(),
2 => {
let mut acc = states[0].as_ref().clone();
acc.extend_ref_and_sort(&states[1]);
acc
}
n => {
let mid = n / 2;
let (mut left, right) = rayon::join(
|| parallel_merge_tree(&states[..mid]),
|| parallel_merge_tree(&states[mid..]),
);
left.extend_ref_and_sort(&right);
left
}
}
}
parallel_merge_tree(states)
}
}
impl AsRef<Self> for HashedPostStateSorted {
@@ -1461,7 +1416,7 @@ mod tests {
storages: B256Map::default(),
};
state1.extend_ref_and_sort(&state2);
state1.extend_ref(&state2);
// Check accounts are merged and sorted
assert_eq!(state1.accounts.len(), 6);

View File

@@ -1,6 +1,6 @@
//! Merkle trie proofs.
use crate::{BranchNodeMasksMap, Nibbles, ProofTrieNode, TrieAccount};
use crate::{BranchNodeMasksMap, Nibbles, TrieAccount};
use alloc::{borrow::Cow, vec::Vec};
use alloy_consensus::constants::KECCAK_EMPTY;
use alloy_primitives::{
@@ -267,12 +267,6 @@ impl MultiProof {
self.account_subtree.extend_from(other.account_subtree);
self.branch_node_masks.extend(other.branch_node_masks);
let reserve = if self.storages.is_empty() {
other.storages.len()
} else {
other.storages.len().div_ceil(2)
};
self.storages.reserve(reserve);
for (hashed_address, storage) in other.storages {
match self.storages.entry(hashed_address) {
hash_map::Entry::Occupied(mut entry) => {
@@ -396,12 +390,6 @@ impl DecodedMultiProof {
self.account_subtree.extend_from(other.account_subtree);
self.branch_node_masks.extend(other.branch_node_masks);
let reserve = if self.storages.is_empty() {
other.storages.len()
} else {
other.storages.len().div_ceil(2)
};
self.storages.reserve(reserve);
for (hashed_address, storage) in other.storages {
match self.storages.entry(hashed_address) {
hash_map::Entry::Occupied(mut entry) => {
@@ -443,33 +431,6 @@ impl TryFrom<MultiProof> for DecodedMultiProof {
}
}
/// V2 decoded multiproof which contains the results of both account and storage V2 proof
/// calculations.
#[derive(Clone, Debug, PartialEq, Eq, Default)]
pub struct DecodedMultiProofV2 {
/// Account trie proof nodes
pub account_proofs: Vec<ProofTrieNode>,
/// Storage trie proof nodes indexed by account
pub storage_proofs: B256Map<Vec<ProofTrieNode>>,
}
impl DecodedMultiProofV2 {
/// Returns true if there are no proofs
pub fn is_empty(&self) -> bool {
self.account_proofs.is_empty() && self.storage_proofs.is_empty()
}
/// Appends the given multiproof's data to this one.
///
/// This implementation does not deduplicate redundant proofs.
pub fn extend(&mut self, other: Self) {
self.account_proofs.extend(other.account_proofs);
for (hashed_address, other_storage_proofs) in other.storage_proofs {
self.storage_proofs.entry(hashed_address).or_default().extend(other_storage_proofs);
}
}
}
/// The merkle multiproof of storage trie.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct StorageMultiProof {

View File

@@ -4,7 +4,6 @@ use crate::{
};
use alloc::{
collections::{btree_map::BTreeMap, btree_set::BTreeSet},
sync::Arc,
vec::Vec,
};
use alloy_primitives::{
@@ -207,7 +206,7 @@ impl TrieUpdates {
}
/// Converts trie updates into [`TrieUpdatesSortedRef`].
pub fn into_sorted_ref(&self) -> TrieUpdatesSortedRef<'_> {
pub fn into_sorted_ref<'a>(&'a self) -> TrieUpdatesSortedRef<'a> {
let mut account_nodes = self.account_nodes.iter().collect::<Vec<_>>();
account_nodes.sort_unstable_by(|a, b| a.0.cmp(b.0));
@@ -217,7 +216,7 @@ impl TrieUpdates {
storage_tries: self
.storage_tries
.iter()
.map(|m| (*m.0, m.1.into_sorted_ref()))
.map(|m| (*m.0, m.1.into_sorted_ref().clone()))
.collect(),
}
}
@@ -606,10 +605,7 @@ impl TrieUpdatesSorted {
/// This merges the account nodes and storage tries from `other` into `self`.
/// Account nodes are merged and re-sorted, with `other`'s values taking precedence
/// for duplicate keys.
///
/// Sorts the account nodes after extending. Sorts the storage tries after extending, for each
/// storage trie.
pub fn extend_ref_and_sort(&mut self, other: &Self) {
pub fn extend_ref(&mut self, other: &Self) {
// Extend account nodes
extend_sorted_vec(&mut self.account_nodes, &other.account_nodes);
@@ -630,57 +626,48 @@ impl TrieUpdatesSorted {
/// Batch-merge sorted trie updates. Iterator yields **newest to oldest**.
///
/// For small batches, uses `extend_ref_and_sort` loop.
/// For large batches, uses k-way merge for O(n log k) complexity.
pub fn merge_batch<T: AsRef<Self> + From<Self>>(iter: impl IntoIterator<Item = T>) -> T {
const THRESHOLD: usize = 30;
let items: alloc::vec::Vec<_> = iter.into_iter().collect();
let k = items.len();
if k == 0 {
return Self::default().into();
}
if k == 1 {
return items.into_iter().next().expect("k == 1");
/// This is more efficient than repeated `extend_ref` calls for large batches,
/// using k-way merge for O(n log k) complexity instead of O(n * k).
pub fn merge_batch<'a>(updates: impl IntoIterator<Item = &'a Self>) -> Self {
let updates: Vec<_> = updates.into_iter().collect();
if updates.is_empty() {
return Self::default();
}
if k < THRESHOLD {
// Small k: extend loop, oldest-to-newest so newer overrides older.
let mut iter = items.iter().rev();
let mut acc = iter.next().expect("k > 0").as_ref().clone();
for next in iter {
acc.extend_ref_and_sort(next.as_ref());
}
return acc.into();
}
// Large k: k-way merge.
let account_nodes =
kway_merge_sorted(items.iter().map(|i| i.as_ref().account_nodes.as_slice()));
// Merge account nodes using k-way merge. Newest (index 0) takes precedence.
let account_nodes = kway_merge_sorted(updates.iter().map(|u| u.account_nodes.as_slice()));
// Accumulator for collecting storage trie slices per address.
// We process updates newest-to-oldest and stop collecting for an address
// once we hit a "deleted" storage (sealed=true), since older data is irrelevant.
struct StorageAcc<'a> {
/// Storage trie was deleted (account removed or cleared).
is_deleted: bool,
/// Stop collecting older slices after seeing a deletion.
sealed: bool,
/// Storage trie node slices to merge, ordered newest to oldest.
slices: Vec<&'a [(Nibbles, Option<BranchNodeCompact>)]>,
}
let mut acc: B256Map<StorageAcc<'_>> = B256Map::default();
for item in &items {
for (addr, storage) in &item.as_ref().storage_tries {
// Collect storage slices per address, respecting deletion boundaries
for update in &updates {
for (addr, storage) in &update.storage_tries {
let entry = acc.entry(*addr).or_insert_with(|| StorageAcc {
is_deleted: false,
sealed: false,
slices: Vec::new(),
});
// Skip if we already hit a deletion for this address (older data is irrelevant)
if entry.sealed {
continue;
}
entry.slices.push(storage.storage_nodes.as_slice());
// If this storage was deleted, mark as deleted and seal to ignore older updates
if storage.is_deleted {
entry.is_deleted = true;
entry.sealed = true;
@@ -688,6 +675,7 @@ impl TrieUpdatesSorted {
}
}
// Merge each address's storage slices using k-way merge
let storage_tries = acc
.into_iter()
.map(|(addr, entry)| {
@@ -696,37 +684,7 @@ impl TrieUpdatesSorted {
})
.collect();
Self { account_nodes, storage_tries }.into()
}
/// Parallel batch-merge sorted trie updates. Slice is **oldest to newest**.
///
/// This is more efficient than sequential `extend_ref` calls when merging many updates,
/// as it processes all updates in parallel with tree reduction using divide-and-conquer.
#[cfg(feature = "rayon")]
pub fn merge_parallel(updates: &[Arc<Self>]) -> Self {
fn parallel_merge_tree(updates: &[Arc<TrieUpdatesSorted>]) -> TrieUpdatesSorted {
match updates.len() {
0 => TrieUpdatesSorted::default(),
1 => updates[0].as_ref().clone(),
2 => {
let mut acc = updates[0].as_ref().clone();
acc.extend_ref_and_sort(&updates[1]);
acc
}
n => {
let mid = n / 2;
let (mut left, right) = rayon::join(
|| parallel_merge_tree(&updates[..mid]),
|| parallel_merge_tree(&updates[mid..]),
);
left.extend_ref_and_sort(&right);
left
}
}
}
parallel_merge_tree(updates)
Self { account_nodes, storage_tries }
}
}
@@ -876,7 +834,7 @@ mod tests {
// Test extending with empty updates
let mut updates1 = TrieUpdatesSorted::default();
let updates2 = TrieUpdatesSorted::default();
updates1.extend_ref_and_sort(&updates2);
updates1.extend_ref(&updates2);
assert_eq!(updates1.account_nodes.len(), 0);
assert_eq!(updates1.storage_tries.len(), 0);
@@ -895,7 +853,7 @@ mod tests {
],
storage_tries: B256Map::default(),
};
updates1.extend_ref_and_sort(&updates2);
updates1.extend_ref(&updates2);
assert_eq!(updates1.account_nodes.len(), 3);
// Should be sorted: 0x01, 0x02, 0x03
assert_eq!(updates1.account_nodes[0].0, Nibbles::from_nibbles_unchecked([0x01]));
@@ -931,7 +889,7 @@ mod tests {
(hashed_address2, storage_trie1),
]),
};
updates1.extend_ref_and_sort(&updates2);
updates1.extend_ref(&updates2);
assert_eq!(updates1.storage_tries.len(), 2);
assert!(updates1.storage_tries.contains_key(&hashed_address1));
assert!(updates1.storage_tries.contains_key(&hashed_address2));

View File

@@ -87,7 +87,7 @@ where
// This reverts all changes from db tip back to just after block-1 was processed
let mut cumulative_state_revert_prev = cumulative_state_revert.clone();
cumulative_state_revert_prev.extend_ref_and_sort(&individual_state_revert);
cumulative_state_revert_prev.extend_ref(&individual_state_revert);
// Step 2: Calculate cumulative trie updates revert for block-1
// This gives us the trie state as it was after block-1 was processed
@@ -469,7 +469,7 @@ impl ChangesetCache {
// Since we iterate newest to oldest, older values are added last
// and overwrite any conflicting newer values (oldest changeset values take
// precedence).
accumulated_reverts.extend_ref_and_sort(&changesets);
accumulated_reverts.extend_ref(&changesets);
}
let elapsed = timer.elapsed();

View File

@@ -14,7 +14,6 @@ workspace = true
[dependencies]
# reth
reth-execution-errors.workspace = true
reth-primitives-traits.workspace = true
reth-provider.workspace = true
reth-storage-errors.workspace = true
reth-trie-common.workspace = true

View File

@@ -22,12 +22,6 @@ pub mod proof;
pub mod proof_task;
/// Async value encoder for V2 proofs.
pub(crate) mod value_encoder;
/// V2 multiproof targets and chunking.
pub mod targets_v2;
/// Parallel state root metrics.
#[cfg(feature = "metrics")]
pub mod metrics;

View File

@@ -41,7 +41,7 @@ use alloy_primitives::{
use alloy_rlp::{BufMut, Encodable};
use crossbeam_channel::{unbounded, Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
use dashmap::DashMap;
use reth_execution_errors::{SparseTrieError, SparseTrieErrorKind, StateProofError};
use reth_execution_errors::{SparseTrieError, SparseTrieErrorKind};
use reth_provider::{DatabaseProviderROFactory, ProviderError, ProviderResult};
use reth_storage_errors::db::DatabaseError;
use reth_trie::{
@@ -305,17 +305,17 @@ impl ProofWorkerHandle {
self.storage_work_tx
.send(StorageWorkerJob::StorageProof { input, proof_result_sender })
.map_err(|err| {
let error =
ProviderError::other(std::io::Error::other("storage workers unavailable"));
if let StorageWorkerJob::StorageProof { proof_result_sender, .. } = err.0 {
let _ = proof_result_sender.send(StorageProofResultMessage {
hashed_address,
result: Err(DatabaseError::Other(
"storage workers unavailable".to_string(),
)
.into()),
result: Err(ParallelStateRootError::Provider(error.clone())),
});
}
ProviderError::other(std::io::Error::other("storage workers unavailable"))
error
})
}
@@ -432,7 +432,7 @@ where
input: StorageProofInput,
trie_cursor_metrics: &mut TrieCursorMetricsCache,
hashed_cursor_metrics: &mut HashedCursorMetricsCache,
) -> Result<StorageProofResult, StateProofError> {
) -> Result<StorageProofResult, ParallelStateRootError> {
// Consume the input so we can move large collections (e.g. target slots) without cloning.
let StorageProofInput::Legacy {
hashed_address,
@@ -469,13 +469,20 @@ where
.with_added_removed_keys(added_removed_keys)
.with_trie_cursor_metrics(trie_cursor_metrics)
.with_hashed_cursor_metrics(hashed_cursor_metrics)
.storage_multiproof(target_slots);
.storage_multiproof(target_slots)
.map_err(|e| ParallelStateRootError::Other(e.to_string()));
trie_cursor_metrics.record_span("trie_cursor");
hashed_cursor_metrics.record_span("hashed_cursor");
// Decode proof into DecodedStorageMultiProof
let decoded_result =
raw_proof_result.and_then(|raw_proof| raw_proof.try_into().map_err(Into::into))?;
let decoded_result = raw_proof_result.and_then(|raw_proof| {
raw_proof.try_into().map_err(|e: alloy_rlp::Error| {
ParallelStateRootError::Other(format!(
"Failed to decode storage proof for {}: {}",
hashed_address, e
))
})
})?;
trace!(
target: "trie::proof_task",
@@ -495,7 +502,7 @@ where
<Provider as TrieCursorFactory>::StorageTrieCursor<'_>,
<Provider as HashedCursorFactory>::StorageCursor<'_>,
>,
) -> Result<StorageProofResult, StateProofError> {
) -> Result<StorageProofResult, ParallelStateRootError> {
let StorageProofInput::V2 { hashed_address, mut targets } = input else {
panic!("compute_v2_storage_proof only accepts StorageProofInput::V2")
};
@@ -710,12 +717,12 @@ pub struct StorageProofResultMessage {
/// The hashed address this storage proof belongs to
pub(crate) hashed_address: B256,
/// The storage proof calculation result
pub(crate) result: Result<StorageProofResult, StateProofError>,
pub(crate) result: Result<StorageProofResult, ParallelStateRootError>,
}
/// Internal message for storage workers.
#[derive(Debug)]
pub(crate) enum StorageWorkerJob {
enum StorageWorkerJob {
/// Storage proof computation request
StorageProof {
/// Storage proof input parameters
@@ -1555,11 +1562,8 @@ fn dispatch_storage_proofs(
let mut storage_proof_receivers =
B256Map::with_capacity_and_hasher(targets.len(), Default::default());
let mut sorted_targets: Vec<_> = targets.iter().collect();
sorted_targets.sort_unstable_by_key(|(addr, _)| *addr);
// Dispatch all storage proofs to worker pool
for (hashed_address, target_slots) in sorted_targets {
for (hashed_address, target_slots) in targets.iter() {
// Create channel for receiving ProofResultMessage
let (result_tx, result_rx) = crossbeam_channel::unbounded();

View File

@@ -1,148 +0,0 @@
//! V2 multiproof targets and chunking.
use alloy_primitives::{map::B256Map, B256};
use reth_trie::proof_v2;
/// A set of account and storage V2 proof targets. The account and storage targets do not need to
/// necessarily overlap.
#[derive(Debug, Default)]
pub struct MultiProofTargetsV2 {
/// The set of account proof targets to generate proofs for.
pub account_targets: Vec<proof_v2::Target>,
/// The sets of storage proof targets to generate proofs for.
pub storage_targets: B256Map<Vec<proof_v2::Target>>,
}
impl MultiProofTargetsV2 {
/// Returns true is there are no account or storage targets.
pub fn is_empty(&self) -> bool {
self.account_targets.is_empty() && self.storage_targets.is_empty()
}
}
/// An iterator that yields chunks of V2 proof targets of at most `size` account and storage
/// targets.
///
/// Unlike legacy chunking, V2 preserves account targets exactly as they were (with their `min_len`
/// metadata). Account targets must appear in a chunk. Storage targets for those accounts are
/// chunked together, but if they exceed the chunk size, subsequent chunks contain only the
/// remaining storage targets without repeating the account target.
#[derive(Debug)]
pub struct ChunkedMultiProofTargetsV2 {
/// Remaining account targets to process
account_targets: std::vec::IntoIter<proof_v2::Target>,
/// Storage targets by account address
storage_targets: B256Map<Vec<proof_v2::Target>>,
/// Current account being processed (if any storage slots remain)
current_account_storage: Option<(B256, std::vec::IntoIter<proof_v2::Target>)>,
/// Chunk size
size: usize,
}
impl ChunkedMultiProofTargetsV2 {
/// Creates a new chunked iterator for the given targets.
pub fn new(targets: MultiProofTargetsV2, size: usize) -> Self {
Self {
account_targets: targets.account_targets.into_iter(),
storage_targets: targets.storage_targets,
current_account_storage: None,
size,
}
}
}
impl Iterator for ChunkedMultiProofTargetsV2 {
type Item = MultiProofTargetsV2;
fn next(&mut self) -> Option<Self::Item> {
let mut chunk = MultiProofTargetsV2::default();
let mut count = 0;
// First, finish any remaining storage slots from previous account
if let Some((account_addr, ref mut storage_iter)) = self.current_account_storage {
let remaining_capacity = self.size - count;
let slots: Vec<_> = storage_iter.by_ref().take(remaining_capacity).collect();
count += slots.len();
chunk.storage_targets.insert(account_addr, slots);
// If iterator is exhausted, clear current_account_storage
if storage_iter.len() == 0 {
self.current_account_storage = None;
}
}
// Process account targets and their storage
while count < self.size {
let Some(account_target) = self.account_targets.next() else {
break;
};
// Add the account target
chunk.account_targets.push(account_target);
count += 1;
// Check if this account has storage targets
let account_addr = account_target.key();
if let Some(storage_slots) = self.storage_targets.remove(&account_addr) {
let remaining_capacity = self.size - count;
if storage_slots.len() <= remaining_capacity {
// Optimization: We can take all slots, just move the vec
count += storage_slots.len();
chunk.storage_targets.insert(account_addr, storage_slots);
} else {
// We need to split the storage slots
let mut storage_iter = storage_slots.into_iter();
let slots_in_chunk: Vec<_> =
storage_iter.by_ref().take(remaining_capacity).collect();
count += slots_in_chunk.len();
chunk.storage_targets.insert(account_addr, slots_in_chunk);
// Save remaining storage slots for next chunk
self.current_account_storage = Some((account_addr, storage_iter));
break;
}
}
}
// Process any remaining storage-only entries (accounts not in account_targets)
while let Some((account_addr, storage_slots)) = self.storage_targets.iter_mut().next() &&
count < self.size
{
let account_addr = *account_addr;
let storage_slots = std::mem::take(storage_slots);
let remaining_capacity = self.size - count;
// Always remove from the map - if there are remaining slots they go to
// current_account_storage
self.storage_targets.remove(&account_addr);
if storage_slots.len() <= remaining_capacity {
// Optimization: We can take all slots, just move the vec
count += storage_slots.len();
chunk.storage_targets.insert(account_addr, storage_slots);
} else {
// We need to split the storage slots
let mut storage_iter = storage_slots.into_iter();
let slots_in_chunk: Vec<_> =
storage_iter.by_ref().take(remaining_capacity).collect();
chunk.storage_targets.insert(account_addr, slots_in_chunk);
// Save remaining storage slots for next chunk
if storage_iter.len() > 0 {
self.current_account_storage = Some((account_addr, storage_iter));
}
break;
}
}
if chunk.account_targets.is_empty() && chunk.storage_targets.is_empty() {
None
} else {
Some(chunk)
}
}
}

View File

@@ -1,185 +0,0 @@
use crate::proof_task::{
StorageProofInput, StorageProofResult, StorageProofResultMessage, StorageWorkerJob,
};
use alloy_primitives::{map::B256Map, B256};
use alloy_rlp::Encodable;
use core::cell::RefCell;
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
use dashmap::DashMap;
use reth_execution_errors::trie::StateProofError;
use reth_primitives_traits::Account;
use reth_storage_errors::db::DatabaseError;
use reth_trie::{
proof_v2::{DeferredValueEncoder, LeafValueEncoder, Target},
ProofTrieNode,
};
use std::{rc::Rc, sync::Arc};
/// Returned from [`AsyncAccountValueEncoder`], used to track an async storage root calculation.
pub(crate) enum AsyncAccountDeferredValueEncoder {
Dispatched {
hashed_address: B256,
account: Account,
proof_result_rx: Result<CrossbeamReceiver<StorageProofResultMessage>, DatabaseError>,
// None if results shouldn't be retained for this dispatched proof.
storage_proof_results: Option<Rc<RefCell<B256Map<Vec<ProofTrieNode>>>>>,
},
FromCache {
account: Account,
root: B256,
},
}
impl DeferredValueEncoder for AsyncAccountDeferredValueEncoder {
fn encode(self, buf: &mut Vec<u8>) -> Result<(), StateProofError> {
let (account, root) = match self {
Self::Dispatched {
hashed_address,
account,
proof_result_rx,
storage_proof_results,
} => {
let result = proof_result_rx?
.recv()
.map_err(|_| {
StateProofError::Database(DatabaseError::Other(format!(
"Storage proof channel closed for {hashed_address:?}",
)))
})?
.result?;
let StorageProofResult::V2 { root: Some(root), proof } = result else {
panic!("StorageProofResult is not V2 with root: {result:?}")
};
if let Some(storage_proof_results) = storage_proof_results.as_ref() {
storage_proof_results.borrow_mut().insert(hashed_address, proof);
}
(account, root)
}
Self::FromCache { account, root } => (account, root),
};
let account = account.into_trie_account(root);
account.encode(buf);
Ok(())
}
}
/// Implements the [`LeafValueEncoder`] trait for accounts using a [`CrossbeamSender`] to dispatch
/// and compute storage roots asynchronously. Can also accept a set of already dispatched account
/// storage proofs, for cases where it's possible to determine some necessary accounts ahead of
/// time.
pub(crate) struct AsyncAccountValueEncoder {
storage_work_tx: CrossbeamSender<StorageWorkerJob>,
/// Storage proof jobs which were dispatched ahead of time.
dispatched: B256Map<CrossbeamReceiver<StorageProofResultMessage>>,
/// Storage roots which have already been computed. This can be used only if a storage proof
/// wasn't dispatched for an account, otherwise we must consume the proof result.
cached_storage_roots: Arc<DashMap<B256, B256>>,
/// Tracks storage proof results received from the storage workers. [`Rc`] + [`RefCell`] is
/// required because [`DeferredValueEncoder`] cannot have a lifetime.
storage_proof_results: Rc<RefCell<B256Map<Vec<ProofTrieNode>>>>,
}
impl AsyncAccountValueEncoder {
/// Initializes a [`Self`] using a `ProofWorkerHandle` which will be used to calculate storage
/// roots asynchronously.
#[expect(dead_code)]
pub(crate) fn new(
storage_work_tx: CrossbeamSender<StorageWorkerJob>,
dispatched: B256Map<CrossbeamReceiver<StorageProofResultMessage>>,
cached_storage_roots: Arc<DashMap<B256, B256>>,
) -> Self {
Self {
storage_work_tx,
dispatched,
cached_storage_roots,
storage_proof_results: Default::default(),
}
}
/// Consume [`Self`] and return all collected storage proofs which had been dispatched.
///
/// # Panics
///
/// This method panics if any deferred encoders produced by [`Self::deferred_encoder`] have not
/// been dropped.
#[expect(dead_code)]
pub(crate) fn into_storage_proofs(
self,
) -> Result<B256Map<Vec<ProofTrieNode>>, StateProofError> {
let mut storage_proof_results = Rc::into_inner(self.storage_proof_results)
.expect("no deferred encoders are still allocated")
.into_inner();
// Any remaining dispatched proofs need to have their results collected
for (hashed_address, rx) in &self.dispatched {
let result = rx
.recv()
.map_err(|_| {
StateProofError::Database(DatabaseError::Other(format!(
"Storage proof channel closed for {hashed_address:?}",
)))
})?
.result?;
let StorageProofResult::V2 { proof, .. } = result else {
panic!("StorageProofResult is not V2: {result:?}")
};
storage_proof_results.insert(*hashed_address, proof);
}
Ok(storage_proof_results)
}
}
impl LeafValueEncoder for AsyncAccountValueEncoder {
type Value = Account;
type DeferredEncoder = AsyncAccountDeferredValueEncoder;
fn deferred_encoder(
&mut self,
hashed_address: B256,
account: Self::Value,
) -> Self::DeferredEncoder {
// If the proof job has already been dispatched for this account then it's not necessary to
// dispatch another.
if let Some(rx) = self.dispatched.remove(&hashed_address) {
return AsyncAccountDeferredValueEncoder::Dispatched {
hashed_address,
account,
proof_result_rx: Ok(rx),
storage_proof_results: Some(self.storage_proof_results.clone()),
}
}
// If the address didn't have a job dispatched for it then we can assume it has no targets,
// and we only need its root.
// If the root is already calculated then just use it directly
if let Some(root) = self.cached_storage_roots.get(&hashed_address) {
return AsyncAccountDeferredValueEncoder::FromCache { account, root: *root }
}
// Create a proof input which targets a bogus key, so that we calculate the root as a
// side-effect.
let input = StorageProofInput::new(hashed_address, vec![Target::new(B256::ZERO)]);
let (tx, rx) = crossbeam_channel::bounded(1);
let proof_result_rx = self
.storage_work_tx
.send(StorageWorkerJob::StorageProof { input, proof_result_sender: tx })
.map_err(|_| DatabaseError::Other("storage workers unavailable".to_string()))
.map(|_| rx);
AsyncAccountDeferredValueEncoder::Dispatched {
hashed_address,
account,
proof_result_rx,
storage_proof_results: None,
}
}
}

View File

@@ -316,86 +316,6 @@ where
}
}
/// Reveals a V2 decoded multiproof.
///
/// V2 multiproofs use a simpler format where proof nodes are stored as vectors rather than
/// hashmaps, with masks already included in the `ProofTrieNode` structure.
#[instrument(
skip_all,
fields(
account_nodes = multiproof.account_proofs.len(),
storages = multiproof.storage_proofs.len()
)
)]
pub fn reveal_decoded_multiproof_v2(
&mut self,
multiproof: reth_trie_common::DecodedMultiProofV2,
) -> SparseStateTrieResult<()> {
// Reveal the account proof nodes
self.reveal_account_v2_proof_nodes(multiproof.account_proofs)?;
#[cfg(not(feature = "std"))]
// If nostd then serially reveal storage proof nodes for each storage trie
{
for (account, storage_proofs) in multiproof.storage_proofs {
self.reveal_storage_v2_proof_nodes(account, storage_proofs)?;
}
Ok(())
}
#[cfg(feature = "std")]
// If std then reveal storage proofs in parallel
{
use rayon::iter::{ParallelBridge, ParallelIterator};
let retain_updates = self.retain_updates;
// Process all storage trie revealings in parallel, having first removed the
// `reveal_nodes` tracking and `SparseTrie`s for each account from their HashMaps.
// These will be returned after processing.
let results: Vec<_> = multiproof
.storage_proofs
.into_iter()
.map(|(account, storage_proofs)| {
let revealed_nodes = self.storage.take_or_create_revealed_paths(&account);
let trie = self.storage.take_or_create_trie(&account);
(account, storage_proofs, revealed_nodes, trie)
})
.par_bridge()
.map(|(account, storage_proofs, mut revealed_nodes, mut trie)| {
let result = Self::reveal_storage_v2_proof_nodes_inner(
account,
storage_proofs,
&mut revealed_nodes,
&mut trie,
retain_updates,
);
(account, result, revealed_nodes, trie)
})
.collect();
let mut any_err = Ok(());
for (account, result, revealed_nodes, trie) in results {
self.storage.revealed_paths.insert(account, revealed_nodes);
self.storage.tries.insert(account, trie);
if let Ok(_metric_values) = result {
#[cfg(feature = "metrics")]
{
self.metrics
.increment_total_storage_nodes(_metric_values.total_nodes as u64);
self.metrics
.increment_skipped_storage_nodes(_metric_values.skipped_nodes as u64);
}
} else {
any_err = result.map(|_| ());
}
}
any_err
}
}
/// Reveals an account multiproof.
pub fn reveal_account_multiproof(
&mut self,
@@ -442,89 +362,6 @@ where
Ok(())
}
/// Reveals account proof nodes from a V2 proof.
///
/// V2 proofs already include the masks in the `ProofTrieNode` structure,
/// so no separate masks map is needed.
pub fn reveal_account_v2_proof_nodes(
&mut self,
nodes: Vec<ProofTrieNode>,
) -> SparseStateTrieResult<()> {
let FilteredV2ProofNodes { root_node, nodes, new_nodes, metric_values: _metric_values } =
filter_revealed_v2_proof_nodes(nodes, &mut self.revealed_account_paths)?;
#[cfg(feature = "metrics")]
{
self.metrics.increment_total_account_nodes(_metric_values.total_nodes as u64);
self.metrics.increment_skipped_account_nodes(_metric_values.skipped_nodes as u64);
}
if let Some(root_node) = root_node {
trace!(target: "trie::sparse", ?root_node, "Revealing root account node from V2 proof");
let trie =
self.state.reveal_root(root_node.node, root_node.masks, self.retain_updates)?;
trie.reserve_nodes(new_nodes);
trace!(target: "trie::sparse", total_nodes = ?nodes.len(), "Revealing account nodes from V2 proof");
trie.reveal_nodes(nodes)?;
}
Ok(())
}
/// Reveals storage proof nodes from a V2 proof for the given address.
///
/// V2 proofs already include the masks in the `ProofTrieNode` structure,
/// so no separate masks map is needed.
pub fn reveal_storage_v2_proof_nodes(
&mut self,
account: B256,
nodes: Vec<ProofTrieNode>,
) -> SparseStateTrieResult<()> {
let (trie, revealed_paths) = self.storage.get_trie_and_revealed_paths_mut(account);
let _metric_values = Self::reveal_storage_v2_proof_nodes_inner(
account,
nodes,
revealed_paths,
trie,
self.retain_updates,
)?;
#[cfg(feature = "metrics")]
{
self.metrics.increment_total_storage_nodes(_metric_values.total_nodes as u64);
self.metrics.increment_skipped_storage_nodes(_metric_values.skipped_nodes as u64);
}
Ok(())
}
/// Reveals storage V2 proof nodes for the given address. This is an internal static function
/// designed to handle a variety of associated public functions.
fn reveal_storage_v2_proof_nodes_inner(
account: B256,
nodes: Vec<ProofTrieNode>,
revealed_nodes: &mut HashSet<Nibbles>,
trie: &mut SparseTrie<S>,
retain_updates: bool,
) -> SparseStateTrieResult<ProofNodesMetricValues> {
let FilteredV2ProofNodes { root_node, nodes, new_nodes, metric_values } =
filter_revealed_v2_proof_nodes(nodes, revealed_nodes)?;
if let Some(root_node) = root_node {
trace!(target: "trie::sparse", ?account, ?root_node, "Revealing root storage node from V2 proof");
let trie = trie.reveal_root(root_node.node, root_node.masks, retain_updates)?;
trie.reserve_nodes(new_nodes);
trace!(target: "trie::sparse", ?account, total_nodes = ?nodes.len(), "Revealing storage nodes from V2 proof");
trie.reveal_nodes(nodes)?;
}
Ok(metric_values)
}
/// Reveals a storage multiproof for the given address.
pub fn reveal_storage_multiproof(
&mut self,
@@ -1163,87 +1000,6 @@ fn filter_map_revealed_nodes(
Ok(result)
}
/// Result of [`filter_revealed_v2_proof_nodes`].
#[derive(Debug, PartialEq, Eq)]
struct FilteredV2ProofNodes {
/// Root node which was pulled out of the original node set to be handled specially.
root_node: Option<ProofTrieNode>,
/// Filtered proof nodes. Root node is removed.
nodes: Vec<ProofTrieNode>,
/// Number of new nodes that will be revealed. This includes all children of branch nodes, even
/// if they are not in the proof.
new_nodes: usize,
/// Values which are being returned so they can be incremented into metrics.
metric_values: ProofNodesMetricValues,
}
/// Filters V2 proof nodes that are already revealed, separates the root node if present, and
/// returns additional information about the number of total, skipped, and new nodes.
///
/// Unlike [`filter_map_revealed_nodes`], V2 proof nodes already have masks included in the
/// `ProofTrieNode` structure, so no separate masks map is needed.
fn filter_revealed_v2_proof_nodes(
proof_nodes: Vec<ProofTrieNode>,
revealed_nodes: &mut HashSet<Nibbles>,
) -> SparseStateTrieResult<FilteredV2ProofNodes> {
let mut result = FilteredV2ProofNodes {
root_node: None,
nodes: Vec::with_capacity(proof_nodes.len()),
new_nodes: 0,
metric_values: Default::default(),
};
// Count non-EmptyRoot nodes for sanity check. When multiple proofs are extended together,
// duplicate EmptyRoot nodes may appear (e.g., storage proofs split across chunks for an
// account with empty storage). We only error if there's an EmptyRoot alongside real nodes.
let non_empty_root_count =
proof_nodes.iter().filter(|n| !matches!(n.node, TrieNode::EmptyRoot)).count();
for node in proof_nodes {
result.metric_values.total_nodes += 1;
let is_root = node.path.is_empty();
// If the node is already revealed, skip it. We don't ever skip the root node, nor do we add
// it to `revealed_nodes`.
if !is_root && !revealed_nodes.insert(node.path) {
result.metric_values.skipped_nodes += 1;
continue
}
result.new_nodes += 1;
// Count children for capacity estimation
match &node.node {
TrieNode::Branch(branch) => {
result.new_nodes += branch.state_mask.count_ones() as usize;
}
TrieNode::Extension(_) => {
result.new_nodes += 1;
}
_ => {}
};
if is_root {
// Perform sanity check: EmptyRoot is only valid if there are no other real nodes.
if matches!(node.node, TrieNode::EmptyRoot) && non_empty_root_count > 0 {
return Err(SparseStateTrieErrorKind::InvalidRootNode {
path: node.path,
node: alloy_rlp::encode(&node.node).into(),
}
.into())
}
result.root_node = Some(node);
continue
}
result.nodes.push(node);
}
Ok(result)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -1418,127 +1174,6 @@ mod tests {
.is_none());
}
#[test]
fn reveal_v2_proof_nodes() {
let provider_factory = DefaultTrieNodeProviderFactory;
let mut sparse = SparseStateTrie::<SerialSparseTrie>::default();
let leaf_value = alloy_rlp::encode(TrieAccount::default());
let leaf_1_node = TrieNode::Leaf(LeafNode::new(Nibbles::default(), leaf_value.clone()));
let leaf_2_node = TrieNode::Leaf(LeafNode::new(Nibbles::default(), leaf_value.clone()));
let branch_node = TrieNode::Branch(BranchNode {
stack: vec![
RlpNode::from_rlp(&alloy_rlp::encode(&leaf_1_node)),
RlpNode::from_rlp(&alloy_rlp::encode(&leaf_2_node)),
],
state_mask: TrieMask::new(0b11),
});
// Create V2 proof nodes with masks already included
let v2_proof_nodes = vec![
ProofTrieNode {
path: Nibbles::default(),
node: branch_node,
masks: Some(BranchNodeMasks {
hash_mask: TrieMask::default(),
tree_mask: TrieMask::default(),
}),
},
ProofTrieNode { path: Nibbles::from_nibbles([0x0]), node: leaf_1_node, masks: None },
ProofTrieNode { path: Nibbles::from_nibbles([0x1]), node: leaf_2_node, masks: None },
];
// Reveal V2 proof nodes
sparse.reveal_account_v2_proof_nodes(v2_proof_nodes.clone()).unwrap();
// Check that the state trie contains the leaf node and value
assert!(sparse
.state_trie_ref()
.unwrap()
.nodes_ref()
.contains_key(&Nibbles::from_nibbles([0x0])));
assert_eq!(
sparse.state_trie_ref().unwrap().get_leaf_value(&Nibbles::from_nibbles([0x0])),
Some(&leaf_value)
);
// Remove the leaf node
sparse.remove_account_leaf(&Nibbles::from_nibbles([0x0]), &provider_factory).unwrap();
assert!(sparse
.state_trie_ref()
.unwrap()
.get_leaf_value(&Nibbles::from_nibbles([0x0]))
.is_none());
// Reveal again - should skip already revealed paths
sparse.reveal_account_v2_proof_nodes(v2_proof_nodes).unwrap();
assert!(sparse
.state_trie_ref()
.unwrap()
.get_leaf_value(&Nibbles::from_nibbles([0x0]))
.is_none());
}
#[test]
fn reveal_storage_v2_proof_nodes() {
let provider_factory = DefaultTrieNodeProviderFactory;
let mut sparse = SparseStateTrie::<SerialSparseTrie>::default();
let storage_value: Vec<u8> = alloy_rlp::encode_fixed_size(&U256::from(42)).to_vec();
let leaf_1_node = TrieNode::Leaf(LeafNode::new(Nibbles::default(), storage_value.clone()));
let leaf_2_node = TrieNode::Leaf(LeafNode::new(Nibbles::default(), storage_value.clone()));
let branch_node = TrieNode::Branch(BranchNode {
stack: vec![
RlpNode::from_rlp(&alloy_rlp::encode(&leaf_1_node)),
RlpNode::from_rlp(&alloy_rlp::encode(&leaf_2_node)),
],
state_mask: TrieMask::new(0b11),
});
let v2_proof_nodes = vec![
ProofTrieNode { path: Nibbles::default(), node: branch_node, masks: None },
ProofTrieNode { path: Nibbles::from_nibbles([0x0]), node: leaf_1_node, masks: None },
ProofTrieNode { path: Nibbles::from_nibbles([0x1]), node: leaf_2_node, masks: None },
];
// Reveal V2 storage proof nodes for account
sparse.reveal_storage_v2_proof_nodes(B256::ZERO, v2_proof_nodes.clone()).unwrap();
// Check that the storage trie contains the leaf node and value
assert!(sparse
.storage_trie_ref(&B256::ZERO)
.unwrap()
.nodes_ref()
.contains_key(&Nibbles::from_nibbles([0x0])));
assert_eq!(
sparse
.storage_trie_ref(&B256::ZERO)
.unwrap()
.get_leaf_value(&Nibbles::from_nibbles([0x0])),
Some(&storage_value)
);
// Remove the leaf node
sparse
.remove_storage_leaf(B256::ZERO, &Nibbles::from_nibbles([0x0]), &provider_factory)
.unwrap();
assert!(sparse
.storage_trie_ref(&B256::ZERO)
.unwrap()
.get_leaf_value(&Nibbles::from_nibbles([0x0]))
.is_none());
// Reveal again - should skip already revealed paths
sparse.reveal_storage_v2_proof_nodes(B256::ZERO, v2_proof_nodes).unwrap();
assert!(sparse
.storage_trie_ref(&B256::ZERO)
.unwrap()
.get_leaf_value(&Nibbles::from_nibbles([0x0]))
.is_none());
}
#[test]
fn take_trie_updates() {
reth_tracing::init_test_tracing();

Some files were not shown because too many files have changed in this diff Show More