Compare commits

..

33 Commits

Author SHA1 Message Date
rakita
0f09a92acc fix: update gas_used/gas_refunded to ResultGas API 2026-02-12 16:47:57 +01:00
rakita
df8ba50cb7 chore: update revm/inspectors/alloy-evm patches and fix Bytecode flatten 2026-02-12 16:37:56 +01:00
rakita
61341a1342 Merge remote-tracking branch 'origin/main' into staging
# Conflicts:
#	Cargo.lock
#	crates/optimism/evm/src/lib.rs
2026-02-12 16:32:00 +01:00
rakita
3d02124e50 Merge origin/main into staging 2026-02-03 13:24:01 +01:00
rakita
953a1b3399 chore: update revm, revm-inspectors, alloy-evm to staging
- revm: 6aa06829d2caa2aa38606ed22b83354a7a7ff98e
- revm-inspectors: cc4f62d8c107b3d2dc42a220690a079cdfc8dfb1
- alloy-evm: 80e6a436ecade2b9bd950fd70bd0d6a45ef0a43e
2026-02-03 13:22:25 +01:00
rakita
a364989c61 Merge origin/main into staging 2026-02-03 13:14:17 +01:00
rakita
f4075f5926 Merge remote staging 2026-01-28 19:12:11 +01:00
rakita
211d3d2924 fix: adapt to revm hasher type changes 2026-01-28 19:11:48 +01:00
rakita
dbc5313a2c Merge origin/main into staging - update patches 2026-01-28 19:09:24 +01:00
rakita
345fc9cfd2 Merge branch 'main' into staging 2026-01-26 22:35:00 +01:00
rakita
33d61c30cb chore: update revm, revm-inspectors, alloy-evm to staging
Updates dependencies:
- revm to 300efbf3e391e1796f5210cd4506508e385a55d2
- revm-inspectors to 9464524e7d983c8601ba7af3048cd60575dad6bd
- alloy-evm to 1c8ff5a179e760517016ab4e4d5af1c4b5923594

Fixes:
- Add slot_num field to BlockEnv for EIP-7843 SLOTNUM opcode
- Update BlockHashCache API: use lowest() instead of keys()
2026-01-26 21:32:08 +01:00
DaniPopes
beb0c5e527 chore: reduce number of nightly builds (#21446) 2026-01-26 21:30:39 +01:00
DaniPopes
73c39279b1 chore: remove unused docker from makefile (#21445) 2026-01-26 21:30:39 +01:00
ethfanWilliam
2fd0a703e2 fix(stages): retain RocksDB TempDir in TestStageDB to prevent premature deletion (#21444) 2026-01-26 21:30:39 +01:00
Dan Cline
7dcd77de95 fix(pruner): prune account and storage changeset static files (#21346) 2026-01-26 21:30:39 +01:00
Dan Cline
da6e6afe78 chore(metrics): add a gas_last metric similar to new_payload_last (#21437) 2026-01-26 21:30:39 +01:00
Brian Picciano
04d9a33c68 refactor(trie): always use ParallelSparseTrie, deprecate config flags (#21435) 2026-01-26 21:30:39 +01:00
Arsenii Kulikov
be1f657b3c perf: use shared channel for prewarm workers (#21429) 2026-01-26 21:30:39 +01:00
Rez
cab4cbf0ea feat: configurable EVM execution limits (#21088)
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
2026-01-26 21:30:39 +01:00
figtracer
29f51abb22 feat(rpc): add transaction hash caching to EthStateCache (#21180)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 21:30:39 +01:00
Matthias Seitz
dd0ee0709c fix(rpc): add block timestamp validation in eth_simulateV1 (#21397)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 21:30:39 +01:00
Matthias Seitz
dcd9e50663 fix(rpc): use correct error codes for eth_simulateV1 reverts and halts (#21412)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 21:30:39 +01:00
Matthias Seitz
59d11378b9 fix(rpc): add block number validation in eth_simulateV1 (#21396)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 21:30:39 +01:00
Matthias Seitz
70ceb755ad fix(rpc): populate block_hash in eth_simulateV1 logs (#21413)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 21:30:39 +01:00
Matthias Seitz
a49991c766 feat(cli): make stopping on invalid block the default for reth import (#21403)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 21:30:39 +01:00
Matthias Seitz
0a77c2aae1 feat(rpc): implement movePrecompileToAddress for eth_simulateV1 (#21414)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 21:30:39 +01:00
Matthias Seitz
93adbf82a0 fix(rpc): set prevrandao to zero for eth_simulateV1 simulated blocks (#21399)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 21:30:39 +01:00
Matthias Seitz
ce08b6f265 fix(rpc): cap simulate_v1 default gas limit to RPC gas cap (#21402)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 21:30:39 +01:00
Matthias Seitz
b03a704a1e fix(engine): only warn for critical capability mismatches (#21398)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 21:30:39 +01:00
Andrey Kolishchak
a51e03fce6 fix(net): FetchFullBlockRangeFuture can get stuck forever after partial body fetch + error (#21411)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 21:30:39 +01:00
rakita
57d7c98f66 chore: merge main and update alloy-evm staging patch 2026-01-26 12:39:58 +01:00
rakita
5d9a43f2d4 Merge remote-tracking branch 'origin/main' into staging 2026-01-26 12:36:44 +01:00
rakita
defd0e8e5c Bump revm to staging and fix breaking changes
- Patch revm and all sub-crates to staging commit 0dc217a9
- Patch revm-inspectors to staging commit fccc4ac5
- Patch alloy-evm to staging commit 625ccc0f
- Add slot_num field to BlockEnv initializers
- Update BlockHashCache usage (no longer has keys method)
2026-01-26 02:41:26 +01:00
159 changed files with 3099 additions and 10892 deletions

View File

@@ -1,4 +0,0 @@
---
---
Improved nightly Docker build failure Slack notification with more detailed formatting and context.

View File

@@ -1,4 +0,0 @@
---
---
Improved documentation overview page with better structure and clarity.

View File

@@ -1,5 +0,0 @@
---
reth-node-events: patch
---
Updated consensus engine log message to be more accurate about received updates.

View File

@@ -1,9 +0,0 @@
---
reth-network-api: minor
reth-network-types: minor
reth-network: minor
reth-node-core: minor
reth: minor
---
Added optional ENR fork ID enforcement to filter out peers from incompatible networks during peer discovery, controlled by the `--enforce-enr-fork-id` CLI flag.

2
.github/CODEOWNERS vendored
View File

@@ -38,7 +38,7 @@ crates/storage/libmdbx-rs/ @shekhirin
crates/storage/nippy-jar/ @joshieDo @shekhirin
crates/storage/provider/ @joshieDo @shekhirin @yongkangc
crates/storage/storage-api/ @joshieDo
crates/tasks/ @mattsse @DaniPopes
crates/tasks/ @mattsse
crates/tokio-util/ @mattsse
crates/tracing/ @mattsse @shekhirin
crates/tracing-otlp/ @mattsse @Rjected

View File

@@ -70,27 +70,18 @@ jobs:
# Add 'latest' tag for non-RC releases
if [[ ! "$VERSION" =~ -rc ]]; then
echo "ethereum_tags=${REGISTRY}/reth:${VERSION},${REGISTRY}/reth:latest" >> "$GITHUB_OUTPUT"
{
echo "ethereum_set<<EOF"
echo "ethereum.tags=${REGISTRY}/reth:${VERSION}"
echo "ethereum.tags=${REGISTRY}/reth:latest"
echo "EOF"
} >> "$GITHUB_OUTPUT"
else
echo "ethereum_tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
fi
elif [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ inputs.build_type }}" == "nightly" ]]; then
echo "targets=nightly" >> "$GITHUB_OUTPUT"
echo "ethereum_tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
else
# git-sha build
echo "targets=ethereum" >> "$GITHUB_OUTPUT"
echo "ethereum_tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
fi
- name: Build and push images
@@ -106,7 +97,7 @@ jobs:
targets: ${{ steps.params.outputs.targets }}
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
set: |
${{ steps.params.outputs.ethereum_set }}
ethereum.tags=${{ steps.params.outputs.ethereum_tags }}
- name: Verify image architectures
env:
@@ -126,18 +117,6 @@ jobs:
- name: Slack Webhook Action
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: danger
SLACK_ICON_EMOJI: ":rotating_light:"
SLACK_USERNAME: "GitHub Actions"
SLACK_TITLE: ":rotating_light: Nightly Docker Build Failed"
SLACK_MESSAGE: |
The scheduled nightly Docker build failed.
*Commit:* `${{ github.sha }}`
*Branch:* `${{ github.ref_name }}`
*Run:* <https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}|View logs>
*Action required:* Re-run the workflow or investigate the build failure.
SLACK_FOOTER: "paradigmxyz/reth · docker.yml"
MSG_MINIMAL: true
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}"
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}

615
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,5 @@
[workspace.package]
version = "1.11.0"
version = "1.10.2"
edition = "2024"
rust-version = "1.88"
license = "MIT OR Apache-2.0"
@@ -110,7 +110,6 @@ members = [
"crates/storage/errors/",
"crates/storage/libmdbx-rs/",
"crates/storage/libmdbx-rs/mdbx-sys/",
"crates/storage/mdbx-viz/",
"crates/storage/nippy-jar/",
"crates/storage/provider/",
"crates/storage/storage-api/",
@@ -378,7 +377,6 @@ reth-invalid-block-hooks = { path = "crates/engine/invalid-block-hooks" }
reth-ipc = { path = "crates/rpc/ipc" }
reth-libmdbx = { path = "crates/storage/libmdbx-rs" }
reth-mdbx-sys = { path = "crates/storage/libmdbx-rs/mdbx-sys" }
reth-mdbx-viz = { path = "crates/storage/mdbx-viz" }
reth-metrics = { path = "crates/metrics" }
reth-net-banlist = { path = "crates/net/banlist" }
reth-net-nat = { path = "crates/net/nat" }
@@ -713,6 +711,28 @@ vergen-git2 = "9.1.0"
ipnet = "2.11"
[patch.crates-io]
# revm staging patches
revm = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
op-revm = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-bytecode = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-context = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-context-interface = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-database = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-database-interface = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-handler = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-inspector = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-interpreter = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-precompile = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-state = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
# revm-inspectors staging patch
revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "e80e2eab72dfa18011e6a99abd37027290a46e83" }
# alloy-evm staging patches
alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "742dc14749ea0279c03ca27b1c26f26ac19fbefb" }
alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "742dc14749ea0279c03ca27b1c26f26ac19fbefb" }
# alloy-consensus = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-contract = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-eips = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }

View File

@@ -192,15 +192,6 @@ impl Command {
parent_header = block.header;
parent_hash = block_hash;
blocks_processed += 1;
let progress = match mode {
RampMode::Blocks(total) => format!("{blocks_processed}/{total}"),
RampMode::TargetGasLimit(target) => {
let pct = (parent_header.gas_limit as f64 / target as f64 * 100.0).min(100.0);
format!("{pct:.1}%")
}
};
info!(target: "reth-bench", progress, block_number = parent_header.number, gas_limit = parent_header.gas_limit, "Block processed");
}
let final_gas_limit = parent_header.gas_limit;

View File

@@ -153,7 +153,6 @@ impl Command {
..
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
let total_blocks = benchmark_mode.total_blocks();
let buffer_size = self.rpc_block_buffer_size;
// Use a oneshot channel to propagate errors from the spawned task
@@ -207,7 +206,6 @@ impl Command {
});
let mut results = Vec::new();
let mut blocks_processed = 0u64;
let total_benchmark_duration = Instant::now();
let mut total_wait_time = Duration::ZERO;
@@ -251,13 +249,8 @@ impl Command {
// Exclude time spent waiting on the block prefetch channel from the benchmark duration.
// We want to measure engine throughput, not RPC fetch latency.
blocks_processed += 1;
let current_duration = total_benchmark_duration.elapsed() - total_wait_time;
let progress = match total_blocks {
Some(total) => format!("{blocks_processed}/{total}"),
None => format!("{blocks_processed}"),
};
info!(target: "reth-bench", progress, %combined_result);
info!(target: "reth-bench", %combined_result);
if let Some(w) = &mut waiter {
w.on_block(block_number).await?;

View File

@@ -52,7 +52,6 @@ impl Command {
..
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
let total_blocks = benchmark_mode.total_blocks();
let buffer_size = self.rpc_block_buffer_size;
// Use a oneshot channel to propagate errors from the spawned task
@@ -83,8 +82,8 @@ impl Command {
}
});
// put results in a summary vec so they can be printed at the end
let mut results = Vec::new();
let mut blocks_processed = 0u64;
let total_benchmark_duration = Instant::now();
let mut total_wait_time = Duration::ZERO;
@@ -106,12 +105,7 @@ impl Command {
call_new_payload(&auth_provider, version, params).await?;
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
blocks_processed += 1;
let progress = match total_blocks {
Some(total) => format!("{blocks_processed}/{total}"),
None => format!("{blocks_processed}"),
};
info!(target: "reth-bench", progress, %new_payload_result);
info!(target: "reth-bench", %new_payload_result);
// current duration since the start of the benchmark minus the time
// waiting for blocks

View File

@@ -341,8 +341,7 @@ impl Command {
};
let current_duration = total_benchmark_duration.elapsed();
let progress = format!("{}/{}", i + 1, payloads.len());
info!(target: "reth-bench", progress, %combined_result);
info!(target: "reth-bench", %combined_result);
if let Some(w) = &mut waiter {
w.on_block(block_number).await?;

View File

@@ -20,19 +20,6 @@ impl BenchMode {
}
}
/// Returns the total number of blocks in the benchmark, if known.
///
/// For [`BenchMode::Range`] this is the length of the range.
/// For [`BenchMode::Continuous`] the total is unbounded, so `None` is returned.
pub const fn total_blocks(&self) -> Option<u64> {
match self {
Self::Continuous(_) => None,
Self::Range(range) => {
Some(range.end().saturating_sub(*range.start()).saturating_add(1))
}
}
}
/// Create a [`BenchMode`] from optional `from` and `to` fields.
pub fn new(from: Option<u64>, to: Option<u64>, latest_block: u64) -> Result<Self, eyre::Error> {
// If neither `--from` nor `--to` are provided, we will run the benchmark continuously,

View File

@@ -190,7 +190,6 @@ min-trace-logs = [
"reth-node-core/min-trace-logs",
]
pageviz = ["reth-db/pageviz", "reth-node-builder/pageviz"]
rocksdb = ["reth-ethereum-cli/rocksdb", "reth-node-core/rocksdb"]
edge = ["rocksdb"]

View File

@@ -1061,14 +1061,6 @@ mod tests {
) -> ProviderResult<Option<StorageValue>> {
Ok(None)
}
fn storage_by_hashed_key(
&self,
_address: Address,
_hashed_storage_key: StorageKey,
) -> ProviderResult<Option<StorageValue>> {
Ok(None)
}
}
impl BytecodeReader for MockStateProvider {

View File

@@ -223,26 +223,6 @@ impl<N: NodePrimitives> StateProvider for MemoryOverlayStateProviderRef<'_, N> {
self.historical.storage(address, storage_key)
}
fn storage_by_hashed_key(
&self,
address: Address,
hashed_storage_key: StorageKey,
) -> ProviderResult<Option<StorageValue>> {
let hashed_address = keccak256(address);
let state = &self.trie_input().state;
if let Some(hs) = state.storages.get(&hashed_address) {
if let Some(value) = hs.storage.get(&hashed_storage_key) {
return Ok(Some(*value));
}
if hs.wiped {
return Ok(Some(StorageValue::ZERO));
}
}
self.historical.storage_by_hashed_key(address, hashed_storage_key)
}
}
impl<N: NodePrimitives> BytecodeReader for MemoryOverlayStateProviderRef<'_, N> {

View File

@@ -87,8 +87,6 @@ tokio-stream.workspace = true
reqwest.workspace = true
url.workspace = true
metrics.workspace = true
memmap2.workspace = true
rayon.workspace = true
# io
fdlimit.workspace = true

View File

@@ -19,7 +19,7 @@ use reth_node_builder::{
Node, NodeComponents, NodeComponentsBuilder, NodeTypes, NodeTypesWithDBAdapter,
};
use reth_node_core::{
args::{DatabaseArgs, DatadirArgs, StaticFilesArgs, StorageArgs},
args::{DatabaseArgs, DatadirArgs, RocksDbArgs, StaticFilesArgs, StorageArgs},
dirs::{ChainPath, DataDirPath},
};
use reth_provider::{
@@ -67,23 +67,62 @@ pub struct EnvironmentArgs<C: ChainSpecParser> {
#[command(flatten)]
pub static_files: StaticFilesArgs,
/// All `RocksDB` related arguments
#[command(flatten)]
pub rocksdb: RocksDbArgs,
/// Storage mode configuration (v2 vs v1/legacy)
#[command(flatten)]
pub storage: StorageArgs,
}
impl<C: ChainSpecParser> EnvironmentArgs<C> {
/// Returns the effective storage settings derived from `--storage.v2`.
/// Returns the effective storage settings derived from `--storage.v2`, static-file, and
/// `RocksDB` CLI args.
///
/// The base storage mode is determined by `--storage.v2`:
/// - When `--storage.v2` is set: uses [`StorageSettings::v2()`] defaults
/// - Otherwise: uses [`StorageSettings::base()`] defaults
/// - Otherwise: uses [`StorageSettings::v1()`] defaults
///
/// Individual `--static-files.*` and `--rocksdb.*` flags override the base when explicitly set.
pub fn storage_settings(&self) -> StorageSettings {
if self.storage.v2 {
StorageSettings::v2()
} else {
StorageSettings::base()
let mut s = if self.storage.v2 { StorageSettings::v2() } else { StorageSettings::base() };
// Apply static files overrides (only when explicitly set)
if let Some(v) = self.static_files.receipts {
s = s.with_receipts_in_static_files(v);
}
if let Some(v) = self.static_files.transaction_senders {
s = s.with_transaction_senders_in_static_files(v);
}
if let Some(v) = self.static_files.account_changesets {
s = s.with_account_changesets_in_static_files(v);
}
if let Some(v) = self.static_files.storage_changesets {
s = s.with_storage_changesets_in_static_files(v);
}
// Apply rocksdb overrides
// --rocksdb.all sets all rocksdb flags to true
if self.rocksdb.all {
s = s
.with_transaction_hash_numbers_in_rocksdb(true)
.with_storages_history_in_rocksdb(true)
.with_account_history_in_rocksdb(true);
}
// Individual rocksdb flags override --rocksdb.all when explicitly set
if let Some(v) = self.rocksdb.tx_hash {
s = s.with_transaction_hash_numbers_in_rocksdb(v);
}
if let Some(v) = self.rocksdb.storages_history {
s = s.with_storages_history_in_rocksdb(v);
}
if let Some(v) = self.rocksdb.account_history {
s = s.with_account_history_in_rocksdb(v);
}
s
}
/// Initializes environment according to [`AccessRights`] and returns an instance of

View File

@@ -5,7 +5,6 @@ use reth_codecs::Compact;
use reth_db_api::{cursor::DbDupCursorRO, database::Database, tables, transaction::DbTx};
use reth_db_common::DbTool;
use reth_node_builder::NodeTypesWithDB;
use reth_storage_api::StorageSettingsCache;
use std::time::{Duration, Instant};
use tracing::info;
@@ -23,94 +22,52 @@ impl Command {
/// Execute `db account-storage` command
pub fn execute<N: NodeTypesWithDB>(self, tool: &DbTool<N>) -> eyre::Result<()> {
let address = self.address;
let use_hashed_state = tool.provider_factory.cached_storage_settings().use_hashed_state();
let (slot_count, plain_size) = tool.provider_factory.db_ref().view(|tx| {
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
let mut count = 0usize;
let mut total_value_bytes = 0usize;
let mut last_log = Instant::now();
let (slot_count, storage_size) = if use_hashed_state {
let hashed_address = keccak256(address);
tool.provider_factory.db_ref().view(|tx| {
let mut cursor = tx.cursor_dup_read::<tables::HashedStorages>()?;
let mut count = 0usize;
let mut total_value_bytes = 0usize;
let mut last_log = Instant::now();
// Walk all storage entries for this address
let walker = cursor.walk_dup(Some(address), None)?;
for entry in walker {
let (_, storage_entry) = entry?;
count += 1;
// StorageEntry encodes as: 32 bytes (key/subkey uncompressed) + compressed U256
let mut buf = Vec::new();
let entry_len = storage_entry.to_compact(&mut buf);
total_value_bytes += entry_len;
let walker = cursor.walk_dup(Some(hashed_address), None)?;
for entry in walker {
let (_, storage_entry) = entry?;
count += 1;
let mut buf = Vec::new();
let entry_len = storage_entry.to_compact(&mut buf);
total_value_bytes += entry_len;
if last_log.elapsed() >= LOG_INTERVAL {
info!(
target: "reth::cli",
address = %address,
slots = count,
key = %storage_entry.key,
"Processing hashed storage slots"
);
last_log = Instant::now();
}
if last_log.elapsed() >= LOG_INTERVAL {
info!(
target: "reth::cli",
address = %address,
slots = count,
key = %storage_entry.key,
"Processing storage slots"
);
last_log = Instant::now();
}
}
let total_size = if count > 0 { 32 + total_value_bytes } else { 0 };
// Add 20 bytes for the Address key (stored once per account in dupsort)
let total_size = if count > 0 { 20 + total_value_bytes } else { 0 };
Ok::<_, eyre::Report>((count, total_size))
})??
} else {
tool.provider_factory.db_ref().view(|tx| {
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
let mut count = 0usize;
let mut total_value_bytes = 0usize;
let mut last_log = Instant::now();
Ok::<_, eyre::Report>((count, total_size))
})??;
// Walk all storage entries for this address
let walker = cursor.walk_dup(Some(address), None)?;
for entry in walker {
let (_, storage_entry) = entry?;
count += 1;
let mut buf = Vec::new();
// StorageEntry encodes as: 32 bytes (key/subkey uncompressed) + compressed U256
let entry_len = storage_entry.to_compact(&mut buf);
total_value_bytes += entry_len;
if last_log.elapsed() >= LOG_INTERVAL {
info!(
target: "reth::cli",
address = %address,
slots = count,
key = %storage_entry.key,
"Processing storage slots"
);
last_log = Instant::now();
}
}
// Add 20 bytes for the Address key (stored once per account in dupsort)
let total_size = if count > 0 { 20 + total_value_bytes } else { 0 };
Ok::<_, eyre::Report>((count, total_size))
})??
};
// Estimate hashed storage size: 32-byte B256 key instead of 20-byte Address
let hashed_size_estimate = if slot_count > 0 { plain_size + 12 } else { 0 };
let total_estimate = plain_size + hashed_size_estimate;
let hashed_address = keccak256(address);
println!("Account: {address}");
println!("Hashed address: {hashed_address}");
println!("Storage slots: {slot_count}");
if use_hashed_state {
println!("Hashed storage size: {} (estimated)", human_bytes(storage_size as f64));
} else {
// Estimate hashed storage size: 32-byte B256 key instead of 20-byte Address
let hashed_size_estimate = if slot_count > 0 { storage_size + 12 } else { 0 };
let total_estimate = storage_size + hashed_size_estimate;
println!("Plain storage size: {} (estimated)", human_bytes(storage_size as f64));
println!(
"Hashed storage size: {} (estimated)",
human_bytes(hashed_size_estimate as f64)
);
println!("Total estimated size: {}", human_bytes(total_estimate as f64));
}
println!("Plain storage size: {} (estimated)", human_bytes(plain_size as f64));
println!("Hashed storage size: {} (estimated)", human_bytes(hashed_size_estimate as f64));
println!("Total estimated size: {}", human_bytes(total_estimate as f64));
Ok(())
}

View File

@@ -1,61 +0,0 @@
use clap::Parser;
use reth_db::mdbx::{self, ffi};
use std::path::PathBuf;
/// Copies the MDBX database to a new location.
///
/// Equivalent to the standalone `mdbx_copy` tool but bundled into reth.
#[derive(Parser, Debug)]
pub struct Command {
/// Destination path for the database copy.
dest: PathBuf,
/// Compact the database while copying (reclaims free space).
#[arg(short, long)]
compact: bool,
/// Force dynamic size for the destination database.
#[arg(short = 'd', long)]
force_dynamic_size: bool,
/// Throttle to avoid MVCC pressure on writers.
#[arg(short = 'p', long)]
throttle_mvcc: bool,
}
impl Command {
/// Execute `db copy` command
pub fn execute(self, db: &mdbx::DatabaseEnv) -> eyre::Result<()> {
let mut flags: ffi::MDBX_copy_flags_t = ffi::MDBX_CP_DEFAULTS;
if self.compact {
flags |= ffi::MDBX_CP_COMPACT;
}
if self.force_dynamic_size {
flags |= ffi::MDBX_CP_FORCE_DYNAMIC_SIZE;
}
if self.throttle_mvcc {
flags |= ffi::MDBX_CP_THROTTLE_MVCC;
}
let dest = self
.dest
.to_str()
.ok_or_else(|| eyre::eyre!("destination path must be valid UTF-8"))?;
let dest_cstr = std::ffi::CString::new(dest)?;
println!("Copying database to {} ...", self.dest.display());
let rc = db.with_raw_env_ptr(|env_ptr| unsafe {
ffi::mdbx_env_copy(env_ptr, dest_cstr.as_ptr(), flags)
});
if rc != 0 {
eyre::bail!("mdbx_env_copy failed with error code {rc}: {}", unsafe {
std::ffi::CStr::from_ptr(ffi::mdbx_strerror(rc)).to_string_lossy()
});
}
println!("Done.");
Ok(())
}
}

View File

@@ -1,564 +0,0 @@
use clap::Parser;
use eyre::Context;
use memmap2::Mmap;
use rayon::prelude::*;
use reth_db_api::tables::Tables;
use reth_db_common::DbTool;
use reth_node_builder::NodeTypesWithDB;
use reth_node_core::dirs::{ChainPath, DataDirPath};
use std::{
collections::HashMap,
fs,
io::Write,
path::PathBuf,
time::Instant,
};
const PAGEHDRSZ: usize = 20;
const NODESIZE: usize = 8;
const P_BRANCH: u16 = 0x01;
const P_LEAF: u16 = 0x02;
const P_LARGE: u16 = 0x04;
const P_META: u16 = 0x08;
const P_DUPFIX: u16 = 0x20;
const N_BIG: u8 = 0x01;
const N_TREE: u8 = 0x02;
const N_DUP: u8 = 0x04;
const P_INVALID: u32 = 0xFFFFFFFF;
const UNREFERENCED: u8 = 0xFF;
const DBI_META: u8 = 0xFE;
const META_GEO: usize = 0x14;
const META_FREE_DB: usize = 0x28;
const META_MAIN_DB: usize = 0x58;
const META_TXNID_A: usize = 0x08;
const META_TXNID_B: usize = 0xB0;
const GEO_NOW: usize = 0x0C;
const TREE_ROOT: usize = 0x08;
const TREE_BRANCH_PAGES: usize = 0x0C;
const TREE_LEAF_PAGES: usize = 0x10;
const TREE_LARGE_PAGES: usize = 0x14;
const TREE_ITEMS: usize = 0x20;
const MDBX_MAGIC: u64 = 0x59659DBDEF4C11;
fn u16_le(buf: &[u8], off: usize) -> u16 {
u16::from_le_bytes([buf[off], buf[off + 1]])
}
fn u32_le(buf: &[u8], off: usize) -> u32 {
u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]])
}
fn u64_le(buf: &[u8], off: usize) -> u64 {
u64::from_le_bytes(buf[off..off + 8].try_into().unwrap())
}
#[derive(Debug, Clone)]
struct TreeDescriptor {
root: u32,
branch_pages: u32,
leaf_pages: u32,
large_pages: u32,
items: u64,
}
impl TreeDescriptor {
fn parse(buf: &[u8], off: usize) -> Self {
Self {
root: u32_le(buf, off + TREE_ROOT),
branch_pages: u32_le(buf, off + TREE_BRANCH_PAGES),
leaf_pages: u32_le(buf, off + TREE_LEAF_PAGES),
large_pages: u32_le(buf, off + TREE_LARGE_PAGES),
items: u64_le(buf, off + TREE_ITEMS),
}
}
fn total_pages(&self) -> u64 {
self.branch_pages as u64 + self.leaf_pages as u64 + self.large_pages as u64
}
}
#[derive(Debug, Clone)]
struct DBIInfo {
name: String,
dbi_index: u8,
tree: TreeDescriptor,
}
fn page_flags(buf: &[u8], page_off: usize) -> u16 {
u16_le(buf, page_off + 0x0A)
}
fn page_nkeys(buf: &[u8], page_off: usize) -> usize {
u16_le(buf, page_off + 0x0C) as usize / 2
}
fn page_overflow_count(buf: &[u8], page_off: usize) -> u32 {
u32_le(buf, page_off + 0x0C)
}
fn mark(owner_map: &mut [u8], pgno: usize, dbi_index: u8, conflicts: &mut u64) -> bool {
if pgno >= owner_map.len() {
return false;
}
let prev = owner_map[pgno];
if prev == UNREFERENCED {
owner_map[pgno] = dbi_index;
return true;
}
if prev != dbi_index {
*conflicts += 1;
}
false
}
fn walk_tree_collect(
buf: &[u8],
ps: usize,
root_pgno: u32,
page_count: usize,
dbi_index: u8,
) -> Vec<(usize, u8)> {
if root_pgno == P_INVALID || root_pgno as usize >= page_count {
return Vec::new();
}
let mut result: Vec<(usize, u8)> = Vec::new();
let mut visited = vec![false; page_count];
let mut stack: Vec<u32> = vec![root_pgno];
while let Some(pgno) = stack.pop() {
let pgno_usize = pgno as usize;
if pgno_usize >= page_count || visited[pgno_usize] {
continue;
}
visited[pgno_usize] = true;
result.push((pgno_usize, dbi_index));
let page_off = pgno_usize * ps;
if page_off + PAGEHDRSZ > buf.len() {
continue;
}
let flags = page_flags(buf, page_off);
if flags & P_BRANCH != 0 {
let nkeys = page_nkeys(buf, page_off);
for i in 0..nkeys {
let idx_off = page_off + PAGEHDRSZ + i * 2;
if idx_off + 2 > buf.len() { break; }
let node_rel = u16_le(buf, idx_off) as usize;
let node_abs = page_off + PAGEHDRSZ + node_rel;
if node_abs + 4 > buf.len() { break; }
stack.push(u32_le(buf, node_abs));
}
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
let nkeys = page_nkeys(buf, page_off);
for i in 0..nkeys {
let idx_off = page_off + PAGEHDRSZ + i * 2;
if idx_off + 2 > buf.len() { break; }
let node_rel = u16_le(buf, idx_off) as usize;
let node_abs = page_off + PAGEHDRSZ + node_rel;
if node_abs + NODESIZE > buf.len() { break; }
let mn_flags = buf[node_abs + 4];
let ksize = u16_le(buf, node_abs + 6) as usize;
let data_off = node_abs + NODESIZE + ksize;
if mn_flags & N_BIG != 0 {
if data_off + 4 > buf.len() { continue; }
let ov_pgno = u32_le(buf, data_off) as usize;
if ov_pgno >= page_count { continue; }
let ov_page_off = ov_pgno * ps;
if ov_page_off + PAGEHDRSZ > buf.len() { continue; }
let ov_count = page_overflow_count(buf, ov_page_off) as usize;
for op in ov_pgno..ov_pgno + ov_count {
if op < page_count && !visited[op] {
visited[op] = true;
result.push((op, dbi_index));
}
}
} else if (mn_flags & (N_DUP | N_TREE)) == (N_DUP | N_TREE) {
if data_off + 48 > buf.len() { continue; }
let sub = TreeDescriptor::parse(buf, data_off);
if sub.root != P_INVALID {
stack.push(sub.root);
}
}
}
} else if flags & P_LARGE != 0 {
let ov_count = page_overflow_count(buf, page_off) as usize;
for op in (pgno_usize + 1)..(pgno_usize + ov_count) {
if op < page_count && !visited[op] {
visited[op] = true;
result.push((op, dbi_index));
}
}
}
}
result
}
fn discover_named_dbis(
buf: &[u8],
ps: usize,
main_root: u32,
owner_map: &mut [u8],
conflicts: &mut u64,
dbi_main: u8,
dbi_start: u8,
) -> Vec<DBIInfo> {
if main_root == P_INVALID || main_root as usize >= owner_map.len() {
return Vec::new();
}
let mut named: Vec<DBIInfo> = Vec::new();
let mut next_index = dbi_start;
let mut stack: Vec<u32> = vec![main_root];
while let Some(pgno) = stack.pop() {
let pgno_usize = pgno as usize;
if pgno_usize >= owner_map.len() { continue; }
if !mark(owner_map, pgno_usize, dbi_main, conflicts) { continue; }
let page_off = pgno_usize * ps;
if page_off + PAGEHDRSZ > buf.len() { continue; }
let flags = page_flags(buf, page_off);
if flags & P_BRANCH != 0 {
let nkeys = page_nkeys(buf, page_off);
for i in 0..nkeys {
let idx_off = page_off + PAGEHDRSZ + i * 2;
if idx_off + 2 > buf.len() { break; }
let node_rel = u16_le(buf, idx_off) as usize;
let node_abs = page_off + PAGEHDRSZ + node_rel;
if node_abs + 4 > buf.len() { break; }
let child_pgno = u32_le(buf, node_abs);
stack.push(child_pgno);
}
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
let nkeys = page_nkeys(buf, page_off);
for i in 0..nkeys {
let idx_off = page_off + PAGEHDRSZ + i * 2;
if idx_off + 2 > buf.len() { break; }
let node_rel = u16_le(buf, idx_off) as usize;
let node_abs = page_off + PAGEHDRSZ + node_rel;
if node_abs + NODESIZE > buf.len() { break; }
let mn_flags = buf[node_abs + 4];
let ksize = u16_le(buf, node_abs + 6) as usize;
let data_off = node_abs + NODESIZE + ksize;
if mn_flags & N_TREE != 0 {
let key_off = node_abs + NODESIZE;
if key_off + ksize > buf.len() || data_off + 48 > buf.len() { continue; }
let name_bytes = &buf[key_off..key_off + ksize];
let name = String::from_utf8_lossy(
name_bytes.split(|&b| b == 0).next().unwrap_or(name_bytes)
).to_string();
let tree = TreeDescriptor::parse(buf, data_off);
named.push(DBIInfo { name, dbi_index: next_index, tree });
next_index = next_index.saturating_add(1);
} else if mn_flags & N_BIG != 0 {
if data_off + 4 > buf.len() { continue; }
let ov_pgno = u32_le(buf, data_off) as usize;
if ov_pgno >= owner_map.len() { continue; }
let ov_page_off = ov_pgno * ps;
if ov_page_off + PAGEHDRSZ > buf.len() { continue; }
let ov_count = page_overflow_count(buf, ov_page_off) as usize;
for op in ov_pgno..ov_pgno + ov_count {
mark(owner_map, op, dbi_main, conflicts);
}
}
}
}
}
named
}
fn mark_free_pages(
buf: &[u8],
ps: usize,
free_root: u32,
owner_map: &mut [u8],
conflicts: &mut u64,
dbi_free: u8,
) -> u64 {
if free_root == P_INVALID || free_root as usize >= owner_map.len() {
return 0;
}
let mut marked: u64 = 0;
let mut stack: Vec<u32> = vec![free_root];
while let Some(pgno) = stack.pop() {
let pgno_usize = pgno as usize;
if pgno_usize >= owner_map.len() { continue; }
if !mark(owner_map, pgno_usize, dbi_free, conflicts) { continue; }
marked += 1;
let page_off = pgno_usize * ps;
if page_off + PAGEHDRSZ > buf.len() { continue; }
let flags = page_flags(buf, page_off);
if flags & P_BRANCH != 0 {
let nkeys = page_nkeys(buf, page_off);
for i in 0..nkeys {
let idx_off = page_off + PAGEHDRSZ + i * 2;
if idx_off + 2 > buf.len() { break; }
let node_rel = u16_le(buf, idx_off) as usize;
let child_pgno = u32_le(buf, page_off + PAGEHDRSZ + node_rel);
stack.push(child_pgno);
}
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
let nkeys = page_nkeys(buf, page_off);
for i in 0..nkeys {
let idx_off = page_off + PAGEHDRSZ + i * 2;
if idx_off + 2 > buf.len() { break; }
let node_rel = u16_le(buf, idx_off) as usize;
let node_abs = page_off + PAGEHDRSZ + node_rel;
if node_abs + NODESIZE > buf.len() { break; }
let mn_flags = buf[node_abs + 4];
let ksize = u16_le(buf, node_abs + 6) as usize;
let dsize = u32_le(buf, node_abs) as usize;
let data_off = node_abs + NODESIZE + ksize;
if mn_flags & N_BIG != 0 {
if data_off + 4 > buf.len() { continue; }
let ov_pgno = u32_le(buf, data_off) as usize;
if ov_pgno >= owner_map.len() { continue; }
let ov_page_off = ov_pgno * ps;
if ov_page_off + PAGEHDRSZ > buf.len() { continue; }
let ov_count = page_overflow_count(buf, ov_page_off) as usize;
for op in ov_pgno..ov_pgno + ov_count {
if mark(owner_map, op, dbi_free, conflicts) { marked += 1; }
}
let ov_data_off = ov_pgno * ps + PAGEHDRSZ;
if dsize >= 4 && ov_data_off + 4 <= buf.len() {
let pnl_count = u32_le(buf, ov_data_off) as usize;
for j in 0..pnl_count {
let fp_off = ov_data_off + 4 + j * 4;
if fp_off + 4 > buf.len() { break; }
let fp = u32_le(buf, fp_off) as usize;
if fp < owner_map.len() && owner_map[fp] == UNREFERENCED {
owner_map[fp] = dbi_free;
marked += 1;
}
}
}
} else {
if dsize >= 4 && data_off + 4 <= buf.len() {
let pnl_count = u32_le(buf, data_off) as usize;
for j in 0..pnl_count {
let fp_off = data_off + 4 + j * 4;
if fp_off + 4 > buf.len() { break; }
let fp = u32_le(buf, fp_off) as usize;
if fp < owner_map.len() && owner_map[fp] == UNREFERENCED {
owner_map[fp] = dbi_free;
marked += 1;
}
}
}
}
}
}
}
marked
}
#[derive(Parser, Debug)]
pub struct Command {
#[arg(short, long, default_value = "owner_map.bin")]
output: PathBuf,
}
impl Command {
pub fn execute<N: NodeTypesWithDB>(
self,
data_dir: ChainPath<DataDirPath>,
_tool: &DbTool<N>,
) -> eyre::Result<()> {
let db_path = data_dir.db().join("mdbx.dat");
eyre::ensure!(db_path.exists(), "mdbx.dat not found at {:?}", db_path);
let t0 = Instant::now();
let file = fs::File::open(&db_path)
.wrap_err_with(|| format!("Failed to open {}", db_path.display()))?;
let mmap = unsafe { Mmap::map(&file)? };
let buf: &[u8] = &mmap;
if buf.len() < PAGEHDRSZ + 0xC0 {
eyre::bail!("File too small for MDBX meta page");
}
let flags0 = u16_le(buf, 0x0A);
if flags0 & P_META == 0 {
eyre::bail!("Page 0 missing P_META flag");
}
let magic = u64_le(buf, PAGEHDRSZ);
if (magic >> 8) != MDBX_MAGIC {
eyre::bail!("MDBX magic mismatch");
}
let candidates = [4096usize, 8192, 16384, 32768, 65536];
let mut ps = 4096usize;
let geo_now_raw = u32_le(buf, PAGEHDRSZ + META_GEO + GEO_NOW) as usize;
for &candidate in &candidates {
let mapped = geo_now_raw * candidate;
if mapped >= buf.len() / 2 && mapped <= buf.len() * 4 {
ps = candidate;
break;
}
}
let mut best_txnid: u64 = 0;
let mut best_meta_base: usize = 0;
for pgno in 0..3usize {
let page_base = pgno * ps;
let meta_base = page_base + PAGEHDRSZ;
if meta_base + 0xC0 > buf.len() { continue; }
let pflags = u16_le(buf, page_base + 0x0A);
if pflags & P_META == 0 { continue; }
let m = u64_le(buf, meta_base);
if (m >> 8) != MDBX_MAGIC { continue; }
let txnid_a = u64_le(buf, meta_base + META_TXNID_A);
let txnid_b = u64_le(buf, meta_base + META_TXNID_B);
if txnid_a == txnid_b && txnid_a > best_txnid {
best_txnid = txnid_a;
best_meta_base = meta_base;
}
}
eyre::ensure!(best_txnid > 0, "No valid meta page found");
let page_count = u32_le(buf, best_meta_base + META_GEO + GEO_NOW) as usize;
let free_tree = TreeDescriptor::parse(buf, best_meta_base + META_FREE_DB);
let main_tree = TreeDescriptor::parse(buf, best_meta_base + META_MAIN_DB);
println!("MDBX owner map generator (parallel)");
println!(" page_size: {ps}");
println!(" page_count: {page_count}");
println!(" txnid: {best_txnid}");
println!(" FreeDB root: {} ({} items)", free_tree.root, free_tree.items);
println!(" MainDB root: {} ({} items)", main_tree.root, main_tree.items);
println!();
let mut owner_map = vec![UNREFERENCED; page_count];
let mut conflicts: u64 = 0;
for pgno in 0..std::cmp::min(3, page_count) {
owner_map[pgno] = DBI_META;
}
println!("Discovering named DBIs via MainDB walk...");
let discovered = discover_named_dbis(
buf, ps, main_tree.root, &mut owner_map, &mut conflicts, 1, 2,
);
let mut name_to_dbi: HashMap<&str, u8> = HashMap::new();
for (idx, table) in Tables::ALL.iter().enumerate() {
name_to_dbi.insert(table.name(), (idx + 2) as u8);
}
let mut named_dbis: Vec<DBIInfo> = Vec::new();
let mut remap: HashMap<u8, u8> = HashMap::new();
for dbi in &discovered {
if let Some(&real_dbi) = name_to_dbi.get(dbi.name.as_str()) {
remap.insert(dbi.dbi_index, real_dbi);
named_dbis.push(DBIInfo {
name: dbi.name.clone(),
dbi_index: real_dbi,
tree: dbi.tree.clone(),
});
} else {
named_dbis.push(dbi.clone());
}
}
for byte in owner_map.iter_mut() {
if let Some(&new_dbi) = remap.get(byte) {
*byte = new_dbi;
}
}
println!("Found {} named DBI(s)", named_dbis.len());
println!("Walking FreeDB / GC...");
let free_marked = mark_free_pages(buf, ps, free_tree.root, &mut owner_map, &mut conflicts, 0);
println!("FreeDB: {} pages marked", free_marked);
println!("Walking {} named DBIs in parallel...", named_dbis.len());
let t_par = Instant::now();
let results: Vec<(String, u8, Vec<(usize, u8)>)> = named_dbis
.par_iter()
.filter(|dbi| dbi.tree.root != P_INVALID)
.map(|dbi| {
let pages = walk_tree_collect(buf, ps, dbi.tree.root, page_count, dbi.dbi_index);
(dbi.name.clone(), dbi.dbi_index, pages)
})
.collect();
for (name, dbi_index, pages) in &results {
let count = pages.len();
for &(pgno, idx) in pages {
mark(&mut owner_map, pgno, idx, &mut conflicts);
}
println!(" [{:2}] {:30} {:>10} pages", dbi_index, name, count);
}
println!("Parallel walk done in {:.2}s", t_par.elapsed().as_secs_f64());
let elapsed = t0.elapsed();
let mut counts: HashMap<u8, u64> = HashMap::new();
let mut unreferenced: u64 = 0;
for &b in &owner_map {
if b == UNREFERENCED {
unreferenced += 1;
} else {
*counts.entry(b).or_default() += 1;
}
}
println!();
println!("Walk complete in {:.2}s", elapsed.as_secs_f64());
println!(" Total pages: {page_count}");
println!(" Unreferenced: {unreferenced}");
println!(" Conflicts: {conflicts}");
println!();
if let Some(&c) = counts.get(&DBI_META) {
println!(" [0xFE] {:30} {:>10} pages", "<meta>", c);
}
if let Some(&c) = counts.get(&0) {
println!(" [ 0] {:30} {:>10} pages", "<free/GC>", c);
}
if let Some(&c) = counts.get(&1) {
println!(" [ 1] {:30} {:>10} pages", "<main>", c);
}
let mut sorted_dbis = named_dbis.clone();
sorted_dbis.sort_by_key(|d| d.dbi_index);
for dbi in &sorted_dbis {
let walked = counts.get(&dbi.dbi_index).copied().unwrap_or(0);
let expected = dbi.tree.total_pages();
let mismatch = if expected > 0 && walked != expected {
format!(" !! MISMATCH (tree_t says {})", expected)
} else {
String::new()
};
println!(" [{:3}] {:30} {:>10} pages{}", dbi.dbi_index, dbi.name, walked, mismatch);
}
println!(" [0xFF] {:30} {:>10} pages", "<unreferenced>", unreferenced);
let mut out = fs::File::create(&self.output)?;
out.write_all(&owner_map)?;
println!();
println!("Written {} bytes to {}", owner_map.len(), self.output.display());
Ok(())
}
}

View File

@@ -98,8 +98,7 @@ impl Command {
)?;
if let Some(entry) = entry {
let se: reth_primitives_traits::StorageEntry = entry.into();
println!("{}", serde_json::to_string_pretty(&se)?);
println!("{}", serde_json::to_string_pretty(&entry)?);
} else {
error!(target: "reth::cli", "No content for the given table key.");
}
@@ -107,14 +106,7 @@ impl Command {
}
let changesets = provider.storage_changeset(key.block_number())?;
let serializable: Vec<_> = changesets
.into_iter()
.map(|(addr, entry)| {
let se: reth_primitives_traits::StorageEntry = entry.into();
(addr, se)
})
.collect();
println!("{}", serde_json::to_string_pretty(&serializable)?);
println!("{}", serde_json::to_string_pretty(&changesets)?);
return Ok(());
}

View File

@@ -12,7 +12,6 @@ use std::{
mod account_storage;
mod checksum;
mod clear;
mod copy;
mod diff;
mod get;
mod list;
@@ -21,7 +20,6 @@ mod settings;
mod state;
mod static_file_header;
mod stats;
pub mod gen_ownermap;
/// DB List TUI
mod tui;
@@ -44,8 +42,6 @@ pub enum Subcommands {
List(list::Command),
/// Calculates the content checksum of a table or static file segment
Checksum(checksum::Command),
/// Copies the MDBX database to a new location (bundled mdbx_copy)
Copy(copy::Command),
/// Create a diff between two database tables or two entire databases.
Diff(diff::Command),
/// Gets the content of a table for the given key
@@ -72,8 +68,6 @@ pub enum Subcommands {
AccountStorage(account_storage::Command),
/// Gets account state and storage at a specific block
State(state::Command),
/// Generate a page owner map for MDBX page visualization
GenOwnermap(gen_ownermap::Command),
}
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C> {
@@ -130,11 +124,6 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
command.execute(&tool)?;
});
}
Subcommands::Copy(command) => {
db_exec!(self.env, tool, N, AccessRights::RO, {
command.execute(tool.provider_factory.db_ref())?;
});
}
Subcommands::Diff(command) => {
db_exec!(self.env, tool, N, AccessRights::RO, {
command.execute(&tool)?;
@@ -217,11 +206,6 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
command.execute(&tool)?;
});
}
Subcommands::GenOwnermap(command) => {
db_exec!(self.env, tool, N, AccessRights::RO, {
command.execute(data_dir, &tool)?;
});
}
}
Ok(())

View File

@@ -39,12 +39,50 @@ enum Subcommands {
#[derive(Debug, Clone, Copy, Subcommand)]
#[clap(rename_all = "snake_case")]
pub enum SetCommand {
/// Enable or disable v2 storage layout
/// Store receipts in static files instead of the database
Receipts {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store transaction senders in static files instead of the database
TransactionSenders {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store account changesets in static files instead of the database
AccountChangesets {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store storage history in rocksdb instead of MDBX
StoragesHistory {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store transaction hash to number mapping in rocksdb instead of MDBX
TransactionHashNumbers {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store account history in rocksdb instead of MDBX
AccountHistory {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store storage changesets in static files instead of the database
StorageChangesets {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Use hashed state tables (HashedAccounts/HashedStorages) as canonical state
///
/// When enabled, uses static files for receipts/senders/changesets and RocksDB for
/// history indices and transaction hashes. When disabled, uses v1/legacy layout (everything in
/// MDBX).
V2 {
/// When enabled, execution writes directly to hashed tables, eliminating need for
/// separate hashing stages. State reads come from hashed tables.
///
/// WARNING: Changing this setting in either direction requires re-syncing the database.
/// Enabling on an existing plain-state database leaves hashed tables empty.
/// Disabling on an existing hashed-state database leaves plain tables empty.
UseHashedState {
#[clap(action(ArgAction::Set))]
value: bool,
},
@@ -87,18 +125,87 @@ impl Command {
println!("No storage settings found, creating new settings.");
}
let mut settings @ StorageSettings { storage_v2: _ } =
settings.unwrap_or_else(StorageSettings::v1);
let mut settings @ StorageSettings {
receipts_in_static_files: _,
transaction_senders_in_static_files: _,
storages_history_in_rocksdb: _,
transaction_hash_numbers_in_rocksdb: _,
account_history_in_rocksdb: _,
account_changesets_in_static_files: _,
storage_changesets_in_static_files: _,
use_hashed_state: _,
} = settings.unwrap_or_else(StorageSettings::v1);
// Update the setting based on the key
match cmd {
SetCommand::V2 { value } => {
if settings.storage_v2 == value {
println!("storage_v2 is already set to {}", value);
SetCommand::Receipts { value } => {
if settings.receipts_in_static_files == value {
println!("receipts_in_static_files is already set to {}", value);
return Ok(());
}
settings.storage_v2 = value;
println!("Set storage_v2 = {}", value);
settings.receipts_in_static_files = value;
println!("Set receipts_in_static_files = {}", value);
}
SetCommand::TransactionSenders { value } => {
if settings.transaction_senders_in_static_files == value {
println!("transaction_senders_in_static_files is already set to {}", value);
return Ok(());
}
settings.transaction_senders_in_static_files = value;
println!("Set transaction_senders_in_static_files = {}", value);
}
SetCommand::AccountChangesets { value } => {
if settings.account_changesets_in_static_files == value {
println!("account_changesets_in_static_files is already set to {}", value);
return Ok(());
}
settings.account_changesets_in_static_files = value;
println!("Set account_changesets_in_static_files = {}", value);
}
SetCommand::StoragesHistory { value } => {
if settings.storages_history_in_rocksdb == value {
println!("storages_history_in_rocksdb is already set to {}", value);
return Ok(());
}
settings.storages_history_in_rocksdb = value;
println!("Set storages_history_in_rocksdb = {}", value);
}
SetCommand::TransactionHashNumbers { value } => {
if settings.transaction_hash_numbers_in_rocksdb == value {
println!("transaction_hash_numbers_in_rocksdb is already set to {}", value);
return Ok(());
}
settings.transaction_hash_numbers_in_rocksdb = value;
println!("Set transaction_hash_numbers_in_rocksdb = {}", value);
}
SetCommand::AccountHistory { value } => {
if settings.account_history_in_rocksdb == value {
println!("account_history_in_rocksdb is already set to {}", value);
return Ok(());
}
settings.account_history_in_rocksdb = value;
println!("Set account_history_in_rocksdb = {}", value);
}
SetCommand::StorageChangesets { value } => {
if settings.storage_changesets_in_static_files == value {
println!("storage_changesets_in_static_files is already set to {}", value);
return Ok(());
}
settings.storage_changesets_in_static_files = value;
println!("Set storage_changesets_in_static_files = {}", value);
}
SetCommand::UseHashedState { value } => {
if settings.use_hashed_state == value {
println!("use_hashed_state is already set to {}", value);
return Ok(());
}
if settings.use_hashed_state && !value {
println!("WARNING: Disabling use_hashed_state on an existing hashed-state database requires a full resync.");
} else {
println!("WARNING: Enabling use_hashed_state on an existing plain-state database requires a full resync.");
}
settings.use_hashed_state = value;
println!("Set use_hashed_state = {}", value);
}
}

View File

@@ -1,4 +1,4 @@
use alloy_primitives::{keccak256, Address, BlockNumber, B256, U256};
use alloy_primitives::{Address, BlockNumber, B256, U256};
use clap::Parser;
use parking_lot::Mutex;
use reth_db_api::{
@@ -63,65 +63,39 @@ impl Command {
address: Address,
limit: usize,
) -> eyre::Result<()> {
let use_hashed_state = tool.provider_factory.cached_storage_settings().use_hashed_state();
let entries = tool.provider_factory.db_ref().view(|tx| {
let (account, walker_entries) = if use_hashed_state {
let hashed_address = keccak256(address);
let account = tx.get::<tables::HashedAccounts>(hashed_address)?;
let mut cursor = tx.cursor_dup_read::<tables::HashedStorages>()?;
let walker = cursor.walk_dup(Some(hashed_address), None)?;
let mut entries = Vec::new();
let mut last_log = Instant::now();
for (idx, entry) in walker.enumerate() {
let (_, storage_entry) = entry?;
if storage_entry.value != U256::ZERO {
entries.push((storage_entry.key, storage_entry.value));
}
if entries.len() >= limit {
break;
}
if last_log.elapsed() >= LOG_INTERVAL {
info!(
target: "reth::cli",
address = %address,
slots_scanned = idx,
"Scanning storage slots"
);
last_log = Instant::now();
}
}
(account, entries)
} else {
// Get account info
let account = tx.get::<tables::PlainAccountState>(address)?;
// Get storage entries
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
let walker = cursor.walk_dup(Some(address), None)?;
let mut entries = Vec::new();
let mut last_log = Instant::now();
for (idx, entry) in walker.enumerate() {
let (_, storage_entry) = entry?;
if storage_entry.value != U256::ZERO {
entries.push((storage_entry.key, storage_entry.value));
}
if entries.len() >= limit {
break;
}
if last_log.elapsed() >= LOG_INTERVAL {
info!(
target: "reth::cli",
address = %address,
slots_scanned = idx,
"Scanning storage slots"
);
last_log = Instant::now();
}
}
(account, entries)
};
// Get account info
let account = tx.get::<tables::PlainAccountState>(address)?;
Ok::<_, eyre::Report>((account, walker_entries))
// Get storage entries
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
let mut entries = Vec::new();
let mut last_log = Instant::now();
let walker = cursor.walk_dup(Some(address), None)?;
for (idx, entry) in walker.enumerate() {
let (_, storage_entry) = entry?;
if storage_entry.value != U256::ZERO {
entries.push((storage_entry.key, storage_entry.value));
}
if entries.len() >= limit {
break;
}
if last_log.elapsed() >= LOG_INTERVAL {
info!(
target: "reth::cli",
address = %address,
slots_scanned = idx,
"Scanning storage slots"
);
last_log = Instant::now();
}
}
Ok::<_, eyre::Report>((account, entries))
})??;
let (account, storage_entries) = entries;
@@ -145,7 +119,7 @@ impl Command {
// Check storage settings to determine where history is stored
let storage_settings = tool.provider_factory.cached_storage_settings();
let history_in_rocksdb = storage_settings.storage_v2;
let history_in_rocksdb = storage_settings.storages_history_in_rocksdb;
// For historical queries, enumerate keys from history indices only
// (not PlainStorageState, which reflects current state)

View File

@@ -37,14 +37,6 @@ pub struct DownloadDefaults {
pub available_snapshots: Vec<Cow<'static, str>>,
/// Default base URL for snapshots
pub default_base_url: Cow<'static, str>,
/// Default base URL for chain-aware snapshots.
///
/// When set, the chain ID is appended to form the full URL: `{base_url}/{chain_id}`.
/// For example, given a base URL of `https://snapshots.example.com` and chain ID `1`,
/// the resulting URL would be `https://snapshots.example.com/1`.
///
/// Falls back to [`default_base_url`](Self::default_base_url) when `None`.
pub default_chain_aware_base_url: Option<Cow<'static, str>>,
/// Optional custom long help text that overrides the generated help
pub long_help: Option<String>,
}
@@ -68,7 +60,6 @@ impl DownloadDefaults {
Cow::Borrowed("https://publicnode.com/snapshots (full nodes & testnets)"),
],
default_base_url: Cow::Borrowed(MERKLE_BASE_URL),
default_chain_aware_base_url: None,
long_help: None,
}
}
@@ -93,11 +84,9 @@ impl DownloadDefaults {
}
help.push_str(
"\nIf no URL is provided, the latest archive snapshot for the selected chain\nwill be proposed for download from ",
);
help.push_str(
self.default_chain_aware_base_url.as_deref().unwrap_or(&self.default_base_url),
"\nIf no URL is provided, the latest mainnet archive snapshot\nwill be proposed for download from ",
);
help.push_str(self.default_base_url.as_ref());
help.push_str(
".\n\nLocal file:// URLs are also supported for extracting snapshots from disk.",
);
@@ -122,12 +111,6 @@ impl DownloadDefaults {
self
}
/// Set the default chain-aware base URL.
pub fn with_chain_aware_base_url(mut self, url: impl Into<Cow<'static, str>>) -> Self {
self.default_chain_aware_base_url = Some(url.into());
self
}
/// Builder: Set custom long help text, overriding the generated help
pub fn with_long_help(mut self, help: impl Into<String>) -> Self {
self.long_help = Some(help.into());
@@ -159,7 +142,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> DownloadCo
let url = match self.url {
Some(url) => url,
None => {
let url = get_latest_snapshot_url(self.env.chain.chain().id()).await?;
let url = get_latest_snapshot_url().await?;
info!(target: "reth::cli", "Using default snapshot URL: {}", url);
url
}
@@ -526,12 +509,8 @@ async fn stream_and_extract(url: &str, target_dir: &Path) -> Result<()> {
}
// Builds default URL for latest mainnet archive snapshot using configured defaults
async fn get_latest_snapshot_url(chain_id: u64) -> Result<String> {
let defaults = DownloadDefaults::get_global();
let base_url = match &defaults.default_chain_aware_base_url {
Some(url) => format!("{url}/{chain_id}"),
None => defaults.default_base_url.to_string(),
};
async fn get_latest_snapshot_url() -> Result<String> {
let base_url = &DownloadDefaults::get_global().default_base_url;
let latest_url = format!("{base_url}/latest.txt");
let filename = Client::new()
.get(latest_url)

View File

@@ -10,8 +10,8 @@ use reth_node_builder::NodeBuilder;
use reth_node_core::{
args::{
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, EraArgs, MetricArgs,
NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, StaticFilesArgs, StorageArgs,
TxPoolArgs,
NetworkArgs, PayloadBuilderArgs, PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs,
StorageArgs, TxPoolArgs,
},
node_config::NodeConfig,
version,
@@ -103,6 +103,10 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
#[command(flatten)]
pub pruning: PruningArgs,
/// All `RocksDB` table routing arguments
#[command(flatten)]
pub rocksdb: RocksDbArgs,
/// Engine cli arguments
#[command(flatten, next_help_heading = "Engine")]
pub engine: EngineArgs,
@@ -115,8 +119,8 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
#[command(flatten, next_help_heading = "Static Files")]
pub static_files: StaticFilesArgs,
/// All storage related arguments with --storage prefix
#[command(flatten, next_help_heading = "Storage")]
/// Storage mode configuration (v2 vs v1/legacy)
#[command(flatten)]
pub storage: StorageArgs,
/// Additional cli arguments
@@ -171,6 +175,7 @@ where
db,
dev,
pruning,
rocksdb,
engine,
era,
static_files,
@@ -178,6 +183,9 @@ where
ext,
} = self;
// Validate RocksDB arguments
rocksdb.validate()?;
// set up node config
let mut node_config = NodeConfig {
datadir,
@@ -193,6 +201,7 @@ where
db,
dev,
pruning,
rocksdb,
engine,
era,
static_files,

View File

@@ -45,16 +45,12 @@ impl<C: ChainSpecParser> Command<C> {
let tool = DbTool::new(provider_factory)?;
let static_file_segments = match self.stage {
StageEnum::Headers => vec![StaticFileSegment::Headers],
StageEnum::Bodies => vec![StaticFileSegment::Transactions],
StageEnum::Execution => vec![
StaticFileSegment::Receipts,
StaticFileSegment::AccountChangeSets,
StaticFileSegment::StorageChangeSets,
],
StageEnum::Senders => vec![StaticFileSegment::TransactionSenders],
_ => vec![],
let static_file_segment = match self.stage {
StageEnum::Headers => Some(StaticFileSegment::Headers),
StageEnum::Bodies => Some(StaticFileSegment::Transactions),
StageEnum::Execution => Some(StaticFileSegment::Receipts),
StageEnum::Senders => Some(StaticFileSegment::TransactionSenders),
_ => None,
};
// Calling `StaticFileProviderRW::prune_*` will instruct the writer to prune rows only
@@ -62,33 +58,35 @@ impl<C: ChainSpecParser> Command<C> {
// deleting the jar files, otherwise if the task were to be interrupted after we
// have deleted them, BUT before we have committed the checkpoints to the database, we'd
// lose essential data.
let static_file_provider = tool.provider_factory.static_file_provider();
for segment in static_file_segments {
if let Some(highest_block) = static_file_provider.get_highest_static_file_block(segment)
if let Some(static_file_segment) = static_file_segment {
let static_file_provider = tool.provider_factory.static_file_provider();
if let Some(highest_block) =
static_file_provider.get_highest_static_file_block(static_file_segment)
{
let mut writer = static_file_provider.latest_writer(segment)?;
let mut writer = static_file_provider.latest_writer(static_file_segment)?;
match segment {
match static_file_segment {
StaticFileSegment::Headers => {
// Prune all headers leaving genesis intact.
writer.prune_headers(highest_block)?;
}
StaticFileSegment::Transactions => {
let to_delete = static_file_provider
.get_highest_static_file_tx(segment)
.get_highest_static_file_tx(static_file_segment)
.map(|tx_num| tx_num + 1)
.unwrap_or_default();
writer.prune_transactions(to_delete, 0)?;
}
StaticFileSegment::Receipts => {
let to_delete = static_file_provider
.get_highest_static_file_tx(segment)
.get_highest_static_file_tx(static_file_segment)
.map(|tx_num| tx_num + 1)
.unwrap_or_default();
writer.prune_receipts(to_delete, 0)?;
}
StaticFileSegment::TransactionSenders => {
let to_delete = static_file_provider
.get_highest_static_file_tx(segment)
.get_highest_static_file_tx(static_file_segment)
.map(|tx_num| tx_num + 1)
.unwrap_or_default();
writer.prune_transaction_senders(to_delete, 0)?;
@@ -133,15 +131,8 @@ impl<C: ChainSpecParser> Command<C> {
reset_stage_checkpoint(tx, StageId::SenderRecovery)?;
}
StageEnum::Execution => {
if provider_rw.cached_storage_settings().use_hashed_state() {
tx.clear::<tables::HashedAccounts>()?;
tx.clear::<tables::HashedStorages>()?;
reset_stage_checkpoint(tx, StageId::AccountHashing)?;
reset_stage_checkpoint(tx, StageId::StorageHashing)?;
} else {
tx.clear::<tables::PlainAccountState>()?;
tx.clear::<tables::PlainStorageState>()?;
}
tx.clear::<tables::PlainAccountState>()?;
tx.clear::<tables::PlainStorageState>()?;
tx.clear::<tables::AccountChangeSets>()?;
tx.clear::<tables::StorageChangeSets>()?;
tx.clear::<tables::Bytecodes>()?;
@@ -187,7 +178,7 @@ impl<C: ChainSpecParser> Command<C> {
let settings = provider_rw.cached_storage_settings();
let rocksdb = tool.provider_factory.rocksdb_provider();
if settings.storage_v2 {
if settings.account_history_in_rocksdb {
rocksdb.clear::<tables::AccountsHistory>()?;
} else {
tx.clear::<tables::AccountsHistory>()?;
@@ -204,7 +195,7 @@ impl<C: ChainSpecParser> Command<C> {
let settings = provider_rw.cached_storage_settings();
let rocksdb = tool.provider_factory.rocksdb_provider();
if settings.storage_v2 {
if settings.storages_history_in_rocksdb {
rocksdb.clear::<tables::StoragesHistory>()?;
} else {
tx.clear::<tables::StoragesHistory>()?;
@@ -218,7 +209,7 @@ impl<C: ChainSpecParser> Command<C> {
)?;
}
StageEnum::TxLookup => {
if provider_rw.cached_storage_settings().storage_v2 {
if provider_rw.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
tool.provider_factory
.rocksdb_provider()
.clear::<tables::TransactionHashNumbers>()?;

View File

@@ -1,6 +1,6 @@
//! Test setup utilities for configuring the initial state.
use crate::{testsuite::Environment, E2ETestSetupBuilder, NodeBuilderHelper};
use crate::{setup_engine_with_connection, testsuite::Environment, NodeBuilderHelper};
use alloy_eips::BlockNumberOrTag;
use alloy_primitives::B256;
use alloy_rpc_types_engine::{ForkchoiceState, PayloadAttributes};
@@ -38,8 +38,6 @@ pub struct Setup<I> {
shutdown_tx: Option<mpsc::Sender<()>>,
/// Is this setup in dev mode
pub is_dev: bool,
/// Whether to use v2 storage mode (hashed keys, static file changesets, rocksdb history)
pub storage_v2: bool,
/// Tracks instance generic.
_phantom: PhantomData<I>,
/// Holds the import result to keep nodes alive when using imported chain
@@ -60,7 +58,6 @@ impl<I> Default for Setup<I> {
tree_config: TreeConfig::default(),
shutdown_tx: None,
is_dev: true,
storage_v2: false,
_phantom: Default::default(),
import_result_holder: None,
import_rlp_path: None,
@@ -129,12 +126,6 @@ where
self
}
/// Enable v2 storage mode (hashed keys, static file changesets, rocksdb history)
pub const fn with_storage_v2(mut self) -> Self {
self.storage_v2 = true;
self
}
/// Apply setup using pre-imported chain data from RLP file
pub async fn apply_with_import<N>(
&mut self,
@@ -203,28 +194,19 @@ where
self.shutdown_tx = Some(shutdown_tx);
let is_dev = self.is_dev;
let storage_v2 = self.storage_v2;
let node_count = self.network.node_count;
let tree_config = self.tree_config.clone();
let attributes_generator = Self::create_static_attributes_generator::<N>();
let mut builder = E2ETestSetupBuilder::<N, _>::new(
let result = setup_engine_with_connection::<N>(
node_count,
Arc::<N::ChainSpec>::new((*chain_spec).clone().into()),
is_dev,
self.tree_config.clone(),
attributes_generator,
self.network.connect_nodes,
)
.with_tree_config_modifier(move |base| {
tree_config.clone().with_cross_block_cache_size(base.cross_block_cache_size())
})
.with_node_config_modifier(move |config| config.set_dev(is_dev))
.with_connect_nodes(self.network.connect_nodes);
if storage_v2 {
builder = builder.with_storage_v2();
}
let result = builder.build().await;
.await;
let mut node_clients = Vec::new();
match result {

View File

@@ -10,6 +10,7 @@ use jsonrpsee::core::client::ClientT;
use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET};
use reth_db::tables;
use reth_e2e_test_utils::{transaction::TransactionTestContext, wallet, E2ETestSetupBuilder};
use reth_node_core::args::RocksDbArgs;
use reth_node_ethereum::EthereumNode;
use reth_payload_builder::EthPayloadBuilderAttributes;
use reth_provider::RocksDBProviderFactory;
@@ -95,6 +96,22 @@ fn test_attributes_generator(timestamp: u64) -> EthPayloadBuilderAttributes {
EthPayloadBuilderAttributes::new(B256::ZERO, attributes)
}
/// Verifies that `RocksDB` CLI defaults are `None` (deferred to storage mode).
#[test]
fn test_rocksdb_defaults_are_none() {
let args = RocksDbArgs::default();
assert!(args.tx_hash.is_none(), "tx_hash default should be None (deferred to --storage.v2)");
assert!(
args.storages_history.is_none(),
"storages_history default should be None (deferred to --storage.v2)"
);
assert!(
args.account_history.is_none(),
"account_history default should be None (deferred to --storage.v2)"
);
}
/// Smoke test: node boots with `RocksDB` routing enabled.
#[tokio::test]
async fn test_rocksdb_node_startup() -> Result<()> {
@@ -460,7 +477,7 @@ async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> {
///
/// This test exercises `unwind_trie_state_from` which previously failed with
/// `UnsortedInput` errors because it read changesets directly from MDBX tables
/// instead of using storage-aware methods that check `is_v2()`.
/// instead of using storage-aware methods that check `storage_changesets_in_static_files`.
#[tokio::test]
async fn test_rocksdb_reorg_unwind() -> Result<()> {
reth_tracing::init_test_tracing();

View File

@@ -179,8 +179,6 @@ pub struct TreeConfig {
sparse_trie_prune_depth: usize,
/// Maximum number of storage tries to retain after pruning.
sparse_trie_max_storage_tries: usize,
/// Whether to fully disable sparse trie cache pruning between blocks.
disable_sparse_trie_cache_pruning: bool,
/// Timeout for the state root task before spawning a sequential fallback computation.
/// If `Some`, after waiting this duration for the state root task, a sequential state root
/// computation is spawned in parallel and whichever finishes first is used.
@@ -218,7 +216,6 @@ impl Default for TreeConfig {
disable_trie_cache: false,
sparse_trie_prune_depth: DEFAULT_SPARSE_TRIE_PRUNE_DEPTH,
sparse_trie_max_storage_tries: DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES,
disable_sparse_trie_cache_pruning: false,
state_root_task_timeout: Some(DEFAULT_STATE_ROOT_TASK_TIMEOUT),
}
}
@@ -284,7 +281,6 @@ impl TreeConfig {
disable_trie_cache: false,
sparse_trie_prune_depth,
sparse_trie_max_storage_tries,
disable_sparse_trie_cache_pruning: false,
state_root_task_timeout,
}
}
@@ -635,17 +631,6 @@ impl TreeConfig {
self
}
/// Returns whether sparse trie cache pruning is disabled.
pub const fn disable_sparse_trie_cache_pruning(&self) -> bool {
self.disable_sparse_trie_cache_pruning
}
/// Setter for whether to disable sparse trie cache pruning.
pub const fn with_disable_sparse_trie_cache_pruning(mut self, value: bool) -> Self {
self.disable_sparse_trie_cache_pruning = value;
self
}
/// Returns the state root task timeout.
pub const fn state_root_task_timeout(&self) -> Option<Duration> {
self.state_root_task_timeout

View File

@@ -20,7 +20,7 @@ use reth_node_types::{BlockTy, NodeTypes};
use reth_payload_builder::PayloadBuilderHandle;
use reth_provider::{
providers::{BlockchainProvider, ProviderNodeTypes},
ProviderFactory, StorageSettingsCache,
ProviderFactory,
};
use reth_prune::PrunerWithFactory;
use reth_stages_api::{MetricEventsSender, Pipeline};
@@ -94,7 +94,6 @@ where
if chain_spec.is_optimism() { EngineApiKind::OpStack } else { EngineApiKind::Ethereum };
let downloader = BasicBlockDownloader::new(client, consensus.clone());
let use_hashed_state = provider.cached_storage_settings().use_hashed_state();
let persistence_handle =
PersistenceHandle::<N::Primitives>::spawn_service(provider, pruner, sync_metrics_tx);
@@ -112,7 +111,6 @@ where
engine_kind,
evm_config,
changeset_cache,
use_hashed_state,
);
let engine_handler = EngineApiRequestHandler::new(to_tree_tx, from_tree);

View File

@@ -73,7 +73,6 @@ reth-prune-types = { workspace = true, optional = true }
reth-stages = { workspace = true, optional = true }
reth-static-file = { workspace = true, optional = true }
reth-tracing = { workspace = true, optional = true }
reth-mdbx-viz = { workspace = true, optional = true }
[dev-dependencies]
# reth
@@ -144,14 +143,6 @@ test-utils = [
"reth-evm-ethereum/test-utils",
"reth-tasks/test-utils",
]
rocksdb = [
"reth-provider/rocksdb",
"reth-prune/rocksdb",
"reth-stages?/rocksdb",
"reth-e2e-test-utils/rocksdb",
]
edge = ["rocksdb"]
pageviz = ["dep:reth-mdbx-viz"]
[[test]]
name = "e2e_testsuite"

View File

@@ -351,14 +351,6 @@ impl<S: StateProvider, const PREWARM: bool> StateProvider for CachedStateProvide
self.state_provider.storage(account, storage_key)
}
}
fn storage_by_hashed_key(
&self,
address: Address,
hashed_storage_key: StorageKey,
) -> ProviderResult<Option<StorageValue>> {
self.state_provider.storage_by_hashed_key(address, hashed_storage_key)
}
}
impl<S: BytecodeReader, const PREWARM: bool> BytecodeReader for CachedStateProvider<S, PREWARM> {

View File

@@ -199,17 +199,6 @@ impl<S: StateProvider> StateProvider for InstrumentedStateProvider<S> {
self.record_storage_fetch(start.elapsed());
res
}
fn storage_by_hashed_key(
&self,
address: Address,
hashed_storage_key: StorageKey,
) -> ProviderResult<Option<StorageValue>> {
let start = Instant::now();
let res = self.state_provider.storage_by_hashed_key(address, hashed_storage_key);
self.record_storage_fetch(start.elapsed());
res
}
}
impl<S: BytecodeReader> BytecodeReader for InstrumentedStateProvider<S> {

View File

@@ -12,13 +12,12 @@ use reth_primitives_traits::constants::gas_units::MEGAGAS;
use reth_trie::updates::TrieUpdates;
use std::time::{Duration, Instant};
/// Upper bounds for each gas bucket. The last bucket is a catch-all for
/// everything above the final threshold: <5M, 5-10M, 10-20M, 20-30M, 30-40M, >40M.
const GAS_BUCKET_THRESHOLDS: [u64; 5] =
[5 * MEGAGAS, 10 * MEGAGAS, 20 * MEGAGAS, 30 * MEGAGAS, 40 * MEGAGAS];
/// Width of each gas bucket in gas units (10 Mgas).
const GAS_BUCKET_SIZE: u64 = 10 * MEGAGAS;
/// Total number of gas buckets (thresholds + 1 catch-all).
const NUM_GAS_BUCKETS: usize = GAS_BUCKET_THRESHOLDS.len() + 1;
/// Number of gas buckets. The last bucket is a catch-all for everything above
/// `(NUM_GAS_BUCKETS - 1) * GAS_BUCKET_SIZE`.
const NUM_GAS_BUCKETS: usize = 5;
/// Metrics for the `EngineApi`.
#[derive(Debug, Default)]
@@ -281,23 +280,21 @@ impl GasBucketMetrics {
}
fn bucket_index(gas_used: u64) -> usize {
GAS_BUCKET_THRESHOLDS
.iter()
.position(|&threshold| gas_used < threshold)
.unwrap_or(GAS_BUCKET_THRESHOLDS.len())
let idx = gas_used / GAS_BUCKET_SIZE;
(idx as usize).min(NUM_GAS_BUCKETS - 1)
}
/// Returns a human-readable label like `<5M`, `5-10M`, … `>40M`.
/// Returns a human-readable label like `<10M`, `10-20M`, … `>40M`.
fn bucket_label(index: usize) -> String {
let m = GAS_BUCKET_SIZE / 1_000_000;
if index == 0 {
let hi = GAS_BUCKET_THRESHOLDS[0] / MEGAGAS;
format!("<{hi}M")
} else if index < GAS_BUCKET_THRESHOLDS.len() {
let lo = GAS_BUCKET_THRESHOLDS[index - 1] / MEGAGAS;
let hi = GAS_BUCKET_THRESHOLDS[index] / MEGAGAS;
format!("<{m}M")
} else if index < NUM_GAS_BUCKETS - 1 {
let lo = m * index as u64;
let hi = lo + m;
format!("{lo}-{hi}M")
} else {
let lo = GAS_BUCKET_THRESHOLDS[GAS_BUCKET_THRESHOLDS.len() - 1] / MEGAGAS;
let lo = m * index as u64;
format!(">{lo}M")
}
}

View File

@@ -32,7 +32,7 @@ use reth_provider::{
BlockExecutionOutput, BlockExecutionResult, BlockReader, ChangeSetReader,
DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader,
StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader,
StorageSettingsCache, TransactionVariant,
TransactionVariant,
};
use reth_revm::database::StateProviderDatabase;
use reth_stages_api::ControlFlow;
@@ -271,9 +271,6 @@ where
evm_config: C,
/// Changeset cache for in-memory trie changesets
changeset_cache: ChangesetCache,
/// Whether the node uses hashed state as canonical storage (v2 mode).
/// Cached at construction to avoid threading `StorageSettingsCache` bounds everywhere.
use_hashed_state: bool,
}
impl<N, P: Debug, T: PayloadTypes + Debug, V: Debug, C> std::fmt::Debug
@@ -299,7 +296,6 @@ where
.field("engine_kind", &self.engine_kind)
.field("evm_config", &self.evm_config)
.field("changeset_cache", &self.changeset_cache)
.field("use_hashed_state", &self.use_hashed_state)
.finish()
}
}
@@ -317,8 +313,7 @@ where
P::Provider: BlockReader<Block = N::Block, Header = N::BlockHeader>
+ StageCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ StorageSettingsCache,
+ StorageChangeSetReader,
C: ConfigureEvm<Primitives = N> + 'static,
T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>,
V: EngineValidator<T>,
@@ -339,7 +334,6 @@ where
engine_kind: EngineApiKind,
evm_config: C,
changeset_cache: ChangesetCache,
use_hashed_state: bool,
) -> Self {
let (incoming_tx, incoming) = crossbeam_channel::unbounded();
@@ -361,7 +355,6 @@ where
engine_kind,
evm_config,
changeset_cache,
use_hashed_state,
}
}
@@ -382,7 +375,6 @@ where
kind: EngineApiKind,
evm_config: C,
changeset_cache: ChangesetCache,
use_hashed_state: bool,
) -> (Sender<FromEngine<EngineApiRequest<T, N>, N::Block>>, UnboundedReceiver<EngineApiEvent<N>>)
{
let best_block_number = provider.best_block_number().unwrap_or(0);
@@ -415,7 +407,6 @@ where
kind,
evm_config,
changeset_cache,
use_hashed_state,
);
let incoming = task.incoming_tx.clone();
spawn_os_thread("engine", || task.run());
@@ -2388,12 +2379,7 @@ where
self.update_reorg_metrics(old.len(), old_first);
self.reinsert_reorged_blocks(new.clone());
// When use_hashed_state is enabled, skip reinserting the old chain — the
// bundle state references plain state reverts which don't exist.
if !self.use_hashed_state {
self.reinsert_reorged_blocks(old.clone());
}
self.reinsert_reorged_blocks(old.clone());
}
// update the tracked in-memory state with the new chain
@@ -2677,23 +2663,8 @@ where
let start = Instant::now();
#[cfg(feature = "pageviz")]
reth_mdbx_viz::pageviz_emit_block_marker(block_num_hash.number, true, 0, 0, 0);
#[cfg(feature = "pageviz")]
let pageviz_start = std::time::Instant::now();
let executed = execute(&mut self.payload_validator, input, ctx)?;
#[cfg(feature = "pageviz")]
reth_mdbx_viz::pageviz_emit_block_marker(
block_num_hash.number,
false,
executed.recovered_block().senders().len() as u16,
pageviz_start.elapsed().as_millis() as u32,
executed.recovered_block().gas_used(),
);
// if the parent is the canonical head, we can insert the block as the pending block
if self.state.tree_state.canonical_block_hash() == executed.recovered_block().parent_hash()
{

View File

@@ -94,9 +94,6 @@ pub const SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY: usize = 1_000_000;
/// 144MB.
pub const SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY: usize = 1_000_000;
/// Blocks with fewer transactions than this skip prewarming, since the fixed overhead of spawning
/// prewarm workers exceeds the execution time saved.
pub const SMALL_BLOCK_TX_THRESHOLD: usize = 5;
/// Type alias for [`PayloadHandle`] returned by payload processor spawn methods.
type IteratorPayloadHandle<Evm, I, N> = PayloadHandle<
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxIterator<Evm>>::Recovered>,
@@ -138,8 +135,6 @@ where
sparse_trie_prune_depth: usize,
/// Maximum storage tries to retain after pruning.
sparse_trie_max_storage_tries: usize,
/// Whether sparse trie cache pruning is fully disabled.
disable_sparse_trie_cache_pruning: bool,
/// Whether to disable cache metrics recording.
disable_cache_metrics: bool,
}
@@ -175,7 +170,6 @@ where
prewarm_max_concurrency: config.prewarm_max_concurrency(),
sparse_trie_prune_depth: config.sparse_trie_prune_depth(),
sparse_trie_max_storage_tries: config.sparse_trie_max_storage_tries(),
disable_sparse_trie_cache_pruning: config.disable_sparse_trie_cache_pruning(),
disable_cache_metrics: config.disable_cache_metrics(),
}
}
@@ -472,8 +466,7 @@ where
where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
{
let skip_prewarm =
self.disable_transaction_prewarming || env.transaction_count < SMALL_BLOCK_TX_THRESHOLD;
let skip_prewarm = self.disable_transaction_prewarming;
let saved_cache = self.disable_state_cache.not().then(|| self.cache_for(env.parent_hash));
@@ -553,7 +546,6 @@ where
let disable_trie_cache = config.disable_trie_cache();
let prune_depth = self.sparse_trie_prune_depth;
let max_storage_tries = self.sparse_trie_max_storage_tries;
let disable_cache_pruning = self.disable_sparse_trie_cache_pruning;
let chunk_size =
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size());
let executor = self.executor.clone();
@@ -650,7 +642,6 @@ where
max_storage_tries,
SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY,
SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY,
disable_cache_pruning,
);
trie_metrics
.into_trie_for_reuse_duration_histogram

View File

@@ -1541,7 +1541,6 @@ mod tests {
providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory,
BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, LatestStateProvider,
PruneCheckpointReader, StageCheckpointReader, StateProviderBox, StorageChangeSetReader,
StorageSettingsCache,
};
use reth_trie::MultiProof;
use reth_trie_db::ChangesetCache;
@@ -1563,7 +1562,6 @@ mod tests {
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ StorageSettingsCache
+ BlockNumReader,
> + Clone
+ Send
@@ -1583,10 +1581,7 @@ mod tests {
fn create_cached_provider<F>(factory: F) -> CachedStateProvider<StateProviderBox>
where
F: DatabaseProviderFactory<
Provider: BlockReader
+ StageCheckpointReader
+ PruneCheckpointReader
+ reth_provider::StorageSettingsCache,
Provider: BlockReader + StageCheckpointReader + PruneCheckpointReader,
> + Clone
+ Send
+ 'static,

View File

@@ -72,7 +72,6 @@ where
max_storage_tries: usize,
max_nodes_capacity: usize,
max_values_capacity: usize,
disable_pruning: bool,
) -> (SparseStateTrie<A, S>, DeferredDrops) {
match self {
Self::Cleared(task) => task.into_cleared_trie(max_nodes_capacity, max_values_capacity),
@@ -81,7 +80,6 @@ where
max_storage_tries,
max_nodes_capacity,
max_values_capacity,
disable_pruning,
),
}
}
@@ -358,23 +356,16 @@ where
/// Prunes and shrinks the trie for reuse in the next payload built on top of this one.
///
/// Should be called after the state root result has been sent.
///
/// When `disable_pruning` is true, the trie is preserved without any node pruning,
/// storage trie eviction, or capacity shrinking, keeping the full cache intact for
/// benchmarking purposes.
pub(super) fn into_trie_for_reuse(
self,
prune_depth: usize,
max_storage_tries: usize,
max_nodes_capacity: usize,
max_values_capacity: usize,
disable_pruning: bool,
) -> (SparseStateTrie<A, S>, DeferredDrops) {
let Self { mut trie, .. } = self;
if !disable_pruning {
trie.prune(prune_depth, max_storage_tries);
trie.shrink_to(max_nodes_capacity, max_values_capacity);
}
trie.prune(prune_depth, max_storage_tries);
trie.shrink_to(max_nodes_capacity, max_values_capacity);
let deferred = trie.take_deferred_drops();
(trie, deferred)
}
@@ -416,9 +407,7 @@ where
let update = match message {
Ok(m) => m,
Err(_) => {
return Err(ParallelStateRootError::Other(
"updates channel disconnected before state root calculation".to_string(),
))
break
}
};

View File

@@ -38,7 +38,7 @@ use reth_provider::{
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader,
ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider,
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache,
StateProviderFactory, StateReader, StorageChangeSetReader,
};
use reth_revm::db::{states::bundle_state::BundleRetention, State};
use reth_trie::{updates::TrieUpdates, HashedPostState, StateRoot};
@@ -146,8 +146,7 @@ where
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader
+ StorageSettingsCache,
+ BlockNumReader,
> + BlockReader<Header = N::BlockHeader>
+ ChangeSetReader
+ BlockNumReader
@@ -1527,8 +1526,7 @@ where
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader
+ StorageSettingsCache,
+ BlockNumReader,
> + BlockReader<Header = N::BlockHeader>
+ StateProviderFactory
+ StateReader

View File

@@ -221,7 +221,6 @@ impl TestHarness {
EngineApiKind::Ethereum,
evm_config,
changeset_cache,
provider.cached_storage_settings().use_hashed_state(),
);
let block_builder = TestBlockBuilder::default().with_chain_spec((*chain_spec).clone());

View File

@@ -2,15 +2,13 @@
mod fcu_finalized_blocks;
use alloy_rpc_types_engine::PayloadStatusEnum;
use eyre::Result;
use reth_chainspec::{ChainSpecBuilder, MAINNET};
use reth_e2e_test_utils::testsuite::{
actions::{
BlockReference, CaptureBlock, CompareNodeChainTips, CreateFork, ExpectFcuStatus,
MakeCanonical, ProduceBlocks, ProduceBlocksLocally, ProduceInvalidBlocks, ReorgTo,
SelectActiveNode, SendForkchoiceUpdate, SendNewPayloads, SetForkBase, UpdateBlockInfo,
ValidateCanonicalTag, WaitForSync,
CaptureBlock, CompareNodeChainTips, CreateFork, ExpectFcuStatus, MakeCanonical,
ProduceBlocks, ProduceBlocksLocally, ProduceInvalidBlocks, ReorgTo, SelectActiveNode,
SendNewPayloads, UpdateBlockInfo, ValidateCanonicalTag, WaitForSync,
},
setup::{NetworkSetup, Setup},
TestBuilder,
@@ -41,14 +39,6 @@ fn default_engine_tree_setup() -> Setup<EthEngineTypes> {
)
}
/// Creates a v2 storage mode setup for engine tree e2e tests.
///
/// v2 mode uses keccak256-hashed slot keys in static file changesets and rocksdb history
/// instead of plain keys in MDBX.
fn v2_engine_tree_setup() -> Setup<EthEngineTypes> {
default_engine_tree_setup().with_storage_v2()
}
/// Test that verifies forkchoice update and canonical chain insertion functionality.
#[tokio::test]
async fn test_engine_tree_fcu_canon_chain_insertion_e2e() -> Result<()> {
@@ -344,152 +334,3 @@ async fn test_engine_tree_live_sync_transition_eventually_canonical_e2e() -> Res
Ok(())
}
// ==================== v2 storage mode variants ====================
/// v2 variant: Verifies forkchoice update and canonical chain insertion in v2 storage mode.
///
/// Exercises the full `save_blocks` → `write_state` → static file changeset path with hashed keys.
#[tokio::test]
async fn test_engine_tree_fcu_canon_chain_insertion_v2_e2e() -> Result<()> {
reth_tracing::init_test_tracing();
let test = TestBuilder::new()
.with_setup(v2_engine_tree_setup())
.with_action(ProduceBlocks::<EthEngineTypes>::new(1))
.with_action(MakeCanonical::new())
.with_action(ProduceBlocks::<EthEngineTypes>::new(3))
.with_action(MakeCanonical::new());
test.run::<EthereumNode>().await?;
Ok(())
}
/// v2 variant: Verifies forkchoice update with a reorg where all blocks are already available.
///
/// Exercises `write_state_reverts` path with hashed changeset keys during CL-driven reorgs.
#[tokio::test]
async fn test_engine_tree_fcu_reorg_with_all_blocks_v2_e2e() -> Result<()> {
reth_tracing::init_test_tracing();
let test = TestBuilder::new()
.with_setup(v2_engine_tree_setup())
.with_action(ProduceBlocks::<EthEngineTypes>::new(5))
.with_action(MakeCanonical::new())
.with_action(CreateFork::<EthEngineTypes>::new(2, 3))
.with_action(CaptureBlock::new("fork_tip"))
.with_action(ReorgTo::<EthEngineTypes>::new_from_tag("fork_tip"));
test.run::<EthereumNode>().await?;
Ok(())
}
/// v2 variant: Verifies progressive canonical chain extension in v2 storage mode.
#[tokio::test]
async fn test_engine_tree_fcu_extends_canon_chain_v2_e2e() -> Result<()> {
reth_tracing::init_test_tracing();
let test = TestBuilder::new()
.with_setup(v2_engine_tree_setup())
.with_action(ProduceBlocks::<EthEngineTypes>::new(1))
.with_action(MakeCanonical::new())
.with_action(ProduceBlocks::<EthEngineTypes>::new(10))
.with_action(CaptureBlock::new("target_block"))
.with_action(ReorgTo::<EthEngineTypes>::new_from_tag("target_block"))
.with_action(MakeCanonical::new());
test.run::<EthereumNode>().await?;
Ok(())
}
/// Creates a 2-node setup for disk-level reorg testing.
///
/// Uses unconnected nodes so fork blocks can be produced independently on Node 1 and then
/// sent to Node 0 via newPayload only (no FCU), keeping Node 0's persisted chain intact
/// until the final `ReorgTo` triggers `find_disk_reorg`.
fn disk_reorg_setup(storage_v2: bool) -> Setup<EthEngineTypes> {
let mut setup = Setup::default()
.with_chain_spec(Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(
serde_json::from_str(include_str!(
"../../../../e2e-test-utils/src/testsuite/assets/genesis.json"
))
.unwrap(),
)
.cancun_activated()
.build(),
))
.with_network(NetworkSetup::multi_node_unconnected(2))
.with_tree_config(
TreeConfig::default().with_legacy_state_root(false).with_has_enough_parallelism(true),
);
if storage_v2 {
setup = setup.with_storage_v2();
}
setup
}
/// Builds a disk-level reorg test scenario.
///
/// 1. Both nodes receive 3 shared blocks
/// 2. Node 0 extends to 10 blocks locally (persisted to disk)
/// 3. Node 1 builds an 8-block fork from block 3 (its canonical head)
/// 4. Fork blocks are sent to Node 0 via newPayload (no FCU, old chain stays on disk)
/// 5. FCU to fork tip on Node 0 triggers `find_disk_reorg` → `RemoveBlocksAbove(3)`
fn disk_reorg_test(storage_v2: bool) -> TestBuilder<EthEngineTypes> {
TestBuilder::new()
.with_setup(disk_reorg_setup(storage_v2))
.with_action(SelectActiveNode::new(0))
.with_action(ProduceBlocks::<EthEngineTypes>::new(3))
.with_action(MakeCanonical::new())
.with_action(ProduceBlocksLocally::<EthEngineTypes>::new(7))
.with_action(MakeCanonical::with_active_node())
.with_action(SelectActiveNode::new(1))
.with_action(SetForkBase::new(3))
.with_action(ProduceBlocksLocally::<EthEngineTypes>::new(8))
.with_action(MakeCanonical::with_active_node())
.with_action(CaptureBlock::new("fork_tip"))
.with_action(
SendNewPayloads::<EthEngineTypes>::new()
.with_source_node(1)
.with_target_node(0)
.with_start_block(4)
.with_total_blocks(8),
)
.with_action(
SendForkchoiceUpdate::<EthEngineTypes>::new(
BlockReference::Tag("fork_tip".into()),
BlockReference::Tag("fork_tip".into()),
BlockReference::Tag("fork_tip".into()),
)
.with_expected_status(PayloadStatusEnum::Valid)
.with_node_idx(0),
)
}
/// Verifies disk-level reorg in v1 (plain key) storage mode.
///
/// Confirms `find_disk_reorg()` detects persisted blocks on the wrong fork and calls
/// `RemoveBlocksAbove` to truncate, then re-persists the correct fork chain.
#[tokio::test]
async fn test_engine_tree_disk_reorg_v1_e2e() -> Result<()> {
reth_tracing::init_test_tracing();
disk_reorg_test(false).run::<EthereumNode>().await?;
Ok(())
}
/// v2 variant: Verifies disk-level reorg in v2 storage mode.
///
/// Same scenario as v1 but with hashed changeset keys in static files and rocksdb history.
/// Exercises `find_disk_reorg()` → `RemoveBlocksAbove` with v2 hashed key format.
#[tokio::test]
async fn test_engine_tree_disk_reorg_v2_e2e() -> Result<()> {
reth_tracing::init_test_tracing();
disk_reorg_test(true).run::<EthereumNode>().await?;
Ok(())
}

View File

@@ -273,6 +273,7 @@ where
gas_limit: payload.payload.gas_limit(),
basefee: payload.payload.saturated_base_fee_per_gas(),
blob_excess_gas_and_price,
slot_num: 0,
};
Ok(EvmEnv { cfg_env, block_env })

View File

@@ -108,8 +108,7 @@ impl<'a, DB: Database, I: Inspector<EthEvmContext<&'a mut State<DB>>>> BlockExec
result: ResultAndState::new(
ExecutionResult::Success {
reason: SuccessReason::Return,
gas_used: 0,
gas_refunded: 0,
gas: Default::default(),
logs: vec![],
output: Output::Call(Bytes::from(vec![])),
},

View File

@@ -18,7 +18,6 @@ use reth_provider::{
};
use reth_revm::database::StateProviderDatabase;
use reth_testing_utils::generators::sign_tx_with_key_pair;
use reth_trie_common::KeccakKeyHasher;
use secp256k1::Keypair;
pub(crate) fn to_execution_outcome(
@@ -78,9 +77,12 @@ where
let execution_outcome = to_execution_outcome(block.number(), &block_execution_output);
// Commit the block's execution outcome to the database
let hashed_state = execution_outcome.hash_state_slow::<KeccakKeyHasher>().into_sorted();
let provider_rw = provider_factory.provider_rw()?;
provider_rw.append_blocks_with_state(vec![block.clone()], &execution_outcome, hashed_state)?;
provider_rw.append_blocks_with_state(
vec![block.clone()],
&execution_outcome,
Default::default(),
)?;
provider_rw.commit()?;
Ok(block_execution_output)
@@ -208,12 +210,11 @@ where
execution_outcome.state_mut().reverts.sort();
// Commit the block's execution outcome to the database
let hashed_state = execution_outcome.hash_state_slow::<KeccakKeyHasher>().into_sorted();
let provider_rw = provider_factory.provider_rw()?;
provider_rw.append_blocks_with_state(
vec![block1.clone(), block2.clone()],
&execution_outcome,
hashed_state,
Default::default(),
)?;
provider_rw.commit()?;

View File

@@ -589,7 +589,7 @@ impl EthMessageID {
/// Returns the max value for the given version.
pub const fn max(version: EthVersion) -> u8 {
if version as u8 >= EthVersion::Eth69 as u8 {
if version.is_eth69() {
Self::BlockRangeUpdate.to_u8()
} else {
Self::Receipts.to_u8()
@@ -937,13 +937,6 @@ mod tests {
assert!(matches!(decoded, StatusMessage::Legacy(s) if s == status));
}
#[test]
fn eth_message_id_max_includes_block_range_update() {
assert_eq!(EthMessageID::max(EthVersion::Eth69), EthMessageID::BlockRangeUpdate.to_u8(),);
assert_eq!(EthMessageID::max(EthVersion::Eth70), EthMessageID::BlockRangeUpdate.to_u8(),);
assert_eq!(EthMessageID::max(EthVersion::Eth68), EthMessageID::Receipts.to_u8());
}
#[test]
fn decode_status_rejects_non_status() {
let msg = EthMessage::<EthNetworkPrimitives>::GetBlockBodies(RequestPair {

View File

@@ -8,7 +8,7 @@ use reth_eth_wire_types::{
};
use reth_ethereum_forks::ForkId;
use reth_network_p2p::error::{RequestError, RequestResult};
use reth_network_peers::{NodeRecord, PeerId};
use reth_network_peers::PeerId;
use reth_network_types::{PeerAddr, PeerKind};
use reth_tokio_util::EventStream;
use std::{
@@ -152,13 +152,8 @@ pub trait NetworkEventListenerProvider: NetworkPeersEvents {
pub enum DiscoveryEvent {
/// Discovered a node
NewNode(DiscoveredEvent),
/// Retrieved a [`ForkId`] from the peer via ENR request.
///
/// Contains the full [`NodeRecord`] (peer ID + address) and the reported [`ForkId`].
/// Used to verify fork compatibility before admitting the peer.
///
/// See also <https://eips.ethereum.org/EIPS/eip-868>
EnrForkId(NodeRecord, ForkId),
/// Retrieved a [`ForkId`] from the peer via ENR request, See <https://eips.ethereum.org/EIPS/eip-868>
EnrForkId(PeerId, ForkId),
}
/// Represents events related to peer discovery in the network.

View File

@@ -172,11 +172,6 @@ pub struct PeersConfig {
/// IPs within the specified CIDR ranges will be allowed.
#[cfg_attr(feature = "serde", serde(skip))]
pub ip_filter: IpFilter,
/// If true, discovered peers without a confirmed ENR [`ForkId`](alloy_eip2124::ForkId)
/// (EIP-868) will not be added to the peer set until their fork ID is verified.
///
/// This filters out peers from other networks that pollute the discovery table.
pub enforce_enr_fork_id: bool,
}
impl Default for PeersConfig {
@@ -196,7 +191,6 @@ impl Default for PeersConfig {
max_backoff_count: 5,
incoming_ip_throttle_duration: INBOUND_IP_THROTTLE_DURATION,
ip_filter: IpFilter::default(),
enforce_enr_fork_id: false,
}
}
}
@@ -320,13 +314,6 @@ impl PeersConfig {
self
}
/// If set, discovered peers without a confirmed ENR [`ForkId`](alloy_eip2124::ForkId) will not
/// be added to the peer set until their fork ID is verified via EIP-868.
pub const fn with_enforce_enr_fork_id(mut self, enforce: bool) -> Self {
self.enforce_enr_fork_id = enforce;
self
}
/// Returns settings for testing
#[cfg(any(test, feature = "test-utils"))]
pub fn test() -> Self {

View File

@@ -240,7 +240,7 @@ impl Discovery {
self.on_node_record_update(record, None);
}
DiscoveryUpdate::EnrForkId(node, fork_id) => {
self.queued_events.push_back(DiscoveryEvent::EnrForkId(node, fork_id))
self.queued_events.push_back(DiscoveryEvent::EnrForkId(node.id, fork_id))
}
DiscoveryUpdate::Removed(peer_id) => {
self.discovered_nodes.remove(&peer_id);

View File

@@ -25,7 +25,7 @@ use crate::{
listener::ConnectionListener,
message::{NewBlockMessage, PeerMessage},
metrics::{
BackedOffPeersMetrics, ClosedSessionsMetrics, DirectionalDisconnectMetrics, NetworkMetrics,
BackedOffPeersMetrics, ClosedSessionsMetrics, DisconnectMetrics, NetworkMetrics,
PendingSessionFailureMetrics, NETWORK_POOL_TRANSACTIONS_SCOPE,
},
network::{NetworkHandle, NetworkHandleMessage},
@@ -140,8 +140,8 @@ pub struct NetworkManager<N: NetworkPrimitives = EthNetworkPrimitives> {
num_active_peers: Arc<AtomicUsize>,
/// Metrics for the Network
metrics: NetworkMetrics,
/// Disconnect metrics for the Network, split by connection direction.
disconnect_metrics: DirectionalDisconnectMetrics,
/// Disconnect metrics for the Network
disconnect_metrics: DisconnectMetrics,
/// Closed sessions metrics, split by direction.
closed_sessions_metrics: ClosedSessionsMetrics,
/// Pending session failure metrics, split by direction.
@@ -864,9 +864,6 @@ impl<N: NetworkPrimitives> NetworkManager<N> {
"Session disconnected"
);
// Capture direction before state is reset to Idle
let is_inbound = self.swarm.state().peers().is_inbound_peer(&peer_id);
let reason = if let Some(ref err) = error {
// If the connection was closed due to an error, we report
// the peer
@@ -890,11 +887,7 @@ impl<N: NetworkPrimitives> NetworkManager<N> {
self.update_active_connection_metrics();
if let Some(reason) = reason {
if is_inbound {
self.disconnect_metrics.increment_inbound(reason);
} else {
self.disconnect_metrics.increment_outbound(reason);
}
self.disconnect_metrics.increment(reason);
}
self.metrics
.backed_off_peers
@@ -917,7 +910,7 @@ impl<N: NetworkPrimitives> NetworkManager<N> {
.on_incoming_pending_session_dropped(remote_addr, err);
self.pending_session_failure_metrics.inbound.increment(1);
if let Some(reason) = err.as_disconnected() {
self.disconnect_metrics.increment_inbound(reason);
self.disconnect_metrics.increment(reason);
}
} else {
self.swarm
@@ -950,7 +943,7 @@ impl<N: NetworkPrimitives> NetworkManager<N> {
BackoffReason::from_disconnect(err.as_disconnected()),
);
if let Some(reason) = err.as_disconnected() {
self.disconnect_metrics.increment_outbound(reason);
self.disconnect_metrics.increment(reason);
}
} else {
self.swarm

View File

@@ -301,34 +301,6 @@ macro_rules! duration_metered_exec {
}};
}
/// Direction-aware wrapper for disconnect metrics.
///
/// Tracks disconnect reasons for inbound and outbound connections separately, in addition to
/// the combined (legacy) counters.
#[derive(Debug, Default)]
pub(crate) struct DirectionalDisconnectMetrics {
/// Combined disconnect metrics (all directions).
pub(crate) total: DisconnectMetrics,
/// Disconnect metrics for inbound connections only.
pub(crate) inbound: InboundDisconnectMetrics,
/// Disconnect metrics for outbound connections only.
pub(crate) outbound: OutboundDisconnectMetrics,
}
impl DirectionalDisconnectMetrics {
/// Increments disconnect counters for an inbound connection.
pub(crate) fn increment_inbound(&self, reason: DisconnectReason) {
self.total.increment(reason);
self.inbound.increment(reason);
}
/// Increments disconnect counters for an outbound connection.
pub(crate) fn increment_outbound(&self, reason: DisconnectReason) {
self.total.increment(reason);
self.outbound.increment(reason);
}
}
/// Metrics for Disconnection types
///
/// These are just counters, and ideally we would implement these metrics on a peer-by-peer basis,
@@ -398,144 +370,6 @@ impl DisconnectMetrics {
}
}
/// Disconnect metrics scoped to inbound connections only.
///
/// These counters track disconnect reasons exclusively for sessions that were initiated by
/// remote peers connecting to this node. This helps operators distinguish between being rejected
/// by remote peers (outbound) vs rejecting incoming peers (inbound).
#[derive(Metrics)]
#[metrics(scope = "network.inbound")]
pub struct InboundDisconnectMetrics {
/// Number of inbound peer disconnects due to `DisconnectRequested` (0x00)
pub(crate) disconnect_requested: Counter,
/// Number of inbound peer disconnects due to `TcpSubsystemError` (0x01)
pub(crate) tcp_subsystem_error: Counter,
/// Number of inbound peer disconnects due to `ProtocolBreach` (0x02)
pub(crate) protocol_breach: Counter,
/// Number of inbound peer disconnects due to `UselessPeer` (0x03)
pub(crate) useless_peer: Counter,
/// Number of inbound peer disconnects due to `TooManyPeers` (0x04)
pub(crate) too_many_peers: Counter,
/// Number of inbound peer disconnects due to `AlreadyConnected` (0x05)
pub(crate) already_connected: Counter,
/// Number of inbound peer disconnects due to `IncompatibleP2PProtocolVersion` (0x06)
pub(crate) incompatible: Counter,
/// Number of inbound peer disconnects due to `NullNodeIdentity` (0x07)
pub(crate) null_node_identity: Counter,
/// Number of inbound peer disconnects due to `ClientQuitting` (0x08)
pub(crate) client_quitting: Counter,
/// Number of inbound peer disconnects due to `UnexpectedHandshakeIdentity` (0x09)
pub(crate) unexpected_identity: Counter,
/// Number of inbound peer disconnects due to `ConnectedToSelf` (0x0a)
pub(crate) connected_to_self: Counter,
/// Number of inbound peer disconnects due to `PingTimeout` (0x0b)
pub(crate) ping_timeout: Counter,
/// Number of inbound peer disconnects due to `SubprotocolSpecific` (0x10)
pub(crate) subprotocol_specific: Counter,
}
impl InboundDisconnectMetrics {
/// Increments the proper counter for the given disconnect reason
pub(crate) fn increment(&self, reason: DisconnectReason) {
match reason {
DisconnectReason::DisconnectRequested => self.disconnect_requested.increment(1),
DisconnectReason::TcpSubsystemError => self.tcp_subsystem_error.increment(1),
DisconnectReason::ProtocolBreach => self.protocol_breach.increment(1),
DisconnectReason::UselessPeer => self.useless_peer.increment(1),
DisconnectReason::TooManyPeers => self.too_many_peers.increment(1),
DisconnectReason::AlreadyConnected => self.already_connected.increment(1),
DisconnectReason::IncompatibleP2PProtocolVersion => self.incompatible.increment(1),
DisconnectReason::NullNodeIdentity => self.null_node_identity.increment(1),
DisconnectReason::ClientQuitting => self.client_quitting.increment(1),
DisconnectReason::UnexpectedHandshakeIdentity => self.unexpected_identity.increment(1),
DisconnectReason::ConnectedToSelf => self.connected_to_self.increment(1),
DisconnectReason::PingTimeout => self.ping_timeout.increment(1),
DisconnectReason::SubprotocolSpecific => self.subprotocol_specific.increment(1),
}
}
}
/// Disconnect metrics scoped to outbound connections only.
///
/// These counters track disconnect reasons exclusively for sessions that this node initiated
/// by dialing out to remote peers. A high `too_many_peers` count here indicates remote peers
/// are rejecting our connection attempts because they are full.
#[derive(Metrics)]
#[metrics(scope = "network.outbound")]
pub struct OutboundDisconnectMetrics {
/// Number of outbound peer disconnects due to `DisconnectRequested` (0x00)
pub(crate) disconnect_requested: Counter,
/// Number of outbound peer disconnects due to `TcpSubsystemError` (0x01)
pub(crate) tcp_subsystem_error: Counter,
/// Number of outbound peer disconnects due to `ProtocolBreach` (0x02)
pub(crate) protocol_breach: Counter,
/// Number of outbound peer disconnects due to `UselessPeer` (0x03)
pub(crate) useless_peer: Counter,
/// Number of outbound peer disconnects due to `TooManyPeers` (0x04)
pub(crate) too_many_peers: Counter,
/// Number of outbound peer disconnects due to `AlreadyConnected` (0x05)
pub(crate) already_connected: Counter,
/// Number of outbound peer disconnects due to `IncompatibleP2PProtocolVersion` (0x06)
pub(crate) incompatible: Counter,
/// Number of outbound peer disconnects due to `NullNodeIdentity` (0x07)
pub(crate) null_node_identity: Counter,
/// Number of outbound peer disconnects due to `ClientQuitting` (0x08)
pub(crate) client_quitting: Counter,
/// Number of outbound peer disconnects due to `UnexpectedHandshakeIdentity` (0x09)
pub(crate) unexpected_identity: Counter,
/// Number of outbound peer disconnects due to `ConnectedToSelf` (0x0a)
pub(crate) connected_to_self: Counter,
/// Number of outbound peer disconnects due to `PingTimeout` (0x0b)
pub(crate) ping_timeout: Counter,
/// Number of outbound peer disconnects due to `SubprotocolSpecific` (0x10)
pub(crate) subprotocol_specific: Counter,
}
impl OutboundDisconnectMetrics {
/// Increments the proper counter for the given disconnect reason
pub(crate) fn increment(&self, reason: DisconnectReason) {
match reason {
DisconnectReason::DisconnectRequested => self.disconnect_requested.increment(1),
DisconnectReason::TcpSubsystemError => self.tcp_subsystem_error.increment(1),
DisconnectReason::ProtocolBreach => self.protocol_breach.increment(1),
DisconnectReason::UselessPeer => self.useless_peer.increment(1),
DisconnectReason::TooManyPeers => self.too_many_peers.increment(1),
DisconnectReason::AlreadyConnected => self.already_connected.increment(1),
DisconnectReason::IncompatibleP2PProtocolVersion => self.incompatible.increment(1),
DisconnectReason::NullNodeIdentity => self.null_node_identity.increment(1),
DisconnectReason::ClientQuitting => self.client_quitting.increment(1),
DisconnectReason::UnexpectedHandshakeIdentity => self.unexpected_identity.increment(1),
DisconnectReason::ConnectedToSelf => self.connected_to_self.increment(1),
DisconnectReason::PingTimeout => self.ping_timeout.increment(1),
DisconnectReason::SubprotocolSpecific => self.subprotocol_specific.increment(1),
}
}
}
/// Metrics for the `EthRequestHandler`
#[derive(Metrics)]
#[metrics(scope = "network")]

View File

@@ -92,9 +92,6 @@ pub struct PeersManager {
incoming_ip_throttle_duration: Duration,
/// IP address filter for restricting network connections to specific IP ranges.
ip_filter: reth_net_banlist::IpFilter,
/// If true, discovered peers without a confirmed ENR fork ID will not be added until their
/// fork ID is verified via EIP-868.
enforce_enr_fork_id: bool,
}
impl PeersManager {
@@ -114,7 +111,6 @@ impl PeersManager {
max_backoff_count,
incoming_ip_throttle_duration,
ip_filter,
enforce_enr_fork_id,
} = config;
let (manager_tx, handle_rx) = mpsc::unbounded_channel();
let now = Instant::now();
@@ -171,7 +167,6 @@ impl PeersManager {
net_connection_state: NetworkConnectionState::default(),
incoming_ip_throttle_duration,
ip_filter,
enforce_enr_fork_id,
}
}
@@ -180,11 +175,6 @@ impl PeersManager {
PeersHandle::new(self.manager_tx.clone())
}
/// Returns `true` if discovered peers must have a confirmed ENR fork ID before being added.
pub(crate) const fn enforce_enr_fork_id(&self) -> bool {
self.enforce_enr_fork_id
}
/// Returns the number of peers in the peer set
#[inline]
pub(crate) fn num_known_peers(&self) -> usize {
@@ -218,13 +208,6 @@ impl PeersManager {
})
}
/// Returns `true` if the given peer is connected via an inbound session.
pub(crate) fn is_inbound_peer(&self, peer_id: &PeerId) -> bool {
self.peers.get(peer_id).is_some_and(|p| {
matches!(p.state, PeerConnectionState::In | PeerConnectionState::DisconnectingIn)
})
}
/// Returns an iterator over all peer ids for peers with the given kind
pub(crate) fn peers_by_kind(&self, kind: PeerKind) -> impl Iterator<Item = PeerId> + '_ {
self.peers.iter().filter_map(move |(peer_id, peer)| (peer.kind == kind).then_some(*peer_id))
@@ -755,6 +738,17 @@ impl PeersManager {
}
}
/// Called as follow-up for a discovered peer.
///
/// The [`ForkId`] is retrieved from an ENR record that the peer announces over the discovery
/// protocol
pub(crate) fn set_discovered_fork_id(&mut self, peer_id: PeerId, fork_id: ForkId) {
if let Some(peer) = self.peers.get_mut(&peer_id) {
trace!(target: "net::peers", ?peer_id, ?fork_id, "set discovered fork id");
peer.fork_id = Some(Box::new(fork_id));
}
}
/// Called for a newly discovered peer.
///
/// If the peer already exists, then the address, kind and `fork_id` will be updated.

View File

@@ -332,19 +332,9 @@ impl<N: NetworkPrimitives> NetworkState<N> {
fork_id,
});
}
DiscoveryEvent::EnrForkId(record, fork_id) => {
let peer_id = record.id;
let tcp_addr = record.tcp_addr();
if tcp_addr.port() == 0 {
return
}
let udp_addr = record.udp_addr();
let addr = PeerAddr::new(tcp_addr, Some(udp_addr));
self.queued_messages.push_back(StateAction::DiscoveredEnrForkId {
peer_id,
addr,
fork_id,
});
DiscoveryEvent::EnrForkId(peer_id, fork_id) => {
self.queued_messages
.push_back(StateAction::DiscoveredEnrForkId { peer_id, fork_id });
}
}
}
@@ -562,8 +552,6 @@ pub(crate) enum StateAction<N: NetworkPrimitives> {
/// Retrieved a [`ForkId`] from the peer via ENR request, See <https://eips.ethereum.org/EIPS/eip-868>
DiscoveredEnrForkId {
peer_id: PeerId,
/// The address of the peer.
addr: PeerAddr,
/// The reported [`ForkId`] by this peer.
fork_id: ForkId,
},

View File

@@ -246,28 +246,18 @@ impl<N: NetworkPrimitives> Swarm<N> {
StateAction::PeerAdded(peer_id) => return Some(SwarmEvent::PeerAdded(peer_id)),
StateAction::PeerRemoved(peer_id) => return Some(SwarmEvent::PeerRemoved(peer_id)),
StateAction::DiscoveredNode { peer_id, addr, fork_id } => {
// Don't try to connect to peer if node is shutting down
if self.is_shutting_down() {
return None
}
// When `enforce_enr_fork_id` is enabled, peers discovered without a confirmed
// fork ID (via EIP-868 ENR) are deferred — they'll only be added once a
// `DiscoveredEnrForkId` event arrives with a validated fork ID.
//
// When disabled (default), peers without a fork ID are admitted immediately.
// Peers that *do* carry a fork ID are always validated against ours.
let enforce = self.state().peers().enforce_enr_fork_id();
let allow = match fork_id {
Some(f) => self.sessions.is_valid_fork_id(f),
None => !enforce,
};
if allow {
// Insert peer only if no fork id or a valid fork id
if fork_id.map_or_else(|| true, |f| self.sessions.is_valid_fork_id(f)) {
self.state_mut().peers_mut().add_peer(peer_id, addr, fork_id);
}
}
StateAction::DiscoveredEnrForkId { peer_id, addr, fork_id } => {
StateAction::DiscoveredEnrForkId { peer_id, fork_id } => {
if self.sessions.is_valid_fork_id(fork_id) {
self.state_mut().peers_mut().add_peer(peer_id, addr, Some(fork_id));
self.state_mut().peers_mut().set_discovered_fork_id(peer_id, fork_id);
} else {
trace!(target: "net", ?peer_id, remote_fork_id=?fork_id, our_fork_id=?self.sessions.fork_id(), "fork id mismatch, removing peer");
self.state_mut().peers_mut().remove_peer(peer_id);

View File

@@ -129,4 +129,3 @@ op = [
"reth-evm/op",
"reth-primitives-traits/op",
]
pageviz = ["reth-engine-tree/pageviz"]

View File

@@ -43,7 +43,6 @@ pub struct DefaultEngineValues {
disable_trie_cache: bool,
sparse_trie_prune_depth: usize,
sparse_trie_max_storage_tries: usize,
disable_sparse_trie_cache_pruning: bool,
state_root_task_timeout: Option<String>,
}
@@ -199,12 +198,6 @@ impl DefaultEngineValues {
self
}
/// Set whether to disable sparse trie cache pruning by default
pub const fn with_disable_sparse_trie_cache_pruning(mut self, v: bool) -> Self {
self.disable_sparse_trie_cache_pruning = v;
self
}
/// Set the default state root task timeout
pub fn with_state_root_task_timeout(mut self, v: Option<String>) -> Self {
self.state_root_task_timeout = v;
@@ -238,7 +231,6 @@ impl Default for DefaultEngineValues {
disable_trie_cache: false,
sparse_trie_prune_depth: DEFAULT_SPARSE_TRIE_PRUNE_DEPTH,
sparse_trie_max_storage_tries: DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES,
disable_sparse_trie_cache_pruning: false,
state_root_task_timeout: Some("1s".to_string()),
}
}
@@ -380,12 +372,6 @@ pub struct EngineArgs {
#[arg(long = "engine.sparse-trie-max-storage-tries", default_value_t = DefaultEngineValues::get_global().sparse_trie_max_storage_tries)]
pub sparse_trie_max_storage_tries: usize,
/// Fully disable sparse trie cache pruning. When set, the cached sparse trie is preserved
/// without any node pruning or storage trie eviction between blocks. Useful for benchmarking
/// the effects of retaining the full trie cache.
#[arg(long = "engine.disable-sparse-trie-cache-pruning", default_value_t = DefaultEngineValues::get_global().disable_sparse_trie_cache_pruning)]
pub disable_sparse_trie_cache_pruning: bool,
/// Configure the timeout for the state root task before spawning a sequential fallback.
/// If the state root task takes longer than this, a sequential computation starts in
/// parallel and whichever finishes first is used.
@@ -429,7 +415,6 @@ impl Default for EngineArgs {
disable_trie_cache,
sparse_trie_prune_depth,
sparse_trie_max_storage_tries,
disable_sparse_trie_cache_pruning,
state_root_task_timeout,
} = DefaultEngineValues::get_global().clone();
Self {
@@ -460,7 +445,6 @@ impl Default for EngineArgs {
disable_trie_cache,
sparse_trie_prune_depth,
sparse_trie_max_storage_tries,
disable_sparse_trie_cache_pruning,
state_root_task_timeout: state_root_task_timeout
.as_deref()
.map(|s| humantime::parse_duration(s).expect("valid default duration")),
@@ -496,7 +480,6 @@ impl EngineArgs {
.with_disable_trie_cache(self.disable_trie_cache)
.with_sparse_trie_prune_depth(self.sparse_trie_prune_depth)
.with_sparse_trie_max_storage_tries(self.sparse_trie_max_storage_tries)
.with_disable_sparse_trie_cache_pruning(self.disable_sparse_trie_cache_pruning)
.with_state_root_task_timeout(self.state_root_task_timeout.filter(|d| !d.is_zero()))
}
}
@@ -551,7 +534,6 @@ mod tests {
disable_trie_cache: true,
sparse_trie_prune_depth: 10,
sparse_trie_max_storage_tries: 100,
disable_sparse_trie_cache_pruning: true,
state_root_task_timeout: Some(Duration::from_secs(2)),
};
@@ -588,7 +570,6 @@ mod tests {
"10",
"--engine.sparse-trie-max-storage-tries",
"100",
"--engine.disable-sparse-trie-cache-pruning",
"--engine.state-root-task-timeout",
"2s",
])

View File

@@ -76,7 +76,11 @@ pub use era::{DefaultEraHost, EraArgs, EraSourceArgs};
mod static_files;
pub use static_files::{StaticFilesArgs, MINIMAL_BLOCKS_PER_FILE};
/// `StorageArgs` for configuring storage settings.
/// `RocksDbArgs` for configuring RocksDB table routing.
mod rocksdb;
pub use rocksdb::{RocksDbArgs, RocksDbArgsError};
/// `StorageArgs` for configuring storage mode (v2 vs v1/legacy).
mod storage;
pub use storage::StorageArgs;

View File

@@ -227,14 +227,6 @@ pub struct NetworkArgs {
/// Example: --netrestrict "192.168.0.0/16,10.0.0.0/8"
#[arg(long, value_name = "NETRESTRICT")]
pub netrestrict: Option<String>,
/// Enforce EIP-868 ENR fork ID validation for discovered peers.
///
/// When enabled, peers discovered without a confirmed fork ID are not added to the peer set
/// until their fork ID is verified via EIP-868 ENR request. This filters out peers from other
/// networks that pollute the discovery table.
#[arg(long)]
pub enforce_enr_fork_id: bool,
}
impl NetworkArgs {
@@ -341,8 +333,7 @@ impl NetworkArgs {
)
.with_max_inbound_opt(self.resolved_max_inbound_peers())
.with_max_outbound_opt(self.resolved_max_outbound_peers())
.with_ip_filter(ip_filter)
.with_enforce_enr_fork_id(self.enforce_enr_fork_id);
.with_ip_filter(ip_filter);
// Configure basic network stack
NetworkConfigBuilder::<N>::new(secret_key)
@@ -500,7 +491,6 @@ impl Default for NetworkArgs {
required_block_hashes: vec![],
network_id: None,
netrestrict: None,
enforce_enr_fork_id: false,
}
}
}

View File

@@ -0,0 +1,160 @@
//! clap [Args](clap::Args) for `RocksDB` table routing configuration
use clap::{ArgAction, Args};
/// Parameters for `RocksDB` table routing configuration.
///
/// These flags control which database tables are stored in `RocksDB` instead of MDBX.
/// All flags are genesis-initialization-only: changing them after genesis requires a re-sync.
///
/// When `--storage.v2` is used, the defaults for these flags change to enable `RocksDB` routing.
/// Individual flags can still override those defaults when explicitly set.
#[derive(Debug, Args, PartialEq, Eq, Clone, Copy, Default)]
#[command(next_help_heading = "RocksDB")]
pub struct RocksDbArgs {
/// Route all supported tables to `RocksDB` instead of MDBX.
///
/// This enables `RocksDB` for `tx-hash`, `storages-history`, and `account-history` tables.
/// Cannot be combined with individual flags set to false.
#[arg(long = "rocksdb.all", action = ArgAction::SetTrue)]
pub all: bool,
/// Route tx hash -> number table to `RocksDB` instead of MDBX.
///
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
/// Defaults to the base storage mode (v1: false, v2: true).
#[arg(long = "rocksdb.tx-hash", action = ArgAction::Set)]
pub tx_hash: Option<bool>,
/// Route storages history tables to `RocksDB` instead of MDBX.
///
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
/// Defaults to the base storage mode (v1: false, v2: true).
#[arg(long = "rocksdb.storages-history", action = ArgAction::Set)]
pub storages_history: Option<bool>,
/// Route account history tables to `RocksDB` instead of MDBX.
///
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
/// Defaults to the base storage mode (v1: false, v2: true).
#[arg(long = "rocksdb.account-history", action = ArgAction::Set)]
pub account_history: Option<bool>,
}
impl RocksDbArgs {
/// Validates the `RocksDB` arguments.
///
/// Returns an error if `--rocksdb.all` is used with any individual flag explicitly set to
/// `false`.
pub const fn validate(&self) -> Result<(), RocksDbArgsError> {
if self.all {
if matches!(self.tx_hash, Some(false)) {
return Err(RocksDbArgsError::ConflictingFlags("tx-hash"));
}
if matches!(self.storages_history, Some(false)) {
return Err(RocksDbArgsError::ConflictingFlags("storages-history"));
}
if matches!(self.account_history, Some(false)) {
return Err(RocksDbArgsError::ConflictingFlags("account-history"));
}
}
Ok(())
}
}
/// Error type for `RocksDB` argument validation.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum RocksDbArgsError {
/// `--rocksdb.all` cannot be combined with an individual flag set to false.
#[error("--rocksdb.all cannot be combined with --rocksdb.{0}=false")]
ConflictingFlags(&'static str),
}
#[cfg(test)]
mod tests {
use super::*;
use clap::Parser;
#[derive(Parser)]
struct CommandParser<T: Args> {
#[command(flatten)]
args: T,
}
#[test]
fn test_default_rocksdb_args() {
let args = CommandParser::<RocksDbArgs>::parse_from(["reth"]).args;
assert_eq!(args, RocksDbArgs::default());
assert!(!args.all);
assert!(args.tx_hash.is_none());
assert!(args.storages_history.is_none());
assert!(args.account_history.is_none());
}
#[test]
fn test_parse_all_flag() {
let args = CommandParser::<RocksDbArgs>::parse_from(["reth", "--rocksdb.all"]).args;
assert!(args.all);
assert!(args.tx_hash.is_none());
}
#[test]
fn test_parse_individual_flags() {
let args = CommandParser::<RocksDbArgs>::parse_from([
"reth",
"--rocksdb.tx-hash=true",
"--rocksdb.storages-history=false",
"--rocksdb.account-history=true",
])
.args;
assert!(!args.all);
assert_eq!(args.tx_hash, Some(true));
assert_eq!(args.storages_history, Some(false));
assert_eq!(args.account_history, Some(true));
}
#[test]
fn test_validate_all_with_none_ok() {
let args =
RocksDbArgs { all: true, tx_hash: None, storages_history: None, account_history: None };
assert!(args.validate().is_ok());
}
#[test]
fn test_validate_all_with_true_ok() {
let args = RocksDbArgs {
all: true,
tx_hash: Some(true),
storages_history: Some(true),
account_history: Some(true),
};
assert!(args.validate().is_ok());
}
#[test]
fn test_validate_all_with_false_errors() {
let args = RocksDbArgs {
all: true,
tx_hash: Some(false),
storages_history: None,
account_history: None,
};
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("tx-hash")));
let args = RocksDbArgs {
all: true,
tx_hash: None,
storages_history: Some(false),
account_history: None,
};
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("storages-history")));
let args = RocksDbArgs {
all: true,
tx_hash: None,
storages_history: None,
account_history: Some(false),
};
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("account-history")));
}
}

View File

@@ -9,6 +9,9 @@ use reth_config::config::{BlocksPerFileConfig, StaticFilesConfig};
pub const MINIMAL_BLOCKS_PER_FILE: u64 = 10000;
/// Parameters for static files configuration
///
/// When `--storage.v2` is used, the defaults for the storage flags change to enable static file
/// storage. Individual flags can still override those defaults when explicitly set.
#[derive(Debug, Args, PartialEq, Eq, Clone, Copy, Default)]
#[command(next_help_heading = "Static Files")]
pub struct StaticFilesArgs {
@@ -35,6 +38,53 @@ pub struct StaticFilesArgs {
/// Number of blocks per file for the storage changesets segment.
#[arg(long = "static-files.blocks-per-file.storage-change-sets")]
pub blocks_per_file_storage_change_sets: Option<u64>,
/// Store receipts in static files instead of the database.
///
/// When enabled, receipts will be written to static files on disk instead of the database.
///
/// Note: This setting can only be configured at genesis initialization. Once
/// the node has been initialized, changing this flag requires re-syncing from scratch.
///
/// Defaults to the base storage mode (v1: false, v2: true).
#[arg(long = "static-files.receipts", action = clap::ArgAction::Set)]
pub receipts: Option<bool>,
/// Store transaction senders in static files instead of the database.
///
/// When enabled, transaction senders will be written to static files on disk instead of the
/// database.
///
/// Note: This setting can only be configured at genesis initialization. Once
/// the node has been initialized, changing this flag requires re-syncing from scratch.
///
/// Defaults to the base storage mode (v1: false, v2: true).
#[arg(long = "static-files.transaction-senders", action = clap::ArgAction::Set)]
pub transaction_senders: Option<bool>,
/// Store account changesets in static files.
///
/// When enabled, account changesets will be written to static files on disk instead of the
/// database.
///
/// Note: This setting can only be configured at genesis initialization. Once
/// the node has been initialized, changing this flag requires re-syncing from scratch.
///
/// Defaults to the base storage mode (v1: false, v2: true).
#[arg(long = "static-files.account-change-sets", action = clap::ArgAction::Set)]
pub account_changesets: Option<bool>,
/// Store storage changesets in static files.
///
/// When enabled, storage changesets will be written to static files on disk instead of the
/// database.
///
/// Note: This setting can only be configured at genesis initialization. Once
/// the node has been initialized, changing this flag requires re-syncing from scratch.
///
/// Defaults to the base storage mode (v1: false, v2: true).
#[arg(long = "static-files.storage-change-sets", action = clap::ArgAction::Set)]
pub storage_changesets: Option<bool>,
}
impl StaticFilesArgs {

View File

@@ -1,13 +1,11 @@
//! clap [Args](clap::Args) for storage configuration
//! clap [Args](clap::Args) for storage mode configuration
use clap::{ArgAction, Args};
/// Parameters for storage configuration.
/// Parameters for storage mode configuration.
///
/// This controls whether the node uses v2 storage defaults (with `RocksDB` and static file
/// optimizations) or v1/legacy storage defaults.
///
/// Individual storage settings can be overridden with `--static-files.*` and `--rocksdb.*` flags.
#[derive(Debug, Args, PartialEq, Eq, Clone, Copy, Default)]
#[command(next_help_heading = "Storage")]
pub struct StorageArgs {
@@ -25,6 +23,16 @@ pub struct StorageArgs {
/// flags.
#[arg(long = "storage.v2", action = ArgAction::SetTrue)]
pub v2: bool,
/// Use hashed state tables (`HashedAccounts`/`HashedStorages`) as canonical state
/// representation instead of plain state tables.
///
/// When enabled, execution writes directly to hashed tables, eliminating the need for
/// separate hashing stages. This should only be enabled for new databases.
///
/// WARNING: Changing this setting on an existing database requires a full resync.
#[arg(long = "storage.use-hashed-state", default_value_t = false)]
pub use_hashed_state: bool,
}
#[cfg(test)]
@@ -32,24 +40,21 @@ mod tests {
use super::*;
use clap::Parser;
/// A helper type to parse Args more easily
#[derive(Parser)]
struct CommandParser<T: Args> {
struct CommandParser {
#[command(flatten)]
args: T,
args: StorageArgs,
}
#[test]
fn test_default_storage_args() {
let default_args = StorageArgs::default();
let args = CommandParser::<StorageArgs>::parse_from(["reth"]).args;
assert_eq!(args, default_args);
let args = CommandParser::parse_from(["reth"]).args;
assert!(!args.v2);
}
#[test]
fn test_parse_v2_flag() {
let args = CommandParser::<StorageArgs>::parse_from(["reth", "--storage.v2"]).args;
let args = CommandParser::parse_from(["reth", "--storage.v2"]).args;
assert!(args.v2);
}
}

View File

@@ -3,7 +3,7 @@
use crate::{
args::{
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, NetworkArgs, PayloadBuilderArgs,
PruningArgs, RpcServerArgs, StaticFilesArgs, StorageArgs, TxPoolArgs,
PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs, StorageArgs, TxPoolArgs,
},
dirs::{ChainPath, DataDirPath},
utils::get_single_header,
@@ -152,7 +152,10 @@ pub struct NodeConfig<ChainSpec> {
/// All static files related arguments
pub static_files: StaticFilesArgs,
/// All storage related arguments with --storage prefix
/// All `RocksDB` table routing arguments
pub rocksdb: RocksDbArgs,
/// Storage mode configuration (v2 vs v1/legacy)
pub storage: StorageArgs,
}
@@ -185,6 +188,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
engine: EngineArgs::default(),
era: EraArgs::default(),
static_files: StaticFilesArgs::default(),
rocksdb: RocksDbArgs::default(),
storage: StorageArgs::default(),
}
}
@@ -260,6 +264,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
engine,
era,
static_files,
rocksdb,
storage,
..
} = self;
@@ -280,6 +285,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
engine,
era,
static_files,
rocksdb,
storage,
}
}
@@ -349,12 +355,6 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
self
}
/// Set the storage args for the node
pub const fn with_storage(mut self, storage: StorageArgs) -> Self {
self.storage = storage;
self
}
/// Returns pruning configuration.
pub fn prune_config(&self) -> Option<PruneConfig>
where
@@ -363,17 +363,42 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
self.pruning.prune_config(&self.chain)
}
/// Returns the effective storage settings derived from `--storage.v2`.
/// Returns the effective storage settings derived from `--storage.v2`, static-file, and
/// `RocksDB` CLI args.
///
/// The base storage mode is determined by `--storage.v2`:
/// - When `--storage.v2` is set: uses [`StorageSettings::v2()`] defaults
/// - Otherwise: uses [`StorageSettings::base()`] defaults
/// - Otherwise: uses [`StorageSettings::v1()`] defaults
///
/// Individual `--static-files.*` and `--rocksdb.*` flags override the base when explicitly set.
pub const fn storage_settings(&self) -> StorageSettings {
if self.storage.v2 {
StorageSettings::v2()
} else {
StorageSettings::base()
let mut s = if self.storage.v2 { StorageSettings::v2() } else { StorageSettings::base() };
// Apply static files overrides (only when explicitly set)
s = s
.with_receipts_in_static_files_opt(self.static_files.receipts)
.with_transaction_senders_in_static_files_opt(self.static_files.transaction_senders)
.with_account_changesets_in_static_files_opt(self.static_files.account_changesets)
.with_storage_changesets_in_static_files_opt(self.static_files.storage_changesets);
// Apply rocksdb overrides
// --rocksdb.all sets all rocksdb flags to true
if self.rocksdb.all {
s = s
.with_transaction_hash_numbers_in_rocksdb(true)
.with_storages_history_in_rocksdb(true)
.with_account_history_in_rocksdb(true);
}
// Individual rocksdb flags override --rocksdb.all when explicitly set
s = s
.with_transaction_hash_numbers_in_rocksdb_opt(self.rocksdb.tx_hash)
.with_storages_history_in_rocksdb_opt(self.rocksdb.storages_history)
.with_account_history_in_rocksdb_opt(self.rocksdb.account_history);
s = s.with_use_hashed_state(self.storage.use_hashed_state);
s
}
/// Returns the max block that the node should run to, looking it up from the network if
@@ -570,6 +595,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
engine: self.engine,
era: self.era,
static_files: self.static_files,
rocksdb: self.rocksdb,
storage: self.storage,
}
}
@@ -612,6 +638,7 @@ impl<ChainSpec> Clone for NodeConfig<ChainSpec> {
engine: self.engine.clone(),
era: self.era.clone(),
static_files: self.static_files,
rocksdb: self.rocksdb,
storage: self.storage,
}
}

View File

@@ -265,7 +265,7 @@ impl NodeState {
warn!(number=block.number(), hash=?block.hash(), "Encountered invalid block");
}
ConsensusEngineEvent::BlockReceived(num_hash) => {
info!(number=num_hash.number, hash=?num_hash.hash, "Received new payload from consensus engine");
info!(number=num_hash.number, hash=?num_hash.hash, "Received block from consensus engine");
}
}
}

View File

@@ -4,7 +4,7 @@ use alloy_genesis::GenesisAccount;
use alloy_primitives::{keccak256, Bytes, B256, U256};
use alloy_trie::TrieAccount;
use derive_more::Deref;
use revm_bytecode::{Bytecode as RevmBytecode, BytecodeDecodeError};
use revm_bytecode::{Bytecode as RevmBytecode, BytecodeDecodeError, BytecodeKind};
use revm_state::AccountInfo;
#[cfg(any(test, feature = "reth-codec"))]
@@ -16,10 +16,10 @@ pub mod compact_ids {
/// Identifier for removed bytecode variant.
pub const REMOVED_BYTECODE_ID: u8 = 1;
/// Identifier for [`LegacyAnalyzed`](revm_bytecode::Bytecode::LegacyAnalyzed).
/// Identifier for [`LegacyAnalyzed`](revm_bytecode::BytecodeKind::LegacyAnalyzed).
pub const LEGACY_ANALYZED_BYTECODE_ID: u8 = 2;
/// Identifier for [`Eip7702`](revm_bytecode::Bytecode::Eip7702).
/// Identifier for [`Eip7702`](revm_bytecode::BytecodeKind::Eip7702).
pub const EIP7702_BYTECODE_ID: u8 = 4;
}
@@ -137,22 +137,19 @@ impl reth_codecs::Compact for Bytecode {
{
use compact_ids::{EIP7702_BYTECODE_ID, LEGACY_ANALYZED_BYTECODE_ID};
let bytecode = match &self.0 {
RevmBytecode::LegacyAnalyzed(analyzed) => analyzed.bytecode(),
RevmBytecode::Eip7702(eip7702) => eip7702.raw(),
};
let bytecode = self.0.bytecode();
buf.put_u32(bytecode.len() as u32);
buf.put_slice(bytecode.as_ref());
let len = match &self.0 {
let len = match self.0.kind() {
// [`REMOVED_BYTECODE_ID`] has been removed.
RevmBytecode::LegacyAnalyzed(analyzed) => {
BytecodeKind::LegacyAnalyzed => {
buf.put_u8(LEGACY_ANALYZED_BYTECODE_ID);
buf.put_u64(analyzed.original_len() as u64);
let map = analyzed.jump_table().as_slice();
buf.put_u64(self.0.len() as u64);
let map = self.0.legacy_jump_table().unwrap().as_slice();
buf.put_slice(map);
1 + 8 + map.len()
}
RevmBytecode::Eip7702(_) => {
BytecodeKind::Eip7702 => {
buf.put_u8(EIP7702_BYTECODE_ID);
1
}
@@ -251,12 +248,10 @@ impl From<Account> for AccountInfo {
#[cfg(test)]
mod tests {
use std::sync::Arc;
use super::*;
use alloy_primitives::{hex_literal::hex, B256, U256};
use reth_codecs::Compact;
use revm_bytecode::{JumpTable, LegacyAnalyzedBytecode};
use revm_bytecode::JumpTable;
#[test]
fn test_account() {
@@ -313,12 +308,11 @@ mod tests {
assert_eq!(len, 17);
let mut buf = vec![];
let bytecode =
Bytecode(RevmBytecode::LegacyAnalyzed(Arc::new(LegacyAnalyzedBytecode::new(
Bytes::from(&hex!("ff00")),
2,
JumpTable::from_slice(&[0], 2),
))));
let bytecode = Bytecode(RevmBytecode::new_analyzed(
Bytes::from(&hex!("ff00")),
2,
JumpTable::from_slice(&[0], 2),
));
let len = bytecode.to_compact(&mut buf);
assert_eq!(len, 16);

View File

@@ -164,7 +164,7 @@ pub use alloy_primitives::{logs_bloom, Log, LogData};
pub mod proofs;
mod storage;
pub use storage::{StorageEntry, StorageSlotKey, ValueWithSubKey};
pub use storage::{StorageEntry, ValueWithSubKey};
pub mod sync;

View File

@@ -1,4 +1,4 @@
use alloy_primitives::{keccak256, B256, U256};
use alloy_primitives::{B256, U256};
/// Trait for `DupSort` table values that contain a subkey.
///
@@ -12,117 +12,6 @@ pub trait ValueWithSubKey {
fn get_subkey(&self) -> Self::SubKey;
}
/// A storage slot key that tracks whether it holds a plain (unhashed) EVM slot
/// or a keccak256-hashed slot.
///
/// This enum replaces the `use_hashed_state: bool` parameter pattern by carrying
/// provenance with the key itself. Once tagged at a read/write boundary, downstream
/// code can call [`Self::to_hashed`] without risk of double-hashing — hashing a
/// [`StorageSlotKey::Hashed`] is a no-op.
///
/// The on-disk encoding is unchanged (raw 32-byte [`B256`]). The variant is set
/// by the code that knows the context (which table, which storage mode).
#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum StorageSlotKey {
/// An unhashed EVM storage slot, as produced by REVM execution.
Plain(B256),
/// A keccak256-hashed storage slot, as stored in `HashedStorages` and
/// in v2-mode `StorageChangeSets`.
Hashed(B256),
}
impl Default for StorageSlotKey {
fn default() -> Self {
Self::Plain(B256::ZERO)
}
}
impl StorageSlotKey {
/// Create a plain slot key from a REVM [`U256`] storage index.
pub const fn from_u256(slot: U256) -> Self {
Self::Plain(B256::new(slot.to_be_bytes()))
}
/// Create a plain slot key from a raw [`B256`].
pub const fn plain(key: B256) -> Self {
Self::Plain(key)
}
/// Create a hashed slot key from a raw [`B256`].
pub const fn hashed(key: B256) -> Self {
Self::Hashed(key)
}
/// Tag a raw [`B256`] based on the storage mode.
///
/// When `use_hashed_state` is true the key is assumed already hashed.
/// When false it is assumed to be a plain slot.
pub const fn from_raw(key: B256, use_hashed_state: bool) -> Self {
if use_hashed_state {
Self::Hashed(key)
} else {
Self::Plain(key)
}
}
/// Returns the raw [`B256`] regardless of variant.
pub const fn as_b256(&self) -> B256 {
match *self {
Self::Plain(b) | Self::Hashed(b) => b,
}
}
/// Returns `true` if this key is already hashed.
pub const fn is_hashed(&self) -> bool {
matches!(self, Self::Hashed(_))
}
/// Returns `true` if this key is plain (unhashed).
pub const fn is_plain(&self) -> bool {
matches!(self, Self::Plain(_))
}
/// Produce the keccak256-hashed form of this slot key.
///
/// - If already [`Hashed`](Self::Hashed), returns the inner value as-is (no double-hash).
/// - If [`Plain`](Self::Plain), applies keccak256 and returns the result.
pub fn to_hashed(&self) -> B256 {
match *self {
Self::Hashed(b) => b,
Self::Plain(b) => keccak256(b),
}
}
/// Convert a plain slot to its changeset representation.
///
/// In v2 mode (`use_hashed_state = true`), the changeset stores hashed keys,
/// so the plain key is hashed. In v1 mode, the plain key is stored as-is.
///
/// Panics (debug) if called on an already-hashed key.
pub fn to_changeset_key(self, use_hashed_state: bool) -> B256 {
debug_assert!(self.is_plain(), "to_changeset_key called on already-hashed key");
if use_hashed_state {
self.to_hashed()
} else {
self.as_b256()
}
}
/// Like [`to_changeset_key`](Self::to_changeset_key) but returns a tagged
/// [`StorageSlotKey`] instead of a raw [`B256`].
///
/// Panics (debug) if called on an already-hashed key.
pub fn to_changeset(self, use_hashed_state: bool) -> Self {
Self::from_raw(self.to_changeset_key(use_hashed_state), use_hashed_state)
}
}
impl From<StorageSlotKey> for B256 {
fn from(key: StorageSlotKey) -> Self {
key.as_b256()
}
}
/// Account storage entry.
///
/// `key` is the subkey when used as a value in the `StorageChangeSets` table.
@@ -142,14 +31,6 @@ impl StorageEntry {
pub const fn new(key: B256, value: U256) -> Self {
Self { key, value }
}
/// Tag this entry's key as a [`StorageSlotKey`] based on the storage mode.
///
/// When `use_hashed_state` is true, the key is tagged as already-hashed.
/// When false, it is tagged as plain.
pub const fn slot_key(&self, use_hashed_state: bool) -> StorageSlotKey {
StorageSlotKey::from_raw(self.key, use_hashed_state)
}
}
impl ValueWithSubKey for StorageEntry {

View File

@@ -75,7 +75,7 @@ where
// Check where account history indices are stored
#[cfg(all(unix, feature = "rocksdb"))]
if provider.cached_storage_settings().storage_v2 {
if provider.cached_storage_settings().account_history_in_rocksdb {
return self.prune_rocksdb(provider, input, range, range_end);
}
@@ -405,7 +405,9 @@ mod tests {
let segment = AccountHistory::new(prune_mode);
let provider = db.factory.database_provider_rw().unwrap();
provider.set_storage_settings_cache(StorageSettings::v1());
provider.set_storage_settings_cache(
StorageSettings::default().with_account_changesets_in_static_files(false),
);
let result = segment.prune(&provider, input).unwrap();
limiter.increment_deleted_entries_count_by(result.pruned);
@@ -506,11 +508,7 @@ mod tests {
test_prune(1400, 3, (PruneProgress::Finished, 804));
}
/// Tests the `prune_static_files` code path. On unix with rocksdb feature, v2 storage
/// routes to `prune_rocksdb` instead, so this test only runs without rocksdb (the
/// `prune_rocksdb_path` test covers that configuration).
#[test]
#[cfg(not(all(unix, feature = "rocksdb")))]
fn prune_static_file() {
let db = TestStageDB::default();
let mut rng = generators::rng();
@@ -566,7 +564,9 @@ mod tests {
let segment = AccountHistory::new(prune_mode);
let provider = db.factory.database_provider_rw().unwrap();
provider.set_storage_settings_cache(StorageSettings::v2());
provider.set_storage_settings_cache(
StorageSettings::default().with_account_changesets_in_static_files(true),
);
let result = segment.prune(&provider, input).unwrap();
limiter.increment_deleted_entries_count_by(result.pruned);
@@ -714,7 +714,11 @@ mod tests {
PruneInput { previous_checkpoint: None, to_block, limiter: PruneLimiter::default() };
let segment = AccountHistory::new(prune_mode);
db.factory.set_storage_settings_cache(StorageSettings::v2());
db.factory.set_storage_settings_cache(
StorageSettings::default()
.with_account_changesets_in_static_files(true)
.with_account_history_in_rocksdb(true),
);
let provider = db.factory.database_provider_rw().unwrap();
let result = segment.prune(&provider, input).unwrap();
@@ -828,7 +832,9 @@ mod tests {
let segment = AccountHistory::new(prune_mode);
let provider = db.factory.database_provider_rw().unwrap();
provider.set_storage_settings_cache(StorageSettings::v1());
provider.set_storage_settings_cache(
StorageSettings::default().with_account_changesets_in_static_files(false),
);
let result = segment.prune(&provider, input).unwrap();
// Should report that there's more data
@@ -886,7 +892,9 @@ mod tests {
};
let provider2 = db.factory.database_provider_rw().unwrap();
provider2.set_storage_settings_cache(StorageSettings::v1());
provider2.set_storage_settings_cache(
StorageSettings::default().with_account_changesets_in_static_files(false),
);
let result2 = segment.prune(&provider2, input2).unwrap();
assert!(result2.progress.is_finished(), "Second run should complete");

View File

@@ -76,7 +76,7 @@ where
// Check where storage history indices are stored
#[cfg(all(unix, feature = "rocksdb"))]
if provider.cached_storage_settings().storage_v2 {
if provider.cached_storage_settings().storages_history_in_rocksdb {
return self.prune_rocksdb(provider, input, range, range_end);
}
@@ -135,7 +135,7 @@ impl StorageHistory {
let (block_address, entry) = result?;
let block_number = block_address.block_number();
let address = block_address.address();
highest_deleted_storages.insert((address, entry.key.as_b256()), block_number);
highest_deleted_storages.insert((address, entry.key), block_number);
last_changeset_pruned_block = Some(block_number);
pruned_changesets += 1;
limiter.increment_deleted_entries_count();
@@ -273,7 +273,7 @@ impl StorageHistory {
let (block_address, entry) = result?;
let block_number = block_address.block_number();
let address = block_address.address();
highest_deleted_storages.insert((address, entry.key.as_b256()), block_number);
highest_deleted_storages.insert((address, entry.key), block_number);
last_changeset_pruned_block = Some(block_number);
changesets_processed += 1;
limiter.increment_deleted_entries_count();
@@ -413,7 +413,9 @@ mod tests {
let segment = StorageHistory::new(prune_mode);
let provider = db.factory.database_provider_rw().unwrap();
provider.set_storage_settings_cache(StorageSettings::v1());
provider.set_storage_settings_cache(
StorageSettings::default().with_storage_changesets_in_static_files(false),
);
let result = segment.prune(&provider, input).unwrap();
limiter.increment_deleted_entries_count_by(result.pruned);
@@ -518,11 +520,7 @@ mod tests {
test_prune(1200, 3, (PruneProgress::Finished, 202));
}
/// Tests the `prune_static_files` code path. On unix with rocksdb feature, v2 storage
/// routes to `prune_rocksdb` instead, so this test only runs without rocksdb (the
/// `prune_rocksdb_path` test covers that configuration).
#[test]
#[cfg(not(all(unix, feature = "rocksdb")))]
fn prune_static_file() {
let db = TestStageDB::default();
let mut rng = generators::rng();
@@ -579,7 +577,9 @@ mod tests {
let segment = StorageHistory::new(prune_mode);
let provider = db.factory.database_provider_rw().unwrap();
provider.set_storage_settings_cache(StorageSettings::v2());
provider.set_storage_settings_cache(
StorageSettings::default().with_storage_changesets_in_static_files(true),
);
let result = segment.prune(&provider, input).unwrap();
limiter.increment_deleted_entries_count_by(result.pruned);
@@ -739,7 +739,9 @@ mod tests {
let segment = StorageHistory::new(prune_mode);
let provider = db.factory.database_provider_rw().unwrap();
provider.set_storage_settings_cache(StorageSettings::v1());
provider.set_storage_settings_cache(
StorageSettings::default().with_storage_changesets_in_static_files(false),
);
let result = segment.prune(&provider, input).unwrap();
// Should report that there's more data
@@ -791,7 +793,9 @@ mod tests {
};
let provider2 = db.factory.database_provider_rw().unwrap();
provider2.set_storage_settings_cache(StorageSettings::v1());
provider2.set_storage_settings_cache(
StorageSettings::default().with_storage_changesets_in_static_files(false),
);
let result2 = segment.prune(&provider2, input2).unwrap();
assert!(result2.progress.is_finished(), "Second run should complete");
@@ -891,7 +895,11 @@ mod tests {
let segment = StorageHistory::new(prune_mode);
let provider = db.factory.database_provider_rw().unwrap();
provider.set_storage_settings_cache(StorageSettings::v2());
provider.set_storage_settings_cache(
StorageSettings::default()
.with_storage_changesets_in_static_files(true)
.with_storages_history_in_rocksdb(true),
);
let result = segment.prune(&provider, input).unwrap();
provider.commit().expect("commit");

View File

@@ -96,7 +96,7 @@ where
// Check where transaction hash numbers are stored
#[cfg(all(unix, feature = "rocksdb"))]
if provider.cached_storage_settings().storage_v2 {
if provider.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
return self.prune_rocksdb(provider, input, start, end);
}
@@ -491,7 +491,9 @@ mod tests {
let segment = TransactionLookup::new(prune_mode);
// Enable RocksDB storage for transaction hash numbers
db.factory.set_storage_settings_cache(StorageSettings::v2());
db.factory.set_storage_settings_cache(
StorageSettings::v1().with_transaction_hash_numbers_in_rocksdb(true),
);
let provider = db.factory.database_provider_rw().unwrap();
let result = segment.prune(&provider, input).unwrap();
@@ -576,7 +578,9 @@ mod tests {
}
// Enable RocksDB storage for transaction hash numbers
db.factory.set_storage_settings_cache(StorageSettings::v2());
db.factory.set_storage_settings_cache(
StorageSettings::v1().with_transaction_hash_numbers_in_rocksdb(true),
);
let to_block: BlockNumber = 6;
let prune_mode = PruneMode::Before(to_block);

View File

@@ -160,14 +160,6 @@ impl StateProvider for StateProviderTest {
) -> ProviderResult<Option<alloy_primitives::StorageValue>> {
Ok(self.accounts.get(&account).and_then(|(storage, _)| storage.get(&storage_key).copied()))
}
fn storage_by_hashed_key(
&self,
_address: Address,
_hashed_storage_key: StorageKey,
) -> ProviderResult<Option<alloy_primitives::StorageValue>> {
Ok(None)
}
}
impl BytecodeReader for StateProviderTest {

View File

@@ -70,8 +70,8 @@ impl ExecutionWitnessRecord {
}
}
}
// BTreeMap keys are ordered, so the first key is the smallest
self.lowest_block_number = statedb.block_hashes.keys().next().copied()
// Get the lowest block number from the cache
self.lowest_block_number = statedb.block_hashes.lowest().map(|(n, _)| n)
}
/// Creates the record from the state after execution.

View File

@@ -172,7 +172,7 @@ pub trait EstimateCall: Call {
};
let gas_refund = match res.result {
ExecutionResult::Success { gas_refunded, .. } => gas_refunded,
ExecutionResult::Success { gas, .. } => gas.inner_refunded(),
ExecutionResult::Halt { reason, .. } => {
// here we don't check for invalid opcode because already executed with highest gas
// limit

View File

@@ -154,14 +154,6 @@ impl StateProvider for StateProviderTraitObjWrapper {
self.0.storage(account, storage_key)
}
fn storage_by_hashed_key(
&self,
address: Address,
hashed_storage_key: alloy_primitives::StorageKey,
) -> reth_errors::ProviderResult<Option<alloy_primitives::StorageValue>> {
self.0.storage_by_hashed_key(address, hashed_storage_key)
}
fn account_code(
&self,
addr: &Address,

View File

@@ -125,9 +125,7 @@ pub trait FromEvmError<Evm: ConfigureEvm>:
match result {
ExecutionResult::Success { output, .. } => Ok(output.into_data()),
ExecutionResult::Revert { output, .. } => Err(Self::from_revert(output)),
ExecutionResult::Halt { reason, gas_used } => {
Err(Self::from_evm_halt(reason, gas_used))
}
ExecutionResult::Halt { reason, gas } => Err(Self::from_evm_halt(reason, gas.used())),
}
}
}

View File

@@ -352,7 +352,7 @@ where
let mut log_index = 0;
for (index, (result, tx)) in results.into_iter().zip(block.body().transactions()).enumerate() {
let call = match result {
ExecutionResult::Halt { reason, gas_used } => {
ExecutionResult::Halt { reason, gas } => {
let error = Err::from_evm_halt(reason, tx.gas_limit());
#[allow(clippy::needless_update)]
SimCallResult {
@@ -362,12 +362,12 @@ where
code: SIMULATE_VM_ERROR_CODE,
..SimulateError::invalid_params()
}),
gas_used,
gas_used: gas.used(),
logs: Vec::new(),
status: false,
}
}
ExecutionResult::Revert { output, gas_used } => {
ExecutionResult::Revert { output, gas } => {
let error = Err::from_revert(output.clone());
#[allow(clippy::needless_update)]
SimCallResult {
@@ -377,15 +377,15 @@ where
code: SIMULATE_REVERT_CODE,
..SimulateError::invalid_params()
}),
gas_used,
gas_used: gas.used(),
status: false,
logs: Vec::new(),
}
}
ExecutionResult::Success { output, gas_used, logs, .. } => SimCallResult {
ExecutionResult::Success { output, gas, logs, .. } => SimCallResult {
return_data: output.into_data(),
error: None,
gas_used,
gas_used: gas.used(),
logs: logs
.into_iter()
.map(|log| {

View File

@@ -22,7 +22,6 @@ use reth_stages_api::{
UnwindInput, UnwindOutput,
};
use reth_static_file_types::StaticFileSegment;
use reth_trie::KeccakKeyHasher;
use std::{
cmp::{max, Ordering},
collections::BTreeMap,
@@ -462,16 +461,9 @@ where
}
}
// Write output. When `use_hashed_state` is enabled, `write_state` skips writing to
// plain account/storage tables and only writes bytecodes and changesets. The hashed
// state is then written separately below.
// write output
provider.write_state(&state, OriginalValuesKnown::Yes, StateWriteConfig::default())?;
if provider.cached_storage_settings().use_hashed_state() {
let hashed_state = state.hash_state_slow::<KeccakKeyHasher>();
provider.write_hashed_state(&hashed_state.into_sorted())?;
}
let db_write_duration = time.elapsed();
debug!(
target: "sync::stages::execution",
@@ -1269,7 +1261,8 @@ mod tests {
// but no receipt data is written.
let factory = create_test_provider_factory();
factory.set_storage_settings_cache(StorageSettings::v2());
factory
.set_storage_settings_cache(StorageSettings::v1().with_receipts_in_static_files(true));
// Setup with block 1
let provider_rw = factory.database_provider_rw().unwrap();

View File

@@ -9,9 +9,7 @@ use reth_db_api::{
};
use reth_etl::Collector;
use reth_primitives_traits::Account;
use reth_provider::{
AccountExtReader, DBProvider, HashingWriter, StatsReader, StorageSettingsCache,
};
use reth_provider::{AccountExtReader, DBProvider, HashingWriter, StatsReader};
use reth_stages_api::{
AccountHashingCheckpoint, EntitiesCheckpoint, ExecInput, ExecOutput, Stage, StageCheckpoint,
StageError, StageId, UnwindInput, UnwindOutput,
@@ -136,11 +134,7 @@ impl Default for AccountHashingStage {
impl<Provider> Stage<Provider> for AccountHashingStage
where
Provider: DBProvider<Tx: DbTxMut>
+ HashingWriter
+ AccountExtReader
+ StatsReader
+ StorageSettingsCache,
Provider: DBProvider<Tx: DbTxMut> + HashingWriter + AccountExtReader + StatsReader,
{
/// Return the id of the stage
fn id(&self) -> StageId {
@@ -148,21 +142,11 @@ where
}
/// Execute the stage.
///
/// When `use_hashed_state` is enabled, this stage is a no-op because the execution stage
/// writes directly to `HashedAccounts`. Otherwise, it hashes plain state to populate hashed
/// tables.
fn execute(&mut self, provider: &Provider, input: ExecInput) -> Result<ExecOutput, StageError> {
if input.target_reached() {
return Ok(ExecOutput::done(input.checkpoint()))
}
// If using hashed state as canonical, execution already writes to `HashedAccounts`,
// so this stage becomes a no-op.
if provider.cached_storage_settings().use_hashed_state() {
return Ok(ExecOutput::done(input.checkpoint().with_block_number(input.target())));
}
let (from_block, to_block) = input.next_block_range().into_inner();
// if there are more blocks then threshold it is faster to go over Plain state and hash all
@@ -250,14 +234,10 @@ where
provider: &Provider,
input: UnwindInput,
) -> Result<UnwindOutput, StageError> {
// NOTE: this runs in both v1 and v2 mode. In v2 mode, execution writes
// directly to `HashedAccounts`, but the unwind must still revert those
// entries here because `MerkleUnwind` runs after this stage (in unwind
// order) and needs `HashedAccounts` to reflect the target block state
// before it can verify the state root.
let (range, unwind_progress, _) =
input.unwind_block_range_with_threshold(self.commit_threshold);
// Aggregate all transition changesets and make a list of accounts that have been changed.
provider.unwind_account_hashing_range(range)?;
let mut stage_checkpoint =

View File

@@ -15,7 +15,6 @@ use reth_stages_api::{
EntitiesCheckpoint, ExecInput, ExecOutput, Stage, StageCheckpoint, StageError, StageId,
StorageHashingCheckpoint, UnwindInput, UnwindOutput,
};
use reth_storage_api::StorageSettingsCache;
use reth_storage_errors::provider::ProviderResult;
use std::{
fmt::Debug,
@@ -69,11 +68,7 @@ impl Default for StorageHashingStage {
impl<Provider> Stage<Provider> for StorageHashingStage
where
Provider: DBProvider<Tx: DbTxMut>
+ StorageReader
+ HashingWriter
+ StatsReader
+ StorageSettingsCache,
Provider: DBProvider<Tx: DbTxMut> + StorageReader + HashingWriter + StatsReader,
{
/// Return the id of the stage
fn id(&self) -> StageId {
@@ -87,12 +82,6 @@ where
return Ok(ExecOutput::done(input.checkpoint()))
}
// If use_hashed_state is enabled, execution writes directly to `HashedStorages`,
// so this stage becomes a no-op.
if provider.cached_storage_settings().use_hashed_state() {
return Ok(ExecOutput::done(input.checkpoint().with_block_number(input.target())));
}
let (from_block, to_block) = input.next_block_range().into_inner();
// if there are more blocks then threshold it is faster to go over Plain state and hash all
@@ -187,11 +176,6 @@ where
provider: &Provider,
input: UnwindInput,
) -> Result<UnwindOutput, StageError> {
// NOTE: this runs in both v1 and v2 mode. In v2 mode, execution writes
// directly to `HashedStorages`, but the unwind must still revert those
// entries here because `MerkleUnwind` runs after this stage (in unwind
// order) and needs `HashedStorages` to reflect the target block state
// before it can verify the state root.
let (range, unwind_progress, _) =
input.unwind_block_range_with_threshold(self.commit_threshold);

View File

@@ -103,7 +103,7 @@ where
let mut range = input.next_block_range();
let first_sync = input.checkpoint().block_number == 0;
let use_rocksdb = provider.cached_storage_settings().storage_v2;
let use_rocksdb = provider.cached_storage_settings().account_history_in_rocksdb;
// On first sync we might have history coming from genesis. We clear the table since it's
// faster to rebuild from scratch.
@@ -122,7 +122,7 @@ where
info!(target: "sync::stages::index_account_history::exec", ?first_sync, ?use_rocksdb, "Collecting indices");
let collector = if provider.cached_storage_settings().storage_v2 {
let collector = if provider.cached_storage_settings().account_changesets_in_static_files {
// Use the provider-based collection that can read from static files.
collect_account_history_indices(provider, range.clone(), &self.etl_config)?
} else {
@@ -666,43 +666,32 @@ mod tests {
#[cfg(all(unix, feature = "rocksdb"))]
mod rocksdb_tests {
use super::*;
use reth_provider::{
providers::StaticFileWriter, RocksDBProviderFactory, StaticFileProviderFactory,
};
use reth_static_file_types::StaticFileSegment;
use reth_provider::RocksDBProviderFactory;
use reth_storage_api::StorageSettings;
/// Sets up v2 account test data: writes block body indices to MDBX and
/// account changesets to static files (matching realistic v2 layout).
fn setup_v2_account_data(db: &TestStageDB, block_range: std::ops::RangeInclusive<u64>) {
db.factory.set_storage_settings_cache(StorageSettings::v2());
db.commit(|tx| {
for block in block_range.clone() {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
}
Ok(())
})
.unwrap();
let static_file_provider = db.factory.static_file_provider();
let mut writer =
static_file_provider.latest_writer(StaticFileSegment::AccountChangeSets).unwrap();
for block in block_range {
writer.append_account_changeset(vec![acc()], block).unwrap();
}
writer.commit().unwrap();
}
/// Test that when `account_history_in_rocksdb` is enabled, the stage
/// writes account history indices to `RocksDB` instead of MDBX.
#[tokio::test]
async fn execute_writes_to_rocksdb_when_enabled() {
// init
let db = TestStageDB::default();
setup_v2_account_data(&db, 0..=10);
// Enable RocksDB for account history
db.factory.set_storage_settings_cache(
StorageSettings::v1().with_account_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=10 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::AccountChangeSets>(block, acc())?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(10), ..Default::default() };
let mut stage = IndexAccountHistoryStage::default();
@@ -732,7 +721,22 @@ mod tests {
#[tokio::test]
async fn unwind_works_when_rocksdb_enabled() {
let db = TestStageDB::default();
setup_v2_account_data(&db, 0..=10);
db.factory.set_storage_settings_cache(
StorageSettings::v1().with_account_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=10 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::AccountChangeSets>(block, acc())?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(10), ..Default::default() };
let mut stage = IndexAccountHistoryStage::default();
@@ -768,7 +772,22 @@ mod tests {
#[tokio::test]
async fn execute_incremental_sync() {
let db = TestStageDB::default();
setup_v2_account_data(&db, 0..=10);
db.factory.set_storage_settings_cache(
StorageSettings::v1().with_account_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=5 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::AccountChangeSets>(block, acc())?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(5), ..Default::default() };
let mut stage = IndexAccountHistoryStage::default();
@@ -783,6 +802,18 @@ mod tests {
let blocks: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks, (0..=5).collect::<Vec<_>>());
db.commit(|tx| {
for block in 6..=10 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::AccountChangeSets>(block, acc())?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(10), checkpoint: Some(StageCheckpoint::new(5)) };
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();

View File

@@ -107,7 +107,7 @@ where
let mut range = input.next_block_range();
let first_sync = input.checkpoint().block_number == 0;
let use_rocksdb = provider.cached_storage_settings().storage_v2;
let use_rocksdb = provider.cached_storage_settings().storages_history_in_rocksdb;
// On first sync we might have history coming from genesis. We clear the table since it's
// faster to rebuild from scratch.
@@ -125,7 +125,7 @@ where
}
info!(target: "sync::stages::index_storage_history::exec", ?first_sync, ?use_rocksdb, "Collecting indices");
let collector = if provider.cached_storage_settings().storage_v2 {
let collector = if provider.cached_storage_settings().storage_changesets_in_static_files {
collect_storage_history_indices(provider, range.clone(), &self.etl_config)?
} else {
collect_history_indices::<_, tables::StorageChangeSets, tables::StoragesHistory, _>(
@@ -694,51 +694,33 @@ mod tests {
#[cfg(all(unix, feature = "rocksdb"))]
mod rocksdb_tests {
use super::*;
use reth_db_api::models::StorageBeforeTx;
use reth_provider::{providers::StaticFileWriter, RocksDBProviderFactory};
use reth_static_file_types::StaticFileSegment;
use reth_provider::RocksDBProviderFactory;
use reth_storage_api::StorageSettings;
/// Sets up v2 storage test data: writes block body indices to MDBX and
/// storage changesets to static files (matching realistic v2 layout).
fn setup_v2_storage_data(db: &TestStageDB, block_range: std::ops::RangeInclusive<u64>) {
db.factory.set_storage_settings_cache(StorageSettings::v2());
db.commit(|tx| {
for block in block_range.clone() {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
}
Ok(())
})
.unwrap();
let static_file_provider = db.factory.static_file_provider();
let mut writer =
static_file_provider.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
for block in block_range {
writer
.append_storage_changeset(
vec![StorageBeforeTx {
address: ADDRESS,
key: STORAGE_KEY,
value: U256::ZERO,
}],
block,
)
.unwrap();
}
writer.commit().unwrap();
}
/// Test that when `storages_history_in_rocksdb` is enabled, the stage
/// writes storage history indices to `RocksDB` instead of MDBX.
#[tokio::test]
async fn execute_writes_to_rocksdb_when_enabled() {
let db = TestStageDB::default();
setup_v2_storage_data(&db, 0..=10);
db.factory.set_storage_settings_cache(
StorageSettings::v1().with_storages_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=10 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(10), ..Default::default() };
let mut stage = IndexStorageHistoryStage::default();
@@ -766,7 +748,25 @@ mod tests {
#[tokio::test]
async fn unwind_works_when_rocksdb_enabled() {
let db = TestStageDB::default();
setup_v2_storage_data(&db, 0..=10);
db.factory.set_storage_settings_cache(
StorageSettings::v1().with_storages_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=10 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(10), ..Default::default() };
let mut stage = IndexStorageHistoryStage::default();
@@ -803,7 +803,25 @@ mod tests {
#[tokio::test]
async fn unwind_to_zero_keeps_block_zero() {
let db = TestStageDB::default();
setup_v2_storage_data(&db, 0..=5);
db.factory.set_storage_settings_cache(
StorageSettings::v1().with_storages_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=5 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(5), ..Default::default() };
let mut stage = IndexStorageHistoryStage::default();
@@ -834,7 +852,25 @@ mod tests {
#[tokio::test]
async fn execute_incremental_sync() {
let db = TestStageDB::default();
setup_v2_storage_data(&db, 0..=10);
db.factory.set_storage_settings_cache(
StorageSettings::v1().with_storages_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=5 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(5), ..Default::default() };
let mut stage = IndexStorageHistoryStage::default();
@@ -849,6 +885,21 @@ mod tests {
let blocks: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks, (0..=5).collect::<Vec<_>>());
db.commit(|tx| {
for block in 6..=10 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(10), checkpoint: Some(StageCheckpoint::new(5)) };
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
@@ -868,8 +919,27 @@ mod tests {
use reth_db_api::models::sharded_key::NUM_OF_INDICES_IN_SHARD;
let db = TestStageDB::default();
db.factory.set_storage_settings_cache(
StorageSettings::v1().with_storages_history_in_rocksdb(true),
);
let num_blocks = (NUM_OF_INDICES_IN_SHARD * 2 + 100) as u64;
setup_v2_storage_data(&db, 0..=num_blocks - 1);
db.commit(|tx| {
for block in 0..num_blocks {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(num_blocks - 1), ..Default::default() };
let mut stage = IndexStorageHistoryStage::default();

View File

@@ -9,7 +9,7 @@ use reth_db_api::{
use reth_primitives_traits::{GotExpected, SealedHeader};
use reth_provider::{
ChangeSetReader, DBProvider, HeaderProvider, ProviderError, StageCheckpointReader,
StageCheckpointWriter, StatsReader, StorageChangeSetReader, StorageSettingsCache, TrieWriter,
StageCheckpointWriter, StatsReader, StorageChangeSetReader, TrieWriter,
};
use reth_stages_api::{
BlockErrorKind, EntitiesCheckpoint, ExecInput, ExecOutput, MerkleCheckpoint, Stage,
@@ -160,7 +160,6 @@ where
+ HeaderProvider
+ ChangeSetReader
+ StorageChangeSetReader
+ StorageSettingsCache
+ StageCheckpointReader
+ StageCheckpointWriter,
{

View File

@@ -540,7 +540,9 @@ mod tests {
let mut rng = generators::rng();
let runner = SenderRecoveryTestRunner::default();
runner.db.factory.set_storage_settings_cache(StorageSettings::v2());
runner.db.factory.set_storage_settings_cache(
StorageSettings::v1().with_transaction_senders_in_static_files(true),
);
let input = ExecInput {
target: Some(target),
checkpoint: Some(StageCheckpoint::new(stage_progress)),

View File

@@ -200,7 +200,7 @@ where
}
#[cfg(all(unix, feature = "rocksdb"))]
if provider.cached_storage_settings().storage_v2 {
if provider.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
provider.commit_pending_rocksdb_batches()?;
provider.rocksdb_provider().flush(&[Tables::TransactionHashNumbers.name()])?;
}
@@ -618,7 +618,9 @@ mod tests {
let runner = TransactionLookupTestRunner::default();
// Enable RocksDB for transaction hash numbers
runner.db.factory.set_storage_settings_cache(StorageSettings::v2());
runner.db.factory.set_storage_settings_cache(
StorageSettings::v1().with_transaction_hash_numbers_in_rocksdb(true),
);
let input = ExecInput {
target: Some(previous_stage),
@@ -684,7 +686,9 @@ mod tests {
let runner = TransactionLookupTestRunner::default();
// Enable RocksDB for transaction hash numbers
runner.db.factory.set_storage_settings_cache(StorageSettings::v2());
runner.db.factory.set_storage_settings_cache(
StorageSettings::v1().with_transaction_hash_numbers_in_rocksdb(true),
);
// Insert blocks with transactions
let blocks = random_block_range(

View File

@@ -208,10 +208,7 @@ where
for (idx, changeset_result) in walker.enumerate() {
let (BlockNumberAddress((block_number, address)), storage) = changeset_result?;
cache
.entry(AddressStorageKey((address, storage.key.as_b256())))
.or_default()
.push(block_number);
cache.entry(AddressStorageKey((address, storage.key))).or_default().push(block_number);
if idx > 0 && idx % interval == 0 && total_changesets > 1000 {
info!(target: "sync::stages::index_history", progress = %format!("{:.4}%", (idx as f64 / total_changesets as f64) * 100.0), "Collecting indices");

View File

@@ -3,7 +3,7 @@
use alloy_consensus::{constants::ETH_TO_WEI, Header, TxEip1559, TxReceipt};
use alloy_eips::eip1559::INITIAL_BASE_FEE;
use alloy_genesis::{Genesis, GenesisAccount};
use alloy_primitives::{bytes, keccak256, Address, Bytes, TxKind, B256, U256};
use alloy_primitives::{bytes, Address, Bytes, TxKind, B256, U256};
use reth_chainspec::{ChainSpecBuilder, ChainSpecProvider, MAINNET};
use reth_config::config::StageConfig;
use reth_consensus::noop::NoopConsensus;
@@ -36,7 +36,7 @@ use reth_stages::sets::DefaultStages;
use reth_stages_api::{Pipeline, StageId};
use reth_static_file::StaticFileProducer;
use reth_storage_api::{
ChangeSetReader, StateProvider, StorageChangeSetReader, StorageSettings, StorageSettingsCache,
ChangeSetReader, StateProvider, StorageChangeSetReader, StorageSettingsCache,
};
use reth_testing_utils::generators::{self, generate_key, sign_tx_with_key_pair};
use reth_trie::{HashedPostState, KeccakKeyHasher, StateRoot};
@@ -79,7 +79,7 @@ fn assert_changesets_queryable(
let settings = provider.cached_storage_settings();
// Verify storage changesets
if settings.storage_v2 {
if settings.storage_changesets_in_static_files {
let static_file_provider = provider_factory.static_file_provider();
static_file_provider.initialize_index()?;
let storage_changesets =
@@ -89,11 +89,6 @@ fn assert_changesets_queryable(
"storage changesets should be queryable from static files for blocks {:?}",
block_range
);
// Verify keys are in hashed format (v2 mode)
for (_, entry) in &storage_changesets {
assert!(entry.key.is_hashed(), "v2: storage changeset keys should be tagged as hashed");
}
} else {
let storage_changesets: Vec<_> = provider
.tx_ref()
@@ -105,20 +100,10 @@ fn assert_changesets_queryable(
"storage changesets should be queryable from MDBX for blocks {:?}",
block_range
);
// Verify keys are plain (not hashed) in v1 mode
for (_, entry) in &storage_changesets {
let key = entry.key;
assert_ne!(
key,
keccak256(key),
"v1: storage changeset key should be plain (not its own keccak256)"
);
}
}
// Verify account changesets
if settings.storage_v2 {
if settings.account_changesets_in_static_files {
let static_file_provider = provider_factory.static_file_provider();
static_file_provider.initialize_index()?;
let account_changesets =
@@ -216,22 +201,19 @@ where
pipeline
}
/// Shared helper for pipeline forward sync and unwind tests.
/// Tests pipeline with ALL stages enabled using both ETH transfers and contract storage changes.
///
/// This test:
/// 1. Pre-funds a signer account and deploys a Counter contract in genesis
/// 2. Each block contains two transactions:
/// - ETH transfer to a recipient (account state changes)
/// - Counter `increment()` call (storage state changes)
/// 3. Runs the full pipeline with ALL stages enabled
/// 4. Forward syncs to `num_blocks`, unwinds to `unwind_target`, then re-syncs back to `num_blocks`
/// 4. Forward syncs to block 5, unwinds to block 2
///
/// When `storage_settings` is `Some`, the pipeline provider factory is configured with the given
/// settings before genesis initialization (e.g. v2 storage mode).
async fn run_pipeline_forward_and_unwind(
storage_settings: Option<StorageSettings>,
num_blocks: u64,
unwind_target: u64,
) -> eyre::Result<()> {
/// This exercises both account and storage hashing/history stages.
#[tokio::test(flavor = "multi_thread")]
async fn test_pipeline() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
// Generate a keypair for signing transactions
@@ -277,6 +259,7 @@ async fn run_pipeline_forward_and_unwind(
let evm_config = EthEvmConfig::new(chain_spec.clone());
// Build blocks by actually executing transactions to get correct state roots
let num_blocks = 5u64;
let mut blocks: Vec<SealedBlock<Block>> = Vec::new();
let mut parent_hash = genesis.hash();
@@ -401,15 +384,11 @@ async fn run_pipeline_forward_and_unwind(
// This is needed because we wrote state during block generation for computing state roots
let pipeline_provider_factory =
create_test_provider_factory_with_chain_spec(chain_spec.clone());
if let Some(settings) = storage_settings {
pipeline_provider_factory.set_storage_settings_cache(settings);
}
init_genesis(&pipeline_provider_factory).expect("init genesis");
let pipeline_genesis =
pipeline_provider_factory.sealed_header(0)?.expect("genesis should exist");
let pipeline_consensus = NoopConsensus::arc();
let blocks_clone = blocks.clone();
let file_client = create_file_client_from_blocks(blocks);
let max_block = file_client.max_block().unwrap();
let tip = file_client.tip().expect("tip");
@@ -438,7 +417,7 @@ async fn run_pipeline_forward_and_unwind(
{
let provider = pipeline_provider_factory.provider()?;
let last_block = provider.last_block_number()?;
assert_eq!(last_block, num_blocks, "should have synced {num_blocks} blocks");
assert_eq!(last_block, 5, "should have synced 5 blocks");
for stage_id in [
StageId::Headers,
@@ -456,28 +435,29 @@ async fn run_pipeline_forward_and_unwind(
let checkpoint = provider.get_stage_checkpoint(stage_id)?;
assert_eq!(
checkpoint.map(|c| c.block_number),
Some(num_blocks),
"{stage_id} checkpoint should be at block {num_blocks}"
Some(5),
"{stage_id} checkpoint should be at block 5"
);
}
// Verify the counter contract's storage was updated
// After num_blocks blocks with 1 increment each, slot 0 should be num_blocks
// After 5 blocks with 1 increment each, slot 0 should be 5
let state = provider.latest();
let counter_storage = state.storage(CONTRACT_ADDRESS, B256::ZERO)?;
assert_eq!(
counter_storage,
Some(U256::from(num_blocks)),
"Counter storage slot 0 should be {num_blocks} after {num_blocks} increments"
Some(U256::from(5)),
"Counter storage slot 0 should be 5 after 5 increments"
);
}
// Verify changesets are queryable before unwind
// This validates that the #21561 fix works - unwind needs to read changesets from the correct
// source
assert_changesets_queryable(&pipeline_provider_factory, 1..=num_blocks)?;
assert_changesets_queryable(&pipeline_provider_factory, 1..=5)?;
// Unwind to unwind_target
// Unwind to block 2
let unwind_target = 2u64;
pipeline.unwind(unwind_target, None)?;
// Verify unwind
@@ -504,114 +484,7 @@ async fn run_pipeline_forward_and_unwind(
);
}
}
let state = provider.latest();
let counter_storage = state.storage(CONTRACT_ADDRESS, B256::ZERO)?;
assert_eq!(
counter_storage,
Some(U256::from(unwind_target)),
"Counter storage slot 0 should be {unwind_target} after unwinding to block {unwind_target}"
);
}
// Re-sync: build a new pipeline starting from unwind_target and sync back to num_blocks
let resync_file_client = create_file_client_from_blocks(blocks_clone);
let resync_consensus = NoopConsensus::arc();
let resync_stages_config = StageConfig::default();
let unwind_head = pipeline_provider_factory
.sealed_header(unwind_target)?
.expect("unwind target header should exist");
let mut resync_header_downloader =
ReverseHeadersDownloaderBuilder::new(resync_stages_config.headers)
.build(resync_file_client.clone(), resync_consensus.clone())
.into_task();
resync_header_downloader.update_local_head(unwind_head);
resync_header_downloader.update_sync_target(SyncTarget::Tip(tip));
let mut resync_body_downloader = BodiesDownloaderBuilder::new(resync_stages_config.bodies)
.build(resync_file_client, resync_consensus, pipeline_provider_factory.clone())
.into_task();
resync_body_downloader
.set_download_range(unwind_target + 1..=max_block)
.expect("set download range");
let resync_pipeline = build_pipeline(
pipeline_provider_factory.clone(),
resync_header_downloader,
resync_body_downloader,
max_block,
tip,
);
let (_resync_pipeline, resync_result) = resync_pipeline.run_as_fut(None).await;
resync_result?;
// Verify re-sync
{
let provider = pipeline_provider_factory.provider()?;
let last_block = provider.last_block_number()?;
assert_eq!(last_block, num_blocks, "should have re-synced to {num_blocks} blocks");
for stage_id in [
StageId::Headers,
StageId::Bodies,
StageId::SenderRecovery,
StageId::Execution,
StageId::AccountHashing,
StageId::StorageHashing,
StageId::MerkleExecute,
StageId::TransactionLookup,
StageId::IndexAccountHistory,
StageId::IndexStorageHistory,
StageId::Finish,
] {
let checkpoint = provider.get_stage_checkpoint(stage_id)?;
assert_eq!(
checkpoint.map(|c| c.block_number),
Some(num_blocks),
"{stage_id} checkpoint should be at block {num_blocks} after re-sync"
);
}
let state = provider.latest();
let counter_storage = state.storage(CONTRACT_ADDRESS, B256::ZERO)?;
assert_eq!(
counter_storage,
Some(U256::from(num_blocks)),
"Counter storage slot 0 should be {num_blocks} after re-sync"
);
}
Ok(())
}
/// Tests pipeline with ALL stages enabled using both ETH transfers and contract storage changes.
///
/// This test:
/// 1. Pre-funds a signer account and deploys a Counter contract in genesis
/// 2. Each block contains two transactions:
/// - ETH transfer to a recipient (account state changes)
/// - Counter `increment()` call (storage state changes)
/// 3. Runs the full pipeline with ALL stages enabled
/// 4. Forward syncs to block 5, unwinds to block 2, then re-syncs to block 5
///
/// This exercises both account and storage hashing/history stages.
#[tokio::test(flavor = "multi_thread")]
async fn test_pipeline() -> eyre::Result<()> {
run_pipeline_forward_and_unwind(None, 5, 2).await
}
/// Same as [`test_pipeline`] but runs with v2 storage settings (`use_hashed_state=true`,
/// `is_v2()=true`, etc.).
///
/// In v2 mode:
/// - The execution stage writes directly to `HashedAccounts`/`HashedStorages`
/// - `AccountHashingStage` and `StorageHashingStage` are no-ops during forward execution
/// - Changesets are stored in static files with pre-hashed storage keys
/// - Unwind must still revert hashed state via the hashing stages before `MerkleUnwind` validates
#[tokio::test(flavor = "multi_thread")]
async fn test_pipeline_v2() -> eyre::Result<()> {
run_pipeline_forward_and_unwind(Some(StorageSettings::v2()), 5, 2).await
}

View File

@@ -5,101 +5,208 @@ use serde::{Deserialize, Serialize};
/// Storage configuration settings for this node.
///
/// Controls whether this node uses v2 storage layout (static files + `RocksDB` routing)
/// or v1/legacy layout (everything in MDBX).
///
/// These should be set during `init_genesis` or `init_db` depending on whether we want dictate
/// behaviour of new or old nodes respectively.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Compact, Serialize, Deserialize)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, Compact)]
#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
#[add_arbitrary_tests(compact)]
pub struct StorageSettings {
/// Whether this node uses v2 storage layout.
/// Whether this node always writes receipts to static files.
///
/// When `true`, enables all v2 storage features:
/// - Receipts and transaction senders in static files
/// - History indices in `RocksDB` (accounts, storages, transaction hashes)
/// - Account and storage changesets in static files
/// - Hashed state tables as canonical state representation
///
/// When `false`, uses v1/legacy layout (everything in MDBX).
pub storage_v2: bool,
/// If this is set to FALSE AND receipt pruning IS ENABLED, all receipts should be written to DB. Otherwise, they should be written to static files. This ensures that older nodes do not need to migrate their current DB tables to static files. For more, read: <https://github.com/paradigmxyz/reth/issues/18890#issuecomment-3457760097>
#[serde(default)]
pub receipts_in_static_files: bool,
/// Whether this node always writes transaction senders to static files.
#[serde(default)]
pub transaction_senders_in_static_files: bool,
/// Whether `StoragesHistory` is stored in `RocksDB`.
#[serde(default)]
pub storages_history_in_rocksdb: bool,
/// Whether `TransactionHashNumbers` is stored in `RocksDB`.
#[serde(default)]
pub transaction_hash_numbers_in_rocksdb: bool,
/// Whether `AccountsHistory` is stored in `RocksDB`.
#[serde(default)]
pub account_history_in_rocksdb: bool,
/// Whether this node should read and write account changesets from static files.
#[serde(default)]
pub account_changesets_in_static_files: bool,
/// Whether this node should read and write storage changesets from static files.
#[serde(default)]
pub storage_changesets_in_static_files: bool,
/// Whether to use hashed state tables (`HashedAccounts`/`HashedStorages`) as the canonical
/// state representation instead of plain state tables.
#[serde(default)]
pub use_hashed_state: bool,
}
impl StorageSettings {
/// Returns the default base `StorageSettings`.
///
/// When the `edge` feature is enabled, returns [`Self::v2()`] so that CI and
/// edge builds automatically use v2 storage defaults. Otherwise returns
/// [`Self::v1()`]. The `--storage.v2` CLI flag can also opt into v2 at runtime
/// regardless of feature flags.
/// Always returns [`Self::v1()`]. Use the `--storage.v2` CLI flag to opt into
/// [`Self::v2()`] at runtime. The `rocksdb` feature only makes the v2 backend
/// *available*; it does not activate it by default.
pub const fn base() -> Self {
#[cfg(feature = "edge")]
{
Self::v2()
}
#[cfg(not(feature = "edge"))]
{
Self::v1()
}
Self::v1()
}
/// Creates `StorageSettings` for v2 nodes with all storage features enabled:
/// - Receipts and transaction senders in static files
/// - History indices in `RocksDB` (storages, accounts, transaction hashes)
/// - Account and storage changesets in static files
/// - Hashed state as canonical state representation
///
/// Use this when the `--storage.v2` CLI flag is set.
pub const fn v2() -> Self {
Self { storage_v2: true }
Self {
receipts_in_static_files: true,
transaction_senders_in_static_files: true,
account_changesets_in_static_files: true,
storage_changesets_in_static_files: true,
storages_history_in_rocksdb: true,
transaction_hash_numbers_in_rocksdb: true,
account_history_in_rocksdb: true,
use_hashed_state: false,
}
}
/// Creates `StorageSettings` for v1/legacy nodes.
///
/// This keeps all data in MDBX, matching the original storage layout.
/// This explicitly sets `receipts_in_static_files` and `transaction_senders_in_static_files` to
/// `false`, ensuring older nodes continue writing receipts and transaction senders to the
/// database when receipt pruning is enabled.
pub const fn v1() -> Self {
Self { storage_v2: false }
Self {
receipts_in_static_files: false,
transaction_senders_in_static_files: false,
storages_history_in_rocksdb: false,
transaction_hash_numbers_in_rocksdb: false,
account_history_in_rocksdb: false,
account_changesets_in_static_files: false,
storage_changesets_in_static_files: false,
use_hashed_state: false,
}
}
/// Returns `true` if this node uses v2 storage layout.
pub const fn is_v2(&self) -> bool {
self.storage_v2
/// Sets the `receipts_in_static_files` flag to the provided value.
pub const fn with_receipts_in_static_files(mut self, value: bool) -> Self {
self.receipts_in_static_files = value;
self
}
/// Whether receipts are stored in static files.
pub const fn receipts_in_static_files(&self) -> bool {
self.storage_v2
/// Sets the `transaction_senders_in_static_files` flag to the provided value.
pub const fn with_transaction_senders_in_static_files(mut self, value: bool) -> Self {
self.transaction_senders_in_static_files = value;
self
}
/// Whether transaction senders are stored in static files.
pub const fn transaction_senders_in_static_files(&self) -> bool {
self.storage_v2
/// Sets the `storages_history_in_rocksdb` flag to the provided value.
pub const fn with_storages_history_in_rocksdb(mut self, value: bool) -> Self {
self.storages_history_in_rocksdb = value;
self
}
/// Whether storages history is stored in `RocksDB`.
pub const fn storages_history_in_rocksdb(&self) -> bool {
self.storage_v2
/// Sets the `transaction_hash_numbers_in_rocksdb` flag to the provided value.
pub const fn with_transaction_hash_numbers_in_rocksdb(mut self, value: bool) -> Self {
self.transaction_hash_numbers_in_rocksdb = value;
self
}
/// Whether transaction hash numbers are stored in `RocksDB`.
pub const fn transaction_hash_numbers_in_rocksdb(&self) -> bool {
self.storage_v2
/// Sets the `account_history_in_rocksdb` flag to the provided value.
pub const fn with_account_history_in_rocksdb(mut self, value: bool) -> Self {
self.account_history_in_rocksdb = value;
self
}
/// Whether account history is stored in `RocksDB`.
pub const fn account_history_in_rocksdb(&self) -> bool {
self.storage_v2
/// Sets the `account_changesets_in_static_files` flag to the provided value.
pub const fn with_account_changesets_in_static_files(mut self, value: bool) -> Self {
self.account_changesets_in_static_files = value;
self
}
/// Whether to use hashed state tables (`HashedAccounts`/`HashedStorages`) as the canonical
/// state representation instead of plain state tables. Implied by v2 storage layout.
pub const fn use_hashed_state(&self) -> bool {
self.storage_v2
/// Sets the `storage_changesets_in_static_files` flag to the provided value.
pub const fn with_storage_changesets_in_static_files(mut self, value: bool) -> Self {
self.storage_changesets_in_static_files = value;
self
}
/// Sets the `use_hashed_state` flag to the provided value.
pub const fn with_use_hashed_state(mut self, value: bool) -> Self {
self.use_hashed_state = value;
self
}
/// Sets `receipts_in_static_files` if `value` is `Some`.
pub const fn with_receipts_in_static_files_opt(mut self, value: Option<bool>) -> Self {
if let Some(v) = value {
self.receipts_in_static_files = v;
}
self
}
/// Sets `transaction_senders_in_static_files` if `value` is `Some`.
pub const fn with_transaction_senders_in_static_files_opt(
mut self,
value: Option<bool>,
) -> Self {
if let Some(v) = value {
self.transaction_senders_in_static_files = v;
}
self
}
/// Sets `account_changesets_in_static_files` if `value` is `Some`.
pub const fn with_account_changesets_in_static_files_opt(
mut self,
value: Option<bool>,
) -> Self {
if let Some(v) = value {
self.account_changesets_in_static_files = v;
}
self
}
/// Sets `storage_changesets_in_static_files` if `value` is `Some`.
pub const fn with_storage_changesets_in_static_files_opt(
mut self,
value: Option<bool>,
) -> Self {
if let Some(v) = value {
self.storage_changesets_in_static_files = v;
}
self
}
/// Sets `transaction_hash_numbers_in_rocksdb` if `value` is `Some`.
pub const fn with_transaction_hash_numbers_in_rocksdb_opt(
mut self,
value: Option<bool>,
) -> Self {
if let Some(v) = value {
self.transaction_hash_numbers_in_rocksdb = v;
}
self
}
/// Sets `storages_history_in_rocksdb` if `value` is `Some`.
pub const fn with_storages_history_in_rocksdb_opt(mut self, value: Option<bool>) -> Self {
if let Some(v) = value {
self.storages_history_in_rocksdb = v;
}
self
}
/// Sets `account_history_in_rocksdb` if `value` is `Some`.
pub const fn with_account_history_in_rocksdb_opt(mut self, value: Option<bool>) -> Self {
if let Some(v) = value {
self.account_history_in_rocksdb = v;
}
self
}
/// Returns `true` if any tables are configured to be stored in `RocksDB`.
pub const fn any_in_rocksdb(&self) -> bool {
self.storage_v2
self.transaction_hash_numbers_in_rocksdb ||
self.account_history_in_rocksdb ||
self.storages_history_in_rocksdb
}
}

View File

@@ -214,13 +214,13 @@ where
// not the genesis block number. This would cause increment_block(N) to fail.
let static_file_provider = provider_rw.static_file_provider();
if genesis_block_number > 0 {
if genesis_storage_settings.storage_v2 {
if genesis_storage_settings.account_changesets_in_static_files {
static_file_provider
.get_writer(genesis_block_number, StaticFileSegment::AccountChangeSets)?
.user_header_mut()
.set_expected_block_start(genesis_block_number);
}
if genesis_storage_settings.storage_v2 {
if genesis_storage_settings.storage_changesets_in_static_files {
static_file_provider
.get_writer(genesis_block_number, StaticFileSegment::StorageChangeSets)?
.user_header_mut()
@@ -259,7 +259,7 @@ where
.user_header_mut()
.set_block_range(genesis_block_number, genesis_block_number);
if genesis_storage_settings.storage_v2 {
if genesis_storage_settings.transaction_senders_in_static_files {
static_file_provider
.get_writer(genesis_block_number, StaticFileSegment::TransactionSenders)?
.user_header_mut()
@@ -1052,7 +1052,7 @@ mod tests {
)
};
let (accounts, storages) = if settings.storage_v2 {
let (accounts, storages) = if settings.account_history_in_rocksdb {
collect_rocksdb(&rocksdb)
} else {
collect_from_mdbx(&factory)
@@ -1075,7 +1075,10 @@ mod tests {
init_genesis_with_settings(&factory, StorageSettings::v1()).unwrap();
// Request different settings - should warn but succeed
let result = init_genesis_with_settings(&factory, StorageSettings::v2());
let result = init_genesis_with_settings(
&factory,
StorageSettings::v1().with_receipts_in_static_files(true),
);
// Should succeed (warning is logged, not an error)
assert!(result.is_ok());
@@ -1084,7 +1087,7 @@ mod tests {
#[test]
fn allow_same_storage_settings() {
let factory = create_test_provider_factory_with_chain_spec(MAINNET.clone());
let settings = StorageSettings::v2();
let settings = StorageSettings::v1().with_receipts_in_static_files(true);
init_genesis_with_settings(&factory, settings).unwrap();
let result = init_genesis_with_settings(&factory, settings);

View File

@@ -27,8 +27,6 @@ alloy-primitives.workspace = true
# mdbx
reth-libmdbx = { workspace = true, optional = true, features = ["return-borrowed", "read-tx-timeouts"] }
eyre = { workspace = true, optional = true }
reth-mdbx-viz = { workspace = true, optional = true }
tokio = { workspace = true, optional = true, features = ["full"] }
# metrics
reth-metrics = { workspace = true, optional = true }
@@ -99,7 +97,6 @@ op = [
"reth-db-api/op",
"reth-primitives-traits/op",
]
pageviz = ["reth-libmdbx/pageviz", "dep:reth-mdbx-viz", "dep:tokio", "mdbx"]
disable-lock = []
[[bench]]

View File

@@ -522,81 +522,6 @@ impl DatabaseEnv {
self
}
/// Starts the real-time page access visualization server.
///
/// This enables the C-level page access instrumentation hooks,
/// starts a background drainer thread that coalesces events,
/// and launches an HTTP+WebSocket server on the given port.
///
/// The server serves an interactive HTML visualization at `http://localhost:{port}/`
/// showing page accesses in real-time with fading highlights:
/// - Blue = read, Red = write, Yellow = free
#[cfg(feature = "pageviz")]
pub fn start_pageviz(&self, port: u16, db_path: std::path::PathBuf) {
use reth_libmdbx::pageviz::PageVizDrainer;
let max_dbi = self.dbis.values().copied().max().unwrap_or(0) as usize;
let mut dbi_names = vec![String::new(); max_dbi + 1];
dbi_names[0] = "FREE_DBI".to_string();
if dbi_names.len() > 1 {
dbi_names[1] = "MAIN_DBI".to_string();
}
for (name, &dbi) in self.dbis.iter() {
dbi_names[dbi as usize] = name.to_string();
}
let mut name_to_dbi = std::collections::HashMap::new();
for (&name, &dbi) in self.dbis.iter() {
name_to_dbi.insert(name, dbi as u8);
}
let mdbx_dat = db_path.join("mdbx.dat");
let walk = match reth_mdbx_viz::walker::walk_mdbx(&mdbx_dat, &name_to_dbi) {
Ok(w) => w,
Err(e) => {
reth_tracing::tracing::error!("pageviz: failed to walk B-tree: {e}");
return;
}
};
let page_count = walk.page_count;
let page_size = walk.page_size;
let mut drainer = PageVizDrainer::new();
drainer.enable();
let rx = drainer.start(60);
let config = reth_mdbx_viz::VizConfig {
port,
page_count: page_count as u64,
page_size: page_size as u32,
dbi_names,
owner_map: walk.owner_map,
tree_info: walk.tree_info,
db_path: mdbx_dat,
};
// Spawn the viz server on a background tokio runtime
std::thread::Builder::new()
.name("mdbx-viz-server".into())
.spawn(move || {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("failed to build tokio runtime for pageviz");
rt.block_on(async {
if let Err(e) = reth_mdbx_viz::start_viz_server(config, rx).await {
reth_tracing::tracing::error!("pageviz server error: {e}");
}
});
// Keep drainer alive while server runs
drop(drainer);
})
.expect("failed to spawn pageviz server thread");
reth_tracing::tracing::info!("pageviz server started on port {port}");
}
/// Creates all the tables defined in [`Tables`], if necessary.
///
/// This keeps tracks of the created table handles and stores them for better efficiency.

View File

@@ -45,22 +45,11 @@ pub fn init_db_for<P: AsRef<Path>, TS: TableSet>(
path: P,
args: DatabaseArguments,
) -> eyre::Result<DatabaseEnv> {
let db_path = path.as_ref().to_path_buf();
let client_version = args.client_version().clone();
let mut db = create_db(path, args)?;
db.create_and_track_tables_for::<TS>()?;
db.record_client_version(client_version)?;
drop_orphan_tables(&db);
#[cfg(feature = "pageviz")]
{
let port = std::env::var("RETH_PAGEVIZ_PORT")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(3141u16);
db.start_pageviz(port, db_path);
}
Ok(db)
}

View File

@@ -28,7 +28,6 @@ dashmap = { workspace = true, features = ["inline"], optional = true }
default = []
return-borrowed = []
read-tx-timeouts = ["dep:dashmap"]
pageviz = ["reth-mdbx-sys/pageviz"]
[dev-dependencies]
criterion.workspace = true

View File

@@ -11,7 +11,3 @@ repository.workspace = true
[build-dependencies]
cc.workspace = true
bindgen = { workspace = true, features = ["runtime"] }
[features]
default = []
pageviz = []

View File

@@ -30,11 +30,6 @@ fn main() {
#[cfg(not(debug_assertions))]
cc.define("MDBX_DEBUG", "0").define("NDEBUG", None);
// Enable page visualization instrumentation
if std::env::var("CARGO_FEATURE_PAGEVIZ").is_ok() {
cc.define("MDBX_PAGEVIZ", "1");
}
// Propagate `-C target-cpu=native`
let rustflags = env::var("CARGO_ENCODED_RUSTFLAGS").unwrap();
if rustflags.contains("target-cpu=native") &&
@@ -43,11 +38,7 @@ fn main() {
cc.flag("-march=native");
}
cc.file(mdbx.join("mdbx.c"));
if std::env::var("CARGO_FEATURE_PAGEVIZ").is_ok() {
cc.file(mdbx.join("mdbx_pageviz.c"));
}
cc.compile("libmdbx.a");
cc.file(mdbx.join("mdbx.c")).compile("libmdbx.a");
}
fn generate_bindings(mdbx: &Path, out_file: &Path) {

View File

@@ -13,7 +13,6 @@
#include MDBX_CONFIG_H
#endif
/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */
#if (defined(MDBX_DEBUG) && MDBX_DEBUG > 0) || (defined(MDBX_FORCE_ASSERTIONS) && MDBX_FORCE_ASSERTIONS)
#undef NDEBUG
@@ -47,7 +46,6 @@
#define _DARWIN_C_SOURCE
#endif /* _DARWIN_C_SOURCE */
#include "mdbx_pageviz.h"
#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && !defined(__USE_MINGW_ANSI_STDIO)
#define __USE_MINGW_ANSI_STDIO 1
#endif /* MinGW */
@@ -6195,7 +6193,6 @@ MDBX_INTERNAL int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const p
static inline int page_touch(MDBX_cursor *mc) {
page_t *const mp = mc->pg[mc->top];
MDBX_txn *txn = mc->txn;
MDBX_PAGEVIZ_WRITE(mc, mp->pgno);
tASSERT(txn, mc->txn->flags & MDBX_TXN_DIRTY);
tASSERT(txn, F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY));
@@ -19929,9 +19926,6 @@ __cold int dxb_resize(MDBX_env *const env, const pgno_t used_pgno, const pgno_t
eASSERT(env, env->dxb_mmap.limit >= env->dxb_mmap.current);
if (rc == MDBX_SUCCESS) {
#if defined(MDBX_PAGEVIZ) && MDBX_PAGEVIZ
mdbx_pageviz_set_mapping(env->dxb_mmap.base, env->dxb_mmap.current, env->ps);
#endif
eASSERT(env, limit_bytes == env->dxb_mmap.limit);
eASSERT(env, size_bytes <= env->dxb_mmap.filesize);
if (mode == explicit_resize)
@@ -20345,10 +20339,6 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, const mdbx_mode_t mode_bit
if (unlikely(err != MDBX_SUCCESS))
return err;
#if defined(MDBX_PAGEVIZ) && MDBX_PAGEVIZ
mdbx_pageviz_set_mapping(env->dxb_mmap.base, env->dxb_mmap.current, env->ps);
#endif
#if defined(MADV_DONTDUMP)
err =
madvise(env->dxb_mmap.base, env->dxb_mmap.limit, MADV_DONTDUMP) ? ignore_enosys_and_eagain(errno) : MDBX_SUCCESS;
@@ -23100,7 +23090,6 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
return err;
for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) {
pnl_append_prereserved(txn->tw.retired_pages, lp->pgno);
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_FREE, 0, lp->pgno);
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
}
@@ -23115,7 +23104,6 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) {
tASSERT(txn, lp->flags == P_LOOSE);
loose[++count] = lp->pgno;
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_FREE, 0, lp->pgno);
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
}
@@ -31638,7 +31626,6 @@ static __always_inline pgr_t page_get_inline(const uint16_t ILL, const MDBX_curs
if (unlikely(r.err != MDBX_SUCCESS))
goto bailout;
#endif /* MDBX_DISABLE_VALIDATION */
MDBX_PAGEVIZ_READ(mc, pgno);
return r;
}
@@ -31855,7 +31842,6 @@ pgr_t page_new(MDBX_cursor *mc, const unsigned flags) {
return ret;
DEBUG("db %zu allocated new page %" PRIaPGNO, cursor_dbi(mc), ret.page->pgno);
MDBX_PAGEVIZ_WRITE(mc, ret.page->pgno);
ret.page->flags = (uint16_t)flags;
cASSERT(mc, *cursor_dbi_state(mc) & DBI_DIRTY);
cASSERT(mc, mc->txn->flags & MDBX_TXN_DIRTY);
@@ -31884,7 +31870,6 @@ pgr_t page_new_large(MDBX_cursor *mc, const size_t npages) {
return ret;
DEBUG("dbi %zu allocated new large-page %" PRIaPGNO ", num %zu", cursor_dbi(mc), ret.page->pgno, npages);
MDBX_PAGEVIZ_WRITE(mc, ret.page->pgno);
ret.page->flags = P_LARGE;
cASSERT(mc, *cursor_dbi_state(mc) & DBI_DIRTY);
cASSERT(mc, mc->txn->flags & MDBX_TXN_DIRTY);
@@ -32030,7 +32015,6 @@ __hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *c
DEBUG("touched db %d page %" PRIaPGNO " -> %" PRIaPGNO, cursor_dbi_dbg(mc), mp->pgno, pgno);
tASSERT(txn, mp->pgno != pgno);
pnl_append_prereserved(txn->tw.retired_pages, mp->pgno);
MDBX_PAGEVIZ_FREE(mc, mp->pgno);
/* Update the parent page, if any, to point to the new page */
if (likely(mc->top)) {
page_t *parent = mc->pg[mc->top - 1];
@@ -32227,7 +32211,6 @@ int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, page_t *mp /* maybe null
unsigned pageflags /* maybe unknown/zero */) {
int rc;
MDBX_txn *const txn = mc->txn;
MDBX_PAGEVIZ_FREE(mc, pgno);
tASSERT(txn, !mp || (mp->pgno == pgno && mp->flags == pageflags));
/* During deleting entire subtrees, it is reasonable and possible to avoid

View File

@@ -1,119 +0,0 @@
#include "mdbx_pageviz.h"
#if defined(MDBX_PAGEVIZ) && MDBX_PAGEVIZ
/* ── Global state ─────────────────────────────────────────────────────── */
mdbx_pageviz_state_t *mdbx_pageviz_global = NULL;
/* ── Thread-local ring index ──────────────────────────────────────────── */
MDBX_PAGEVIZ_TLS uint32_t mdbx_pageviz_tls_ring = MDBX_PAGEVIZ_TLS_UNREGISTERED;
/* ── Lifecycle ────────────────────────────────────────────────────────── */
mdbx_pageviz_state_t *mdbx_pageviz_create(void) {
mdbx_pageviz_state_t *state = calloc(1, sizeof(mdbx_pageviz_state_t));
if (state)
mdbx_pageviz_global = state;
return state;
}
void mdbx_pageviz_destroy(mdbx_pageviz_state_t *state) {
if (!state)
return;
if (mdbx_pageviz_global == state)
mdbx_pageviz_global = NULL;
free(state);
}
/* ── Enable / Disable ─────────────────────────────────────────────────── */
void mdbx_pageviz_enable(mdbx_pageviz_state_t *state) {
atomic_store_explicit(&state->enabled, 1, memory_order_release);
}
void mdbx_pageviz_disable(mdbx_pageviz_state_t *state) {
atomic_store_explicit(&state->enabled, 0, memory_order_release);
}
/* ── Drain ────────────────────────────────────────────────────────────── */
uint32_t mdbx_pageviz_drain(mdbx_pageviz_state_t *state, uint32_t ring_idx,
uint64_t *out_buf, uint32_t max_count) {
if (!state || ring_idx >= MDBX_PAGEVIZ_MAX_RINGS || !out_buf || max_count == 0)
return 0;
mdbx_pageviz_ring_t *ring = &state->rings[ring_idx];
uint32_t head = atomic_load_explicit(&ring->published_head,
memory_order_acquire);
uint32_t tail = atomic_load_explicit(&ring->consumer_tail,
memory_order_relaxed);
if (head == tail)
return 0;
uint32_t avail = head - tail;
uint32_t count = avail < max_count ? avail : max_count;
for (uint32_t i = 0; i < count; i++) {
uint32_t slot = (tail + i) & MDBX_PAGEVIZ_RING_MASK;
out_buf[i] = ring->events[slot];
}
atomic_store_explicit(&ring->consumer_tail, tail + count,
memory_order_release);
return count;
}
/* ── Queries ──────────────────────────────────────────────────────────── */
uint32_t mdbx_pageviz_ring_count(mdbx_pageviz_state_t *state) {
return atomic_load_explicit(&state->ring_count, memory_order_relaxed);
}
uint64_t mdbx_pageviz_dropped(mdbx_pageviz_state_t *state, uint32_t ring_idx) {
return atomic_load_explicit(&state->rings[ring_idx].dropped,
memory_order_relaxed);
}
/* ── Mapping info ────────────────────────────────────────────────────── */
void mdbx_pageviz_set_mapping(void *base, size_t len, uint32_t mdbx_page_size) {
mdbx_pageviz_state_t *state = mdbx_pageviz_global;
if (!state)
return;
state->mapping.mdbx_page_size = mdbx_page_size;
state->mapping.sys_page_size = (uint32_t)sysconf(_SC_PAGESIZE);
state->mapping.len = len;
/* base written last so consumers see consistent len+page_size first */
__atomic_store_n((void *volatile *)&state->mapping.base, base, __ATOMIC_RELEASE);
}
int mdbx_pageviz_get_mapping(void **out_base, size_t *out_len,
uint32_t *out_mdbx_ps, uint32_t *out_sys_ps) {
mdbx_pageviz_state_t *state = mdbx_pageviz_global;
if (!state)
return 0;
void *b = __atomic_load_n((void *volatile *)&state->mapping.base, __ATOMIC_ACQUIRE);
if (!b)
return 0;
*out_base = b;
*out_len = state->mapping.len;
*out_mdbx_ps = state->mapping.mdbx_page_size;
*out_sys_ps = state->mapping.sys_page_size;
return 1;
}
/* ── Block marker emit (non-inline wrapper for Rust FFI) ──────────────── */
void mdbx_pageviz_emit_block_marker(uint8_t op, uint32_t block_number,
uint16_t tx_count, uint8_t duration_encoded,
uint8_t gas_encoded) {
mdbx_pageviz_emit(op, ((uint32_t)gas_encoded << 24) | ((uint32_t)duration_encoded << 16) | (uint32_t)tx_count, block_number);
}
#else /* !MDBX_PAGEVIZ */
typedef int mdbx_pageviz_empty_tu_;
#endif /* MDBX_PAGEVIZ */

View File

@@ -1,195 +0,0 @@
/* mdbx_pageviz.h — Standalone page-access visualization ring buffer for MDBX.
*
* Self-contained: no dependencies on MDBX internals.
* Gate everything behind MDBX_PAGEVIZ; when not defined, all macros expand to
* nothing (zero overhead).
*
* Compatible with C11 and C17. */
#ifndef MDBX_PAGEVIZ_H
#define MDBX_PAGEVIZ_H
/* ── Disabled stub macros ─────────────────────────────────────────────── */
#if !defined(MDBX_PAGEVIZ) || !(MDBX_PAGEVIZ)
#define MDBX_PAGEVIZ_READ(mc, pgno) ((void)0)
#define MDBX_PAGEVIZ_WRITE(mc, pgno) ((void)0)
#define MDBX_PAGEVIZ_FREE(mc, pgno) ((void)0)
#else /* MDBX_PAGEVIZ enabled ───────────────────────────────────────────── */
#include <stdatomic.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
/* ── Tunables ─────────────────────────────────────────────────────────── */
#define MDBX_PAGEVIZ_RING_CAPACITY 65536u /* must be power-of-2 */
#define MDBX_PAGEVIZ_MAX_RINGS 128u
#define MDBX_PAGEVIZ_PUBLISH_INTERVAL 32u
/* ── Event encoding ───────────────────────────────────────────────────── */
/* Layout of a 64-bit event word:
* bits 63..56 op (1=READ, 2=WRITE, 3=FREE)
* bits 55..32 dbi (lower 24 bits of a uint32_t)
* bits 31..0 pgno (uint32_t page number) */
#define MDBX_PAGEVIZ_OP_READ 1
#define MDBX_PAGEVIZ_OP_WRITE 2
#define MDBX_PAGEVIZ_OP_FREE 3
#define MDBX_PAGEVIZ_OP_BLOCK_START 4
#define MDBX_PAGEVIZ_OP_BLOCK_END 5
#define MDBX_PAGEVIZ_ENCODE(op, dbi, pgno) \
(((uint64_t)(op) << 56) | ((uint64_t)((dbi) & 0x00FFFFFFu) << 32) | \
(uint64_t)(uint32_t)(pgno))
#define MDBX_PAGEVIZ_DECODE_OP(ev) ((uint8_t)((ev) >> 56))
#define MDBX_PAGEVIZ_DECODE_DBI(ev) ((uint32_t)(((ev) >> 32) & 0x00FFFFFFu))
#define MDBX_PAGEVIZ_DECODE_PGNO(ev) ((uint32_t)(ev))
/* ── Ring-buffer mask (power-of-2 capacity) ───────────────────────────── */
#define MDBX_PAGEVIZ_RING_MASK (MDBX_PAGEVIZ_RING_CAPACITY - 1u)
/* ── Platform TLS ─────────────────────────────────────────────────────── */
#if defined(_MSC_VER)
# define MDBX_PAGEVIZ_TLS __declspec(thread)
#elif defined(__GNUC__) || defined(__clang__)
# define MDBX_PAGEVIZ_TLS __thread
#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
# define MDBX_PAGEVIZ_TLS _Thread_local
#else
# error "No thread-local storage support detected"
#endif
/* ── Structures ───────────────────────────────────────────────────────── */
typedef struct mdbx_pageviz_ring {
uint64_t events[MDBX_PAGEVIZ_RING_CAPACITY];
_Atomic uint32_t published_head; /* consumer reads (acquire) */
uint32_t local_head; /* producer-only, no atomic needed */
_Atomic uint32_t consumer_tail; /* consumer updates after draining */
_Atomic uint64_t dropped; /* unused, kept for ABI compat */
uint32_t _pad[10]; /* pad to separate cache lines */
} mdbx_pageviz_ring_t;
typedef struct mdbx_pageviz_mapping {
volatile void *base;
volatile size_t len;
volatile uint32_t mdbx_page_size;
volatile uint32_t sys_page_size;
} mdbx_pageviz_mapping_t;
typedef struct mdbx_pageviz_state {
mdbx_pageviz_ring_t rings[MDBX_PAGEVIZ_MAX_RINGS];
_Atomic uint32_t ring_count; /* how many rings registered */
_Atomic uint32_t enabled; /* runtime enable/disable */
mdbx_pageviz_mapping_t mapping;
} mdbx_pageviz_state_t;
/* ── Global state (defined in .c) ─────────────────────────────────────── */
extern mdbx_pageviz_state_t *mdbx_pageviz_global;
/* ── Thread-local ring index (defined in .c) ──────────────────────────── */
#define MDBX_PAGEVIZ_TLS_UNREGISTERED UINT32_MAX
extern MDBX_PAGEVIZ_TLS uint32_t mdbx_pageviz_tls_ring;
/* ── Inline helpers (private) ─────────────────────────────────────────── */
static inline uint32_t mdbx_pageviz_register_ring_(mdbx_pageviz_state_t *s) {
uint32_t idx =
atomic_fetch_add_explicit(&s->ring_count, 1, memory_order_relaxed);
if (idx >= MDBX_PAGEVIZ_MAX_RINGS) {
atomic_fetch_sub_explicit(&s->ring_count, 1, memory_order_relaxed);
return MDBX_PAGEVIZ_TLS_UNREGISTERED;
}
mdbx_pageviz_ring_t *r = &s->rings[idx];
atomic_store_explicit(&r->published_head, 0, memory_order_relaxed);
r->local_head = 0;
atomic_store_explicit(&r->consumer_tail, 0, memory_order_relaxed);
atomic_store_explicit(&r->dropped, 0, memory_order_relaxed);
return idx;
}
/* ── Hot-path emit (inlined) ──────────────────────────────────────────── */
static inline void mdbx_pageviz_emit(uint8_t op, uint32_t dbi,
uint32_t pgno) {
mdbx_pageviz_state_t *state = mdbx_pageviz_global;
if (__builtin_expect(
state == NULL ||
!atomic_load_explicit(&state->enabled, memory_order_relaxed),
1))
return;
uint32_t idx = mdbx_pageviz_tls_ring;
if (__builtin_expect(idx == MDBX_PAGEVIZ_TLS_UNREGISTERED, 0)) {
idx = mdbx_pageviz_register_ring_(state);
if (__builtin_expect(idx == MDBX_PAGEVIZ_TLS_UNREGISTERED, 0))
return;
mdbx_pageviz_tls_ring = idx;
}
mdbx_pageviz_ring_t *r = &state->rings[idx];
/* Block (spin) until the consumer drains enough space. */
while (__builtin_expect(
(uint32_t)(r->local_head -
atomic_load_explicit(&r->consumer_tail, memory_order_acquire))
>= MDBX_PAGEVIZ_RING_CAPACITY, 0)) {
/* Flush our writes so the consumer can make progress. */
atomic_store_explicit(&r->published_head, r->local_head,
memory_order_release);
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)
__builtin_ia32_pause();
#elif defined(__aarch64__) || defined(_M_ARM64)
__asm__ volatile("yield");
#endif
}
r->events[r->local_head & MDBX_PAGEVIZ_RING_MASK] =
MDBX_PAGEVIZ_ENCODE(op, dbi, pgno);
r->local_head++;
if ((r->local_head & (MDBX_PAGEVIZ_PUBLISH_INTERVAL - 1u)) == 0)
atomic_store_explicit(&r->published_head, r->local_head,
memory_order_release);
}
/* ── Macro hooks (inserted into mdbx.c) ──────────────────────────────── */
#define MDBX_PAGEVIZ_READ(mc, pgno) \
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_READ, (uint32_t)cursor_dbi(mc), (pgno))
#define MDBX_PAGEVIZ_WRITE(mc, pgno) \
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_WRITE, (uint32_t)cursor_dbi(mc), (pgno))
#define MDBX_PAGEVIZ_FREE(mc, pgno) \
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_FREE, (uint32_t)cursor_dbi(mc), (pgno))
/* ── Public API (defined in .c) ───────────────────────────────────────── */
mdbx_pageviz_state_t *mdbx_pageviz_create(void);
void mdbx_pageviz_destroy(mdbx_pageviz_state_t *state);
void mdbx_pageviz_enable(mdbx_pageviz_state_t *state);
void mdbx_pageviz_disable(mdbx_pageviz_state_t *state);
uint32_t mdbx_pageviz_drain(mdbx_pageviz_state_t *state, uint32_t ring_idx,
uint64_t *out_buf, uint32_t max_count);
uint32_t mdbx_pageviz_ring_count(mdbx_pageviz_state_t *state);
uint64_t mdbx_pageviz_dropped(mdbx_pageviz_state_t *state, uint32_t ring_idx);
void mdbx_pageviz_set_mapping(void *base, size_t len, uint32_t mdbx_page_size);
int mdbx_pageviz_get_mapping(void **out_base, size_t *out_len,
uint32_t *out_mdbx_ps, uint32_t *out_sys_ps);
void mdbx_pageviz_emit_block_marker(uint8_t op, uint32_t block_number,
uint16_t tx_count, uint8_t duration_encoded,
uint8_t gas_encoded);
#endif /* MDBX_PAGEVIZ */
#endif /* MDBX_PAGEVIZ_H */

View File

@@ -36,9 +36,6 @@ mod flags;
mod transaction;
mod txn_manager;
#[cfg(feature = "pageviz")]
pub mod pageviz;
#[cfg(test)]
mod test_utils {
use super::*;

View File

@@ -1,226 +0,0 @@
use std::{
collections::HashMap,
sync::{
atomic::{AtomicBool, Ordering},
mpsc, Arc,
},
time::{Duration, Instant},
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub enum PageOp {
Read = 1,
Write = 2,
Free = 3,
BlockStart = 4,
BlockEnd = 5,
}
impl PageOp {
fn from_u8(v: u8) -> Option<Self> {
match v {
1 => Some(Self::Read),
2 => Some(Self::Write),
3 => Some(Self::Free),
4 => Some(Self::BlockStart),
5 => Some(Self::BlockEnd),
_ => None,
}
}
pub fn is_block_marker(self) -> bool {
matches!(self, Self::BlockStart | Self::BlockEnd)
}
}
#[derive(Debug, Clone, Copy)]
pub struct PageEvent {
pub pgno: u32,
pub dbi: u32,
pub op: PageOp,
}
impl PageEvent {
fn decode(raw: u64) -> Option<Self> {
let op_byte = (raw >> 56) as u8;
let op = PageOp::from_u8(op_byte)?;
let dbi = ((raw >> 32) & 0x00FF_FFFF) as u32;
let pgno = raw as u32;
Some(Self { pgno, dbi, op })
}
}
unsafe extern "C" {
fn mdbx_pageviz_create() -> *mut std::ffi::c_void;
fn mdbx_pageviz_destroy(state: *mut std::ffi::c_void);
fn mdbx_pageviz_enable(state: *mut std::ffi::c_void);
fn mdbx_pageviz_disable(state: *mut std::ffi::c_void);
fn mdbx_pageviz_drain(
state: *mut std::ffi::c_void,
ring_idx: u32,
out_buf: *mut u64,
max_count: u32,
) -> u32;
fn mdbx_pageviz_ring_count(state: *mut std::ffi::c_void) -> u32;
fn mdbx_pageviz_dropped(state: *mut std::ffi::c_void, ring_idx: u32) -> u64;
}
#[derive(Debug)]
pub struct PageVizStats {
pub ring_count: u32,
pub total_dropped: u64,
}
#[derive(Debug, Clone, Copy)]
struct StatePtr(*mut std::ffi::c_void);
// SAFETY: The C state is thread-safe (uses atomics internally).
unsafe impl Send for StatePtr {}
unsafe impl Sync for StatePtr {}
#[derive(Debug)]
pub struct PageVizDrainer {
state: StatePtr,
running: Arc<AtomicBool>,
handle: Option<std::thread::JoinHandle<()>>,
}
const DRAIN_BUF_LEN: usize = 4096;
const OVERLOAD_THRESHOLD: usize = 200_000;
const READ_SAMPLE_RATE: usize = 8;
impl PageVizDrainer {
pub fn new() -> Self {
let state = StatePtr(unsafe { mdbx_pageviz_create() });
Self { state, running: Arc::new(AtomicBool::new(false)), handle: None }
}
pub fn enable(&self) {
unsafe { mdbx_pageviz_enable(self.state.0) }
}
pub fn disable(&self) {
unsafe { mdbx_pageviz_disable(self.state.0) }
}
pub fn start(&mut self, tick_hz: u32) -> mpsc::Receiver<Vec<PageEvent>> {
let (tx, rx) = mpsc::channel();
let running = self.running.clone();
running.store(true, Ordering::SeqCst);
let state_addr = self.state.0 as usize;
let tick_interval = Duration::from_micros(1_000_000 / u64::from(tick_hz));
let handle = std::thread::Builder::new()
.name("mdbx-pageviz".into())
.spawn(move || {
let state = state_addr as *mut std::ffi::c_void;
let mut buf = [0u64; DRAIN_BUF_LEN];
let mut coalesced: HashMap<u32, PageEvent> = HashMap::new();
let mut markers: Vec<PageEvent> = Vec::new();
let mut read_counter: usize = 0;
while running.load(Ordering::Relaxed) {
let tick_start = Instant::now();
let ring_count = unsafe { mdbx_pageviz_ring_count(state) };
for ring_idx in 0..ring_count {
let dropped = unsafe { mdbx_pageviz_dropped(state, ring_idx) };
if dropped > 0 {
tracing::warn!(
target: "libmdbx::pageviz",
ring_idx,
dropped,
"pageviz ring dropped events"
);
}
loop {
let count = unsafe {
mdbx_pageviz_drain(
state,
ring_idx,
buf.as_mut_ptr(),
DRAIN_BUF_LEN as u32,
)
};
if count == 0 {
break;
}
for &raw in &buf[..count as usize] {
if let Some(evt) = PageEvent::decode(raw) {
if evt.op.is_block_marker() {
markers.push(evt);
} else {
coalesced.insert(evt.pgno, evt);
}
}
}
}
}
if !coalesced.is_empty() || !markers.is_empty() {
let mut events: Vec<PageEvent> = if coalesced.len() > OVERLOAD_THRESHOLD {
coalesced
.drain()
.filter(|(_, evt)| {
if evt.op == PageOp::Read {
read_counter += 1;
read_counter % READ_SAMPLE_RATE == 0
} else {
true
}
})
.map(|(_, evt)| evt)
.collect()
} else {
coalesced.drain().map(|(_, evt)| evt).collect()
};
events.extend(markers.drain(..));
if tx.send(events).is_err() {
break;
}
}
let elapsed = tick_start.elapsed();
if let Some(remaining) = tick_interval.checked_sub(elapsed) {
std::thread::sleep(remaining);
}
}
})
.expect("failed to spawn mdbx-pageviz thread");
self.handle = Some(handle);
rx
}
pub fn stop(&mut self) {
self.running.store(false, Ordering::SeqCst);
if let Some(handle) = self.handle.take() {
let _ = handle.join();
}
}
pub fn stats(&self) -> PageVizStats {
let ring_count = unsafe { mdbx_pageviz_ring_count(self.state.0) };
let mut total_dropped = 0u64;
for ring_idx in 0..ring_count {
total_dropped += unsafe { mdbx_pageviz_dropped(self.state.0, ring_idx) };
}
PageVizStats { ring_count, total_dropped }
}
}
impl Drop for PageVizDrainer {
fn drop(&mut self) {
self.stop();
unsafe {
mdbx_pageviz_disable(self.state.0);
mdbx_pageviz_destroy(self.state.0);
}
}
}

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More