mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
33 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0f09a92acc | ||
|
|
df8ba50cb7 | ||
|
|
61341a1342 | ||
|
|
3d02124e50 | ||
|
|
953a1b3399 | ||
|
|
a364989c61 | ||
|
|
f4075f5926 | ||
|
|
211d3d2924 | ||
|
|
dbc5313a2c | ||
|
|
345fc9cfd2 | ||
|
|
33d61c30cb | ||
|
|
beb0c5e527 | ||
|
|
73c39279b1 | ||
|
|
2fd0a703e2 | ||
|
|
7dcd77de95 | ||
|
|
da6e6afe78 | ||
|
|
04d9a33c68 | ||
|
|
be1f657b3c | ||
|
|
cab4cbf0ea | ||
|
|
29f51abb22 | ||
|
|
dd0ee0709c | ||
|
|
dcd9e50663 | ||
|
|
59d11378b9 | ||
|
|
70ceb755ad | ||
|
|
a49991c766 | ||
|
|
0a77c2aae1 | ||
|
|
93adbf82a0 | ||
|
|
ce08b6f265 | ||
|
|
b03a704a1e | ||
|
|
a51e03fce6 | ||
|
|
57d7c98f66 | ||
|
|
5d9a43f2d4 | ||
|
|
defd0e8e5c |
@@ -1,4 +0,0 @@
|
||||
---
|
||||
---
|
||||
|
||||
Improved nightly Docker build failure Slack notification with more detailed formatting and context.
|
||||
@@ -1,4 +0,0 @@
|
||||
---
|
||||
---
|
||||
|
||||
Improved documentation overview page with better structure and clarity.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
reth-node-events: patch
|
||||
---
|
||||
|
||||
Updated consensus engine log message to be more accurate about received updates.
|
||||
@@ -1,9 +0,0 @@
|
||||
---
|
||||
reth-network-api: minor
|
||||
reth-network-types: minor
|
||||
reth-network: minor
|
||||
reth-node-core: minor
|
||||
reth: minor
|
||||
---
|
||||
|
||||
Added optional ENR fork ID enforcement to filter out peers from incompatible networks during peer discovery, controlled by the `--enforce-enr-fork-id` CLI flag.
|
||||
2
.github/CODEOWNERS
vendored
2
.github/CODEOWNERS
vendored
@@ -38,7 +38,7 @@ crates/storage/libmdbx-rs/ @shekhirin
|
||||
crates/storage/nippy-jar/ @joshieDo @shekhirin
|
||||
crates/storage/provider/ @joshieDo @shekhirin @yongkangc
|
||||
crates/storage/storage-api/ @joshieDo
|
||||
crates/tasks/ @mattsse @DaniPopes
|
||||
crates/tasks/ @mattsse
|
||||
crates/tokio-util/ @mattsse
|
||||
crates/tracing/ @mattsse @shekhirin
|
||||
crates/tracing-otlp/ @mattsse @Rjected
|
||||
|
||||
27
.github/workflows/docker.yml
vendored
27
.github/workflows/docker.yml
vendored
@@ -70,27 +70,18 @@ jobs:
|
||||
# Add 'latest' tag for non-RC releases
|
||||
if [[ ! "$VERSION" =~ -rc ]]; then
|
||||
echo "ethereum_tags=${REGISTRY}/reth:${VERSION},${REGISTRY}/reth:latest" >> "$GITHUB_OUTPUT"
|
||||
{
|
||||
echo "ethereum_set<<EOF"
|
||||
echo "ethereum.tags=${REGISTRY}/reth:${VERSION}"
|
||||
echo "ethereum.tags=${REGISTRY}/reth:latest"
|
||||
echo "EOF"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "ethereum_tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
elif [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ inputs.build_type }}" == "nightly" ]]; then
|
||||
echo "targets=nightly" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
|
||||
|
||||
else
|
||||
# git-sha build
|
||||
echo "targets=ethereum" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build and push images
|
||||
@@ -106,7 +97,7 @@ jobs:
|
||||
targets: ${{ steps.params.outputs.targets }}
|
||||
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
|
||||
set: |
|
||||
${{ steps.params.outputs.ethereum_set }}
|
||||
ethereum.tags=${{ steps.params.outputs.ethereum_tags }}
|
||||
|
||||
- name: Verify image architectures
|
||||
env:
|
||||
@@ -126,18 +117,6 @@ jobs:
|
||||
- name: Slack Webhook Action
|
||||
uses: rtCamp/action-slack-notify@v2
|
||||
env:
|
||||
SLACK_COLOR: danger
|
||||
SLACK_ICON_EMOJI: ":rotating_light:"
|
||||
SLACK_USERNAME: "GitHub Actions"
|
||||
SLACK_TITLE: ":rotating_light: Nightly Docker Build Failed"
|
||||
SLACK_MESSAGE: |
|
||||
The scheduled nightly Docker build failed.
|
||||
|
||||
*Commit:* `${{ github.sha }}`
|
||||
*Branch:* `${{ github.ref_name }}`
|
||||
*Run:* <https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}|View logs>
|
||||
|
||||
*Action required:* Re-run the workflow or investigate the build failure.
|
||||
SLACK_FOOTER: "paradigmxyz/reth · docker.yml"
|
||||
MSG_MINIMAL: true
|
||||
SLACK_COLOR: ${{ job.status }}
|
||||
SLACK_MESSAGE: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}"
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
|
||||
615
Cargo.lock
generated
615
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
26
Cargo.toml
26
Cargo.toml
@@ -1,5 +1,5 @@
|
||||
[workspace.package]
|
||||
version = "1.11.0"
|
||||
version = "1.10.2"
|
||||
edition = "2024"
|
||||
rust-version = "1.88"
|
||||
license = "MIT OR Apache-2.0"
|
||||
@@ -110,7 +110,6 @@ members = [
|
||||
"crates/storage/errors/",
|
||||
"crates/storage/libmdbx-rs/",
|
||||
"crates/storage/libmdbx-rs/mdbx-sys/",
|
||||
"crates/storage/mdbx-viz/",
|
||||
"crates/storage/nippy-jar/",
|
||||
"crates/storage/provider/",
|
||||
"crates/storage/storage-api/",
|
||||
@@ -378,7 +377,6 @@ reth-invalid-block-hooks = { path = "crates/engine/invalid-block-hooks" }
|
||||
reth-ipc = { path = "crates/rpc/ipc" }
|
||||
reth-libmdbx = { path = "crates/storage/libmdbx-rs" }
|
||||
reth-mdbx-sys = { path = "crates/storage/libmdbx-rs/mdbx-sys" }
|
||||
reth-mdbx-viz = { path = "crates/storage/mdbx-viz" }
|
||||
reth-metrics = { path = "crates/metrics" }
|
||||
reth-net-banlist = { path = "crates/net/banlist" }
|
||||
reth-net-nat = { path = "crates/net/nat" }
|
||||
@@ -713,6 +711,28 @@ vergen-git2 = "9.1.0"
|
||||
ipnet = "2.11"
|
||||
|
||||
[patch.crates-io]
|
||||
# revm staging patches
|
||||
revm = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
|
||||
op-revm = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
|
||||
revm-bytecode = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
|
||||
revm-context = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
|
||||
revm-context-interface = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
|
||||
revm-database = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
|
||||
revm-database-interface = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
|
||||
revm-handler = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
|
||||
revm-inspector = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
|
||||
revm-interpreter = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
|
||||
revm-precompile = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
|
||||
revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
|
||||
revm-state = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
|
||||
|
||||
# revm-inspectors staging patch
|
||||
revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "e80e2eab72dfa18011e6a99abd37027290a46e83" }
|
||||
|
||||
# alloy-evm staging patches
|
||||
alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "742dc14749ea0279c03ca27b1c26f26ac19fbefb" }
|
||||
alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "742dc14749ea0279c03ca27b1c26f26ac19fbefb" }
|
||||
|
||||
# alloy-consensus = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-contract = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-eips = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
|
||||
@@ -192,15 +192,6 @@ impl Command {
|
||||
parent_header = block.header;
|
||||
parent_hash = block_hash;
|
||||
blocks_processed += 1;
|
||||
|
||||
let progress = match mode {
|
||||
RampMode::Blocks(total) => format!("{blocks_processed}/{total}"),
|
||||
RampMode::TargetGasLimit(target) => {
|
||||
let pct = (parent_header.gas_limit as f64 / target as f64 * 100.0).min(100.0);
|
||||
format!("{pct:.1}%")
|
||||
}
|
||||
};
|
||||
info!(target: "reth-bench", progress, block_number = parent_header.number, gas_limit = parent_header.gas_limit, "Block processed");
|
||||
}
|
||||
|
||||
let final_gas_limit = parent_header.gas_limit;
|
||||
|
||||
@@ -153,7 +153,6 @@ impl Command {
|
||||
..
|
||||
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
|
||||
|
||||
let total_blocks = benchmark_mode.total_blocks();
|
||||
let buffer_size = self.rpc_block_buffer_size;
|
||||
|
||||
// Use a oneshot channel to propagate errors from the spawned task
|
||||
@@ -207,7 +206,6 @@ impl Command {
|
||||
});
|
||||
|
||||
let mut results = Vec::new();
|
||||
let mut blocks_processed = 0u64;
|
||||
let total_benchmark_duration = Instant::now();
|
||||
let mut total_wait_time = Duration::ZERO;
|
||||
|
||||
@@ -251,13 +249,8 @@ impl Command {
|
||||
|
||||
// Exclude time spent waiting on the block prefetch channel from the benchmark duration.
|
||||
// We want to measure engine throughput, not RPC fetch latency.
|
||||
blocks_processed += 1;
|
||||
let current_duration = total_benchmark_duration.elapsed() - total_wait_time;
|
||||
let progress = match total_blocks {
|
||||
Some(total) => format!("{blocks_processed}/{total}"),
|
||||
None => format!("{blocks_processed}"),
|
||||
};
|
||||
info!(target: "reth-bench", progress, %combined_result);
|
||||
info!(target: "reth-bench", %combined_result);
|
||||
|
||||
if let Some(w) = &mut waiter {
|
||||
w.on_block(block_number).await?;
|
||||
|
||||
@@ -52,7 +52,6 @@ impl Command {
|
||||
..
|
||||
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
|
||||
|
||||
let total_blocks = benchmark_mode.total_blocks();
|
||||
let buffer_size = self.rpc_block_buffer_size;
|
||||
|
||||
// Use a oneshot channel to propagate errors from the spawned task
|
||||
@@ -83,8 +82,8 @@ impl Command {
|
||||
}
|
||||
});
|
||||
|
||||
// put results in a summary vec so they can be printed at the end
|
||||
let mut results = Vec::new();
|
||||
let mut blocks_processed = 0u64;
|
||||
let total_benchmark_duration = Instant::now();
|
||||
let mut total_wait_time = Duration::ZERO;
|
||||
|
||||
@@ -106,12 +105,7 @@ impl Command {
|
||||
call_new_payload(&auth_provider, version, params).await?;
|
||||
|
||||
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
|
||||
blocks_processed += 1;
|
||||
let progress = match total_blocks {
|
||||
Some(total) => format!("{blocks_processed}/{total}"),
|
||||
None => format!("{blocks_processed}"),
|
||||
};
|
||||
info!(target: "reth-bench", progress, %new_payload_result);
|
||||
info!(target: "reth-bench", %new_payload_result);
|
||||
|
||||
// current duration since the start of the benchmark minus the time
|
||||
// waiting for blocks
|
||||
|
||||
@@ -341,8 +341,7 @@ impl Command {
|
||||
};
|
||||
|
||||
let current_duration = total_benchmark_duration.elapsed();
|
||||
let progress = format!("{}/{}", i + 1, payloads.len());
|
||||
info!(target: "reth-bench", progress, %combined_result);
|
||||
info!(target: "reth-bench", %combined_result);
|
||||
|
||||
if let Some(w) = &mut waiter {
|
||||
w.on_block(block_number).await?;
|
||||
|
||||
@@ -20,19 +20,6 @@ impl BenchMode {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the total number of blocks in the benchmark, if known.
|
||||
///
|
||||
/// For [`BenchMode::Range`] this is the length of the range.
|
||||
/// For [`BenchMode::Continuous`] the total is unbounded, so `None` is returned.
|
||||
pub const fn total_blocks(&self) -> Option<u64> {
|
||||
match self {
|
||||
Self::Continuous(_) => None,
|
||||
Self::Range(range) => {
|
||||
Some(range.end().saturating_sub(*range.start()).saturating_add(1))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a [`BenchMode`] from optional `from` and `to` fields.
|
||||
pub fn new(from: Option<u64>, to: Option<u64>, latest_block: u64) -> Result<Self, eyre::Error> {
|
||||
// If neither `--from` nor `--to` are provided, we will run the benchmark continuously,
|
||||
|
||||
@@ -190,7 +190,6 @@ min-trace-logs = [
|
||||
"reth-node-core/min-trace-logs",
|
||||
]
|
||||
|
||||
pageviz = ["reth-db/pageviz", "reth-node-builder/pageviz"]
|
||||
rocksdb = ["reth-ethereum-cli/rocksdb", "reth-node-core/rocksdb"]
|
||||
edge = ["rocksdb"]
|
||||
|
||||
|
||||
@@ -1061,14 +1061,6 @@ mod tests {
|
||||
) -> ProviderResult<Option<StorageValue>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn storage_by_hashed_key(
|
||||
&self,
|
||||
_address: Address,
|
||||
_hashed_storage_key: StorageKey,
|
||||
) -> ProviderResult<Option<StorageValue>> {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
impl BytecodeReader for MockStateProvider {
|
||||
|
||||
@@ -223,26 +223,6 @@ impl<N: NodePrimitives> StateProvider for MemoryOverlayStateProviderRef<'_, N> {
|
||||
|
||||
self.historical.storage(address, storage_key)
|
||||
}
|
||||
|
||||
fn storage_by_hashed_key(
|
||||
&self,
|
||||
address: Address,
|
||||
hashed_storage_key: StorageKey,
|
||||
) -> ProviderResult<Option<StorageValue>> {
|
||||
let hashed_address = keccak256(address);
|
||||
let state = &self.trie_input().state;
|
||||
|
||||
if let Some(hs) = state.storages.get(&hashed_address) {
|
||||
if let Some(value) = hs.storage.get(&hashed_storage_key) {
|
||||
return Ok(Some(*value));
|
||||
}
|
||||
if hs.wiped {
|
||||
return Ok(Some(StorageValue::ZERO));
|
||||
}
|
||||
}
|
||||
|
||||
self.historical.storage_by_hashed_key(address, hashed_storage_key)
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives> BytecodeReader for MemoryOverlayStateProviderRef<'_, N> {
|
||||
|
||||
@@ -87,8 +87,6 @@ tokio-stream.workspace = true
|
||||
reqwest.workspace = true
|
||||
url.workspace = true
|
||||
metrics.workspace = true
|
||||
memmap2.workspace = true
|
||||
rayon.workspace = true
|
||||
|
||||
# io
|
||||
fdlimit.workspace = true
|
||||
|
||||
@@ -19,7 +19,7 @@ use reth_node_builder::{
|
||||
Node, NodeComponents, NodeComponentsBuilder, NodeTypes, NodeTypesWithDBAdapter,
|
||||
};
|
||||
use reth_node_core::{
|
||||
args::{DatabaseArgs, DatadirArgs, StaticFilesArgs, StorageArgs},
|
||||
args::{DatabaseArgs, DatadirArgs, RocksDbArgs, StaticFilesArgs, StorageArgs},
|
||||
dirs::{ChainPath, DataDirPath},
|
||||
};
|
||||
use reth_provider::{
|
||||
@@ -67,23 +67,62 @@ pub struct EnvironmentArgs<C: ChainSpecParser> {
|
||||
#[command(flatten)]
|
||||
pub static_files: StaticFilesArgs,
|
||||
|
||||
/// All `RocksDB` related arguments
|
||||
#[command(flatten)]
|
||||
pub rocksdb: RocksDbArgs,
|
||||
|
||||
/// Storage mode configuration (v2 vs v1/legacy)
|
||||
#[command(flatten)]
|
||||
pub storage: StorageArgs,
|
||||
}
|
||||
|
||||
impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
/// Returns the effective storage settings derived from `--storage.v2`.
|
||||
/// Returns the effective storage settings derived from `--storage.v2`, static-file, and
|
||||
/// `RocksDB` CLI args.
|
||||
///
|
||||
/// The base storage mode is determined by `--storage.v2`:
|
||||
/// - When `--storage.v2` is set: uses [`StorageSettings::v2()`] defaults
|
||||
/// - Otherwise: uses [`StorageSettings::base()`] defaults
|
||||
/// - Otherwise: uses [`StorageSettings::v1()`] defaults
|
||||
///
|
||||
/// Individual `--static-files.*` and `--rocksdb.*` flags override the base when explicitly set.
|
||||
pub fn storage_settings(&self) -> StorageSettings {
|
||||
if self.storage.v2 {
|
||||
StorageSettings::v2()
|
||||
} else {
|
||||
StorageSettings::base()
|
||||
let mut s = if self.storage.v2 { StorageSettings::v2() } else { StorageSettings::base() };
|
||||
|
||||
// Apply static files overrides (only when explicitly set)
|
||||
if let Some(v) = self.static_files.receipts {
|
||||
s = s.with_receipts_in_static_files(v);
|
||||
}
|
||||
if let Some(v) = self.static_files.transaction_senders {
|
||||
s = s.with_transaction_senders_in_static_files(v);
|
||||
}
|
||||
if let Some(v) = self.static_files.account_changesets {
|
||||
s = s.with_account_changesets_in_static_files(v);
|
||||
}
|
||||
if let Some(v) = self.static_files.storage_changesets {
|
||||
s = s.with_storage_changesets_in_static_files(v);
|
||||
}
|
||||
|
||||
// Apply rocksdb overrides
|
||||
// --rocksdb.all sets all rocksdb flags to true
|
||||
if self.rocksdb.all {
|
||||
s = s
|
||||
.with_transaction_hash_numbers_in_rocksdb(true)
|
||||
.with_storages_history_in_rocksdb(true)
|
||||
.with_account_history_in_rocksdb(true);
|
||||
}
|
||||
|
||||
// Individual rocksdb flags override --rocksdb.all when explicitly set
|
||||
if let Some(v) = self.rocksdb.tx_hash {
|
||||
s = s.with_transaction_hash_numbers_in_rocksdb(v);
|
||||
}
|
||||
if let Some(v) = self.rocksdb.storages_history {
|
||||
s = s.with_storages_history_in_rocksdb(v);
|
||||
}
|
||||
if let Some(v) = self.rocksdb.account_history {
|
||||
s = s.with_account_history_in_rocksdb(v);
|
||||
}
|
||||
|
||||
s
|
||||
}
|
||||
|
||||
/// Initializes environment according to [`AccessRights`] and returns an instance of
|
||||
|
||||
@@ -5,7 +5,6 @@ use reth_codecs::Compact;
|
||||
use reth_db_api::{cursor::DbDupCursorRO, database::Database, tables, transaction::DbTx};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::NodeTypesWithDB;
|
||||
use reth_storage_api::StorageSettingsCache;
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::info;
|
||||
|
||||
@@ -23,94 +22,52 @@ impl Command {
|
||||
/// Execute `db account-storage` command
|
||||
pub fn execute<N: NodeTypesWithDB>(self, tool: &DbTool<N>) -> eyre::Result<()> {
|
||||
let address = self.address;
|
||||
let use_hashed_state = tool.provider_factory.cached_storage_settings().use_hashed_state();
|
||||
let (slot_count, plain_size) = tool.provider_factory.db_ref().view(|tx| {
|
||||
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
|
||||
let mut count = 0usize;
|
||||
let mut total_value_bytes = 0usize;
|
||||
let mut last_log = Instant::now();
|
||||
|
||||
let (slot_count, storage_size) = if use_hashed_state {
|
||||
let hashed_address = keccak256(address);
|
||||
tool.provider_factory.db_ref().view(|tx| {
|
||||
let mut cursor = tx.cursor_dup_read::<tables::HashedStorages>()?;
|
||||
let mut count = 0usize;
|
||||
let mut total_value_bytes = 0usize;
|
||||
let mut last_log = Instant::now();
|
||||
// Walk all storage entries for this address
|
||||
let walker = cursor.walk_dup(Some(address), None)?;
|
||||
for entry in walker {
|
||||
let (_, storage_entry) = entry?;
|
||||
count += 1;
|
||||
// StorageEntry encodes as: 32 bytes (key/subkey uncompressed) + compressed U256
|
||||
let mut buf = Vec::new();
|
||||
let entry_len = storage_entry.to_compact(&mut buf);
|
||||
total_value_bytes += entry_len;
|
||||
|
||||
let walker = cursor.walk_dup(Some(hashed_address), None)?;
|
||||
for entry in walker {
|
||||
let (_, storage_entry) = entry?;
|
||||
count += 1;
|
||||
let mut buf = Vec::new();
|
||||
let entry_len = storage_entry.to_compact(&mut buf);
|
||||
total_value_bytes += entry_len;
|
||||
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots = count,
|
||||
key = %storage_entry.key,
|
||||
"Processing hashed storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots = count,
|
||||
key = %storage_entry.key,
|
||||
"Processing storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
let total_size = if count > 0 { 32 + total_value_bytes } else { 0 };
|
||||
// Add 20 bytes for the Address key (stored once per account in dupsort)
|
||||
let total_size = if count > 0 { 20 + total_value_bytes } else { 0 };
|
||||
|
||||
Ok::<_, eyre::Report>((count, total_size))
|
||||
})??
|
||||
} else {
|
||||
tool.provider_factory.db_ref().view(|tx| {
|
||||
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
|
||||
let mut count = 0usize;
|
||||
let mut total_value_bytes = 0usize;
|
||||
let mut last_log = Instant::now();
|
||||
Ok::<_, eyre::Report>((count, total_size))
|
||||
})??;
|
||||
|
||||
// Walk all storage entries for this address
|
||||
let walker = cursor.walk_dup(Some(address), None)?;
|
||||
for entry in walker {
|
||||
let (_, storage_entry) = entry?;
|
||||
count += 1;
|
||||
let mut buf = Vec::new();
|
||||
// StorageEntry encodes as: 32 bytes (key/subkey uncompressed) + compressed U256
|
||||
let entry_len = storage_entry.to_compact(&mut buf);
|
||||
total_value_bytes += entry_len;
|
||||
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots = count,
|
||||
key = %storage_entry.key,
|
||||
"Processing storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
// Add 20 bytes for the Address key (stored once per account in dupsort)
|
||||
let total_size = if count > 0 { 20 + total_value_bytes } else { 0 };
|
||||
|
||||
Ok::<_, eyre::Report>((count, total_size))
|
||||
})??
|
||||
};
|
||||
// Estimate hashed storage size: 32-byte B256 key instead of 20-byte Address
|
||||
let hashed_size_estimate = if slot_count > 0 { plain_size + 12 } else { 0 };
|
||||
let total_estimate = plain_size + hashed_size_estimate;
|
||||
|
||||
let hashed_address = keccak256(address);
|
||||
|
||||
println!("Account: {address}");
|
||||
println!("Hashed address: {hashed_address}");
|
||||
println!("Storage slots: {slot_count}");
|
||||
if use_hashed_state {
|
||||
println!("Hashed storage size: {} (estimated)", human_bytes(storage_size as f64));
|
||||
} else {
|
||||
// Estimate hashed storage size: 32-byte B256 key instead of 20-byte Address
|
||||
let hashed_size_estimate = if slot_count > 0 { storage_size + 12 } else { 0 };
|
||||
let total_estimate = storage_size + hashed_size_estimate;
|
||||
println!("Plain storage size: {} (estimated)", human_bytes(storage_size as f64));
|
||||
println!(
|
||||
"Hashed storage size: {} (estimated)",
|
||||
human_bytes(hashed_size_estimate as f64)
|
||||
);
|
||||
println!("Total estimated size: {}", human_bytes(total_estimate as f64));
|
||||
}
|
||||
println!("Plain storage size: {} (estimated)", human_bytes(plain_size as f64));
|
||||
println!("Hashed storage size: {} (estimated)", human_bytes(hashed_size_estimate as f64));
|
||||
println!("Total estimated size: {}", human_bytes(total_estimate as f64));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
use clap::Parser;
|
||||
use reth_db::mdbx::{self, ffi};
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// Copies the MDBX database to a new location.
|
||||
///
|
||||
/// Equivalent to the standalone `mdbx_copy` tool but bundled into reth.
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct Command {
|
||||
/// Destination path for the database copy.
|
||||
dest: PathBuf,
|
||||
|
||||
/// Compact the database while copying (reclaims free space).
|
||||
#[arg(short, long)]
|
||||
compact: bool,
|
||||
|
||||
/// Force dynamic size for the destination database.
|
||||
#[arg(short = 'd', long)]
|
||||
force_dynamic_size: bool,
|
||||
|
||||
/// Throttle to avoid MVCC pressure on writers.
|
||||
#[arg(short = 'p', long)]
|
||||
throttle_mvcc: bool,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `db copy` command
|
||||
pub fn execute(self, db: &mdbx::DatabaseEnv) -> eyre::Result<()> {
|
||||
let mut flags: ffi::MDBX_copy_flags_t = ffi::MDBX_CP_DEFAULTS;
|
||||
if self.compact {
|
||||
flags |= ffi::MDBX_CP_COMPACT;
|
||||
}
|
||||
if self.force_dynamic_size {
|
||||
flags |= ffi::MDBX_CP_FORCE_DYNAMIC_SIZE;
|
||||
}
|
||||
if self.throttle_mvcc {
|
||||
flags |= ffi::MDBX_CP_THROTTLE_MVCC;
|
||||
}
|
||||
|
||||
let dest = self
|
||||
.dest
|
||||
.to_str()
|
||||
.ok_or_else(|| eyre::eyre!("destination path must be valid UTF-8"))?;
|
||||
let dest_cstr = std::ffi::CString::new(dest)?;
|
||||
|
||||
println!("Copying database to {} ...", self.dest.display());
|
||||
|
||||
let rc = db.with_raw_env_ptr(|env_ptr| unsafe {
|
||||
ffi::mdbx_env_copy(env_ptr, dest_cstr.as_ptr(), flags)
|
||||
});
|
||||
|
||||
if rc != 0 {
|
||||
eyre::bail!("mdbx_env_copy failed with error code {rc}: {}", unsafe {
|
||||
std::ffi::CStr::from_ptr(ffi::mdbx_strerror(rc)).to_string_lossy()
|
||||
});
|
||||
}
|
||||
|
||||
println!("Done.");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,564 +0,0 @@
|
||||
use clap::Parser;
|
||||
use eyre::Context;
|
||||
use memmap2::Mmap;
|
||||
use rayon::prelude::*;
|
||||
use reth_db_api::tables::Tables;
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::NodeTypesWithDB;
|
||||
use reth_node_core::dirs::{ChainPath, DataDirPath};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fs,
|
||||
io::Write,
|
||||
path::PathBuf,
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
const PAGEHDRSZ: usize = 20;
|
||||
const NODESIZE: usize = 8;
|
||||
const P_BRANCH: u16 = 0x01;
|
||||
const P_LEAF: u16 = 0x02;
|
||||
const P_LARGE: u16 = 0x04;
|
||||
const P_META: u16 = 0x08;
|
||||
const P_DUPFIX: u16 = 0x20;
|
||||
const N_BIG: u8 = 0x01;
|
||||
const N_TREE: u8 = 0x02;
|
||||
const N_DUP: u8 = 0x04;
|
||||
const P_INVALID: u32 = 0xFFFFFFFF;
|
||||
const UNREFERENCED: u8 = 0xFF;
|
||||
const DBI_META: u8 = 0xFE;
|
||||
|
||||
const META_GEO: usize = 0x14;
|
||||
const META_FREE_DB: usize = 0x28;
|
||||
const META_MAIN_DB: usize = 0x58;
|
||||
const META_TXNID_A: usize = 0x08;
|
||||
const META_TXNID_B: usize = 0xB0;
|
||||
const GEO_NOW: usize = 0x0C;
|
||||
const TREE_ROOT: usize = 0x08;
|
||||
const TREE_BRANCH_PAGES: usize = 0x0C;
|
||||
const TREE_LEAF_PAGES: usize = 0x10;
|
||||
const TREE_LARGE_PAGES: usize = 0x14;
|
||||
const TREE_ITEMS: usize = 0x20;
|
||||
const MDBX_MAGIC: u64 = 0x59659DBDEF4C11;
|
||||
|
||||
fn u16_le(buf: &[u8], off: usize) -> u16 {
|
||||
u16::from_le_bytes([buf[off], buf[off + 1]])
|
||||
}
|
||||
|
||||
fn u32_le(buf: &[u8], off: usize) -> u32 {
|
||||
u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]])
|
||||
}
|
||||
|
||||
fn u64_le(buf: &[u8], off: usize) -> u64 {
|
||||
u64::from_le_bytes(buf[off..off + 8].try_into().unwrap())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct TreeDescriptor {
|
||||
root: u32,
|
||||
branch_pages: u32,
|
||||
leaf_pages: u32,
|
||||
large_pages: u32,
|
||||
items: u64,
|
||||
}
|
||||
|
||||
impl TreeDescriptor {
|
||||
fn parse(buf: &[u8], off: usize) -> Self {
|
||||
Self {
|
||||
root: u32_le(buf, off + TREE_ROOT),
|
||||
branch_pages: u32_le(buf, off + TREE_BRANCH_PAGES),
|
||||
leaf_pages: u32_le(buf, off + TREE_LEAF_PAGES),
|
||||
large_pages: u32_le(buf, off + TREE_LARGE_PAGES),
|
||||
items: u64_le(buf, off + TREE_ITEMS),
|
||||
}
|
||||
}
|
||||
|
||||
fn total_pages(&self) -> u64 {
|
||||
self.branch_pages as u64 + self.leaf_pages as u64 + self.large_pages as u64
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct DBIInfo {
|
||||
name: String,
|
||||
dbi_index: u8,
|
||||
tree: TreeDescriptor,
|
||||
}
|
||||
|
||||
fn page_flags(buf: &[u8], page_off: usize) -> u16 {
|
||||
u16_le(buf, page_off + 0x0A)
|
||||
}
|
||||
|
||||
fn page_nkeys(buf: &[u8], page_off: usize) -> usize {
|
||||
u16_le(buf, page_off + 0x0C) as usize / 2
|
||||
}
|
||||
|
||||
fn page_overflow_count(buf: &[u8], page_off: usize) -> u32 {
|
||||
u32_le(buf, page_off + 0x0C)
|
||||
}
|
||||
|
||||
fn mark(owner_map: &mut [u8], pgno: usize, dbi_index: u8, conflicts: &mut u64) -> bool {
|
||||
if pgno >= owner_map.len() {
|
||||
return false;
|
||||
}
|
||||
let prev = owner_map[pgno];
|
||||
if prev == UNREFERENCED {
|
||||
owner_map[pgno] = dbi_index;
|
||||
return true;
|
||||
}
|
||||
if prev != dbi_index {
|
||||
*conflicts += 1;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
|
||||
fn walk_tree_collect(
|
||||
buf: &[u8],
|
||||
ps: usize,
|
||||
root_pgno: u32,
|
||||
page_count: usize,
|
||||
dbi_index: u8,
|
||||
) -> Vec<(usize, u8)> {
|
||||
if root_pgno == P_INVALID || root_pgno as usize >= page_count {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut result: Vec<(usize, u8)> = Vec::new();
|
||||
let mut visited = vec![false; page_count];
|
||||
let mut stack: Vec<u32> = vec![root_pgno];
|
||||
|
||||
while let Some(pgno) = stack.pop() {
|
||||
let pgno_usize = pgno as usize;
|
||||
if pgno_usize >= page_count || visited[pgno_usize] {
|
||||
continue;
|
||||
}
|
||||
visited[pgno_usize] = true;
|
||||
result.push((pgno_usize, dbi_index));
|
||||
|
||||
let page_off = pgno_usize * ps;
|
||||
if page_off + PAGEHDRSZ > buf.len() {
|
||||
continue;
|
||||
}
|
||||
let flags = page_flags(buf, page_off);
|
||||
|
||||
if flags & P_BRANCH != 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let node_abs = page_off + PAGEHDRSZ + node_rel;
|
||||
if node_abs + 4 > buf.len() { break; }
|
||||
stack.push(u32_le(buf, node_abs));
|
||||
}
|
||||
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let node_abs = page_off + PAGEHDRSZ + node_rel;
|
||||
if node_abs + NODESIZE > buf.len() { break; }
|
||||
|
||||
let mn_flags = buf[node_abs + 4];
|
||||
let ksize = u16_le(buf, node_abs + 6) as usize;
|
||||
let data_off = node_abs + NODESIZE + ksize;
|
||||
|
||||
if mn_flags & N_BIG != 0 {
|
||||
if data_off + 4 > buf.len() { continue; }
|
||||
let ov_pgno = u32_le(buf, data_off) as usize;
|
||||
if ov_pgno >= page_count { continue; }
|
||||
let ov_page_off = ov_pgno * ps;
|
||||
if ov_page_off + PAGEHDRSZ > buf.len() { continue; }
|
||||
let ov_count = page_overflow_count(buf, ov_page_off) as usize;
|
||||
for op in ov_pgno..ov_pgno + ov_count {
|
||||
if op < page_count && !visited[op] {
|
||||
visited[op] = true;
|
||||
result.push((op, dbi_index));
|
||||
}
|
||||
}
|
||||
} else if (mn_flags & (N_DUP | N_TREE)) == (N_DUP | N_TREE) {
|
||||
if data_off + 48 > buf.len() { continue; }
|
||||
let sub = TreeDescriptor::parse(buf, data_off);
|
||||
if sub.root != P_INVALID {
|
||||
stack.push(sub.root);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if flags & P_LARGE != 0 {
|
||||
let ov_count = page_overflow_count(buf, page_off) as usize;
|
||||
for op in (pgno_usize + 1)..(pgno_usize + ov_count) {
|
||||
if op < page_count && !visited[op] {
|
||||
visited[op] = true;
|
||||
result.push((op, dbi_index));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
fn discover_named_dbis(
|
||||
buf: &[u8],
|
||||
ps: usize,
|
||||
main_root: u32,
|
||||
owner_map: &mut [u8],
|
||||
conflicts: &mut u64,
|
||||
dbi_main: u8,
|
||||
dbi_start: u8,
|
||||
) -> Vec<DBIInfo> {
|
||||
if main_root == P_INVALID || main_root as usize >= owner_map.len() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut named: Vec<DBIInfo> = Vec::new();
|
||||
let mut next_index = dbi_start;
|
||||
let mut stack: Vec<u32> = vec![main_root];
|
||||
|
||||
while let Some(pgno) = stack.pop() {
|
||||
let pgno_usize = pgno as usize;
|
||||
if pgno_usize >= owner_map.len() { continue; }
|
||||
if !mark(owner_map, pgno_usize, dbi_main, conflicts) { continue; }
|
||||
|
||||
let page_off = pgno_usize * ps;
|
||||
if page_off + PAGEHDRSZ > buf.len() { continue; }
|
||||
let flags = page_flags(buf, page_off);
|
||||
|
||||
if flags & P_BRANCH != 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let node_abs = page_off + PAGEHDRSZ + node_rel;
|
||||
if node_abs + 4 > buf.len() { break; }
|
||||
let child_pgno = u32_le(buf, node_abs);
|
||||
stack.push(child_pgno);
|
||||
}
|
||||
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let node_abs = page_off + PAGEHDRSZ + node_rel;
|
||||
if node_abs + NODESIZE > buf.len() { break; }
|
||||
|
||||
let mn_flags = buf[node_abs + 4];
|
||||
let ksize = u16_le(buf, node_abs + 6) as usize;
|
||||
let data_off = node_abs + NODESIZE + ksize;
|
||||
|
||||
if mn_flags & N_TREE != 0 {
|
||||
let key_off = node_abs + NODESIZE;
|
||||
if key_off + ksize > buf.len() || data_off + 48 > buf.len() { continue; }
|
||||
let name_bytes = &buf[key_off..key_off + ksize];
|
||||
let name = String::from_utf8_lossy(
|
||||
name_bytes.split(|&b| b == 0).next().unwrap_or(name_bytes)
|
||||
).to_string();
|
||||
let tree = TreeDescriptor::parse(buf, data_off);
|
||||
named.push(DBIInfo { name, dbi_index: next_index, tree });
|
||||
next_index = next_index.saturating_add(1);
|
||||
} else if mn_flags & N_BIG != 0 {
|
||||
if data_off + 4 > buf.len() { continue; }
|
||||
let ov_pgno = u32_le(buf, data_off) as usize;
|
||||
if ov_pgno >= owner_map.len() { continue; }
|
||||
let ov_page_off = ov_pgno * ps;
|
||||
if ov_page_off + PAGEHDRSZ > buf.len() { continue; }
|
||||
let ov_count = page_overflow_count(buf, ov_page_off) as usize;
|
||||
for op in ov_pgno..ov_pgno + ov_count {
|
||||
mark(owner_map, op, dbi_main, conflicts);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
named
|
||||
}
|
||||
|
||||
fn mark_free_pages(
|
||||
buf: &[u8],
|
||||
ps: usize,
|
||||
free_root: u32,
|
||||
owner_map: &mut [u8],
|
||||
conflicts: &mut u64,
|
||||
dbi_free: u8,
|
||||
) -> u64 {
|
||||
if free_root == P_INVALID || free_root as usize >= owner_map.len() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let mut marked: u64 = 0;
|
||||
let mut stack: Vec<u32> = vec![free_root];
|
||||
|
||||
while let Some(pgno) = stack.pop() {
|
||||
let pgno_usize = pgno as usize;
|
||||
if pgno_usize >= owner_map.len() { continue; }
|
||||
if !mark(owner_map, pgno_usize, dbi_free, conflicts) { continue; }
|
||||
marked += 1;
|
||||
|
||||
let page_off = pgno_usize * ps;
|
||||
if page_off + PAGEHDRSZ > buf.len() { continue; }
|
||||
let flags = page_flags(buf, page_off);
|
||||
|
||||
if flags & P_BRANCH != 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let child_pgno = u32_le(buf, page_off + PAGEHDRSZ + node_rel);
|
||||
stack.push(child_pgno);
|
||||
}
|
||||
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let node_abs = page_off + PAGEHDRSZ + node_rel;
|
||||
if node_abs + NODESIZE > buf.len() { break; }
|
||||
|
||||
let mn_flags = buf[node_abs + 4];
|
||||
let ksize = u16_le(buf, node_abs + 6) as usize;
|
||||
let dsize = u32_le(buf, node_abs) as usize;
|
||||
let data_off = node_abs + NODESIZE + ksize;
|
||||
|
||||
if mn_flags & N_BIG != 0 {
|
||||
if data_off + 4 > buf.len() { continue; }
|
||||
let ov_pgno = u32_le(buf, data_off) as usize;
|
||||
if ov_pgno >= owner_map.len() { continue; }
|
||||
let ov_page_off = ov_pgno * ps;
|
||||
if ov_page_off + PAGEHDRSZ > buf.len() { continue; }
|
||||
let ov_count = page_overflow_count(buf, ov_page_off) as usize;
|
||||
for op in ov_pgno..ov_pgno + ov_count {
|
||||
if mark(owner_map, op, dbi_free, conflicts) { marked += 1; }
|
||||
}
|
||||
let ov_data_off = ov_pgno * ps + PAGEHDRSZ;
|
||||
if dsize >= 4 && ov_data_off + 4 <= buf.len() {
|
||||
let pnl_count = u32_le(buf, ov_data_off) as usize;
|
||||
for j in 0..pnl_count {
|
||||
let fp_off = ov_data_off + 4 + j * 4;
|
||||
if fp_off + 4 > buf.len() { break; }
|
||||
let fp = u32_le(buf, fp_off) as usize;
|
||||
if fp < owner_map.len() && owner_map[fp] == UNREFERENCED {
|
||||
owner_map[fp] = dbi_free;
|
||||
marked += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if dsize >= 4 && data_off + 4 <= buf.len() {
|
||||
let pnl_count = u32_le(buf, data_off) as usize;
|
||||
for j in 0..pnl_count {
|
||||
let fp_off = data_off + 4 + j * 4;
|
||||
if fp_off + 4 > buf.len() { break; }
|
||||
let fp = u32_le(buf, fp_off) as usize;
|
||||
if fp < owner_map.len() && owner_map[fp] == UNREFERENCED {
|
||||
owner_map[fp] = dbi_free;
|
||||
marked += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
marked
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct Command {
|
||||
#[arg(short, long, default_value = "owner_map.bin")]
|
||||
output: PathBuf,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
pub fn execute<N: NodeTypesWithDB>(
|
||||
self,
|
||||
data_dir: ChainPath<DataDirPath>,
|
||||
_tool: &DbTool<N>,
|
||||
) -> eyre::Result<()> {
|
||||
let db_path = data_dir.db().join("mdbx.dat");
|
||||
eyre::ensure!(db_path.exists(), "mdbx.dat not found at {:?}", db_path);
|
||||
|
||||
let t0 = Instant::now();
|
||||
|
||||
let file = fs::File::open(&db_path)
|
||||
.wrap_err_with(|| format!("Failed to open {}", db_path.display()))?;
|
||||
let mmap = unsafe { Mmap::map(&file)? };
|
||||
let buf: &[u8] = &mmap;
|
||||
|
||||
if buf.len() < PAGEHDRSZ + 0xC0 {
|
||||
eyre::bail!("File too small for MDBX meta page");
|
||||
}
|
||||
let flags0 = u16_le(buf, 0x0A);
|
||||
if flags0 & P_META == 0 {
|
||||
eyre::bail!("Page 0 missing P_META flag");
|
||||
}
|
||||
let magic = u64_le(buf, PAGEHDRSZ);
|
||||
if (magic >> 8) != MDBX_MAGIC {
|
||||
eyre::bail!("MDBX magic mismatch");
|
||||
}
|
||||
|
||||
let candidates = [4096usize, 8192, 16384, 32768, 65536];
|
||||
let mut ps = 4096usize;
|
||||
let geo_now_raw = u32_le(buf, PAGEHDRSZ + META_GEO + GEO_NOW) as usize;
|
||||
for &candidate in &candidates {
|
||||
let mapped = geo_now_raw * candidate;
|
||||
if mapped >= buf.len() / 2 && mapped <= buf.len() * 4 {
|
||||
ps = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let mut best_txnid: u64 = 0;
|
||||
let mut best_meta_base: usize = 0;
|
||||
for pgno in 0..3usize {
|
||||
let page_base = pgno * ps;
|
||||
let meta_base = page_base + PAGEHDRSZ;
|
||||
if meta_base + 0xC0 > buf.len() { continue; }
|
||||
let pflags = u16_le(buf, page_base + 0x0A);
|
||||
if pflags & P_META == 0 { continue; }
|
||||
let m = u64_le(buf, meta_base);
|
||||
if (m >> 8) != MDBX_MAGIC { continue; }
|
||||
let txnid_a = u64_le(buf, meta_base + META_TXNID_A);
|
||||
let txnid_b = u64_le(buf, meta_base + META_TXNID_B);
|
||||
if txnid_a == txnid_b && txnid_a > best_txnid {
|
||||
best_txnid = txnid_a;
|
||||
best_meta_base = meta_base;
|
||||
}
|
||||
}
|
||||
eyre::ensure!(best_txnid > 0, "No valid meta page found");
|
||||
|
||||
let page_count = u32_le(buf, best_meta_base + META_GEO + GEO_NOW) as usize;
|
||||
let free_tree = TreeDescriptor::parse(buf, best_meta_base + META_FREE_DB);
|
||||
let main_tree = TreeDescriptor::parse(buf, best_meta_base + META_MAIN_DB);
|
||||
|
||||
println!("MDBX owner map generator (parallel)");
|
||||
println!(" page_size: {ps}");
|
||||
println!(" page_count: {page_count}");
|
||||
println!(" txnid: {best_txnid}");
|
||||
println!(" FreeDB root: {} ({} items)", free_tree.root, free_tree.items);
|
||||
println!(" MainDB root: {} ({} items)", main_tree.root, main_tree.items);
|
||||
println!();
|
||||
|
||||
let mut owner_map = vec![UNREFERENCED; page_count];
|
||||
let mut conflicts: u64 = 0;
|
||||
|
||||
for pgno in 0..std::cmp::min(3, page_count) {
|
||||
owner_map[pgno] = DBI_META;
|
||||
}
|
||||
|
||||
println!("Discovering named DBIs via MainDB walk...");
|
||||
let discovered = discover_named_dbis(
|
||||
buf, ps, main_tree.root, &mut owner_map, &mut conflicts, 1, 2,
|
||||
);
|
||||
|
||||
let mut name_to_dbi: HashMap<&str, u8> = HashMap::new();
|
||||
for (idx, table) in Tables::ALL.iter().enumerate() {
|
||||
name_to_dbi.insert(table.name(), (idx + 2) as u8);
|
||||
}
|
||||
|
||||
let mut named_dbis: Vec<DBIInfo> = Vec::new();
|
||||
let mut remap: HashMap<u8, u8> = HashMap::new();
|
||||
for dbi in &discovered {
|
||||
if let Some(&real_dbi) = name_to_dbi.get(dbi.name.as_str()) {
|
||||
remap.insert(dbi.dbi_index, real_dbi);
|
||||
named_dbis.push(DBIInfo {
|
||||
name: dbi.name.clone(),
|
||||
dbi_index: real_dbi,
|
||||
tree: dbi.tree.clone(),
|
||||
});
|
||||
} else {
|
||||
named_dbis.push(dbi.clone());
|
||||
}
|
||||
}
|
||||
|
||||
for byte in owner_map.iter_mut() {
|
||||
if let Some(&new_dbi) = remap.get(byte) {
|
||||
*byte = new_dbi;
|
||||
}
|
||||
}
|
||||
|
||||
println!("Found {} named DBI(s)", named_dbis.len());
|
||||
|
||||
println!("Walking FreeDB / GC...");
|
||||
let free_marked = mark_free_pages(buf, ps, free_tree.root, &mut owner_map, &mut conflicts, 0);
|
||||
println!("FreeDB: {} pages marked", free_marked);
|
||||
|
||||
println!("Walking {} named DBIs in parallel...", named_dbis.len());
|
||||
let t_par = Instant::now();
|
||||
|
||||
let results: Vec<(String, u8, Vec<(usize, u8)>)> = named_dbis
|
||||
.par_iter()
|
||||
.filter(|dbi| dbi.tree.root != P_INVALID)
|
||||
.map(|dbi| {
|
||||
let pages = walk_tree_collect(buf, ps, dbi.tree.root, page_count, dbi.dbi_index);
|
||||
(dbi.name.clone(), dbi.dbi_index, pages)
|
||||
})
|
||||
.collect();
|
||||
|
||||
for (name, dbi_index, pages) in &results {
|
||||
let count = pages.len();
|
||||
for &(pgno, idx) in pages {
|
||||
mark(&mut owner_map, pgno, idx, &mut conflicts);
|
||||
}
|
||||
println!(" [{:2}] {:30} {:>10} pages", dbi_index, name, count);
|
||||
}
|
||||
|
||||
println!("Parallel walk done in {:.2}s", t_par.elapsed().as_secs_f64());
|
||||
|
||||
let elapsed = t0.elapsed();
|
||||
|
||||
let mut counts: HashMap<u8, u64> = HashMap::new();
|
||||
let mut unreferenced: u64 = 0;
|
||||
for &b in &owner_map {
|
||||
if b == UNREFERENCED {
|
||||
unreferenced += 1;
|
||||
} else {
|
||||
*counts.entry(b).or_default() += 1;
|
||||
}
|
||||
}
|
||||
|
||||
println!();
|
||||
println!("Walk complete in {:.2}s", elapsed.as_secs_f64());
|
||||
println!(" Total pages: {page_count}");
|
||||
println!(" Unreferenced: {unreferenced}");
|
||||
println!(" Conflicts: {conflicts}");
|
||||
println!();
|
||||
|
||||
if let Some(&c) = counts.get(&DBI_META) {
|
||||
println!(" [0xFE] {:30} {:>10} pages", "<meta>", c);
|
||||
}
|
||||
if let Some(&c) = counts.get(&0) {
|
||||
println!(" [ 0] {:30} {:>10} pages", "<free/GC>", c);
|
||||
}
|
||||
if let Some(&c) = counts.get(&1) {
|
||||
println!(" [ 1] {:30} {:>10} pages", "<main>", c);
|
||||
}
|
||||
let mut sorted_dbis = named_dbis.clone();
|
||||
sorted_dbis.sort_by_key(|d| d.dbi_index);
|
||||
for dbi in &sorted_dbis {
|
||||
let walked = counts.get(&dbi.dbi_index).copied().unwrap_or(0);
|
||||
let expected = dbi.tree.total_pages();
|
||||
let mismatch = if expected > 0 && walked != expected {
|
||||
format!(" !! MISMATCH (tree_t says {})", expected)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
println!(" [{:3}] {:30} {:>10} pages{}", dbi.dbi_index, dbi.name, walked, mismatch);
|
||||
}
|
||||
println!(" [0xFF] {:30} {:>10} pages", "<unreferenced>", unreferenced);
|
||||
|
||||
let mut out = fs::File::create(&self.output)?;
|
||||
out.write_all(&owner_map)?;
|
||||
println!();
|
||||
println!("Written {} bytes to {}", owner_map.len(), self.output.display());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -98,8 +98,7 @@ impl Command {
|
||||
)?;
|
||||
|
||||
if let Some(entry) = entry {
|
||||
let se: reth_primitives_traits::StorageEntry = entry.into();
|
||||
println!("{}", serde_json::to_string_pretty(&se)?);
|
||||
println!("{}", serde_json::to_string_pretty(&entry)?);
|
||||
} else {
|
||||
error!(target: "reth::cli", "No content for the given table key.");
|
||||
}
|
||||
@@ -107,14 +106,7 @@ impl Command {
|
||||
}
|
||||
|
||||
let changesets = provider.storage_changeset(key.block_number())?;
|
||||
let serializable: Vec<_> = changesets
|
||||
.into_iter()
|
||||
.map(|(addr, entry)| {
|
||||
let se: reth_primitives_traits::StorageEntry = entry.into();
|
||||
(addr, se)
|
||||
})
|
||||
.collect();
|
||||
println!("{}", serde_json::to_string_pretty(&serializable)?);
|
||||
println!("{}", serde_json::to_string_pretty(&changesets)?);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@ use std::{
|
||||
mod account_storage;
|
||||
mod checksum;
|
||||
mod clear;
|
||||
mod copy;
|
||||
mod diff;
|
||||
mod get;
|
||||
mod list;
|
||||
@@ -21,7 +20,6 @@ mod settings;
|
||||
mod state;
|
||||
mod static_file_header;
|
||||
mod stats;
|
||||
pub mod gen_ownermap;
|
||||
/// DB List TUI
|
||||
mod tui;
|
||||
|
||||
@@ -44,8 +42,6 @@ pub enum Subcommands {
|
||||
List(list::Command),
|
||||
/// Calculates the content checksum of a table or static file segment
|
||||
Checksum(checksum::Command),
|
||||
/// Copies the MDBX database to a new location (bundled mdbx_copy)
|
||||
Copy(copy::Command),
|
||||
/// Create a diff between two database tables or two entire databases.
|
||||
Diff(diff::Command),
|
||||
/// Gets the content of a table for the given key
|
||||
@@ -72,8 +68,6 @@ pub enum Subcommands {
|
||||
AccountStorage(account_storage::Command),
|
||||
/// Gets account state and storage at a specific block
|
||||
State(state::Command),
|
||||
/// Generate a page owner map for MDBX page visualization
|
||||
GenOwnermap(gen_ownermap::Command),
|
||||
}
|
||||
|
||||
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C> {
|
||||
@@ -130,11 +124,6 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
command.execute(&tool)?;
|
||||
});
|
||||
}
|
||||
Subcommands::Copy(command) => {
|
||||
db_exec!(self.env, tool, N, AccessRights::RO, {
|
||||
command.execute(tool.provider_factory.db_ref())?;
|
||||
});
|
||||
}
|
||||
Subcommands::Diff(command) => {
|
||||
db_exec!(self.env, tool, N, AccessRights::RO, {
|
||||
command.execute(&tool)?;
|
||||
@@ -217,11 +206,6 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
command.execute(&tool)?;
|
||||
});
|
||||
}
|
||||
Subcommands::GenOwnermap(command) => {
|
||||
db_exec!(self.env, tool, N, AccessRights::RO, {
|
||||
command.execute(data_dir, &tool)?;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -39,12 +39,50 @@ enum Subcommands {
|
||||
#[derive(Debug, Clone, Copy, Subcommand)]
|
||||
#[clap(rename_all = "snake_case")]
|
||||
pub enum SetCommand {
|
||||
/// Enable or disable v2 storage layout
|
||||
/// Store receipts in static files instead of the database
|
||||
Receipts {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store transaction senders in static files instead of the database
|
||||
TransactionSenders {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store account changesets in static files instead of the database
|
||||
AccountChangesets {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store storage history in rocksdb instead of MDBX
|
||||
StoragesHistory {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store transaction hash to number mapping in rocksdb instead of MDBX
|
||||
TransactionHashNumbers {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store account history in rocksdb instead of MDBX
|
||||
AccountHistory {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store storage changesets in static files instead of the database
|
||||
StorageChangesets {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Use hashed state tables (HashedAccounts/HashedStorages) as canonical state
|
||||
///
|
||||
/// When enabled, uses static files for receipts/senders/changesets and RocksDB for
|
||||
/// history indices and transaction hashes. When disabled, uses v1/legacy layout (everything in
|
||||
/// MDBX).
|
||||
V2 {
|
||||
/// When enabled, execution writes directly to hashed tables, eliminating need for
|
||||
/// separate hashing stages. State reads come from hashed tables.
|
||||
///
|
||||
/// WARNING: Changing this setting in either direction requires re-syncing the database.
|
||||
/// Enabling on an existing plain-state database leaves hashed tables empty.
|
||||
/// Disabling on an existing hashed-state database leaves plain tables empty.
|
||||
UseHashedState {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
@@ -87,18 +125,87 @@ impl Command {
|
||||
println!("No storage settings found, creating new settings.");
|
||||
}
|
||||
|
||||
let mut settings @ StorageSettings { storage_v2: _ } =
|
||||
settings.unwrap_or_else(StorageSettings::v1);
|
||||
let mut settings @ StorageSettings {
|
||||
receipts_in_static_files: _,
|
||||
transaction_senders_in_static_files: _,
|
||||
storages_history_in_rocksdb: _,
|
||||
transaction_hash_numbers_in_rocksdb: _,
|
||||
account_history_in_rocksdb: _,
|
||||
account_changesets_in_static_files: _,
|
||||
storage_changesets_in_static_files: _,
|
||||
use_hashed_state: _,
|
||||
} = settings.unwrap_or_else(StorageSettings::v1);
|
||||
|
||||
// Update the setting based on the key
|
||||
match cmd {
|
||||
SetCommand::V2 { value } => {
|
||||
if settings.storage_v2 == value {
|
||||
println!("storage_v2 is already set to {}", value);
|
||||
SetCommand::Receipts { value } => {
|
||||
if settings.receipts_in_static_files == value {
|
||||
println!("receipts_in_static_files is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.storage_v2 = value;
|
||||
println!("Set storage_v2 = {}", value);
|
||||
settings.receipts_in_static_files = value;
|
||||
println!("Set receipts_in_static_files = {}", value);
|
||||
}
|
||||
SetCommand::TransactionSenders { value } => {
|
||||
if settings.transaction_senders_in_static_files == value {
|
||||
println!("transaction_senders_in_static_files is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.transaction_senders_in_static_files = value;
|
||||
println!("Set transaction_senders_in_static_files = {}", value);
|
||||
}
|
||||
SetCommand::AccountChangesets { value } => {
|
||||
if settings.account_changesets_in_static_files == value {
|
||||
println!("account_changesets_in_static_files is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.account_changesets_in_static_files = value;
|
||||
println!("Set account_changesets_in_static_files = {}", value);
|
||||
}
|
||||
SetCommand::StoragesHistory { value } => {
|
||||
if settings.storages_history_in_rocksdb == value {
|
||||
println!("storages_history_in_rocksdb is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.storages_history_in_rocksdb = value;
|
||||
println!("Set storages_history_in_rocksdb = {}", value);
|
||||
}
|
||||
SetCommand::TransactionHashNumbers { value } => {
|
||||
if settings.transaction_hash_numbers_in_rocksdb == value {
|
||||
println!("transaction_hash_numbers_in_rocksdb is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.transaction_hash_numbers_in_rocksdb = value;
|
||||
println!("Set transaction_hash_numbers_in_rocksdb = {}", value);
|
||||
}
|
||||
SetCommand::AccountHistory { value } => {
|
||||
if settings.account_history_in_rocksdb == value {
|
||||
println!("account_history_in_rocksdb is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.account_history_in_rocksdb = value;
|
||||
println!("Set account_history_in_rocksdb = {}", value);
|
||||
}
|
||||
SetCommand::StorageChangesets { value } => {
|
||||
if settings.storage_changesets_in_static_files == value {
|
||||
println!("storage_changesets_in_static_files is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.storage_changesets_in_static_files = value;
|
||||
println!("Set storage_changesets_in_static_files = {}", value);
|
||||
}
|
||||
SetCommand::UseHashedState { value } => {
|
||||
if settings.use_hashed_state == value {
|
||||
println!("use_hashed_state is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
if settings.use_hashed_state && !value {
|
||||
println!("WARNING: Disabling use_hashed_state on an existing hashed-state database requires a full resync.");
|
||||
} else {
|
||||
println!("WARNING: Enabling use_hashed_state on an existing plain-state database requires a full resync.");
|
||||
}
|
||||
settings.use_hashed_state = value;
|
||||
println!("Set use_hashed_state = {}", value);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use alloy_primitives::{keccak256, Address, BlockNumber, B256, U256};
|
||||
use alloy_primitives::{Address, BlockNumber, B256, U256};
|
||||
use clap::Parser;
|
||||
use parking_lot::Mutex;
|
||||
use reth_db_api::{
|
||||
@@ -63,65 +63,39 @@ impl Command {
|
||||
address: Address,
|
||||
limit: usize,
|
||||
) -> eyre::Result<()> {
|
||||
let use_hashed_state = tool.provider_factory.cached_storage_settings().use_hashed_state();
|
||||
|
||||
let entries = tool.provider_factory.db_ref().view(|tx| {
|
||||
let (account, walker_entries) = if use_hashed_state {
|
||||
let hashed_address = keccak256(address);
|
||||
let account = tx.get::<tables::HashedAccounts>(hashed_address)?;
|
||||
let mut cursor = tx.cursor_dup_read::<tables::HashedStorages>()?;
|
||||
let walker = cursor.walk_dup(Some(hashed_address), None)?;
|
||||
let mut entries = Vec::new();
|
||||
let mut last_log = Instant::now();
|
||||
for (idx, entry) in walker.enumerate() {
|
||||
let (_, storage_entry) = entry?;
|
||||
if storage_entry.value != U256::ZERO {
|
||||
entries.push((storage_entry.key, storage_entry.value));
|
||||
}
|
||||
if entries.len() >= limit {
|
||||
break;
|
||||
}
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots_scanned = idx,
|
||||
"Scanning storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
(account, entries)
|
||||
} else {
|
||||
// Get account info
|
||||
let account = tx.get::<tables::PlainAccountState>(address)?;
|
||||
// Get storage entries
|
||||
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
|
||||
let walker = cursor.walk_dup(Some(address), None)?;
|
||||
let mut entries = Vec::new();
|
||||
let mut last_log = Instant::now();
|
||||
for (idx, entry) in walker.enumerate() {
|
||||
let (_, storage_entry) = entry?;
|
||||
if storage_entry.value != U256::ZERO {
|
||||
entries.push((storage_entry.key, storage_entry.value));
|
||||
}
|
||||
if entries.len() >= limit {
|
||||
break;
|
||||
}
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots_scanned = idx,
|
||||
"Scanning storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
(account, entries)
|
||||
};
|
||||
// Get account info
|
||||
let account = tx.get::<tables::PlainAccountState>(address)?;
|
||||
|
||||
Ok::<_, eyre::Report>((account, walker_entries))
|
||||
// Get storage entries
|
||||
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
|
||||
let mut entries = Vec::new();
|
||||
let mut last_log = Instant::now();
|
||||
|
||||
let walker = cursor.walk_dup(Some(address), None)?;
|
||||
for (idx, entry) in walker.enumerate() {
|
||||
let (_, storage_entry) = entry?;
|
||||
|
||||
if storage_entry.value != U256::ZERO {
|
||||
entries.push((storage_entry.key, storage_entry.value));
|
||||
}
|
||||
|
||||
if entries.len() >= limit {
|
||||
break;
|
||||
}
|
||||
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots_scanned = idx,
|
||||
"Scanning storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
Ok::<_, eyre::Report>((account, entries))
|
||||
})??;
|
||||
|
||||
let (account, storage_entries) = entries;
|
||||
@@ -145,7 +119,7 @@ impl Command {
|
||||
|
||||
// Check storage settings to determine where history is stored
|
||||
let storage_settings = tool.provider_factory.cached_storage_settings();
|
||||
let history_in_rocksdb = storage_settings.storage_v2;
|
||||
let history_in_rocksdb = storage_settings.storages_history_in_rocksdb;
|
||||
|
||||
// For historical queries, enumerate keys from history indices only
|
||||
// (not PlainStorageState, which reflects current state)
|
||||
|
||||
@@ -37,14 +37,6 @@ pub struct DownloadDefaults {
|
||||
pub available_snapshots: Vec<Cow<'static, str>>,
|
||||
/// Default base URL for snapshots
|
||||
pub default_base_url: Cow<'static, str>,
|
||||
/// Default base URL for chain-aware snapshots.
|
||||
///
|
||||
/// When set, the chain ID is appended to form the full URL: `{base_url}/{chain_id}`.
|
||||
/// For example, given a base URL of `https://snapshots.example.com` and chain ID `1`,
|
||||
/// the resulting URL would be `https://snapshots.example.com/1`.
|
||||
///
|
||||
/// Falls back to [`default_base_url`](Self::default_base_url) when `None`.
|
||||
pub default_chain_aware_base_url: Option<Cow<'static, str>>,
|
||||
/// Optional custom long help text that overrides the generated help
|
||||
pub long_help: Option<String>,
|
||||
}
|
||||
@@ -68,7 +60,6 @@ impl DownloadDefaults {
|
||||
Cow::Borrowed("https://publicnode.com/snapshots (full nodes & testnets)"),
|
||||
],
|
||||
default_base_url: Cow::Borrowed(MERKLE_BASE_URL),
|
||||
default_chain_aware_base_url: None,
|
||||
long_help: None,
|
||||
}
|
||||
}
|
||||
@@ -93,11 +84,9 @@ impl DownloadDefaults {
|
||||
}
|
||||
|
||||
help.push_str(
|
||||
"\nIf no URL is provided, the latest archive snapshot for the selected chain\nwill be proposed for download from ",
|
||||
);
|
||||
help.push_str(
|
||||
self.default_chain_aware_base_url.as_deref().unwrap_or(&self.default_base_url),
|
||||
"\nIf no URL is provided, the latest mainnet archive snapshot\nwill be proposed for download from ",
|
||||
);
|
||||
help.push_str(self.default_base_url.as_ref());
|
||||
help.push_str(
|
||||
".\n\nLocal file:// URLs are also supported for extracting snapshots from disk.",
|
||||
);
|
||||
@@ -122,12 +111,6 @@ impl DownloadDefaults {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the default chain-aware base URL.
|
||||
pub fn with_chain_aware_base_url(mut self, url: impl Into<Cow<'static, str>>) -> Self {
|
||||
self.default_chain_aware_base_url = Some(url.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Builder: Set custom long help text, overriding the generated help
|
||||
pub fn with_long_help(mut self, help: impl Into<String>) -> Self {
|
||||
self.long_help = Some(help.into());
|
||||
@@ -159,7 +142,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> DownloadCo
|
||||
let url = match self.url {
|
||||
Some(url) => url,
|
||||
None => {
|
||||
let url = get_latest_snapshot_url(self.env.chain.chain().id()).await?;
|
||||
let url = get_latest_snapshot_url().await?;
|
||||
info!(target: "reth::cli", "Using default snapshot URL: {}", url);
|
||||
url
|
||||
}
|
||||
@@ -526,12 +509,8 @@ async fn stream_and_extract(url: &str, target_dir: &Path) -> Result<()> {
|
||||
}
|
||||
|
||||
// Builds default URL for latest mainnet archive snapshot using configured defaults
|
||||
async fn get_latest_snapshot_url(chain_id: u64) -> Result<String> {
|
||||
let defaults = DownloadDefaults::get_global();
|
||||
let base_url = match &defaults.default_chain_aware_base_url {
|
||||
Some(url) => format!("{url}/{chain_id}"),
|
||||
None => defaults.default_base_url.to_string(),
|
||||
};
|
||||
async fn get_latest_snapshot_url() -> Result<String> {
|
||||
let base_url = &DownloadDefaults::get_global().default_base_url;
|
||||
let latest_url = format!("{base_url}/latest.txt");
|
||||
let filename = Client::new()
|
||||
.get(latest_url)
|
||||
|
||||
@@ -10,8 +10,8 @@ use reth_node_builder::NodeBuilder;
|
||||
use reth_node_core::{
|
||||
args::{
|
||||
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, EraArgs, MetricArgs,
|
||||
NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, StaticFilesArgs, StorageArgs,
|
||||
TxPoolArgs,
|
||||
NetworkArgs, PayloadBuilderArgs, PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs,
|
||||
StorageArgs, TxPoolArgs,
|
||||
},
|
||||
node_config::NodeConfig,
|
||||
version,
|
||||
@@ -103,6 +103,10 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
|
||||
#[command(flatten)]
|
||||
pub pruning: PruningArgs,
|
||||
|
||||
/// All `RocksDB` table routing arguments
|
||||
#[command(flatten)]
|
||||
pub rocksdb: RocksDbArgs,
|
||||
|
||||
/// Engine cli arguments
|
||||
#[command(flatten, next_help_heading = "Engine")]
|
||||
pub engine: EngineArgs,
|
||||
@@ -115,8 +119,8 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
|
||||
#[command(flatten, next_help_heading = "Static Files")]
|
||||
pub static_files: StaticFilesArgs,
|
||||
|
||||
/// All storage related arguments with --storage prefix
|
||||
#[command(flatten, next_help_heading = "Storage")]
|
||||
/// Storage mode configuration (v2 vs v1/legacy)
|
||||
#[command(flatten)]
|
||||
pub storage: StorageArgs,
|
||||
|
||||
/// Additional cli arguments
|
||||
@@ -171,6 +175,7 @@ where
|
||||
db,
|
||||
dev,
|
||||
pruning,
|
||||
rocksdb,
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
@@ -178,6 +183,9 @@ where
|
||||
ext,
|
||||
} = self;
|
||||
|
||||
// Validate RocksDB arguments
|
||||
rocksdb.validate()?;
|
||||
|
||||
// set up node config
|
||||
let mut node_config = NodeConfig {
|
||||
datadir,
|
||||
@@ -193,6 +201,7 @@ where
|
||||
db,
|
||||
dev,
|
||||
pruning,
|
||||
rocksdb,
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
|
||||
@@ -45,16 +45,12 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
|
||||
let tool = DbTool::new(provider_factory)?;
|
||||
|
||||
let static_file_segments = match self.stage {
|
||||
StageEnum::Headers => vec![StaticFileSegment::Headers],
|
||||
StageEnum::Bodies => vec![StaticFileSegment::Transactions],
|
||||
StageEnum::Execution => vec![
|
||||
StaticFileSegment::Receipts,
|
||||
StaticFileSegment::AccountChangeSets,
|
||||
StaticFileSegment::StorageChangeSets,
|
||||
],
|
||||
StageEnum::Senders => vec![StaticFileSegment::TransactionSenders],
|
||||
_ => vec![],
|
||||
let static_file_segment = match self.stage {
|
||||
StageEnum::Headers => Some(StaticFileSegment::Headers),
|
||||
StageEnum::Bodies => Some(StaticFileSegment::Transactions),
|
||||
StageEnum::Execution => Some(StaticFileSegment::Receipts),
|
||||
StageEnum::Senders => Some(StaticFileSegment::TransactionSenders),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
// Calling `StaticFileProviderRW::prune_*` will instruct the writer to prune rows only
|
||||
@@ -62,33 +58,35 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
// deleting the jar files, otherwise if the task were to be interrupted after we
|
||||
// have deleted them, BUT before we have committed the checkpoints to the database, we'd
|
||||
// lose essential data.
|
||||
let static_file_provider = tool.provider_factory.static_file_provider();
|
||||
for segment in static_file_segments {
|
||||
if let Some(highest_block) = static_file_provider.get_highest_static_file_block(segment)
|
||||
if let Some(static_file_segment) = static_file_segment {
|
||||
let static_file_provider = tool.provider_factory.static_file_provider();
|
||||
if let Some(highest_block) =
|
||||
static_file_provider.get_highest_static_file_block(static_file_segment)
|
||||
{
|
||||
let mut writer = static_file_provider.latest_writer(segment)?;
|
||||
let mut writer = static_file_provider.latest_writer(static_file_segment)?;
|
||||
|
||||
match segment {
|
||||
match static_file_segment {
|
||||
StaticFileSegment::Headers => {
|
||||
// Prune all headers leaving genesis intact.
|
||||
writer.prune_headers(highest_block)?;
|
||||
}
|
||||
StaticFileSegment::Transactions => {
|
||||
let to_delete = static_file_provider
|
||||
.get_highest_static_file_tx(segment)
|
||||
.get_highest_static_file_tx(static_file_segment)
|
||||
.map(|tx_num| tx_num + 1)
|
||||
.unwrap_or_default();
|
||||
writer.prune_transactions(to_delete, 0)?;
|
||||
}
|
||||
StaticFileSegment::Receipts => {
|
||||
let to_delete = static_file_provider
|
||||
.get_highest_static_file_tx(segment)
|
||||
.get_highest_static_file_tx(static_file_segment)
|
||||
.map(|tx_num| tx_num + 1)
|
||||
.unwrap_or_default();
|
||||
writer.prune_receipts(to_delete, 0)?;
|
||||
}
|
||||
StaticFileSegment::TransactionSenders => {
|
||||
let to_delete = static_file_provider
|
||||
.get_highest_static_file_tx(segment)
|
||||
.get_highest_static_file_tx(static_file_segment)
|
||||
.map(|tx_num| tx_num + 1)
|
||||
.unwrap_or_default();
|
||||
writer.prune_transaction_senders(to_delete, 0)?;
|
||||
@@ -133,15 +131,8 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
reset_stage_checkpoint(tx, StageId::SenderRecovery)?;
|
||||
}
|
||||
StageEnum::Execution => {
|
||||
if provider_rw.cached_storage_settings().use_hashed_state() {
|
||||
tx.clear::<tables::HashedAccounts>()?;
|
||||
tx.clear::<tables::HashedStorages>()?;
|
||||
reset_stage_checkpoint(tx, StageId::AccountHashing)?;
|
||||
reset_stage_checkpoint(tx, StageId::StorageHashing)?;
|
||||
} else {
|
||||
tx.clear::<tables::PlainAccountState>()?;
|
||||
tx.clear::<tables::PlainStorageState>()?;
|
||||
}
|
||||
tx.clear::<tables::PlainAccountState>()?;
|
||||
tx.clear::<tables::PlainStorageState>()?;
|
||||
tx.clear::<tables::AccountChangeSets>()?;
|
||||
tx.clear::<tables::StorageChangeSets>()?;
|
||||
tx.clear::<tables::Bytecodes>()?;
|
||||
@@ -187,7 +178,7 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
let settings = provider_rw.cached_storage_settings();
|
||||
let rocksdb = tool.provider_factory.rocksdb_provider();
|
||||
|
||||
if settings.storage_v2 {
|
||||
if settings.account_history_in_rocksdb {
|
||||
rocksdb.clear::<tables::AccountsHistory>()?;
|
||||
} else {
|
||||
tx.clear::<tables::AccountsHistory>()?;
|
||||
@@ -204,7 +195,7 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
let settings = provider_rw.cached_storage_settings();
|
||||
let rocksdb = tool.provider_factory.rocksdb_provider();
|
||||
|
||||
if settings.storage_v2 {
|
||||
if settings.storages_history_in_rocksdb {
|
||||
rocksdb.clear::<tables::StoragesHistory>()?;
|
||||
} else {
|
||||
tx.clear::<tables::StoragesHistory>()?;
|
||||
@@ -218,7 +209,7 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
)?;
|
||||
}
|
||||
StageEnum::TxLookup => {
|
||||
if provider_rw.cached_storage_settings().storage_v2 {
|
||||
if provider_rw.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
|
||||
tool.provider_factory
|
||||
.rocksdb_provider()
|
||||
.clear::<tables::TransactionHashNumbers>()?;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Test setup utilities for configuring the initial state.
|
||||
|
||||
use crate::{testsuite::Environment, E2ETestSetupBuilder, NodeBuilderHelper};
|
||||
use crate::{setup_engine_with_connection, testsuite::Environment, NodeBuilderHelper};
|
||||
use alloy_eips::BlockNumberOrTag;
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rpc_types_engine::{ForkchoiceState, PayloadAttributes};
|
||||
@@ -38,8 +38,6 @@ pub struct Setup<I> {
|
||||
shutdown_tx: Option<mpsc::Sender<()>>,
|
||||
/// Is this setup in dev mode
|
||||
pub is_dev: bool,
|
||||
/// Whether to use v2 storage mode (hashed keys, static file changesets, rocksdb history)
|
||||
pub storage_v2: bool,
|
||||
/// Tracks instance generic.
|
||||
_phantom: PhantomData<I>,
|
||||
/// Holds the import result to keep nodes alive when using imported chain
|
||||
@@ -60,7 +58,6 @@ impl<I> Default for Setup<I> {
|
||||
tree_config: TreeConfig::default(),
|
||||
shutdown_tx: None,
|
||||
is_dev: true,
|
||||
storage_v2: false,
|
||||
_phantom: Default::default(),
|
||||
import_result_holder: None,
|
||||
import_rlp_path: None,
|
||||
@@ -129,12 +126,6 @@ where
|
||||
self
|
||||
}
|
||||
|
||||
/// Enable v2 storage mode (hashed keys, static file changesets, rocksdb history)
|
||||
pub const fn with_storage_v2(mut self) -> Self {
|
||||
self.storage_v2 = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// Apply setup using pre-imported chain data from RLP file
|
||||
pub async fn apply_with_import<N>(
|
||||
&mut self,
|
||||
@@ -203,28 +194,19 @@ where
|
||||
self.shutdown_tx = Some(shutdown_tx);
|
||||
|
||||
let is_dev = self.is_dev;
|
||||
let storage_v2 = self.storage_v2;
|
||||
let node_count = self.network.node_count;
|
||||
let tree_config = self.tree_config.clone();
|
||||
|
||||
let attributes_generator = Self::create_static_attributes_generator::<N>();
|
||||
|
||||
let mut builder = E2ETestSetupBuilder::<N, _>::new(
|
||||
let result = setup_engine_with_connection::<N>(
|
||||
node_count,
|
||||
Arc::<N::ChainSpec>::new((*chain_spec).clone().into()),
|
||||
is_dev,
|
||||
self.tree_config.clone(),
|
||||
attributes_generator,
|
||||
self.network.connect_nodes,
|
||||
)
|
||||
.with_tree_config_modifier(move |base| {
|
||||
tree_config.clone().with_cross_block_cache_size(base.cross_block_cache_size())
|
||||
})
|
||||
.with_node_config_modifier(move |config| config.set_dev(is_dev))
|
||||
.with_connect_nodes(self.network.connect_nodes);
|
||||
|
||||
if storage_v2 {
|
||||
builder = builder.with_storage_v2();
|
||||
}
|
||||
|
||||
let result = builder.build().await;
|
||||
.await;
|
||||
|
||||
let mut node_clients = Vec::new();
|
||||
match result {
|
||||
|
||||
@@ -10,6 +10,7 @@ use jsonrpsee::core::client::ClientT;
|
||||
use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET};
|
||||
use reth_db::tables;
|
||||
use reth_e2e_test_utils::{transaction::TransactionTestContext, wallet, E2ETestSetupBuilder};
|
||||
use reth_node_core::args::RocksDbArgs;
|
||||
use reth_node_ethereum::EthereumNode;
|
||||
use reth_payload_builder::EthPayloadBuilderAttributes;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
@@ -95,6 +96,22 @@ fn test_attributes_generator(timestamp: u64) -> EthPayloadBuilderAttributes {
|
||||
EthPayloadBuilderAttributes::new(B256::ZERO, attributes)
|
||||
}
|
||||
|
||||
/// Verifies that `RocksDB` CLI defaults are `None` (deferred to storage mode).
|
||||
#[test]
|
||||
fn test_rocksdb_defaults_are_none() {
|
||||
let args = RocksDbArgs::default();
|
||||
|
||||
assert!(args.tx_hash.is_none(), "tx_hash default should be None (deferred to --storage.v2)");
|
||||
assert!(
|
||||
args.storages_history.is_none(),
|
||||
"storages_history default should be None (deferred to --storage.v2)"
|
||||
);
|
||||
assert!(
|
||||
args.account_history.is_none(),
|
||||
"account_history default should be None (deferred to --storage.v2)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Smoke test: node boots with `RocksDB` routing enabled.
|
||||
#[tokio::test]
|
||||
async fn test_rocksdb_node_startup() -> Result<()> {
|
||||
@@ -460,7 +477,7 @@ async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> {
|
||||
///
|
||||
/// This test exercises `unwind_trie_state_from` which previously failed with
|
||||
/// `UnsortedInput` errors because it read changesets directly from MDBX tables
|
||||
/// instead of using storage-aware methods that check `is_v2()`.
|
||||
/// instead of using storage-aware methods that check `storage_changesets_in_static_files`.
|
||||
#[tokio::test]
|
||||
async fn test_rocksdb_reorg_unwind() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
@@ -179,8 +179,6 @@ pub struct TreeConfig {
|
||||
sparse_trie_prune_depth: usize,
|
||||
/// Maximum number of storage tries to retain after pruning.
|
||||
sparse_trie_max_storage_tries: usize,
|
||||
/// Whether to fully disable sparse trie cache pruning between blocks.
|
||||
disable_sparse_trie_cache_pruning: bool,
|
||||
/// Timeout for the state root task before spawning a sequential fallback computation.
|
||||
/// If `Some`, after waiting this duration for the state root task, a sequential state root
|
||||
/// computation is spawned in parallel and whichever finishes first is used.
|
||||
@@ -218,7 +216,6 @@ impl Default for TreeConfig {
|
||||
disable_trie_cache: false,
|
||||
sparse_trie_prune_depth: DEFAULT_SPARSE_TRIE_PRUNE_DEPTH,
|
||||
sparse_trie_max_storage_tries: DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES,
|
||||
disable_sparse_trie_cache_pruning: false,
|
||||
state_root_task_timeout: Some(DEFAULT_STATE_ROOT_TASK_TIMEOUT),
|
||||
}
|
||||
}
|
||||
@@ -284,7 +281,6 @@ impl TreeConfig {
|
||||
disable_trie_cache: false,
|
||||
sparse_trie_prune_depth,
|
||||
sparse_trie_max_storage_tries,
|
||||
disable_sparse_trie_cache_pruning: false,
|
||||
state_root_task_timeout,
|
||||
}
|
||||
}
|
||||
@@ -635,17 +631,6 @@ impl TreeConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether sparse trie cache pruning is disabled.
|
||||
pub const fn disable_sparse_trie_cache_pruning(&self) -> bool {
|
||||
self.disable_sparse_trie_cache_pruning
|
||||
}
|
||||
|
||||
/// Setter for whether to disable sparse trie cache pruning.
|
||||
pub const fn with_disable_sparse_trie_cache_pruning(mut self, value: bool) -> Self {
|
||||
self.disable_sparse_trie_cache_pruning = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns the state root task timeout.
|
||||
pub const fn state_root_task_timeout(&self) -> Option<Duration> {
|
||||
self.state_root_task_timeout
|
||||
|
||||
@@ -20,7 +20,7 @@ use reth_node_types::{BlockTy, NodeTypes};
|
||||
use reth_payload_builder::PayloadBuilderHandle;
|
||||
use reth_provider::{
|
||||
providers::{BlockchainProvider, ProviderNodeTypes},
|
||||
ProviderFactory, StorageSettingsCache,
|
||||
ProviderFactory,
|
||||
};
|
||||
use reth_prune::PrunerWithFactory;
|
||||
use reth_stages_api::{MetricEventsSender, Pipeline};
|
||||
@@ -94,7 +94,6 @@ where
|
||||
if chain_spec.is_optimism() { EngineApiKind::OpStack } else { EngineApiKind::Ethereum };
|
||||
|
||||
let downloader = BasicBlockDownloader::new(client, consensus.clone());
|
||||
let use_hashed_state = provider.cached_storage_settings().use_hashed_state();
|
||||
|
||||
let persistence_handle =
|
||||
PersistenceHandle::<N::Primitives>::spawn_service(provider, pruner, sync_metrics_tx);
|
||||
@@ -112,7 +111,6 @@ where
|
||||
engine_kind,
|
||||
evm_config,
|
||||
changeset_cache,
|
||||
use_hashed_state,
|
||||
);
|
||||
|
||||
let engine_handler = EngineApiRequestHandler::new(to_tree_tx, from_tree);
|
||||
|
||||
@@ -73,7 +73,6 @@ reth-prune-types = { workspace = true, optional = true }
|
||||
reth-stages = { workspace = true, optional = true }
|
||||
reth-static-file = { workspace = true, optional = true }
|
||||
reth-tracing = { workspace = true, optional = true }
|
||||
reth-mdbx-viz = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
# reth
|
||||
@@ -144,14 +143,6 @@ test-utils = [
|
||||
"reth-evm-ethereum/test-utils",
|
||||
"reth-tasks/test-utils",
|
||||
]
|
||||
rocksdb = [
|
||||
"reth-provider/rocksdb",
|
||||
"reth-prune/rocksdb",
|
||||
"reth-stages?/rocksdb",
|
||||
"reth-e2e-test-utils/rocksdb",
|
||||
]
|
||||
edge = ["rocksdb"]
|
||||
pageviz = ["dep:reth-mdbx-viz"]
|
||||
|
||||
[[test]]
|
||||
name = "e2e_testsuite"
|
||||
|
||||
@@ -351,14 +351,6 @@ impl<S: StateProvider, const PREWARM: bool> StateProvider for CachedStateProvide
|
||||
self.state_provider.storage(account, storage_key)
|
||||
}
|
||||
}
|
||||
|
||||
fn storage_by_hashed_key(
|
||||
&self,
|
||||
address: Address,
|
||||
hashed_storage_key: StorageKey,
|
||||
) -> ProviderResult<Option<StorageValue>> {
|
||||
self.state_provider.storage_by_hashed_key(address, hashed_storage_key)
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: BytecodeReader, const PREWARM: bool> BytecodeReader for CachedStateProvider<S, PREWARM> {
|
||||
|
||||
@@ -199,17 +199,6 @@ impl<S: StateProvider> StateProvider for InstrumentedStateProvider<S> {
|
||||
self.record_storage_fetch(start.elapsed());
|
||||
res
|
||||
}
|
||||
|
||||
fn storage_by_hashed_key(
|
||||
&self,
|
||||
address: Address,
|
||||
hashed_storage_key: StorageKey,
|
||||
) -> ProviderResult<Option<StorageValue>> {
|
||||
let start = Instant::now();
|
||||
let res = self.state_provider.storage_by_hashed_key(address, hashed_storage_key);
|
||||
self.record_storage_fetch(start.elapsed());
|
||||
res
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: BytecodeReader> BytecodeReader for InstrumentedStateProvider<S> {
|
||||
|
||||
@@ -12,13 +12,12 @@ use reth_primitives_traits::constants::gas_units::MEGAGAS;
|
||||
use reth_trie::updates::TrieUpdates;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Upper bounds for each gas bucket. The last bucket is a catch-all for
|
||||
/// everything above the final threshold: <5M, 5-10M, 10-20M, 20-30M, 30-40M, >40M.
|
||||
const GAS_BUCKET_THRESHOLDS: [u64; 5] =
|
||||
[5 * MEGAGAS, 10 * MEGAGAS, 20 * MEGAGAS, 30 * MEGAGAS, 40 * MEGAGAS];
|
||||
/// Width of each gas bucket in gas units (10 Mgas).
|
||||
const GAS_BUCKET_SIZE: u64 = 10 * MEGAGAS;
|
||||
|
||||
/// Total number of gas buckets (thresholds + 1 catch-all).
|
||||
const NUM_GAS_BUCKETS: usize = GAS_BUCKET_THRESHOLDS.len() + 1;
|
||||
/// Number of gas buckets. The last bucket is a catch-all for everything above
|
||||
/// `(NUM_GAS_BUCKETS - 1) * GAS_BUCKET_SIZE`.
|
||||
const NUM_GAS_BUCKETS: usize = 5;
|
||||
|
||||
/// Metrics for the `EngineApi`.
|
||||
#[derive(Debug, Default)]
|
||||
@@ -281,23 +280,21 @@ impl GasBucketMetrics {
|
||||
}
|
||||
|
||||
fn bucket_index(gas_used: u64) -> usize {
|
||||
GAS_BUCKET_THRESHOLDS
|
||||
.iter()
|
||||
.position(|&threshold| gas_used < threshold)
|
||||
.unwrap_or(GAS_BUCKET_THRESHOLDS.len())
|
||||
let idx = gas_used / GAS_BUCKET_SIZE;
|
||||
(idx as usize).min(NUM_GAS_BUCKETS - 1)
|
||||
}
|
||||
|
||||
/// Returns a human-readable label like `<5M`, `5-10M`, … `>40M`.
|
||||
/// Returns a human-readable label like `<10M`, `10-20M`, … `>40M`.
|
||||
fn bucket_label(index: usize) -> String {
|
||||
let m = GAS_BUCKET_SIZE / 1_000_000;
|
||||
if index == 0 {
|
||||
let hi = GAS_BUCKET_THRESHOLDS[0] / MEGAGAS;
|
||||
format!("<{hi}M")
|
||||
} else if index < GAS_BUCKET_THRESHOLDS.len() {
|
||||
let lo = GAS_BUCKET_THRESHOLDS[index - 1] / MEGAGAS;
|
||||
let hi = GAS_BUCKET_THRESHOLDS[index] / MEGAGAS;
|
||||
format!("<{m}M")
|
||||
} else if index < NUM_GAS_BUCKETS - 1 {
|
||||
let lo = m * index as u64;
|
||||
let hi = lo + m;
|
||||
format!("{lo}-{hi}M")
|
||||
} else {
|
||||
let lo = GAS_BUCKET_THRESHOLDS[GAS_BUCKET_THRESHOLDS.len() - 1] / MEGAGAS;
|
||||
let lo = m * index as u64;
|
||||
format!(">{lo}M")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ use reth_provider::{
|
||||
BlockExecutionOutput, BlockExecutionResult, BlockReader, ChangeSetReader,
|
||||
DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader,
|
||||
StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader,
|
||||
StorageSettingsCache, TransactionVariant,
|
||||
TransactionVariant,
|
||||
};
|
||||
use reth_revm::database::StateProviderDatabase;
|
||||
use reth_stages_api::ControlFlow;
|
||||
@@ -271,9 +271,6 @@ where
|
||||
evm_config: C,
|
||||
/// Changeset cache for in-memory trie changesets
|
||||
changeset_cache: ChangesetCache,
|
||||
/// Whether the node uses hashed state as canonical storage (v2 mode).
|
||||
/// Cached at construction to avoid threading `StorageSettingsCache` bounds everywhere.
|
||||
use_hashed_state: bool,
|
||||
}
|
||||
|
||||
impl<N, P: Debug, T: PayloadTypes + Debug, V: Debug, C> std::fmt::Debug
|
||||
@@ -299,7 +296,6 @@ where
|
||||
.field("engine_kind", &self.engine_kind)
|
||||
.field("evm_config", &self.evm_config)
|
||||
.field("changeset_cache", &self.changeset_cache)
|
||||
.field("use_hashed_state", &self.use_hashed_state)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -317,8 +313,7 @@ where
|
||||
P::Provider: BlockReader<Block = N::Block, Header = N::BlockHeader>
|
||||
+ StageCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ StorageSettingsCache,
|
||||
+ StorageChangeSetReader,
|
||||
C: ConfigureEvm<Primitives = N> + 'static,
|
||||
T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>,
|
||||
V: EngineValidator<T>,
|
||||
@@ -339,7 +334,6 @@ where
|
||||
engine_kind: EngineApiKind,
|
||||
evm_config: C,
|
||||
changeset_cache: ChangesetCache,
|
||||
use_hashed_state: bool,
|
||||
) -> Self {
|
||||
let (incoming_tx, incoming) = crossbeam_channel::unbounded();
|
||||
|
||||
@@ -361,7 +355,6 @@ where
|
||||
engine_kind,
|
||||
evm_config,
|
||||
changeset_cache,
|
||||
use_hashed_state,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -382,7 +375,6 @@ where
|
||||
kind: EngineApiKind,
|
||||
evm_config: C,
|
||||
changeset_cache: ChangesetCache,
|
||||
use_hashed_state: bool,
|
||||
) -> (Sender<FromEngine<EngineApiRequest<T, N>, N::Block>>, UnboundedReceiver<EngineApiEvent<N>>)
|
||||
{
|
||||
let best_block_number = provider.best_block_number().unwrap_or(0);
|
||||
@@ -415,7 +407,6 @@ where
|
||||
kind,
|
||||
evm_config,
|
||||
changeset_cache,
|
||||
use_hashed_state,
|
||||
);
|
||||
let incoming = task.incoming_tx.clone();
|
||||
spawn_os_thread("engine", || task.run());
|
||||
@@ -2388,12 +2379,7 @@ where
|
||||
|
||||
self.update_reorg_metrics(old.len(), old_first);
|
||||
self.reinsert_reorged_blocks(new.clone());
|
||||
|
||||
// When use_hashed_state is enabled, skip reinserting the old chain — the
|
||||
// bundle state references plain state reverts which don't exist.
|
||||
if !self.use_hashed_state {
|
||||
self.reinsert_reorged_blocks(old.clone());
|
||||
}
|
||||
self.reinsert_reorged_blocks(old.clone());
|
||||
}
|
||||
|
||||
// update the tracked in-memory state with the new chain
|
||||
@@ -2677,23 +2663,8 @@ where
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
#[cfg(feature = "pageviz")]
|
||||
reth_mdbx_viz::pageviz_emit_block_marker(block_num_hash.number, true, 0, 0, 0);
|
||||
|
||||
#[cfg(feature = "pageviz")]
|
||||
let pageviz_start = std::time::Instant::now();
|
||||
|
||||
let executed = execute(&mut self.payload_validator, input, ctx)?;
|
||||
|
||||
#[cfg(feature = "pageviz")]
|
||||
reth_mdbx_viz::pageviz_emit_block_marker(
|
||||
block_num_hash.number,
|
||||
false,
|
||||
executed.recovered_block().senders().len() as u16,
|
||||
pageviz_start.elapsed().as_millis() as u32,
|
||||
executed.recovered_block().gas_used(),
|
||||
);
|
||||
|
||||
// if the parent is the canonical head, we can insert the block as the pending block
|
||||
if self.state.tree_state.canonical_block_hash() == executed.recovered_block().parent_hash()
|
||||
{
|
||||
|
||||
@@ -94,9 +94,6 @@ pub const SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY: usize = 1_000_000;
|
||||
/// 144MB.
|
||||
pub const SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY: usize = 1_000_000;
|
||||
|
||||
/// Blocks with fewer transactions than this skip prewarming, since the fixed overhead of spawning
|
||||
/// prewarm workers exceeds the execution time saved.
|
||||
pub const SMALL_BLOCK_TX_THRESHOLD: usize = 5;
|
||||
/// Type alias for [`PayloadHandle`] returned by payload processor spawn methods.
|
||||
type IteratorPayloadHandle<Evm, I, N> = PayloadHandle<
|
||||
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxIterator<Evm>>::Recovered>,
|
||||
@@ -138,8 +135,6 @@ where
|
||||
sparse_trie_prune_depth: usize,
|
||||
/// Maximum storage tries to retain after pruning.
|
||||
sparse_trie_max_storage_tries: usize,
|
||||
/// Whether sparse trie cache pruning is fully disabled.
|
||||
disable_sparse_trie_cache_pruning: bool,
|
||||
/// Whether to disable cache metrics recording.
|
||||
disable_cache_metrics: bool,
|
||||
}
|
||||
@@ -175,7 +170,6 @@ where
|
||||
prewarm_max_concurrency: config.prewarm_max_concurrency(),
|
||||
sparse_trie_prune_depth: config.sparse_trie_prune_depth(),
|
||||
sparse_trie_max_storage_tries: config.sparse_trie_max_storage_tries(),
|
||||
disable_sparse_trie_cache_pruning: config.disable_sparse_trie_cache_pruning(),
|
||||
disable_cache_metrics: config.disable_cache_metrics(),
|
||||
}
|
||||
}
|
||||
@@ -472,8 +466,7 @@ where
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
{
|
||||
let skip_prewarm =
|
||||
self.disable_transaction_prewarming || env.transaction_count < SMALL_BLOCK_TX_THRESHOLD;
|
||||
let skip_prewarm = self.disable_transaction_prewarming;
|
||||
|
||||
let saved_cache = self.disable_state_cache.not().then(|| self.cache_for(env.parent_hash));
|
||||
|
||||
@@ -553,7 +546,6 @@ where
|
||||
let disable_trie_cache = config.disable_trie_cache();
|
||||
let prune_depth = self.sparse_trie_prune_depth;
|
||||
let max_storage_tries = self.sparse_trie_max_storage_tries;
|
||||
let disable_cache_pruning = self.disable_sparse_trie_cache_pruning;
|
||||
let chunk_size =
|
||||
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size());
|
||||
let executor = self.executor.clone();
|
||||
@@ -650,7 +642,6 @@ where
|
||||
max_storage_tries,
|
||||
SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY,
|
||||
SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY,
|
||||
disable_cache_pruning,
|
||||
);
|
||||
trie_metrics
|
||||
.into_trie_for_reuse_duration_histogram
|
||||
|
||||
@@ -1541,7 +1541,6 @@ mod tests {
|
||||
providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory,
|
||||
BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, LatestStateProvider,
|
||||
PruneCheckpointReader, StageCheckpointReader, StateProviderBox, StorageChangeSetReader,
|
||||
StorageSettingsCache,
|
||||
};
|
||||
use reth_trie::MultiProof;
|
||||
use reth_trie_db::ChangesetCache;
|
||||
@@ -1563,7 +1562,6 @@ mod tests {
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ StorageSettingsCache
|
||||
+ BlockNumReader,
|
||||
> + Clone
|
||||
+ Send
|
||||
@@ -1583,10 +1581,7 @@ mod tests {
|
||||
fn create_cached_provider<F>(factory: F) -> CachedStateProvider<StateProviderBox>
|
||||
where
|
||||
F: DatabaseProviderFactory<
|
||||
Provider: BlockReader
|
||||
+ StageCheckpointReader
|
||||
+ PruneCheckpointReader
|
||||
+ reth_provider::StorageSettingsCache,
|
||||
Provider: BlockReader + StageCheckpointReader + PruneCheckpointReader,
|
||||
> + Clone
|
||||
+ Send
|
||||
+ 'static,
|
||||
|
||||
@@ -72,7 +72,6 @@ where
|
||||
max_storage_tries: usize,
|
||||
max_nodes_capacity: usize,
|
||||
max_values_capacity: usize,
|
||||
disable_pruning: bool,
|
||||
) -> (SparseStateTrie<A, S>, DeferredDrops) {
|
||||
match self {
|
||||
Self::Cleared(task) => task.into_cleared_trie(max_nodes_capacity, max_values_capacity),
|
||||
@@ -81,7 +80,6 @@ where
|
||||
max_storage_tries,
|
||||
max_nodes_capacity,
|
||||
max_values_capacity,
|
||||
disable_pruning,
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -358,23 +356,16 @@ where
|
||||
/// Prunes and shrinks the trie for reuse in the next payload built on top of this one.
|
||||
///
|
||||
/// Should be called after the state root result has been sent.
|
||||
///
|
||||
/// When `disable_pruning` is true, the trie is preserved without any node pruning,
|
||||
/// storage trie eviction, or capacity shrinking, keeping the full cache intact for
|
||||
/// benchmarking purposes.
|
||||
pub(super) fn into_trie_for_reuse(
|
||||
self,
|
||||
prune_depth: usize,
|
||||
max_storage_tries: usize,
|
||||
max_nodes_capacity: usize,
|
||||
max_values_capacity: usize,
|
||||
disable_pruning: bool,
|
||||
) -> (SparseStateTrie<A, S>, DeferredDrops) {
|
||||
let Self { mut trie, .. } = self;
|
||||
if !disable_pruning {
|
||||
trie.prune(prune_depth, max_storage_tries);
|
||||
trie.shrink_to(max_nodes_capacity, max_values_capacity);
|
||||
}
|
||||
trie.prune(prune_depth, max_storage_tries);
|
||||
trie.shrink_to(max_nodes_capacity, max_values_capacity);
|
||||
let deferred = trie.take_deferred_drops();
|
||||
(trie, deferred)
|
||||
}
|
||||
@@ -416,9 +407,7 @@ where
|
||||
let update = match message {
|
||||
Ok(m) => m,
|
||||
Err(_) => {
|
||||
return Err(ParallelStateRootError::Other(
|
||||
"updates channel disconnected before state root calculation".to_string(),
|
||||
))
|
||||
break
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ use reth_provider::{
|
||||
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader,
|
||||
ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider,
|
||||
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
|
||||
StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache,
|
||||
StateProviderFactory, StateReader, StorageChangeSetReader,
|
||||
};
|
||||
use reth_revm::db::{states::bundle_state::BundleRetention, State};
|
||||
use reth_trie::{updates::TrieUpdates, HashedPostState, StateRoot};
|
||||
@@ -146,8 +146,7 @@ where
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader
|
||||
+ StorageSettingsCache,
|
||||
+ BlockNumReader,
|
||||
> + BlockReader<Header = N::BlockHeader>
|
||||
+ ChangeSetReader
|
||||
+ BlockNumReader
|
||||
@@ -1527,8 +1526,7 @@ where
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader
|
||||
+ StorageSettingsCache,
|
||||
+ BlockNumReader,
|
||||
> + BlockReader<Header = N::BlockHeader>
|
||||
+ StateProviderFactory
|
||||
+ StateReader
|
||||
|
||||
@@ -221,7 +221,6 @@ impl TestHarness {
|
||||
EngineApiKind::Ethereum,
|
||||
evm_config,
|
||||
changeset_cache,
|
||||
provider.cached_storage_settings().use_hashed_state(),
|
||||
);
|
||||
|
||||
let block_builder = TestBlockBuilder::default().with_chain_spec((*chain_spec).clone());
|
||||
|
||||
@@ -2,15 +2,13 @@
|
||||
|
||||
mod fcu_finalized_blocks;
|
||||
|
||||
use alloy_rpc_types_engine::PayloadStatusEnum;
|
||||
use eyre::Result;
|
||||
use reth_chainspec::{ChainSpecBuilder, MAINNET};
|
||||
use reth_e2e_test_utils::testsuite::{
|
||||
actions::{
|
||||
BlockReference, CaptureBlock, CompareNodeChainTips, CreateFork, ExpectFcuStatus,
|
||||
MakeCanonical, ProduceBlocks, ProduceBlocksLocally, ProduceInvalidBlocks, ReorgTo,
|
||||
SelectActiveNode, SendForkchoiceUpdate, SendNewPayloads, SetForkBase, UpdateBlockInfo,
|
||||
ValidateCanonicalTag, WaitForSync,
|
||||
CaptureBlock, CompareNodeChainTips, CreateFork, ExpectFcuStatus, MakeCanonical,
|
||||
ProduceBlocks, ProduceBlocksLocally, ProduceInvalidBlocks, ReorgTo, SelectActiveNode,
|
||||
SendNewPayloads, UpdateBlockInfo, ValidateCanonicalTag, WaitForSync,
|
||||
},
|
||||
setup::{NetworkSetup, Setup},
|
||||
TestBuilder,
|
||||
@@ -41,14 +39,6 @@ fn default_engine_tree_setup() -> Setup<EthEngineTypes> {
|
||||
)
|
||||
}
|
||||
|
||||
/// Creates a v2 storage mode setup for engine tree e2e tests.
|
||||
///
|
||||
/// v2 mode uses keccak256-hashed slot keys in static file changesets and rocksdb history
|
||||
/// instead of plain keys in MDBX.
|
||||
fn v2_engine_tree_setup() -> Setup<EthEngineTypes> {
|
||||
default_engine_tree_setup().with_storage_v2()
|
||||
}
|
||||
|
||||
/// Test that verifies forkchoice update and canonical chain insertion functionality.
|
||||
#[tokio::test]
|
||||
async fn test_engine_tree_fcu_canon_chain_insertion_e2e() -> Result<()> {
|
||||
@@ -344,152 +334,3 @@ async fn test_engine_tree_live_sync_transition_eventually_canonical_e2e() -> Res
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ==================== v2 storage mode variants ====================
|
||||
|
||||
/// v2 variant: Verifies forkchoice update and canonical chain insertion in v2 storage mode.
|
||||
///
|
||||
/// Exercises the full `save_blocks` → `write_state` → static file changeset path with hashed keys.
|
||||
#[tokio::test]
|
||||
async fn test_engine_tree_fcu_canon_chain_insertion_v2_e2e() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let test = TestBuilder::new()
|
||||
.with_setup(v2_engine_tree_setup())
|
||||
.with_action(ProduceBlocks::<EthEngineTypes>::new(1))
|
||||
.with_action(MakeCanonical::new())
|
||||
.with_action(ProduceBlocks::<EthEngineTypes>::new(3))
|
||||
.with_action(MakeCanonical::new());
|
||||
|
||||
test.run::<EthereumNode>().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// v2 variant: Verifies forkchoice update with a reorg where all blocks are already available.
|
||||
///
|
||||
/// Exercises `write_state_reverts` path with hashed changeset keys during CL-driven reorgs.
|
||||
#[tokio::test]
|
||||
async fn test_engine_tree_fcu_reorg_with_all_blocks_v2_e2e() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let test = TestBuilder::new()
|
||||
.with_setup(v2_engine_tree_setup())
|
||||
.with_action(ProduceBlocks::<EthEngineTypes>::new(5))
|
||||
.with_action(MakeCanonical::new())
|
||||
.with_action(CreateFork::<EthEngineTypes>::new(2, 3))
|
||||
.with_action(CaptureBlock::new("fork_tip"))
|
||||
.with_action(ReorgTo::<EthEngineTypes>::new_from_tag("fork_tip"));
|
||||
|
||||
test.run::<EthereumNode>().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// v2 variant: Verifies progressive canonical chain extension in v2 storage mode.
|
||||
#[tokio::test]
|
||||
async fn test_engine_tree_fcu_extends_canon_chain_v2_e2e() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let test = TestBuilder::new()
|
||||
.with_setup(v2_engine_tree_setup())
|
||||
.with_action(ProduceBlocks::<EthEngineTypes>::new(1))
|
||||
.with_action(MakeCanonical::new())
|
||||
.with_action(ProduceBlocks::<EthEngineTypes>::new(10))
|
||||
.with_action(CaptureBlock::new("target_block"))
|
||||
.with_action(ReorgTo::<EthEngineTypes>::new_from_tag("target_block"))
|
||||
.with_action(MakeCanonical::new());
|
||||
|
||||
test.run::<EthereumNode>().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Creates a 2-node setup for disk-level reorg testing.
|
||||
///
|
||||
/// Uses unconnected nodes so fork blocks can be produced independently on Node 1 and then
|
||||
/// sent to Node 0 via newPayload only (no FCU), keeping Node 0's persisted chain intact
|
||||
/// until the final `ReorgTo` triggers `find_disk_reorg`.
|
||||
fn disk_reorg_setup(storage_v2: bool) -> Setup<EthEngineTypes> {
|
||||
let mut setup = Setup::default()
|
||||
.with_chain_spec(Arc::new(
|
||||
ChainSpecBuilder::default()
|
||||
.chain(MAINNET.chain)
|
||||
.genesis(
|
||||
serde_json::from_str(include_str!(
|
||||
"../../../../e2e-test-utils/src/testsuite/assets/genesis.json"
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.cancun_activated()
|
||||
.build(),
|
||||
))
|
||||
.with_network(NetworkSetup::multi_node_unconnected(2))
|
||||
.with_tree_config(
|
||||
TreeConfig::default().with_legacy_state_root(false).with_has_enough_parallelism(true),
|
||||
);
|
||||
if storage_v2 {
|
||||
setup = setup.with_storage_v2();
|
||||
}
|
||||
setup
|
||||
}
|
||||
|
||||
/// Builds a disk-level reorg test scenario.
|
||||
///
|
||||
/// 1. Both nodes receive 3 shared blocks
|
||||
/// 2. Node 0 extends to 10 blocks locally (persisted to disk)
|
||||
/// 3. Node 1 builds an 8-block fork from block 3 (its canonical head)
|
||||
/// 4. Fork blocks are sent to Node 0 via newPayload (no FCU, old chain stays on disk)
|
||||
/// 5. FCU to fork tip on Node 0 triggers `find_disk_reorg` → `RemoveBlocksAbove(3)`
|
||||
fn disk_reorg_test(storage_v2: bool) -> TestBuilder<EthEngineTypes> {
|
||||
TestBuilder::new()
|
||||
.with_setup(disk_reorg_setup(storage_v2))
|
||||
.with_action(SelectActiveNode::new(0))
|
||||
.with_action(ProduceBlocks::<EthEngineTypes>::new(3))
|
||||
.with_action(MakeCanonical::new())
|
||||
.with_action(ProduceBlocksLocally::<EthEngineTypes>::new(7))
|
||||
.with_action(MakeCanonical::with_active_node())
|
||||
.with_action(SelectActiveNode::new(1))
|
||||
.with_action(SetForkBase::new(3))
|
||||
.with_action(ProduceBlocksLocally::<EthEngineTypes>::new(8))
|
||||
.with_action(MakeCanonical::with_active_node())
|
||||
.with_action(CaptureBlock::new("fork_tip"))
|
||||
.with_action(
|
||||
SendNewPayloads::<EthEngineTypes>::new()
|
||||
.with_source_node(1)
|
||||
.with_target_node(0)
|
||||
.with_start_block(4)
|
||||
.with_total_blocks(8),
|
||||
)
|
||||
.with_action(
|
||||
SendForkchoiceUpdate::<EthEngineTypes>::new(
|
||||
BlockReference::Tag("fork_tip".into()),
|
||||
BlockReference::Tag("fork_tip".into()),
|
||||
BlockReference::Tag("fork_tip".into()),
|
||||
)
|
||||
.with_expected_status(PayloadStatusEnum::Valid)
|
||||
.with_node_idx(0),
|
||||
)
|
||||
}
|
||||
|
||||
/// Verifies disk-level reorg in v1 (plain key) storage mode.
|
||||
///
|
||||
/// Confirms `find_disk_reorg()` detects persisted blocks on the wrong fork and calls
|
||||
/// `RemoveBlocksAbove` to truncate, then re-persists the correct fork chain.
|
||||
#[tokio::test]
|
||||
async fn test_engine_tree_disk_reorg_v1_e2e() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
disk_reorg_test(false).run::<EthereumNode>().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// v2 variant: Verifies disk-level reorg in v2 storage mode.
|
||||
///
|
||||
/// Same scenario as v1 but with hashed changeset keys in static files and rocksdb history.
|
||||
/// Exercises `find_disk_reorg()` → `RemoveBlocksAbove` with v2 hashed key format.
|
||||
#[tokio::test]
|
||||
async fn test_engine_tree_disk_reorg_v2_e2e() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
disk_reorg_test(true).run::<EthereumNode>().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -273,6 +273,7 @@ where
|
||||
gas_limit: payload.payload.gas_limit(),
|
||||
basefee: payload.payload.saturated_base_fee_per_gas(),
|
||||
blob_excess_gas_and_price,
|
||||
slot_num: 0,
|
||||
};
|
||||
|
||||
Ok(EvmEnv { cfg_env, block_env })
|
||||
|
||||
@@ -108,8 +108,7 @@ impl<'a, DB: Database, I: Inspector<EthEvmContext<&'a mut State<DB>>>> BlockExec
|
||||
result: ResultAndState::new(
|
||||
ExecutionResult::Success {
|
||||
reason: SuccessReason::Return,
|
||||
gas_used: 0,
|
||||
gas_refunded: 0,
|
||||
gas: Default::default(),
|
||||
logs: vec![],
|
||||
output: Output::Call(Bytes::from(vec![])),
|
||||
},
|
||||
|
||||
@@ -18,7 +18,6 @@ use reth_provider::{
|
||||
};
|
||||
use reth_revm::database::StateProviderDatabase;
|
||||
use reth_testing_utils::generators::sign_tx_with_key_pair;
|
||||
use reth_trie_common::KeccakKeyHasher;
|
||||
use secp256k1::Keypair;
|
||||
|
||||
pub(crate) fn to_execution_outcome(
|
||||
@@ -78,9 +77,12 @@ where
|
||||
let execution_outcome = to_execution_outcome(block.number(), &block_execution_output);
|
||||
|
||||
// Commit the block's execution outcome to the database
|
||||
let hashed_state = execution_outcome.hash_state_slow::<KeccakKeyHasher>().into_sorted();
|
||||
let provider_rw = provider_factory.provider_rw()?;
|
||||
provider_rw.append_blocks_with_state(vec![block.clone()], &execution_outcome, hashed_state)?;
|
||||
provider_rw.append_blocks_with_state(
|
||||
vec![block.clone()],
|
||||
&execution_outcome,
|
||||
Default::default(),
|
||||
)?;
|
||||
provider_rw.commit()?;
|
||||
|
||||
Ok(block_execution_output)
|
||||
@@ -208,12 +210,11 @@ where
|
||||
execution_outcome.state_mut().reverts.sort();
|
||||
|
||||
// Commit the block's execution outcome to the database
|
||||
let hashed_state = execution_outcome.hash_state_slow::<KeccakKeyHasher>().into_sorted();
|
||||
let provider_rw = provider_factory.provider_rw()?;
|
||||
provider_rw.append_blocks_with_state(
|
||||
vec![block1.clone(), block2.clone()],
|
||||
&execution_outcome,
|
||||
hashed_state,
|
||||
Default::default(),
|
||||
)?;
|
||||
provider_rw.commit()?;
|
||||
|
||||
|
||||
@@ -589,7 +589,7 @@ impl EthMessageID {
|
||||
|
||||
/// Returns the max value for the given version.
|
||||
pub const fn max(version: EthVersion) -> u8 {
|
||||
if version as u8 >= EthVersion::Eth69 as u8 {
|
||||
if version.is_eth69() {
|
||||
Self::BlockRangeUpdate.to_u8()
|
||||
} else {
|
||||
Self::Receipts.to_u8()
|
||||
@@ -937,13 +937,6 @@ mod tests {
|
||||
assert!(matches!(decoded, StatusMessage::Legacy(s) if s == status));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eth_message_id_max_includes_block_range_update() {
|
||||
assert_eq!(EthMessageID::max(EthVersion::Eth69), EthMessageID::BlockRangeUpdate.to_u8(),);
|
||||
assert_eq!(EthMessageID::max(EthVersion::Eth70), EthMessageID::BlockRangeUpdate.to_u8(),);
|
||||
assert_eq!(EthMessageID::max(EthVersion::Eth68), EthMessageID::Receipts.to_u8());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_status_rejects_non_status() {
|
||||
let msg = EthMessage::<EthNetworkPrimitives>::GetBlockBodies(RequestPair {
|
||||
|
||||
@@ -8,7 +8,7 @@ use reth_eth_wire_types::{
|
||||
};
|
||||
use reth_ethereum_forks::ForkId;
|
||||
use reth_network_p2p::error::{RequestError, RequestResult};
|
||||
use reth_network_peers::{NodeRecord, PeerId};
|
||||
use reth_network_peers::PeerId;
|
||||
use reth_network_types::{PeerAddr, PeerKind};
|
||||
use reth_tokio_util::EventStream;
|
||||
use std::{
|
||||
@@ -152,13 +152,8 @@ pub trait NetworkEventListenerProvider: NetworkPeersEvents {
|
||||
pub enum DiscoveryEvent {
|
||||
/// Discovered a node
|
||||
NewNode(DiscoveredEvent),
|
||||
/// Retrieved a [`ForkId`] from the peer via ENR request.
|
||||
///
|
||||
/// Contains the full [`NodeRecord`] (peer ID + address) and the reported [`ForkId`].
|
||||
/// Used to verify fork compatibility before admitting the peer.
|
||||
///
|
||||
/// See also <https://eips.ethereum.org/EIPS/eip-868>
|
||||
EnrForkId(NodeRecord, ForkId),
|
||||
/// Retrieved a [`ForkId`] from the peer via ENR request, See <https://eips.ethereum.org/EIPS/eip-868>
|
||||
EnrForkId(PeerId, ForkId),
|
||||
}
|
||||
|
||||
/// Represents events related to peer discovery in the network.
|
||||
|
||||
@@ -172,11 +172,6 @@ pub struct PeersConfig {
|
||||
/// IPs within the specified CIDR ranges will be allowed.
|
||||
#[cfg_attr(feature = "serde", serde(skip))]
|
||||
pub ip_filter: IpFilter,
|
||||
/// If true, discovered peers without a confirmed ENR [`ForkId`](alloy_eip2124::ForkId)
|
||||
/// (EIP-868) will not be added to the peer set until their fork ID is verified.
|
||||
///
|
||||
/// This filters out peers from other networks that pollute the discovery table.
|
||||
pub enforce_enr_fork_id: bool,
|
||||
}
|
||||
|
||||
impl Default for PeersConfig {
|
||||
@@ -196,7 +191,6 @@ impl Default for PeersConfig {
|
||||
max_backoff_count: 5,
|
||||
incoming_ip_throttle_duration: INBOUND_IP_THROTTLE_DURATION,
|
||||
ip_filter: IpFilter::default(),
|
||||
enforce_enr_fork_id: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -320,13 +314,6 @@ impl PeersConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// If set, discovered peers without a confirmed ENR [`ForkId`](alloy_eip2124::ForkId) will not
|
||||
/// be added to the peer set until their fork ID is verified via EIP-868.
|
||||
pub const fn with_enforce_enr_fork_id(mut self, enforce: bool) -> Self {
|
||||
self.enforce_enr_fork_id = enforce;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns settings for testing
|
||||
#[cfg(any(test, feature = "test-utils"))]
|
||||
pub fn test() -> Self {
|
||||
|
||||
@@ -240,7 +240,7 @@ impl Discovery {
|
||||
self.on_node_record_update(record, None);
|
||||
}
|
||||
DiscoveryUpdate::EnrForkId(node, fork_id) => {
|
||||
self.queued_events.push_back(DiscoveryEvent::EnrForkId(node, fork_id))
|
||||
self.queued_events.push_back(DiscoveryEvent::EnrForkId(node.id, fork_id))
|
||||
}
|
||||
DiscoveryUpdate::Removed(peer_id) => {
|
||||
self.discovered_nodes.remove(&peer_id);
|
||||
|
||||
@@ -25,7 +25,7 @@ use crate::{
|
||||
listener::ConnectionListener,
|
||||
message::{NewBlockMessage, PeerMessage},
|
||||
metrics::{
|
||||
BackedOffPeersMetrics, ClosedSessionsMetrics, DirectionalDisconnectMetrics, NetworkMetrics,
|
||||
BackedOffPeersMetrics, ClosedSessionsMetrics, DisconnectMetrics, NetworkMetrics,
|
||||
PendingSessionFailureMetrics, NETWORK_POOL_TRANSACTIONS_SCOPE,
|
||||
},
|
||||
network::{NetworkHandle, NetworkHandleMessage},
|
||||
@@ -140,8 +140,8 @@ pub struct NetworkManager<N: NetworkPrimitives = EthNetworkPrimitives> {
|
||||
num_active_peers: Arc<AtomicUsize>,
|
||||
/// Metrics for the Network
|
||||
metrics: NetworkMetrics,
|
||||
/// Disconnect metrics for the Network, split by connection direction.
|
||||
disconnect_metrics: DirectionalDisconnectMetrics,
|
||||
/// Disconnect metrics for the Network
|
||||
disconnect_metrics: DisconnectMetrics,
|
||||
/// Closed sessions metrics, split by direction.
|
||||
closed_sessions_metrics: ClosedSessionsMetrics,
|
||||
/// Pending session failure metrics, split by direction.
|
||||
@@ -864,9 +864,6 @@ impl<N: NetworkPrimitives> NetworkManager<N> {
|
||||
"Session disconnected"
|
||||
);
|
||||
|
||||
// Capture direction before state is reset to Idle
|
||||
let is_inbound = self.swarm.state().peers().is_inbound_peer(&peer_id);
|
||||
|
||||
let reason = if let Some(ref err) = error {
|
||||
// If the connection was closed due to an error, we report
|
||||
// the peer
|
||||
@@ -890,11 +887,7 @@ impl<N: NetworkPrimitives> NetworkManager<N> {
|
||||
self.update_active_connection_metrics();
|
||||
|
||||
if let Some(reason) = reason {
|
||||
if is_inbound {
|
||||
self.disconnect_metrics.increment_inbound(reason);
|
||||
} else {
|
||||
self.disconnect_metrics.increment_outbound(reason);
|
||||
}
|
||||
self.disconnect_metrics.increment(reason);
|
||||
}
|
||||
self.metrics
|
||||
.backed_off_peers
|
||||
@@ -917,7 +910,7 @@ impl<N: NetworkPrimitives> NetworkManager<N> {
|
||||
.on_incoming_pending_session_dropped(remote_addr, err);
|
||||
self.pending_session_failure_metrics.inbound.increment(1);
|
||||
if let Some(reason) = err.as_disconnected() {
|
||||
self.disconnect_metrics.increment_inbound(reason);
|
||||
self.disconnect_metrics.increment(reason);
|
||||
}
|
||||
} else {
|
||||
self.swarm
|
||||
@@ -950,7 +943,7 @@ impl<N: NetworkPrimitives> NetworkManager<N> {
|
||||
BackoffReason::from_disconnect(err.as_disconnected()),
|
||||
);
|
||||
if let Some(reason) = err.as_disconnected() {
|
||||
self.disconnect_metrics.increment_outbound(reason);
|
||||
self.disconnect_metrics.increment(reason);
|
||||
}
|
||||
} else {
|
||||
self.swarm
|
||||
|
||||
@@ -301,34 +301,6 @@ macro_rules! duration_metered_exec {
|
||||
}};
|
||||
}
|
||||
|
||||
/// Direction-aware wrapper for disconnect metrics.
|
||||
///
|
||||
/// Tracks disconnect reasons for inbound and outbound connections separately, in addition to
|
||||
/// the combined (legacy) counters.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct DirectionalDisconnectMetrics {
|
||||
/// Combined disconnect metrics (all directions).
|
||||
pub(crate) total: DisconnectMetrics,
|
||||
/// Disconnect metrics for inbound connections only.
|
||||
pub(crate) inbound: InboundDisconnectMetrics,
|
||||
/// Disconnect metrics for outbound connections only.
|
||||
pub(crate) outbound: OutboundDisconnectMetrics,
|
||||
}
|
||||
|
||||
impl DirectionalDisconnectMetrics {
|
||||
/// Increments disconnect counters for an inbound connection.
|
||||
pub(crate) fn increment_inbound(&self, reason: DisconnectReason) {
|
||||
self.total.increment(reason);
|
||||
self.inbound.increment(reason);
|
||||
}
|
||||
|
||||
/// Increments disconnect counters for an outbound connection.
|
||||
pub(crate) fn increment_outbound(&self, reason: DisconnectReason) {
|
||||
self.total.increment(reason);
|
||||
self.outbound.increment(reason);
|
||||
}
|
||||
}
|
||||
|
||||
/// Metrics for Disconnection types
|
||||
///
|
||||
/// These are just counters, and ideally we would implement these metrics on a peer-by-peer basis,
|
||||
@@ -398,144 +370,6 @@ impl DisconnectMetrics {
|
||||
}
|
||||
}
|
||||
|
||||
/// Disconnect metrics scoped to inbound connections only.
|
||||
///
|
||||
/// These counters track disconnect reasons exclusively for sessions that were initiated by
|
||||
/// remote peers connecting to this node. This helps operators distinguish between being rejected
|
||||
/// by remote peers (outbound) vs rejecting incoming peers (inbound).
|
||||
#[derive(Metrics)]
|
||||
#[metrics(scope = "network.inbound")]
|
||||
pub struct InboundDisconnectMetrics {
|
||||
/// Number of inbound peer disconnects due to `DisconnectRequested` (0x00)
|
||||
pub(crate) disconnect_requested: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `TcpSubsystemError` (0x01)
|
||||
pub(crate) tcp_subsystem_error: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `ProtocolBreach` (0x02)
|
||||
pub(crate) protocol_breach: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `UselessPeer` (0x03)
|
||||
pub(crate) useless_peer: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `TooManyPeers` (0x04)
|
||||
pub(crate) too_many_peers: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `AlreadyConnected` (0x05)
|
||||
pub(crate) already_connected: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `IncompatibleP2PProtocolVersion` (0x06)
|
||||
pub(crate) incompatible: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `NullNodeIdentity` (0x07)
|
||||
pub(crate) null_node_identity: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `ClientQuitting` (0x08)
|
||||
pub(crate) client_quitting: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `UnexpectedHandshakeIdentity` (0x09)
|
||||
pub(crate) unexpected_identity: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `ConnectedToSelf` (0x0a)
|
||||
pub(crate) connected_to_self: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `PingTimeout` (0x0b)
|
||||
pub(crate) ping_timeout: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `SubprotocolSpecific` (0x10)
|
||||
pub(crate) subprotocol_specific: Counter,
|
||||
}
|
||||
|
||||
impl InboundDisconnectMetrics {
|
||||
/// Increments the proper counter for the given disconnect reason
|
||||
pub(crate) fn increment(&self, reason: DisconnectReason) {
|
||||
match reason {
|
||||
DisconnectReason::DisconnectRequested => self.disconnect_requested.increment(1),
|
||||
DisconnectReason::TcpSubsystemError => self.tcp_subsystem_error.increment(1),
|
||||
DisconnectReason::ProtocolBreach => self.protocol_breach.increment(1),
|
||||
DisconnectReason::UselessPeer => self.useless_peer.increment(1),
|
||||
DisconnectReason::TooManyPeers => self.too_many_peers.increment(1),
|
||||
DisconnectReason::AlreadyConnected => self.already_connected.increment(1),
|
||||
DisconnectReason::IncompatibleP2PProtocolVersion => self.incompatible.increment(1),
|
||||
DisconnectReason::NullNodeIdentity => self.null_node_identity.increment(1),
|
||||
DisconnectReason::ClientQuitting => self.client_quitting.increment(1),
|
||||
DisconnectReason::UnexpectedHandshakeIdentity => self.unexpected_identity.increment(1),
|
||||
DisconnectReason::ConnectedToSelf => self.connected_to_self.increment(1),
|
||||
DisconnectReason::PingTimeout => self.ping_timeout.increment(1),
|
||||
DisconnectReason::SubprotocolSpecific => self.subprotocol_specific.increment(1),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Disconnect metrics scoped to outbound connections only.
|
||||
///
|
||||
/// These counters track disconnect reasons exclusively for sessions that this node initiated
|
||||
/// by dialing out to remote peers. A high `too_many_peers` count here indicates remote peers
|
||||
/// are rejecting our connection attempts because they are full.
|
||||
#[derive(Metrics)]
|
||||
#[metrics(scope = "network.outbound")]
|
||||
pub struct OutboundDisconnectMetrics {
|
||||
/// Number of outbound peer disconnects due to `DisconnectRequested` (0x00)
|
||||
pub(crate) disconnect_requested: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `TcpSubsystemError` (0x01)
|
||||
pub(crate) tcp_subsystem_error: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `ProtocolBreach` (0x02)
|
||||
pub(crate) protocol_breach: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `UselessPeer` (0x03)
|
||||
pub(crate) useless_peer: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `TooManyPeers` (0x04)
|
||||
pub(crate) too_many_peers: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `AlreadyConnected` (0x05)
|
||||
pub(crate) already_connected: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `IncompatibleP2PProtocolVersion` (0x06)
|
||||
pub(crate) incompatible: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `NullNodeIdentity` (0x07)
|
||||
pub(crate) null_node_identity: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `ClientQuitting` (0x08)
|
||||
pub(crate) client_quitting: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `UnexpectedHandshakeIdentity` (0x09)
|
||||
pub(crate) unexpected_identity: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `ConnectedToSelf` (0x0a)
|
||||
pub(crate) connected_to_self: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `PingTimeout` (0x0b)
|
||||
pub(crate) ping_timeout: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `SubprotocolSpecific` (0x10)
|
||||
pub(crate) subprotocol_specific: Counter,
|
||||
}
|
||||
|
||||
impl OutboundDisconnectMetrics {
|
||||
/// Increments the proper counter for the given disconnect reason
|
||||
pub(crate) fn increment(&self, reason: DisconnectReason) {
|
||||
match reason {
|
||||
DisconnectReason::DisconnectRequested => self.disconnect_requested.increment(1),
|
||||
DisconnectReason::TcpSubsystemError => self.tcp_subsystem_error.increment(1),
|
||||
DisconnectReason::ProtocolBreach => self.protocol_breach.increment(1),
|
||||
DisconnectReason::UselessPeer => self.useless_peer.increment(1),
|
||||
DisconnectReason::TooManyPeers => self.too_many_peers.increment(1),
|
||||
DisconnectReason::AlreadyConnected => self.already_connected.increment(1),
|
||||
DisconnectReason::IncompatibleP2PProtocolVersion => self.incompatible.increment(1),
|
||||
DisconnectReason::NullNodeIdentity => self.null_node_identity.increment(1),
|
||||
DisconnectReason::ClientQuitting => self.client_quitting.increment(1),
|
||||
DisconnectReason::UnexpectedHandshakeIdentity => self.unexpected_identity.increment(1),
|
||||
DisconnectReason::ConnectedToSelf => self.connected_to_self.increment(1),
|
||||
DisconnectReason::PingTimeout => self.ping_timeout.increment(1),
|
||||
DisconnectReason::SubprotocolSpecific => self.subprotocol_specific.increment(1),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Metrics for the `EthRequestHandler`
|
||||
#[derive(Metrics)]
|
||||
#[metrics(scope = "network")]
|
||||
|
||||
@@ -92,9 +92,6 @@ pub struct PeersManager {
|
||||
incoming_ip_throttle_duration: Duration,
|
||||
/// IP address filter for restricting network connections to specific IP ranges.
|
||||
ip_filter: reth_net_banlist::IpFilter,
|
||||
/// If true, discovered peers without a confirmed ENR fork ID will not be added until their
|
||||
/// fork ID is verified via EIP-868.
|
||||
enforce_enr_fork_id: bool,
|
||||
}
|
||||
|
||||
impl PeersManager {
|
||||
@@ -114,7 +111,6 @@ impl PeersManager {
|
||||
max_backoff_count,
|
||||
incoming_ip_throttle_duration,
|
||||
ip_filter,
|
||||
enforce_enr_fork_id,
|
||||
} = config;
|
||||
let (manager_tx, handle_rx) = mpsc::unbounded_channel();
|
||||
let now = Instant::now();
|
||||
@@ -171,7 +167,6 @@ impl PeersManager {
|
||||
net_connection_state: NetworkConnectionState::default(),
|
||||
incoming_ip_throttle_duration,
|
||||
ip_filter,
|
||||
enforce_enr_fork_id,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,11 +175,6 @@ impl PeersManager {
|
||||
PeersHandle::new(self.manager_tx.clone())
|
||||
}
|
||||
|
||||
/// Returns `true` if discovered peers must have a confirmed ENR fork ID before being added.
|
||||
pub(crate) const fn enforce_enr_fork_id(&self) -> bool {
|
||||
self.enforce_enr_fork_id
|
||||
}
|
||||
|
||||
/// Returns the number of peers in the peer set
|
||||
#[inline]
|
||||
pub(crate) fn num_known_peers(&self) -> usize {
|
||||
@@ -218,13 +208,6 @@ impl PeersManager {
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns `true` if the given peer is connected via an inbound session.
|
||||
pub(crate) fn is_inbound_peer(&self, peer_id: &PeerId) -> bool {
|
||||
self.peers.get(peer_id).is_some_and(|p| {
|
||||
matches!(p.state, PeerConnectionState::In | PeerConnectionState::DisconnectingIn)
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns an iterator over all peer ids for peers with the given kind
|
||||
pub(crate) fn peers_by_kind(&self, kind: PeerKind) -> impl Iterator<Item = PeerId> + '_ {
|
||||
self.peers.iter().filter_map(move |(peer_id, peer)| (peer.kind == kind).then_some(*peer_id))
|
||||
@@ -755,6 +738,17 @@ impl PeersManager {
|
||||
}
|
||||
}
|
||||
|
||||
/// Called as follow-up for a discovered peer.
|
||||
///
|
||||
/// The [`ForkId`] is retrieved from an ENR record that the peer announces over the discovery
|
||||
/// protocol
|
||||
pub(crate) fn set_discovered_fork_id(&mut self, peer_id: PeerId, fork_id: ForkId) {
|
||||
if let Some(peer) = self.peers.get_mut(&peer_id) {
|
||||
trace!(target: "net::peers", ?peer_id, ?fork_id, "set discovered fork id");
|
||||
peer.fork_id = Some(Box::new(fork_id));
|
||||
}
|
||||
}
|
||||
|
||||
/// Called for a newly discovered peer.
|
||||
///
|
||||
/// If the peer already exists, then the address, kind and `fork_id` will be updated.
|
||||
|
||||
@@ -332,19 +332,9 @@ impl<N: NetworkPrimitives> NetworkState<N> {
|
||||
fork_id,
|
||||
});
|
||||
}
|
||||
DiscoveryEvent::EnrForkId(record, fork_id) => {
|
||||
let peer_id = record.id;
|
||||
let tcp_addr = record.tcp_addr();
|
||||
if tcp_addr.port() == 0 {
|
||||
return
|
||||
}
|
||||
let udp_addr = record.udp_addr();
|
||||
let addr = PeerAddr::new(tcp_addr, Some(udp_addr));
|
||||
self.queued_messages.push_back(StateAction::DiscoveredEnrForkId {
|
||||
peer_id,
|
||||
addr,
|
||||
fork_id,
|
||||
});
|
||||
DiscoveryEvent::EnrForkId(peer_id, fork_id) => {
|
||||
self.queued_messages
|
||||
.push_back(StateAction::DiscoveredEnrForkId { peer_id, fork_id });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -562,8 +552,6 @@ pub(crate) enum StateAction<N: NetworkPrimitives> {
|
||||
/// Retrieved a [`ForkId`] from the peer via ENR request, See <https://eips.ethereum.org/EIPS/eip-868>
|
||||
DiscoveredEnrForkId {
|
||||
peer_id: PeerId,
|
||||
/// The address of the peer.
|
||||
addr: PeerAddr,
|
||||
/// The reported [`ForkId`] by this peer.
|
||||
fork_id: ForkId,
|
||||
},
|
||||
|
||||
@@ -246,28 +246,18 @@ impl<N: NetworkPrimitives> Swarm<N> {
|
||||
StateAction::PeerAdded(peer_id) => return Some(SwarmEvent::PeerAdded(peer_id)),
|
||||
StateAction::PeerRemoved(peer_id) => return Some(SwarmEvent::PeerRemoved(peer_id)),
|
||||
StateAction::DiscoveredNode { peer_id, addr, fork_id } => {
|
||||
// Don't try to connect to peer if node is shutting down
|
||||
if self.is_shutting_down() {
|
||||
return None
|
||||
}
|
||||
|
||||
// When `enforce_enr_fork_id` is enabled, peers discovered without a confirmed
|
||||
// fork ID (via EIP-868 ENR) are deferred — they'll only be added once a
|
||||
// `DiscoveredEnrForkId` event arrives with a validated fork ID.
|
||||
//
|
||||
// When disabled (default), peers without a fork ID are admitted immediately.
|
||||
// Peers that *do* carry a fork ID are always validated against ours.
|
||||
let enforce = self.state().peers().enforce_enr_fork_id();
|
||||
let allow = match fork_id {
|
||||
Some(f) => self.sessions.is_valid_fork_id(f),
|
||||
None => !enforce,
|
||||
};
|
||||
if allow {
|
||||
// Insert peer only if no fork id or a valid fork id
|
||||
if fork_id.map_or_else(|| true, |f| self.sessions.is_valid_fork_id(f)) {
|
||||
self.state_mut().peers_mut().add_peer(peer_id, addr, fork_id);
|
||||
}
|
||||
}
|
||||
StateAction::DiscoveredEnrForkId { peer_id, addr, fork_id } => {
|
||||
StateAction::DiscoveredEnrForkId { peer_id, fork_id } => {
|
||||
if self.sessions.is_valid_fork_id(fork_id) {
|
||||
self.state_mut().peers_mut().add_peer(peer_id, addr, Some(fork_id));
|
||||
self.state_mut().peers_mut().set_discovered_fork_id(peer_id, fork_id);
|
||||
} else {
|
||||
trace!(target: "net", ?peer_id, remote_fork_id=?fork_id, our_fork_id=?self.sessions.fork_id(), "fork id mismatch, removing peer");
|
||||
self.state_mut().peers_mut().remove_peer(peer_id);
|
||||
|
||||
@@ -129,4 +129,3 @@ op = [
|
||||
"reth-evm/op",
|
||||
"reth-primitives-traits/op",
|
||||
]
|
||||
pageviz = ["reth-engine-tree/pageviz"]
|
||||
|
||||
@@ -43,7 +43,6 @@ pub struct DefaultEngineValues {
|
||||
disable_trie_cache: bool,
|
||||
sparse_trie_prune_depth: usize,
|
||||
sparse_trie_max_storage_tries: usize,
|
||||
disable_sparse_trie_cache_pruning: bool,
|
||||
state_root_task_timeout: Option<String>,
|
||||
}
|
||||
|
||||
@@ -199,12 +198,6 @@ impl DefaultEngineValues {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether to disable sparse trie cache pruning by default
|
||||
pub const fn with_disable_sparse_trie_cache_pruning(mut self, v: bool) -> Self {
|
||||
self.disable_sparse_trie_cache_pruning = v;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the default state root task timeout
|
||||
pub fn with_state_root_task_timeout(mut self, v: Option<String>) -> Self {
|
||||
self.state_root_task_timeout = v;
|
||||
@@ -238,7 +231,6 @@ impl Default for DefaultEngineValues {
|
||||
disable_trie_cache: false,
|
||||
sparse_trie_prune_depth: DEFAULT_SPARSE_TRIE_PRUNE_DEPTH,
|
||||
sparse_trie_max_storage_tries: DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES,
|
||||
disable_sparse_trie_cache_pruning: false,
|
||||
state_root_task_timeout: Some("1s".to_string()),
|
||||
}
|
||||
}
|
||||
@@ -380,12 +372,6 @@ pub struct EngineArgs {
|
||||
#[arg(long = "engine.sparse-trie-max-storage-tries", default_value_t = DefaultEngineValues::get_global().sparse_trie_max_storage_tries)]
|
||||
pub sparse_trie_max_storage_tries: usize,
|
||||
|
||||
/// Fully disable sparse trie cache pruning. When set, the cached sparse trie is preserved
|
||||
/// without any node pruning or storage trie eviction between blocks. Useful for benchmarking
|
||||
/// the effects of retaining the full trie cache.
|
||||
#[arg(long = "engine.disable-sparse-trie-cache-pruning", default_value_t = DefaultEngineValues::get_global().disable_sparse_trie_cache_pruning)]
|
||||
pub disable_sparse_trie_cache_pruning: bool,
|
||||
|
||||
/// Configure the timeout for the state root task before spawning a sequential fallback.
|
||||
/// If the state root task takes longer than this, a sequential computation starts in
|
||||
/// parallel and whichever finishes first is used.
|
||||
@@ -429,7 +415,6 @@ impl Default for EngineArgs {
|
||||
disable_trie_cache,
|
||||
sparse_trie_prune_depth,
|
||||
sparse_trie_max_storage_tries,
|
||||
disable_sparse_trie_cache_pruning,
|
||||
state_root_task_timeout,
|
||||
} = DefaultEngineValues::get_global().clone();
|
||||
Self {
|
||||
@@ -460,7 +445,6 @@ impl Default for EngineArgs {
|
||||
disable_trie_cache,
|
||||
sparse_trie_prune_depth,
|
||||
sparse_trie_max_storage_tries,
|
||||
disable_sparse_trie_cache_pruning,
|
||||
state_root_task_timeout: state_root_task_timeout
|
||||
.as_deref()
|
||||
.map(|s| humantime::parse_duration(s).expect("valid default duration")),
|
||||
@@ -496,7 +480,6 @@ impl EngineArgs {
|
||||
.with_disable_trie_cache(self.disable_trie_cache)
|
||||
.with_sparse_trie_prune_depth(self.sparse_trie_prune_depth)
|
||||
.with_sparse_trie_max_storage_tries(self.sparse_trie_max_storage_tries)
|
||||
.with_disable_sparse_trie_cache_pruning(self.disable_sparse_trie_cache_pruning)
|
||||
.with_state_root_task_timeout(self.state_root_task_timeout.filter(|d| !d.is_zero()))
|
||||
}
|
||||
}
|
||||
@@ -551,7 +534,6 @@ mod tests {
|
||||
disable_trie_cache: true,
|
||||
sparse_trie_prune_depth: 10,
|
||||
sparse_trie_max_storage_tries: 100,
|
||||
disable_sparse_trie_cache_pruning: true,
|
||||
state_root_task_timeout: Some(Duration::from_secs(2)),
|
||||
};
|
||||
|
||||
@@ -588,7 +570,6 @@ mod tests {
|
||||
"10",
|
||||
"--engine.sparse-trie-max-storage-tries",
|
||||
"100",
|
||||
"--engine.disable-sparse-trie-cache-pruning",
|
||||
"--engine.state-root-task-timeout",
|
||||
"2s",
|
||||
])
|
||||
|
||||
@@ -76,7 +76,11 @@ pub use era::{DefaultEraHost, EraArgs, EraSourceArgs};
|
||||
mod static_files;
|
||||
pub use static_files::{StaticFilesArgs, MINIMAL_BLOCKS_PER_FILE};
|
||||
|
||||
/// `StorageArgs` for configuring storage settings.
|
||||
/// `RocksDbArgs` for configuring RocksDB table routing.
|
||||
mod rocksdb;
|
||||
pub use rocksdb::{RocksDbArgs, RocksDbArgsError};
|
||||
|
||||
/// `StorageArgs` for configuring storage mode (v2 vs v1/legacy).
|
||||
mod storage;
|
||||
pub use storage::StorageArgs;
|
||||
|
||||
|
||||
@@ -227,14 +227,6 @@ pub struct NetworkArgs {
|
||||
/// Example: --netrestrict "192.168.0.0/16,10.0.0.0/8"
|
||||
#[arg(long, value_name = "NETRESTRICT")]
|
||||
pub netrestrict: Option<String>,
|
||||
|
||||
/// Enforce EIP-868 ENR fork ID validation for discovered peers.
|
||||
///
|
||||
/// When enabled, peers discovered without a confirmed fork ID are not added to the peer set
|
||||
/// until their fork ID is verified via EIP-868 ENR request. This filters out peers from other
|
||||
/// networks that pollute the discovery table.
|
||||
#[arg(long)]
|
||||
pub enforce_enr_fork_id: bool,
|
||||
}
|
||||
|
||||
impl NetworkArgs {
|
||||
@@ -341,8 +333,7 @@ impl NetworkArgs {
|
||||
)
|
||||
.with_max_inbound_opt(self.resolved_max_inbound_peers())
|
||||
.with_max_outbound_opt(self.resolved_max_outbound_peers())
|
||||
.with_ip_filter(ip_filter)
|
||||
.with_enforce_enr_fork_id(self.enforce_enr_fork_id);
|
||||
.with_ip_filter(ip_filter);
|
||||
|
||||
// Configure basic network stack
|
||||
NetworkConfigBuilder::<N>::new(secret_key)
|
||||
@@ -500,7 +491,6 @@ impl Default for NetworkArgs {
|
||||
required_block_hashes: vec![],
|
||||
network_id: None,
|
||||
netrestrict: None,
|
||||
enforce_enr_fork_id: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
160
crates/node/core/src/args/rocksdb.rs
Normal file
160
crates/node/core/src/args/rocksdb.rs
Normal file
@@ -0,0 +1,160 @@
|
||||
//! clap [Args](clap::Args) for `RocksDB` table routing configuration
|
||||
|
||||
use clap::{ArgAction, Args};
|
||||
|
||||
/// Parameters for `RocksDB` table routing configuration.
|
||||
///
|
||||
/// These flags control which database tables are stored in `RocksDB` instead of MDBX.
|
||||
/// All flags are genesis-initialization-only: changing them after genesis requires a re-sync.
|
||||
///
|
||||
/// When `--storage.v2` is used, the defaults for these flags change to enable `RocksDB` routing.
|
||||
/// Individual flags can still override those defaults when explicitly set.
|
||||
#[derive(Debug, Args, PartialEq, Eq, Clone, Copy, Default)]
|
||||
#[command(next_help_heading = "RocksDB")]
|
||||
pub struct RocksDbArgs {
|
||||
/// Route all supported tables to `RocksDB` instead of MDBX.
|
||||
///
|
||||
/// This enables `RocksDB` for `tx-hash`, `storages-history`, and `account-history` tables.
|
||||
/// Cannot be combined with individual flags set to false.
|
||||
#[arg(long = "rocksdb.all", action = ArgAction::SetTrue)]
|
||||
pub all: bool,
|
||||
|
||||
/// Route tx hash -> number table to `RocksDB` instead of MDBX.
|
||||
///
|
||||
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
|
||||
/// Defaults to the base storage mode (v1: false, v2: true).
|
||||
#[arg(long = "rocksdb.tx-hash", action = ArgAction::Set)]
|
||||
pub tx_hash: Option<bool>,
|
||||
|
||||
/// Route storages history tables to `RocksDB` instead of MDBX.
|
||||
///
|
||||
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
|
||||
/// Defaults to the base storage mode (v1: false, v2: true).
|
||||
#[arg(long = "rocksdb.storages-history", action = ArgAction::Set)]
|
||||
pub storages_history: Option<bool>,
|
||||
|
||||
/// Route account history tables to `RocksDB` instead of MDBX.
|
||||
///
|
||||
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
|
||||
/// Defaults to the base storage mode (v1: false, v2: true).
|
||||
#[arg(long = "rocksdb.account-history", action = ArgAction::Set)]
|
||||
pub account_history: Option<bool>,
|
||||
}
|
||||
|
||||
impl RocksDbArgs {
|
||||
/// Validates the `RocksDB` arguments.
|
||||
///
|
||||
/// Returns an error if `--rocksdb.all` is used with any individual flag explicitly set to
|
||||
/// `false`.
|
||||
pub const fn validate(&self) -> Result<(), RocksDbArgsError> {
|
||||
if self.all {
|
||||
if matches!(self.tx_hash, Some(false)) {
|
||||
return Err(RocksDbArgsError::ConflictingFlags("tx-hash"));
|
||||
}
|
||||
if matches!(self.storages_history, Some(false)) {
|
||||
return Err(RocksDbArgsError::ConflictingFlags("storages-history"));
|
||||
}
|
||||
if matches!(self.account_history, Some(false)) {
|
||||
return Err(RocksDbArgsError::ConflictingFlags("account-history"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Error type for `RocksDB` argument validation.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum RocksDbArgsError {
|
||||
/// `--rocksdb.all` cannot be combined with an individual flag set to false.
|
||||
#[error("--rocksdb.all cannot be combined with --rocksdb.{0}=false")]
|
||||
ConflictingFlags(&'static str),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use clap::Parser;
|
||||
|
||||
#[derive(Parser)]
|
||||
struct CommandParser<T: Args> {
|
||||
#[command(flatten)]
|
||||
args: T,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_default_rocksdb_args() {
|
||||
let args = CommandParser::<RocksDbArgs>::parse_from(["reth"]).args;
|
||||
assert_eq!(args, RocksDbArgs::default());
|
||||
assert!(!args.all);
|
||||
assert!(args.tx_hash.is_none());
|
||||
assert!(args.storages_history.is_none());
|
||||
assert!(args.account_history.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_all_flag() {
|
||||
let args = CommandParser::<RocksDbArgs>::parse_from(["reth", "--rocksdb.all"]).args;
|
||||
assert!(args.all);
|
||||
assert!(args.tx_hash.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_individual_flags() {
|
||||
let args = CommandParser::<RocksDbArgs>::parse_from([
|
||||
"reth",
|
||||
"--rocksdb.tx-hash=true",
|
||||
"--rocksdb.storages-history=false",
|
||||
"--rocksdb.account-history=true",
|
||||
])
|
||||
.args;
|
||||
assert!(!args.all);
|
||||
assert_eq!(args.tx_hash, Some(true));
|
||||
assert_eq!(args.storages_history, Some(false));
|
||||
assert_eq!(args.account_history, Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_all_with_none_ok() {
|
||||
let args =
|
||||
RocksDbArgs { all: true, tx_hash: None, storages_history: None, account_history: None };
|
||||
assert!(args.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_all_with_true_ok() {
|
||||
let args = RocksDbArgs {
|
||||
all: true,
|
||||
tx_hash: Some(true),
|
||||
storages_history: Some(true),
|
||||
account_history: Some(true),
|
||||
};
|
||||
assert!(args.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_all_with_false_errors() {
|
||||
let args = RocksDbArgs {
|
||||
all: true,
|
||||
tx_hash: Some(false),
|
||||
storages_history: None,
|
||||
account_history: None,
|
||||
};
|
||||
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("tx-hash")));
|
||||
|
||||
let args = RocksDbArgs {
|
||||
all: true,
|
||||
tx_hash: None,
|
||||
storages_history: Some(false),
|
||||
account_history: None,
|
||||
};
|
||||
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("storages-history")));
|
||||
|
||||
let args = RocksDbArgs {
|
||||
all: true,
|
||||
tx_hash: None,
|
||||
storages_history: None,
|
||||
account_history: Some(false),
|
||||
};
|
||||
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("account-history")));
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,9 @@ use reth_config::config::{BlocksPerFileConfig, StaticFilesConfig};
|
||||
pub const MINIMAL_BLOCKS_PER_FILE: u64 = 10000;
|
||||
|
||||
/// Parameters for static files configuration
|
||||
///
|
||||
/// When `--storage.v2` is used, the defaults for the storage flags change to enable static file
|
||||
/// storage. Individual flags can still override those defaults when explicitly set.
|
||||
#[derive(Debug, Args, PartialEq, Eq, Clone, Copy, Default)]
|
||||
#[command(next_help_heading = "Static Files")]
|
||||
pub struct StaticFilesArgs {
|
||||
@@ -35,6 +38,53 @@ pub struct StaticFilesArgs {
|
||||
/// Number of blocks per file for the storage changesets segment.
|
||||
#[arg(long = "static-files.blocks-per-file.storage-change-sets")]
|
||||
pub blocks_per_file_storage_change_sets: Option<u64>,
|
||||
|
||||
/// Store receipts in static files instead of the database.
|
||||
///
|
||||
/// When enabled, receipts will be written to static files on disk instead of the database.
|
||||
///
|
||||
/// Note: This setting can only be configured at genesis initialization. Once
|
||||
/// the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
///
|
||||
/// Defaults to the base storage mode (v1: false, v2: true).
|
||||
#[arg(long = "static-files.receipts", action = clap::ArgAction::Set)]
|
||||
pub receipts: Option<bool>,
|
||||
|
||||
/// Store transaction senders in static files instead of the database.
|
||||
///
|
||||
/// When enabled, transaction senders will be written to static files on disk instead of the
|
||||
/// database.
|
||||
///
|
||||
/// Note: This setting can only be configured at genesis initialization. Once
|
||||
/// the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
///
|
||||
/// Defaults to the base storage mode (v1: false, v2: true).
|
||||
#[arg(long = "static-files.transaction-senders", action = clap::ArgAction::Set)]
|
||||
pub transaction_senders: Option<bool>,
|
||||
|
||||
/// Store account changesets in static files.
|
||||
///
|
||||
/// When enabled, account changesets will be written to static files on disk instead of the
|
||||
/// database.
|
||||
///
|
||||
/// Note: This setting can only be configured at genesis initialization. Once
|
||||
/// the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
///
|
||||
/// Defaults to the base storage mode (v1: false, v2: true).
|
||||
#[arg(long = "static-files.account-change-sets", action = clap::ArgAction::Set)]
|
||||
pub account_changesets: Option<bool>,
|
||||
|
||||
/// Store storage changesets in static files.
|
||||
///
|
||||
/// When enabled, storage changesets will be written to static files on disk instead of the
|
||||
/// database.
|
||||
///
|
||||
/// Note: This setting can only be configured at genesis initialization. Once
|
||||
/// the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
///
|
||||
/// Defaults to the base storage mode (v1: false, v2: true).
|
||||
#[arg(long = "static-files.storage-change-sets", action = clap::ArgAction::Set)]
|
||||
pub storage_changesets: Option<bool>,
|
||||
}
|
||||
|
||||
impl StaticFilesArgs {
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
//! clap [Args](clap::Args) for storage configuration
|
||||
//! clap [Args](clap::Args) for storage mode configuration
|
||||
|
||||
use clap::{ArgAction, Args};
|
||||
|
||||
/// Parameters for storage configuration.
|
||||
/// Parameters for storage mode configuration.
|
||||
///
|
||||
/// This controls whether the node uses v2 storage defaults (with `RocksDB` and static file
|
||||
/// optimizations) or v1/legacy storage defaults.
|
||||
///
|
||||
/// Individual storage settings can be overridden with `--static-files.*` and `--rocksdb.*` flags.
|
||||
#[derive(Debug, Args, PartialEq, Eq, Clone, Copy, Default)]
|
||||
#[command(next_help_heading = "Storage")]
|
||||
pub struct StorageArgs {
|
||||
@@ -25,6 +23,16 @@ pub struct StorageArgs {
|
||||
/// flags.
|
||||
#[arg(long = "storage.v2", action = ArgAction::SetTrue)]
|
||||
pub v2: bool,
|
||||
|
||||
/// Use hashed state tables (`HashedAccounts`/`HashedStorages`) as canonical state
|
||||
/// representation instead of plain state tables.
|
||||
///
|
||||
/// When enabled, execution writes directly to hashed tables, eliminating the need for
|
||||
/// separate hashing stages. This should only be enabled for new databases.
|
||||
///
|
||||
/// WARNING: Changing this setting on an existing database requires a full resync.
|
||||
#[arg(long = "storage.use-hashed-state", default_value_t = false)]
|
||||
pub use_hashed_state: bool,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -32,24 +40,21 @@ mod tests {
|
||||
use super::*;
|
||||
use clap::Parser;
|
||||
|
||||
/// A helper type to parse Args more easily
|
||||
#[derive(Parser)]
|
||||
struct CommandParser<T: Args> {
|
||||
struct CommandParser {
|
||||
#[command(flatten)]
|
||||
args: T,
|
||||
args: StorageArgs,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_default_storage_args() {
|
||||
let default_args = StorageArgs::default();
|
||||
let args = CommandParser::<StorageArgs>::parse_from(["reth"]).args;
|
||||
assert_eq!(args, default_args);
|
||||
let args = CommandParser::parse_from(["reth"]).args;
|
||||
assert!(!args.v2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_v2_flag() {
|
||||
let args = CommandParser::<StorageArgs>::parse_from(["reth", "--storage.v2"]).args;
|
||||
let args = CommandParser::parse_from(["reth", "--storage.v2"]).args;
|
||||
assert!(args.v2);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use crate::{
|
||||
args::{
|
||||
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, NetworkArgs, PayloadBuilderArgs,
|
||||
PruningArgs, RpcServerArgs, StaticFilesArgs, StorageArgs, TxPoolArgs,
|
||||
PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs, StorageArgs, TxPoolArgs,
|
||||
},
|
||||
dirs::{ChainPath, DataDirPath},
|
||||
utils::get_single_header,
|
||||
@@ -152,7 +152,10 @@ pub struct NodeConfig<ChainSpec> {
|
||||
/// All static files related arguments
|
||||
pub static_files: StaticFilesArgs,
|
||||
|
||||
/// All storage related arguments with --storage prefix
|
||||
/// All `RocksDB` table routing arguments
|
||||
pub rocksdb: RocksDbArgs,
|
||||
|
||||
/// Storage mode configuration (v2 vs v1/legacy)
|
||||
pub storage: StorageArgs,
|
||||
}
|
||||
|
||||
@@ -185,6 +188,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
engine: EngineArgs::default(),
|
||||
era: EraArgs::default(),
|
||||
static_files: StaticFilesArgs::default(),
|
||||
rocksdb: RocksDbArgs::default(),
|
||||
storage: StorageArgs::default(),
|
||||
}
|
||||
}
|
||||
@@ -260,6 +264,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
rocksdb,
|
||||
storage,
|
||||
..
|
||||
} = self;
|
||||
@@ -280,6 +285,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
rocksdb,
|
||||
storage,
|
||||
}
|
||||
}
|
||||
@@ -349,12 +355,6 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the storage args for the node
|
||||
pub const fn with_storage(mut self, storage: StorageArgs) -> Self {
|
||||
self.storage = storage;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns pruning configuration.
|
||||
pub fn prune_config(&self) -> Option<PruneConfig>
|
||||
where
|
||||
@@ -363,17 +363,42 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
self.pruning.prune_config(&self.chain)
|
||||
}
|
||||
|
||||
/// Returns the effective storage settings derived from `--storage.v2`.
|
||||
/// Returns the effective storage settings derived from `--storage.v2`, static-file, and
|
||||
/// `RocksDB` CLI args.
|
||||
///
|
||||
/// The base storage mode is determined by `--storage.v2`:
|
||||
/// - When `--storage.v2` is set: uses [`StorageSettings::v2()`] defaults
|
||||
/// - Otherwise: uses [`StorageSettings::base()`] defaults
|
||||
/// - Otherwise: uses [`StorageSettings::v1()`] defaults
|
||||
///
|
||||
/// Individual `--static-files.*` and `--rocksdb.*` flags override the base when explicitly set.
|
||||
pub const fn storage_settings(&self) -> StorageSettings {
|
||||
if self.storage.v2 {
|
||||
StorageSettings::v2()
|
||||
} else {
|
||||
StorageSettings::base()
|
||||
let mut s = if self.storage.v2 { StorageSettings::v2() } else { StorageSettings::base() };
|
||||
|
||||
// Apply static files overrides (only when explicitly set)
|
||||
s = s
|
||||
.with_receipts_in_static_files_opt(self.static_files.receipts)
|
||||
.with_transaction_senders_in_static_files_opt(self.static_files.transaction_senders)
|
||||
.with_account_changesets_in_static_files_opt(self.static_files.account_changesets)
|
||||
.with_storage_changesets_in_static_files_opt(self.static_files.storage_changesets);
|
||||
|
||||
// Apply rocksdb overrides
|
||||
// --rocksdb.all sets all rocksdb flags to true
|
||||
if self.rocksdb.all {
|
||||
s = s
|
||||
.with_transaction_hash_numbers_in_rocksdb(true)
|
||||
.with_storages_history_in_rocksdb(true)
|
||||
.with_account_history_in_rocksdb(true);
|
||||
}
|
||||
|
||||
// Individual rocksdb flags override --rocksdb.all when explicitly set
|
||||
s = s
|
||||
.with_transaction_hash_numbers_in_rocksdb_opt(self.rocksdb.tx_hash)
|
||||
.with_storages_history_in_rocksdb_opt(self.rocksdb.storages_history)
|
||||
.with_account_history_in_rocksdb_opt(self.rocksdb.account_history);
|
||||
|
||||
s = s.with_use_hashed_state(self.storage.use_hashed_state);
|
||||
|
||||
s
|
||||
}
|
||||
|
||||
/// Returns the max block that the node should run to, looking it up from the network if
|
||||
@@ -570,6 +595,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
engine: self.engine,
|
||||
era: self.era,
|
||||
static_files: self.static_files,
|
||||
rocksdb: self.rocksdb,
|
||||
storage: self.storage,
|
||||
}
|
||||
}
|
||||
@@ -612,6 +638,7 @@ impl<ChainSpec> Clone for NodeConfig<ChainSpec> {
|
||||
engine: self.engine.clone(),
|
||||
era: self.era.clone(),
|
||||
static_files: self.static_files,
|
||||
rocksdb: self.rocksdb,
|
||||
storage: self.storage,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -265,7 +265,7 @@ impl NodeState {
|
||||
warn!(number=block.number(), hash=?block.hash(), "Encountered invalid block");
|
||||
}
|
||||
ConsensusEngineEvent::BlockReceived(num_hash) => {
|
||||
info!(number=num_hash.number, hash=?num_hash.hash, "Received new payload from consensus engine");
|
||||
info!(number=num_hash.number, hash=?num_hash.hash, "Received block from consensus engine");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ use alloy_genesis::GenesisAccount;
|
||||
use alloy_primitives::{keccak256, Bytes, B256, U256};
|
||||
use alloy_trie::TrieAccount;
|
||||
use derive_more::Deref;
|
||||
use revm_bytecode::{Bytecode as RevmBytecode, BytecodeDecodeError};
|
||||
use revm_bytecode::{Bytecode as RevmBytecode, BytecodeDecodeError, BytecodeKind};
|
||||
use revm_state::AccountInfo;
|
||||
|
||||
#[cfg(any(test, feature = "reth-codec"))]
|
||||
@@ -16,10 +16,10 @@ pub mod compact_ids {
|
||||
/// Identifier for removed bytecode variant.
|
||||
pub const REMOVED_BYTECODE_ID: u8 = 1;
|
||||
|
||||
/// Identifier for [`LegacyAnalyzed`](revm_bytecode::Bytecode::LegacyAnalyzed).
|
||||
/// Identifier for [`LegacyAnalyzed`](revm_bytecode::BytecodeKind::LegacyAnalyzed).
|
||||
pub const LEGACY_ANALYZED_BYTECODE_ID: u8 = 2;
|
||||
|
||||
/// Identifier for [`Eip7702`](revm_bytecode::Bytecode::Eip7702).
|
||||
/// Identifier for [`Eip7702`](revm_bytecode::BytecodeKind::Eip7702).
|
||||
pub const EIP7702_BYTECODE_ID: u8 = 4;
|
||||
}
|
||||
|
||||
@@ -137,22 +137,19 @@ impl reth_codecs::Compact for Bytecode {
|
||||
{
|
||||
use compact_ids::{EIP7702_BYTECODE_ID, LEGACY_ANALYZED_BYTECODE_ID};
|
||||
|
||||
let bytecode = match &self.0 {
|
||||
RevmBytecode::LegacyAnalyzed(analyzed) => analyzed.bytecode(),
|
||||
RevmBytecode::Eip7702(eip7702) => eip7702.raw(),
|
||||
};
|
||||
let bytecode = self.0.bytecode();
|
||||
buf.put_u32(bytecode.len() as u32);
|
||||
buf.put_slice(bytecode.as_ref());
|
||||
let len = match &self.0 {
|
||||
let len = match self.0.kind() {
|
||||
// [`REMOVED_BYTECODE_ID`] has been removed.
|
||||
RevmBytecode::LegacyAnalyzed(analyzed) => {
|
||||
BytecodeKind::LegacyAnalyzed => {
|
||||
buf.put_u8(LEGACY_ANALYZED_BYTECODE_ID);
|
||||
buf.put_u64(analyzed.original_len() as u64);
|
||||
let map = analyzed.jump_table().as_slice();
|
||||
buf.put_u64(self.0.len() as u64);
|
||||
let map = self.0.legacy_jump_table().unwrap().as_slice();
|
||||
buf.put_slice(map);
|
||||
1 + 8 + map.len()
|
||||
}
|
||||
RevmBytecode::Eip7702(_) => {
|
||||
BytecodeKind::Eip7702 => {
|
||||
buf.put_u8(EIP7702_BYTECODE_ID);
|
||||
1
|
||||
}
|
||||
@@ -251,12 +248,10 @@ impl From<Account> for AccountInfo {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::*;
|
||||
use alloy_primitives::{hex_literal::hex, B256, U256};
|
||||
use reth_codecs::Compact;
|
||||
use revm_bytecode::{JumpTable, LegacyAnalyzedBytecode};
|
||||
use revm_bytecode::JumpTable;
|
||||
|
||||
#[test]
|
||||
fn test_account() {
|
||||
@@ -313,12 +308,11 @@ mod tests {
|
||||
assert_eq!(len, 17);
|
||||
|
||||
let mut buf = vec![];
|
||||
let bytecode =
|
||||
Bytecode(RevmBytecode::LegacyAnalyzed(Arc::new(LegacyAnalyzedBytecode::new(
|
||||
Bytes::from(&hex!("ff00")),
|
||||
2,
|
||||
JumpTable::from_slice(&[0], 2),
|
||||
))));
|
||||
let bytecode = Bytecode(RevmBytecode::new_analyzed(
|
||||
Bytes::from(&hex!("ff00")),
|
||||
2,
|
||||
JumpTable::from_slice(&[0], 2),
|
||||
));
|
||||
let len = bytecode.to_compact(&mut buf);
|
||||
assert_eq!(len, 16);
|
||||
|
||||
|
||||
@@ -164,7 +164,7 @@ pub use alloy_primitives::{logs_bloom, Log, LogData};
|
||||
pub mod proofs;
|
||||
|
||||
mod storage;
|
||||
pub use storage::{StorageEntry, StorageSlotKey, ValueWithSubKey};
|
||||
pub use storage::{StorageEntry, ValueWithSubKey};
|
||||
|
||||
pub mod sync;
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use alloy_primitives::{keccak256, B256, U256};
|
||||
use alloy_primitives::{B256, U256};
|
||||
|
||||
/// Trait for `DupSort` table values that contain a subkey.
|
||||
///
|
||||
@@ -12,117 +12,6 @@ pub trait ValueWithSubKey {
|
||||
fn get_subkey(&self) -> Self::SubKey;
|
||||
}
|
||||
|
||||
/// A storage slot key that tracks whether it holds a plain (unhashed) EVM slot
|
||||
/// or a keccak256-hashed slot.
|
||||
///
|
||||
/// This enum replaces the `use_hashed_state: bool` parameter pattern by carrying
|
||||
/// provenance with the key itself. Once tagged at a read/write boundary, downstream
|
||||
/// code can call [`Self::to_hashed`] without risk of double-hashing — hashing a
|
||||
/// [`StorageSlotKey::Hashed`] is a no-op.
|
||||
///
|
||||
/// The on-disk encoding is unchanged (raw 32-byte [`B256`]). The variant is set
|
||||
/// by the code that knows the context (which table, which storage mode).
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum StorageSlotKey {
|
||||
/// An unhashed EVM storage slot, as produced by REVM execution.
|
||||
Plain(B256),
|
||||
/// A keccak256-hashed storage slot, as stored in `HashedStorages` and
|
||||
/// in v2-mode `StorageChangeSets`.
|
||||
Hashed(B256),
|
||||
}
|
||||
|
||||
impl Default for StorageSlotKey {
|
||||
fn default() -> Self {
|
||||
Self::Plain(B256::ZERO)
|
||||
}
|
||||
}
|
||||
|
||||
impl StorageSlotKey {
|
||||
/// Create a plain slot key from a REVM [`U256`] storage index.
|
||||
pub const fn from_u256(slot: U256) -> Self {
|
||||
Self::Plain(B256::new(slot.to_be_bytes()))
|
||||
}
|
||||
|
||||
/// Create a plain slot key from a raw [`B256`].
|
||||
pub const fn plain(key: B256) -> Self {
|
||||
Self::Plain(key)
|
||||
}
|
||||
|
||||
/// Create a hashed slot key from a raw [`B256`].
|
||||
pub const fn hashed(key: B256) -> Self {
|
||||
Self::Hashed(key)
|
||||
}
|
||||
|
||||
/// Tag a raw [`B256`] based on the storage mode.
|
||||
///
|
||||
/// When `use_hashed_state` is true the key is assumed already hashed.
|
||||
/// When false it is assumed to be a plain slot.
|
||||
pub const fn from_raw(key: B256, use_hashed_state: bool) -> Self {
|
||||
if use_hashed_state {
|
||||
Self::Hashed(key)
|
||||
} else {
|
||||
Self::Plain(key)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the raw [`B256`] regardless of variant.
|
||||
pub const fn as_b256(&self) -> B256 {
|
||||
match *self {
|
||||
Self::Plain(b) | Self::Hashed(b) => b,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `true` if this key is already hashed.
|
||||
pub const fn is_hashed(&self) -> bool {
|
||||
matches!(self, Self::Hashed(_))
|
||||
}
|
||||
|
||||
/// Returns `true` if this key is plain (unhashed).
|
||||
pub const fn is_plain(&self) -> bool {
|
||||
matches!(self, Self::Plain(_))
|
||||
}
|
||||
|
||||
/// Produce the keccak256-hashed form of this slot key.
|
||||
///
|
||||
/// - If already [`Hashed`](Self::Hashed), returns the inner value as-is (no double-hash).
|
||||
/// - If [`Plain`](Self::Plain), applies keccak256 and returns the result.
|
||||
pub fn to_hashed(&self) -> B256 {
|
||||
match *self {
|
||||
Self::Hashed(b) => b,
|
||||
Self::Plain(b) => keccak256(b),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a plain slot to its changeset representation.
|
||||
///
|
||||
/// In v2 mode (`use_hashed_state = true`), the changeset stores hashed keys,
|
||||
/// so the plain key is hashed. In v1 mode, the plain key is stored as-is.
|
||||
///
|
||||
/// Panics (debug) if called on an already-hashed key.
|
||||
pub fn to_changeset_key(self, use_hashed_state: bool) -> B256 {
|
||||
debug_assert!(self.is_plain(), "to_changeset_key called on already-hashed key");
|
||||
if use_hashed_state {
|
||||
self.to_hashed()
|
||||
} else {
|
||||
self.as_b256()
|
||||
}
|
||||
}
|
||||
|
||||
/// Like [`to_changeset_key`](Self::to_changeset_key) but returns a tagged
|
||||
/// [`StorageSlotKey`] instead of a raw [`B256`].
|
||||
///
|
||||
/// Panics (debug) if called on an already-hashed key.
|
||||
pub fn to_changeset(self, use_hashed_state: bool) -> Self {
|
||||
Self::from_raw(self.to_changeset_key(use_hashed_state), use_hashed_state)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<StorageSlotKey> for B256 {
|
||||
fn from(key: StorageSlotKey) -> Self {
|
||||
key.as_b256()
|
||||
}
|
||||
}
|
||||
|
||||
/// Account storage entry.
|
||||
///
|
||||
/// `key` is the subkey when used as a value in the `StorageChangeSets` table.
|
||||
@@ -142,14 +31,6 @@ impl StorageEntry {
|
||||
pub const fn new(key: B256, value: U256) -> Self {
|
||||
Self { key, value }
|
||||
}
|
||||
|
||||
/// Tag this entry's key as a [`StorageSlotKey`] based on the storage mode.
|
||||
///
|
||||
/// When `use_hashed_state` is true, the key is tagged as already-hashed.
|
||||
/// When false, it is tagged as plain.
|
||||
pub const fn slot_key(&self, use_hashed_state: bool) -> StorageSlotKey {
|
||||
StorageSlotKey::from_raw(self.key, use_hashed_state)
|
||||
}
|
||||
}
|
||||
|
||||
impl ValueWithSubKey for StorageEntry {
|
||||
|
||||
@@ -75,7 +75,7 @@ where
|
||||
|
||||
// Check where account history indices are stored
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
if provider.cached_storage_settings().account_history_in_rocksdb {
|
||||
return self.prune_rocksdb(provider, input, range, range_end);
|
||||
}
|
||||
|
||||
@@ -405,7 +405,9 @@ mod tests {
|
||||
let segment = AccountHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(StorageSettings::v1());
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default().with_account_changesets_in_static_files(false),
|
||||
);
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
limiter.increment_deleted_entries_count_by(result.pruned);
|
||||
|
||||
@@ -506,11 +508,7 @@ mod tests {
|
||||
test_prune(1400, 3, (PruneProgress::Finished, 804));
|
||||
}
|
||||
|
||||
/// Tests the `prune_static_files` code path. On unix with rocksdb feature, v2 storage
|
||||
/// routes to `prune_rocksdb` instead, so this test only runs without rocksdb (the
|
||||
/// `prune_rocksdb_path` test covers that configuration).
|
||||
#[test]
|
||||
#[cfg(not(all(unix, feature = "rocksdb")))]
|
||||
fn prune_static_file() {
|
||||
let db = TestStageDB::default();
|
||||
let mut rng = generators::rng();
|
||||
@@ -566,7 +564,9 @@ mod tests {
|
||||
let segment = AccountHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(StorageSettings::v2());
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default().with_account_changesets_in_static_files(true),
|
||||
);
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
limiter.increment_deleted_entries_count_by(result.pruned);
|
||||
|
||||
@@ -714,7 +714,11 @@ mod tests {
|
||||
PruneInput { previous_checkpoint: None, to_block, limiter: PruneLimiter::default() };
|
||||
let segment = AccountHistory::new(prune_mode);
|
||||
|
||||
db.factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::default()
|
||||
.with_account_changesets_in_static_files(true)
|
||||
.with_account_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
@@ -828,7 +832,9 @@ mod tests {
|
||||
let segment = AccountHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(StorageSettings::v1());
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default().with_account_changesets_in_static_files(false),
|
||||
);
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
|
||||
// Should report that there's more data
|
||||
@@ -886,7 +892,9 @@ mod tests {
|
||||
};
|
||||
|
||||
let provider2 = db.factory.database_provider_rw().unwrap();
|
||||
provider2.set_storage_settings_cache(StorageSettings::v1());
|
||||
provider2.set_storage_settings_cache(
|
||||
StorageSettings::default().with_account_changesets_in_static_files(false),
|
||||
);
|
||||
let result2 = segment.prune(&provider2, input2).unwrap();
|
||||
|
||||
assert!(result2.progress.is_finished(), "Second run should complete");
|
||||
|
||||
@@ -76,7 +76,7 @@ where
|
||||
|
||||
// Check where storage history indices are stored
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
if provider.cached_storage_settings().storages_history_in_rocksdb {
|
||||
return self.prune_rocksdb(provider, input, range, range_end);
|
||||
}
|
||||
|
||||
@@ -135,7 +135,7 @@ impl StorageHistory {
|
||||
let (block_address, entry) = result?;
|
||||
let block_number = block_address.block_number();
|
||||
let address = block_address.address();
|
||||
highest_deleted_storages.insert((address, entry.key.as_b256()), block_number);
|
||||
highest_deleted_storages.insert((address, entry.key), block_number);
|
||||
last_changeset_pruned_block = Some(block_number);
|
||||
pruned_changesets += 1;
|
||||
limiter.increment_deleted_entries_count();
|
||||
@@ -273,7 +273,7 @@ impl StorageHistory {
|
||||
let (block_address, entry) = result?;
|
||||
let block_number = block_address.block_number();
|
||||
let address = block_address.address();
|
||||
highest_deleted_storages.insert((address, entry.key.as_b256()), block_number);
|
||||
highest_deleted_storages.insert((address, entry.key), block_number);
|
||||
last_changeset_pruned_block = Some(block_number);
|
||||
changesets_processed += 1;
|
||||
limiter.increment_deleted_entries_count();
|
||||
@@ -413,7 +413,9 @@ mod tests {
|
||||
let segment = StorageHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(StorageSettings::v1());
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default().with_storage_changesets_in_static_files(false),
|
||||
);
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
limiter.increment_deleted_entries_count_by(result.pruned);
|
||||
|
||||
@@ -518,11 +520,7 @@ mod tests {
|
||||
test_prune(1200, 3, (PruneProgress::Finished, 202));
|
||||
}
|
||||
|
||||
/// Tests the `prune_static_files` code path. On unix with rocksdb feature, v2 storage
|
||||
/// routes to `prune_rocksdb` instead, so this test only runs without rocksdb (the
|
||||
/// `prune_rocksdb_path` test covers that configuration).
|
||||
#[test]
|
||||
#[cfg(not(all(unix, feature = "rocksdb")))]
|
||||
fn prune_static_file() {
|
||||
let db = TestStageDB::default();
|
||||
let mut rng = generators::rng();
|
||||
@@ -579,7 +577,9 @@ mod tests {
|
||||
let segment = StorageHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(StorageSettings::v2());
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default().with_storage_changesets_in_static_files(true),
|
||||
);
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
limiter.increment_deleted_entries_count_by(result.pruned);
|
||||
|
||||
@@ -739,7 +739,9 @@ mod tests {
|
||||
let segment = StorageHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(StorageSettings::v1());
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default().with_storage_changesets_in_static_files(false),
|
||||
);
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
|
||||
// Should report that there's more data
|
||||
@@ -791,7 +793,9 @@ mod tests {
|
||||
};
|
||||
|
||||
let provider2 = db.factory.database_provider_rw().unwrap();
|
||||
provider2.set_storage_settings_cache(StorageSettings::v1());
|
||||
provider2.set_storage_settings_cache(
|
||||
StorageSettings::default().with_storage_changesets_in_static_files(false),
|
||||
);
|
||||
let result2 = segment.prune(&provider2, input2).unwrap();
|
||||
|
||||
assert!(result2.progress.is_finished(), "Second run should complete");
|
||||
@@ -891,7 +895,11 @@ mod tests {
|
||||
let segment = StorageHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(StorageSettings::v2());
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default()
|
||||
.with_storage_changesets_in_static_files(true)
|
||||
.with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
provider.commit().expect("commit");
|
||||
|
||||
|
||||
@@ -96,7 +96,7 @@ where
|
||||
|
||||
// Check where transaction hash numbers are stored
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
if provider.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
|
||||
return self.prune_rocksdb(provider, input, start, end);
|
||||
}
|
||||
|
||||
@@ -491,7 +491,9 @@ mod tests {
|
||||
let segment = TransactionLookup::new(prune_mode);
|
||||
|
||||
// Enable RocksDB storage for transaction hash numbers
|
||||
db.factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
@@ -576,7 +578,9 @@ mod tests {
|
||||
}
|
||||
|
||||
// Enable RocksDB storage for transaction hash numbers
|
||||
db.factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let to_block: BlockNumber = 6;
|
||||
let prune_mode = PruneMode::Before(to_block);
|
||||
|
||||
@@ -160,14 +160,6 @@ impl StateProvider for StateProviderTest {
|
||||
) -> ProviderResult<Option<alloy_primitives::StorageValue>> {
|
||||
Ok(self.accounts.get(&account).and_then(|(storage, _)| storage.get(&storage_key).copied()))
|
||||
}
|
||||
|
||||
fn storage_by_hashed_key(
|
||||
&self,
|
||||
_address: Address,
|
||||
_hashed_storage_key: StorageKey,
|
||||
) -> ProviderResult<Option<alloy_primitives::StorageValue>> {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
impl BytecodeReader for StateProviderTest {
|
||||
|
||||
@@ -70,8 +70,8 @@ impl ExecutionWitnessRecord {
|
||||
}
|
||||
}
|
||||
}
|
||||
// BTreeMap keys are ordered, so the first key is the smallest
|
||||
self.lowest_block_number = statedb.block_hashes.keys().next().copied()
|
||||
// Get the lowest block number from the cache
|
||||
self.lowest_block_number = statedb.block_hashes.lowest().map(|(n, _)| n)
|
||||
}
|
||||
|
||||
/// Creates the record from the state after execution.
|
||||
|
||||
@@ -172,7 +172,7 @@ pub trait EstimateCall: Call {
|
||||
};
|
||||
|
||||
let gas_refund = match res.result {
|
||||
ExecutionResult::Success { gas_refunded, .. } => gas_refunded,
|
||||
ExecutionResult::Success { gas, .. } => gas.inner_refunded(),
|
||||
ExecutionResult::Halt { reason, .. } => {
|
||||
// here we don't check for invalid opcode because already executed with highest gas
|
||||
// limit
|
||||
|
||||
8
crates/rpc/rpc-eth-types/src/cache/db.rs
vendored
8
crates/rpc/rpc-eth-types/src/cache/db.rs
vendored
@@ -154,14 +154,6 @@ impl StateProvider for StateProviderTraitObjWrapper {
|
||||
self.0.storage(account, storage_key)
|
||||
}
|
||||
|
||||
fn storage_by_hashed_key(
|
||||
&self,
|
||||
address: Address,
|
||||
hashed_storage_key: alloy_primitives::StorageKey,
|
||||
) -> reth_errors::ProviderResult<Option<alloy_primitives::StorageValue>> {
|
||||
self.0.storage_by_hashed_key(address, hashed_storage_key)
|
||||
}
|
||||
|
||||
fn account_code(
|
||||
&self,
|
||||
addr: &Address,
|
||||
|
||||
@@ -125,9 +125,7 @@ pub trait FromEvmError<Evm: ConfigureEvm>:
|
||||
match result {
|
||||
ExecutionResult::Success { output, .. } => Ok(output.into_data()),
|
||||
ExecutionResult::Revert { output, .. } => Err(Self::from_revert(output)),
|
||||
ExecutionResult::Halt { reason, gas_used } => {
|
||||
Err(Self::from_evm_halt(reason, gas_used))
|
||||
}
|
||||
ExecutionResult::Halt { reason, gas } => Err(Self::from_evm_halt(reason, gas.used())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -352,7 +352,7 @@ where
|
||||
let mut log_index = 0;
|
||||
for (index, (result, tx)) in results.into_iter().zip(block.body().transactions()).enumerate() {
|
||||
let call = match result {
|
||||
ExecutionResult::Halt { reason, gas_used } => {
|
||||
ExecutionResult::Halt { reason, gas } => {
|
||||
let error = Err::from_evm_halt(reason, tx.gas_limit());
|
||||
#[allow(clippy::needless_update)]
|
||||
SimCallResult {
|
||||
@@ -362,12 +362,12 @@ where
|
||||
code: SIMULATE_VM_ERROR_CODE,
|
||||
..SimulateError::invalid_params()
|
||||
}),
|
||||
gas_used,
|
||||
gas_used: gas.used(),
|
||||
logs: Vec::new(),
|
||||
status: false,
|
||||
}
|
||||
}
|
||||
ExecutionResult::Revert { output, gas_used } => {
|
||||
ExecutionResult::Revert { output, gas } => {
|
||||
let error = Err::from_revert(output.clone());
|
||||
#[allow(clippy::needless_update)]
|
||||
SimCallResult {
|
||||
@@ -377,15 +377,15 @@ where
|
||||
code: SIMULATE_REVERT_CODE,
|
||||
..SimulateError::invalid_params()
|
||||
}),
|
||||
gas_used,
|
||||
gas_used: gas.used(),
|
||||
status: false,
|
||||
logs: Vec::new(),
|
||||
}
|
||||
}
|
||||
ExecutionResult::Success { output, gas_used, logs, .. } => SimCallResult {
|
||||
ExecutionResult::Success { output, gas, logs, .. } => SimCallResult {
|
||||
return_data: output.into_data(),
|
||||
error: None,
|
||||
gas_used,
|
||||
gas_used: gas.used(),
|
||||
logs: logs
|
||||
.into_iter()
|
||||
.map(|log| {
|
||||
|
||||
@@ -22,7 +22,6 @@ use reth_stages_api::{
|
||||
UnwindInput, UnwindOutput,
|
||||
};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_trie::KeccakKeyHasher;
|
||||
use std::{
|
||||
cmp::{max, Ordering},
|
||||
collections::BTreeMap,
|
||||
@@ -462,16 +461,9 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// Write output. When `use_hashed_state` is enabled, `write_state` skips writing to
|
||||
// plain account/storage tables and only writes bytecodes and changesets. The hashed
|
||||
// state is then written separately below.
|
||||
// write output
|
||||
provider.write_state(&state, OriginalValuesKnown::Yes, StateWriteConfig::default())?;
|
||||
|
||||
if provider.cached_storage_settings().use_hashed_state() {
|
||||
let hashed_state = state.hash_state_slow::<KeccakKeyHasher>();
|
||||
provider.write_hashed_state(&hashed_state.into_sorted())?;
|
||||
}
|
||||
|
||||
let db_write_duration = time.elapsed();
|
||||
debug!(
|
||||
target: "sync::stages::execution",
|
||||
@@ -1269,7 +1261,8 @@ mod tests {
|
||||
// but no receipt data is written.
|
||||
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
factory
|
||||
.set_storage_settings_cache(StorageSettings::v1().with_receipts_in_static_files(true));
|
||||
|
||||
// Setup with block 1
|
||||
let provider_rw = factory.database_provider_rw().unwrap();
|
||||
|
||||
@@ -9,9 +9,7 @@ use reth_db_api::{
|
||||
};
|
||||
use reth_etl::Collector;
|
||||
use reth_primitives_traits::Account;
|
||||
use reth_provider::{
|
||||
AccountExtReader, DBProvider, HashingWriter, StatsReader, StorageSettingsCache,
|
||||
};
|
||||
use reth_provider::{AccountExtReader, DBProvider, HashingWriter, StatsReader};
|
||||
use reth_stages_api::{
|
||||
AccountHashingCheckpoint, EntitiesCheckpoint, ExecInput, ExecOutput, Stage, StageCheckpoint,
|
||||
StageError, StageId, UnwindInput, UnwindOutput,
|
||||
@@ -136,11 +134,7 @@ impl Default for AccountHashingStage {
|
||||
|
||||
impl<Provider> Stage<Provider> for AccountHashingStage
|
||||
where
|
||||
Provider: DBProvider<Tx: DbTxMut>
|
||||
+ HashingWriter
|
||||
+ AccountExtReader
|
||||
+ StatsReader
|
||||
+ StorageSettingsCache,
|
||||
Provider: DBProvider<Tx: DbTxMut> + HashingWriter + AccountExtReader + StatsReader,
|
||||
{
|
||||
/// Return the id of the stage
|
||||
fn id(&self) -> StageId {
|
||||
@@ -148,21 +142,11 @@ where
|
||||
}
|
||||
|
||||
/// Execute the stage.
|
||||
///
|
||||
/// When `use_hashed_state` is enabled, this stage is a no-op because the execution stage
|
||||
/// writes directly to `HashedAccounts`. Otherwise, it hashes plain state to populate hashed
|
||||
/// tables.
|
||||
fn execute(&mut self, provider: &Provider, input: ExecInput) -> Result<ExecOutput, StageError> {
|
||||
if input.target_reached() {
|
||||
return Ok(ExecOutput::done(input.checkpoint()))
|
||||
}
|
||||
|
||||
// If using hashed state as canonical, execution already writes to `HashedAccounts`,
|
||||
// so this stage becomes a no-op.
|
||||
if provider.cached_storage_settings().use_hashed_state() {
|
||||
return Ok(ExecOutput::done(input.checkpoint().with_block_number(input.target())));
|
||||
}
|
||||
|
||||
let (from_block, to_block) = input.next_block_range().into_inner();
|
||||
|
||||
// if there are more blocks then threshold it is faster to go over Plain state and hash all
|
||||
@@ -250,14 +234,10 @@ where
|
||||
provider: &Provider,
|
||||
input: UnwindInput,
|
||||
) -> Result<UnwindOutput, StageError> {
|
||||
// NOTE: this runs in both v1 and v2 mode. In v2 mode, execution writes
|
||||
// directly to `HashedAccounts`, but the unwind must still revert those
|
||||
// entries here because `MerkleUnwind` runs after this stage (in unwind
|
||||
// order) and needs `HashedAccounts` to reflect the target block state
|
||||
// before it can verify the state root.
|
||||
let (range, unwind_progress, _) =
|
||||
input.unwind_block_range_with_threshold(self.commit_threshold);
|
||||
|
||||
// Aggregate all transition changesets and make a list of accounts that have been changed.
|
||||
provider.unwind_account_hashing_range(range)?;
|
||||
|
||||
let mut stage_checkpoint =
|
||||
|
||||
@@ -15,7 +15,6 @@ use reth_stages_api::{
|
||||
EntitiesCheckpoint, ExecInput, ExecOutput, Stage, StageCheckpoint, StageError, StageId,
|
||||
StorageHashingCheckpoint, UnwindInput, UnwindOutput,
|
||||
};
|
||||
use reth_storage_api::StorageSettingsCache;
|
||||
use reth_storage_errors::provider::ProviderResult;
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
@@ -69,11 +68,7 @@ impl Default for StorageHashingStage {
|
||||
|
||||
impl<Provider> Stage<Provider> for StorageHashingStage
|
||||
where
|
||||
Provider: DBProvider<Tx: DbTxMut>
|
||||
+ StorageReader
|
||||
+ HashingWriter
|
||||
+ StatsReader
|
||||
+ StorageSettingsCache,
|
||||
Provider: DBProvider<Tx: DbTxMut> + StorageReader + HashingWriter + StatsReader,
|
||||
{
|
||||
/// Return the id of the stage
|
||||
fn id(&self) -> StageId {
|
||||
@@ -87,12 +82,6 @@ where
|
||||
return Ok(ExecOutput::done(input.checkpoint()))
|
||||
}
|
||||
|
||||
// If use_hashed_state is enabled, execution writes directly to `HashedStorages`,
|
||||
// so this stage becomes a no-op.
|
||||
if provider.cached_storage_settings().use_hashed_state() {
|
||||
return Ok(ExecOutput::done(input.checkpoint().with_block_number(input.target())));
|
||||
}
|
||||
|
||||
let (from_block, to_block) = input.next_block_range().into_inner();
|
||||
|
||||
// if there are more blocks then threshold it is faster to go over Plain state and hash all
|
||||
@@ -187,11 +176,6 @@ where
|
||||
provider: &Provider,
|
||||
input: UnwindInput,
|
||||
) -> Result<UnwindOutput, StageError> {
|
||||
// NOTE: this runs in both v1 and v2 mode. In v2 mode, execution writes
|
||||
// directly to `HashedStorages`, but the unwind must still revert those
|
||||
// entries here because `MerkleUnwind` runs after this stage (in unwind
|
||||
// order) and needs `HashedStorages` to reflect the target block state
|
||||
// before it can verify the state root.
|
||||
let (range, unwind_progress, _) =
|
||||
input.unwind_block_range_with_threshold(self.commit_threshold);
|
||||
|
||||
|
||||
@@ -103,7 +103,7 @@ where
|
||||
|
||||
let mut range = input.next_block_range();
|
||||
let first_sync = input.checkpoint().block_number == 0;
|
||||
let use_rocksdb = provider.cached_storage_settings().storage_v2;
|
||||
let use_rocksdb = provider.cached_storage_settings().account_history_in_rocksdb;
|
||||
|
||||
// On first sync we might have history coming from genesis. We clear the table since it's
|
||||
// faster to rebuild from scratch.
|
||||
@@ -122,7 +122,7 @@ where
|
||||
|
||||
info!(target: "sync::stages::index_account_history::exec", ?first_sync, ?use_rocksdb, "Collecting indices");
|
||||
|
||||
let collector = if provider.cached_storage_settings().storage_v2 {
|
||||
let collector = if provider.cached_storage_settings().account_changesets_in_static_files {
|
||||
// Use the provider-based collection that can read from static files.
|
||||
collect_account_history_indices(provider, range.clone(), &self.etl_config)?
|
||||
} else {
|
||||
@@ -666,43 +666,32 @@ mod tests {
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
mod rocksdb_tests {
|
||||
use super::*;
|
||||
use reth_provider::{
|
||||
providers::StaticFileWriter, RocksDBProviderFactory, StaticFileProviderFactory,
|
||||
};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use reth_storage_api::StorageSettings;
|
||||
|
||||
/// Sets up v2 account test data: writes block body indices to MDBX and
|
||||
/// account changesets to static files (matching realistic v2 layout).
|
||||
fn setup_v2_account_data(db: &TestStageDB, block_range: std::ops::RangeInclusive<u64>) {
|
||||
db.factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in block_range.clone() {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let static_file_provider = db.factory.static_file_provider();
|
||||
let mut writer =
|
||||
static_file_provider.latest_writer(StaticFileSegment::AccountChangeSets).unwrap();
|
||||
for block in block_range {
|
||||
writer.append_account_changeset(vec![acc()], block).unwrap();
|
||||
}
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
/// Test that when `account_history_in_rocksdb` is enabled, the stage
|
||||
/// writes account history indices to `RocksDB` instead of MDBX.
|
||||
#[tokio::test]
|
||||
async fn execute_writes_to_rocksdb_when_enabled() {
|
||||
// init
|
||||
let db = TestStageDB::default();
|
||||
setup_v2_account_data(&db, 0..=10);
|
||||
|
||||
// Enable RocksDB for account history
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_account_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), ..Default::default() };
|
||||
let mut stage = IndexAccountHistoryStage::default();
|
||||
@@ -732,7 +721,22 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn unwind_works_when_rocksdb_enabled() {
|
||||
let db = TestStageDB::default();
|
||||
setup_v2_account_data(&db, 0..=10);
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_account_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), ..Default::default() };
|
||||
let mut stage = IndexAccountHistoryStage::default();
|
||||
@@ -768,7 +772,22 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn execute_incremental_sync() {
|
||||
let db = TestStageDB::default();
|
||||
setup_v2_account_data(&db, 0..=10);
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_account_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=5 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(5), ..Default::default() };
|
||||
let mut stage = IndexAccountHistoryStage::default();
|
||||
@@ -783,6 +802,18 @@ mod tests {
|
||||
let blocks: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks, (0..=5).collect::<Vec<_>>());
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 6..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), checkpoint: Some(StageCheckpoint::new(5)) };
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
|
||||
@@ -107,7 +107,7 @@ where
|
||||
|
||||
let mut range = input.next_block_range();
|
||||
let first_sync = input.checkpoint().block_number == 0;
|
||||
let use_rocksdb = provider.cached_storage_settings().storage_v2;
|
||||
let use_rocksdb = provider.cached_storage_settings().storages_history_in_rocksdb;
|
||||
|
||||
// On first sync we might have history coming from genesis. We clear the table since it's
|
||||
// faster to rebuild from scratch.
|
||||
@@ -125,7 +125,7 @@ where
|
||||
}
|
||||
|
||||
info!(target: "sync::stages::index_storage_history::exec", ?first_sync, ?use_rocksdb, "Collecting indices");
|
||||
let collector = if provider.cached_storage_settings().storage_v2 {
|
||||
let collector = if provider.cached_storage_settings().storage_changesets_in_static_files {
|
||||
collect_storage_history_indices(provider, range.clone(), &self.etl_config)?
|
||||
} else {
|
||||
collect_history_indices::<_, tables::StorageChangeSets, tables::StoragesHistory, _>(
|
||||
@@ -694,51 +694,33 @@ mod tests {
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
mod rocksdb_tests {
|
||||
use super::*;
|
||||
use reth_db_api::models::StorageBeforeTx;
|
||||
use reth_provider::{providers::StaticFileWriter, RocksDBProviderFactory};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use reth_storage_api::StorageSettings;
|
||||
|
||||
/// Sets up v2 storage test data: writes block body indices to MDBX and
|
||||
/// storage changesets to static files (matching realistic v2 layout).
|
||||
fn setup_v2_storage_data(db: &TestStageDB, block_range: std::ops::RangeInclusive<u64>) {
|
||||
db.factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in block_range.clone() {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let static_file_provider = db.factory.static_file_provider();
|
||||
let mut writer =
|
||||
static_file_provider.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
|
||||
for block in block_range {
|
||||
writer
|
||||
.append_storage_changeset(
|
||||
vec![StorageBeforeTx {
|
||||
address: ADDRESS,
|
||||
key: STORAGE_KEY,
|
||||
value: U256::ZERO,
|
||||
}],
|
||||
block,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
/// Test that when `storages_history_in_rocksdb` is enabled, the stage
|
||||
/// writes storage history indices to `RocksDB` instead of MDBX.
|
||||
#[tokio::test]
|
||||
async fn execute_writes_to_rocksdb_when_enabled() {
|
||||
let db = TestStageDB::default();
|
||||
setup_v2_storage_data(&db, 0..=10);
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
@@ -766,7 +748,25 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn unwind_works_when_rocksdb_enabled() {
|
||||
let db = TestStageDB::default();
|
||||
setup_v2_storage_data(&db, 0..=10);
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
@@ -803,7 +803,25 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn unwind_to_zero_keeps_block_zero() {
|
||||
let db = TestStageDB::default();
|
||||
setup_v2_storage_data(&db, 0..=5);
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=5 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(5), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
@@ -834,7 +852,25 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn execute_incremental_sync() {
|
||||
let db = TestStageDB::default();
|
||||
setup_v2_storage_data(&db, 0..=10);
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=5 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(5), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
@@ -849,6 +885,21 @@ mod tests {
|
||||
let blocks: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks, (0..=5).collect::<Vec<_>>());
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 6..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), checkpoint: Some(StageCheckpoint::new(5)) };
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
@@ -868,8 +919,27 @@ mod tests {
|
||||
use reth_db_api::models::sharded_key::NUM_OF_INDICES_IN_SHARD;
|
||||
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let num_blocks = (NUM_OF_INDICES_IN_SHARD * 2 + 100) as u64;
|
||||
setup_v2_storage_data(&db, 0..=num_blocks - 1);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..num_blocks {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(num_blocks - 1), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
|
||||
@@ -9,7 +9,7 @@ use reth_db_api::{
|
||||
use reth_primitives_traits::{GotExpected, SealedHeader};
|
||||
use reth_provider::{
|
||||
ChangeSetReader, DBProvider, HeaderProvider, ProviderError, StageCheckpointReader,
|
||||
StageCheckpointWriter, StatsReader, StorageChangeSetReader, StorageSettingsCache, TrieWriter,
|
||||
StageCheckpointWriter, StatsReader, StorageChangeSetReader, TrieWriter,
|
||||
};
|
||||
use reth_stages_api::{
|
||||
BlockErrorKind, EntitiesCheckpoint, ExecInput, ExecOutput, MerkleCheckpoint, Stage,
|
||||
@@ -160,7 +160,6 @@ where
|
||||
+ HeaderProvider
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ StorageSettingsCache
|
||||
+ StageCheckpointReader
|
||||
+ StageCheckpointWriter,
|
||||
{
|
||||
|
||||
@@ -540,7 +540,9 @@ mod tests {
|
||||
let mut rng = generators::rng();
|
||||
|
||||
let runner = SenderRecoveryTestRunner::default();
|
||||
runner.db.factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
runner.db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_transaction_senders_in_static_files(true),
|
||||
);
|
||||
let input = ExecInput {
|
||||
target: Some(target),
|
||||
checkpoint: Some(StageCheckpoint::new(stage_progress)),
|
||||
|
||||
@@ -200,7 +200,7 @@ where
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
if provider.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
|
||||
provider.commit_pending_rocksdb_batches()?;
|
||||
provider.rocksdb_provider().flush(&[Tables::TransactionHashNumbers.name()])?;
|
||||
}
|
||||
@@ -618,7 +618,9 @@ mod tests {
|
||||
let runner = TransactionLookupTestRunner::default();
|
||||
|
||||
// Enable RocksDB for transaction hash numbers
|
||||
runner.db.factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
runner.db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let input = ExecInput {
|
||||
target: Some(previous_stage),
|
||||
@@ -684,7 +686,9 @@ mod tests {
|
||||
let runner = TransactionLookupTestRunner::default();
|
||||
|
||||
// Enable RocksDB for transaction hash numbers
|
||||
runner.db.factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
runner.db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Insert blocks with transactions
|
||||
let blocks = random_block_range(
|
||||
|
||||
@@ -208,10 +208,7 @@ where
|
||||
|
||||
for (idx, changeset_result) in walker.enumerate() {
|
||||
let (BlockNumberAddress((block_number, address)), storage) = changeset_result?;
|
||||
cache
|
||||
.entry(AddressStorageKey((address, storage.key.as_b256())))
|
||||
.or_default()
|
||||
.push(block_number);
|
||||
cache.entry(AddressStorageKey((address, storage.key))).or_default().push(block_number);
|
||||
|
||||
if idx > 0 && idx % interval == 0 && total_changesets > 1000 {
|
||||
info!(target: "sync::stages::index_history", progress = %format!("{:.4}%", (idx as f64 / total_changesets as f64) * 100.0), "Collecting indices");
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use alloy_consensus::{constants::ETH_TO_WEI, Header, TxEip1559, TxReceipt};
|
||||
use alloy_eips::eip1559::INITIAL_BASE_FEE;
|
||||
use alloy_genesis::{Genesis, GenesisAccount};
|
||||
use alloy_primitives::{bytes, keccak256, Address, Bytes, TxKind, B256, U256};
|
||||
use alloy_primitives::{bytes, Address, Bytes, TxKind, B256, U256};
|
||||
use reth_chainspec::{ChainSpecBuilder, ChainSpecProvider, MAINNET};
|
||||
use reth_config::config::StageConfig;
|
||||
use reth_consensus::noop::NoopConsensus;
|
||||
@@ -36,7 +36,7 @@ use reth_stages::sets::DefaultStages;
|
||||
use reth_stages_api::{Pipeline, StageId};
|
||||
use reth_static_file::StaticFileProducer;
|
||||
use reth_storage_api::{
|
||||
ChangeSetReader, StateProvider, StorageChangeSetReader, StorageSettings, StorageSettingsCache,
|
||||
ChangeSetReader, StateProvider, StorageChangeSetReader, StorageSettingsCache,
|
||||
};
|
||||
use reth_testing_utils::generators::{self, generate_key, sign_tx_with_key_pair};
|
||||
use reth_trie::{HashedPostState, KeccakKeyHasher, StateRoot};
|
||||
@@ -79,7 +79,7 @@ fn assert_changesets_queryable(
|
||||
let settings = provider.cached_storage_settings();
|
||||
|
||||
// Verify storage changesets
|
||||
if settings.storage_v2 {
|
||||
if settings.storage_changesets_in_static_files {
|
||||
let static_file_provider = provider_factory.static_file_provider();
|
||||
static_file_provider.initialize_index()?;
|
||||
let storage_changesets =
|
||||
@@ -89,11 +89,6 @@ fn assert_changesets_queryable(
|
||||
"storage changesets should be queryable from static files for blocks {:?}",
|
||||
block_range
|
||||
);
|
||||
|
||||
// Verify keys are in hashed format (v2 mode)
|
||||
for (_, entry) in &storage_changesets {
|
||||
assert!(entry.key.is_hashed(), "v2: storage changeset keys should be tagged as hashed");
|
||||
}
|
||||
} else {
|
||||
let storage_changesets: Vec<_> = provider
|
||||
.tx_ref()
|
||||
@@ -105,20 +100,10 @@ fn assert_changesets_queryable(
|
||||
"storage changesets should be queryable from MDBX for blocks {:?}",
|
||||
block_range
|
||||
);
|
||||
|
||||
// Verify keys are plain (not hashed) in v1 mode
|
||||
for (_, entry) in &storage_changesets {
|
||||
let key = entry.key;
|
||||
assert_ne!(
|
||||
key,
|
||||
keccak256(key),
|
||||
"v1: storage changeset key should be plain (not its own keccak256)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify account changesets
|
||||
if settings.storage_v2 {
|
||||
if settings.account_changesets_in_static_files {
|
||||
let static_file_provider = provider_factory.static_file_provider();
|
||||
static_file_provider.initialize_index()?;
|
||||
let account_changesets =
|
||||
@@ -216,22 +201,19 @@ where
|
||||
pipeline
|
||||
}
|
||||
|
||||
/// Shared helper for pipeline forward sync and unwind tests.
|
||||
/// Tests pipeline with ALL stages enabled using both ETH transfers and contract storage changes.
|
||||
///
|
||||
/// This test:
|
||||
/// 1. Pre-funds a signer account and deploys a Counter contract in genesis
|
||||
/// 2. Each block contains two transactions:
|
||||
/// - ETH transfer to a recipient (account state changes)
|
||||
/// - Counter `increment()` call (storage state changes)
|
||||
/// 3. Runs the full pipeline with ALL stages enabled
|
||||
/// 4. Forward syncs to `num_blocks`, unwinds to `unwind_target`, then re-syncs back to `num_blocks`
|
||||
/// 4. Forward syncs to block 5, unwinds to block 2
|
||||
///
|
||||
/// When `storage_settings` is `Some`, the pipeline provider factory is configured with the given
|
||||
/// settings before genesis initialization (e.g. v2 storage mode).
|
||||
async fn run_pipeline_forward_and_unwind(
|
||||
storage_settings: Option<StorageSettings>,
|
||||
num_blocks: u64,
|
||||
unwind_target: u64,
|
||||
) -> eyre::Result<()> {
|
||||
/// This exercises both account and storage hashing/history stages.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_pipeline() -> eyre::Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
// Generate a keypair for signing transactions
|
||||
@@ -277,6 +259,7 @@ async fn run_pipeline_forward_and_unwind(
|
||||
let evm_config = EthEvmConfig::new(chain_spec.clone());
|
||||
|
||||
// Build blocks by actually executing transactions to get correct state roots
|
||||
let num_blocks = 5u64;
|
||||
let mut blocks: Vec<SealedBlock<Block>> = Vec::new();
|
||||
let mut parent_hash = genesis.hash();
|
||||
|
||||
@@ -401,15 +384,11 @@ async fn run_pipeline_forward_and_unwind(
|
||||
// This is needed because we wrote state during block generation for computing state roots
|
||||
let pipeline_provider_factory =
|
||||
create_test_provider_factory_with_chain_spec(chain_spec.clone());
|
||||
if let Some(settings) = storage_settings {
|
||||
pipeline_provider_factory.set_storage_settings_cache(settings);
|
||||
}
|
||||
init_genesis(&pipeline_provider_factory).expect("init genesis");
|
||||
let pipeline_genesis =
|
||||
pipeline_provider_factory.sealed_header(0)?.expect("genesis should exist");
|
||||
let pipeline_consensus = NoopConsensus::arc();
|
||||
|
||||
let blocks_clone = blocks.clone();
|
||||
let file_client = create_file_client_from_blocks(blocks);
|
||||
let max_block = file_client.max_block().unwrap();
|
||||
let tip = file_client.tip().expect("tip");
|
||||
@@ -438,7 +417,7 @@ async fn run_pipeline_forward_and_unwind(
|
||||
{
|
||||
let provider = pipeline_provider_factory.provider()?;
|
||||
let last_block = provider.last_block_number()?;
|
||||
assert_eq!(last_block, num_blocks, "should have synced {num_blocks} blocks");
|
||||
assert_eq!(last_block, 5, "should have synced 5 blocks");
|
||||
|
||||
for stage_id in [
|
||||
StageId::Headers,
|
||||
@@ -456,28 +435,29 @@ async fn run_pipeline_forward_and_unwind(
|
||||
let checkpoint = provider.get_stage_checkpoint(stage_id)?;
|
||||
assert_eq!(
|
||||
checkpoint.map(|c| c.block_number),
|
||||
Some(num_blocks),
|
||||
"{stage_id} checkpoint should be at block {num_blocks}"
|
||||
Some(5),
|
||||
"{stage_id} checkpoint should be at block 5"
|
||||
);
|
||||
}
|
||||
|
||||
// Verify the counter contract's storage was updated
|
||||
// After num_blocks blocks with 1 increment each, slot 0 should be num_blocks
|
||||
// After 5 blocks with 1 increment each, slot 0 should be 5
|
||||
let state = provider.latest();
|
||||
let counter_storage = state.storage(CONTRACT_ADDRESS, B256::ZERO)?;
|
||||
assert_eq!(
|
||||
counter_storage,
|
||||
Some(U256::from(num_blocks)),
|
||||
"Counter storage slot 0 should be {num_blocks} after {num_blocks} increments"
|
||||
Some(U256::from(5)),
|
||||
"Counter storage slot 0 should be 5 after 5 increments"
|
||||
);
|
||||
}
|
||||
|
||||
// Verify changesets are queryable before unwind
|
||||
// This validates that the #21561 fix works - unwind needs to read changesets from the correct
|
||||
// source
|
||||
assert_changesets_queryable(&pipeline_provider_factory, 1..=num_blocks)?;
|
||||
assert_changesets_queryable(&pipeline_provider_factory, 1..=5)?;
|
||||
|
||||
// Unwind to unwind_target
|
||||
// Unwind to block 2
|
||||
let unwind_target = 2u64;
|
||||
pipeline.unwind(unwind_target, None)?;
|
||||
|
||||
// Verify unwind
|
||||
@@ -504,114 +484,7 @@ async fn run_pipeline_forward_and_unwind(
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let state = provider.latest();
|
||||
let counter_storage = state.storage(CONTRACT_ADDRESS, B256::ZERO)?;
|
||||
assert_eq!(
|
||||
counter_storage,
|
||||
Some(U256::from(unwind_target)),
|
||||
"Counter storage slot 0 should be {unwind_target} after unwinding to block {unwind_target}"
|
||||
);
|
||||
}
|
||||
|
||||
// Re-sync: build a new pipeline starting from unwind_target and sync back to num_blocks
|
||||
let resync_file_client = create_file_client_from_blocks(blocks_clone);
|
||||
let resync_consensus = NoopConsensus::arc();
|
||||
let resync_stages_config = StageConfig::default();
|
||||
|
||||
let unwind_head = pipeline_provider_factory
|
||||
.sealed_header(unwind_target)?
|
||||
.expect("unwind target header should exist");
|
||||
|
||||
let mut resync_header_downloader =
|
||||
ReverseHeadersDownloaderBuilder::new(resync_stages_config.headers)
|
||||
.build(resync_file_client.clone(), resync_consensus.clone())
|
||||
.into_task();
|
||||
resync_header_downloader.update_local_head(unwind_head);
|
||||
resync_header_downloader.update_sync_target(SyncTarget::Tip(tip));
|
||||
|
||||
let mut resync_body_downloader = BodiesDownloaderBuilder::new(resync_stages_config.bodies)
|
||||
.build(resync_file_client, resync_consensus, pipeline_provider_factory.clone())
|
||||
.into_task();
|
||||
resync_body_downloader
|
||||
.set_download_range(unwind_target + 1..=max_block)
|
||||
.expect("set download range");
|
||||
|
||||
let resync_pipeline = build_pipeline(
|
||||
pipeline_provider_factory.clone(),
|
||||
resync_header_downloader,
|
||||
resync_body_downloader,
|
||||
max_block,
|
||||
tip,
|
||||
);
|
||||
|
||||
let (_resync_pipeline, resync_result) = resync_pipeline.run_as_fut(None).await;
|
||||
resync_result?;
|
||||
|
||||
// Verify re-sync
|
||||
{
|
||||
let provider = pipeline_provider_factory.provider()?;
|
||||
let last_block = provider.last_block_number()?;
|
||||
assert_eq!(last_block, num_blocks, "should have re-synced to {num_blocks} blocks");
|
||||
|
||||
for stage_id in [
|
||||
StageId::Headers,
|
||||
StageId::Bodies,
|
||||
StageId::SenderRecovery,
|
||||
StageId::Execution,
|
||||
StageId::AccountHashing,
|
||||
StageId::StorageHashing,
|
||||
StageId::MerkleExecute,
|
||||
StageId::TransactionLookup,
|
||||
StageId::IndexAccountHistory,
|
||||
StageId::IndexStorageHistory,
|
||||
StageId::Finish,
|
||||
] {
|
||||
let checkpoint = provider.get_stage_checkpoint(stage_id)?;
|
||||
assert_eq!(
|
||||
checkpoint.map(|c| c.block_number),
|
||||
Some(num_blocks),
|
||||
"{stage_id} checkpoint should be at block {num_blocks} after re-sync"
|
||||
);
|
||||
}
|
||||
|
||||
let state = provider.latest();
|
||||
let counter_storage = state.storage(CONTRACT_ADDRESS, B256::ZERO)?;
|
||||
assert_eq!(
|
||||
counter_storage,
|
||||
Some(U256::from(num_blocks)),
|
||||
"Counter storage slot 0 should be {num_blocks} after re-sync"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Tests pipeline with ALL stages enabled using both ETH transfers and contract storage changes.
|
||||
///
|
||||
/// This test:
|
||||
/// 1. Pre-funds a signer account and deploys a Counter contract in genesis
|
||||
/// 2. Each block contains two transactions:
|
||||
/// - ETH transfer to a recipient (account state changes)
|
||||
/// - Counter `increment()` call (storage state changes)
|
||||
/// 3. Runs the full pipeline with ALL stages enabled
|
||||
/// 4. Forward syncs to block 5, unwinds to block 2, then re-syncs to block 5
|
||||
///
|
||||
/// This exercises both account and storage hashing/history stages.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_pipeline() -> eyre::Result<()> {
|
||||
run_pipeline_forward_and_unwind(None, 5, 2).await
|
||||
}
|
||||
|
||||
/// Same as [`test_pipeline`] but runs with v2 storage settings (`use_hashed_state=true`,
|
||||
/// `is_v2()=true`, etc.).
|
||||
///
|
||||
/// In v2 mode:
|
||||
/// - The execution stage writes directly to `HashedAccounts`/`HashedStorages`
|
||||
/// - `AccountHashingStage` and `StorageHashingStage` are no-ops during forward execution
|
||||
/// - Changesets are stored in static files with pre-hashed storage keys
|
||||
/// - Unwind must still revert hashed state via the hashing stages before `MerkleUnwind` validates
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_pipeline_v2() -> eyre::Result<()> {
|
||||
run_pipeline_forward_and_unwind(Some(StorageSettings::v2()), 5, 2).await
|
||||
}
|
||||
|
||||
@@ -5,101 +5,208 @@ use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Storage configuration settings for this node.
|
||||
///
|
||||
/// Controls whether this node uses v2 storage layout (static files + `RocksDB` routing)
|
||||
/// or v1/legacy layout (everything in MDBX).
|
||||
///
|
||||
/// These should be set during `init_genesis` or `init_db` depending on whether we want dictate
|
||||
/// behaviour of new or old nodes respectively.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Compact, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, Compact)]
|
||||
#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
|
||||
#[add_arbitrary_tests(compact)]
|
||||
pub struct StorageSettings {
|
||||
/// Whether this node uses v2 storage layout.
|
||||
/// Whether this node always writes receipts to static files.
|
||||
///
|
||||
/// When `true`, enables all v2 storage features:
|
||||
/// - Receipts and transaction senders in static files
|
||||
/// - History indices in `RocksDB` (accounts, storages, transaction hashes)
|
||||
/// - Account and storage changesets in static files
|
||||
/// - Hashed state tables as canonical state representation
|
||||
///
|
||||
/// When `false`, uses v1/legacy layout (everything in MDBX).
|
||||
pub storage_v2: bool,
|
||||
/// If this is set to FALSE AND receipt pruning IS ENABLED, all receipts should be written to DB. Otherwise, they should be written to static files. This ensures that older nodes do not need to migrate their current DB tables to static files. For more, read: <https://github.com/paradigmxyz/reth/issues/18890#issuecomment-3457760097>
|
||||
#[serde(default)]
|
||||
pub receipts_in_static_files: bool,
|
||||
/// Whether this node always writes transaction senders to static files.
|
||||
#[serde(default)]
|
||||
pub transaction_senders_in_static_files: bool,
|
||||
/// Whether `StoragesHistory` is stored in `RocksDB`.
|
||||
#[serde(default)]
|
||||
pub storages_history_in_rocksdb: bool,
|
||||
/// Whether `TransactionHashNumbers` is stored in `RocksDB`.
|
||||
#[serde(default)]
|
||||
pub transaction_hash_numbers_in_rocksdb: bool,
|
||||
/// Whether `AccountsHistory` is stored in `RocksDB`.
|
||||
#[serde(default)]
|
||||
pub account_history_in_rocksdb: bool,
|
||||
/// Whether this node should read and write account changesets from static files.
|
||||
#[serde(default)]
|
||||
pub account_changesets_in_static_files: bool,
|
||||
/// Whether this node should read and write storage changesets from static files.
|
||||
#[serde(default)]
|
||||
pub storage_changesets_in_static_files: bool,
|
||||
/// Whether to use hashed state tables (`HashedAccounts`/`HashedStorages`) as the canonical
|
||||
/// state representation instead of plain state tables.
|
||||
#[serde(default)]
|
||||
pub use_hashed_state: bool,
|
||||
}
|
||||
|
||||
impl StorageSettings {
|
||||
/// Returns the default base `StorageSettings`.
|
||||
///
|
||||
/// When the `edge` feature is enabled, returns [`Self::v2()`] so that CI and
|
||||
/// edge builds automatically use v2 storage defaults. Otherwise returns
|
||||
/// [`Self::v1()`]. The `--storage.v2` CLI flag can also opt into v2 at runtime
|
||||
/// regardless of feature flags.
|
||||
/// Always returns [`Self::v1()`]. Use the `--storage.v2` CLI flag to opt into
|
||||
/// [`Self::v2()`] at runtime. The `rocksdb` feature only makes the v2 backend
|
||||
/// *available*; it does not activate it by default.
|
||||
pub const fn base() -> Self {
|
||||
#[cfg(feature = "edge")]
|
||||
{
|
||||
Self::v2()
|
||||
}
|
||||
#[cfg(not(feature = "edge"))]
|
||||
{
|
||||
Self::v1()
|
||||
}
|
||||
Self::v1()
|
||||
}
|
||||
|
||||
/// Creates `StorageSettings` for v2 nodes with all storage features enabled:
|
||||
/// - Receipts and transaction senders in static files
|
||||
/// - History indices in `RocksDB` (storages, accounts, transaction hashes)
|
||||
/// - Account and storage changesets in static files
|
||||
/// - Hashed state as canonical state representation
|
||||
///
|
||||
/// Use this when the `--storage.v2` CLI flag is set.
|
||||
pub const fn v2() -> Self {
|
||||
Self { storage_v2: true }
|
||||
Self {
|
||||
receipts_in_static_files: true,
|
||||
transaction_senders_in_static_files: true,
|
||||
account_changesets_in_static_files: true,
|
||||
storage_changesets_in_static_files: true,
|
||||
storages_history_in_rocksdb: true,
|
||||
transaction_hash_numbers_in_rocksdb: true,
|
||||
account_history_in_rocksdb: true,
|
||||
use_hashed_state: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates `StorageSettings` for v1/legacy nodes.
|
||||
///
|
||||
/// This keeps all data in MDBX, matching the original storage layout.
|
||||
/// This explicitly sets `receipts_in_static_files` and `transaction_senders_in_static_files` to
|
||||
/// `false`, ensuring older nodes continue writing receipts and transaction senders to the
|
||||
/// database when receipt pruning is enabled.
|
||||
pub const fn v1() -> Self {
|
||||
Self { storage_v2: false }
|
||||
Self {
|
||||
receipts_in_static_files: false,
|
||||
transaction_senders_in_static_files: false,
|
||||
storages_history_in_rocksdb: false,
|
||||
transaction_hash_numbers_in_rocksdb: false,
|
||||
account_history_in_rocksdb: false,
|
||||
account_changesets_in_static_files: false,
|
||||
storage_changesets_in_static_files: false,
|
||||
use_hashed_state: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `true` if this node uses v2 storage layout.
|
||||
pub const fn is_v2(&self) -> bool {
|
||||
self.storage_v2
|
||||
/// Sets the `receipts_in_static_files` flag to the provided value.
|
||||
pub const fn with_receipts_in_static_files(mut self, value: bool) -> Self {
|
||||
self.receipts_in_static_files = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Whether receipts are stored in static files.
|
||||
pub const fn receipts_in_static_files(&self) -> bool {
|
||||
self.storage_v2
|
||||
/// Sets the `transaction_senders_in_static_files` flag to the provided value.
|
||||
pub const fn with_transaction_senders_in_static_files(mut self, value: bool) -> Self {
|
||||
self.transaction_senders_in_static_files = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Whether transaction senders are stored in static files.
|
||||
pub const fn transaction_senders_in_static_files(&self) -> bool {
|
||||
self.storage_v2
|
||||
/// Sets the `storages_history_in_rocksdb` flag to the provided value.
|
||||
pub const fn with_storages_history_in_rocksdb(mut self, value: bool) -> Self {
|
||||
self.storages_history_in_rocksdb = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Whether storages history is stored in `RocksDB`.
|
||||
pub const fn storages_history_in_rocksdb(&self) -> bool {
|
||||
self.storage_v2
|
||||
/// Sets the `transaction_hash_numbers_in_rocksdb` flag to the provided value.
|
||||
pub const fn with_transaction_hash_numbers_in_rocksdb(mut self, value: bool) -> Self {
|
||||
self.transaction_hash_numbers_in_rocksdb = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Whether transaction hash numbers are stored in `RocksDB`.
|
||||
pub const fn transaction_hash_numbers_in_rocksdb(&self) -> bool {
|
||||
self.storage_v2
|
||||
/// Sets the `account_history_in_rocksdb` flag to the provided value.
|
||||
pub const fn with_account_history_in_rocksdb(mut self, value: bool) -> Self {
|
||||
self.account_history_in_rocksdb = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Whether account history is stored in `RocksDB`.
|
||||
pub const fn account_history_in_rocksdb(&self) -> bool {
|
||||
self.storage_v2
|
||||
/// Sets the `account_changesets_in_static_files` flag to the provided value.
|
||||
pub const fn with_account_changesets_in_static_files(mut self, value: bool) -> Self {
|
||||
self.account_changesets_in_static_files = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Whether to use hashed state tables (`HashedAccounts`/`HashedStorages`) as the canonical
|
||||
/// state representation instead of plain state tables. Implied by v2 storage layout.
|
||||
pub const fn use_hashed_state(&self) -> bool {
|
||||
self.storage_v2
|
||||
/// Sets the `storage_changesets_in_static_files` flag to the provided value.
|
||||
pub const fn with_storage_changesets_in_static_files(mut self, value: bool) -> Self {
|
||||
self.storage_changesets_in_static_files = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the `use_hashed_state` flag to the provided value.
|
||||
pub const fn with_use_hashed_state(mut self, value: bool) -> Self {
|
||||
self.use_hashed_state = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets `receipts_in_static_files` if `value` is `Some`.
|
||||
pub const fn with_receipts_in_static_files_opt(mut self, value: Option<bool>) -> Self {
|
||||
if let Some(v) = value {
|
||||
self.receipts_in_static_files = v;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets `transaction_senders_in_static_files` if `value` is `Some`.
|
||||
pub const fn with_transaction_senders_in_static_files_opt(
|
||||
mut self,
|
||||
value: Option<bool>,
|
||||
) -> Self {
|
||||
if let Some(v) = value {
|
||||
self.transaction_senders_in_static_files = v;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets `account_changesets_in_static_files` if `value` is `Some`.
|
||||
pub const fn with_account_changesets_in_static_files_opt(
|
||||
mut self,
|
||||
value: Option<bool>,
|
||||
) -> Self {
|
||||
if let Some(v) = value {
|
||||
self.account_changesets_in_static_files = v;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets `storage_changesets_in_static_files` if `value` is `Some`.
|
||||
pub const fn with_storage_changesets_in_static_files_opt(
|
||||
mut self,
|
||||
value: Option<bool>,
|
||||
) -> Self {
|
||||
if let Some(v) = value {
|
||||
self.storage_changesets_in_static_files = v;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets `transaction_hash_numbers_in_rocksdb` if `value` is `Some`.
|
||||
pub const fn with_transaction_hash_numbers_in_rocksdb_opt(
|
||||
mut self,
|
||||
value: Option<bool>,
|
||||
) -> Self {
|
||||
if let Some(v) = value {
|
||||
self.transaction_hash_numbers_in_rocksdb = v;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets `storages_history_in_rocksdb` if `value` is `Some`.
|
||||
pub const fn with_storages_history_in_rocksdb_opt(mut self, value: Option<bool>) -> Self {
|
||||
if let Some(v) = value {
|
||||
self.storages_history_in_rocksdb = v;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets `account_history_in_rocksdb` if `value` is `Some`.
|
||||
pub const fn with_account_history_in_rocksdb_opt(mut self, value: Option<bool>) -> Self {
|
||||
if let Some(v) = value {
|
||||
self.account_history_in_rocksdb = v;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns `true` if any tables are configured to be stored in `RocksDB`.
|
||||
pub const fn any_in_rocksdb(&self) -> bool {
|
||||
self.storage_v2
|
||||
self.transaction_hash_numbers_in_rocksdb ||
|
||||
self.account_history_in_rocksdb ||
|
||||
self.storages_history_in_rocksdb
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,13 +214,13 @@ where
|
||||
// not the genesis block number. This would cause increment_block(N) to fail.
|
||||
let static_file_provider = provider_rw.static_file_provider();
|
||||
if genesis_block_number > 0 {
|
||||
if genesis_storage_settings.storage_v2 {
|
||||
if genesis_storage_settings.account_changesets_in_static_files {
|
||||
static_file_provider
|
||||
.get_writer(genesis_block_number, StaticFileSegment::AccountChangeSets)?
|
||||
.user_header_mut()
|
||||
.set_expected_block_start(genesis_block_number);
|
||||
}
|
||||
if genesis_storage_settings.storage_v2 {
|
||||
if genesis_storage_settings.storage_changesets_in_static_files {
|
||||
static_file_provider
|
||||
.get_writer(genesis_block_number, StaticFileSegment::StorageChangeSets)?
|
||||
.user_header_mut()
|
||||
@@ -259,7 +259,7 @@ where
|
||||
.user_header_mut()
|
||||
.set_block_range(genesis_block_number, genesis_block_number);
|
||||
|
||||
if genesis_storage_settings.storage_v2 {
|
||||
if genesis_storage_settings.transaction_senders_in_static_files {
|
||||
static_file_provider
|
||||
.get_writer(genesis_block_number, StaticFileSegment::TransactionSenders)?
|
||||
.user_header_mut()
|
||||
@@ -1052,7 +1052,7 @@ mod tests {
|
||||
)
|
||||
};
|
||||
|
||||
let (accounts, storages) = if settings.storage_v2 {
|
||||
let (accounts, storages) = if settings.account_history_in_rocksdb {
|
||||
collect_rocksdb(&rocksdb)
|
||||
} else {
|
||||
collect_from_mdbx(&factory)
|
||||
@@ -1075,7 +1075,10 @@ mod tests {
|
||||
init_genesis_with_settings(&factory, StorageSettings::v1()).unwrap();
|
||||
|
||||
// Request different settings - should warn but succeed
|
||||
let result = init_genesis_with_settings(&factory, StorageSettings::v2());
|
||||
let result = init_genesis_with_settings(
|
||||
&factory,
|
||||
StorageSettings::v1().with_receipts_in_static_files(true),
|
||||
);
|
||||
|
||||
// Should succeed (warning is logged, not an error)
|
||||
assert!(result.is_ok());
|
||||
@@ -1084,7 +1087,7 @@ mod tests {
|
||||
#[test]
|
||||
fn allow_same_storage_settings() {
|
||||
let factory = create_test_provider_factory_with_chain_spec(MAINNET.clone());
|
||||
let settings = StorageSettings::v2();
|
||||
let settings = StorageSettings::v1().with_receipts_in_static_files(true);
|
||||
init_genesis_with_settings(&factory, settings).unwrap();
|
||||
|
||||
let result = init_genesis_with_settings(&factory, settings);
|
||||
|
||||
@@ -27,8 +27,6 @@ alloy-primitives.workspace = true
|
||||
# mdbx
|
||||
reth-libmdbx = { workspace = true, optional = true, features = ["return-borrowed", "read-tx-timeouts"] }
|
||||
eyre = { workspace = true, optional = true }
|
||||
reth-mdbx-viz = { workspace = true, optional = true }
|
||||
tokio = { workspace = true, optional = true, features = ["full"] }
|
||||
|
||||
# metrics
|
||||
reth-metrics = { workspace = true, optional = true }
|
||||
@@ -99,7 +97,6 @@ op = [
|
||||
"reth-db-api/op",
|
||||
"reth-primitives-traits/op",
|
||||
]
|
||||
pageviz = ["reth-libmdbx/pageviz", "dep:reth-mdbx-viz", "dep:tokio", "mdbx"]
|
||||
disable-lock = []
|
||||
|
||||
[[bench]]
|
||||
|
||||
@@ -522,81 +522,6 @@ impl DatabaseEnv {
|
||||
self
|
||||
}
|
||||
|
||||
/// Starts the real-time page access visualization server.
|
||||
///
|
||||
/// This enables the C-level page access instrumentation hooks,
|
||||
/// starts a background drainer thread that coalesces events,
|
||||
/// and launches an HTTP+WebSocket server on the given port.
|
||||
///
|
||||
/// The server serves an interactive HTML visualization at `http://localhost:{port}/`
|
||||
/// showing page accesses in real-time with fading highlights:
|
||||
/// - Blue = read, Red = write, Yellow = free
|
||||
#[cfg(feature = "pageviz")]
|
||||
pub fn start_pageviz(&self, port: u16, db_path: std::path::PathBuf) {
|
||||
use reth_libmdbx::pageviz::PageVizDrainer;
|
||||
|
||||
let max_dbi = self.dbis.values().copied().max().unwrap_or(0) as usize;
|
||||
let mut dbi_names = vec![String::new(); max_dbi + 1];
|
||||
dbi_names[0] = "FREE_DBI".to_string();
|
||||
if dbi_names.len() > 1 {
|
||||
dbi_names[1] = "MAIN_DBI".to_string();
|
||||
}
|
||||
for (name, &dbi) in self.dbis.iter() {
|
||||
dbi_names[dbi as usize] = name.to_string();
|
||||
}
|
||||
|
||||
let mut name_to_dbi = std::collections::HashMap::new();
|
||||
for (&name, &dbi) in self.dbis.iter() {
|
||||
name_to_dbi.insert(name, dbi as u8);
|
||||
}
|
||||
|
||||
let mdbx_dat = db_path.join("mdbx.dat");
|
||||
let walk = match reth_mdbx_viz::walker::walk_mdbx(&mdbx_dat, &name_to_dbi) {
|
||||
Ok(w) => w,
|
||||
Err(e) => {
|
||||
reth_tracing::tracing::error!("pageviz: failed to walk B-tree: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let page_count = walk.page_count;
|
||||
let page_size = walk.page_size;
|
||||
|
||||
let mut drainer = PageVizDrainer::new();
|
||||
drainer.enable();
|
||||
let rx = drainer.start(60);
|
||||
|
||||
let config = reth_mdbx_viz::VizConfig {
|
||||
port,
|
||||
page_count: page_count as u64,
|
||||
page_size: page_size as u32,
|
||||
dbi_names,
|
||||
owner_map: walk.owner_map,
|
||||
tree_info: walk.tree_info,
|
||||
db_path: mdbx_dat,
|
||||
};
|
||||
|
||||
// Spawn the viz server on a background tokio runtime
|
||||
std::thread::Builder::new()
|
||||
.name("mdbx-viz-server".into())
|
||||
.spawn(move || {
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("failed to build tokio runtime for pageviz");
|
||||
rt.block_on(async {
|
||||
if let Err(e) = reth_mdbx_viz::start_viz_server(config, rx).await {
|
||||
reth_tracing::tracing::error!("pageviz server error: {e}");
|
||||
}
|
||||
});
|
||||
// Keep drainer alive while server runs
|
||||
drop(drainer);
|
||||
})
|
||||
.expect("failed to spawn pageviz server thread");
|
||||
|
||||
reth_tracing::tracing::info!("pageviz server started on port {port}");
|
||||
}
|
||||
|
||||
/// Creates all the tables defined in [`Tables`], if necessary.
|
||||
///
|
||||
/// This keeps tracks of the created table handles and stores them for better efficiency.
|
||||
|
||||
@@ -45,22 +45,11 @@ pub fn init_db_for<P: AsRef<Path>, TS: TableSet>(
|
||||
path: P,
|
||||
args: DatabaseArguments,
|
||||
) -> eyre::Result<DatabaseEnv> {
|
||||
let db_path = path.as_ref().to_path_buf();
|
||||
let client_version = args.client_version().clone();
|
||||
let mut db = create_db(path, args)?;
|
||||
db.create_and_track_tables_for::<TS>()?;
|
||||
db.record_client_version(client_version)?;
|
||||
drop_orphan_tables(&db);
|
||||
|
||||
#[cfg(feature = "pageviz")]
|
||||
{
|
||||
let port = std::env::var("RETH_PAGEVIZ_PORT")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(3141u16);
|
||||
db.start_pageviz(port, db_path);
|
||||
}
|
||||
|
||||
Ok(db)
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,6 @@ dashmap = { workspace = true, features = ["inline"], optional = true }
|
||||
default = []
|
||||
return-borrowed = []
|
||||
read-tx-timeouts = ["dep:dashmap"]
|
||||
pageviz = ["reth-mdbx-sys/pageviz"]
|
||||
|
||||
[dev-dependencies]
|
||||
criterion.workspace = true
|
||||
|
||||
@@ -11,7 +11,3 @@ repository.workspace = true
|
||||
[build-dependencies]
|
||||
cc.workspace = true
|
||||
bindgen = { workspace = true, features = ["runtime"] }
|
||||
|
||||
[features]
|
||||
default = []
|
||||
pageviz = []
|
||||
|
||||
@@ -30,11 +30,6 @@ fn main() {
|
||||
#[cfg(not(debug_assertions))]
|
||||
cc.define("MDBX_DEBUG", "0").define("NDEBUG", None);
|
||||
|
||||
// Enable page visualization instrumentation
|
||||
if std::env::var("CARGO_FEATURE_PAGEVIZ").is_ok() {
|
||||
cc.define("MDBX_PAGEVIZ", "1");
|
||||
}
|
||||
|
||||
// Propagate `-C target-cpu=native`
|
||||
let rustflags = env::var("CARGO_ENCODED_RUSTFLAGS").unwrap();
|
||||
if rustflags.contains("target-cpu=native") &&
|
||||
@@ -43,11 +38,7 @@ fn main() {
|
||||
cc.flag("-march=native");
|
||||
}
|
||||
|
||||
cc.file(mdbx.join("mdbx.c"));
|
||||
if std::env::var("CARGO_FEATURE_PAGEVIZ").is_ok() {
|
||||
cc.file(mdbx.join("mdbx_pageviz.c"));
|
||||
}
|
||||
cc.compile("libmdbx.a");
|
||||
cc.file(mdbx.join("mdbx.c")).compile("libmdbx.a");
|
||||
}
|
||||
|
||||
fn generate_bindings(mdbx: &Path, out_file: &Path) {
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
#include MDBX_CONFIG_H
|
||||
#endif
|
||||
|
||||
|
||||
/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */
|
||||
#if (defined(MDBX_DEBUG) && MDBX_DEBUG > 0) || (defined(MDBX_FORCE_ASSERTIONS) && MDBX_FORCE_ASSERTIONS)
|
||||
#undef NDEBUG
|
||||
@@ -47,7 +46,6 @@
|
||||
#define _DARWIN_C_SOURCE
|
||||
#endif /* _DARWIN_C_SOURCE */
|
||||
|
||||
#include "mdbx_pageviz.h"
|
||||
#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && !defined(__USE_MINGW_ANSI_STDIO)
|
||||
#define __USE_MINGW_ANSI_STDIO 1
|
||||
#endif /* MinGW */
|
||||
@@ -6195,7 +6193,6 @@ MDBX_INTERNAL int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const p
|
||||
static inline int page_touch(MDBX_cursor *mc) {
|
||||
page_t *const mp = mc->pg[mc->top];
|
||||
MDBX_txn *txn = mc->txn;
|
||||
MDBX_PAGEVIZ_WRITE(mc, mp->pgno);
|
||||
|
||||
tASSERT(txn, mc->txn->flags & MDBX_TXN_DIRTY);
|
||||
tASSERT(txn, F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY));
|
||||
@@ -19929,9 +19926,6 @@ __cold int dxb_resize(MDBX_env *const env, const pgno_t used_pgno, const pgno_t
|
||||
eASSERT(env, env->dxb_mmap.limit >= env->dxb_mmap.current);
|
||||
|
||||
if (rc == MDBX_SUCCESS) {
|
||||
#if defined(MDBX_PAGEVIZ) && MDBX_PAGEVIZ
|
||||
mdbx_pageviz_set_mapping(env->dxb_mmap.base, env->dxb_mmap.current, env->ps);
|
||||
#endif
|
||||
eASSERT(env, limit_bytes == env->dxb_mmap.limit);
|
||||
eASSERT(env, size_bytes <= env->dxb_mmap.filesize);
|
||||
if (mode == explicit_resize)
|
||||
@@ -20345,10 +20339,6 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, const mdbx_mode_t mode_bit
|
||||
if (unlikely(err != MDBX_SUCCESS))
|
||||
return err;
|
||||
|
||||
#if defined(MDBX_PAGEVIZ) && MDBX_PAGEVIZ
|
||||
mdbx_pageviz_set_mapping(env->dxb_mmap.base, env->dxb_mmap.current, env->ps);
|
||||
#endif
|
||||
|
||||
#if defined(MADV_DONTDUMP)
|
||||
err =
|
||||
madvise(env->dxb_mmap.base, env->dxb_mmap.limit, MADV_DONTDUMP) ? ignore_enosys_and_eagain(errno) : MDBX_SUCCESS;
|
||||
@@ -23100,7 +23090,6 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
|
||||
return err;
|
||||
for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) {
|
||||
pnl_append_prereserved(txn->tw.retired_pages, lp->pgno);
|
||||
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_FREE, 0, lp->pgno);
|
||||
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
|
||||
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
|
||||
}
|
||||
@@ -23115,7 +23104,6 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
|
||||
for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) {
|
||||
tASSERT(txn, lp->flags == P_LOOSE);
|
||||
loose[++count] = lp->pgno;
|
||||
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_FREE, 0, lp->pgno);
|
||||
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
|
||||
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
|
||||
}
|
||||
@@ -31638,7 +31626,6 @@ static __always_inline pgr_t page_get_inline(const uint16_t ILL, const MDBX_curs
|
||||
if (unlikely(r.err != MDBX_SUCCESS))
|
||||
goto bailout;
|
||||
#endif /* MDBX_DISABLE_VALIDATION */
|
||||
MDBX_PAGEVIZ_READ(mc, pgno);
|
||||
return r;
|
||||
}
|
||||
|
||||
@@ -31855,7 +31842,6 @@ pgr_t page_new(MDBX_cursor *mc, const unsigned flags) {
|
||||
return ret;
|
||||
|
||||
DEBUG("db %zu allocated new page %" PRIaPGNO, cursor_dbi(mc), ret.page->pgno);
|
||||
MDBX_PAGEVIZ_WRITE(mc, ret.page->pgno);
|
||||
ret.page->flags = (uint16_t)flags;
|
||||
cASSERT(mc, *cursor_dbi_state(mc) & DBI_DIRTY);
|
||||
cASSERT(mc, mc->txn->flags & MDBX_TXN_DIRTY);
|
||||
@@ -31884,7 +31870,6 @@ pgr_t page_new_large(MDBX_cursor *mc, const size_t npages) {
|
||||
return ret;
|
||||
|
||||
DEBUG("dbi %zu allocated new large-page %" PRIaPGNO ", num %zu", cursor_dbi(mc), ret.page->pgno, npages);
|
||||
MDBX_PAGEVIZ_WRITE(mc, ret.page->pgno);
|
||||
ret.page->flags = P_LARGE;
|
||||
cASSERT(mc, *cursor_dbi_state(mc) & DBI_DIRTY);
|
||||
cASSERT(mc, mc->txn->flags & MDBX_TXN_DIRTY);
|
||||
@@ -32030,7 +32015,6 @@ __hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *c
|
||||
DEBUG("touched db %d page %" PRIaPGNO " -> %" PRIaPGNO, cursor_dbi_dbg(mc), mp->pgno, pgno);
|
||||
tASSERT(txn, mp->pgno != pgno);
|
||||
pnl_append_prereserved(txn->tw.retired_pages, mp->pgno);
|
||||
MDBX_PAGEVIZ_FREE(mc, mp->pgno);
|
||||
/* Update the parent page, if any, to point to the new page */
|
||||
if (likely(mc->top)) {
|
||||
page_t *parent = mc->pg[mc->top - 1];
|
||||
@@ -32227,7 +32211,6 @@ int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, page_t *mp /* maybe null
|
||||
unsigned pageflags /* maybe unknown/zero */) {
|
||||
int rc;
|
||||
MDBX_txn *const txn = mc->txn;
|
||||
MDBX_PAGEVIZ_FREE(mc, pgno);
|
||||
tASSERT(txn, !mp || (mp->pgno == pgno && mp->flags == pageflags));
|
||||
|
||||
/* During deleting entire subtrees, it is reasonable and possible to avoid
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
#include "mdbx_pageviz.h"
|
||||
|
||||
#if defined(MDBX_PAGEVIZ) && MDBX_PAGEVIZ
|
||||
|
||||
/* ── Global state ─────────────────────────────────────────────────────── */
|
||||
|
||||
mdbx_pageviz_state_t *mdbx_pageviz_global = NULL;
|
||||
|
||||
/* ── Thread-local ring index ──────────────────────────────────────────── */
|
||||
|
||||
MDBX_PAGEVIZ_TLS uint32_t mdbx_pageviz_tls_ring = MDBX_PAGEVIZ_TLS_UNREGISTERED;
|
||||
|
||||
/* ── Lifecycle ────────────────────────────────────────────────────────── */
|
||||
|
||||
mdbx_pageviz_state_t *mdbx_pageviz_create(void) {
|
||||
mdbx_pageviz_state_t *state = calloc(1, sizeof(mdbx_pageviz_state_t));
|
||||
if (state)
|
||||
mdbx_pageviz_global = state;
|
||||
return state;
|
||||
}
|
||||
|
||||
void mdbx_pageviz_destroy(mdbx_pageviz_state_t *state) {
|
||||
if (!state)
|
||||
return;
|
||||
if (mdbx_pageviz_global == state)
|
||||
mdbx_pageviz_global = NULL;
|
||||
free(state);
|
||||
}
|
||||
|
||||
/* ── Enable / Disable ─────────────────────────────────────────────────── */
|
||||
|
||||
void mdbx_pageviz_enable(mdbx_pageviz_state_t *state) {
|
||||
atomic_store_explicit(&state->enabled, 1, memory_order_release);
|
||||
}
|
||||
|
||||
void mdbx_pageviz_disable(mdbx_pageviz_state_t *state) {
|
||||
atomic_store_explicit(&state->enabled, 0, memory_order_release);
|
||||
}
|
||||
|
||||
/* ── Drain ────────────────────────────────────────────────────────────── */
|
||||
|
||||
uint32_t mdbx_pageviz_drain(mdbx_pageviz_state_t *state, uint32_t ring_idx,
|
||||
uint64_t *out_buf, uint32_t max_count) {
|
||||
if (!state || ring_idx >= MDBX_PAGEVIZ_MAX_RINGS || !out_buf || max_count == 0)
|
||||
return 0;
|
||||
|
||||
mdbx_pageviz_ring_t *ring = &state->rings[ring_idx];
|
||||
uint32_t head = atomic_load_explicit(&ring->published_head,
|
||||
memory_order_acquire);
|
||||
uint32_t tail = atomic_load_explicit(&ring->consumer_tail,
|
||||
memory_order_relaxed);
|
||||
|
||||
if (head == tail)
|
||||
return 0;
|
||||
|
||||
uint32_t avail = head - tail;
|
||||
uint32_t count = avail < max_count ? avail : max_count;
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
uint32_t slot = (tail + i) & MDBX_PAGEVIZ_RING_MASK;
|
||||
out_buf[i] = ring->events[slot];
|
||||
}
|
||||
|
||||
atomic_store_explicit(&ring->consumer_tail, tail + count,
|
||||
memory_order_release);
|
||||
return count;
|
||||
}
|
||||
|
||||
/* ── Queries ──────────────────────────────────────────────────────────── */
|
||||
|
||||
uint32_t mdbx_pageviz_ring_count(mdbx_pageviz_state_t *state) {
|
||||
return atomic_load_explicit(&state->ring_count, memory_order_relaxed);
|
||||
}
|
||||
|
||||
uint64_t mdbx_pageviz_dropped(mdbx_pageviz_state_t *state, uint32_t ring_idx) {
|
||||
return atomic_load_explicit(&state->rings[ring_idx].dropped,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
|
||||
/* ── Mapping info ────────────────────────────────────────────────────── */
|
||||
|
||||
void mdbx_pageviz_set_mapping(void *base, size_t len, uint32_t mdbx_page_size) {
|
||||
mdbx_pageviz_state_t *state = mdbx_pageviz_global;
|
||||
if (!state)
|
||||
return;
|
||||
state->mapping.mdbx_page_size = mdbx_page_size;
|
||||
state->mapping.sys_page_size = (uint32_t)sysconf(_SC_PAGESIZE);
|
||||
state->mapping.len = len;
|
||||
/* base written last so consumers see consistent len+page_size first */
|
||||
__atomic_store_n((void *volatile *)&state->mapping.base, base, __ATOMIC_RELEASE);
|
||||
}
|
||||
|
||||
int mdbx_pageviz_get_mapping(void **out_base, size_t *out_len,
|
||||
uint32_t *out_mdbx_ps, uint32_t *out_sys_ps) {
|
||||
mdbx_pageviz_state_t *state = mdbx_pageviz_global;
|
||||
if (!state)
|
||||
return 0;
|
||||
void *b = __atomic_load_n((void *volatile *)&state->mapping.base, __ATOMIC_ACQUIRE);
|
||||
if (!b)
|
||||
return 0;
|
||||
*out_base = b;
|
||||
*out_len = state->mapping.len;
|
||||
*out_mdbx_ps = state->mapping.mdbx_page_size;
|
||||
*out_sys_ps = state->mapping.sys_page_size;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* ── Block marker emit (non-inline wrapper for Rust FFI) ──────────────── */
|
||||
|
||||
void mdbx_pageviz_emit_block_marker(uint8_t op, uint32_t block_number,
|
||||
uint16_t tx_count, uint8_t duration_encoded,
|
||||
uint8_t gas_encoded) {
|
||||
mdbx_pageviz_emit(op, ((uint32_t)gas_encoded << 24) | ((uint32_t)duration_encoded << 16) | (uint32_t)tx_count, block_number);
|
||||
}
|
||||
|
||||
#else /* !MDBX_PAGEVIZ */
|
||||
|
||||
typedef int mdbx_pageviz_empty_tu_;
|
||||
|
||||
#endif /* MDBX_PAGEVIZ */
|
||||
@@ -1,195 +0,0 @@
|
||||
/* mdbx_pageviz.h — Standalone page-access visualization ring buffer for MDBX.
|
||||
*
|
||||
* Self-contained: no dependencies on MDBX internals.
|
||||
* Gate everything behind MDBX_PAGEVIZ; when not defined, all macros expand to
|
||||
* nothing (zero overhead).
|
||||
*
|
||||
* Compatible with C11 and C17. */
|
||||
|
||||
#ifndef MDBX_PAGEVIZ_H
|
||||
#define MDBX_PAGEVIZ_H
|
||||
|
||||
/* ── Disabled stub macros ─────────────────────────────────────────────── */
|
||||
|
||||
#if !defined(MDBX_PAGEVIZ) || !(MDBX_PAGEVIZ)
|
||||
|
||||
#define MDBX_PAGEVIZ_READ(mc, pgno) ((void)0)
|
||||
#define MDBX_PAGEVIZ_WRITE(mc, pgno) ((void)0)
|
||||
#define MDBX_PAGEVIZ_FREE(mc, pgno) ((void)0)
|
||||
|
||||
#else /* MDBX_PAGEVIZ enabled ───────────────────────────────────────────── */
|
||||
|
||||
#include <stdatomic.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* ── Tunables ─────────────────────────────────────────────────────────── */
|
||||
|
||||
#define MDBX_PAGEVIZ_RING_CAPACITY 65536u /* must be power-of-2 */
|
||||
#define MDBX_PAGEVIZ_MAX_RINGS 128u
|
||||
#define MDBX_PAGEVIZ_PUBLISH_INTERVAL 32u
|
||||
|
||||
/* ── Event encoding ───────────────────────────────────────────────────── */
|
||||
|
||||
/* Layout of a 64-bit event word:
|
||||
* bits 63..56 op (1=READ, 2=WRITE, 3=FREE)
|
||||
* bits 55..32 dbi (lower 24 bits of a uint32_t)
|
||||
* bits 31..0 pgno (uint32_t page number) */
|
||||
|
||||
#define MDBX_PAGEVIZ_OP_READ 1
|
||||
#define MDBX_PAGEVIZ_OP_WRITE 2
|
||||
#define MDBX_PAGEVIZ_OP_FREE 3
|
||||
#define MDBX_PAGEVIZ_OP_BLOCK_START 4
|
||||
#define MDBX_PAGEVIZ_OP_BLOCK_END 5
|
||||
|
||||
#define MDBX_PAGEVIZ_ENCODE(op, dbi, pgno) \
|
||||
(((uint64_t)(op) << 56) | ((uint64_t)((dbi) & 0x00FFFFFFu) << 32) | \
|
||||
(uint64_t)(uint32_t)(pgno))
|
||||
|
||||
#define MDBX_PAGEVIZ_DECODE_OP(ev) ((uint8_t)((ev) >> 56))
|
||||
#define MDBX_PAGEVIZ_DECODE_DBI(ev) ((uint32_t)(((ev) >> 32) & 0x00FFFFFFu))
|
||||
#define MDBX_PAGEVIZ_DECODE_PGNO(ev) ((uint32_t)(ev))
|
||||
|
||||
/* ── Ring-buffer mask (power-of-2 capacity) ───────────────────────────── */
|
||||
|
||||
#define MDBX_PAGEVIZ_RING_MASK (MDBX_PAGEVIZ_RING_CAPACITY - 1u)
|
||||
|
||||
/* ── Platform TLS ─────────────────────────────────────────────────────── */
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
# define MDBX_PAGEVIZ_TLS __declspec(thread)
|
||||
#elif defined(__GNUC__) || defined(__clang__)
|
||||
# define MDBX_PAGEVIZ_TLS __thread
|
||||
#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
|
||||
# define MDBX_PAGEVIZ_TLS _Thread_local
|
||||
#else
|
||||
# error "No thread-local storage support detected"
|
||||
#endif
|
||||
|
||||
/* ── Structures ───────────────────────────────────────────────────────── */
|
||||
|
||||
typedef struct mdbx_pageviz_ring {
|
||||
uint64_t events[MDBX_PAGEVIZ_RING_CAPACITY];
|
||||
_Atomic uint32_t published_head; /* consumer reads (acquire) */
|
||||
uint32_t local_head; /* producer-only, no atomic needed */
|
||||
_Atomic uint32_t consumer_tail; /* consumer updates after draining */
|
||||
_Atomic uint64_t dropped; /* unused, kept for ABI compat */
|
||||
uint32_t _pad[10]; /* pad to separate cache lines */
|
||||
} mdbx_pageviz_ring_t;
|
||||
|
||||
typedef struct mdbx_pageviz_mapping {
|
||||
volatile void *base;
|
||||
volatile size_t len;
|
||||
volatile uint32_t mdbx_page_size;
|
||||
volatile uint32_t sys_page_size;
|
||||
} mdbx_pageviz_mapping_t;
|
||||
|
||||
typedef struct mdbx_pageviz_state {
|
||||
mdbx_pageviz_ring_t rings[MDBX_PAGEVIZ_MAX_RINGS];
|
||||
_Atomic uint32_t ring_count; /* how many rings registered */
|
||||
_Atomic uint32_t enabled; /* runtime enable/disable */
|
||||
mdbx_pageviz_mapping_t mapping;
|
||||
} mdbx_pageviz_state_t;
|
||||
|
||||
/* ── Global state (defined in .c) ─────────────────────────────────────── */
|
||||
|
||||
extern mdbx_pageviz_state_t *mdbx_pageviz_global;
|
||||
|
||||
/* ── Thread-local ring index (defined in .c) ──────────────────────────── */
|
||||
|
||||
#define MDBX_PAGEVIZ_TLS_UNREGISTERED UINT32_MAX
|
||||
|
||||
extern MDBX_PAGEVIZ_TLS uint32_t mdbx_pageviz_tls_ring;
|
||||
|
||||
/* ── Inline helpers (private) ─────────────────────────────────────────── */
|
||||
|
||||
static inline uint32_t mdbx_pageviz_register_ring_(mdbx_pageviz_state_t *s) {
|
||||
uint32_t idx =
|
||||
atomic_fetch_add_explicit(&s->ring_count, 1, memory_order_relaxed);
|
||||
if (idx >= MDBX_PAGEVIZ_MAX_RINGS) {
|
||||
atomic_fetch_sub_explicit(&s->ring_count, 1, memory_order_relaxed);
|
||||
return MDBX_PAGEVIZ_TLS_UNREGISTERED;
|
||||
}
|
||||
mdbx_pageviz_ring_t *r = &s->rings[idx];
|
||||
atomic_store_explicit(&r->published_head, 0, memory_order_relaxed);
|
||||
r->local_head = 0;
|
||||
atomic_store_explicit(&r->consumer_tail, 0, memory_order_relaxed);
|
||||
atomic_store_explicit(&r->dropped, 0, memory_order_relaxed);
|
||||
return idx;
|
||||
}
|
||||
|
||||
/* ── Hot-path emit (inlined) ──────────────────────────────────────────── */
|
||||
|
||||
static inline void mdbx_pageviz_emit(uint8_t op, uint32_t dbi,
|
||||
uint32_t pgno) {
|
||||
mdbx_pageviz_state_t *state = mdbx_pageviz_global;
|
||||
if (__builtin_expect(
|
||||
state == NULL ||
|
||||
!atomic_load_explicit(&state->enabled, memory_order_relaxed),
|
||||
1))
|
||||
return;
|
||||
|
||||
uint32_t idx = mdbx_pageviz_tls_ring;
|
||||
if (__builtin_expect(idx == MDBX_PAGEVIZ_TLS_UNREGISTERED, 0)) {
|
||||
idx = mdbx_pageviz_register_ring_(state);
|
||||
if (__builtin_expect(idx == MDBX_PAGEVIZ_TLS_UNREGISTERED, 0))
|
||||
return;
|
||||
mdbx_pageviz_tls_ring = idx;
|
||||
}
|
||||
|
||||
mdbx_pageviz_ring_t *r = &state->rings[idx];
|
||||
|
||||
/* Block (spin) until the consumer drains enough space. */
|
||||
while (__builtin_expect(
|
||||
(uint32_t)(r->local_head -
|
||||
atomic_load_explicit(&r->consumer_tail, memory_order_acquire))
|
||||
>= MDBX_PAGEVIZ_RING_CAPACITY, 0)) {
|
||||
/* Flush our writes so the consumer can make progress. */
|
||||
atomic_store_explicit(&r->published_head, r->local_head,
|
||||
memory_order_release);
|
||||
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)
|
||||
__builtin_ia32_pause();
|
||||
#elif defined(__aarch64__) || defined(_M_ARM64)
|
||||
__asm__ volatile("yield");
|
||||
#endif
|
||||
}
|
||||
|
||||
r->events[r->local_head & MDBX_PAGEVIZ_RING_MASK] =
|
||||
MDBX_PAGEVIZ_ENCODE(op, dbi, pgno);
|
||||
r->local_head++;
|
||||
|
||||
if ((r->local_head & (MDBX_PAGEVIZ_PUBLISH_INTERVAL - 1u)) == 0)
|
||||
atomic_store_explicit(&r->published_head, r->local_head,
|
||||
memory_order_release);
|
||||
}
|
||||
|
||||
/* ── Macro hooks (inserted into mdbx.c) ──────────────────────────────── */
|
||||
|
||||
#define MDBX_PAGEVIZ_READ(mc, pgno) \
|
||||
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_READ, (uint32_t)cursor_dbi(mc), (pgno))
|
||||
#define MDBX_PAGEVIZ_WRITE(mc, pgno) \
|
||||
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_WRITE, (uint32_t)cursor_dbi(mc), (pgno))
|
||||
#define MDBX_PAGEVIZ_FREE(mc, pgno) \
|
||||
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_FREE, (uint32_t)cursor_dbi(mc), (pgno))
|
||||
|
||||
/* ── Public API (defined in .c) ───────────────────────────────────────── */
|
||||
|
||||
mdbx_pageviz_state_t *mdbx_pageviz_create(void);
|
||||
void mdbx_pageviz_destroy(mdbx_pageviz_state_t *state);
|
||||
void mdbx_pageviz_enable(mdbx_pageviz_state_t *state);
|
||||
void mdbx_pageviz_disable(mdbx_pageviz_state_t *state);
|
||||
uint32_t mdbx_pageviz_drain(mdbx_pageviz_state_t *state, uint32_t ring_idx,
|
||||
uint64_t *out_buf, uint32_t max_count);
|
||||
uint32_t mdbx_pageviz_ring_count(mdbx_pageviz_state_t *state);
|
||||
uint64_t mdbx_pageviz_dropped(mdbx_pageviz_state_t *state, uint32_t ring_idx);
|
||||
void mdbx_pageviz_set_mapping(void *base, size_t len, uint32_t mdbx_page_size);
|
||||
int mdbx_pageviz_get_mapping(void **out_base, size_t *out_len,
|
||||
uint32_t *out_mdbx_ps, uint32_t *out_sys_ps);
|
||||
void mdbx_pageviz_emit_block_marker(uint8_t op, uint32_t block_number,
|
||||
uint16_t tx_count, uint8_t duration_encoded,
|
||||
uint8_t gas_encoded);
|
||||
|
||||
#endif /* MDBX_PAGEVIZ */
|
||||
#endif /* MDBX_PAGEVIZ_H */
|
||||
@@ -36,9 +36,6 @@ mod flags;
|
||||
mod transaction;
|
||||
mod txn_manager;
|
||||
|
||||
#[cfg(feature = "pageviz")]
|
||||
pub mod pageviz;
|
||||
|
||||
#[cfg(test)]
|
||||
mod test_utils {
|
||||
use super::*;
|
||||
|
||||
@@ -1,226 +0,0 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
mpsc, Arc,
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[repr(u8)]
|
||||
pub enum PageOp {
|
||||
Read = 1,
|
||||
Write = 2,
|
||||
Free = 3,
|
||||
BlockStart = 4,
|
||||
BlockEnd = 5,
|
||||
}
|
||||
|
||||
impl PageOp {
|
||||
fn from_u8(v: u8) -> Option<Self> {
|
||||
match v {
|
||||
1 => Some(Self::Read),
|
||||
2 => Some(Self::Write),
|
||||
3 => Some(Self::Free),
|
||||
4 => Some(Self::BlockStart),
|
||||
5 => Some(Self::BlockEnd),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_block_marker(self) -> bool {
|
||||
matches!(self, Self::BlockStart | Self::BlockEnd)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct PageEvent {
|
||||
pub pgno: u32,
|
||||
pub dbi: u32,
|
||||
pub op: PageOp,
|
||||
}
|
||||
|
||||
impl PageEvent {
|
||||
fn decode(raw: u64) -> Option<Self> {
|
||||
let op_byte = (raw >> 56) as u8;
|
||||
let op = PageOp::from_u8(op_byte)?;
|
||||
let dbi = ((raw >> 32) & 0x00FF_FFFF) as u32;
|
||||
let pgno = raw as u32;
|
||||
Some(Self { pgno, dbi, op })
|
||||
}
|
||||
}
|
||||
|
||||
unsafe extern "C" {
|
||||
fn mdbx_pageviz_create() -> *mut std::ffi::c_void;
|
||||
fn mdbx_pageviz_destroy(state: *mut std::ffi::c_void);
|
||||
fn mdbx_pageviz_enable(state: *mut std::ffi::c_void);
|
||||
fn mdbx_pageviz_disable(state: *mut std::ffi::c_void);
|
||||
fn mdbx_pageviz_drain(
|
||||
state: *mut std::ffi::c_void,
|
||||
ring_idx: u32,
|
||||
out_buf: *mut u64,
|
||||
max_count: u32,
|
||||
) -> u32;
|
||||
fn mdbx_pageviz_ring_count(state: *mut std::ffi::c_void) -> u32;
|
||||
fn mdbx_pageviz_dropped(state: *mut std::ffi::c_void, ring_idx: u32) -> u64;
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct PageVizStats {
|
||||
pub ring_count: u32,
|
||||
pub total_dropped: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct StatePtr(*mut std::ffi::c_void);
|
||||
|
||||
// SAFETY: The C state is thread-safe (uses atomics internally).
|
||||
unsafe impl Send for StatePtr {}
|
||||
unsafe impl Sync for StatePtr {}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct PageVizDrainer {
|
||||
state: StatePtr,
|
||||
running: Arc<AtomicBool>,
|
||||
handle: Option<std::thread::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
const DRAIN_BUF_LEN: usize = 4096;
|
||||
const OVERLOAD_THRESHOLD: usize = 200_000;
|
||||
const READ_SAMPLE_RATE: usize = 8;
|
||||
|
||||
impl PageVizDrainer {
|
||||
pub fn new() -> Self {
|
||||
let state = StatePtr(unsafe { mdbx_pageviz_create() });
|
||||
Self { state, running: Arc::new(AtomicBool::new(false)), handle: None }
|
||||
}
|
||||
|
||||
pub fn enable(&self) {
|
||||
unsafe { mdbx_pageviz_enable(self.state.0) }
|
||||
}
|
||||
|
||||
pub fn disable(&self) {
|
||||
unsafe { mdbx_pageviz_disable(self.state.0) }
|
||||
}
|
||||
|
||||
pub fn start(&mut self, tick_hz: u32) -> mpsc::Receiver<Vec<PageEvent>> {
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let running = self.running.clone();
|
||||
running.store(true, Ordering::SeqCst);
|
||||
|
||||
let state_addr = self.state.0 as usize;
|
||||
let tick_interval = Duration::from_micros(1_000_000 / u64::from(tick_hz));
|
||||
|
||||
let handle = std::thread::Builder::new()
|
||||
.name("mdbx-pageviz".into())
|
||||
.spawn(move || {
|
||||
let state = state_addr as *mut std::ffi::c_void;
|
||||
let mut buf = [0u64; DRAIN_BUF_LEN];
|
||||
let mut coalesced: HashMap<u32, PageEvent> = HashMap::new();
|
||||
let mut markers: Vec<PageEvent> = Vec::new();
|
||||
let mut read_counter: usize = 0;
|
||||
|
||||
while running.load(Ordering::Relaxed) {
|
||||
let tick_start = Instant::now();
|
||||
|
||||
let ring_count = unsafe { mdbx_pageviz_ring_count(state) };
|
||||
|
||||
for ring_idx in 0..ring_count {
|
||||
let dropped = unsafe { mdbx_pageviz_dropped(state, ring_idx) };
|
||||
if dropped > 0 {
|
||||
tracing::warn!(
|
||||
target: "libmdbx::pageviz",
|
||||
ring_idx,
|
||||
dropped,
|
||||
"pageviz ring dropped events"
|
||||
);
|
||||
}
|
||||
|
||||
loop {
|
||||
let count = unsafe {
|
||||
mdbx_pageviz_drain(
|
||||
state,
|
||||
ring_idx,
|
||||
buf.as_mut_ptr(),
|
||||
DRAIN_BUF_LEN as u32,
|
||||
)
|
||||
};
|
||||
if count == 0 {
|
||||
break;
|
||||
}
|
||||
for &raw in &buf[..count as usize] {
|
||||
if let Some(evt) = PageEvent::decode(raw) {
|
||||
if evt.op.is_block_marker() {
|
||||
markers.push(evt);
|
||||
} else {
|
||||
coalesced.insert(evt.pgno, evt);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !coalesced.is_empty() || !markers.is_empty() {
|
||||
let mut events: Vec<PageEvent> = if coalesced.len() > OVERLOAD_THRESHOLD {
|
||||
coalesced
|
||||
.drain()
|
||||
.filter(|(_, evt)| {
|
||||
if evt.op == PageOp::Read {
|
||||
read_counter += 1;
|
||||
read_counter % READ_SAMPLE_RATE == 0
|
||||
} else {
|
||||
true
|
||||
}
|
||||
})
|
||||
.map(|(_, evt)| evt)
|
||||
.collect()
|
||||
} else {
|
||||
coalesced.drain().map(|(_, evt)| evt).collect()
|
||||
};
|
||||
|
||||
events.extend(markers.drain(..));
|
||||
|
||||
if tx.send(events).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let elapsed = tick_start.elapsed();
|
||||
if let Some(remaining) = tick_interval.checked_sub(elapsed) {
|
||||
std::thread::sleep(remaining);
|
||||
}
|
||||
}
|
||||
})
|
||||
.expect("failed to spawn mdbx-pageviz thread");
|
||||
|
||||
self.handle = Some(handle);
|
||||
rx
|
||||
}
|
||||
|
||||
pub fn stop(&mut self) {
|
||||
self.running.store(false, Ordering::SeqCst);
|
||||
if let Some(handle) = self.handle.take() {
|
||||
let _ = handle.join();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> PageVizStats {
|
||||
let ring_count = unsafe { mdbx_pageviz_ring_count(self.state.0) };
|
||||
let mut total_dropped = 0u64;
|
||||
for ring_idx in 0..ring_count {
|
||||
total_dropped += unsafe { mdbx_pageviz_dropped(self.state.0, ring_idx) };
|
||||
}
|
||||
PageVizStats { ring_count, total_dropped }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PageVizDrainer {
|
||||
fn drop(&mut self) {
|
||||
self.stop();
|
||||
unsafe {
|
||||
mdbx_pageviz_disable(self.state.0);
|
||||
mdbx_pageviz_destroy(self.state.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
1381
crates/storage/mdbx-viz/Cargo.lock
generated
1381
crates/storage/mdbx-viz/Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user