mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
61 Commits
docs/rocks
...
wt-pr2a
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
082daa7fc1 | ||
|
|
5c3e45cd6b | ||
|
|
68fdba32d2 | ||
|
|
8f6a0a2992 | ||
|
|
ec9c7f8d3e | ||
|
|
dbdaf068f0 | ||
|
|
055bf63ee9 | ||
|
|
2305c3ebeb | ||
|
|
eb55c3c3da | ||
|
|
72e1467ba3 | ||
|
|
74edce0089 | ||
|
|
8c645d5762 | ||
|
|
b7d2ee2566 | ||
|
|
7609deddda | ||
|
|
ec50fd40b3 | ||
|
|
624ddc5779 | ||
|
|
dd72cfe23e | ||
|
|
ff8ac97e33 | ||
|
|
0974485863 | ||
|
|
274394e777 | ||
|
|
1954c91a60 | ||
|
|
9cf82c8403 | ||
|
|
f85fcba872 | ||
|
|
ebaa4bda3a | ||
|
|
04d4c9a02f | ||
|
|
3065a328f9 | ||
|
|
43a84f1231 | ||
|
|
5a5c21cc1b | ||
|
|
8a8a9126d6 | ||
|
|
6f73c2447d | ||
|
|
2cae438642 | ||
|
|
37b5db0d47 | ||
|
|
238433e146 | ||
|
|
660964a0f5 | ||
|
|
22b465dd64 | ||
|
|
3ff575b877 | ||
|
|
d12752dc8a | ||
|
|
869b5d0851 | ||
|
|
78de3d8f61 | ||
|
|
bc79cc44c9 | ||
|
|
ff8f434dcd | ||
|
|
9662dc5271 | ||
|
|
3ba37082dc | ||
|
|
7934294988 | ||
|
|
7371bd3f29 | ||
|
|
80980b8e4d | ||
|
|
2e2cd67663 | ||
|
|
4f009728e2 | ||
|
|
39d5ae73e8 | ||
|
|
5ef200eaad | ||
|
|
d002dacc13 | ||
|
|
bb39cba504 | ||
|
|
bd144a4c42 | ||
|
|
a0845bab18 | ||
|
|
346cc0da71 | ||
|
|
ea3d4663ae | ||
|
|
3667d3b5aa | ||
|
|
7cfb19c98e | ||
|
|
5a38871489 | ||
|
|
c825c8c187 | ||
|
|
8f37cd08fc |
43
.github/CODEOWNERS
vendored
43
.github/CODEOWNERS
vendored
@@ -1,45 +1,52 @@
|
||||
* @gakonst
|
||||
crates/blockchain-tree-api/ @rakita @mattsse @Rjected
|
||||
crates/blockchain-tree/ @rakita @mattsse @Rjected
|
||||
crates/chain-state/ @fgimenez @mattsse
|
||||
crates/chainspec/ @Rjected @joshieDo @mattsse
|
||||
crates/cli/ @mattsse
|
||||
crates/config/ @shekhirin @mattsse @Rjected
|
||||
crates/consensus/ @mattsse @Rjected
|
||||
crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez
|
||||
crates/engine/ @mattsse @Rjected @fgimenez @mediocregopher @yongkangc
|
||||
crates/era/ @mattsse @RomanHodulak
|
||||
crates/engine/ @mattsse @Rjected @mediocregopher @yongkangc
|
||||
crates/era/ @mattsse
|
||||
crates/era-downloader/ @mattsse
|
||||
crates/era-utils/ @mattsse
|
||||
crates/errors/ @mattsse
|
||||
crates/ethereum-forks/ @mattsse @Rjected
|
||||
crates/ethereum/ @mattsse @Rjected
|
||||
crates/etl/ @joshieDo @shekhirin
|
||||
crates/evm/ @rakita @mattsse @Rjected
|
||||
crates/evm/ @mattsse @Rjected @klkvr
|
||||
crates/exex/ @shekhirin
|
||||
crates/fs-util/ @mattsse
|
||||
crates/metrics/ @mattsse @Rjected
|
||||
crates/net/ @mattsse @Rjected
|
||||
crates/net/downloaders/ @Rjected
|
||||
crates/node/ @mattsse @Rjected @klkvr
|
||||
crates/optimism/ @mattsse @Rjected @fgimenez
|
||||
crates/optimism/ @mattsse @Rjected
|
||||
crates/payload/ @mattsse @Rjected
|
||||
crates/primitives-traits/ @Rjected @RomanHodulak @mattsse @klkvr
|
||||
crates/primitives-traits/ @Rjected @mattsse @klkvr
|
||||
crates/primitives/ @Rjected @mattsse @klkvr
|
||||
crates/prune/ @shekhirin @joshieDo
|
||||
crates/ress @shekhirin @Rjected
|
||||
crates/revm/ @mattsse @rakita
|
||||
crates/rpc/ @mattsse @Rjected @RomanHodulak
|
||||
crates/ress/ @shekhirin @Rjected
|
||||
crates/revm/ @mattsse
|
||||
crates/rpc/ @mattsse @Rjected
|
||||
crates/stages/ @shekhirin @mediocregopher
|
||||
crates/static-file/ @joshieDo @shekhirin
|
||||
crates/stateless/ @mattsse
|
||||
crates/storage/codecs/ @joshieDo
|
||||
crates/storage/db-api/ @joshieDo @rakita
|
||||
crates/storage/db-api/ @joshieDo
|
||||
crates/storage/db-common/ @Rjected
|
||||
crates/storage/db/ @joshieDo @rakita
|
||||
crates/storage/errors/ @rakita
|
||||
crates/storage/libmdbx-rs/ @rakita @shekhirin
|
||||
crates/storage/db/ @joshieDo
|
||||
crates/storage/errors/ @joshieDo
|
||||
crates/storage/libmdbx-rs/ @shekhirin
|
||||
crates/storage/nippy-jar/ @joshieDo @shekhirin
|
||||
crates/storage/provider/ @rakita @joshieDo @shekhirin
|
||||
crates/storage/provider/ @joshieDo @shekhirin
|
||||
crates/storage/storage-api/ @joshieDo
|
||||
crates/tasks/ @mattsse
|
||||
crates/tokio-util/ @fgimenez
|
||||
crates/tokio-util/ @mattsse
|
||||
crates/tracing/ @mattsse @shekhirin
|
||||
crates/tracing-otlp/ @mattsse @Rjected
|
||||
crates/transaction-pool/ @mattsse @yongkangc
|
||||
crates/trie/ @Rjected @shekhirin @mediocregopher
|
||||
crates/trie/ @Rjected @shekhirin @mediocregopher @yongkangc
|
||||
bin/reth/ @mattsse @shekhirin @Rjected
|
||||
bin/reth-bench/ @mattsse @Rjected @shekhirin @yongkangc
|
||||
bin/reth-bench-compare/ @mediocregopher @shekhirin @yongkangc
|
||||
etc/ @Rjected @shekhirin
|
||||
.github/ @gakonst @DaniPopes
|
||||
|
||||
14
.github/scripts/codspeed-build.sh
vendored
14
.github/scripts/codspeed-build.sh
vendored
@@ -1,14 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eo pipefail
|
||||
|
||||
# TODO: Benchmarks run WAY too slow due to excessive amount of iterations.
|
||||
|
||||
cmd=(cargo codspeed build --profile profiling)
|
||||
crates=(
|
||||
-p reth-primitives
|
||||
-p reth-trie
|
||||
-p reth-trie-common
|
||||
-p reth-trie-sparse
|
||||
)
|
||||
|
||||
"${cmd[@]}" --features test-utils "${crates[@]}"
|
||||
14
.github/workflows/bench.yml
vendored
14
.github/workflows/bench.yml
vendored
@@ -17,6 +17,16 @@ name: bench
|
||||
jobs:
|
||||
codspeed:
|
||||
runs-on: depot-ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
partition: [1, 2]
|
||||
total_partitions: [2]
|
||||
include:
|
||||
- partition: 1
|
||||
crates: "-p reth-primitives -p reth-trie-common -p reth-trie-sparse"
|
||||
- partition: 2
|
||||
crates: "-p reth-trie"
|
||||
name: codspeed (${{ matrix.partition }}/${{ matrix.total_partitions }})
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
@@ -32,10 +42,10 @@ jobs:
|
||||
with:
|
||||
tool: cargo-codspeed
|
||||
- name: Build the benchmark target(s)
|
||||
run: ./.github/scripts/codspeed-build.sh
|
||||
run: cargo codspeed build --profile profiling --features test-utils ${{ matrix.crates }}
|
||||
- name: Run the benchmarks
|
||||
uses: CodSpeedHQ/action@v4
|
||||
with:
|
||||
run: cargo codspeed run --workspace
|
||||
run: cargo codspeed run ${{ matrix.crates }}
|
||||
mode: instrumentation
|
||||
token: ${{ secrets.CODSPEED_TOKEN }}
|
||||
|
||||
66
.github/workflows/check-alloy.yml
vendored
Normal file
66
.github/workflows/check-alloy.yml
vendored
Normal file
@@ -0,0 +1,66 @@
|
||||
# Checks reth compilation against alloy branches to detect breaking changes.
|
||||
# Run on-demand via workflow_dispatch.
|
||||
|
||||
name: Check Alloy Breaking Changes
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
alloy_branch:
|
||||
description: 'Branch/rev for alloy-rs/alloy (leave empty to skip)'
|
||||
required: false
|
||||
type: string
|
||||
alloy_evm_branch:
|
||||
description: 'Branch/rev for alloy-rs/evm (alloy-evm, alloy-op-evm) (leave empty to skip)'
|
||||
required: false
|
||||
type: string
|
||||
op_alloy_branch:
|
||||
description: 'Branch/rev for alloy-rs/op-alloy (leave empty to skip)'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Check compilation with patched alloy
|
||||
runs-on: depot-ubuntu-latest-16
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
|
||||
- name: Apply alloy patches
|
||||
run: |
|
||||
ARGS=""
|
||||
if [ -n "${{ inputs.alloy_branch }}" ]; then
|
||||
ARGS="$ARGS --alloy ${{ inputs.alloy_branch }}"
|
||||
fi
|
||||
if [ -n "${{ inputs.alloy_evm_branch }}" ]; then
|
||||
ARGS="$ARGS --evm ${{ inputs.alloy_evm_branch }}"
|
||||
fi
|
||||
if [ -n "${{ inputs.op_alloy_branch }}" ]; then
|
||||
ARGS="$ARGS --op ${{ inputs.op_alloy_branch }}"
|
||||
fi
|
||||
|
||||
if [ -z "$ARGS" ]; then
|
||||
echo "No branches specified, nothing to patch"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
./scripts/patch-alloy.sh $ARGS
|
||||
|
||||
echo "=== Final patch section ==="
|
||||
tail -50 Cargo.toml
|
||||
|
||||
- name: Check workspace
|
||||
run: cargo check --workspace --all-features
|
||||
|
||||
- name: Check Optimism
|
||||
run: cargo check -p reth-optimism-node --all-features
|
||||
424
Cargo.lock
generated
424
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
31
Cargo.toml
31
Cargo.toml
@@ -1,5 +1,5 @@
|
||||
[workspace.package]
|
||||
version = "1.10.1"
|
||||
version = "1.10.2"
|
||||
edition = "2024"
|
||||
rust-version = "1.88"
|
||||
license = "MIT OR Apache-2.0"
|
||||
@@ -797,3 +797,32 @@ ipnet = "2.11"
|
||||
|
||||
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
|
||||
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
|
||||
|
||||
# Patched by patch-alloy.sh
|
||||
alloy-consensus = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-contract = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-eips = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-genesis = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-network = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-provider = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-serde = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-signer = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-transport = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
|
||||
@@ -22,12 +22,42 @@ use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIM
|
||||
use std::{path::PathBuf, time::Instant};
|
||||
use tracing::info;
|
||||
|
||||
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
|
||||
///
|
||||
/// Examples: "30000000", "30M", "1G", "2G"
|
||||
fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
|
||||
let s = s.trim();
|
||||
if s.is_empty() {
|
||||
return Err(eyre::eyre!("empty value"));
|
||||
}
|
||||
|
||||
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
|
||||
(prefix, 1_000_000_000u64)
|
||||
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
|
||||
(prefix, 1_000_000u64)
|
||||
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
|
||||
(prefix, 1_000u64)
|
||||
} else {
|
||||
(s, 1u64)
|
||||
};
|
||||
|
||||
let base: u64 = num_str.trim().parse()?;
|
||||
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
|
||||
}
|
||||
|
||||
/// `reth benchmark gas-limit-ramp` command.
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Command {
|
||||
/// Number of blocks to generate.
|
||||
#[arg(long, value_name = "BLOCKS")]
|
||||
blocks: u64,
|
||||
/// Number of blocks to generate. Mutually exclusive with --target-gas-limit.
|
||||
#[arg(long, value_name = "BLOCKS", conflicts_with = "target_gas_limit")]
|
||||
blocks: Option<u64>,
|
||||
|
||||
/// Target gas limit to ramp up to. The benchmark will generate blocks until the gas limit
|
||||
/// reaches or exceeds this value. Mutually exclusive with --blocks.
|
||||
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 2G = 2
|
||||
/// billion).
|
||||
#[arg(long, value_name = "TARGET_GAS_LIMIT", conflicts_with = "blocks", value_parser = parse_gas_limit)]
|
||||
target_gas_limit: Option<u64>,
|
||||
|
||||
/// The Engine API RPC URL.
|
||||
#[arg(long = "engine-rpc-url", value_name = "ENGINE_RPC_URL")]
|
||||
@@ -42,12 +72,37 @@ pub struct Command {
|
||||
output: PathBuf,
|
||||
}
|
||||
|
||||
/// Mode for determining when to stop ramping.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum RampMode {
|
||||
/// Ramp for a fixed number of blocks.
|
||||
Blocks(u64),
|
||||
/// Ramp until reaching or exceeding target gas limit.
|
||||
TargetGasLimit(u64),
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `benchmark gas-limit-ramp` command.
|
||||
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
|
||||
if self.blocks == 0 {
|
||||
return Err(eyre::eyre!("--blocks must be greater than 0"));
|
||||
}
|
||||
let mode = match (self.blocks, self.target_gas_limit) {
|
||||
(Some(blocks), None) => {
|
||||
if blocks == 0 {
|
||||
return Err(eyre::eyre!("--blocks must be greater than 0"));
|
||||
}
|
||||
RampMode::Blocks(blocks)
|
||||
}
|
||||
(None, Some(target)) => {
|
||||
if target == 0 {
|
||||
return Err(eyre::eyre!("--target-gas-limit must be greater than 0"));
|
||||
}
|
||||
RampMode::TargetGasLimit(target)
|
||||
}
|
||||
_ => {
|
||||
return Err(eyre::eyre!(
|
||||
"Exactly one of --blocks or --target-gas-limit must be specified"
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
// Ensure output directory exists
|
||||
if self.output.is_file() {
|
||||
@@ -84,14 +139,31 @@ impl Command {
|
||||
|
||||
let canonical_parent = parent_header.number;
|
||||
let start_block = canonical_parent + 1;
|
||||
let end_block = start_block + self.blocks - 1;
|
||||
|
||||
info!(canonical_parent, start_block, end_block, "Starting gas limit ramp benchmark");
|
||||
match mode {
|
||||
RampMode::Blocks(blocks) => {
|
||||
info!(
|
||||
canonical_parent,
|
||||
start_block,
|
||||
end_block = start_block + blocks - 1,
|
||||
"Starting gas limit ramp benchmark (block count mode)"
|
||||
);
|
||||
}
|
||||
RampMode::TargetGasLimit(target) => {
|
||||
info!(
|
||||
canonical_parent,
|
||||
start_block,
|
||||
current_gas_limit = parent_header.gas_limit,
|
||||
target_gas_limit = target,
|
||||
"Starting gas limit ramp benchmark (target gas limit mode)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut next_block_number = start_block;
|
||||
let mut blocks_processed = 0u64;
|
||||
let total_benchmark_duration = Instant::now();
|
||||
|
||||
while next_block_number <= end_block {
|
||||
while !should_stop(mode, blocks_processed, parent_header.gas_limit) {
|
||||
let timestamp = parent_header.timestamp.saturating_add(1);
|
||||
|
||||
let request = prepare_payload_request(&chain_spec, timestamp, parent_hash);
|
||||
@@ -140,13 +212,13 @@ impl Command {
|
||||
|
||||
parent_header = block.header;
|
||||
parent_hash = block_hash;
|
||||
next_block_number += 1;
|
||||
blocks_processed += 1;
|
||||
}
|
||||
|
||||
let final_gas_limit = parent_header.gas_limit;
|
||||
info!(
|
||||
total_duration=?total_benchmark_duration.elapsed(),
|
||||
blocks_processed = self.blocks,
|
||||
blocks_processed,
|
||||
final_gas_limit,
|
||||
"Benchmark complete"
|
||||
);
|
||||
@@ -158,3 +230,57 @@ impl Command {
|
||||
const fn max_gas_limit_increase(parent_gas_limit: u64) -> u64 {
|
||||
(parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR).saturating_sub(1)
|
||||
}
|
||||
|
||||
const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u64) -> bool {
|
||||
match mode {
|
||||
RampMode::Blocks(target_blocks) => blocks_processed >= target_blocks,
|
||||
RampMode::TargetGasLimit(target) => current_gas_limit >= target,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_plain_number() {
|
||||
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
|
||||
assert_eq!(parse_gas_limit("1").unwrap(), 1);
|
||||
assert_eq!(parse_gas_limit("0").unwrap(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_k_suffix() {
|
||||
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
|
||||
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
|
||||
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_m_suffix() {
|
||||
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
|
||||
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
|
||||
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_g_suffix() {
|
||||
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
|
||||
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
|
||||
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_with_whitespace() {
|
||||
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
|
||||
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_errors() {
|
||||
assert!(parse_gas_limit("").is_err());
|
||||
assert!(parse_gas_limit("abc").is_err());
|
||||
assert!(parse_gas_limit("G").is_err());
|
||||
assert!(parse_gas_limit("-1G").is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,7 +180,7 @@ impl Command {
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| {
|
||||
e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
|
||||
e.file_name().to_string_lossy().starts_with("payload_")
|
||||
e.file_name().to_string_lossy().starts_with("payload_block_")
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -191,7 +191,7 @@ impl Command {
|
||||
let name = e.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
// Extract index from "payload_NNN.json"
|
||||
let index_str = name_str.strip_prefix("payload_")?.strip_suffix(".json")?;
|
||||
let index_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
|
||||
let index: u64 = index_str.parse().ok()?;
|
||||
Some((index, e.path()))
|
||||
})
|
||||
|
||||
@@ -41,6 +41,7 @@ derive_more.workspace = true
|
||||
metrics.workspace = true
|
||||
parking_lot.workspace = true
|
||||
pin-project.workspace = true
|
||||
rayon = { workspace = true, optional = true }
|
||||
serde = { workspace = true, optional = true }
|
||||
|
||||
# optional deps for test-utils
|
||||
@@ -84,6 +85,7 @@ test-utils = [
|
||||
"reth-trie/test-utils",
|
||||
"reth-ethereum-primitives/test-utils",
|
||||
]
|
||||
rayon = ["dep:rayon"]
|
||||
|
||||
[[bench]]
|
||||
name = "canonical_hashes_range"
|
||||
|
||||
@@ -163,14 +163,29 @@ impl DeferredTrieData {
|
||||
anchor_hash: B256,
|
||||
ancestors: &[Self],
|
||||
) -> ComputedTrieData {
|
||||
let sorted_hashed_state = match Arc::try_unwrap(hashed_state) {
|
||||
Ok(state) => state.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
};
|
||||
let sorted_trie_updates = match Arc::try_unwrap(trie_updates) {
|
||||
Ok(updates) => updates.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
};
|
||||
#[cfg(feature = "rayon")]
|
||||
let (sorted_hashed_state, sorted_trie_updates) = rayon::join(
|
||||
|| match Arc::try_unwrap(hashed_state) {
|
||||
Ok(state) => state.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
},
|
||||
|| match Arc::try_unwrap(trie_updates) {
|
||||
Ok(updates) => updates.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
},
|
||||
);
|
||||
|
||||
#[cfg(not(feature = "rayon"))]
|
||||
let (sorted_hashed_state, sorted_trie_updates) = (
|
||||
match Arc::try_unwrap(hashed_state) {
|
||||
Ok(state) => state.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
},
|
||||
match Arc::try_unwrap(trie_updates) {
|
||||
Ok(updates) => updates.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
},
|
||||
);
|
||||
|
||||
// Reuse parent's overlay if available and anchors match.
|
||||
// We can only reuse the parent's overlay if it was built on top of the same
|
||||
@@ -228,8 +243,53 @@ impl DeferredTrieData {
|
||||
/// In normal operation, the parent always has a cached overlay and this
|
||||
/// function is never called.
|
||||
///
|
||||
/// Iterates ancestors oldest -> newest, then extends with current block's data,
|
||||
/// so later state takes precedence.
|
||||
/// When the `rayon` feature is enabled, uses parallel collection and merge:
|
||||
/// 1. Collects ancestor data in parallel (each `wait_cloned()` may compute)
|
||||
/// 2. Merges hashed state and trie updates in parallel with each other
|
||||
/// 3. Uses tree reduction within each merge for O(log n) depth
|
||||
#[cfg(feature = "rayon")]
|
||||
fn merge_ancestors_into_overlay(
|
||||
ancestors: &[Self],
|
||||
sorted_hashed_state: &HashedPostStateSorted,
|
||||
sorted_trie_updates: &TrieUpdatesSorted,
|
||||
) -> TrieInputSorted {
|
||||
// Early exit: no ancestors means just wrap current block's data
|
||||
if ancestors.is_empty() {
|
||||
return TrieInputSorted::new(
|
||||
Arc::new(sorted_trie_updates.clone()),
|
||||
Arc::new(sorted_hashed_state.clone()),
|
||||
Default::default(),
|
||||
);
|
||||
}
|
||||
|
||||
// Collect ancestor data, unzipping states and updates into Arc slices
|
||||
let (states, updates): (Vec<_>, Vec<_>) = ancestors
|
||||
.iter()
|
||||
.map(|a| {
|
||||
let data = a.wait_cloned();
|
||||
(data.hashed_state, data.trie_updates)
|
||||
})
|
||||
.unzip();
|
||||
|
||||
// Merge state and nodes in parallel with each other using tree reduction
|
||||
let (state, nodes) = rayon::join(
|
||||
|| {
|
||||
let mut merged = HashedPostStateSorted::merge_parallel(&states);
|
||||
merged.extend_ref_and_sort(sorted_hashed_state);
|
||||
merged
|
||||
},
|
||||
|| {
|
||||
let mut merged = TrieUpdatesSorted::merge_parallel(&updates);
|
||||
merged.extend_ref_and_sort(sorted_trie_updates);
|
||||
merged
|
||||
},
|
||||
);
|
||||
|
||||
TrieInputSorted::new(Arc::new(nodes), Arc::new(state), Default::default())
|
||||
}
|
||||
|
||||
/// Merge all ancestors and current block's data into a single overlay (sequential fallback).
|
||||
#[cfg(not(feature = "rayon"))]
|
||||
fn merge_ancestors_into_overlay(
|
||||
ancestors: &[Self],
|
||||
sorted_hashed_state: &HashedPostStateSorted,
|
||||
|
||||
@@ -123,60 +123,18 @@ impl LazyOverlay {
|
||||
|
||||
/// Merge all blocks' trie data into a single [`TrieInputSorted`].
|
||||
///
|
||||
/// Blocks are ordered newest to oldest. Uses hybrid merge algorithm that
|
||||
/// switches between `extend_ref` (small batches) and k-way merge (large batches).
|
||||
/// Blocks are ordered newest to oldest.
|
||||
fn merge_blocks(blocks: &[DeferredTrieData]) -> TrieInputSorted {
|
||||
const MERGE_BATCH_THRESHOLD: usize = 64;
|
||||
|
||||
if blocks.is_empty() {
|
||||
return TrieInputSorted::default();
|
||||
}
|
||||
|
||||
// Single block: use its data directly (no allocation)
|
||||
if blocks.len() == 1 {
|
||||
let data = blocks[0].wait_cloned();
|
||||
return TrieInputSorted {
|
||||
state: data.hashed_state,
|
||||
nodes: data.trie_updates,
|
||||
prefix_sets: Default::default(),
|
||||
};
|
||||
}
|
||||
let state =
|
||||
HashedPostStateSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().hashed_state));
|
||||
let nodes =
|
||||
TrieUpdatesSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().trie_updates));
|
||||
|
||||
if blocks.len() < MERGE_BATCH_THRESHOLD {
|
||||
// Small k: extend_ref loop with Arc::make_mut is faster.
|
||||
// Uses copy-on-write - only clones inner data if Arc has multiple refs.
|
||||
// Iterate oldest->newest so newer values override older ones.
|
||||
let mut blocks_iter = blocks.iter().rev();
|
||||
let first = blocks_iter.next().expect("blocks is non-empty");
|
||||
let data = first.wait_cloned();
|
||||
|
||||
let mut state = data.hashed_state;
|
||||
let mut nodes = data.trie_updates;
|
||||
|
||||
for block in blocks_iter {
|
||||
let block_data = block.wait_cloned();
|
||||
Arc::make_mut(&mut state).extend_ref_and_sort(block_data.hashed_state.as_ref());
|
||||
Arc::make_mut(&mut nodes).extend_ref_and_sort(block_data.trie_updates.as_ref());
|
||||
}
|
||||
|
||||
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
|
||||
} else {
|
||||
// Large k: k-way merge is faster (O(n log k)).
|
||||
// Collect is unavoidable here - we need all data materialized for k-way merge.
|
||||
let trie_data: Vec<_> = blocks.iter().map(|b| b.wait_cloned()).collect();
|
||||
|
||||
let merged_state = HashedPostStateSorted::merge_batch(
|
||||
trie_data.iter().map(|d| d.hashed_state.as_ref()),
|
||||
);
|
||||
let merged_nodes =
|
||||
TrieUpdatesSorted::merge_batch(trie_data.iter().map(|d| d.trie_updates.as_ref()));
|
||||
|
||||
TrieInputSorted {
|
||||
state: Arc::new(merged_state),
|
||||
nodes: Arc::new(merged_nodes),
|
||||
prefix_sets: Default::default(),
|
||||
}
|
||||
}
|
||||
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -278,6 +278,7 @@ pub fn create_chain_config(
|
||||
// Check if DAO fork is supported (it has an activation block)
|
||||
let dao_fork_support = hardforks.fork(EthereumHardfork::Dao) != ForkCondition::Never;
|
||||
|
||||
#[allow(clippy::needless_update)]
|
||||
ChainConfig {
|
||||
chain_id: chain.map(|c| c.id()).unwrap_or(0),
|
||||
homestead_block: block_num(EthereumHardfork::Homestead),
|
||||
@@ -313,6 +314,7 @@ pub fn create_chain_config(
|
||||
extra_fields: Default::default(),
|
||||
deposit_contract_address,
|
||||
blob_schedule,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -131,4 +131,4 @@ arbitrary = [
|
||||
"reth-ethereum-primitives/arbitrary",
|
||||
]
|
||||
|
||||
edge = ["reth-db-common/edge", "reth-stages/rocksdb"]
|
||||
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb"]
|
||||
|
||||
@@ -19,7 +19,7 @@ use reth_node_builder::{
|
||||
Node, NodeComponents, NodeComponentsBuilder, NodeTypes, NodeTypesWithDBAdapter,
|
||||
};
|
||||
use reth_node_core::{
|
||||
args::{DatabaseArgs, DatadirArgs, StaticFilesArgs},
|
||||
args::{DatabaseArgs, DatadirArgs, RocksDbArgs, StaticFilesArgs},
|
||||
dirs::{ChainPath, DataDirPath},
|
||||
};
|
||||
use reth_provider::{
|
||||
@@ -27,7 +27,7 @@ use reth_provider::{
|
||||
BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider,
|
||||
StaticFileProviderBuilder,
|
||||
},
|
||||
ProviderFactory, StaticFileProviderFactory,
|
||||
ProviderFactory, StaticFileProviderFactory, StorageSettings,
|
||||
};
|
||||
use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget};
|
||||
use reth_static_file::StaticFileProducer;
|
||||
@@ -66,9 +66,24 @@ pub struct EnvironmentArgs<C: ChainSpecParser> {
|
||||
/// All static files related arguments
|
||||
#[command(flatten)]
|
||||
pub static_files: StaticFilesArgs,
|
||||
|
||||
/// All `RocksDB` related arguments
|
||||
#[command(flatten)]
|
||||
pub rocksdb: RocksDbArgs,
|
||||
}
|
||||
|
||||
impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
/// Returns the effective storage settings derived from static-file and `RocksDB` CLI args.
|
||||
pub fn storage_settings(&self) -> StorageSettings {
|
||||
StorageSettings::base()
|
||||
.with_receipts_in_static_files(self.static_files.receipts)
|
||||
.with_transaction_senders_in_static_files(self.static_files.transaction_senders)
|
||||
.with_account_changesets_in_static_files(self.static_files.account_changesets)
|
||||
.with_transaction_hash_numbers_in_rocksdb(self.rocksdb.all || self.rocksdb.tx_hash)
|
||||
.with_storages_history_in_rocksdb(self.rocksdb.all || self.rocksdb.storages_history)
|
||||
.with_account_history_in_rocksdb(self.rocksdb.all || self.rocksdb.account_history)
|
||||
}
|
||||
|
||||
/// Initializes environment according to [`AccessRights`] and returns an instance of
|
||||
/// [`Environment`].
|
||||
pub fn init<N: CliNodeTypes>(&self, access: AccessRights) -> eyre::Result<Environment<N>>
|
||||
@@ -121,17 +136,17 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
})
|
||||
}
|
||||
};
|
||||
// TransactionDB only support read-write mode
|
||||
let rocksdb_provider = RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level)
|
||||
.with_read_only(!access.is_read_write())
|
||||
.build()?;
|
||||
|
||||
let provider_factory =
|
||||
self.create_provider_factory(&config, db, sfp, rocksdb_provider, access)?;
|
||||
if access.is_read_write() {
|
||||
debug!(target: "reth::cli", chain=%self.chain.chain(), genesis=?self.chain.genesis_hash(), "Initializing genesis");
|
||||
init_genesis_with_settings(&provider_factory, self.static_files.to_settings())?;
|
||||
init_genesis_with_settings(&provider_factory, self.storage_settings())?;
|
||||
}
|
||||
|
||||
Ok(Environment { config, provider_factory, data_dir })
|
||||
|
||||
@@ -1,145 +0,0 @@
|
||||
use crate::{
|
||||
common::CliNodeTypes,
|
||||
db::get::{maybe_json_value_parser, table_key},
|
||||
};
|
||||
use alloy_primitives::map::foldhash::fast::FixedState;
|
||||
use clap::Parser;
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_db::DatabaseEnv;
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
|
||||
Tables,
|
||||
};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
|
||||
use reth_provider::{providers::ProviderNodeTypes, DBProvider};
|
||||
use std::{
|
||||
hash::{BuildHasher, Hasher},
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
/// The arguments for the `reth db checksum` command
|
||||
pub struct Command {
|
||||
/// The table name
|
||||
table: Tables,
|
||||
|
||||
/// The start of the range to checksum.
|
||||
#[arg(long, value_parser = maybe_json_value_parser)]
|
||||
start_key: Option<String>,
|
||||
|
||||
/// The end of the range to checksum.
|
||||
#[arg(long, value_parser = maybe_json_value_parser)]
|
||||
end_key: Option<String>,
|
||||
|
||||
/// The maximum number of records that are queried and used to compute the
|
||||
/// checksum.
|
||||
#[arg(long)]
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `db checksum` command
|
||||
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
self,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
) -> eyre::Result<()> {
|
||||
warn!("This command should be run without the node running!");
|
||||
self.table.view(&ChecksumViewer {
|
||||
tool,
|
||||
start_key: self.start_key,
|
||||
end_key: self.end_key,
|
||||
limit: self.limit,
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
|
||||
tool: &'a DbTool<N>,
|
||||
start_key: Option<String>,
|
||||
end_key: Option<String>,
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
|
||||
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
|
||||
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
|
||||
type Error = eyre::Report;
|
||||
|
||||
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
|
||||
let provider =
|
||||
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
|
||||
let tx = provider.tx_ref();
|
||||
info!(
|
||||
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
|
||||
self.start_key, self.end_key, self.limit
|
||||
);
|
||||
|
||||
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
|
||||
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
|
||||
(Some(start), Some(end)) => {
|
||||
let start_key = table_key::<T>(start).map(RawKey::new)?;
|
||||
let end_key = table_key::<T>(end).map(RawKey::new)?;
|
||||
cursor.walk_range(start_key..=end_key)?
|
||||
}
|
||||
(None, Some(end)) => {
|
||||
let end_key = table_key::<T>(end).map(RawKey::new)?;
|
||||
|
||||
cursor.walk_range(..=end_key)?
|
||||
}
|
||||
(Some(start), None) => {
|
||||
let start_key = table_key::<T>(start).map(RawKey::new)?;
|
||||
cursor.walk_range(start_key..)?
|
||||
}
|
||||
(None, None) => cursor.walk_range(..)?,
|
||||
};
|
||||
|
||||
let start_time = Instant::now();
|
||||
let mut hasher = FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher();
|
||||
let mut total = 0;
|
||||
|
||||
let limit = self.limit.unwrap_or(usize::MAX);
|
||||
let mut enumerate_start_key = None;
|
||||
let mut enumerate_end_key = None;
|
||||
for (index, entry) in walker.enumerate() {
|
||||
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
|
||||
|
||||
if index.is_multiple_of(100_000) {
|
||||
info!("Hashed {index} entries.");
|
||||
}
|
||||
|
||||
hasher.write(k.raw_key());
|
||||
hasher.write(v.raw_value());
|
||||
|
||||
if enumerate_start_key.is_none() {
|
||||
enumerate_start_key = Some(k.clone());
|
||||
}
|
||||
enumerate_end_key = Some(k);
|
||||
|
||||
total = index + 1;
|
||||
if total >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
info!("Hashed {total} entries.");
|
||||
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
|
||||
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
|
||||
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
|
||||
}
|
||||
|
||||
let checksum = hasher.finish();
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
|
||||
|
||||
Ok((checksum, elapsed))
|
||||
}
|
||||
}
|
||||
288
crates/cli/commands/src/db/checksum/mod.rs
Normal file
288
crates/cli/commands/src/db/checksum/mod.rs
Normal file
@@ -0,0 +1,288 @@
|
||||
use crate::{
|
||||
common::CliNodeTypes,
|
||||
db::get::{maybe_json_value_parser, table_key},
|
||||
};
|
||||
use alloy_primitives::map::foldhash::fast::FixedState;
|
||||
use clap::Parser;
|
||||
use itertools::Itertools;
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_db::{static_file::iter_static_files, DatabaseEnv};
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
|
||||
Tables,
|
||||
};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
|
||||
use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProviderFactory};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use std::{
|
||||
hash::{BuildHasher, Hasher},
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
#[cfg(all(unix, feature = "edge"))]
|
||||
mod rocksdb;
|
||||
|
||||
/// Interval for logging progress during checksum computation.
|
||||
const PROGRESS_LOG_INTERVAL: usize = 100_000;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
/// The arguments for the `reth db checksum` command
|
||||
pub struct Command {
|
||||
#[command(subcommand)]
|
||||
subcommand: Subcommand,
|
||||
}
|
||||
|
||||
#[derive(clap::Subcommand, Debug)]
|
||||
enum Subcommand {
|
||||
/// Calculates the checksum of a database table
|
||||
Mdbx {
|
||||
/// The table name
|
||||
table: Tables,
|
||||
|
||||
/// The start of the range to checksum.
|
||||
#[arg(long, value_parser = maybe_json_value_parser)]
|
||||
start_key: Option<String>,
|
||||
|
||||
/// The end of the range to checksum.
|
||||
#[arg(long, value_parser = maybe_json_value_parser)]
|
||||
end_key: Option<String>,
|
||||
|
||||
/// The maximum number of records that are queried and used to compute the
|
||||
/// checksum.
|
||||
#[arg(long)]
|
||||
limit: Option<usize>,
|
||||
},
|
||||
/// Calculates the checksum of a static file segment
|
||||
StaticFile {
|
||||
/// The static file segment
|
||||
#[arg(value_enum)]
|
||||
segment: StaticFileSegment,
|
||||
|
||||
/// The block number to start from (inclusive).
|
||||
#[arg(long)]
|
||||
start_block: Option<u64>,
|
||||
|
||||
/// The block number to end at (inclusive).
|
||||
#[arg(long)]
|
||||
end_block: Option<u64>,
|
||||
|
||||
/// The maximum number of rows to checksum.
|
||||
#[arg(long)]
|
||||
limit: Option<usize>,
|
||||
},
|
||||
/// Calculates the checksum of a RocksDB table
|
||||
#[cfg(all(unix, feature = "edge"))]
|
||||
Rocksdb {
|
||||
/// The RocksDB table
|
||||
#[arg(value_enum)]
|
||||
table: rocksdb::RocksDbTable,
|
||||
|
||||
/// The maximum number of records to checksum.
|
||||
#[arg(long)]
|
||||
limit: Option<usize>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `db checksum` command
|
||||
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
self,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
) -> eyre::Result<()> {
|
||||
warn!("This command should be run without the node running!");
|
||||
|
||||
match self.subcommand {
|
||||
Subcommand::Mdbx { table, start_key, end_key, limit } => {
|
||||
table.view(&ChecksumViewer { tool, start_key, end_key, limit })?;
|
||||
}
|
||||
Subcommand::StaticFile { segment, start_block, end_block, limit } => {
|
||||
checksum_static_file(tool, segment, start_block, end_block, limit)?;
|
||||
}
|
||||
#[cfg(all(unix, feature = "edge"))]
|
||||
Subcommand::Rocksdb { table, limit } => {
|
||||
rocksdb::checksum_rocksdb(tool, table, limit)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a new hasher with the standard seed used for checksum computation.
|
||||
fn checksum_hasher() -> impl Hasher {
|
||||
FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher()
|
||||
}
|
||||
|
||||
fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
segment: StaticFileSegment,
|
||||
start_block: Option<u64>,
|
||||
end_block: Option<u64>,
|
||||
limit: Option<usize>,
|
||||
) -> eyre::Result<()> {
|
||||
let static_file_provider = tool.provider_factory.static_file_provider();
|
||||
if let Err(err) = static_file_provider.check_consistency(&tool.provider_factory.provider()?) {
|
||||
warn!("Error checking consistency of static files: {err}");
|
||||
}
|
||||
|
||||
let static_files = iter_static_files(static_file_provider.directory())?;
|
||||
|
||||
let ranges = static_files
|
||||
.get(segment)
|
||||
.ok_or_else(|| eyre::eyre!("No static files found for segment: {}", segment))?;
|
||||
|
||||
let start_time = Instant::now();
|
||||
let mut hasher = checksum_hasher();
|
||||
let mut total = 0usize;
|
||||
let limit = limit.unwrap_or(usize::MAX);
|
||||
|
||||
let start_block = start_block.unwrap_or(0);
|
||||
let end_block = end_block.unwrap_or(u64::MAX);
|
||||
|
||||
info!(
|
||||
"Computing checksum for {} static files, start_block={}, end_block={}, limit={:?}",
|
||||
segment,
|
||||
start_block,
|
||||
end_block,
|
||||
if limit == usize::MAX { None } else { Some(limit) }
|
||||
);
|
||||
|
||||
'outer: for (block_range, _header) in ranges.iter().sorted_by_key(|(range, _)| range.start()) {
|
||||
if block_range.end() < start_block || block_range.start() > end_block {
|
||||
continue;
|
||||
}
|
||||
|
||||
let fixed_block_range = static_file_provider.find_fixed_range(segment, block_range.start());
|
||||
let jar_provider = static_file_provider
|
||||
.get_segment_provider_for_range(segment, || Some(fixed_block_range), None)?
|
||||
.ok_or_else(|| {
|
||||
eyre::eyre!(
|
||||
"Failed to get segment provider for segment {} at range {}",
|
||||
segment,
|
||||
block_range
|
||||
)
|
||||
})?;
|
||||
|
||||
let mut cursor = jar_provider.cursor()?;
|
||||
|
||||
while let Ok(Some(row)) = cursor.next_row() {
|
||||
for col_data in row.iter() {
|
||||
hasher.write(col_data);
|
||||
}
|
||||
|
||||
total += 1;
|
||||
|
||||
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
|
||||
info!("Hashed {total} entries.");
|
||||
}
|
||||
|
||||
if total >= limit {
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
|
||||
// Explicitly drop provider before removing from cache to avoid deadlock
|
||||
drop(jar_provider);
|
||||
static_file_provider.remove_cached_provider(segment, fixed_block_range.end());
|
||||
}
|
||||
|
||||
let checksum = hasher.finish();
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
info!(
|
||||
"Checksum for static file segment `{}`: {:#x} ({} entries, elapsed: {:?})",
|
||||
segment, checksum, total, elapsed
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
|
||||
tool: &'a DbTool<N>,
|
||||
start_key: Option<String>,
|
||||
end_key: Option<String>,
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
|
||||
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
|
||||
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
|
||||
type Error = eyre::Report;
|
||||
|
||||
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
|
||||
let provider =
|
||||
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
|
||||
let tx = provider.tx_ref();
|
||||
info!(
|
||||
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
|
||||
self.start_key, self.end_key, self.limit
|
||||
);
|
||||
|
||||
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
|
||||
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
|
||||
(Some(start), Some(end)) => {
|
||||
let start_key = table_key::<T>(start).map(RawKey::new)?;
|
||||
let end_key = table_key::<T>(end).map(RawKey::new)?;
|
||||
cursor.walk_range(start_key..=end_key)?
|
||||
}
|
||||
(None, Some(end)) => {
|
||||
let end_key = table_key::<T>(end).map(RawKey::new)?;
|
||||
|
||||
cursor.walk_range(..=end_key)?
|
||||
}
|
||||
(Some(start), None) => {
|
||||
let start_key = table_key::<T>(start).map(RawKey::new)?;
|
||||
cursor.walk_range(start_key..)?
|
||||
}
|
||||
(None, None) => cursor.walk_range(..)?,
|
||||
};
|
||||
|
||||
let start_time = Instant::now();
|
||||
let mut hasher = checksum_hasher();
|
||||
let mut total = 0;
|
||||
|
||||
let limit = self.limit.unwrap_or(usize::MAX);
|
||||
let mut enumerate_start_key = None;
|
||||
let mut enumerate_end_key = None;
|
||||
for (index, entry) in walker.enumerate() {
|
||||
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
|
||||
|
||||
if index.is_multiple_of(PROGRESS_LOG_INTERVAL) {
|
||||
info!("Hashed {index} entries.");
|
||||
}
|
||||
|
||||
hasher.write(k.raw_key());
|
||||
hasher.write(v.raw_value());
|
||||
|
||||
if enumerate_start_key.is_none() {
|
||||
enumerate_start_key = Some(k.clone());
|
||||
}
|
||||
enumerate_end_key = Some(k);
|
||||
|
||||
total = index + 1;
|
||||
if total >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
info!("Hashed {total} entries.");
|
||||
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
|
||||
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
|
||||
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
|
||||
}
|
||||
|
||||
let checksum = hasher.finish();
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
|
||||
|
||||
Ok((checksum, elapsed))
|
||||
}
|
||||
}
|
||||
106
crates/cli/commands/src/db/checksum/rocksdb.rs
Normal file
106
crates/cli/commands/src/db/checksum/rocksdb.rs
Normal file
@@ -0,0 +1,106 @@
|
||||
//! RocksDB checksum implementation.
|
||||
|
||||
use super::{checksum_hasher, PROGRESS_LOG_INTERVAL};
|
||||
use crate::common::CliNodeTypes;
|
||||
use clap::ValueEnum;
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_db::{tables, DatabaseEnv};
|
||||
use reth_db_api::table::Table;
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::NodeTypesWithDBAdapter;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use std::{hash::Hasher, sync::Arc, time::Instant};
|
||||
use tracing::info;
|
||||
|
||||
/// RocksDB tables that can be checksummed.
|
||||
#[derive(Debug, Clone, Copy, ValueEnum)]
|
||||
pub enum RocksDbTable {
|
||||
/// Transaction hash to transaction number mapping
|
||||
TransactionHashNumbers,
|
||||
/// Account history indices
|
||||
AccountsHistory,
|
||||
/// Storage history indices
|
||||
StoragesHistory,
|
||||
}
|
||||
|
||||
impl RocksDbTable {
|
||||
/// Returns the table name as a string
|
||||
const fn name(&self) -> &'static str {
|
||||
match self {
|
||||
Self::TransactionHashNumbers => tables::TransactionHashNumbers::NAME,
|
||||
Self::AccountsHistory => tables::AccountsHistory::NAME,
|
||||
Self::StoragesHistory => tables::StoragesHistory::NAME,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Computes a checksum for a RocksDB table.
|
||||
pub fn checksum_rocksdb<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
table: RocksDbTable,
|
||||
limit: Option<usize>,
|
||||
) -> eyre::Result<()> {
|
||||
let rocksdb = tool.provider_factory.rocksdb_provider();
|
||||
|
||||
let start_time = Instant::now();
|
||||
let limit = limit.unwrap_or(usize::MAX);
|
||||
|
||||
info!(
|
||||
"Computing checksum for RocksDB table `{}`, limit={:?}",
|
||||
table.name(),
|
||||
if limit == usize::MAX { None } else { Some(limit) }
|
||||
);
|
||||
|
||||
let (checksum, total) = match table {
|
||||
RocksDbTable::TransactionHashNumbers => {
|
||||
checksum_rocksdb_table::<tables::TransactionHashNumbers>(&rocksdb, limit)?
|
||||
}
|
||||
RocksDbTable::AccountsHistory => {
|
||||
checksum_rocksdb_table::<tables::AccountsHistory>(&rocksdb, limit)?
|
||||
}
|
||||
RocksDbTable::StoragesHistory => {
|
||||
checksum_rocksdb_table::<tables::StoragesHistory>(&rocksdb, limit)?
|
||||
}
|
||||
};
|
||||
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
info!(
|
||||
"Checksum for RocksDB table `{}`: {:#x} ({} entries, elapsed: {:?})",
|
||||
table.name(),
|
||||
checksum,
|
||||
total,
|
||||
elapsed
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Computes checksum for a specific RocksDB table by iterating over rows.
|
||||
fn checksum_rocksdb_table<T: Table>(
|
||||
rocksdb: &reth_provider::providers::RocksDBProvider,
|
||||
limit: usize,
|
||||
) -> eyre::Result<(u64, usize)> {
|
||||
let iter = rocksdb.raw_iter::<T>()?;
|
||||
let mut hasher = checksum_hasher();
|
||||
let mut total = 0usize;
|
||||
|
||||
for entry in iter {
|
||||
let (key_bytes, value_bytes) = entry?;
|
||||
|
||||
hasher.write(&key_bytes);
|
||||
hasher.write(&value_bytes);
|
||||
|
||||
total += 1;
|
||||
|
||||
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
|
||||
info!("Hashed {total} entries.");
|
||||
}
|
||||
|
||||
if total >= limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok((hasher.finish(), total))
|
||||
}
|
||||
@@ -39,7 +39,7 @@ pub enum Subcommands {
|
||||
Stats(stats::Command),
|
||||
/// Lists the contents of a table
|
||||
List(list::Command),
|
||||
/// Calculates the content checksum of a table
|
||||
/// Calculates the content checksum of a table or static file segment
|
||||
Checksum(checksum::Command),
|
||||
/// Create a diff between two database tables or two entire databases.
|
||||
Diff(diff::Command),
|
||||
|
||||
@@ -11,7 +11,10 @@ use reth_db_common::DbTool;
|
||||
use reth_fs_util as fs;
|
||||
use reth_node_builder::{NodePrimitives, NodeTypesWithDB, NodeTypesWithDBAdapter};
|
||||
use reth_node_core::dirs::{ChainPath, DataDirPath};
|
||||
use reth_provider::providers::{ProviderNodeTypes, StaticFileProvider};
|
||||
use reth_provider::{
|
||||
providers::{ProviderNodeTypes, StaticFileProvider},
|
||||
RocksDBProviderFactory,
|
||||
};
|
||||
use reth_static_file_types::SegmentRangeInclusive;
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
@@ -61,6 +64,11 @@ impl Command {
|
||||
let db_stats_table = self.db_stats_table(tool)?;
|
||||
println!("{db_stats_table}");
|
||||
|
||||
println!("\n");
|
||||
|
||||
let rocksdb_stats_table = self.rocksdb_stats_table(tool);
|
||||
println!("{rocksdb_stats_table}");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -148,6 +156,60 @@ impl Command {
|
||||
Ok(table)
|
||||
}
|
||||
|
||||
fn rocksdb_stats_table<N: NodeTypesWithDB>(&self, tool: &DbTool<N>) -> ComfyTable {
|
||||
let mut table = ComfyTable::new();
|
||||
table.load_preset(comfy_table::presets::ASCII_MARKDOWN);
|
||||
table.set_header([
|
||||
"RocksDB Table Name",
|
||||
"# Entries",
|
||||
"SST Size",
|
||||
"Memtable Size",
|
||||
"Total Size",
|
||||
"Pending Compaction",
|
||||
]);
|
||||
|
||||
let stats = tool.provider_factory.rocksdb_provider().table_stats();
|
||||
let mut total_sst: u64 = 0;
|
||||
let mut total_memtable: u64 = 0;
|
||||
let mut total_size: u64 = 0;
|
||||
let mut total_pending: u64 = 0;
|
||||
|
||||
for stat in &stats {
|
||||
total_sst += stat.sst_size_bytes;
|
||||
total_memtable += stat.memtable_size_bytes;
|
||||
total_size += stat.estimated_size_bytes;
|
||||
total_pending += stat.pending_compaction_bytes;
|
||||
let mut row = Row::new();
|
||||
row.add_cell(Cell::new(&stat.name))
|
||||
.add_cell(Cell::new(stat.estimated_num_keys))
|
||||
.add_cell(Cell::new(human_bytes(stat.sst_size_bytes as f64)))
|
||||
.add_cell(Cell::new(human_bytes(stat.memtable_size_bytes as f64)))
|
||||
.add_cell(Cell::new(human_bytes(stat.estimated_size_bytes as f64)))
|
||||
.add_cell(Cell::new(human_bytes(stat.pending_compaction_bytes as f64)));
|
||||
table.add_row(row);
|
||||
}
|
||||
|
||||
if !stats.is_empty() {
|
||||
let max_widths = table.column_max_content_widths();
|
||||
let mut separator = Row::new();
|
||||
for width in max_widths {
|
||||
separator.add_cell(Cell::new("-".repeat(width as usize)));
|
||||
}
|
||||
table.add_row(separator);
|
||||
|
||||
let mut row = Row::new();
|
||||
row.add_cell(Cell::new("RocksDB Total"))
|
||||
.add_cell(Cell::new(""))
|
||||
.add_cell(Cell::new(human_bytes(total_sst as f64)))
|
||||
.add_cell(Cell::new(human_bytes(total_memtable as f64)))
|
||||
.add_cell(Cell::new(human_bytes(total_size as f64)))
|
||||
.add_cell(Cell::new(human_bytes(total_pending as f64)));
|
||||
table.add_row(row);
|
||||
}
|
||||
|
||||
table
|
||||
}
|
||||
|
||||
fn static_files_stats_table<N: NodePrimitives>(
|
||||
&self,
|
||||
data_dir: ChainPath<DataDirPath>,
|
||||
|
||||
@@ -10,7 +10,8 @@ use reth_node_builder::NodeBuilder;
|
||||
use reth_node_core::{
|
||||
args::{
|
||||
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, EraArgs, MetricArgs,
|
||||
NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
|
||||
NetworkArgs, PayloadBuilderArgs, PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs,
|
||||
TxPoolArgs,
|
||||
},
|
||||
node_config::NodeConfig,
|
||||
version,
|
||||
@@ -102,6 +103,10 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
|
||||
#[command(flatten)]
|
||||
pub pruning: PruningArgs,
|
||||
|
||||
/// All `RocksDB` table routing arguments
|
||||
#[command(flatten)]
|
||||
pub rocksdb: RocksDbArgs,
|
||||
|
||||
/// Engine cli arguments
|
||||
#[command(flatten, next_help_heading = "Engine")]
|
||||
pub engine: EngineArgs,
|
||||
@@ -166,12 +171,16 @@ where
|
||||
db,
|
||||
dev,
|
||||
pruning,
|
||||
rocksdb,
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
ext,
|
||||
} = self;
|
||||
|
||||
// Validate RocksDB arguments
|
||||
rocksdb.validate()?;
|
||||
|
||||
// set up node config
|
||||
let mut node_config = NodeConfig {
|
||||
datadir,
|
||||
@@ -187,6 +196,7 @@ where
|
||||
db,
|
||||
dev,
|
||||
pruning,
|
||||
rocksdb,
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
|
||||
@@ -121,7 +121,16 @@ pub fn install() {
|
||||
unsafe {
|
||||
let alt_stack_size: usize = min_sigstack_size() + 64 * 1024;
|
||||
let mut alt_stack: libc::stack_t = mem::zeroed();
|
||||
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, 1).unwrap()).cast();
|
||||
// Both SysV AMD64 ABI and aarch64 ABI require 16 bytes alignment. We are going to be
|
||||
// generous here and just use a size of a page.
|
||||
let raw_page_sz = libc::sysconf(libc::_SC_PAGESIZE);
|
||||
let page_sz = if raw_page_sz == -1 {
|
||||
// Fallback alignment in case sysconf fails.
|
||||
4096_usize
|
||||
} else {
|
||||
raw_page_sz as usize
|
||||
};
|
||||
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, page_sz).unwrap()).cast();
|
||||
alt_stack.ss_size = alt_stack_size;
|
||||
libc::sigaltstack(&raw const alt_stack, ptr::null_mut());
|
||||
|
||||
|
||||
@@ -137,6 +137,8 @@ pub struct TreeConfig {
|
||||
account_worker_count: usize,
|
||||
/// Whether to enable V2 storage proofs.
|
||||
enable_proof_v2: bool,
|
||||
/// Whether to disable cache metrics recording (can be expensive with large cached state).
|
||||
disable_cache_metrics: bool,
|
||||
}
|
||||
|
||||
impl Default for TreeConfig {
|
||||
@@ -166,6 +168,7 @@ impl Default for TreeConfig {
|
||||
storage_worker_count: default_storage_worker_count(),
|
||||
account_worker_count: default_account_worker_count(),
|
||||
enable_proof_v2: false,
|
||||
disable_cache_metrics: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -198,6 +201,7 @@ impl TreeConfig {
|
||||
storage_worker_count: usize,
|
||||
account_worker_count: usize,
|
||||
enable_proof_v2: bool,
|
||||
disable_cache_metrics: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
persistence_threshold,
|
||||
@@ -224,6 +228,7 @@ impl TreeConfig {
|
||||
storage_worker_count,
|
||||
account_worker_count,
|
||||
enable_proof_v2,
|
||||
disable_cache_metrics,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -516,4 +521,15 @@ impl TreeConfig {
|
||||
self.enable_proof_v2 = enable_proof_v2;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether cache metrics recording is disabled.
|
||||
pub const fn disable_cache_metrics(&self) -> bool {
|
||||
self.disable_cache_metrics
|
||||
}
|
||||
|
||||
/// Setter for whether to disable cache metrics recording.
|
||||
pub const fn without_cache_metrics(mut self, disable_cache_metrics: bool) -> Self {
|
||||
self.disable_cache_metrics = disable_cache_metrics;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
# reth
|
||||
reth-chain-state.workspace = true
|
||||
reth-chain-state = { workspace = true, features = ["rayon"] }
|
||||
reth-chainspec = { workspace = true, optional = true }
|
||||
reth-consensus.workspace = true
|
||||
reth-db.workspace = true
|
||||
|
||||
@@ -606,12 +606,21 @@ pub(crate) struct SavedCache {
|
||||
/// A guard to track in-flight usage of this cache.
|
||||
/// The cache is considered available if the strong count is 1.
|
||||
usage_guard: Arc<()>,
|
||||
|
||||
/// Whether to skip cache metrics recording (can be expensive with large cached state).
|
||||
disable_cache_metrics: bool,
|
||||
}
|
||||
|
||||
impl SavedCache {
|
||||
/// Creates a new instance with the internals
|
||||
pub(super) fn new(hash: B256, caches: ExecutionCache, metrics: CachedStateMetrics) -> Self {
|
||||
Self { hash, caches, metrics, usage_guard: Arc::new(()) }
|
||||
Self { hash, caches, metrics, usage_guard: Arc::new(()), disable_cache_metrics: false }
|
||||
}
|
||||
|
||||
/// Sets whether to disable cache metrics recording.
|
||||
pub(super) const fn with_disable_cache_metrics(mut self, disable: bool) -> Self {
|
||||
self.disable_cache_metrics = disable;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns the hash for this cache
|
||||
@@ -619,9 +628,9 @@ impl SavedCache {
|
||||
self.hash
|
||||
}
|
||||
|
||||
/// Splits the cache into its caches and metrics, consuming it.
|
||||
pub(crate) fn split(self) -> (ExecutionCache, CachedStateMetrics) {
|
||||
(self.caches, self.metrics)
|
||||
/// Splits the cache into its caches, metrics, and `disable_cache_metrics` flag, consuming it.
|
||||
pub(crate) fn split(self) -> (ExecutionCache, CachedStateMetrics, bool) {
|
||||
(self.caches, self.metrics, self.disable_cache_metrics)
|
||||
}
|
||||
|
||||
/// Returns true if the cache is available for use (no other tasks are currently using it).
|
||||
@@ -645,7 +654,13 @@ impl SavedCache {
|
||||
}
|
||||
|
||||
/// Updates the metrics for the [`ExecutionCache`].
|
||||
///
|
||||
/// Note: This can be expensive with large cached state as it iterates over
|
||||
/// all storage entries. Use `with_disable_cache_metrics(true)` to skip.
|
||||
pub(crate) fn update_metrics(&self) {
|
||||
if self.disable_cache_metrics {
|
||||
return;
|
||||
}
|
||||
self.metrics.storage_cache_size.set(self.caches.total_storage_slots() as f64);
|
||||
self.metrics.account_cache_size.set(self.caches.account_cache.entry_count() as f64);
|
||||
self.metrics.code_cache_size.set(self.caches.code_cache.entry_count() as f64);
|
||||
|
||||
@@ -1,25 +1,15 @@
|
||||
use crate::tree::{error::InsertBlockFatalError, MeteredStateHook, TreeOutcome};
|
||||
use alloy_consensus::transaction::TxHashRef;
|
||||
use alloy_evm::{
|
||||
block::{BlockExecutor, ExecutableTx},
|
||||
Evm,
|
||||
};
|
||||
use crate::tree::{error::InsertBlockFatalError, TreeOutcome};
|
||||
use alloy_rpc_types_engine::{PayloadStatus, PayloadStatusEnum};
|
||||
use core::borrow::BorrowMut;
|
||||
use reth_engine_primitives::{ForkchoiceStatus, OnForkChoiceUpdated};
|
||||
use reth_errors::{BlockExecutionError, ProviderError};
|
||||
use reth_evm::{metrics::ExecutorMetrics, OnStateHook};
|
||||
use reth_errors::ProviderError;
|
||||
use reth_evm::metrics::ExecutorMetrics;
|
||||
use reth_execution_types::BlockExecutionOutput;
|
||||
use reth_metrics::{
|
||||
metrics::{Counter, Gauge, Histogram},
|
||||
Metrics,
|
||||
};
|
||||
use reth_primitives_traits::SignedTransaction;
|
||||
use reth_trie::updates::TrieUpdates;
|
||||
use revm::database::{states::bundle_state::BundleRetention, State};
|
||||
use revm_primitives::Address;
|
||||
use std::time::Instant;
|
||||
use tracing::{debug_span, trace};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Metrics for the `EngineApi`.
|
||||
#[derive(Debug, Default)]
|
||||
@@ -35,114 +25,24 @@ pub(crate) struct EngineApiMetrics {
|
||||
}
|
||||
|
||||
impl EngineApiMetrics {
|
||||
/// Helper function for metered execution
|
||||
fn metered<F, R>(&self, f: F) -> R
|
||||
where
|
||||
F: FnOnce() -> (u64, R),
|
||||
{
|
||||
// Execute the block and record the elapsed time.
|
||||
let execute_start = Instant::now();
|
||||
let (gas_used, output) = f();
|
||||
let execution_duration = execute_start.elapsed().as_secs_f64();
|
||||
|
||||
// Update gas metrics.
|
||||
self.executor.gas_processed_total.increment(gas_used);
|
||||
self.executor.gas_per_second.set(gas_used as f64 / execution_duration);
|
||||
self.executor.gas_used_histogram.record(gas_used as f64);
|
||||
self.executor.execution_histogram.record(execution_duration);
|
||||
self.executor.execution_duration.set(execution_duration);
|
||||
|
||||
output
|
||||
}
|
||||
|
||||
/// Execute the given block using the provided [`BlockExecutor`] and update metrics for the
|
||||
/// execution.
|
||||
/// Records metrics for block execution.
|
||||
///
|
||||
/// This method updates metrics for execution time, gas usage, and the number
|
||||
/// of accounts, storage slots and bytecodes loaded and updated.
|
||||
///
|
||||
/// The optional `on_receipt` callback is invoked after each transaction with the receipt
|
||||
/// index and a reference to all receipts collected so far. This allows callers to stream
|
||||
/// receipts to a background task for incremental receipt root computation.
|
||||
pub(crate) fn execute_metered<E, DB, F>(
|
||||
/// of accounts, storage slots and bytecodes updated.
|
||||
pub(crate) fn record_block_execution<R>(
|
||||
&self,
|
||||
executor: E,
|
||||
mut transactions: impl Iterator<Item = Result<impl ExecutableTx<E>, BlockExecutionError>>,
|
||||
transaction_count: usize,
|
||||
state_hook: Box<dyn OnStateHook>,
|
||||
mut on_receipt: F,
|
||||
) -> Result<(BlockExecutionOutput<E::Receipt>, Vec<Address>), BlockExecutionError>
|
||||
where
|
||||
DB: alloy_evm::Database,
|
||||
E: BlockExecutor<Evm: Evm<DB: BorrowMut<State<DB>>>, Transaction: SignedTransaction>,
|
||||
F: FnMut(&[E::Receipt]),
|
||||
{
|
||||
// clone here is cheap, all the metrics are Option<Arc<_>>. additionally
|
||||
// they are globally registered so that the data recorded in the hook will
|
||||
// be accessible.
|
||||
let wrapper = MeteredStateHook { metrics: self.executor.clone(), inner_hook: state_hook };
|
||||
output: &BlockExecutionOutput<R>,
|
||||
execution_duration: Duration,
|
||||
) {
|
||||
let execution_secs = execution_duration.as_secs_f64();
|
||||
let gas_used = output.result.gas_used;
|
||||
|
||||
let mut senders = Vec::with_capacity(transaction_count);
|
||||
let mut executor = executor.with_state_hook(Some(Box::new(wrapper)));
|
||||
|
||||
let f = || {
|
||||
let start = Instant::now();
|
||||
debug_span!(target: "engine::tree", "pre execution")
|
||||
.entered()
|
||||
.in_scope(|| executor.apply_pre_execution_changes())?;
|
||||
self.executor.pre_execution_histogram.record(start.elapsed());
|
||||
|
||||
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
|
||||
loop {
|
||||
let start = Instant::now();
|
||||
let Some(tx) = transactions.next() else { break };
|
||||
self.executor.transaction_wait_histogram.record(start.elapsed());
|
||||
|
||||
let tx = tx?;
|
||||
senders.push(*tx.signer());
|
||||
|
||||
let span = debug_span!(
|
||||
target: "engine::tree",
|
||||
"execute tx",
|
||||
tx_hash = ?tx.tx().tx_hash(),
|
||||
gas_used = tracing::field::Empty,
|
||||
);
|
||||
let enter = span.entered();
|
||||
trace!(target: "engine::tree", "Executing transaction");
|
||||
let start = Instant::now();
|
||||
let gas_used = executor.execute_transaction(tx)?;
|
||||
self.executor.transaction_execution_histogram.record(start.elapsed());
|
||||
|
||||
// Invoke callback with the latest receipt
|
||||
on_receipt(executor.receipts());
|
||||
|
||||
// record the tx gas used
|
||||
enter.record("gas_used", gas_used);
|
||||
}
|
||||
drop(exec_span);
|
||||
|
||||
let start = Instant::now();
|
||||
let result = debug_span!(target: "engine::tree", "finish")
|
||||
.entered()
|
||||
.in_scope(|| executor.finish())
|
||||
.map(|(evm, result)| (evm.into_db(), result));
|
||||
self.executor.post_execution_histogram.record(start.elapsed());
|
||||
|
||||
result
|
||||
};
|
||||
|
||||
// Use metered to execute and track timing/gas metrics
|
||||
let (mut db, result) = self.metered(|| {
|
||||
let res = f();
|
||||
let gas_used = res.as_ref().map(|r| r.1.gas_used).unwrap_or(0);
|
||||
(gas_used, res)
|
||||
})?;
|
||||
|
||||
// merge transitions into bundle state
|
||||
debug_span!(target: "engine::tree", "merge transitions")
|
||||
.entered()
|
||||
.in_scope(|| db.borrow_mut().merge_transitions(BundleRetention::Reverts));
|
||||
let output = BlockExecutionOutput { result, state: db.borrow_mut().take_bundle() };
|
||||
// Update gas metrics
|
||||
self.executor.gas_processed_total.increment(gas_used);
|
||||
self.executor.gas_per_second.set(gas_used as f64 / execution_secs);
|
||||
self.executor.gas_used_histogram.record(gas_used as f64);
|
||||
self.executor.execution_histogram.record(execution_secs);
|
||||
self.executor.execution_duration.set(execution_secs);
|
||||
|
||||
// Update the metrics for the number of accounts, storage slots and bytecodes updated
|
||||
let accounts = output.state.state.len();
|
||||
@@ -153,8 +53,31 @@ impl EngineApiMetrics {
|
||||
self.executor.accounts_updated_histogram.record(accounts as f64);
|
||||
self.executor.storage_slots_updated_histogram.record(storage_slots as f64);
|
||||
self.executor.bytecodes_updated_histogram.record(bytecodes as f64);
|
||||
}
|
||||
|
||||
Ok((output, senders))
|
||||
/// Returns a reference to the executor metrics for use in state hooks.
|
||||
pub(crate) const fn executor_metrics(&self) -> &ExecutorMetrics {
|
||||
&self.executor
|
||||
}
|
||||
|
||||
/// Records the duration of block pre-execution changes (e.g., beacon root update).
|
||||
pub(crate) fn record_pre_execution(&self, elapsed: Duration) {
|
||||
self.executor.pre_execution_histogram.record(elapsed);
|
||||
}
|
||||
|
||||
/// Records the duration of block post-execution changes (e.g., finalization).
|
||||
pub(crate) fn record_post_execution(&self, elapsed: Duration) {
|
||||
self.executor.post_execution_histogram.record(elapsed);
|
||||
}
|
||||
|
||||
/// Records the time spent waiting for the next transaction from the iterator.
|
||||
pub(crate) fn record_transaction_wait(&self, elapsed: Duration) {
|
||||
self.executor.transaction_wait_histogram.record(elapsed);
|
||||
}
|
||||
|
||||
/// Records the duration of a single transaction execution.
|
||||
pub(crate) fn record_transaction_execution(&self, elapsed: Duration) {
|
||||
self.executor.transaction_execution_histogram.record(elapsed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -210,6 +133,12 @@ pub(crate) struct EngineMetrics {
|
||||
#[derive(Metrics)]
|
||||
#[metrics(scope = "consensus.engine.beacon")]
|
||||
pub(crate) struct ForkchoiceUpdatedMetrics {
|
||||
/// Finish time of the latest forkchoice updated call.
|
||||
#[metric(skip)]
|
||||
pub(crate) latest_finish_at: Option<Instant>,
|
||||
/// Start time of the latest forkchoice updated call.
|
||||
#[metric(skip)]
|
||||
pub(crate) latest_start_at: Option<Instant>,
|
||||
/// The total count of forkchoice updated messages received.
|
||||
pub(crate) forkchoice_updated_messages: Counter,
|
||||
/// The total count of forkchoice updated messages with payload received.
|
||||
@@ -232,18 +161,35 @@ pub(crate) struct ForkchoiceUpdatedMetrics {
|
||||
pub(crate) forkchoice_updated_last: Gauge,
|
||||
/// Time diff between new payload call response and the next forkchoice updated call request.
|
||||
pub(crate) new_payload_forkchoice_updated_time_diff: Histogram,
|
||||
/// Time from previous forkchoice updated finish to current forkchoice updated start (idle
|
||||
/// time).
|
||||
pub(crate) time_between_forkchoice_updated: Histogram,
|
||||
/// Time from previous forkchoice updated start to current forkchoice updated start (total
|
||||
/// interval).
|
||||
pub(crate) forkchoice_updated_interval: Histogram,
|
||||
}
|
||||
|
||||
impl ForkchoiceUpdatedMetrics {
|
||||
/// Increment the forkchoiceUpdated counter based on the given result
|
||||
pub(crate) fn update_response_metrics(
|
||||
&self,
|
||||
&mut self,
|
||||
start: Instant,
|
||||
latest_new_payload_at: &mut Option<Instant>,
|
||||
has_attrs: bool,
|
||||
result: &Result<TreeOutcome<OnForkChoiceUpdated>, ProviderError>,
|
||||
) {
|
||||
let elapsed = start.elapsed();
|
||||
let finish = Instant::now();
|
||||
let elapsed = finish - start;
|
||||
|
||||
if let Some(prev_finish) = self.latest_finish_at {
|
||||
self.time_between_forkchoice_updated.record(start - prev_finish);
|
||||
}
|
||||
if let Some(prev_start) = self.latest_start_at {
|
||||
self.forkchoice_updated_interval.record(start - prev_start);
|
||||
}
|
||||
self.latest_finish_at = Some(finish);
|
||||
self.latest_start_at = Some(start);
|
||||
|
||||
match result {
|
||||
Ok(outcome) => match outcome.outcome.forkchoice_status() {
|
||||
ForkchoiceStatus::Valid => self.forkchoice_updated_valid.increment(1),
|
||||
@@ -410,138 +356,10 @@ pub(crate) struct BlockBufferMetrics {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_eips::eip7685::Requests;
|
||||
use alloy_evm::block::StateChangeSource;
|
||||
use alloy_primitives::{B256, U256};
|
||||
use metrics_util::debugging::{DebuggingRecorder, Snapshotter};
|
||||
use reth_ethereum_primitives::{Receipt, TransactionSigned};
|
||||
use reth_evm_ethereum::EthEvm;
|
||||
use reth_ethereum_primitives::Receipt;
|
||||
use reth_execution_types::BlockExecutionResult;
|
||||
use reth_primitives_traits::RecoveredBlock;
|
||||
use revm::{
|
||||
context::result::{ExecutionResult, Output, ResultAndState, SuccessReason},
|
||||
database::State,
|
||||
database_interface::EmptyDB,
|
||||
inspector::NoOpInspector,
|
||||
state::{Account, AccountInfo, AccountStatus, EvmState, EvmStorage, EvmStorageSlot},
|
||||
Context, MainBuilder, MainContext,
|
||||
};
|
||||
use revm_primitives::Bytes;
|
||||
use std::sync::mpsc;
|
||||
|
||||
/// A simple mock executor for testing that doesn't require complex EVM setup
|
||||
struct MockExecutor {
|
||||
state: EvmState,
|
||||
receipts: Vec<Receipt>,
|
||||
hook: Option<Box<dyn OnStateHook>>,
|
||||
}
|
||||
|
||||
impl MockExecutor {
|
||||
fn new(state: EvmState) -> Self {
|
||||
Self { state, receipts: vec![], hook: None }
|
||||
}
|
||||
}
|
||||
|
||||
// Mock Evm type for testing
|
||||
type MockEvm = EthEvm<State<EmptyDB>, NoOpInspector>;
|
||||
|
||||
impl BlockExecutor for MockExecutor {
|
||||
type Transaction = TransactionSigned;
|
||||
type Receipt = Receipt;
|
||||
type Evm = MockEvm;
|
||||
|
||||
fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn execute_transaction_without_commit(
|
||||
&mut self,
|
||||
_tx: impl ExecutableTx<Self>,
|
||||
) -> Result<ResultAndState<<Self::Evm as Evm>::HaltReason>, BlockExecutionError> {
|
||||
// Call hook with our mock state for each transaction
|
||||
if let Some(hook) = self.hook.as_mut() {
|
||||
hook.on_state(StateChangeSource::Transaction(0), &self.state);
|
||||
}
|
||||
|
||||
Ok(ResultAndState::new(
|
||||
ExecutionResult::Success {
|
||||
reason: SuccessReason::Return,
|
||||
gas_used: 1000, // Mock gas used
|
||||
gas_refunded: 0,
|
||||
logs: vec![],
|
||||
output: Output::Call(Bytes::from(vec![])),
|
||||
},
|
||||
Default::default(),
|
||||
))
|
||||
}
|
||||
|
||||
fn commit_transaction(
|
||||
&mut self,
|
||||
_output: ResultAndState<<Self::Evm as Evm>::HaltReason>,
|
||||
_tx: impl ExecutableTx<Self>,
|
||||
) -> Result<u64, BlockExecutionError> {
|
||||
Ok(1000)
|
||||
}
|
||||
|
||||
fn finish(
|
||||
self,
|
||||
) -> Result<(Self::Evm, BlockExecutionResult<Self::Receipt>), BlockExecutionError> {
|
||||
let Self { hook, state, .. } = self;
|
||||
|
||||
// Call hook with our mock state
|
||||
if let Some(mut hook) = hook {
|
||||
hook.on_state(StateChangeSource::Transaction(0), &state);
|
||||
}
|
||||
|
||||
// Create a mock EVM
|
||||
let db = State::builder()
|
||||
.with_database(EmptyDB::default())
|
||||
.with_bundle_update()
|
||||
.without_state_clear()
|
||||
.build();
|
||||
let evm = EthEvm::new(
|
||||
Context::mainnet().with_db(db).build_mainnet_with_inspector(NoOpInspector {}),
|
||||
false,
|
||||
);
|
||||
|
||||
// Return successful result like the original tests
|
||||
Ok((
|
||||
evm,
|
||||
BlockExecutionResult {
|
||||
receipts: vec![],
|
||||
requests: Requests::default(),
|
||||
gas_used: 1000,
|
||||
blob_gas_used: 0,
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
fn set_state_hook(&mut self, hook: Option<Box<dyn OnStateHook>>) {
|
||||
self.hook = hook;
|
||||
}
|
||||
|
||||
fn evm_mut(&mut self) -> &mut Self::Evm {
|
||||
panic!("Mock executor evm_mut() not implemented")
|
||||
}
|
||||
|
||||
fn evm(&self) -> &Self::Evm {
|
||||
panic!("Mock executor evm() not implemented")
|
||||
}
|
||||
|
||||
fn receipts(&self) -> &[Self::Receipt] {
|
||||
&self.receipts
|
||||
}
|
||||
}
|
||||
|
||||
struct ChannelStateHook {
|
||||
output: i32,
|
||||
sender: mpsc::Sender<i32>,
|
||||
}
|
||||
|
||||
impl OnStateHook for ChannelStateHook {
|
||||
fn on_state(&mut self, _source: StateChangeSource, _state: &EvmState) {
|
||||
let _ = self.sender.send(self.output);
|
||||
}
|
||||
}
|
||||
use reth_revm::db::BundleState;
|
||||
|
||||
fn setup_test_recorder() -> Snapshotter {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
@@ -551,38 +369,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_executor_metrics_hook_called() {
|
||||
let metrics = EngineApiMetrics::default();
|
||||
let input = RecoveredBlock::<reth_ethereum_primitives::Block>::default();
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let expected_output = 42;
|
||||
let state_hook = Box::new(ChannelStateHook { sender: tx, output: expected_output });
|
||||
|
||||
let state = EvmState::default();
|
||||
let executor = MockExecutor::new(state);
|
||||
|
||||
// This will fail to create the EVM but should still call the hook
|
||||
let _result = metrics.execute_metered::<_, EmptyDB, _>(
|
||||
executor,
|
||||
input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>),
|
||||
input.transaction_count(),
|
||||
state_hook,
|
||||
|_| {},
|
||||
);
|
||||
|
||||
// Check if hook was called (it might not be if finish() fails early)
|
||||
match rx.try_recv() {
|
||||
Ok(actual_output) => assert_eq!(actual_output, expected_output),
|
||||
Err(_) => {
|
||||
// Hook wasn't called, which is expected if the mock fails early
|
||||
// The test still validates that the code compiles and runs
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_executor_metrics_hook_metrics_recorded() {
|
||||
fn test_record_block_execution_metrics() {
|
||||
let snapshotter = setup_test_recorder();
|
||||
let metrics = EngineApiMetrics::default();
|
||||
|
||||
@@ -591,45 +378,17 @@ mod tests {
|
||||
metrics.executor.gas_per_second.set(0.0);
|
||||
metrics.executor.gas_used_histogram.record(0.0);
|
||||
|
||||
let input = RecoveredBlock::<reth_ethereum_primitives::Block>::default();
|
||||
|
||||
let (tx, _rx) = mpsc::channel();
|
||||
let state_hook = Box::new(ChannelStateHook { sender: tx, output: 42 });
|
||||
|
||||
// Create a state with some data
|
||||
let state = {
|
||||
let mut state = EvmState::default();
|
||||
let storage =
|
||||
EvmStorage::from_iter([(U256::from(1), EvmStorageSlot::new(U256::from(2), 0))]);
|
||||
state.insert(
|
||||
Default::default(),
|
||||
Account {
|
||||
info: AccountInfo {
|
||||
balance: U256::from(100),
|
||||
nonce: 10,
|
||||
code_hash: B256::random(),
|
||||
code: Default::default(),
|
||||
account_id: None,
|
||||
},
|
||||
original_info: Box::new(AccountInfo::default()),
|
||||
storage,
|
||||
status: AccountStatus::default(),
|
||||
transaction_id: 0,
|
||||
},
|
||||
);
|
||||
state
|
||||
let output = BlockExecutionOutput::<Receipt> {
|
||||
state: BundleState::default(),
|
||||
result: BlockExecutionResult {
|
||||
receipts: vec![],
|
||||
requests: Requests::default(),
|
||||
gas_used: 21000,
|
||||
blob_gas_used: 0,
|
||||
},
|
||||
};
|
||||
|
||||
let executor = MockExecutor::new(state);
|
||||
|
||||
// Execute (will fail but should still update some metrics)
|
||||
let _result = metrics.execute_metered::<_, EmptyDB, _>(
|
||||
executor,
|
||||
input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>),
|
||||
input.transaction_count(),
|
||||
state_hook,
|
||||
|_| {},
|
||||
);
|
||||
metrics.record_block_execution(&output, Duration::from_millis(100));
|
||||
|
||||
let snapshot = snapshotter.snapshot().into_vec();
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ use alloy_evm::{block::StateChangeSource, ToTxEnv};
|
||||
use alloy_primitives::B256;
|
||||
use crossbeam_channel::Sender as CrossbeamSender;
|
||||
use executor::WorkloadExecutor;
|
||||
use metrics::Counter;
|
||||
use multiproof::{SparseTrieUpdate, *};
|
||||
use parking_lot::RwLock;
|
||||
use prewarm::PrewarmMetrics;
|
||||
@@ -28,6 +29,7 @@ use reth_evm::{
|
||||
ConfigureEvm, EvmEnvFor, ExecutableTxIterator, ExecutableTxTuple, OnStateHook, SpecFor,
|
||||
TxEnvFor,
|
||||
};
|
||||
use reth_metrics::Metrics;
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_provider::{
|
||||
BlockExecutionOutput, BlockReader, DatabaseProviderROFactory, StateProvider,
|
||||
@@ -139,6 +141,8 @@ where
|
||||
disable_parallel_sparse_trie: bool,
|
||||
/// Maximum concurrency for prewarm task.
|
||||
prewarm_max_concurrency: usize,
|
||||
/// Whether to disable cache metrics recording.
|
||||
disable_cache_metrics: bool,
|
||||
}
|
||||
|
||||
impl<N, Evm> PayloadProcessor<Evm>
|
||||
@@ -171,6 +175,7 @@ where
|
||||
sparse_state_trie: Arc::default(),
|
||||
disable_parallel_sparse_trie: config.disable_parallel_sparse_trie(),
|
||||
prewarm_max_concurrency: config.prewarm_max_concurrency(),
|
||||
disable_cache_metrics: config.disable_cache_metrics(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -300,7 +305,7 @@ where
|
||||
// Build a state provider for the multiproof task
|
||||
let provider = provider_builder.build().expect("failed to build provider");
|
||||
let provider = if let Some(saved_cache) = saved_cache {
|
||||
let (cache, metrics) = saved_cache.split();
|
||||
let (cache, metrics, _) = saved_cache.split();
|
||||
Box::new(CachedStateProvider::new(provider, cache, metrics))
|
||||
as Box<dyn StateProvider>
|
||||
} else {
|
||||
@@ -477,6 +482,7 @@ where
|
||||
debug!("creating new execution cache on cache miss");
|
||||
let cache = ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size);
|
||||
SavedCache::new(parent_hash, cache, CachedStateMetrics::zeroed())
|
||||
.with_disable_cache_metrics(self.disable_cache_metrics)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -558,6 +564,7 @@ where
|
||||
block_with_parent: BlockWithParent,
|
||||
bundle_state: &BundleState,
|
||||
) {
|
||||
let disable_cache_metrics = self.disable_cache_metrics;
|
||||
self.execution_cache.update_with_guard(|cached| {
|
||||
if cached.as_ref().is_some_and(|c| c.executed_block_hash() != block_with_parent.parent) {
|
||||
debug!(
|
||||
@@ -571,7 +578,8 @@ where
|
||||
// Take existing cache (if any) or create fresh caches
|
||||
let (caches, cache_metrics) = match cached.take() {
|
||||
Some(existing) => {
|
||||
existing.split()
|
||||
let (c, m, _) = existing.split();
|
||||
(c, m)
|
||||
}
|
||||
None => (
|
||||
ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size),
|
||||
@@ -580,7 +588,8 @@ where
|
||||
};
|
||||
|
||||
// Insert the block's bundle state into cache
|
||||
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics);
|
||||
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics)
|
||||
.with_disable_cache_metrics(disable_cache_metrics);
|
||||
if new_cache.cache().insert_state(bundle_state).is_err() {
|
||||
*cached = None;
|
||||
debug!(target: "engine::caching", "cleared execution cache on update error");
|
||||
@@ -770,6 +779,8 @@ impl<R> Drop for CacheTaskHandle<R> {
|
||||
struct ExecutionCache {
|
||||
/// Guarded cloneable cache identified by a block hash.
|
||||
inner: Arc<RwLock<Option<SavedCache>>>,
|
||||
/// Metrics for cache operations.
|
||||
metrics: ExecutionCacheMetrics,
|
||||
}
|
||||
|
||||
impl ExecutionCache {
|
||||
@@ -811,6 +822,10 @@ impl ExecutionCache {
|
||||
if hash_matches && available {
|
||||
return Some(c.clone());
|
||||
}
|
||||
|
||||
if hash_matches && !available {
|
||||
self.metrics.execution_cache_in_use.increment(1);
|
||||
}
|
||||
} else {
|
||||
debug!(target: "engine::caching", %parent_hash, "No cache found");
|
||||
}
|
||||
@@ -846,6 +861,15 @@ impl ExecutionCache {
|
||||
}
|
||||
}
|
||||
|
||||
/// Metrics for execution cache operations.
|
||||
#[derive(Metrics, Clone)]
|
||||
#[metrics(scope = "consensus.engine.beacon")]
|
||||
pub(crate) struct ExecutionCacheMetrics {
|
||||
/// Counter for when the execution cache was unavailable because other threads
|
||||
/// (e.g., prewarming) are still using it.
|
||||
pub(crate) execution_cache_in_use: Counter,
|
||||
}
|
||||
|
||||
/// EVM context required to execute a block.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ExecutionEnv<Evm: ConfigureEvm> {
|
||||
|
||||
@@ -278,8 +278,9 @@ where
|
||||
execution_cache.update_with_guard(|cached| {
|
||||
// consumes the `SavedCache` held by the prewarming task, which releases its usage
|
||||
// guard
|
||||
let (caches, cache_metrics) = saved_cache.split();
|
||||
let new_cache = SavedCache::new(hash, caches, cache_metrics);
|
||||
let (caches, cache_metrics, disable_cache_metrics) = saved_cache.split();
|
||||
let new_cache = SavedCache::new(hash, caches, cache_metrics)
|
||||
.with_disable_cache_metrics(disable_cache_metrics);
|
||||
|
||||
// Insert state into cache while holding the lock
|
||||
// Access the BundleState through the shared ExecutionOutcome
|
||||
@@ -296,6 +297,11 @@ where
|
||||
// Replace the shared cache with the new one; the previous cache (if any) is
|
||||
// dropped.
|
||||
*cached = Some(new_cache);
|
||||
} else {
|
||||
// Block was invalid; caches were already mutated by insert_state above,
|
||||
// so we must clear to prevent using polluted state
|
||||
*cached = None;
|
||||
debug!(target: "engine::caching", "cleared execution cache on invalid block");
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -64,14 +64,11 @@ impl<R: Receipt> ReceiptRootTaskHandle<R> {
|
||||
///
|
||||
/// * `receipts_len` - The total number of receipts expected. This is needed to correctly order
|
||||
/// the trie keys according to RLP encoding rules.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if the number of receipts received doesn't match `receipts_len`.
|
||||
pub fn run(self, receipts_len: usize) {
|
||||
let mut builder = OrderedTrieRootEncodedBuilder::new(receipts_len);
|
||||
let mut aggregated_bloom = Bloom::ZERO;
|
||||
let mut encode_buf = Vec::new();
|
||||
let mut received_count = 0usize;
|
||||
|
||||
for indexed_receipt in self.receipt_rx {
|
||||
let receipt_with_bloom = indexed_receipt.receipt.with_bloom_ref();
|
||||
@@ -81,9 +78,21 @@ impl<R: Receipt> ReceiptRootTaskHandle<R> {
|
||||
|
||||
aggregated_bloom |= *receipt_with_bloom.bloom_ref();
|
||||
builder.push_unchecked(indexed_receipt.index, &encode_buf);
|
||||
received_count += 1;
|
||||
}
|
||||
|
||||
let root = builder.finalize().expect("receipt root builder incomplete");
|
||||
let Ok(root) = builder.finalize() else {
|
||||
// Finalize fails if we didn't receive exactly `receipts_len` receipts. This can
|
||||
// happen if execution was aborted early (e.g., invalid transaction encountered).
|
||||
// We return without sending a result, allowing the caller to handle the abort.
|
||||
tracing::error!(
|
||||
target: "engine::tree::payload_processor",
|
||||
expected = receipts_len,
|
||||
received = received_count,
|
||||
"Receipt root task received incomplete receipts, execution likely aborted"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let _ = self.result_tx.send((root, aggregated_bloom));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,10 +7,10 @@ use crate::tree::{
|
||||
payload_processor::{executor::WorkloadExecutor, PayloadProcessor},
|
||||
precompile_cache::{CachedPrecompile, CachedPrecompileMetrics, PrecompileCacheMap},
|
||||
sparse_trie::StateRootComputeOutcome,
|
||||
EngineApiMetrics, EngineApiTreeState, ExecutionEnv, PayloadHandle, StateProviderBuilder,
|
||||
StateProviderDatabase, TreeConfig,
|
||||
EngineApiMetrics, EngineApiTreeState, ExecutionEnv, MeteredStateHook, PayloadHandle,
|
||||
StateProviderBuilder, StateProviderDatabase, TreeConfig,
|
||||
};
|
||||
use alloy_consensus::transaction::Either;
|
||||
use alloy_consensus::transaction::{Either, TxHashRef};
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::{eip1898::BlockWithParent, NumHash};
|
||||
use alloy_evm::Evm;
|
||||
@@ -41,7 +41,7 @@ use reth_provider::{
|
||||
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
|
||||
StateProviderFactory, StateReader,
|
||||
};
|
||||
use reth_revm::db::State;
|
||||
use reth_revm::db::{states::bundle_state::BundleRetention, State};
|
||||
use reth_trie::{updates::TrieUpdates, HashedPostState, StateRoot};
|
||||
use reth_trie_db::ChangesetCache;
|
||||
use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError};
|
||||
@@ -479,11 +479,15 @@ where
|
||||
let block = self.convert_to_block(input)?.with_senders(senders);
|
||||
|
||||
// Wait for the receipt root computation to complete.
|
||||
let receipt_root_bloom = Some(
|
||||
receipt_root_rx
|
||||
.blocking_recv()
|
||||
.expect("receipt root task dropped sender without result"),
|
||||
);
|
||||
let receipt_root_bloom = receipt_root_rx
|
||||
.blocking_recv()
|
||||
.inspect_err(|_| {
|
||||
tracing::error!(
|
||||
target: "engine::tree::payload_validator",
|
||||
"Receipt root task dropped sender without result, receipt root calculation likely aborted"
|
||||
);
|
||||
})
|
||||
.ok();
|
||||
|
||||
let hashed_state = ensure_ok_post_block!(
|
||||
self.validate_post_execution(
|
||||
@@ -638,7 +642,13 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Executes a block with the given state provider
|
||||
/// Executes a block with the given state provider.
|
||||
///
|
||||
/// This method orchestrates block execution:
|
||||
/// 1. Sets up the EVM with state database and precompile caching
|
||||
/// 2. Spawns a background task for incremental receipt root computation
|
||||
/// 3. Executes transactions with metrics collection via state hooks
|
||||
/// 4. Merges state transitions and records execution metrics
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)]
|
||||
#[expect(clippy::type_complexity)]
|
||||
fn execute_block<S, Err, T>(
|
||||
@@ -701,31 +711,117 @@ where
|
||||
let task_handle = ReceiptRootTaskHandle::new(receipt_rx, result_tx);
|
||||
self.payload_processor.executor().spawn_blocking(move || task_handle.run(receipts_len));
|
||||
|
||||
// Wrap the state hook with metrics collection
|
||||
let inner_hook = Box::new(handle.state_hook());
|
||||
let state_hook =
|
||||
MeteredStateHook { metrics: self.metrics.executor_metrics().clone(), inner_hook };
|
||||
|
||||
let transaction_count = input.transaction_count();
|
||||
let executor = executor.with_state_hook(Some(Box::new(state_hook)));
|
||||
|
||||
let execution_start = Instant::now();
|
||||
let state_hook = Box::new(handle.state_hook());
|
||||
let (output, senders) = self.metrics.execute_metered(
|
||||
|
||||
// Execute all transactions and finalize
|
||||
let (executor, senders) = self.execute_transactions(
|
||||
executor,
|
||||
handle.iter_transactions().map(|res| res.map_err(BlockExecutionError::other)),
|
||||
input.transaction_count(),
|
||||
state_hook,
|
||||
|receipts| {
|
||||
// Send the latest receipt to the background task for incremental root computation.
|
||||
// The receipt is cloned here; encoding happens in the background thread.
|
||||
if let Some(receipt) = receipts.last() {
|
||||
// Infer tx_index from the number of receipts collected so far
|
||||
let tx_index = receipts.len() - 1;
|
||||
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
|
||||
}
|
||||
},
|
||||
transaction_count,
|
||||
handle.iter_transactions(),
|
||||
&receipt_tx,
|
||||
)?;
|
||||
drop(receipt_tx);
|
||||
|
||||
let execution_finish = Instant::now();
|
||||
let execution_time = execution_finish.duration_since(execution_start);
|
||||
debug!(target: "engine::tree::payload_validator", elapsed = ?execution_time, "Executed block");
|
||||
// Finish execution and get the result
|
||||
let post_exec_start = Instant::now();
|
||||
let (_evm, result) = debug_span!(target: "engine::tree", "finish")
|
||||
.in_scope(|| executor.finish())
|
||||
.map(|(evm, result)| (evm.into_db(), result))?;
|
||||
self.metrics.record_post_execution(post_exec_start.elapsed());
|
||||
|
||||
// Merge transitions into bundle state
|
||||
debug_span!(target: "engine::tree", "merge transitions")
|
||||
.in_scope(|| db.merge_transitions(BundleRetention::Reverts));
|
||||
|
||||
let output = BlockExecutionOutput { result, state: db.take_bundle() };
|
||||
|
||||
let execution_duration = execution_start.elapsed();
|
||||
self.metrics.record_block_execution(&output, execution_duration);
|
||||
|
||||
debug!(target: "engine::tree::payload_validator", elapsed = ?execution_duration, "Executed block");
|
||||
Ok((output, senders, result_rx))
|
||||
}
|
||||
|
||||
/// Executes transactions and collects senders, streaming receipts to a background task.
|
||||
///
|
||||
/// This method handles:
|
||||
/// - Applying pre-execution changes (e.g., beacon root updates)
|
||||
/// - Executing each transaction with timing metrics
|
||||
/// - Streaming receipts to the receipt root computation task
|
||||
/// - Collecting transaction senders for later use
|
||||
///
|
||||
/// Returns the executor (for finalization) and the collected senders.
|
||||
fn execute_transactions<E, Tx, InnerTx, Err>(
|
||||
&self,
|
||||
mut executor: E,
|
||||
transaction_count: usize,
|
||||
transactions: impl Iterator<Item = Result<Tx, Err>>,
|
||||
receipt_tx: &crossbeam_channel::Sender<IndexedReceipt<N::Receipt>>,
|
||||
) -> Result<(E, Vec<Address>), BlockExecutionError>
|
||||
where
|
||||
E: BlockExecutor<Receipt = N::Receipt>,
|
||||
Tx: alloy_evm::block::ExecutableTx<E> + alloy_evm::RecoveredTx<InnerTx>,
|
||||
InnerTx: TxHashRef,
|
||||
Err: core::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
let mut senders = Vec::with_capacity(transaction_count);
|
||||
|
||||
// Apply pre-execution changes (e.g., beacon root update)
|
||||
let pre_exec_start = Instant::now();
|
||||
debug_span!(target: "engine::tree", "pre execution")
|
||||
.in_scope(|| executor.apply_pre_execution_changes())?;
|
||||
self.metrics.record_pre_execution(pre_exec_start.elapsed());
|
||||
|
||||
// Execute transactions
|
||||
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
|
||||
let mut transactions = transactions.into_iter();
|
||||
loop {
|
||||
// Measure time spent waiting for next transaction from iterator
|
||||
// (e.g., parallel signature recovery)
|
||||
let wait_start = Instant::now();
|
||||
let Some(tx_result) = transactions.next() else { break };
|
||||
self.metrics.record_transaction_wait(wait_start.elapsed());
|
||||
|
||||
let tx = tx_result.map_err(BlockExecutionError::other)?;
|
||||
let tx_signer = *<Tx as alloy_evm::RecoveredTx<InnerTx>>::signer(&tx);
|
||||
let tx_hash = <Tx as alloy_evm::RecoveredTx<InnerTx>>::tx(&tx).tx_hash();
|
||||
|
||||
senders.push(tx_signer);
|
||||
|
||||
let span = debug_span!(
|
||||
target: "engine::tree",
|
||||
"execute tx",
|
||||
?tx_hash,
|
||||
gas_used = tracing::field::Empty,
|
||||
);
|
||||
let enter = span.entered();
|
||||
trace!(target: "engine::tree", "Executing transaction");
|
||||
|
||||
let tx_start = Instant::now();
|
||||
let gas_used = executor.execute_transaction(tx)?;
|
||||
self.metrics.record_transaction_execution(tx_start.elapsed());
|
||||
|
||||
// Send the latest receipt to the background task for incremental root computation
|
||||
if let Some(receipt) = executor.receipts().last() {
|
||||
let tx_index = executor.receipts().len() - 1;
|
||||
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
|
||||
}
|
||||
|
||||
enter.record("gas_used", gas_used);
|
||||
}
|
||||
drop(exec_span);
|
||||
|
||||
Ok((executor, senders))
|
||||
}
|
||||
|
||||
/// Compute state root for the given hashed post state in parallel.
|
||||
///
|
||||
/// Uses an overlay factory which provides the state of the parent block, along with the
|
||||
|
||||
@@ -148,20 +148,6 @@ pub trait Executor<DB: Database>: Sized {
|
||||
fn size_hint(&self) -> usize;
|
||||
}
|
||||
|
||||
/// Helper type for the output of executing a block.
|
||||
///
|
||||
/// Deprecated: this type is unused within reth and will be removed in the next
|
||||
/// major release. Use `reth_execution_types::BlockExecutionResult` or
|
||||
/// `reth_execution_types::BlockExecutionOutput`.
|
||||
#[deprecated(note = "Use reth_execution_types::BlockExecutionResult or BlockExecutionOutput")]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ExecuteOutput<R> {
|
||||
/// Receipts obtained after executing a block.
|
||||
pub receipts: Vec<R>,
|
||||
/// Cumulative gas used in the block execution.
|
||||
pub gas_used: u64,
|
||||
}
|
||||
|
||||
/// Input for block building. Consumed by [`BlockAssembler`].
|
||||
///
|
||||
/// This struct contains all the data needed by the [`BlockAssembler`] to create
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
use crate::ExecutionOutcome;
|
||||
use alloc::{borrow::Cow, collections::BTreeMap, vec::Vec};
|
||||
use alloy_consensus::{transaction::Recovered, BlockHeader};
|
||||
use alloy_consensus::{transaction::Recovered, BlockHeader, TxReceipt};
|
||||
use alloy_eips::{eip1898::ForkBlock, eip2718::Encodable2718, BlockNumHash};
|
||||
use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash};
|
||||
use alloy_primitives::{Address, BlockHash, BlockNumber, Log, TxHash};
|
||||
use core::{fmt, ops::RangeInclusive};
|
||||
use reth_primitives_traits::{
|
||||
transaction::signed::SignedTransaction, Block, BlockBody, IndexedTx, NodePrimitives,
|
||||
@@ -184,6 +184,19 @@ impl<N: NodePrimitives> Chain<N> {
|
||||
self.execution_outcome.receipts().iter()
|
||||
}
|
||||
|
||||
/// Returns an iterator over all receipts in the chain.
|
||||
pub fn receipts_iter(&self) -> impl Iterator<Item = &N::Receipt> + '_ {
|
||||
self.block_receipts_iter().flatten()
|
||||
}
|
||||
|
||||
/// Returns an iterator over all logs in the chain.
|
||||
pub fn logs_iter(&self) -> impl Iterator<Item = &Log> + '_
|
||||
where
|
||||
N::Receipt: TxReceipt<Log = Log>,
|
||||
{
|
||||
self.receipts_iter().flat_map(|receipt| receipt.logs())
|
||||
}
|
||||
|
||||
/// Returns an iterator over all blocks in the chain with increasing block number.
|
||||
pub fn blocks_iter(&self) -> impl Iterator<Item = &RecoveredBlock<N::Block>> + '_ {
|
||||
self.blocks().iter().map(|block| block.1)
|
||||
|
||||
@@ -676,19 +676,13 @@ where
|
||||
|
||||
/// Convenience function to [`Self::init_genesis`]
|
||||
pub fn with_genesis(self) -> Result<Self, InitStorageError> {
|
||||
init_genesis_with_settings(
|
||||
self.provider_factory(),
|
||||
self.node_config().static_files.to_settings(),
|
||||
)?;
|
||||
init_genesis_with_settings(self.provider_factory(), self.node_config().storage_settings())?;
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
/// Write the genesis block and state if it has not already been written
|
||||
pub fn init_genesis(&self) -> Result<B256, InitStorageError> {
|
||||
init_genesis_with_settings(
|
||||
self.provider_factory(),
|
||||
self.node_config().static_files.to_settings(),
|
||||
)
|
||||
init_genesis_with_settings(self.provider_factory(), self.node_config().storage_settings())
|
||||
}
|
||||
|
||||
/// Creates a new `WithMeteredProvider` container and attaches it to the
|
||||
@@ -1283,6 +1277,10 @@ pub fn metrics_hooks<N: NodeTypesWithDB>(provider_factory: &ProviderFactory<N>)
|
||||
})
|
||||
}
|
||||
})
|
||||
.with_hook({
|
||||
let rocksdb = provider_factory.rocksdb_provider();
|
||||
move || throttle!(Duration::from_secs(5 * 60), || rocksdb.report_metrics())
|
||||
})
|
||||
.build()
|
||||
}
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ use reth_node_core::{
|
||||
use reth_node_events::node;
|
||||
use reth_provider::{
|
||||
providers::{BlockchainProvider, NodeTypesForProvider},
|
||||
BlockNumReader, MetadataProvider,
|
||||
BlockNumReader, StorageSettingsCache,
|
||||
};
|
||||
use reth_tasks::TaskExecutor;
|
||||
use reth_tokio_util::EventSender;
|
||||
@@ -41,7 +41,6 @@ use reth_trie_db::ChangesetCache;
|
||||
use std::{future::Future, pin::Pin, sync::Arc};
|
||||
use tokio::sync::{mpsc::unbounded_channel, oneshot};
|
||||
use tokio_stream::wrappers::UnboundedReceiverStream;
|
||||
use tracing::warn;
|
||||
|
||||
/// The engine node launcher.
|
||||
#[derive(Debug)]
|
||||
@@ -104,24 +103,8 @@ impl EngineNodeLauncher {
|
||||
.with_adjusted_configs()
|
||||
// Create the provider factory with changeset cache
|
||||
.with_provider_factory::<_, <CB::Components as NodeComponents<T>>::Evm>(changeset_cache.clone()).await?
|
||||
.inspect(|ctx| {
|
||||
.inspect(|_| {
|
||||
info!(target: "reth::cli", "Database opened");
|
||||
match ctx.provider_factory().storage_settings() {
|
||||
Ok(settings) => {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
?settings,
|
||||
"Storage settings"
|
||||
);
|
||||
},
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "reth::cli",
|
||||
?err,
|
||||
"Failed to get storage settings"
|
||||
);
|
||||
},
|
||||
}
|
||||
})
|
||||
.with_prometheus_server().await?
|
||||
.inspect(|this| {
|
||||
@@ -130,6 +113,8 @@ impl EngineNodeLauncher {
|
||||
.with_genesis()?
|
||||
.inspect(|this: &LaunchContextWith<Attached<WithConfigs<<T::Types as NodeTypes>::ChainSpec>, _>>| {
|
||||
info!(target: "reth::cli", "\n{}", this.chain_spec().display_hardforks());
|
||||
let settings = this.provider_factory().cached_storage_settings();
|
||||
info!(target: "reth::cli", ?settings, "Loaded storage settings");
|
||||
})
|
||||
.with_metrics_task()
|
||||
// passing FullNodeTypes as type parameter here so that we can build
|
||||
|
||||
@@ -19,7 +19,6 @@ reth-cli-util.workspace = true
|
||||
reth-db = { workspace = true, features = ["mdbx"] }
|
||||
reth-storage-errors.workspace = true
|
||||
reth-storage-api = { workspace = true, features = ["std", "db-api"] }
|
||||
reth-provider.workspace = true
|
||||
reth-network = { workspace = true, features = ["serde"] }
|
||||
reth-network-p2p.workspace = true
|
||||
reth-rpc-eth-types.workspace = true
|
||||
@@ -92,7 +91,7 @@ min-debug-logs = ["tracing/release_max_level_debug"]
|
||||
min-trace-logs = ["tracing/release_max_level_trace"]
|
||||
|
||||
# Marker feature for edge/unstable builds - captured by vergen in build.rs
|
||||
edge = []
|
||||
edge = ["reth-storage-api/edge"]
|
||||
|
||||
[build-dependencies]
|
||||
vergen = { workspace = true, features = ["build", "cargo", "emit_and_set"] }
|
||||
|
||||
@@ -37,6 +37,7 @@ pub struct DefaultEngineValues {
|
||||
storage_worker_count: Option<usize>,
|
||||
account_worker_count: Option<usize>,
|
||||
enable_proof_v2: bool,
|
||||
cache_metrics_disabled: bool,
|
||||
}
|
||||
|
||||
impl DefaultEngineValues {
|
||||
@@ -172,6 +173,12 @@ impl DefaultEngineValues {
|
||||
self.enable_proof_v2 = v;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether to disable cache metrics by default
|
||||
pub const fn with_cache_metrics_disabled(mut self, v: bool) -> Self {
|
||||
self.cache_metrics_disabled = v;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DefaultEngineValues {
|
||||
@@ -197,6 +204,7 @@ impl Default for DefaultEngineValues {
|
||||
storage_worker_count: None,
|
||||
account_worker_count: None,
|
||||
enable_proof_v2: false,
|
||||
cache_metrics_disabled: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -320,6 +328,10 @@ pub struct EngineArgs {
|
||||
/// Enable V2 storage proofs for state root calculations
|
||||
#[arg(long = "engine.enable-proof-v2", default_value_t = DefaultEngineValues::get_global().enable_proof_v2)]
|
||||
pub enable_proof_v2: bool,
|
||||
|
||||
/// Disable cache metrics recording, which can take up to 50ms with large cached state.
|
||||
#[arg(long = "engine.disable-cache-metrics", default_value_t = DefaultEngineValues::get_global().cache_metrics_disabled)]
|
||||
pub cache_metrics_disabled: bool,
|
||||
}
|
||||
|
||||
#[allow(deprecated)]
|
||||
@@ -346,6 +358,7 @@ impl Default for EngineArgs {
|
||||
storage_worker_count,
|
||||
account_worker_count,
|
||||
enable_proof_v2,
|
||||
cache_metrics_disabled,
|
||||
} = DefaultEngineValues::get_global().clone();
|
||||
Self {
|
||||
persistence_threshold,
|
||||
@@ -371,6 +384,7 @@ impl Default for EngineArgs {
|
||||
storage_worker_count,
|
||||
account_worker_count,
|
||||
enable_proof_v2,
|
||||
cache_metrics_disabled,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -407,6 +421,7 @@ impl EngineArgs {
|
||||
}
|
||||
|
||||
config = config.with_enable_proof_v2(self.enable_proof_v2);
|
||||
config = config.without_cache_metrics(self.cache_metrics_disabled);
|
||||
|
||||
config
|
||||
}
|
||||
@@ -458,6 +473,7 @@ mod tests {
|
||||
storage_worker_count: Some(16),
|
||||
account_worker_count: Some(8),
|
||||
enable_proof_v2: false,
|
||||
cache_metrics_disabled: true,
|
||||
};
|
||||
|
||||
let parsed_args = CommandParser::<EngineArgs>::parse_from([
|
||||
@@ -488,6 +504,7 @@ mod tests {
|
||||
"16",
|
||||
"--engine.account-worker-count",
|
||||
"8",
|
||||
"--engine.disable-cache-metrics",
|
||||
])
|
||||
.args;
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ pub use dev::DevArgs;
|
||||
|
||||
/// PruneArgs for configuring the pruning and full node
|
||||
mod pruning;
|
||||
pub use pruning::PruningArgs;
|
||||
pub use pruning::{DefaultPruningValues, PruningArgs};
|
||||
|
||||
/// DatadirArgs for configuring data storage paths
|
||||
mod datadir_args;
|
||||
@@ -80,5 +80,9 @@ pub use era::{DefaultEraHost, EraArgs, EraSourceArgs};
|
||||
mod static_files;
|
||||
pub use static_files::{StaticFilesArgs, MINIMAL_BLOCKS_PER_FILE};
|
||||
|
||||
/// `RocksDbArgs` for configuring RocksDB table routing.
|
||||
mod rocksdb;
|
||||
pub use rocksdb::{RocksDbArgs, RocksDbArgsError};
|
||||
|
||||
mod error;
|
||||
pub mod types;
|
||||
|
||||
@@ -6,7 +6,88 @@ use clap::{builder::RangedU64ValueParser, Args};
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_config::config::PruneConfig;
|
||||
use reth_prune_types::{PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_PRUNING_DISTANCE};
|
||||
use std::{collections::BTreeMap, ops::Not};
|
||||
use std::{collections::BTreeMap, ops::Not, sync::OnceLock};
|
||||
|
||||
/// Global static pruning defaults
|
||||
static PRUNING_DEFAULTS: OnceLock<DefaultPruningValues> = OnceLock::new();
|
||||
|
||||
/// Default values for `--full` and `--minimal` pruning modes that can be customized.
|
||||
///
|
||||
/// Global defaults can be set via [`DefaultPruningValues::try_init`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DefaultPruningValues {
|
||||
/// Prune modes for `--full` flag.
|
||||
///
|
||||
/// Note: `bodies_history` is ignored when `full_bodies_history_use_pre_merge` is `true`.
|
||||
pub full_prune_modes: PruneModes,
|
||||
/// If `true`, `--full` will set `bodies_history` to prune everything before the merge block
|
||||
/// (Paris hardfork). If `false`, uses `full_prune_modes.bodies_history` directly.
|
||||
pub full_bodies_history_use_pre_merge: bool,
|
||||
/// Prune modes for `--minimal` flag.
|
||||
pub minimal_prune_modes: PruneModes,
|
||||
}
|
||||
|
||||
impl DefaultPruningValues {
|
||||
/// Initialize the global pruning defaults with this configuration.
|
||||
///
|
||||
/// Returns `Err(self)` if already initialized.
|
||||
pub fn try_init(self) -> Result<(), Self> {
|
||||
PRUNING_DEFAULTS.set(self)
|
||||
}
|
||||
|
||||
/// Get a reference to the global pruning defaults.
|
||||
pub fn get_global() -> &'static Self {
|
||||
PRUNING_DEFAULTS.get_or_init(Self::default)
|
||||
}
|
||||
|
||||
/// Set the prune modes for `--full` flag.
|
||||
pub fn with_full_prune_modes(mut self, modes: PruneModes) -> Self {
|
||||
self.full_prune_modes = modes;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether `--full` should use pre-merge pruning for bodies history.
|
||||
///
|
||||
/// When `true` (default), bodies are pruned before the Paris hardfork block.
|
||||
/// When `false`, uses `full_prune_modes.bodies_history` directly.
|
||||
pub const fn with_full_bodies_history_use_pre_merge(mut self, use_pre_merge: bool) -> Self {
|
||||
self.full_bodies_history_use_pre_merge = use_pre_merge;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the prune modes for `--minimal` flag.
|
||||
pub fn with_minimal_prune_modes(mut self, modes: PruneModes) -> Self {
|
||||
self.minimal_prune_modes = modes;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DefaultPruningValues {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
full_prune_modes: PruneModes {
|
||||
sender_recovery: Some(PruneMode::Full),
|
||||
transaction_lookup: None,
|
||||
receipts: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
account_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
storage_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
// This field is ignored when full_bodies_history_use_pre_merge is true
|
||||
bodies_history: None,
|
||||
receipts_log_filter: Default::default(),
|
||||
},
|
||||
full_bodies_history_use_pre_merge: true,
|
||||
minimal_prune_modes: PruneModes {
|
||||
sender_recovery: Some(PruneMode::Full),
|
||||
transaction_lookup: Some(PruneMode::Full),
|
||||
receipts: Some(PruneMode::Full),
|
||||
account_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
storage_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
bodies_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
receipts_log_filter: Default::default(),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parameters for pruning and full node
|
||||
#[derive(Debug, Clone, Args, PartialEq, Eq, Default)]
|
||||
@@ -128,36 +209,22 @@ impl PruningArgs {
|
||||
|
||||
// If --full is set, use full node defaults.
|
||||
if self.full {
|
||||
config = PruneConfig {
|
||||
block_interval: config.block_interval,
|
||||
segments: PruneModes {
|
||||
sender_recovery: Some(PruneMode::Full),
|
||||
transaction_lookup: None,
|
||||
receipts: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
account_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
storage_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
bodies_history: chain_spec
|
||||
.ethereum_fork_activation(EthereumHardfork::Paris)
|
||||
.block_number()
|
||||
.map(PruneMode::Before),
|
||||
receipts_log_filter: Default::default(),
|
||||
},
|
||||
let defaults = DefaultPruningValues::get_global();
|
||||
let mut segments = defaults.full_prune_modes.clone();
|
||||
if defaults.full_bodies_history_use_pre_merge {
|
||||
segments.bodies_history = chain_spec
|
||||
.ethereum_fork_activation(EthereumHardfork::Paris)
|
||||
.block_number()
|
||||
.map(PruneMode::Before);
|
||||
}
|
||||
config = PruneConfig { block_interval: config.block_interval, segments }
|
||||
}
|
||||
|
||||
// If --minimal is set, use minimal storage mode with aggressive pruning.
|
||||
if self.minimal {
|
||||
config = PruneConfig {
|
||||
block_interval: config.block_interval,
|
||||
segments: PruneModes {
|
||||
sender_recovery: Some(PruneMode::Full),
|
||||
transaction_lookup: Some(PruneMode::Full),
|
||||
receipts: Some(PruneMode::Full),
|
||||
account_history: Some(PruneMode::Distance(10064)),
|
||||
storage_history: Some(PruneMode::Distance(10064)),
|
||||
bodies_history: Some(PruneMode::Distance(10064)),
|
||||
receipts_log_filter: Default::default(),
|
||||
},
|
||||
segments: DefaultPruningValues::get_global().minimal_prune_modes.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
160
crates/node/core/src/args/rocksdb.rs
Normal file
160
crates/node/core/src/args/rocksdb.rs
Normal file
@@ -0,0 +1,160 @@
|
||||
//! clap [Args](clap::Args) for `RocksDB` table routing configuration
|
||||
|
||||
use clap::{ArgAction, Args};
|
||||
|
||||
/// Default value for `RocksDB` routing flags.
|
||||
///
|
||||
/// When the `edge` feature is enabled, defaults to `true` to enable edge storage features.
|
||||
/// Otherwise defaults to `false` for legacy behavior.
|
||||
const fn default_rocksdb_flag() -> bool {
|
||||
cfg!(feature = "edge")
|
||||
}
|
||||
|
||||
/// Parameters for `RocksDB` table routing configuration.
|
||||
///
|
||||
/// These flags control which database tables are stored in `RocksDB` instead of MDBX.
|
||||
/// All flags are genesis-initialization-only: changing them after genesis requires a re-sync.
|
||||
#[derive(Debug, Args, PartialEq, Eq, Clone, Copy)]
|
||||
#[command(next_help_heading = "RocksDB")]
|
||||
pub struct RocksDbArgs {
|
||||
/// Route all supported tables to `RocksDB` instead of MDBX.
|
||||
///
|
||||
/// This enables `RocksDB` for `tx-hash`, `storages-history`, and `account-history` tables.
|
||||
/// Cannot be combined with individual flags set to false.
|
||||
#[arg(long = "rocksdb.all", action = ArgAction::SetTrue)]
|
||||
pub all: bool,
|
||||
|
||||
/// Route tx hash -> number table to `RocksDB` instead of MDBX.
|
||||
///
|
||||
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
|
||||
/// Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
#[arg(long = "rocksdb.tx-hash", default_value_t = default_rocksdb_flag(), action = ArgAction::Set)]
|
||||
pub tx_hash: bool,
|
||||
|
||||
/// Route storages history tables to `RocksDB` instead of MDBX.
|
||||
///
|
||||
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
|
||||
/// Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
#[arg(long = "rocksdb.storages-history", default_value_t = default_rocksdb_flag(), action = ArgAction::Set)]
|
||||
pub storages_history: bool,
|
||||
|
||||
/// Route account history tables to `RocksDB` instead of MDBX.
|
||||
///
|
||||
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
|
||||
/// Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
#[arg(long = "rocksdb.account-history", default_value_t = default_rocksdb_flag(), action = ArgAction::Set)]
|
||||
pub account_history: bool,
|
||||
}
|
||||
|
||||
impl Default for RocksDbArgs {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
all: false,
|
||||
tx_hash: default_rocksdb_flag(),
|
||||
storages_history: default_rocksdb_flag(),
|
||||
account_history: default_rocksdb_flag(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RocksDbArgs {
|
||||
/// Validates the `RocksDB` arguments.
|
||||
///
|
||||
/// Returns an error if `--rocksdb.all` is used with any individual flag set to `false`.
|
||||
pub const fn validate(&self) -> Result<(), RocksDbArgsError> {
|
||||
if self.all {
|
||||
if !self.tx_hash {
|
||||
return Err(RocksDbArgsError::ConflictingFlags("tx-hash"));
|
||||
}
|
||||
if !self.storages_history {
|
||||
return Err(RocksDbArgsError::ConflictingFlags("storages-history"));
|
||||
}
|
||||
if !self.account_history {
|
||||
return Err(RocksDbArgsError::ConflictingFlags("account-history"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Error type for `RocksDB` argument validation.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum RocksDbArgsError {
|
||||
/// `--rocksdb.all` cannot be combined with an individual flag set to false.
|
||||
#[error("--rocksdb.all cannot be combined with --rocksdb.{0}=false")]
|
||||
ConflictingFlags(&'static str),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use clap::Parser;
|
||||
|
||||
#[derive(Parser)]
|
||||
struct CommandParser<T: Args> {
|
||||
#[command(flatten)]
|
||||
args: T,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_default_rocksdb_args() {
|
||||
let args = CommandParser::<RocksDbArgs>::parse_from(["reth"]).args;
|
||||
assert_eq!(args, RocksDbArgs::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_all_flag() {
|
||||
let args = CommandParser::<RocksDbArgs>::parse_from(["reth", "--rocksdb.all"]).args;
|
||||
assert!(args.all);
|
||||
assert_eq!(args.tx_hash, default_rocksdb_flag());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_individual_flags() {
|
||||
let args = CommandParser::<RocksDbArgs>::parse_from([
|
||||
"reth",
|
||||
"--rocksdb.tx-hash=true",
|
||||
"--rocksdb.storages-history=false",
|
||||
"--rocksdb.account-history=true",
|
||||
])
|
||||
.args;
|
||||
assert!(!args.all);
|
||||
assert!(args.tx_hash);
|
||||
assert!(!args.storages_history);
|
||||
assert!(args.account_history);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_all_with_true_ok() {
|
||||
let args =
|
||||
RocksDbArgs { all: true, tx_hash: true, storages_history: true, account_history: true };
|
||||
assert!(args.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_all_with_false_errors() {
|
||||
let args = RocksDbArgs {
|
||||
all: true,
|
||||
tx_hash: false,
|
||||
storages_history: true,
|
||||
account_history: true,
|
||||
};
|
||||
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("tx-hash")));
|
||||
|
||||
let args = RocksDbArgs {
|
||||
all: true,
|
||||
tx_hash: true,
|
||||
storages_history: false,
|
||||
account_history: true,
|
||||
};
|
||||
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("storages-history")));
|
||||
|
||||
let args = RocksDbArgs {
|
||||
all: true,
|
||||
tx_hash: true,
|
||||
storages_history: true,
|
||||
account_history: false,
|
||||
};
|
||||
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("account-history")));
|
||||
}
|
||||
}
|
||||
@@ -2,15 +2,22 @@
|
||||
|
||||
use clap::Args;
|
||||
use reth_config::config::{BlocksPerFileConfig, StaticFilesConfig};
|
||||
use reth_provider::StorageSettings;
|
||||
|
||||
/// Blocks per static file when running in `--minimal` node.
|
||||
///
|
||||
/// 10000 blocks per static file allows us to prune all history every 10k blocks.
|
||||
pub const MINIMAL_BLOCKS_PER_FILE: u64 = 10000;
|
||||
|
||||
/// Default value for static file storage flags.
|
||||
///
|
||||
/// When the `edge` feature is enabled, defaults to `true` to enable edge storage features.
|
||||
/// Otherwise defaults to `false` for legacy behavior.
|
||||
const fn default_static_file_flag() -> bool {
|
||||
cfg!(feature = "edge")
|
||||
}
|
||||
|
||||
/// Parameters for static files configuration
|
||||
#[derive(Debug, Args, PartialEq, Eq, Default, Clone, Copy)]
|
||||
#[derive(Debug, Args, PartialEq, Eq, Clone, Copy)]
|
||||
#[command(next_help_heading = "Static Files")]
|
||||
pub struct StaticFilesArgs {
|
||||
/// Number of blocks per file for the headers segment.
|
||||
@@ -39,7 +46,7 @@ pub struct StaticFilesArgs {
|
||||
///
|
||||
/// Note: This setting can only be configured at genesis initialization. Once
|
||||
/// the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
#[arg(long = "static-files.receipts")]
|
||||
#[arg(long = "static-files.receipts", default_value_t = default_static_file_flag(), action = clap::ArgAction::Set)]
|
||||
pub receipts: bool,
|
||||
|
||||
/// Store transaction senders in static files instead of the database.
|
||||
@@ -49,7 +56,7 @@ pub struct StaticFilesArgs {
|
||||
///
|
||||
/// Note: This setting can only be configured at genesis initialization. Once
|
||||
/// the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
#[arg(long = "static-files.transaction-senders")]
|
||||
#[arg(long = "static-files.transaction-senders", default_value_t = default_static_file_flag(), action = clap::ArgAction::Set)]
|
||||
pub transaction_senders: bool,
|
||||
|
||||
/// Store account changesets in static files.
|
||||
@@ -59,7 +66,7 @@ pub struct StaticFilesArgs {
|
||||
///
|
||||
/// Note: This setting can only be configured at genesis initialization. Once
|
||||
/// the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
#[arg(long = "static-files.account-change-sets")]
|
||||
#[arg(long = "static-files.account-change-sets", default_value_t = default_static_file_flag(), action = clap::ArgAction::Set)]
|
||||
pub account_changesets: bool,
|
||||
}
|
||||
|
||||
@@ -94,12 +101,19 @@ impl StaticFilesArgs {
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts the static files arguments into [`StorageSettings`].
|
||||
pub const fn to_settings(&self) -> StorageSettings {
|
||||
StorageSettings::legacy()
|
||||
.with_receipts_in_static_files(self.receipts)
|
||||
.with_transaction_senders_in_static_files(self.transaction_senders)
|
||||
.with_account_changesets_in_static_files(self.account_changesets)
|
||||
impl Default for StaticFilesArgs {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
blocks_per_file_headers: None,
|
||||
blocks_per_file_transactions: None,
|
||||
blocks_per_file_receipts: None,
|
||||
blocks_per_file_transaction_senders: None,
|
||||
blocks_per_file_account_change_sets: None,
|
||||
receipts: default_static_file_flag(),
|
||||
transaction_senders: default_static_file_flag(),
|
||||
account_changesets: default_static_file_flag(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use crate::{
|
||||
args::{
|
||||
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, NetworkArgs, PayloadBuilderArgs,
|
||||
PruningArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
|
||||
PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
|
||||
},
|
||||
dirs::{ChainPath, DataDirPath},
|
||||
utils::get_single_header,
|
||||
@@ -21,6 +21,7 @@ use reth_primitives_traits::SealedHeader;
|
||||
use reth_stages_types::StageId;
|
||||
use reth_storage_api::{
|
||||
BlockHashReader, DatabaseProviderFactory, HeaderProvider, StageCheckpointReader,
|
||||
StorageSettings,
|
||||
};
|
||||
use reth_storage_errors::provider::ProviderResult;
|
||||
use reth_transaction_pool::TransactionPool;
|
||||
@@ -150,6 +151,9 @@ pub struct NodeConfig<ChainSpec> {
|
||||
|
||||
/// All static files related arguments
|
||||
pub static_files: StaticFilesArgs,
|
||||
|
||||
/// All `RocksDB` table routing arguments
|
||||
pub rocksdb: RocksDbArgs,
|
||||
}
|
||||
|
||||
impl NodeConfig<ChainSpec> {
|
||||
@@ -181,6 +185,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
engine: EngineArgs::default(),
|
||||
era: EraArgs::default(),
|
||||
static_files: StaticFilesArgs::default(),
|
||||
rocksdb: RocksDbArgs::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -255,6 +260,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
rocksdb,
|
||||
..
|
||||
} = self;
|
||||
NodeConfig {
|
||||
@@ -274,6 +280,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
rocksdb,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -350,6 +357,17 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
self.pruning.prune_config(&self.chain)
|
||||
}
|
||||
|
||||
/// Returns the effective storage settings derived from static-file and `RocksDB` CLI args.
|
||||
pub const fn storage_settings(&self) -> StorageSettings {
|
||||
StorageSettings::base()
|
||||
.with_receipts_in_static_files(self.static_files.receipts)
|
||||
.with_transaction_senders_in_static_files(self.static_files.transaction_senders)
|
||||
.with_account_changesets_in_static_files(self.static_files.account_changesets)
|
||||
.with_transaction_hash_numbers_in_rocksdb(self.rocksdb.all || self.rocksdb.tx_hash)
|
||||
.with_storages_history_in_rocksdb(self.rocksdb.all || self.rocksdb.storages_history)
|
||||
.with_account_history_in_rocksdb(self.rocksdb.all || self.rocksdb.account_history)
|
||||
}
|
||||
|
||||
/// Returns the max block that the node should run to, looking it up from the network if
|
||||
/// necessary
|
||||
pub async fn max_block<Provider, Client>(
|
||||
@@ -544,6 +562,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
engine: self.engine,
|
||||
era: self.era,
|
||||
static_files: self.static_files,
|
||||
rocksdb: self.rocksdb,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -585,6 +604,7 @@ impl<ChainSpec> Clone for NodeConfig<ChainSpec> {
|
||||
engine: self.engine.clone(),
|
||||
era: self.era.clone(),
|
||||
static_files: self.static_files,
|
||||
rocksdb: self.rocksdb,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,6 +106,7 @@ impl MetricServer {
|
||||
// Describe metrics after recorder installation
|
||||
describe_db_metrics();
|
||||
describe_static_file_metrics();
|
||||
describe_rocksdb_metrics();
|
||||
Collector::default().describe();
|
||||
describe_memory_stats();
|
||||
describe_io_stats();
|
||||
@@ -238,6 +239,26 @@ fn describe_static_file_metrics() {
|
||||
);
|
||||
}
|
||||
|
||||
fn describe_rocksdb_metrics() {
|
||||
describe_gauge!(
|
||||
"rocksdb.table_size",
|
||||
Unit::Bytes,
|
||||
"The estimated size of a RocksDB table (SST + memtable)"
|
||||
);
|
||||
describe_gauge!("rocksdb.table_entries", "The estimated number of keys in a RocksDB table");
|
||||
describe_gauge!(
|
||||
"rocksdb.pending_compaction_bytes",
|
||||
Unit::Bytes,
|
||||
"Bytes pending compaction for a RocksDB table"
|
||||
);
|
||||
describe_gauge!("rocksdb.sst_size", Unit::Bytes, "The size of SST files for a RocksDB table");
|
||||
describe_gauge!(
|
||||
"rocksdb.memtable_size",
|
||||
Unit::Bytes,
|
||||
"The size of memtables for a RocksDB table"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "jemalloc", unix))]
|
||||
fn describe_memory_stats() {
|
||||
describe_gauge!(
|
||||
|
||||
@@ -510,27 +510,12 @@ mod tests {
|
||||
fn test_sealed_block_rlp_roundtrip() {
|
||||
// Create a sample block using alloy_consensus::Block
|
||||
let header = alloy_consensus::Header {
|
||||
parent_hash: B256::ZERO,
|
||||
ommers_hash: B256::ZERO,
|
||||
beneficiary: Address::ZERO,
|
||||
state_root: B256::ZERO,
|
||||
transactions_root: B256::ZERO,
|
||||
receipts_root: B256::ZERO,
|
||||
logs_bloom: Default::default(),
|
||||
difficulty: Default::default(),
|
||||
number: 42,
|
||||
gas_limit: 30_000_000,
|
||||
gas_used: 21_000,
|
||||
timestamp: 1_000_000,
|
||||
extra_data: Default::default(),
|
||||
mix_hash: B256::ZERO,
|
||||
nonce: Default::default(),
|
||||
base_fee_per_gas: Some(1_000_000_000),
|
||||
withdrawals_root: None,
|
||||
blob_gas_used: None,
|
||||
excess_blob_gas: None,
|
||||
parent_beacon_block_root: None,
|
||||
requests_hash: None,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Create a simple transaction
|
||||
@@ -585,27 +570,12 @@ mod tests {
|
||||
fn test_decode_sealed_produces_correct_hash() {
|
||||
// Create a sample block using alloy_consensus::Block
|
||||
let header = alloy_consensus::Header {
|
||||
parent_hash: B256::ZERO,
|
||||
ommers_hash: B256::ZERO,
|
||||
beneficiary: Address::ZERO,
|
||||
state_root: B256::ZERO,
|
||||
transactions_root: B256::ZERO,
|
||||
receipts_root: B256::ZERO,
|
||||
logs_bloom: Default::default(),
|
||||
difficulty: Default::default(),
|
||||
number: 42,
|
||||
gas_limit: 30_000_000,
|
||||
gas_used: 21_000,
|
||||
timestamp: 1_000_000,
|
||||
extra_data: Default::default(),
|
||||
mix_hash: B256::ZERO,
|
||||
nonce: Default::default(),
|
||||
base_fee_per_gas: Some(1_000_000_000),
|
||||
withdrawals_root: None,
|
||||
blob_gas_used: None,
|
||||
excess_blob_gas: None,
|
||||
parent_beacon_block_root: None,
|
||||
requests_hash: None,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Create a simple transaction
|
||||
|
||||
@@ -84,7 +84,14 @@ where
|
||||
.into_inner();
|
||||
let tx_range = start..=
|
||||
Some(end)
|
||||
.min(input.limiter.deleted_entries_limit_left().map(|left| start + left as u64 - 1))
|
||||
.min(
|
||||
input
|
||||
.limiter
|
||||
.deleted_entries_limit_left()
|
||||
// Use saturating addition here to avoid panicking on
|
||||
// `deleted_entries_limit == usize::MAX`
|
||||
.map(|left| start.saturating_add(left as u64) - 1),
|
||||
)
|
||||
.unwrap();
|
||||
let tx_range_end = *tx_range.end();
|
||||
|
||||
|
||||
@@ -252,6 +252,18 @@ pub trait EngineApi<Engine: EngineTypes> {
|
||||
&self,
|
||||
versioned_hashes: Vec<B256>,
|
||||
) -> RpcResult<Option<Vec<Option<BlobAndProofV2>>>>;
|
||||
|
||||
/// Returns the Block Access Lists for the given block hashes.
|
||||
///
|
||||
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
|
||||
#[method(name = "getBALsByHashV1")]
|
||||
async fn get_bals_by_hash_v1(&self, block_hashes: Vec<BlockHash>) -> RpcResult<Vec<Bytes>>;
|
||||
|
||||
/// Returns the Block Access Lists for the given block range.
|
||||
///
|
||||
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
|
||||
#[method(name = "getBALsByRangeV1")]
|
||||
async fn get_bals_by_range_v1(&self, start: U64, count: U64) -> RpcResult<Vec<Bytes>>;
|
||||
}
|
||||
|
||||
/// A subset of the ETH rpc interface: <https://ethereum.github.io/execution-apis/api-documentation>
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
use std::collections::HashSet;
|
||||
//! Engine API capabilities.
|
||||
|
||||
/// The list of all supported Engine capabilities available over the engine endpoint.
|
||||
use std::collections::HashSet;
|
||||
use tracing::warn;
|
||||
|
||||
/// All Engine API capabilities supported by Reth (Ethereum mainnet).
|
||||
///
|
||||
/// See <https://github.com/ethereum/execution-apis/tree/main/src/engine> for updates.
|
||||
pub const CAPABILITIES: &[&str] = &[
|
||||
"engine_forkchoiceUpdatedV1",
|
||||
"engine_forkchoiceUpdatedV2",
|
||||
@@ -22,43 +27,150 @@ pub const CAPABILITIES: &[&str] = &[
|
||||
"engine_getBlobsV3",
|
||||
];
|
||||
|
||||
// The list of all supported Engine capabilities available over the engine endpoint.
|
||||
///
|
||||
/// Latest spec: Prague
|
||||
/// Engine API capabilities set.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EngineCapabilities {
|
||||
inner: HashSet<String>,
|
||||
}
|
||||
|
||||
impl EngineCapabilities {
|
||||
/// Creates a new `EngineCapabilities` instance with the given capabilities.
|
||||
pub fn new(capabilities: impl IntoIterator<Item: Into<String>>) -> Self {
|
||||
/// Creates from an iterator of capability strings.
|
||||
pub fn new(capabilities: impl IntoIterator<Item = impl Into<String>>) -> Self {
|
||||
Self { inner: capabilities.into_iter().map(Into::into).collect() }
|
||||
}
|
||||
|
||||
/// Returns the list of all supported Engine capabilities for Prague spec.
|
||||
fn prague() -> Self {
|
||||
Self { inner: CAPABILITIES.iter().copied().map(str::to_owned).collect() }
|
||||
}
|
||||
|
||||
/// Returns the list of all supported Engine capabilities.
|
||||
/// Returns the capabilities as a list of strings.
|
||||
pub fn list(&self) -> Vec<String> {
|
||||
self.inner.iter().cloned().collect()
|
||||
}
|
||||
|
||||
/// Inserts a new capability.
|
||||
pub fn add_capability(&mut self, capability: impl Into<String>) {
|
||||
self.inner.insert(capability.into());
|
||||
/// Returns a reference to the inner set.
|
||||
pub const fn as_set(&self) -> &HashSet<String> {
|
||||
&self.inner
|
||||
}
|
||||
|
||||
/// Removes a capability.
|
||||
pub fn remove_capability(&mut self, capability: &str) -> Option<String> {
|
||||
self.inner.take(capability)
|
||||
/// Compares CL capabilities with this EL's capabilities and returns any mismatches.
|
||||
///
|
||||
/// Called during `engine_exchangeCapabilities` to detect version mismatches
|
||||
/// between the consensus layer and execution layer.
|
||||
pub fn get_capability_mismatches(&self, cl_capabilities: &[String]) -> CapabilityMismatches {
|
||||
let cl_set: HashSet<&str> = cl_capabilities.iter().map(String::as_str).collect();
|
||||
|
||||
// CL has methods EL doesn't support
|
||||
let mut missing_in_el: Vec<_> = cl_capabilities
|
||||
.iter()
|
||||
.filter(|cap| !self.inner.contains(cap.as_str()))
|
||||
.cloned()
|
||||
.collect();
|
||||
missing_in_el.sort();
|
||||
|
||||
// EL has methods CL doesn't support
|
||||
let mut missing_in_cl: Vec<_> =
|
||||
self.inner.iter().filter(|cap| !cl_set.contains(cap.as_str())).cloned().collect();
|
||||
missing_in_cl.sort();
|
||||
|
||||
CapabilityMismatches { missing_in_el, missing_in_cl }
|
||||
}
|
||||
|
||||
/// Logs warnings if CL and EL capabilities don't match.
|
||||
///
|
||||
/// Called during `engine_exchangeCapabilities` to warn operators about
|
||||
/// version mismatches between the consensus layer and execution layer.
|
||||
pub fn log_capability_mismatches(&self, cl_capabilities: &[String]) {
|
||||
let mismatches = self.get_capability_mismatches(cl_capabilities);
|
||||
|
||||
if !mismatches.missing_in_el.is_empty() {
|
||||
warn!(
|
||||
target: "rpc::engine",
|
||||
missing = ?mismatches.missing_in_el,
|
||||
"CL supports Engine API methods that Reth doesn't. Consider upgrading Reth."
|
||||
);
|
||||
}
|
||||
|
||||
if !mismatches.missing_in_cl.is_empty() {
|
||||
warn!(
|
||||
target: "rpc::engine",
|
||||
missing = ?mismatches.missing_in_cl,
|
||||
"Reth supports Engine API methods that CL doesn't. Consider upgrading your consensus client."
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for EngineCapabilities {
|
||||
fn default() -> Self {
|
||||
Self::prague()
|
||||
Self::new(CAPABILITIES.iter().copied())
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of comparing CL and EL capabilities.
|
||||
#[derive(Debug, Default, PartialEq, Eq)]
|
||||
pub struct CapabilityMismatches {
|
||||
/// Methods supported by CL but not by EL (Reth).
|
||||
/// Operators should consider upgrading Reth.
|
||||
pub missing_in_el: Vec<String>,
|
||||
/// Methods supported by EL (Reth) but not by CL.
|
||||
/// Operators should consider upgrading their consensus client.
|
||||
pub missing_in_cl: Vec<String>,
|
||||
}
|
||||
|
||||
impl CapabilityMismatches {
|
||||
/// Returns `true` if there are no mismatches.
|
||||
pub const fn is_empty(&self) -> bool {
|
||||
self.missing_in_el.is_empty() && self.missing_in_cl.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_no_mismatches() {
|
||||
let el = EngineCapabilities::new(["method_a", "method_b"]);
|
||||
let cl = vec!["method_a".to_string(), "method_b".to_string()];
|
||||
|
||||
let result = el.get_capability_mismatches(&cl);
|
||||
assert!(result.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cl_has_extra_methods() {
|
||||
let el = EngineCapabilities::new(["method_a"]);
|
||||
let cl = vec!["method_a".to_string(), "method_b".to_string()];
|
||||
|
||||
let result = el.get_capability_mismatches(&cl);
|
||||
assert_eq!(result.missing_in_el, vec!["method_b"]);
|
||||
assert!(result.missing_in_cl.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_el_has_extra_methods() {
|
||||
let el = EngineCapabilities::new(["method_a", "method_b"]);
|
||||
let cl = vec!["method_a".to_string()];
|
||||
|
||||
let result = el.get_capability_mismatches(&cl);
|
||||
assert!(result.missing_in_el.is_empty());
|
||||
assert_eq!(result.missing_in_cl, vec!["method_b"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_both_have_extra_methods() {
|
||||
let el = EngineCapabilities::new(["method_a", "method_c"]);
|
||||
let cl = vec!["method_a".to_string(), "method_b".to_string()];
|
||||
|
||||
let result = el.get_capability_mismatches(&cl);
|
||||
assert_eq!(result.missing_in_el, vec!["method_b"]);
|
||||
assert_eq!(result.missing_in_cl, vec!["method_c"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_results_are_sorted() {
|
||||
let el = EngineCapabilities::new(["z_method", "a_method"]);
|
||||
let cl = vec!["z_other".to_string(), "a_other".to_string()];
|
||||
|
||||
let result = el.get_capability_mismatches(&cl);
|
||||
assert_eq!(result.missing_in_el, vec!["a_other", "z_other"]);
|
||||
assert_eq!(result.missing_in_cl, vec!["a_method", "z_method"]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1134,8 +1134,13 @@ where
|
||||
|
||||
/// Handler for `engine_exchangeCapabilitiesV1`
|
||||
/// See also <https://github.com/ethereum/execution-apis/blob/6452a6b194d7db269bf1dbd087a267251d3cc7f8/src/engine/common.md#capabilities>
|
||||
async fn exchange_capabilities(&self, _capabilities: Vec<String>) -> RpcResult<Vec<String>> {
|
||||
Ok(self.capabilities().list())
|
||||
async fn exchange_capabilities(&self, capabilities: Vec<String>) -> RpcResult<Vec<String>> {
|
||||
trace!(target: "rpc::engine", "Serving engine_exchangeCapabilities");
|
||||
|
||||
let el_caps = self.capabilities();
|
||||
el_caps.log_capability_mismatches(&capabilities);
|
||||
|
||||
Ok(el_caps.list())
|
||||
}
|
||||
|
||||
async fn get_blobs_v1(
|
||||
@@ -1161,6 +1166,33 @@ where
|
||||
trace!(target: "rpc::engine", "Serving engine_getBlobsV3");
|
||||
Ok(self.get_blobs_v3_metered(versioned_hashes)?)
|
||||
}
|
||||
|
||||
/// Handler for `engine_getBALsByHashV1`
|
||||
///
|
||||
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
|
||||
async fn get_bals_by_hash_v1(
|
||||
&self,
|
||||
_block_hashes: Vec<BlockHash>,
|
||||
) -> RpcResult<Vec<alloy_primitives::Bytes>> {
|
||||
trace!(target: "rpc::engine", "Serving engine_getBALsByHashV1");
|
||||
Err(EngineApiError::EngineObjectValidationError(
|
||||
reth_payload_primitives::EngineObjectValidationError::UnsupportedFork,
|
||||
))?
|
||||
}
|
||||
|
||||
/// Handler for `engine_getBALsByRangeV1`
|
||||
///
|
||||
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
|
||||
async fn get_bals_by_range_v1(
|
||||
&self,
|
||||
_start: U64,
|
||||
_count: U64,
|
||||
) -> RpcResult<Vec<alloy_primitives::Bytes>> {
|
||||
trace!(target: "rpc::engine", "Serving engine_getBALsByRangeV1");
|
||||
Err(EngineApiError::EngineObjectValidationError(
|
||||
reth_payload_primitives::EngineObjectValidationError::UnsupportedFork,
|
||||
))?
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider, EngineT, Pool, Validator, ChainSpec> IntoEngineApiRpcModule
|
||||
|
||||
@@ -507,6 +507,7 @@ impl From<reth_errors::ProviderError> for EthApiError {
|
||||
ProviderError::BlockNumberForTransactionIndexNotFound => Self::UnknownBlockOrTxIndex,
|
||||
ProviderError::FinalizedBlockNotFound => Self::HeaderNotFound(BlockId::finalized()),
|
||||
ProviderError::SafeBlockNotFound => Self::HeaderNotFound(BlockId::safe()),
|
||||
ProviderError::BlockExpired { .. } => Self::PrunedHistoryUnavailable,
|
||||
err => Self::Internal(err.into()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,6 +86,14 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Clear all ETL state. Called on error paths to prevent buffer pollution on retry.
|
||||
fn clear_etl_state(&mut self) {
|
||||
self.sync_gap = None;
|
||||
self.hash_collector.clear();
|
||||
self.header_collector.clear();
|
||||
self.is_etl_ready = false;
|
||||
}
|
||||
|
||||
/// Write downloaded headers to storage from ETL.
|
||||
///
|
||||
/// Writes to static files ( `Header | HeaderTD | HeaderHash` ) and [`tables::HeaderNumbers`]
|
||||
@@ -258,7 +266,7 @@ where
|
||||
}
|
||||
Some(Err(HeadersDownloaderError::DetachedHead { local_head, header, error })) => {
|
||||
error!(target: "sync::stages::headers", %error, "Cannot attach header to head");
|
||||
self.sync_gap = None;
|
||||
self.clear_etl_state();
|
||||
return Poll::Ready(Err(StageError::DetachedHead {
|
||||
local_head: Box::new(local_head.block_with_parent()),
|
||||
header: Box::new(header.block_with_parent()),
|
||||
@@ -266,7 +274,7 @@ where
|
||||
}))
|
||||
}
|
||||
None => {
|
||||
self.sync_gap = None;
|
||||
self.clear_etl_state();
|
||||
return Poll::Ready(Err(StageError::ChannelClosed))
|
||||
}
|
||||
}
|
||||
@@ -324,7 +332,7 @@ where
|
||||
provider: &Provider,
|
||||
input: UnwindInput,
|
||||
) -> Result<UnwindOutput, StageError> {
|
||||
self.sync_gap.take();
|
||||
self.clear_etl_state();
|
||||
|
||||
// First unwind the db tables, until the unwind_to block number. use the walker to unwind
|
||||
// HeaderNumbers based on the index in CanonicalHeaders
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
use crate::stages::utils::collect_history_indices;
|
||||
|
||||
use super::{collect_account_history_indices, load_history_indices};
|
||||
use alloy_primitives::Address;
|
||||
use super::collect_account_history_indices;
|
||||
use crate::stages::utils::{collect_history_indices, load_account_history};
|
||||
use reth_config::config::{EtlConfig, IndexHistoryConfig};
|
||||
use reth_db_api::{models::ShardedKey, table::Decode, tables, transaction::DbTxMut};
|
||||
use reth_db_api::{models::ShardedKey, tables, transaction::DbTxMut};
|
||||
use reth_provider::{
|
||||
DBProvider, HistoryWriter, PruneCheckpointReader, PruneCheckpointWriter, StorageSettingsCache,
|
||||
DBProvider, EitherWriter, HistoryWriter, PruneCheckpointReader, PruneCheckpointWriter,
|
||||
RocksDBProviderFactory, StorageSettingsCache,
|
||||
};
|
||||
use reth_prune_types::{PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment};
|
||||
use reth_stages_api::{
|
||||
@@ -53,7 +52,8 @@ where
|
||||
+ PruneCheckpointWriter
|
||||
+ reth_storage_api::ChangeSetReader
|
||||
+ reth_provider::StaticFileProviderFactory
|
||||
+ StorageSettingsCache,
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory,
|
||||
{
|
||||
/// Return the id of the stage
|
||||
fn id(&self) -> StageId {
|
||||
@@ -101,15 +101,25 @@ where
|
||||
|
||||
let mut range = input.next_block_range();
|
||||
let first_sync = input.checkpoint().block_number == 0;
|
||||
let use_rocksdb = provider.cached_storage_settings().account_history_in_rocksdb;
|
||||
|
||||
// On first sync we might have history coming from genesis. We clear the table since it's
|
||||
// faster to rebuild from scratch.
|
||||
if first_sync {
|
||||
provider.tx_ref().clear::<tables::AccountsHistory>()?;
|
||||
if use_rocksdb {
|
||||
// Note: RocksDB clear() executes immediately (not deferred to commit like MDBX),
|
||||
// but this is safe for first_sync because if we crash before commit, the
|
||||
// checkpoint stays at 0 and we'll just clear and rebuild again on restart. The
|
||||
// source data (changesets) is intact.
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
provider.rocksdb_provider().clear::<tables::AccountsHistory>()?;
|
||||
} else {
|
||||
provider.tx_ref().clear::<tables::AccountsHistory>()?;
|
||||
}
|
||||
range = 0..=*input.next_block_range().end();
|
||||
}
|
||||
|
||||
info!(target: "sync::stages::index_account_history::exec", ?first_sync, "Collecting indices");
|
||||
info!(target: "sync::stages::index_account_history::exec", ?first_sync, ?use_rocksdb, "Collecting indices");
|
||||
|
||||
let collector = if provider.cached_storage_settings().account_changesets_in_static_files {
|
||||
// Use the provider-based collection that can read from static files.
|
||||
@@ -125,14 +135,13 @@ where
|
||||
};
|
||||
|
||||
info!(target: "sync::stages::index_account_history::exec", "Loading indices into database");
|
||||
load_history_indices::<_, tables::AccountsHistory, _>(
|
||||
provider,
|
||||
collector,
|
||||
first_sync,
|
||||
ShardedKey::new,
|
||||
ShardedKey::<Address>::decode_owned,
|
||||
|key| key.key,
|
||||
)?;
|
||||
|
||||
provider.with_rocksdb_batch(|rocksdb_batch| {
|
||||
let mut writer = EitherWriter::new_accounts_history(provider, rocksdb_batch)?;
|
||||
load_account_history(collector, first_sync, &mut writer)
|
||||
.map_err(|e| reth_provider::ProviderError::other(Box::new(e)))?;
|
||||
Ok(((), writer.into_raw_rocksdb_batch()))
|
||||
})?;
|
||||
|
||||
Ok(ExecOutput { checkpoint: StageCheckpoint::new(*range.end()), done: true })
|
||||
}
|
||||
@@ -160,7 +169,7 @@ mod tests {
|
||||
stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, TestRunnerError,
|
||||
TestStageDB, UnwindStageTestRunner,
|
||||
};
|
||||
use alloy_primitives::{address, BlockNumber, B256};
|
||||
use alloy_primitives::{address, Address, BlockNumber, B256};
|
||||
use itertools::Itertools;
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO,
|
||||
@@ -646,4 +655,169 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
mod rocksdb_tests {
|
||||
use super::*;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use reth_storage_api::StorageSettings;
|
||||
|
||||
/// Test that when `account_history_in_rocksdb` is enabled, the stage
|
||||
/// writes account history indices to `RocksDB` instead of MDBX.
|
||||
#[tokio::test]
|
||||
async fn execute_writes_to_rocksdb_when_enabled() {
|
||||
// init
|
||||
let db = TestStageDB::default();
|
||||
|
||||
// Enable RocksDB for account history
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_account_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), ..Default::default() };
|
||||
let mut stage = IndexAccountHistoryStage::default();
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
// Verify MDBX table is empty (data should be in RocksDB)
|
||||
let mdbx_table = db.table::<tables::AccountsHistory>().unwrap();
|
||||
assert!(
|
||||
mdbx_table.is_empty(),
|
||||
"MDBX AccountsHistory should be empty when RocksDB is enabled"
|
||||
);
|
||||
|
||||
// Verify RocksDB has the data
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should contain account history");
|
||||
|
||||
let block_list = result.unwrap();
|
||||
let blocks: Vec<u64> = block_list.iter().collect();
|
||||
assert_eq!(blocks, (0..=10).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
/// Test that unwind works correctly when `account_history_in_rocksdb` is enabled.
|
||||
#[tokio::test]
|
||||
async fn unwind_works_when_rocksdb_enabled() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_account_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), ..Default::default() };
|
||||
let mut stage = IndexAccountHistoryStage::default();
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
// Verify RocksDB has blocks 0-10 before unwind
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should have data before unwind");
|
||||
let blocks_before: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks_before, (0..=10).collect::<Vec<_>>());
|
||||
|
||||
// Unwind to block 5 (remove blocks 6-10)
|
||||
let unwind_input =
|
||||
UnwindInput { checkpoint: StageCheckpoint::new(10), unwind_to: 5, bad_block: None };
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.unwind(&provider, unwind_input).unwrap();
|
||||
assert_eq!(out, UnwindOutput { checkpoint: StageCheckpoint::new(5) });
|
||||
provider.commit().unwrap();
|
||||
|
||||
// Verify RocksDB now only has blocks 0-5 (blocks 6-10 removed)
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should still have data after unwind");
|
||||
let blocks_after: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks_after, (0..=5).collect::<Vec<_>>(), "Should only have blocks 0-5");
|
||||
}
|
||||
|
||||
/// Test incremental sync merges new data with existing shards.
|
||||
#[tokio::test]
|
||||
async fn execute_incremental_sync() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_account_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=5 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(5), ..Default::default() };
|
||||
let mut stage = IndexAccountHistoryStage::default();
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(5), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some());
|
||||
let blocks: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks, (0..=5).collect::<Vec<_>>());
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 6..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), checkpoint: Some(StageCheckpoint::new(5)) };
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should have merged data");
|
||||
let blocks: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks, (0..=10).collect::<Vec<_>>());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,19 +1,21 @@
|
||||
use super::{collect_history_indices, load_history_indices};
|
||||
use crate::{StageCheckpoint, StageId};
|
||||
use super::collect_history_indices;
|
||||
use crate::{stages::utils::load_storage_history, StageCheckpoint, StageId};
|
||||
use reth_config::config::{EtlConfig, IndexHistoryConfig};
|
||||
use reth_db_api::{
|
||||
models::{storage_sharded_key::StorageShardedKey, AddressStorageKey, BlockNumberAddress},
|
||||
table::Decode,
|
||||
tables,
|
||||
transaction::DbTxMut,
|
||||
};
|
||||
use reth_provider::{DBProvider, HistoryWriter, PruneCheckpointReader, PruneCheckpointWriter};
|
||||
use reth_provider::{
|
||||
DBProvider, EitherWriter, HistoryWriter, PruneCheckpointReader, PruneCheckpointWriter,
|
||||
RocksDBProviderFactory, StorageSettingsCache,
|
||||
};
|
||||
use reth_prune_types::{PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment};
|
||||
use reth_stages_api::{ExecInput, ExecOutput, Stage, StageError, UnwindInput, UnwindOutput};
|
||||
use std::fmt::Debug;
|
||||
use tracing::info;
|
||||
|
||||
/// Stage is indexing history the account changesets generated in
|
||||
/// Stage is indexing history the storage changesets generated in
|
||||
/// [`ExecutionStage`][crate::stages::ExecutionStage]. For more information
|
||||
/// on index sharding take a look at [`tables::StoragesHistory`].
|
||||
#[derive(Debug)]
|
||||
@@ -34,7 +36,7 @@ impl IndexStorageHistoryStage {
|
||||
etl_config: EtlConfig,
|
||||
prune_mode: Option<PruneMode>,
|
||||
) -> Self {
|
||||
Self { commit_threshold: config.commit_threshold, prune_mode, etl_config }
|
||||
Self { commit_threshold: config.commit_threshold, etl_config, prune_mode }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,8 +48,13 @@ impl Default for IndexStorageHistoryStage {
|
||||
|
||||
impl<Provider> Stage<Provider> for IndexStorageHistoryStage
|
||||
where
|
||||
Provider:
|
||||
DBProvider<Tx: DbTxMut> + PruneCheckpointWriter + HistoryWriter + PruneCheckpointReader,
|
||||
Provider: DBProvider<Tx: DbTxMut>
|
||||
+ HistoryWriter
|
||||
+ PruneCheckpointReader
|
||||
+ PruneCheckpointWriter
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory
|
||||
+ reth_provider::NodePrimitivesProvider,
|
||||
{
|
||||
/// Return the id of the stage
|
||||
fn id(&self) -> StageId {
|
||||
@@ -95,15 +102,25 @@ where
|
||||
|
||||
let mut range = input.next_block_range();
|
||||
let first_sync = input.checkpoint().block_number == 0;
|
||||
let use_rocksdb = provider.cached_storage_settings().storages_history_in_rocksdb;
|
||||
|
||||
// On first sync we might have history coming from genesis. We clear the table since it's
|
||||
// faster to rebuild from scratch.
|
||||
if first_sync {
|
||||
provider.tx_ref().clear::<tables::StoragesHistory>()?;
|
||||
if use_rocksdb {
|
||||
// Note: RocksDB clear() executes immediately (not deferred to commit like MDBX),
|
||||
// but this is safe for first_sync because if we crash before commit, the
|
||||
// checkpoint stays at 0 and we'll just clear and rebuild again on restart. The
|
||||
// source data (changesets) is intact.
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
provider.rocksdb_provider().clear::<tables::StoragesHistory>()?;
|
||||
} else {
|
||||
provider.tx_ref().clear::<tables::StoragesHistory>()?;
|
||||
}
|
||||
range = 0..=*input.next_block_range().end();
|
||||
}
|
||||
|
||||
info!(target: "sync::stages::index_storage_history::exec", ?first_sync, "Collecting indices");
|
||||
info!(target: "sync::stages::index_storage_history::exec", ?first_sync, ?use_rocksdb, "Collecting indices");
|
||||
let collector =
|
||||
collect_history_indices::<_, tables::StorageChangeSets, tables::StoragesHistory, _>(
|
||||
provider,
|
||||
@@ -116,16 +133,13 @@ where
|
||||
)?;
|
||||
|
||||
info!(target: "sync::stages::index_storage_history::exec", "Loading indices into database");
|
||||
load_history_indices::<_, tables::StoragesHistory, _>(
|
||||
provider,
|
||||
collector,
|
||||
first_sync,
|
||||
|AddressStorageKey((address, storage_key)), highest_block_number| {
|
||||
StorageShardedKey::new(address, storage_key, highest_block_number)
|
||||
},
|
||||
StorageShardedKey::decode_owned,
|
||||
|key| AddressStorageKey((key.address, key.sharded_key.key)),
|
||||
)?;
|
||||
|
||||
provider.with_rocksdb_batch(|rocksdb_batch| {
|
||||
let mut writer = EitherWriter::new_storages_history(provider, rocksdb_batch)?;
|
||||
load_storage_history(collector, first_sync, &mut writer)
|
||||
.map_err(|e| reth_provider::ProviderError::other(Box::new(e)))?;
|
||||
Ok(((), writer.into_raw_rocksdb_batch()))
|
||||
})?;
|
||||
|
||||
Ok(ExecOutput { checkpoint: StageCheckpoint::new(*range.end()), done: true })
|
||||
}
|
||||
@@ -382,12 +396,12 @@ mod tests {
|
||||
async fn insert_index_second_half_shard() {
|
||||
// init
|
||||
let db = TestStageDB::default();
|
||||
let mut close_full_list = (1..=LAST_BLOCK_IN_FULL_SHARD - 1).collect::<Vec<_>>();
|
||||
let mut almost_full_list = (1..=LAST_BLOCK_IN_FULL_SHARD - 1).collect::<Vec<_>>();
|
||||
|
||||
// setup
|
||||
partial_setup(&db);
|
||||
db.commit(|tx| {
|
||||
tx.put::<tables::StoragesHistory>(shard(u64::MAX), list(&close_full_list)).unwrap();
|
||||
tx.put::<tables::StoragesHistory>(shard(u64::MAX), list(&almost_full_list)).unwrap();
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
@@ -396,12 +410,12 @@ mod tests {
|
||||
run(&db, LAST_BLOCK_IN_FULL_SHARD + 1, Some(LAST_BLOCK_IN_FULL_SHARD - 1));
|
||||
|
||||
// verify
|
||||
close_full_list.push(LAST_BLOCK_IN_FULL_SHARD);
|
||||
almost_full_list.push(LAST_BLOCK_IN_FULL_SHARD);
|
||||
let table = cast(db.table::<tables::StoragesHistory>().unwrap());
|
||||
assert_eq!(
|
||||
table,
|
||||
BTreeMap::from([
|
||||
(shard(LAST_BLOCK_IN_FULL_SHARD), close_full_list.clone()),
|
||||
(shard(LAST_BLOCK_IN_FULL_SHARD), almost_full_list.clone()),
|
||||
(shard(u64::MAX), vec![LAST_BLOCK_IN_FULL_SHARD + 1])
|
||||
])
|
||||
);
|
||||
@@ -410,9 +424,9 @@ mod tests {
|
||||
unwind(&db, LAST_BLOCK_IN_FULL_SHARD, LAST_BLOCK_IN_FULL_SHARD - 1);
|
||||
|
||||
// verify initial state
|
||||
close_full_list.pop();
|
||||
almost_full_list.pop();
|
||||
let table = cast(db.table::<tables::StoragesHistory>().unwrap());
|
||||
assert_eq!(table, BTreeMap::from([(shard(u64::MAX), close_full_list)]));
|
||||
assert_eq!(table, BTreeMap::from([(shard(u64::MAX), almost_full_list)]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -663,4 +677,294 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
mod rocksdb_tests {
|
||||
use super::*;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use reth_storage_api::StorageSettings;
|
||||
|
||||
/// Test that when `storages_history_in_rocksdb` is enabled, the stage
|
||||
/// writes storage history indices to `RocksDB` instead of MDBX.
|
||||
#[tokio::test]
|
||||
async fn execute_writes_to_rocksdb_when_enabled() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let mdbx_table = db.table::<tables::StoragesHistory>().unwrap();
|
||||
assert!(
|
||||
mdbx_table.is_empty(),
|
||||
"MDBX StoragesHistory should be empty when RocksDB is enabled"
|
||||
);
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should contain storage history");
|
||||
|
||||
let block_list = result.unwrap();
|
||||
let blocks: Vec<u64> = block_list.iter().collect();
|
||||
assert_eq!(blocks, (0..=10).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
/// Test that unwind works correctly when `storages_history_in_rocksdb` is enabled.
|
||||
#[tokio::test]
|
||||
async fn unwind_works_when_rocksdb_enabled() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should have data before unwind");
|
||||
let blocks_before: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks_before, (0..=10).collect::<Vec<_>>());
|
||||
|
||||
let unwind_input =
|
||||
UnwindInput { checkpoint: StageCheckpoint::new(10), unwind_to: 5, bad_block: None };
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.unwind(&provider, unwind_input).unwrap();
|
||||
assert_eq!(out, UnwindOutput { checkpoint: StageCheckpoint::new(5) });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should still have data after partial unwind");
|
||||
let blocks_after: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(
|
||||
blocks_after,
|
||||
(0..=5).collect::<Vec<_>>(),
|
||||
"Should only have blocks 0-5 after unwind to block 5"
|
||||
);
|
||||
}
|
||||
|
||||
/// Test that unwind to block 0 keeps only block 0's history.
|
||||
#[tokio::test]
|
||||
async fn unwind_to_zero_keeps_block_zero() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=5 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(5), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(5), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should have data before unwind");
|
||||
|
||||
let unwind_input =
|
||||
UnwindInput { checkpoint: StageCheckpoint::new(5), unwind_to: 0, bad_block: None };
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.unwind(&provider, unwind_input).unwrap();
|
||||
assert_eq!(out, UnwindOutput { checkpoint: StageCheckpoint::new(0) });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should still have block 0 history");
|
||||
let blocks_after: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks_after, vec![0], "Should only have block 0 after unwinding to 0");
|
||||
}
|
||||
|
||||
/// Test incremental sync merges new data with existing shards.
|
||||
#[tokio::test]
|
||||
async fn execute_incremental_sync() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=5 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(5), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(5), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some());
|
||||
let blocks: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks, (0..=5).collect::<Vec<_>>());
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 6..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), checkpoint: Some(StageCheckpoint::new(5)) };
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should have merged data");
|
||||
let blocks: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks, (0..=10).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
/// Test multi-shard unwind correctly handles shards that span across unwind boundary.
|
||||
#[tokio::test]
|
||||
async fn unwind_multi_shard() {
|
||||
use reth_db_api::models::sharded_key::NUM_OF_INDICES_IN_SHARD;
|
||||
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let num_blocks = (NUM_OF_INDICES_IN_SHARD * 2 + 100) as u64;
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..num_blocks {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(num_blocks - 1), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(
|
||||
out,
|
||||
ExecOutput { checkpoint: StageCheckpoint::new(num_blocks - 1), done: true }
|
||||
);
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let shards = rocksdb.storage_history_shards(ADDRESS, STORAGE_KEY).unwrap();
|
||||
assert!(shards.len() >= 2, "Should have at least 2 shards for {} blocks", num_blocks);
|
||||
|
||||
let unwind_to = NUM_OF_INDICES_IN_SHARD as u64 + 50;
|
||||
let unwind_input = UnwindInput {
|
||||
checkpoint: StageCheckpoint::new(num_blocks - 1),
|
||||
unwind_to,
|
||||
bad_block: None,
|
||||
};
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.unwind(&provider, unwind_input).unwrap();
|
||||
assert_eq!(out, UnwindOutput { checkpoint: StageCheckpoint::new(unwind_to) });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let shards_after = rocksdb.storage_history_shards(ADDRESS, STORAGE_KEY).unwrap();
|
||||
assert!(!shards_after.is_empty(), "Should still have shards after unwind");
|
||||
|
||||
let all_blocks: Vec<u64> =
|
||||
shards_after.iter().flat_map(|(_, list)| list.iter()).collect();
|
||||
assert_eq!(
|
||||
all_blocks,
|
||||
(0..=unwind_to).collect::<Vec<_>>(),
|
||||
"Should only have blocks 0 to {} after unwind",
|
||||
unwind_to
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,20 @@
|
||||
//! Utils for `stages`.
|
||||
use alloy_primitives::{Address, BlockNumber, TxNumber};
|
||||
use alloy_primitives::{Address, BlockNumber, TxNumber, B256};
|
||||
use reth_config::config::EtlConfig;
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbCursorRW},
|
||||
models::{sharded_key::NUM_OF_INDICES_IN_SHARD, AccountBeforeTx, ShardedKey},
|
||||
table::{Decompress, Table},
|
||||
transaction::{DbTx, DbTxMut},
|
||||
BlockNumberList, DatabaseError,
|
||||
models::{
|
||||
sharded_key::NUM_OF_INDICES_IN_SHARD, storage_sharded_key::StorageShardedKey,
|
||||
AccountBeforeTx, ShardedKey,
|
||||
},
|
||||
table::{Decode, Decompress, Table},
|
||||
transaction::DbTx,
|
||||
BlockNumberList,
|
||||
};
|
||||
use reth_etl::Collector;
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_provider::{
|
||||
providers::StaticFileProvider, to_range, BlockReader, DBProvider, ProviderError,
|
||||
providers::StaticFileProvider, to_range, BlockReader, DBProvider, EitherWriter, ProviderError,
|
||||
StaticFileProviderFactory,
|
||||
};
|
||||
use reth_stages_api::StageError;
|
||||
@@ -108,7 +112,7 @@ where
|
||||
for (address, indices) in cache {
|
||||
insert_fn(address, indices)?
|
||||
}
|
||||
Ok::<(), StageError>(())
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Collects account history indices using a provider that implements `ChangeSetReader`.
|
||||
@@ -124,12 +128,12 @@ where
|
||||
let mut cache: HashMap<Address, Vec<u64>> = HashMap::default();
|
||||
|
||||
let mut insert_fn = |address: Address, indices: Vec<u64>| {
|
||||
let last = indices.last().expect("qed");
|
||||
let last = indices.last().expect("indices is non-empty");
|
||||
collector.insert(
|
||||
ShardedKey::new(address, *last),
|
||||
BlockNumberList::new_pre_sorted(indices.into_iter()),
|
||||
)?;
|
||||
Ok::<(), StageError>(())
|
||||
Ok(())
|
||||
};
|
||||
|
||||
// Convert range bounds to concrete range
|
||||
@@ -170,154 +174,176 @@ where
|
||||
Ok(collector)
|
||||
}
|
||||
|
||||
/// Given a [`Collector`] created by [`collect_history_indices`] it iterates all entries, loading
|
||||
/// the indices into the database in shards.
|
||||
/// Loads account history indices into the database via `EitherWriter`.
|
||||
///
|
||||
/// ## Process
|
||||
/// Iterates over elements, grouping indices by their partial keys (e.g., `Address` or
|
||||
/// `Address.StorageKey`). It flushes indices to disk when reaching a shard's max length
|
||||
/// (`NUM_OF_INDICES_IN_SHARD`) or when the partial key changes, ensuring the last previous partial
|
||||
/// key shard is stored.
|
||||
pub(crate) fn load_history_indices<Provider, H, P>(
|
||||
provider: &Provider,
|
||||
mut collector: Collector<H::Key, H::Value>,
|
||||
/// Works with [`EitherWriter`] to support both MDBX and `RocksDB` backends.
|
||||
///
|
||||
/// ## Process
|
||||
/// Iterates over elements, grouping indices by their address. It flushes indices to disk
|
||||
/// when reaching a shard's max length (`NUM_OF_INDICES_IN_SHARD`) or when the address changes,
|
||||
/// ensuring the last previous address shard is stored.
|
||||
///
|
||||
/// Uses `Option<Address>` instead of `Address::default()` as the sentinel to avoid
|
||||
/// incorrectly treating `Address::ZERO` as "no previous address".
|
||||
pub(crate) fn load_account_history<N, CURSOR>(
|
||||
mut collector: Collector<ShardedKey<Address>, BlockNumberList>,
|
||||
append_only: bool,
|
||||
sharded_key_factory: impl Clone + Fn(P, u64) -> <H as Table>::Key,
|
||||
decode_key: impl Fn(Vec<u8>) -> Result<<H as Table>::Key, DatabaseError>,
|
||||
get_partial: impl Fn(<H as Table>::Key) -> P,
|
||||
writer: &mut EitherWriter<'_, CURSOR, N>,
|
||||
) -> Result<(), StageError>
|
||||
where
|
||||
Provider: DBProvider<Tx: DbTxMut>,
|
||||
H: Table<Value = BlockNumberList>,
|
||||
P: Copy + Default + Eq,
|
||||
N: NodePrimitives,
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::AccountsHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::AccountsHistory>,
|
||||
{
|
||||
let mut write_cursor = provider.tx_ref().cursor_write::<H>()?;
|
||||
let mut current_partial = P::default();
|
||||
let mut current_address: Option<Address> = None;
|
||||
// Accumulator for block numbers where the current address changed.
|
||||
let mut current_list = Vec::<u64>::new();
|
||||
|
||||
// observability
|
||||
let total_entries = collector.len();
|
||||
let interval = (total_entries / 10).max(1);
|
||||
|
||||
for (index, element) in collector.iter()?.enumerate() {
|
||||
let (k, v) = element?;
|
||||
let sharded_key = decode_key(k)?;
|
||||
let sharded_key = ShardedKey::<Address>::decode_owned(k)?;
|
||||
let new_list = BlockNumberList::decompress_owned(v)?;
|
||||
|
||||
if index > 0 && index.is_multiple_of(interval) && total_entries > 10 {
|
||||
info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices");
|
||||
}
|
||||
|
||||
// AccountsHistory: `Address`.
|
||||
// StorageHistory: `Address.StorageKey`.
|
||||
let partial_key = get_partial(sharded_key);
|
||||
let address = sharded_key.key;
|
||||
|
||||
if current_partial != partial_key {
|
||||
// We have reached the end of this subset of keys so
|
||||
// we need to flush its last indice shard.
|
||||
load_indices(
|
||||
&mut write_cursor,
|
||||
current_partial,
|
||||
&mut current_list,
|
||||
&sharded_key_factory,
|
||||
append_only,
|
||||
LoadMode::Flush,
|
||||
)?;
|
||||
// When address changes, flush the previous address's shards and start fresh.
|
||||
if current_address != Some(address) {
|
||||
// Flush all remaining shards for the previous address (uses u64::MAX for last shard).
|
||||
if let Some(prev_addr) = current_address {
|
||||
flush_account_history_shards(prev_addr, &mut current_list, append_only, writer)?;
|
||||
}
|
||||
|
||||
current_partial = partial_key;
|
||||
current_address = Some(address);
|
||||
current_list.clear();
|
||||
|
||||
// If it's not the first sync, there might an existing shard already, so we need to
|
||||
// merge it with the one coming from the collector
|
||||
// On incremental sync, merge with the existing last shard from the database.
|
||||
// The last shard is stored with key (address, u64::MAX) so we can find it.
|
||||
if !append_only &&
|
||||
let Some((_, last_database_shard)) =
|
||||
write_cursor.seek_exact(sharded_key_factory(current_partial, u64::MAX))?
|
||||
let Some(last_shard) = writer.get_last_account_history_shard(address)?
|
||||
{
|
||||
current_list.extend(last_database_shard.iter());
|
||||
current_list.extend(last_shard.iter());
|
||||
}
|
||||
}
|
||||
|
||||
// Append new block numbers to the accumulator.
|
||||
current_list.extend(new_list.iter());
|
||||
load_indices(
|
||||
&mut write_cursor,
|
||||
current_partial,
|
||||
&mut current_list,
|
||||
&sharded_key_factory,
|
||||
append_only,
|
||||
LoadMode::KeepLast,
|
||||
)?;
|
||||
|
||||
// Flush complete shards, keeping the last (partial) shard buffered.
|
||||
flush_account_history_shards_partial(address, &mut current_list, append_only, writer)?;
|
||||
}
|
||||
|
||||
// There will be one remaining shard that needs to be flushed to DB.
|
||||
load_indices(
|
||||
&mut write_cursor,
|
||||
current_partial,
|
||||
&mut current_list,
|
||||
&sharded_key_factory,
|
||||
append_only,
|
||||
LoadMode::Flush,
|
||||
)?;
|
||||
// Flush the final address's remaining shard.
|
||||
if let Some(addr) = current_address {
|
||||
flush_account_history_shards(addr, &mut current_list, append_only, writer)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Shard and insert the indices list according to [`LoadMode`] and its length.
|
||||
pub(crate) fn load_indices<H, C, P>(
|
||||
cursor: &mut C,
|
||||
partial_key: P,
|
||||
list: &mut Vec<BlockNumber>,
|
||||
sharded_key_factory: &impl Fn(P, BlockNumber) -> <H as Table>::Key,
|
||||
/// Flushes complete shards for account history, keeping the trailing partial shard buffered.
|
||||
///
|
||||
/// Only flushes when we have more than one shard's worth of data, keeping the last
|
||||
/// (possibly partial) shard for continued accumulation. This avoids writing a shard
|
||||
/// that may need to be updated when more indices arrive.
|
||||
fn flush_account_history_shards_partial<N, CURSOR>(
|
||||
address: Address,
|
||||
list: &mut Vec<u64>,
|
||||
append_only: bool,
|
||||
mode: LoadMode,
|
||||
writer: &mut EitherWriter<'_, CURSOR, N>,
|
||||
) -> Result<(), StageError>
|
||||
where
|
||||
C: DbCursorRO<H> + DbCursorRW<H>,
|
||||
H: Table<Value = BlockNumberList>,
|
||||
P: Copy,
|
||||
N: NodePrimitives,
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::AccountsHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::AccountsHistory>,
|
||||
{
|
||||
if list.len() > NUM_OF_INDICES_IN_SHARD || mode.is_flush() {
|
||||
let chunks = list
|
||||
.chunks(NUM_OF_INDICES_IN_SHARD)
|
||||
.map(|chunks| chunks.to_vec())
|
||||
.collect::<Vec<Vec<u64>>>();
|
||||
// Nothing to flush if we haven't filled a complete shard yet.
|
||||
if list.len() <= NUM_OF_INDICES_IN_SHARD {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut iter = chunks.into_iter().peekable();
|
||||
while let Some(chunk) = iter.next() {
|
||||
let mut highest = *chunk.last().expect("at least one index");
|
||||
let num_full_shards = list.len() / NUM_OF_INDICES_IN_SHARD;
|
||||
|
||||
if !mode.is_flush() && iter.peek().is_none() {
|
||||
*list = chunk;
|
||||
} else {
|
||||
if iter.peek().is_none() {
|
||||
highest = u64::MAX;
|
||||
}
|
||||
let key = sharded_key_factory(partial_key, highest);
|
||||
let value = BlockNumberList::new_pre_sorted(chunk);
|
||||
// Always keep at least one shard buffered for continued accumulation.
|
||||
// If len is exact multiple of shard size, keep the last full shard.
|
||||
let shards_to_flush = if list.len().is_multiple_of(NUM_OF_INDICES_IN_SHARD) {
|
||||
num_full_shards - 1
|
||||
} else {
|
||||
num_full_shards
|
||||
};
|
||||
|
||||
if append_only {
|
||||
cursor.append(key, &value)?;
|
||||
} else {
|
||||
cursor.upsert(key, &value)?;
|
||||
}
|
||||
}
|
||||
if shards_to_flush == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Split: flush the first N shards, keep the remainder buffered.
|
||||
let flush_len = shards_to_flush * NUM_OF_INDICES_IN_SHARD;
|
||||
let remainder = list.split_off(flush_len);
|
||||
|
||||
// Write each complete shard with its highest block number as the key.
|
||||
for chunk in list.chunks(NUM_OF_INDICES_IN_SHARD) {
|
||||
let highest = *chunk.last().expect("chunk is non-empty");
|
||||
let key = ShardedKey::new(address, highest);
|
||||
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
|
||||
|
||||
if append_only {
|
||||
writer.append_account_history(key, &value)?;
|
||||
} else {
|
||||
writer.upsert_account_history(key, &value)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the remaining indices for the next iteration.
|
||||
*list = remainder;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Mode on how to load index shards into the database.
|
||||
pub(crate) enum LoadMode {
|
||||
/// Keep the last shard in memory and don't flush it to the database.
|
||||
KeepLast,
|
||||
/// Flush all shards into the database.
|
||||
Flush,
|
||||
}
|
||||
|
||||
impl LoadMode {
|
||||
const fn is_flush(&self) -> bool {
|
||||
matches!(self, Self::Flush)
|
||||
/// Flushes all remaining shards for account history, using `u64::MAX` for the last shard.
|
||||
///
|
||||
/// The `u64::MAX` key for the final shard is an invariant that allows `seek_exact(address,
|
||||
/// u64::MAX)` to find the last shard during incremental sync for merging with new indices.
|
||||
fn flush_account_history_shards<N, CURSOR>(
|
||||
address: Address,
|
||||
list: &mut Vec<u64>,
|
||||
append_only: bool,
|
||||
writer: &mut EitherWriter<'_, CURSOR, N>,
|
||||
) -> Result<(), StageError>
|
||||
where
|
||||
N: NodePrimitives,
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::AccountsHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::AccountsHistory>,
|
||||
{
|
||||
if list.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let num_chunks = list.len().div_ceil(NUM_OF_INDICES_IN_SHARD);
|
||||
|
||||
for (i, chunk) in list.chunks(NUM_OF_INDICES_IN_SHARD).enumerate() {
|
||||
let is_last = i == num_chunks - 1;
|
||||
|
||||
// Use u64::MAX for the final shard's key. This invariant allows incremental sync
|
||||
// to find the last shard via seek_exact(address, u64::MAX) for merging.
|
||||
let highest = if is_last { u64::MAX } else { *chunk.last().expect("chunk is non-empty") };
|
||||
|
||||
let key = ShardedKey::new(address, highest);
|
||||
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
|
||||
|
||||
if append_only {
|
||||
writer.append_account_history(key, &value)?;
|
||||
} else {
|
||||
writer.upsert_account_history(key, &value)?;
|
||||
}
|
||||
}
|
||||
|
||||
list.clear();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Called when database is ahead of static files. Attempts to find the first block we are missing
|
||||
@@ -355,3 +381,191 @@ where
|
||||
segment,
|
||||
})
|
||||
}
|
||||
|
||||
/// Loads storage history indices into the database via `EitherWriter`.
|
||||
///
|
||||
/// Works with [`EitherWriter`] to support both MDBX and `RocksDB` backends.
|
||||
///
|
||||
/// ## Process
|
||||
/// Iterates over elements, grouping indices by their (address, `storage_key`) pairs. It flushes
|
||||
/// indices to disk when reaching a shard's max length (`NUM_OF_INDICES_IN_SHARD`) or when the
|
||||
/// (address, `storage_key`) pair changes, ensuring the last previous shard is stored.
|
||||
///
|
||||
/// Uses `Option<(Address, B256)>` instead of default values as the sentinel to avoid
|
||||
/// incorrectly treating `(Address::ZERO, B256::ZERO)` as "no previous key".
|
||||
pub(crate) fn load_storage_history<N, CURSOR>(
|
||||
mut collector: Collector<StorageShardedKey, BlockNumberList>,
|
||||
append_only: bool,
|
||||
writer: &mut EitherWriter<'_, CURSOR, N>,
|
||||
) -> Result<(), StageError>
|
||||
where
|
||||
N: NodePrimitives,
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::StoragesHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::StoragesHistory>,
|
||||
{
|
||||
let mut current_key: Option<(Address, B256)> = None;
|
||||
// Accumulator for block numbers where the current (address, storage_key) changed.
|
||||
let mut current_list = Vec::<u64>::new();
|
||||
|
||||
let total_entries = collector.len();
|
||||
let interval = (total_entries / 10).max(1);
|
||||
|
||||
for (index, element) in collector.iter()?.enumerate() {
|
||||
let (k, v) = element?;
|
||||
let sharded_key = StorageShardedKey::decode_owned(k)?;
|
||||
let new_list = BlockNumberList::decompress_owned(v)?;
|
||||
|
||||
if index > 0 && index.is_multiple_of(interval) && total_entries > 10 {
|
||||
info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices");
|
||||
}
|
||||
|
||||
let partial_key = (sharded_key.address, sharded_key.sharded_key.key);
|
||||
|
||||
// When (address, storage_key) changes, flush the previous key's shards and start fresh.
|
||||
if current_key != Some(partial_key) {
|
||||
// Flush all remaining shards for the previous key (uses u64::MAX for last shard).
|
||||
if let Some((prev_addr, prev_storage_key)) = current_key {
|
||||
flush_storage_history_shards(
|
||||
prev_addr,
|
||||
prev_storage_key,
|
||||
&mut current_list,
|
||||
append_only,
|
||||
writer,
|
||||
)?;
|
||||
}
|
||||
|
||||
current_key = Some(partial_key);
|
||||
current_list.clear();
|
||||
|
||||
// On incremental sync, merge with the existing last shard from the database.
|
||||
// The last shard is stored with key (address, storage_key, u64::MAX) so we can find it.
|
||||
if !append_only &&
|
||||
let Some(last_shard) =
|
||||
writer.get_last_storage_history_shard(partial_key.0, partial_key.1)?
|
||||
{
|
||||
current_list.extend(last_shard.iter());
|
||||
}
|
||||
}
|
||||
|
||||
// Append new block numbers to the accumulator.
|
||||
current_list.extend(new_list.iter());
|
||||
|
||||
// Flush complete shards, keeping the last (partial) shard buffered.
|
||||
flush_storage_history_shards_partial(
|
||||
partial_key.0,
|
||||
partial_key.1,
|
||||
&mut current_list,
|
||||
append_only,
|
||||
writer,
|
||||
)?;
|
||||
}
|
||||
|
||||
// Flush the final key's remaining shard.
|
||||
if let Some((addr, storage_key)) = current_key {
|
||||
flush_storage_history_shards(addr, storage_key, &mut current_list, append_only, writer)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flushes complete shards for storage history, keeping the trailing partial shard buffered.
|
||||
///
|
||||
/// Only flushes when we have more than one shard's worth of data, keeping the last
|
||||
/// (possibly partial) shard for continued accumulation. This avoids writing a shard
|
||||
/// that may need to be updated when more indices arrive.
|
||||
fn flush_storage_history_shards_partial<N, CURSOR>(
|
||||
address: Address,
|
||||
storage_key: B256,
|
||||
list: &mut Vec<u64>,
|
||||
append_only: bool,
|
||||
writer: &mut EitherWriter<'_, CURSOR, N>,
|
||||
) -> Result<(), StageError>
|
||||
where
|
||||
N: NodePrimitives,
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::StoragesHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::StoragesHistory>,
|
||||
{
|
||||
// Nothing to flush if we haven't filled a complete shard yet.
|
||||
if list.len() <= NUM_OF_INDICES_IN_SHARD {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let num_full_shards = list.len() / NUM_OF_INDICES_IN_SHARD;
|
||||
|
||||
// Always keep at least one shard buffered for continued accumulation.
|
||||
// If len is exact multiple of shard size, keep the last full shard.
|
||||
let shards_to_flush = if list.len().is_multiple_of(NUM_OF_INDICES_IN_SHARD) {
|
||||
num_full_shards - 1
|
||||
} else {
|
||||
num_full_shards
|
||||
};
|
||||
|
||||
if shards_to_flush == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Split: flush the first N shards, keep the remainder buffered.
|
||||
let flush_len = shards_to_flush * NUM_OF_INDICES_IN_SHARD;
|
||||
let remainder = list.split_off(flush_len);
|
||||
|
||||
// Write each complete shard with its highest block number as the key.
|
||||
for chunk in list.chunks(NUM_OF_INDICES_IN_SHARD) {
|
||||
let highest = *chunk.last().expect("chunk is non-empty");
|
||||
let key = StorageShardedKey::new(address, storage_key, highest);
|
||||
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
|
||||
|
||||
if append_only {
|
||||
writer.append_storage_history(key, &value)?;
|
||||
} else {
|
||||
writer.upsert_storage_history(key, &value)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the remaining indices for the next iteration.
|
||||
*list = remainder;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flushes all remaining shards for storage history, using `u64::MAX` for the last shard.
|
||||
///
|
||||
/// The `u64::MAX` key for the final shard is an invariant that allows
|
||||
/// `seek_exact(address, storage_key, u64::MAX)` to find the last shard during incremental
|
||||
/// sync for merging with new indices.
|
||||
fn flush_storage_history_shards<N, CURSOR>(
|
||||
address: Address,
|
||||
storage_key: B256,
|
||||
list: &mut Vec<u64>,
|
||||
append_only: bool,
|
||||
writer: &mut EitherWriter<'_, CURSOR, N>,
|
||||
) -> Result<(), StageError>
|
||||
where
|
||||
N: NodePrimitives,
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::StoragesHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::StoragesHistory>,
|
||||
{
|
||||
if list.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let num_chunks = list.len().div_ceil(NUM_OF_INDICES_IN_SHARD);
|
||||
|
||||
for (i, chunk) in list.chunks(NUM_OF_INDICES_IN_SHARD).enumerate() {
|
||||
let is_last = i == num_chunks - 1;
|
||||
|
||||
// Use u64::MAX for the final shard's key. This invariant allows incremental sync
|
||||
// to find the last shard via seek_exact(address, storage_key, u64::MAX) for merging.
|
||||
let highest = if is_last { u64::MAX } else { *chunk.last().expect("chunk is non-empty") };
|
||||
|
||||
let key = StorageShardedKey::new(address, storage_key, highest);
|
||||
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
|
||||
|
||||
if append_only {
|
||||
writer.append_storage_history(key, &value)?;
|
||||
} else {
|
||||
writer.upsert_storage_history(key, &value)?;
|
||||
}
|
||||
}
|
||||
|
||||
list.clear();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -40,6 +40,7 @@ serde = { workspace = true, default-features = false }
|
||||
metrics.workspace = true
|
||||
|
||||
# misc
|
||||
arrayvec.workspace = true
|
||||
derive_more.workspace = true
|
||||
bytes.workspace = true
|
||||
|
||||
|
||||
@@ -34,6 +34,21 @@ pub struct StorageSettings {
|
||||
}
|
||||
|
||||
impl StorageSettings {
|
||||
/// Returns the default base `StorageSettings` for this build.
|
||||
///
|
||||
/// When the `edge` feature is enabled, returns [`Self::edge()`].
|
||||
/// Otherwise, returns [`Self::legacy()`].
|
||||
pub const fn base() -> Self {
|
||||
#[cfg(feature = "edge")]
|
||||
{
|
||||
Self::edge()
|
||||
}
|
||||
#[cfg(not(feature = "edge"))]
|
||||
{
|
||||
Self::legacy()
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates `StorageSettings` for edge nodes with all storage features enabled:
|
||||
/// - Receipts and transaction senders in static files
|
||||
/// - History indices in `RocksDB` (storages, accounts, transaction hashes)
|
||||
@@ -45,7 +60,7 @@ impl StorageSettings {
|
||||
transaction_senders_in_static_files: true,
|
||||
account_changesets_in_static_files: true,
|
||||
storages_history_in_rocksdb: false,
|
||||
transaction_hash_numbers_in_rocksdb: false,
|
||||
transaction_hash_numbers_in_rocksdb: true,
|
||||
account_history_in_rocksdb: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,13 +126,10 @@ impl Decode for String {
|
||||
}
|
||||
|
||||
impl Encode for StoredNibbles {
|
||||
type Encoded = Vec<u8>;
|
||||
type Encoded = arrayvec::ArrayVec<u8, 64>;
|
||||
|
||||
// Delegate to the Compact implementation
|
||||
fn encode(self) -> Self::Encoded {
|
||||
// NOTE: This used to be `to_compact`, but all it does is append the bytes to the buffer,
|
||||
// so we can just use the implementation of `Into<Vec<u8>>` to reuse the buffer.
|
||||
self.0.to_vec()
|
||||
self.0.iter().collect()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,13 +3,16 @@ use crate::{
|
||||
table::{Decode, Encode},
|
||||
DatabaseError,
|
||||
};
|
||||
use alloy_primitives::BlockNumber;
|
||||
use alloy_primitives::{Address, BlockNumber};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::hash::Hash;
|
||||
|
||||
/// Number of indices in one shard.
|
||||
pub const NUM_OF_INDICES_IN_SHARD: usize = 2_000;
|
||||
|
||||
/// Size of `BlockNumber` in bytes (u64 = 8 bytes).
|
||||
const BLOCK_NUMBER_SIZE: usize = std::mem::size_of::<BlockNumber>();
|
||||
|
||||
/// Sometimes data can be too big to be saved for a single key. This helps out by dividing the data
|
||||
/// into different shards. Example:
|
||||
///
|
||||
@@ -43,21 +46,68 @@ impl<T> ShardedKey<T> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Encode> Encode for ShardedKey<T> {
|
||||
type Encoded = Vec<u8>;
|
||||
/// Stack-allocated encoded key for `ShardedKey<Address>`.
|
||||
///
|
||||
/// This avoids heap allocation in hot database paths. The key layout is:
|
||||
/// - 20 bytes: `Address`
|
||||
/// - 8 bytes: `BlockNumber` (big-endian)
|
||||
pub type ShardedKeyAddressEncoded = [u8; 20 + BLOCK_NUMBER_SIZE];
|
||||
|
||||
impl Encode for ShardedKey<Address> {
|
||||
type Encoded = ShardedKeyAddressEncoded;
|
||||
|
||||
#[inline]
|
||||
fn encode(self) -> Self::Encoded {
|
||||
let mut buf: Vec<u8> = Encode::encode(self.key).into();
|
||||
buf.extend_from_slice(&self.highest_block_number.to_be_bytes());
|
||||
let mut buf = [0u8; 20 + BLOCK_NUMBER_SIZE];
|
||||
buf[..20].copy_from_slice(self.key.as_slice());
|
||||
buf[20..].copy_from_slice(&self.highest_block_number.to_be_bytes());
|
||||
buf
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Decode> Decode for ShardedKey<T> {
|
||||
impl Decode for ShardedKey<Address> {
|
||||
fn decode(value: &[u8]) -> Result<Self, DatabaseError> {
|
||||
let (key, highest_tx_number) = value.split_last_chunk().ok_or(DatabaseError::Decode)?;
|
||||
let key = T::decode(key)?;
|
||||
let highest_tx_number = u64::from_be_bytes(*highest_tx_number);
|
||||
Ok(Self::new(key, highest_tx_number))
|
||||
if value.len() != 20 + BLOCK_NUMBER_SIZE {
|
||||
return Err(DatabaseError::Decode);
|
||||
}
|
||||
let key = Address::from_slice(&value[..20]);
|
||||
let highest_block_number =
|
||||
u64::from_be_bytes(value[20..].try_into().map_err(|_| DatabaseError::Decode)?);
|
||||
Ok(Self::new(key, highest_block_number))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_primitives::address;
|
||||
|
||||
#[test]
|
||||
fn sharded_key_address_encode_decode_roundtrip() {
|
||||
let addr = address!("0102030405060708091011121314151617181920");
|
||||
let block_num = 0x123456789ABCDEF0u64;
|
||||
let key = ShardedKey::new(addr, block_num);
|
||||
|
||||
let encoded = key.encode();
|
||||
|
||||
// Verify it's stack-allocated (28 bytes)
|
||||
assert_eq!(encoded.len(), 28);
|
||||
assert_eq!(std::mem::size_of_val(&encoded), 28);
|
||||
|
||||
// Verify roundtrip (check against expected values since key was consumed)
|
||||
let decoded = ShardedKey::<Address>::decode(&encoded).unwrap();
|
||||
assert_eq!(decoded.key, address!("0102030405060708091011121314151617181920"));
|
||||
assert_eq!(decoded.highest_block_number, 0x123456789ABCDEF0u64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sharded_key_last_works() {
|
||||
let addr = address!("0102030405060708091011121314151617181920");
|
||||
let key = ShardedKey::<Address>::last(addr);
|
||||
assert_eq!(key.highest_block_number, u64::MAX);
|
||||
|
||||
let encoded = key.encode();
|
||||
let decoded = ShardedKey::<Address>::decode(&encoded).unwrap();
|
||||
assert_eq!(decoded.highest_block_number, u64::MAX);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,14 @@ pub const NUM_OF_INDICES_IN_SHARD: usize = 2_000;
|
||||
/// The fields are: 20-byte address, 32-byte key, and 8-byte block number
|
||||
const STORAGE_SHARD_KEY_BYTES_SIZE: usize = 20 + 32 + 8;
|
||||
|
||||
/// Stack-allocated encoded key for `StorageShardedKey`.
|
||||
///
|
||||
/// This avoids heap allocation in hot database paths. The key layout is:
|
||||
/// - 20 bytes: `Address`
|
||||
/// - 32 bytes: `B256` storage key
|
||||
/// - 8 bytes: `BlockNumber` (big-endian)
|
||||
pub type StorageShardedKeyEncoded = [u8; STORAGE_SHARD_KEY_BYTES_SIZE];
|
||||
|
||||
/// Sometimes data can be too big to be saved for a single key. This helps out by dividing the data
|
||||
/// into different shards. Example:
|
||||
///
|
||||
@@ -54,13 +62,14 @@ impl StorageShardedKey {
|
||||
}
|
||||
|
||||
impl Encode for StorageShardedKey {
|
||||
type Encoded = Vec<u8>;
|
||||
type Encoded = StorageShardedKeyEncoded;
|
||||
|
||||
#[inline]
|
||||
fn encode(self) -> Self::Encoded {
|
||||
let mut buf: Vec<u8> = Vec::with_capacity(STORAGE_SHARD_KEY_BYTES_SIZE);
|
||||
buf.extend_from_slice(&Encode::encode(self.address));
|
||||
buf.extend_from_slice(&Encode::encode(self.sharded_key.key));
|
||||
buf.extend_from_slice(&self.sharded_key.highest_block_number.to_be_bytes());
|
||||
let mut buf = [0u8; STORAGE_SHARD_KEY_BYTES_SIZE];
|
||||
buf[..20].copy_from_slice(self.address.as_slice());
|
||||
buf[20..52].copy_from_slice(self.sharded_key.key.as_slice());
|
||||
buf[52..].copy_from_slice(&self.sharded_key.highest_block_number.to_be_bytes());
|
||||
buf
|
||||
}
|
||||
}
|
||||
@@ -81,3 +90,44 @@ impl Decode for StorageShardedKey {
|
||||
Ok(Self { address, sharded_key: ShardedKey::new(storage_key, highest_block_number) })
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_primitives::{address, b256};
|
||||
|
||||
#[test]
|
||||
fn storage_sharded_key_encode_decode_roundtrip() {
|
||||
let addr = address!("0102030405060708091011121314151617181920");
|
||||
let storage_key = b256!("0001020304050607080910111213141516171819202122232425262728293031");
|
||||
let block_num = 0x123456789ABCDEFu64;
|
||||
let key = StorageShardedKey::new(addr, storage_key, block_num);
|
||||
|
||||
let encoded = key.encode();
|
||||
|
||||
// Verify it's stack-allocated (60 bytes)
|
||||
assert_eq!(encoded.len(), 60);
|
||||
assert_eq!(std::mem::size_of_val(&encoded), 60);
|
||||
|
||||
// Verify roundtrip (check against expected values since key was consumed)
|
||||
let decoded = StorageShardedKey::decode(&encoded).unwrap();
|
||||
assert_eq!(decoded.address, address!("0102030405060708091011121314151617181920"));
|
||||
assert_eq!(
|
||||
decoded.sharded_key.key,
|
||||
b256!("0001020304050607080910111213141516171819202122232425262728293031")
|
||||
);
|
||||
assert_eq!(decoded.sharded_key.highest_block_number, 0x123456789ABCDEFu64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn storage_sharded_key_last_works() {
|
||||
let addr = address!("0102030405060708091011121314151617181920");
|
||||
let storage_key = b256!("0001020304050607080910111213141516171819202122232425262728293031");
|
||||
let key = StorageShardedKey::last(addr, storage_key);
|
||||
assert_eq!(key.sharded_key.highest_block_number, u64::MAX);
|
||||
|
||||
let encoded = key.encode();
|
||||
let decoded = StorageShardedKey::decode(&encoded).unwrap();
|
||||
assert_eq!(decoded.sharded_key.highest_block_number, u64::MAX);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,10 +46,40 @@ pub trait Decompress: Send + Sync + Sized + Debug {
|
||||
}
|
||||
}
|
||||
|
||||
/// Trait for converting encoded types to `Vec<u8>`.
|
||||
///
|
||||
/// This is implemented for all `AsRef<[u8]>` types. For `Vec<u8>` this is a no-op,
|
||||
/// for other types like `ArrayVec` or fixed arrays it performs a copy.
|
||||
pub trait IntoVec: AsRef<[u8]> {
|
||||
/// Convert to a `Vec<u8>`.
|
||||
fn into_vec(self) -> Vec<u8>;
|
||||
}
|
||||
|
||||
impl IntoVec for Vec<u8> {
|
||||
#[inline]
|
||||
fn into_vec(self) -> Vec<u8> {
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl<const N: usize> IntoVec for [u8; N] {
|
||||
#[inline]
|
||||
fn into_vec(self) -> Vec<u8> {
|
||||
self.to_vec()
|
||||
}
|
||||
}
|
||||
|
||||
impl<const N: usize> IntoVec for arrayvec::ArrayVec<u8, N> {
|
||||
#[inline]
|
||||
fn into_vec(self) -> Vec<u8> {
|
||||
self.to_vec()
|
||||
}
|
||||
}
|
||||
|
||||
/// Trait that will transform the data to be saved in the DB.
|
||||
pub trait Encode: Send + Sync + Sized + Debug {
|
||||
/// Encoded type.
|
||||
type Encoded: AsRef<[u8]> + Into<Vec<u8>> + Send + Sync + Ord + Debug;
|
||||
type Encoded: AsRef<[u8]> + IntoVec + Send + Sync + Ord + Debug;
|
||||
|
||||
/// Encodes data going into the database.
|
||||
fn encode(self) -> Self::Encoded;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use crate::{
|
||||
table::{Compress, Decode, Decompress, DupSort, Encode, Key, Table, Value},
|
||||
table::{Compress, Decode, Decompress, DupSort, Encode, IntoVec, Key, Table, Value},
|
||||
DatabaseError,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -52,7 +52,7 @@ pub struct RawKey<K: Key> {
|
||||
impl<K: Key> RawKey<K> {
|
||||
/// Create new raw key.
|
||||
pub fn new(key: K) -> Self {
|
||||
Self { key: K::encode(key).into(), _phantom: std::marker::PhantomData }
|
||||
Self { key: K::encode(key).into_vec(), _phantom: std::marker::PhantomData }
|
||||
}
|
||||
|
||||
/// Creates a raw key from an existing `Vec`. Useful when we already have the encoded
|
||||
|
||||
@@ -11,7 +11,7 @@ use reth_db_api::{
|
||||
DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW, DupWalker, RangeWalker,
|
||||
ReverseWalker, Walker,
|
||||
},
|
||||
table::{Compress, Decode, Decompress, DupSort, Encode, Table},
|
||||
table::{Compress, Decode, Decompress, DupSort, Encode, IntoVec, Table},
|
||||
};
|
||||
use reth_libmdbx::{Error as MDBXError, TransactionKind, WriteFlags, RO, RW};
|
||||
use reth_storage_errors::db::{DatabaseErrorInfo, DatabaseWriteError, DatabaseWriteOperation};
|
||||
@@ -215,27 +215,26 @@ impl<K: TransactionKind, T: DupSort> DbDupCursorRO<T> for Cursor<K, T> {
|
||||
) -> Result<DupWalker<'_, T, Self>, DatabaseError> {
|
||||
let start = match (key, subkey) {
|
||||
(Some(key), Some(subkey)) => {
|
||||
// encode key and decode it after.
|
||||
let key: Vec<u8> = key.encode().into();
|
||||
let encoded_key = key.encode();
|
||||
self.inner
|
||||
.get_both_range(key.as_ref(), subkey.encode().as_ref())
|
||||
.get_both_range(encoded_key.as_ref(), subkey.encode().as_ref())
|
||||
.map_err(|e| DatabaseError::Read(e.into()))?
|
||||
.map(|val| decoder::<T>((Cow::Owned(key), val)))
|
||||
.map(|val| decoder::<T>((Cow::Borrowed(encoded_key.as_ref()), val)))
|
||||
}
|
||||
(Some(key), None) => {
|
||||
let key: Vec<u8> = key.encode().into();
|
||||
let encoded_key = key.encode();
|
||||
self.inner
|
||||
.set(key.as_ref())
|
||||
.set(encoded_key.as_ref())
|
||||
.map_err(|e| DatabaseError::Read(e.into()))?
|
||||
.map(|val| decoder::<T>((Cow::Owned(key), val)))
|
||||
.map(|val| decoder::<T>((Cow::Borrowed(encoded_key.as_ref()), val)))
|
||||
}
|
||||
(None, Some(subkey)) => {
|
||||
if let Some((key, _)) = self.first()? {
|
||||
let key: Vec<u8> = key.encode().into();
|
||||
let encoded_key = key.encode();
|
||||
self.inner
|
||||
.get_both_range(key.as_ref(), subkey.encode().as_ref())
|
||||
.get_both_range(encoded_key.as_ref(), subkey.encode().as_ref())
|
||||
.map_err(|e| DatabaseError::Read(e.into()))?
|
||||
.map(|val| decoder::<T>((Cow::Owned(key), val)))
|
||||
.map(|val| decoder::<T>((Cow::Borrowed(encoded_key.as_ref()), val)))
|
||||
} else {
|
||||
Some(Err(DatabaseError::Read(MDBXError::NotFound.into())))
|
||||
}
|
||||
@@ -269,7 +268,7 @@ impl<T: Table> DbCursorRW<T> for Cursor<RW, T> {
|
||||
info: e.into(),
|
||||
operation: DatabaseWriteOperation::CursorUpsert,
|
||||
table_name: T::NAME,
|
||||
key: key.into(),
|
||||
key: key.into_vec(),
|
||||
}
|
||||
.into()
|
||||
})
|
||||
@@ -291,7 +290,7 @@ impl<T: Table> DbCursorRW<T> for Cursor<RW, T> {
|
||||
info: e.into(),
|
||||
operation: DatabaseWriteOperation::CursorInsert,
|
||||
table_name: T::NAME,
|
||||
key: key.into(),
|
||||
key: key.into_vec(),
|
||||
}
|
||||
.into()
|
||||
})
|
||||
@@ -315,7 +314,7 @@ impl<T: Table> DbCursorRW<T> for Cursor<RW, T> {
|
||||
info: e.into(),
|
||||
operation: DatabaseWriteOperation::CursorAppend,
|
||||
table_name: T::NAME,
|
||||
key: key.into(),
|
||||
key: key.into_vec(),
|
||||
}
|
||||
.into()
|
||||
})
|
||||
@@ -351,7 +350,7 @@ impl<T: DupSort> DbDupCursorRW<T> for Cursor<RW, T> {
|
||||
info: e.into(),
|
||||
operation: DatabaseWriteOperation::CursorAppendDup,
|
||||
table_name: T::NAME,
|
||||
key: key.into(),
|
||||
key: key.into_vec(),
|
||||
}
|
||||
.into()
|
||||
})
|
||||
|
||||
@@ -6,7 +6,7 @@ use crate::{
|
||||
DatabaseError,
|
||||
};
|
||||
use reth_db_api::{
|
||||
table::{Compress, DupSort, Encode, Table, TableImporter},
|
||||
table::{Compress, DupSort, Encode, IntoVec, Table, TableImporter},
|
||||
transaction::{DbTx, DbTxMut},
|
||||
};
|
||||
use reth_libmdbx::{ffi::MDBX_dbi, CommitLatency, Transaction, TransactionKind, WriteFlags, RW};
|
||||
@@ -387,7 +387,7 @@ impl Tx<RW> {
|
||||
info: e.into(),
|
||||
operation: write_operation,
|
||||
table_name: T::NAME,
|
||||
key: key.into(),
|
||||
key: key.into_vec(),
|
||||
}
|
||||
.into()
|
||||
})
|
||||
|
||||
@@ -104,6 +104,16 @@ pub enum ProviderError {
|
||||
/// State is not available for the given block number because it is pruned.
|
||||
#[error("state at block #{_0} is pruned")]
|
||||
StateAtBlockPruned(BlockNumber),
|
||||
/// Block data is not available because history has expired.
|
||||
///
|
||||
/// The requested block number is below the earliest available block.
|
||||
#[error("block #{requested} is not available, history has expired (earliest available: #{earliest_available})")]
|
||||
BlockExpired {
|
||||
/// The block number that was requested.
|
||||
requested: BlockNumber,
|
||||
/// The earliest available block number.
|
||||
earliest_available: BlockNumber,
|
||||
},
|
||||
/// Provider does not support this particular request.
|
||||
#[error("this provider does not support this request")]
|
||||
UnsupportedProvider,
|
||||
|
||||
@@ -223,7 +223,11 @@ where
|
||||
let mut flags: c_uint = 0;
|
||||
unsafe {
|
||||
self.txn_execute(|txn| {
|
||||
mdbx_result(ffi::mdbx_dbi_flags_ex(txn, dbi, &mut flags, ptr::null_mut()))
|
||||
// `mdbx_dbi_flags_ex` requires `status` to be a non-NULL ptr, otherwise it will
|
||||
// return an EINVAL and panic below, so we just provide a placeholder variable
|
||||
// which we discard immediately.
|
||||
let mut _status: c_uint = 0;
|
||||
mdbx_result(ffi::mdbx_dbi_flags_ex(txn, dbi, &mut flags, &mut _status))
|
||||
})??;
|
||||
}
|
||||
|
||||
|
||||
@@ -85,6 +85,7 @@ rand.workspace = true
|
||||
tokio = { workspace = true, features = ["sync", "macros", "rt-multi-thread"] }
|
||||
|
||||
[features]
|
||||
edge = ["reth-storage-api/edge", "rocksdb"]
|
||||
rocksdb = ["dep:rocksdb"]
|
||||
test-utils = [
|
||||
"reth-db/test-utils",
|
||||
|
||||
@@ -13,7 +13,7 @@ use crate::{
|
||||
providers::{history_info, HistoryInfo, StaticFileProvider, StaticFileProviderRWRefMut},
|
||||
StaticFileProviderFactory,
|
||||
};
|
||||
use alloy_primitives::{map::HashMap, Address, BlockNumber, TxHash, TxNumber};
|
||||
use alloy_primitives::{map::HashMap, Address, BlockNumber, TxHash, TxNumber, B256};
|
||||
use rayon::slice::ParallelSliceMut;
|
||||
use reth_db::{
|
||||
cursor::{DbCursorRO, DbDupCursorRW},
|
||||
@@ -512,14 +512,71 @@ where
|
||||
Self::RocksDB(batch) => batch.delete::<tables::StoragesHistory>(key),
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends a storage history entry (for first sync - more efficient).
|
||||
pub fn append_storage_history(
|
||||
&mut self,
|
||||
key: StorageShardedKey,
|
||||
value: &BlockNumberList,
|
||||
) -> ProviderResult<()> {
|
||||
match self {
|
||||
Self::Database(cursor) => Ok(cursor.append(key, value)?),
|
||||
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(batch) => batch.put::<tables::StoragesHistory>(key, value),
|
||||
}
|
||||
}
|
||||
|
||||
/// Upserts a storage history entry (for incremental sync).
|
||||
pub fn upsert_storage_history(
|
||||
&mut self,
|
||||
key: StorageShardedKey,
|
||||
value: &BlockNumberList,
|
||||
) -> ProviderResult<()> {
|
||||
match self {
|
||||
Self::Database(cursor) => Ok(cursor.upsert(key, value)?),
|
||||
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(batch) => batch.put::<tables::StoragesHistory>(key, value),
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets the last shard for an address and storage key (keyed with `u64::MAX`).
|
||||
pub fn get_last_storage_history_shard(
|
||||
&mut self,
|
||||
address: Address,
|
||||
storage_key: B256,
|
||||
) -> ProviderResult<Option<BlockNumberList>> {
|
||||
let key = StorageShardedKey::last(address, storage_key);
|
||||
match self {
|
||||
Self::Database(cursor) => Ok(cursor.seek_exact(key)?.map(|(_, v)| v)),
|
||||
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(batch) => batch.get::<tables::StoragesHistory>(key),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N>
|
||||
where
|
||||
CURSOR: DbCursorRW<tables::AccountsHistory> + DbCursorRO<tables::AccountsHistory>,
|
||||
{
|
||||
/// Puts an account history entry.
|
||||
pub fn put_account_history(
|
||||
/// Appends an account history entry (for first sync - more efficient).
|
||||
pub fn append_account_history(
|
||||
&mut self,
|
||||
key: ShardedKey<Address>,
|
||||
value: &BlockNumberList,
|
||||
) -> ProviderResult<()> {
|
||||
match self {
|
||||
Self::Database(cursor) => Ok(cursor.append(key, value)?),
|
||||
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(batch) => batch.put::<tables::AccountsHistory>(key, value),
|
||||
}
|
||||
}
|
||||
|
||||
/// Upserts an account history entry (for incremental sync).
|
||||
pub fn upsert_account_history(
|
||||
&mut self,
|
||||
key: ShardedKey<Address>,
|
||||
value: &BlockNumberList,
|
||||
@@ -532,6 +589,21 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets the last shard for an address (keyed with `u64::MAX`).
|
||||
pub fn get_last_account_history_shard(
|
||||
&mut self,
|
||||
address: Address,
|
||||
) -> ProviderResult<Option<BlockNumberList>> {
|
||||
match self {
|
||||
Self::Database(cursor) => {
|
||||
Ok(cursor.seek_exact(ShardedKey::last(address))?.map(|(_, v)| v))
|
||||
}
|
||||
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(batch) => batch.get::<tables::AccountsHistory>(ShardedKey::last(address)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Deletes an account history entry.
|
||||
pub fn delete_account_history(&mut self, key: ShardedKey<Address>) -> ProviderResult<()> {
|
||||
match self {
|
||||
@@ -1266,8 +1338,8 @@ mod rocksdb_tests {
|
||||
for (highest_block, blocks) in shards {
|
||||
let key = ShardedKey::new(address, *highest_block);
|
||||
let value = IntegerList::new(blocks.clone()).unwrap();
|
||||
mdbx_writer.put_account_history(key.clone(), &value).unwrap();
|
||||
rocks_writer.put_account_history(key, &value).unwrap();
|
||||
mdbx_writer.upsert_account_history(key.clone(), &value).unwrap();
|
||||
rocks_writer.upsert_account_history(key, &value).unwrap();
|
||||
}
|
||||
|
||||
// Commit both backends
|
||||
|
||||
@@ -560,43 +560,12 @@ impl<TX: DbTx + DbTxMut + 'static, N: NodeTypesForProvider> DatabaseProvider<TX,
|
||||
|
||||
// Write all trie updates in a single batch.
|
||||
// This reduces cursor open/close overhead from N calls to 1.
|
||||
// Uses hybrid algorithm: extend_ref for small batches, k-way merge for large.
|
||||
if save_mode.with_state() {
|
||||
const MERGE_BATCH_THRESHOLD: usize = 30;
|
||||
|
||||
let start = Instant::now();
|
||||
let num_blocks = blocks.len();
|
||||
|
||||
let merged = if num_blocks == 0 {
|
||||
TrieUpdatesSorted::default()
|
||||
} else if num_blocks == 1 {
|
||||
// Single block: use directly (Arc::try_unwrap avoids clone if refcount is 1)
|
||||
match Arc::try_unwrap(blocks[0].trie_updates()) {
|
||||
Ok(owned) => owned,
|
||||
Err(arc) => (*arc).clone(),
|
||||
}
|
||||
} else if num_blocks < MERGE_BATCH_THRESHOLD {
|
||||
// Small k: extend_ref with Arc::make_mut (copy-on-write).
|
||||
// Blocks are oldest-to-newest, iterate forward so newest overrides.
|
||||
let mut blocks_iter = blocks.iter();
|
||||
let mut result = blocks_iter.next().expect("non-empty").trie_updates();
|
||||
|
||||
for block in blocks_iter {
|
||||
Arc::make_mut(&mut result)
|
||||
.extend_ref_and_sort(block.trie_updates().as_ref());
|
||||
}
|
||||
|
||||
match Arc::try_unwrap(result) {
|
||||
Ok(owned) => owned,
|
||||
Err(arc) => (*arc).clone(),
|
||||
}
|
||||
} else {
|
||||
// Large k: k-way merge is faster (O(n log k)).
|
||||
// Collect Arcs first to extend lifetime, then pass refs.
|
||||
// Blocks are oldest-to-newest, merge_batch expects newest-to-oldest.
|
||||
let arcs: Vec<_> = blocks.iter().rev().map(|b| b.trie_updates()).collect();
|
||||
TrieUpdatesSorted::merge_batch(arcs.iter().map(|arc| arc.as_ref()))
|
||||
};
|
||||
// Blocks are oldest-to-newest, merge_batch expects newest-to-oldest.
|
||||
let merged =
|
||||
TrieUpdatesSorted::merge_batch(blocks.iter().rev().map(|b| b.trie_updates()));
|
||||
|
||||
if !merged.is_empty() {
|
||||
self.write_trie_updates_sorted(&merged)?;
|
||||
@@ -1535,10 +1504,17 @@ impl<TX: DbTx + 'static, N: NodeTypesForProvider> BlockReader for DatabaseProvid
|
||||
/// If the header for this block is not found, this returns `None`.
|
||||
/// If the header is found, but the transactions either do not exist, or are not indexed, this
|
||||
/// will return None.
|
||||
///
|
||||
/// Returns an error if the requested block is below the earliest available history.
|
||||
fn block(&self, id: BlockHashOrNumber) -> ProviderResult<Option<Self::Block>> {
|
||||
if let Some(number) = self.convert_hash_or_number(id)? &&
|
||||
let Some(header) = self.header_by_number(number)?
|
||||
{
|
||||
if let Some(number) = self.convert_hash_or_number(id)? {
|
||||
let earliest_available = self.static_file_provider.earliest_history_height();
|
||||
if number < earliest_available {
|
||||
return Err(ProviderError::BlockExpired { requested: number, earliest_available })
|
||||
}
|
||||
|
||||
let Some(header) = self.header_by_number(number)? else { return Ok(None) };
|
||||
|
||||
// If the body indices are not found, this means that the transactions either do not
|
||||
// exist in the database yet, or they do exit but are not indexed.
|
||||
// If they exist but are not indexed, we don't have enough
|
||||
@@ -2970,25 +2946,33 @@ impl<TX: DbTxMut + DbTx + 'static, N: NodeTypes> HistoryWriter for DatabaseProvi
|
||||
.into_iter()
|
||||
.map(|(index, account)| (account.address, *index))
|
||||
.collect::<Vec<_>>();
|
||||
last_indices.sort_by_key(|(a, _)| *a);
|
||||
last_indices.sort_unstable_by_key(|(a, _)| *a);
|
||||
|
||||
// Unwind the account history index.
|
||||
let mut cursor = self.tx.cursor_write::<tables::AccountsHistory>()?;
|
||||
for &(address, rem_index) in &last_indices {
|
||||
let partial_shard = unwind_history_shards::<_, tables::AccountsHistory, _>(
|
||||
&mut cursor,
|
||||
ShardedKey::last(address),
|
||||
rem_index,
|
||||
|sharded_key| sharded_key.key == address,
|
||||
)?;
|
||||
|
||||
// Check the last returned partial shard.
|
||||
// If it's not empty, the shard needs to be reinserted.
|
||||
if !partial_shard.is_empty() {
|
||||
cursor.insert(
|
||||
if self.cached_storage_settings().account_history_in_rocksdb {
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
{
|
||||
let batch = self.rocksdb_provider.unwind_account_history_indices(&last_indices)?;
|
||||
self.pending_rocksdb_batches.lock().push(batch);
|
||||
}
|
||||
} else {
|
||||
// Unwind the account history index in MDBX.
|
||||
let mut cursor = self.tx.cursor_write::<tables::AccountsHistory>()?;
|
||||
for &(address, rem_index) in &last_indices {
|
||||
let partial_shard = unwind_history_shards::<_, tables::AccountsHistory, _>(
|
||||
&mut cursor,
|
||||
ShardedKey::last(address),
|
||||
&BlockNumberList::new_pre_sorted(partial_shard),
|
||||
rem_index,
|
||||
|sharded_key| sharded_key.key == address,
|
||||
)?;
|
||||
|
||||
// Check the last returned partial shard.
|
||||
// If it's not empty, the shard needs to be reinserted.
|
||||
if !partial_shard.is_empty() {
|
||||
cursor.insert(
|
||||
ShardedKey::last(address),
|
||||
&BlockNumberList::new_pre_sorted(partial_shard),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3028,25 +3012,35 @@ impl<TX: DbTxMut + DbTx + 'static, N: NodeTypes> HistoryWriter for DatabaseProvi
|
||||
.collect::<Vec<_>>();
|
||||
storage_changesets.sort_by_key(|(address, key, _)| (*address, *key));
|
||||
|
||||
let mut cursor = self.tx.cursor_write::<tables::StoragesHistory>()?;
|
||||
for &(address, storage_key, rem_index) in &storage_changesets {
|
||||
let partial_shard = unwind_history_shards::<_, tables::StoragesHistory, _>(
|
||||
&mut cursor,
|
||||
StorageShardedKey::last(address, storage_key),
|
||||
rem_index,
|
||||
|storage_sharded_key| {
|
||||
storage_sharded_key.address == address &&
|
||||
storage_sharded_key.sharded_key.key == storage_key
|
||||
},
|
||||
)?;
|
||||
|
||||
// Check the last returned partial shard.
|
||||
// If it's not empty, the shard needs to be reinserted.
|
||||
if !partial_shard.is_empty() {
|
||||
cursor.insert(
|
||||
if self.cached_storage_settings().storages_history_in_rocksdb {
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
{
|
||||
let batch =
|
||||
self.rocksdb_provider.unwind_storage_history_indices(&storage_changesets)?;
|
||||
self.pending_rocksdb_batches.lock().push(batch);
|
||||
}
|
||||
} else {
|
||||
// Unwind the storage history index in MDBX.
|
||||
let mut cursor = self.tx.cursor_write::<tables::StoragesHistory>()?;
|
||||
for &(address, storage_key, rem_index) in &storage_changesets {
|
||||
let partial_shard = unwind_history_shards::<_, tables::StoragesHistory, _>(
|
||||
&mut cursor,
|
||||
StorageShardedKey::last(address, storage_key),
|
||||
&BlockNumberList::new_pre_sorted(partial_shard),
|
||||
rem_index,
|
||||
|storage_sharded_key| {
|
||||
storage_sharded_key.address == address &&
|
||||
storage_sharded_key.sharded_key.key == storage_key
|
||||
},
|
||||
)?;
|
||||
|
||||
// Check the last returned partial shard.
|
||||
// If it's not empty, the shard needs to be reinserted.
|
||||
if !partial_shard.is_empty() {
|
||||
cursor.insert(
|
||||
StorageShardedKey::last(address, storage_key),
|
||||
&BlockNumberList::new_pre_sorted(partial_shard),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -38,7 +38,9 @@ pub use consistent::ConsistentProvider;
|
||||
#[cfg_attr(not(all(unix, feature = "rocksdb")), path = "rocksdb_stub.rs")]
|
||||
pub(crate) mod rocksdb;
|
||||
|
||||
pub use rocksdb::{RocksDBBatch, RocksDBBuilder, RocksDBProvider, RocksTx};
|
||||
pub use rocksdb::{
|
||||
RocksDBBatch, RocksDBBuilder, RocksDBProvider, RocksDBRawIter, RocksDBTableStats, RocksTx,
|
||||
};
|
||||
|
||||
/// Helper trait to bound [`NodeTypes`] so that combined with database they satisfy
|
||||
/// [`ProviderNodeTypes`].
|
||||
|
||||
@@ -6,7 +6,7 @@ use reth_db::Tables;
|
||||
use reth_metrics::Metrics;
|
||||
use strum::{EnumIter, IntoEnumIterator};
|
||||
|
||||
const ROCKSDB_TABLES: &[&str] = &[
|
||||
pub(super) const ROCKSDB_TABLES: &[&str] = &[
|
||||
Tables::TransactionHashNumbers.name(),
|
||||
Tables::StoragesHistory.name(),
|
||||
Tables::AccountsHistory.name(),
|
||||
|
||||
@@ -5,4 +5,6 @@ mod metrics;
|
||||
mod provider;
|
||||
|
||||
pub(crate) use provider::{PendingRocksDBBatches, RocksDBWriteCtx};
|
||||
pub use provider::{RocksDBBatch, RocksDBBuilder, RocksDBProvider, RocksTx};
|
||||
pub use provider::{
|
||||
RocksDBBatch, RocksDBBuilder, RocksDBProvider, RocksDBRawIter, RocksDBTableStats, RocksTx,
|
||||
};
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,8 +5,9 @@
|
||||
//! All method calls are cfg-guarded in the calling code, so only type definitions are needed here.
|
||||
|
||||
use alloy_primitives::BlockNumber;
|
||||
use metrics::Label;
|
||||
use parking_lot::Mutex;
|
||||
use reth_db_api::models::StorageSettings;
|
||||
use reth_db_api::{database_metrics::DatabaseMetrics, models::StorageSettings};
|
||||
use reth_prune_types::PruneMode;
|
||||
use reth_storage_errors::{db::LogLevel, provider::ProviderResult};
|
||||
use std::{path::Path, sync::Arc};
|
||||
@@ -14,6 +15,23 @@ use std::{path::Path, sync::Arc};
|
||||
/// Pending `RocksDB` batches type alias (stub - uses unit type).
|
||||
pub(crate) type PendingRocksDBBatches = Arc<Mutex<Vec<()>>>;
|
||||
|
||||
/// Statistics for a single `RocksDB` table (column family) - stub.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RocksDBTableStats {
|
||||
/// Size of SST files on disk in bytes.
|
||||
pub sst_size_bytes: u64,
|
||||
/// Size of memtables in memory in bytes.
|
||||
pub memtable_size_bytes: u64,
|
||||
/// Name of the table/column family.
|
||||
pub name: String,
|
||||
/// Estimated number of keys in the table.
|
||||
pub estimated_num_keys: u64,
|
||||
/// Estimated size of live data in bytes (SST files + memtables).
|
||||
pub estimated_size_bytes: u64,
|
||||
/// Estimated bytes pending compaction (reclaimable space).
|
||||
pub pending_compaction_bytes: u64,
|
||||
}
|
||||
|
||||
/// Context for `RocksDB` block writes (stub).
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
@@ -56,6 +74,19 @@ impl RocksDBProvider {
|
||||
) -> ProviderResult<Option<BlockNumber>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Returns statistics for all column families in the database (stub implementation).
|
||||
///
|
||||
/// Returns an empty vector since there is no `RocksDB` when the feature is disabled.
|
||||
pub const fn table_stats(&self) -> Vec<RocksDBTableStats> {
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl DatabaseMetrics for RocksDBProvider {
|
||||
fn gauge_metrics(&self) -> Vec<(&'static str, f64, Vec<Label>)> {
|
||||
vec![]
|
||||
}
|
||||
}
|
||||
|
||||
/// A stub batch writer for `RocksDB`.
|
||||
@@ -102,6 +133,11 @@ impl RocksDBBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets read-only mode (stub implementation).
|
||||
pub const fn with_read_only(self, _read_only: bool) -> Self {
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the `RocksDB` provider (stub implementation).
|
||||
pub const fn build(self) -> ProviderResult<RocksDBProvider> {
|
||||
Ok(RocksDBProvider)
|
||||
@@ -111,3 +147,7 @@ impl RocksDBBuilder {
|
||||
/// A stub transaction for `RocksDB`.
|
||||
#[derive(Debug)]
|
||||
pub struct RocksTx;
|
||||
|
||||
/// A stub raw iterator for `RocksDB`.
|
||||
#[derive(Debug)]
|
||||
pub struct RocksDBRawIter;
|
||||
|
||||
@@ -129,6 +129,8 @@ impl<F> OverlayStateProviderFactory<F> {
|
||||
/// This overlay will be applied on top of any reverts applied via `with_block_hash`.
|
||||
pub fn with_overlay_source(mut self, source: Option<OverlaySource>) -> Self {
|
||||
self.overlay_source = source;
|
||||
// Clear the overlay cache since we've updated the source.
|
||||
self.overlay_cache = Default::default();
|
||||
self
|
||||
}
|
||||
|
||||
@@ -137,6 +139,8 @@ impl<F> OverlayStateProviderFactory<F> {
|
||||
/// Convenience method that wraps the lazy overlay in `OverlaySource::Lazy`.
|
||||
pub fn with_lazy_overlay(mut self, lazy_overlay: Option<LazyOverlay>) -> Self {
|
||||
self.overlay_source = lazy_overlay.map(OverlaySource::Lazy);
|
||||
// Clear the overlay cache since we've updated the source.
|
||||
self.overlay_cache = Default::default();
|
||||
self
|
||||
}
|
||||
|
||||
@@ -152,6 +156,8 @@ impl<F> OverlayStateProviderFactory<F> {
|
||||
trie: Arc::new(TrieUpdatesSorted::default()),
|
||||
state,
|
||||
});
|
||||
// Clear the overlay cache since we've updated the source.
|
||||
self.overlay_cache = Default::default();
|
||||
}
|
||||
self
|
||||
}
|
||||
@@ -178,6 +184,8 @@ impl<F> OverlayStateProviderFactory<F> {
|
||||
});
|
||||
}
|
||||
}
|
||||
// Clear the overlay cache since we've updated the source.
|
||||
self.overlay_cache = Default::default();
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
@@ -528,7 +528,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
}
|
||||
|
||||
/// Writes headers for all blocks to the static file segment.
|
||||
#[instrument(level = "debug", target = "providers::db", skip_all)]
|
||||
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
|
||||
fn write_headers(
|
||||
w: &mut StaticFileProviderRWRefMut<'_, N>,
|
||||
blocks: &[ExecutedBlock<N>],
|
||||
@@ -541,7 +541,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
}
|
||||
|
||||
/// Writes transactions for all blocks to the static file segment.
|
||||
#[instrument(level = "debug", target = "providers::db", skip_all)]
|
||||
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
|
||||
fn write_transactions(
|
||||
w: &mut StaticFileProviderRWRefMut<'_, N>,
|
||||
blocks: &[ExecutedBlock<N>],
|
||||
@@ -558,7 +558,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
}
|
||||
|
||||
/// Writes transaction senders for all blocks to the static file segment.
|
||||
#[instrument(level = "debug", target = "providers::db", skip_all)]
|
||||
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
|
||||
fn write_transaction_senders(
|
||||
w: &mut StaticFileProviderRWRefMut<'_, N>,
|
||||
blocks: &[ExecutedBlock<N>],
|
||||
@@ -575,7 +575,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
}
|
||||
|
||||
/// Writes receipts for all blocks to the static file segment.
|
||||
#[instrument(level = "debug", target = "providers::db", skip_all)]
|
||||
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
|
||||
fn write_receipts(
|
||||
w: &mut StaticFileProviderRWRefMut<'_, N>,
|
||||
blocks: &[ExecutedBlock<N>],
|
||||
@@ -602,7 +602,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
}
|
||||
|
||||
/// Writes account changesets for all blocks to the static file segment.
|
||||
#[instrument(level = "debug", target = "providers::db", skip_all)]
|
||||
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
|
||||
fn write_account_changesets(
|
||||
w: &mut StaticFileProviderRWRefMut<'_, N>,
|
||||
blocks: &[ExecutedBlock<N>],
|
||||
@@ -647,7 +647,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
///
|
||||
/// This spawns separate threads for each segment type and each thread calls `sync_all()` on its
|
||||
/// writer when done.
|
||||
#[instrument(level = "debug", target = "providers::db", skip_all)]
|
||||
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
|
||||
pub fn write_blocks_data(
|
||||
&self,
|
||||
blocks: &[ExecutedBlock<N>],
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
use crate::{either_writer::RocksTxRefArg, providers::RocksDBProvider};
|
||||
use crate::{
|
||||
either_writer::{RawRocksDBBatch, RocksBatchArg, RocksTxRefArg},
|
||||
providers::RocksDBProvider,
|
||||
};
|
||||
use reth_storage_errors::provider::ProviderResult;
|
||||
|
||||
/// `RocksDB` provider factory.
|
||||
@@ -31,4 +34,28 @@ pub trait RocksDBProviderFactory {
|
||||
#[cfg(not(all(unix, feature = "rocksdb")))]
|
||||
f(())
|
||||
}
|
||||
|
||||
/// Executes a closure with a `RocksDB` batch, automatically registering it for commit.
|
||||
///
|
||||
/// This helper encapsulates all the cfg-gated `RocksDB` batch handling.
|
||||
fn with_rocksdb_batch<F, R>(&self, f: F) -> ProviderResult<R>
|
||||
where
|
||||
F: FnOnce(RocksBatchArg<'_>) -> ProviderResult<(R, Option<RawRocksDBBatch>)>,
|
||||
{
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
{
|
||||
let rocksdb = self.rocksdb_provider();
|
||||
let batch = rocksdb.batch();
|
||||
let (result, raw_batch) = f(batch)?;
|
||||
if let Some(b) = raw_batch {
|
||||
self.set_pending_rocksdb_batch(b);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
#[cfg(not(all(unix, feature = "rocksdb")))]
|
||||
{
|
||||
let (result, _) = f(())?;
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ serde_json = { workspace = true, optional = true }
|
||||
|
||||
[features]
|
||||
default = ["std"]
|
||||
edge = ["reth-db-api/edge"]
|
||||
std = [
|
||||
"reth-chainspec/std",
|
||||
"alloy-consensus/std",
|
||||
|
||||
@@ -746,6 +746,18 @@ impl<T: PoolTransaction> Default for AllPoolTransactions<T> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: PoolTransaction> IntoIterator for AllPoolTransactions<T> {
|
||||
type Item = Arc<ValidPoolTransaction<T>>;
|
||||
type IntoIter = std::iter::Chain<
|
||||
std::vec::IntoIter<Arc<ValidPoolTransaction<T>>>,
|
||||
std::vec::IntoIter<Arc<ValidPoolTransaction<T>>>,
|
||||
>;
|
||||
|
||||
fn into_iter(self) -> Self::IntoIter {
|
||||
self.pending.into_iter().chain(self.queued)
|
||||
}
|
||||
}
|
||||
|
||||
/// Represents transactions that were propagated over the network.
|
||||
#[derive(Debug, Clone, Eq, PartialEq, Default)]
|
||||
pub struct PropagatedTransactions(pub HashMap<TxHash, Vec<PropagateKind>>);
|
||||
|
||||
@@ -6,7 +6,7 @@ use crate::{
|
||||
utils::{extend_sorted_vec, kway_merge_sorted},
|
||||
KeyHasher, MultiProofTargets, Nibbles,
|
||||
};
|
||||
use alloc::{borrow::Cow, vec::Vec};
|
||||
use alloc::{borrow::Cow, sync::Arc, vec::Vec};
|
||||
use alloy_primitives::{
|
||||
keccak256,
|
||||
map::{hash_map, B256Map, HashMap, HashSet},
|
||||
@@ -638,31 +638,44 @@ impl HashedPostStateSorted {
|
||||
|
||||
/// Batch-merge sorted hashed post states. Iterator yields **newest to oldest**.
|
||||
///
|
||||
/// Uses k-way merge for O(n log k) complexity and one-pass accumulation for storages.
|
||||
pub fn merge_batch<'a>(states: impl IntoIterator<Item = &'a Self>) -> Self {
|
||||
let states: Vec<_> = states.into_iter().collect();
|
||||
if states.is_empty() {
|
||||
return Self::default();
|
||||
/// For small batches, uses `extend_ref_and_sort` loop.
|
||||
/// For large batches, uses k-way merge for O(n log k) complexity.
|
||||
pub fn merge_batch<T: AsRef<Self> + From<Self>>(iter: impl IntoIterator<Item = T>) -> T {
|
||||
const THRESHOLD: usize = 30;
|
||||
|
||||
let items: alloc::vec::Vec<_> = iter.into_iter().collect();
|
||||
let k = items.len();
|
||||
|
||||
if k == 0 {
|
||||
return Self::default().into();
|
||||
}
|
||||
if k == 1 {
|
||||
return items.into_iter().next().expect("k == 1");
|
||||
}
|
||||
|
||||
let accounts = kway_merge_sorted(states.iter().map(|s| s.accounts.as_slice()));
|
||||
if k < THRESHOLD {
|
||||
// Small k: extend loop, oldest-to-newest so newer overrides older.
|
||||
let mut iter = items.iter().rev();
|
||||
let mut acc = iter.next().expect("k > 0").as_ref().clone();
|
||||
for next in iter {
|
||||
acc.extend_ref_and_sort(next.as_ref());
|
||||
}
|
||||
return acc.into();
|
||||
}
|
||||
|
||||
// Large k: k-way merge.
|
||||
let accounts = kway_merge_sorted(items.iter().map(|i| i.as_ref().accounts.as_slice()));
|
||||
|
||||
struct StorageAcc<'a> {
|
||||
/// Account storage was cleared (e.g., SELFDESTRUCT).
|
||||
wiped: bool,
|
||||
/// Stop collecting older slices after seeing a wipe.
|
||||
sealed: bool,
|
||||
/// Storage slot slices to merge, ordered newest to oldest.
|
||||
slices: Vec<&'a [(B256, U256)]>,
|
||||
}
|
||||
|
||||
let mut acc: B256Map<StorageAcc<'_>> = B256Map::default();
|
||||
|
||||
// Accumulate storage slices per address from newest to oldest state.
|
||||
// Once we see a `wiped` flag, the account was cleared at that point,
|
||||
// so older storage slots are irrelevant - we "seal" and stop collecting.
|
||||
for state in &states {
|
||||
for (addr, storage) in &state.storages {
|
||||
for item in &items {
|
||||
for (addr, storage) in &item.as_ref().storages {
|
||||
let entry = acc.entry(*addr).or_insert_with(|| StorageAcc {
|
||||
wiped: false,
|
||||
sealed: false,
|
||||
@@ -689,7 +702,7 @@ impl HashedPostStateSorted {
|
||||
})
|
||||
.collect();
|
||||
|
||||
Self { accounts, storages }
|
||||
Self { accounts, storages }.into()
|
||||
}
|
||||
|
||||
/// Clears all accounts and storage data.
|
||||
@@ -697,6 +710,36 @@ impl HashedPostStateSorted {
|
||||
self.accounts.clear();
|
||||
self.storages.clear();
|
||||
}
|
||||
|
||||
/// Parallel batch-merge sorted hashed post states. Slice is **oldest to newest**.
|
||||
///
|
||||
/// This is more efficient than sequential `extend_ref` calls when merging many states,
|
||||
/// as it processes all states in parallel with tree reduction using divide-and-conquer.
|
||||
#[cfg(feature = "rayon")]
|
||||
pub fn merge_parallel(states: &[Arc<Self>]) -> Self {
|
||||
fn parallel_merge_tree(states: &[Arc<HashedPostStateSorted>]) -> HashedPostStateSorted {
|
||||
match states.len() {
|
||||
0 => HashedPostStateSorted::default(),
|
||||
1 => states[0].as_ref().clone(),
|
||||
2 => {
|
||||
let mut acc = states[0].as_ref().clone();
|
||||
acc.extend_ref_and_sort(&states[1]);
|
||||
acc
|
||||
}
|
||||
n => {
|
||||
let mid = n / 2;
|
||||
let (mut left, right) = rayon::join(
|
||||
|| parallel_merge_tree(&states[..mid]),
|
||||
|| parallel_merge_tree(&states[mid..]),
|
||||
);
|
||||
left.extend_ref_and_sort(&right);
|
||||
left
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
parallel_merge_tree(states)
|
||||
}
|
||||
}
|
||||
|
||||
impl AsRef<Self> for HashedPostStateSorted {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Merkle trie proofs.
|
||||
|
||||
use crate::{BranchNodeMasksMap, Nibbles, TrieAccount};
|
||||
use crate::{BranchNodeMasksMap, Nibbles, ProofTrieNode, TrieAccount};
|
||||
use alloc::{borrow::Cow, vec::Vec};
|
||||
use alloy_consensus::constants::KECCAK_EMPTY;
|
||||
use alloy_primitives::{
|
||||
@@ -267,6 +267,12 @@ impl MultiProof {
|
||||
self.account_subtree.extend_from(other.account_subtree);
|
||||
self.branch_node_masks.extend(other.branch_node_masks);
|
||||
|
||||
let reserve = if self.storages.is_empty() {
|
||||
other.storages.len()
|
||||
} else {
|
||||
other.storages.len().div_ceil(2)
|
||||
};
|
||||
self.storages.reserve(reserve);
|
||||
for (hashed_address, storage) in other.storages {
|
||||
match self.storages.entry(hashed_address) {
|
||||
hash_map::Entry::Occupied(mut entry) => {
|
||||
@@ -390,6 +396,12 @@ impl DecodedMultiProof {
|
||||
self.account_subtree.extend_from(other.account_subtree);
|
||||
self.branch_node_masks.extend(other.branch_node_masks);
|
||||
|
||||
let reserve = if self.storages.is_empty() {
|
||||
other.storages.len()
|
||||
} else {
|
||||
other.storages.len().div_ceil(2)
|
||||
};
|
||||
self.storages.reserve(reserve);
|
||||
for (hashed_address, storage) in other.storages {
|
||||
match self.storages.entry(hashed_address) {
|
||||
hash_map::Entry::Occupied(mut entry) => {
|
||||
@@ -431,6 +443,33 @@ impl TryFrom<MultiProof> for DecodedMultiProof {
|
||||
}
|
||||
}
|
||||
|
||||
/// V2 decoded multiproof which contains the results of both account and storage V2 proof
|
||||
/// calculations.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Default)]
|
||||
pub struct DecodedMultiProofV2 {
|
||||
/// Account trie proof nodes
|
||||
pub account_proofs: Vec<ProofTrieNode>,
|
||||
/// Storage trie proof nodes indexed by account
|
||||
pub storage_proofs: B256Map<Vec<ProofTrieNode>>,
|
||||
}
|
||||
|
||||
impl DecodedMultiProofV2 {
|
||||
/// Returns true if there are no proofs
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.account_proofs.is_empty() && self.storage_proofs.is_empty()
|
||||
}
|
||||
|
||||
/// Appends the given multiproof's data to this one.
|
||||
///
|
||||
/// This implementation does not deduplicate redundant proofs.
|
||||
pub fn extend(&mut self, other: Self) {
|
||||
self.account_proofs.extend(other.account_proofs);
|
||||
for (hashed_address, other_storage_proofs) in other.storage_proofs {
|
||||
self.storage_proofs.entry(hashed_address).or_default().extend(other_storage_proofs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The merkle multiproof of storage trie.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct StorageMultiProof {
|
||||
|
||||
@@ -4,6 +4,7 @@ use crate::{
|
||||
};
|
||||
use alloc::{
|
||||
collections::{btree_map::BTreeMap, btree_set::BTreeSet},
|
||||
sync::Arc,
|
||||
vec::Vec,
|
||||
};
|
||||
use alloy_primitives::{
|
||||
@@ -206,7 +207,7 @@ impl TrieUpdates {
|
||||
}
|
||||
|
||||
/// Converts trie updates into [`TrieUpdatesSortedRef`].
|
||||
pub fn into_sorted_ref<'a>(&'a self) -> TrieUpdatesSortedRef<'a> {
|
||||
pub fn into_sorted_ref(&self) -> TrieUpdatesSortedRef<'_> {
|
||||
let mut account_nodes = self.account_nodes.iter().collect::<Vec<_>>();
|
||||
account_nodes.sort_unstable_by(|a, b| a.0.cmp(b.0));
|
||||
|
||||
@@ -216,7 +217,7 @@ impl TrieUpdates {
|
||||
storage_tries: self
|
||||
.storage_tries
|
||||
.iter()
|
||||
.map(|m| (*m.0, m.1.into_sorted_ref().clone()))
|
||||
.map(|m| (*m.0, m.1.into_sorted_ref()))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
@@ -629,48 +630,57 @@ impl TrieUpdatesSorted {
|
||||
|
||||
/// Batch-merge sorted trie updates. Iterator yields **newest to oldest**.
|
||||
///
|
||||
/// This is more efficient than repeated `extend_ref` calls for large batches,
|
||||
/// using k-way merge for O(n log k) complexity instead of O(n * k).
|
||||
pub fn merge_batch<'a>(updates: impl IntoIterator<Item = &'a Self>) -> Self {
|
||||
let updates: Vec<_> = updates.into_iter().collect();
|
||||
if updates.is_empty() {
|
||||
return Self::default();
|
||||
/// For small batches, uses `extend_ref_and_sort` loop.
|
||||
/// For large batches, uses k-way merge for O(n log k) complexity.
|
||||
pub fn merge_batch<T: AsRef<Self> + From<Self>>(iter: impl IntoIterator<Item = T>) -> T {
|
||||
const THRESHOLD: usize = 30;
|
||||
|
||||
let items: alloc::vec::Vec<_> = iter.into_iter().collect();
|
||||
let k = items.len();
|
||||
|
||||
if k == 0 {
|
||||
return Self::default().into();
|
||||
}
|
||||
if k == 1 {
|
||||
return items.into_iter().next().expect("k == 1");
|
||||
}
|
||||
|
||||
// Merge account nodes using k-way merge. Newest (index 0) takes precedence.
|
||||
let account_nodes = kway_merge_sorted(updates.iter().map(|u| u.account_nodes.as_slice()));
|
||||
if k < THRESHOLD {
|
||||
// Small k: extend loop, oldest-to-newest so newer overrides older.
|
||||
let mut iter = items.iter().rev();
|
||||
let mut acc = iter.next().expect("k > 0").as_ref().clone();
|
||||
for next in iter {
|
||||
acc.extend_ref_and_sort(next.as_ref());
|
||||
}
|
||||
return acc.into();
|
||||
}
|
||||
|
||||
// Large k: k-way merge.
|
||||
let account_nodes =
|
||||
kway_merge_sorted(items.iter().map(|i| i.as_ref().account_nodes.as_slice()));
|
||||
|
||||
// Accumulator for collecting storage trie slices per address.
|
||||
// We process updates newest-to-oldest and stop collecting for an address
|
||||
// once we hit a "deleted" storage (sealed=true), since older data is irrelevant.
|
||||
struct StorageAcc<'a> {
|
||||
/// Storage trie was deleted (account removed or cleared).
|
||||
is_deleted: bool,
|
||||
/// Stop collecting older slices after seeing a deletion.
|
||||
sealed: bool,
|
||||
/// Storage trie node slices to merge, ordered newest to oldest.
|
||||
slices: Vec<&'a [(Nibbles, Option<BranchNodeCompact>)]>,
|
||||
}
|
||||
|
||||
let mut acc: B256Map<StorageAcc<'_>> = B256Map::default();
|
||||
|
||||
// Collect storage slices per address, respecting deletion boundaries
|
||||
for update in &updates {
|
||||
for (addr, storage) in &update.storage_tries {
|
||||
for item in &items {
|
||||
for (addr, storage) in &item.as_ref().storage_tries {
|
||||
let entry = acc.entry(*addr).or_insert_with(|| StorageAcc {
|
||||
is_deleted: false,
|
||||
sealed: false,
|
||||
slices: Vec::new(),
|
||||
});
|
||||
|
||||
// Skip if we already hit a deletion for this address (older data is irrelevant)
|
||||
if entry.sealed {
|
||||
continue;
|
||||
}
|
||||
|
||||
entry.slices.push(storage.storage_nodes.as_slice());
|
||||
|
||||
// If this storage was deleted, mark as deleted and seal to ignore older updates
|
||||
if storage.is_deleted {
|
||||
entry.is_deleted = true;
|
||||
entry.sealed = true;
|
||||
@@ -678,7 +688,6 @@ impl TrieUpdatesSorted {
|
||||
}
|
||||
}
|
||||
|
||||
// Merge each address's storage slices using k-way merge
|
||||
let storage_tries = acc
|
||||
.into_iter()
|
||||
.map(|(addr, entry)| {
|
||||
@@ -687,7 +696,37 @@ impl TrieUpdatesSorted {
|
||||
})
|
||||
.collect();
|
||||
|
||||
Self { account_nodes, storage_tries }
|
||||
Self { account_nodes, storage_tries }.into()
|
||||
}
|
||||
|
||||
/// Parallel batch-merge sorted trie updates. Slice is **oldest to newest**.
|
||||
///
|
||||
/// This is more efficient than sequential `extend_ref` calls when merging many updates,
|
||||
/// as it processes all updates in parallel with tree reduction using divide-and-conquer.
|
||||
#[cfg(feature = "rayon")]
|
||||
pub fn merge_parallel(updates: &[Arc<Self>]) -> Self {
|
||||
fn parallel_merge_tree(updates: &[Arc<TrieUpdatesSorted>]) -> TrieUpdatesSorted {
|
||||
match updates.len() {
|
||||
0 => TrieUpdatesSorted::default(),
|
||||
1 => updates[0].as_ref().clone(),
|
||||
2 => {
|
||||
let mut acc = updates[0].as_ref().clone();
|
||||
acc.extend_ref_and_sort(&updates[1]);
|
||||
acc
|
||||
}
|
||||
n => {
|
||||
let mid = n / 2;
|
||||
let (mut left, right) = rayon::join(
|
||||
|| parallel_merge_tree(&updates[..mid]),
|
||||
|| parallel_merge_tree(&updates[mid..]),
|
||||
);
|
||||
left.extend_ref_and_sort(&right);
|
||||
left
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
parallel_merge_tree(updates)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ workspace = true
|
||||
[dependencies]
|
||||
# reth
|
||||
reth-execution-errors.workspace = true
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-storage-errors.workspace = true
|
||||
reth-trie-common.workspace = true
|
||||
|
||||
@@ -22,6 +22,12 @@ pub mod proof;
|
||||
|
||||
pub mod proof_task;
|
||||
|
||||
/// Async value encoder for V2 proofs.
|
||||
pub(crate) mod value_encoder;
|
||||
|
||||
/// V2 multiproof targets and chunking.
|
||||
pub mod targets_v2;
|
||||
|
||||
/// Parallel state root metrics.
|
||||
#[cfg(feature = "metrics")]
|
||||
pub mod metrics;
|
||||
|
||||
@@ -41,7 +41,7 @@ use alloy_primitives::{
|
||||
use alloy_rlp::{BufMut, Encodable};
|
||||
use crossbeam_channel::{unbounded, Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
|
||||
use dashmap::DashMap;
|
||||
use reth_execution_errors::{SparseTrieError, SparseTrieErrorKind};
|
||||
use reth_execution_errors::{SparseTrieError, SparseTrieErrorKind, StateProofError};
|
||||
use reth_provider::{DatabaseProviderROFactory, ProviderError, ProviderResult};
|
||||
use reth_storage_errors::db::DatabaseError;
|
||||
use reth_trie::{
|
||||
@@ -305,17 +305,17 @@ impl ProofWorkerHandle {
|
||||
self.storage_work_tx
|
||||
.send(StorageWorkerJob::StorageProof { input, proof_result_sender })
|
||||
.map_err(|err| {
|
||||
let error =
|
||||
ProviderError::other(std::io::Error::other("storage workers unavailable"));
|
||||
|
||||
if let StorageWorkerJob::StorageProof { proof_result_sender, .. } = err.0 {
|
||||
let _ = proof_result_sender.send(StorageProofResultMessage {
|
||||
hashed_address,
|
||||
result: Err(ParallelStateRootError::Provider(error.clone())),
|
||||
result: Err(DatabaseError::Other(
|
||||
"storage workers unavailable".to_string(),
|
||||
)
|
||||
.into()),
|
||||
});
|
||||
}
|
||||
|
||||
error
|
||||
ProviderError::other(std::io::Error::other("storage workers unavailable"))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -432,7 +432,7 @@ where
|
||||
input: StorageProofInput,
|
||||
trie_cursor_metrics: &mut TrieCursorMetricsCache,
|
||||
hashed_cursor_metrics: &mut HashedCursorMetricsCache,
|
||||
) -> Result<StorageProofResult, ParallelStateRootError> {
|
||||
) -> Result<StorageProofResult, StateProofError> {
|
||||
// Consume the input so we can move large collections (e.g. target slots) without cloning.
|
||||
let StorageProofInput::Legacy {
|
||||
hashed_address,
|
||||
@@ -469,20 +469,13 @@ where
|
||||
.with_added_removed_keys(added_removed_keys)
|
||||
.with_trie_cursor_metrics(trie_cursor_metrics)
|
||||
.with_hashed_cursor_metrics(hashed_cursor_metrics)
|
||||
.storage_multiproof(target_slots)
|
||||
.map_err(|e| ParallelStateRootError::Other(e.to_string()));
|
||||
.storage_multiproof(target_slots);
|
||||
trie_cursor_metrics.record_span("trie_cursor");
|
||||
hashed_cursor_metrics.record_span("hashed_cursor");
|
||||
|
||||
// Decode proof into DecodedStorageMultiProof
|
||||
let decoded_result = raw_proof_result.and_then(|raw_proof| {
|
||||
raw_proof.try_into().map_err(|e: alloy_rlp::Error| {
|
||||
ParallelStateRootError::Other(format!(
|
||||
"Failed to decode storage proof for {}: {}",
|
||||
hashed_address, e
|
||||
))
|
||||
})
|
||||
})?;
|
||||
let decoded_result =
|
||||
raw_proof_result.and_then(|raw_proof| raw_proof.try_into().map_err(Into::into))?;
|
||||
|
||||
trace!(
|
||||
target: "trie::proof_task",
|
||||
@@ -502,7 +495,7 @@ where
|
||||
<Provider as TrieCursorFactory>::StorageTrieCursor<'_>,
|
||||
<Provider as HashedCursorFactory>::StorageCursor<'_>,
|
||||
>,
|
||||
) -> Result<StorageProofResult, ParallelStateRootError> {
|
||||
) -> Result<StorageProofResult, StateProofError> {
|
||||
let StorageProofInput::V2 { hashed_address, mut targets } = input else {
|
||||
panic!("compute_v2_storage_proof only accepts StorageProofInput::V2")
|
||||
};
|
||||
@@ -717,12 +710,12 @@ pub struct StorageProofResultMessage {
|
||||
/// The hashed address this storage proof belongs to
|
||||
pub(crate) hashed_address: B256,
|
||||
/// The storage proof calculation result
|
||||
pub(crate) result: Result<StorageProofResult, ParallelStateRootError>,
|
||||
pub(crate) result: Result<StorageProofResult, StateProofError>,
|
||||
}
|
||||
|
||||
/// Internal message for storage workers.
|
||||
#[derive(Debug)]
|
||||
enum StorageWorkerJob {
|
||||
pub(crate) enum StorageWorkerJob {
|
||||
/// Storage proof computation request
|
||||
StorageProof {
|
||||
/// Storage proof input parameters
|
||||
@@ -1562,8 +1555,11 @@ fn dispatch_storage_proofs(
|
||||
let mut storage_proof_receivers =
|
||||
B256Map::with_capacity_and_hasher(targets.len(), Default::default());
|
||||
|
||||
let mut sorted_targets: Vec<_> = targets.iter().collect();
|
||||
sorted_targets.sort_unstable_by_key(|(addr, _)| *addr);
|
||||
|
||||
// Dispatch all storage proofs to worker pool
|
||||
for (hashed_address, target_slots) in targets.iter() {
|
||||
for (hashed_address, target_slots) in sorted_targets {
|
||||
// Create channel for receiving ProofResultMessage
|
||||
let (result_tx, result_rx) = crossbeam_channel::unbounded();
|
||||
|
||||
|
||||
148
crates/trie/parallel/src/targets_v2.rs
Normal file
148
crates/trie/parallel/src/targets_v2.rs
Normal file
@@ -0,0 +1,148 @@
|
||||
//! V2 multiproof targets and chunking.
|
||||
|
||||
use alloy_primitives::{map::B256Map, B256};
|
||||
use reth_trie::proof_v2;
|
||||
|
||||
/// A set of account and storage V2 proof targets. The account and storage targets do not need to
|
||||
/// necessarily overlap.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct MultiProofTargetsV2 {
|
||||
/// The set of account proof targets to generate proofs for.
|
||||
pub account_targets: Vec<proof_v2::Target>,
|
||||
/// The sets of storage proof targets to generate proofs for.
|
||||
pub storage_targets: B256Map<Vec<proof_v2::Target>>,
|
||||
}
|
||||
|
||||
impl MultiProofTargetsV2 {
|
||||
/// Returns true is there are no account or storage targets.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.account_targets.is_empty() && self.storage_targets.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// An iterator that yields chunks of V2 proof targets of at most `size` account and storage
|
||||
/// targets.
|
||||
///
|
||||
/// Unlike legacy chunking, V2 preserves account targets exactly as they were (with their `min_len`
|
||||
/// metadata). Account targets must appear in a chunk. Storage targets for those accounts are
|
||||
/// chunked together, but if they exceed the chunk size, subsequent chunks contain only the
|
||||
/// remaining storage targets without repeating the account target.
|
||||
#[derive(Debug)]
|
||||
pub struct ChunkedMultiProofTargetsV2 {
|
||||
/// Remaining account targets to process
|
||||
account_targets: std::vec::IntoIter<proof_v2::Target>,
|
||||
/// Storage targets by account address
|
||||
storage_targets: B256Map<Vec<proof_v2::Target>>,
|
||||
/// Current account being processed (if any storage slots remain)
|
||||
current_account_storage: Option<(B256, std::vec::IntoIter<proof_v2::Target>)>,
|
||||
/// Chunk size
|
||||
size: usize,
|
||||
}
|
||||
|
||||
impl ChunkedMultiProofTargetsV2 {
|
||||
/// Creates a new chunked iterator for the given targets.
|
||||
pub fn new(targets: MultiProofTargetsV2, size: usize) -> Self {
|
||||
Self {
|
||||
account_targets: targets.account_targets.into_iter(),
|
||||
storage_targets: targets.storage_targets,
|
||||
current_account_storage: None,
|
||||
size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Iterator for ChunkedMultiProofTargetsV2 {
|
||||
type Item = MultiProofTargetsV2;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
let mut chunk = MultiProofTargetsV2::default();
|
||||
let mut count = 0;
|
||||
|
||||
// First, finish any remaining storage slots from previous account
|
||||
if let Some((account_addr, ref mut storage_iter)) = self.current_account_storage {
|
||||
let remaining_capacity = self.size - count;
|
||||
let slots: Vec<_> = storage_iter.by_ref().take(remaining_capacity).collect();
|
||||
|
||||
count += slots.len();
|
||||
chunk.storage_targets.insert(account_addr, slots);
|
||||
|
||||
// If iterator is exhausted, clear current_account_storage
|
||||
if storage_iter.len() == 0 {
|
||||
self.current_account_storage = None;
|
||||
}
|
||||
}
|
||||
|
||||
// Process account targets and their storage
|
||||
while count < self.size {
|
||||
let Some(account_target) = self.account_targets.next() else {
|
||||
break;
|
||||
};
|
||||
|
||||
// Add the account target
|
||||
chunk.account_targets.push(account_target);
|
||||
count += 1;
|
||||
|
||||
// Check if this account has storage targets
|
||||
let account_addr = account_target.key();
|
||||
if let Some(storage_slots) = self.storage_targets.remove(&account_addr) {
|
||||
let remaining_capacity = self.size - count;
|
||||
|
||||
if storage_slots.len() <= remaining_capacity {
|
||||
// Optimization: We can take all slots, just move the vec
|
||||
count += storage_slots.len();
|
||||
chunk.storage_targets.insert(account_addr, storage_slots);
|
||||
} else {
|
||||
// We need to split the storage slots
|
||||
let mut storage_iter = storage_slots.into_iter();
|
||||
let slots_in_chunk: Vec<_> =
|
||||
storage_iter.by_ref().take(remaining_capacity).collect();
|
||||
count += slots_in_chunk.len();
|
||||
|
||||
chunk.storage_targets.insert(account_addr, slots_in_chunk);
|
||||
|
||||
// Save remaining storage slots for next chunk
|
||||
self.current_account_storage = Some((account_addr, storage_iter));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process any remaining storage-only entries (accounts not in account_targets)
|
||||
while let Some((account_addr, storage_slots)) = self.storage_targets.iter_mut().next() &&
|
||||
count < self.size
|
||||
{
|
||||
let account_addr = *account_addr;
|
||||
let storage_slots = std::mem::take(storage_slots);
|
||||
let remaining_capacity = self.size - count;
|
||||
|
||||
// Always remove from the map - if there are remaining slots they go to
|
||||
// current_account_storage
|
||||
self.storage_targets.remove(&account_addr);
|
||||
|
||||
if storage_slots.len() <= remaining_capacity {
|
||||
// Optimization: We can take all slots, just move the vec
|
||||
count += storage_slots.len();
|
||||
chunk.storage_targets.insert(account_addr, storage_slots);
|
||||
} else {
|
||||
// We need to split the storage slots
|
||||
let mut storage_iter = storage_slots.into_iter();
|
||||
let slots_in_chunk: Vec<_> =
|
||||
storage_iter.by_ref().take(remaining_capacity).collect();
|
||||
|
||||
chunk.storage_targets.insert(account_addr, slots_in_chunk);
|
||||
|
||||
// Save remaining storage slots for next chunk
|
||||
if storage_iter.len() > 0 {
|
||||
self.current_account_storage = Some((account_addr, storage_iter));
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if chunk.account_targets.is_empty() && chunk.storage_targets.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(chunk)
|
||||
}
|
||||
}
|
||||
}
|
||||
185
crates/trie/parallel/src/value_encoder.rs
Normal file
185
crates/trie/parallel/src/value_encoder.rs
Normal file
@@ -0,0 +1,185 @@
|
||||
use crate::proof_task::{
|
||||
StorageProofInput, StorageProofResult, StorageProofResultMessage, StorageWorkerJob,
|
||||
};
|
||||
use alloy_primitives::{map::B256Map, B256};
|
||||
use alloy_rlp::Encodable;
|
||||
use core::cell::RefCell;
|
||||
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
|
||||
use dashmap::DashMap;
|
||||
use reth_execution_errors::trie::StateProofError;
|
||||
use reth_primitives_traits::Account;
|
||||
use reth_storage_errors::db::DatabaseError;
|
||||
use reth_trie::{
|
||||
proof_v2::{DeferredValueEncoder, LeafValueEncoder, Target},
|
||||
ProofTrieNode,
|
||||
};
|
||||
use std::{rc::Rc, sync::Arc};
|
||||
|
||||
/// Returned from [`AsyncAccountValueEncoder`], used to track an async storage root calculation.
|
||||
pub(crate) enum AsyncAccountDeferredValueEncoder {
|
||||
Dispatched {
|
||||
hashed_address: B256,
|
||||
account: Account,
|
||||
proof_result_rx: Result<CrossbeamReceiver<StorageProofResultMessage>, DatabaseError>,
|
||||
// None if results shouldn't be retained for this dispatched proof.
|
||||
storage_proof_results: Option<Rc<RefCell<B256Map<Vec<ProofTrieNode>>>>>,
|
||||
},
|
||||
FromCache {
|
||||
account: Account,
|
||||
root: B256,
|
||||
},
|
||||
}
|
||||
|
||||
impl DeferredValueEncoder for AsyncAccountDeferredValueEncoder {
|
||||
fn encode(self, buf: &mut Vec<u8>) -> Result<(), StateProofError> {
|
||||
let (account, root) = match self {
|
||||
Self::Dispatched {
|
||||
hashed_address,
|
||||
account,
|
||||
proof_result_rx,
|
||||
storage_proof_results,
|
||||
} => {
|
||||
let result = proof_result_rx?
|
||||
.recv()
|
||||
.map_err(|_| {
|
||||
StateProofError::Database(DatabaseError::Other(format!(
|
||||
"Storage proof channel closed for {hashed_address:?}",
|
||||
)))
|
||||
})?
|
||||
.result?;
|
||||
|
||||
let StorageProofResult::V2 { root: Some(root), proof } = result else {
|
||||
panic!("StorageProofResult is not V2 with root: {result:?}")
|
||||
};
|
||||
|
||||
if let Some(storage_proof_results) = storage_proof_results.as_ref() {
|
||||
storage_proof_results.borrow_mut().insert(hashed_address, proof);
|
||||
}
|
||||
|
||||
(account, root)
|
||||
}
|
||||
Self::FromCache { account, root } => (account, root),
|
||||
};
|
||||
|
||||
let account = account.into_trie_account(root);
|
||||
account.encode(buf);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Implements the [`LeafValueEncoder`] trait for accounts using a [`CrossbeamSender`] to dispatch
|
||||
/// and compute storage roots asynchronously. Can also accept a set of already dispatched account
|
||||
/// storage proofs, for cases where it's possible to determine some necessary accounts ahead of
|
||||
/// time.
|
||||
pub(crate) struct AsyncAccountValueEncoder {
|
||||
storage_work_tx: CrossbeamSender<StorageWorkerJob>,
|
||||
/// Storage proof jobs which were dispatched ahead of time.
|
||||
dispatched: B256Map<CrossbeamReceiver<StorageProofResultMessage>>,
|
||||
/// Storage roots which have already been computed. This can be used only if a storage proof
|
||||
/// wasn't dispatched for an account, otherwise we must consume the proof result.
|
||||
cached_storage_roots: Arc<DashMap<B256, B256>>,
|
||||
/// Tracks storage proof results received from the storage workers. [`Rc`] + [`RefCell`] is
|
||||
/// required because [`DeferredValueEncoder`] cannot have a lifetime.
|
||||
storage_proof_results: Rc<RefCell<B256Map<Vec<ProofTrieNode>>>>,
|
||||
}
|
||||
|
||||
impl AsyncAccountValueEncoder {
|
||||
/// Initializes a [`Self`] using a `ProofWorkerHandle` which will be used to calculate storage
|
||||
/// roots asynchronously.
|
||||
#[expect(dead_code)]
|
||||
pub(crate) fn new(
|
||||
storage_work_tx: CrossbeamSender<StorageWorkerJob>,
|
||||
dispatched: B256Map<CrossbeamReceiver<StorageProofResultMessage>>,
|
||||
cached_storage_roots: Arc<DashMap<B256, B256>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
storage_work_tx,
|
||||
dispatched,
|
||||
cached_storage_roots,
|
||||
storage_proof_results: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Consume [`Self`] and return all collected storage proofs which had been dispatched.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// This method panics if any deferred encoders produced by [`Self::deferred_encoder`] have not
|
||||
/// been dropped.
|
||||
#[expect(dead_code)]
|
||||
pub(crate) fn into_storage_proofs(
|
||||
self,
|
||||
) -> Result<B256Map<Vec<ProofTrieNode>>, StateProofError> {
|
||||
let mut storage_proof_results = Rc::into_inner(self.storage_proof_results)
|
||||
.expect("no deferred encoders are still allocated")
|
||||
.into_inner();
|
||||
|
||||
// Any remaining dispatched proofs need to have their results collected
|
||||
for (hashed_address, rx) in &self.dispatched {
|
||||
let result = rx
|
||||
.recv()
|
||||
.map_err(|_| {
|
||||
StateProofError::Database(DatabaseError::Other(format!(
|
||||
"Storage proof channel closed for {hashed_address:?}",
|
||||
)))
|
||||
})?
|
||||
.result?;
|
||||
|
||||
let StorageProofResult::V2 { proof, .. } = result else {
|
||||
panic!("StorageProofResult is not V2: {result:?}")
|
||||
};
|
||||
|
||||
storage_proof_results.insert(*hashed_address, proof);
|
||||
}
|
||||
|
||||
Ok(storage_proof_results)
|
||||
}
|
||||
}
|
||||
|
||||
impl LeafValueEncoder for AsyncAccountValueEncoder {
|
||||
type Value = Account;
|
||||
type DeferredEncoder = AsyncAccountDeferredValueEncoder;
|
||||
|
||||
fn deferred_encoder(
|
||||
&mut self,
|
||||
hashed_address: B256,
|
||||
account: Self::Value,
|
||||
) -> Self::DeferredEncoder {
|
||||
// If the proof job has already been dispatched for this account then it's not necessary to
|
||||
// dispatch another.
|
||||
if let Some(rx) = self.dispatched.remove(&hashed_address) {
|
||||
return AsyncAccountDeferredValueEncoder::Dispatched {
|
||||
hashed_address,
|
||||
account,
|
||||
proof_result_rx: Ok(rx),
|
||||
storage_proof_results: Some(self.storage_proof_results.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
// If the address didn't have a job dispatched for it then we can assume it has no targets,
|
||||
// and we only need its root.
|
||||
|
||||
// If the root is already calculated then just use it directly
|
||||
if let Some(root) = self.cached_storage_roots.get(&hashed_address) {
|
||||
return AsyncAccountDeferredValueEncoder::FromCache { account, root: *root }
|
||||
}
|
||||
|
||||
// Create a proof input which targets a bogus key, so that we calculate the root as a
|
||||
// side-effect.
|
||||
let input = StorageProofInput::new(hashed_address, vec![Target::new(B256::ZERO)]);
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
let proof_result_rx = self
|
||||
.storage_work_tx
|
||||
.send(StorageWorkerJob::StorageProof { input, proof_result_sender: tx })
|
||||
.map_err(|_| DatabaseError::Other("storage workers unavailable".to_string()))
|
||||
.map(|_| rx);
|
||||
|
||||
AsyncAccountDeferredValueEncoder::Dispatched {
|
||||
hashed_address,
|
||||
account,
|
||||
proof_result_rx,
|
||||
storage_proof_results: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -316,6 +316,86 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Reveals a V2 decoded multiproof.
|
||||
///
|
||||
/// V2 multiproofs use a simpler format where proof nodes are stored as vectors rather than
|
||||
/// hashmaps, with masks already included in the `ProofTrieNode` structure.
|
||||
#[instrument(
|
||||
skip_all,
|
||||
fields(
|
||||
account_nodes = multiproof.account_proofs.len(),
|
||||
storages = multiproof.storage_proofs.len()
|
||||
)
|
||||
)]
|
||||
pub fn reveal_decoded_multiproof_v2(
|
||||
&mut self,
|
||||
multiproof: reth_trie_common::DecodedMultiProofV2,
|
||||
) -> SparseStateTrieResult<()> {
|
||||
// Reveal the account proof nodes
|
||||
self.reveal_account_v2_proof_nodes(multiproof.account_proofs)?;
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
// If nostd then serially reveal storage proof nodes for each storage trie
|
||||
{
|
||||
for (account, storage_proofs) in multiproof.storage_proofs {
|
||||
self.reveal_storage_v2_proof_nodes(account, storage_proofs)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "std")]
|
||||
// If std then reveal storage proofs in parallel
|
||||
{
|
||||
use rayon::iter::{ParallelBridge, ParallelIterator};
|
||||
|
||||
let retain_updates = self.retain_updates;
|
||||
|
||||
// Process all storage trie revealings in parallel, having first removed the
|
||||
// `reveal_nodes` tracking and `SparseTrie`s for each account from their HashMaps.
|
||||
// These will be returned after processing.
|
||||
let results: Vec<_> = multiproof
|
||||
.storage_proofs
|
||||
.into_iter()
|
||||
.map(|(account, storage_proofs)| {
|
||||
let revealed_nodes = self.storage.take_or_create_revealed_paths(&account);
|
||||
let trie = self.storage.take_or_create_trie(&account);
|
||||
(account, storage_proofs, revealed_nodes, trie)
|
||||
})
|
||||
.par_bridge()
|
||||
.map(|(account, storage_proofs, mut revealed_nodes, mut trie)| {
|
||||
let result = Self::reveal_storage_v2_proof_nodes_inner(
|
||||
account,
|
||||
storage_proofs,
|
||||
&mut revealed_nodes,
|
||||
&mut trie,
|
||||
retain_updates,
|
||||
);
|
||||
(account, result, revealed_nodes, trie)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut any_err = Ok(());
|
||||
for (account, result, revealed_nodes, trie) in results {
|
||||
self.storage.revealed_paths.insert(account, revealed_nodes);
|
||||
self.storage.tries.insert(account, trie);
|
||||
if let Ok(_metric_values) = result {
|
||||
#[cfg(feature = "metrics")]
|
||||
{
|
||||
self.metrics
|
||||
.increment_total_storage_nodes(_metric_values.total_nodes as u64);
|
||||
self.metrics
|
||||
.increment_skipped_storage_nodes(_metric_values.skipped_nodes as u64);
|
||||
}
|
||||
} else {
|
||||
any_err = result.map(|_| ());
|
||||
}
|
||||
}
|
||||
|
||||
any_err
|
||||
}
|
||||
}
|
||||
|
||||
/// Reveals an account multiproof.
|
||||
pub fn reveal_account_multiproof(
|
||||
&mut self,
|
||||
@@ -362,6 +442,89 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reveals account proof nodes from a V2 proof.
|
||||
///
|
||||
/// V2 proofs already include the masks in the `ProofTrieNode` structure,
|
||||
/// so no separate masks map is needed.
|
||||
pub fn reveal_account_v2_proof_nodes(
|
||||
&mut self,
|
||||
nodes: Vec<ProofTrieNode>,
|
||||
) -> SparseStateTrieResult<()> {
|
||||
let FilteredV2ProofNodes { root_node, nodes, new_nodes, metric_values: _metric_values } =
|
||||
filter_revealed_v2_proof_nodes(nodes, &mut self.revealed_account_paths)?;
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
{
|
||||
self.metrics.increment_total_account_nodes(_metric_values.total_nodes as u64);
|
||||
self.metrics.increment_skipped_account_nodes(_metric_values.skipped_nodes as u64);
|
||||
}
|
||||
|
||||
if let Some(root_node) = root_node {
|
||||
trace!(target: "trie::sparse", ?root_node, "Revealing root account node from V2 proof");
|
||||
let trie =
|
||||
self.state.reveal_root(root_node.node, root_node.masks, self.retain_updates)?;
|
||||
|
||||
trie.reserve_nodes(new_nodes);
|
||||
|
||||
trace!(target: "trie::sparse", total_nodes = ?nodes.len(), "Revealing account nodes from V2 proof");
|
||||
trie.reveal_nodes(nodes)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reveals storage proof nodes from a V2 proof for the given address.
|
||||
///
|
||||
/// V2 proofs already include the masks in the `ProofTrieNode` structure,
|
||||
/// so no separate masks map is needed.
|
||||
pub fn reveal_storage_v2_proof_nodes(
|
||||
&mut self,
|
||||
account: B256,
|
||||
nodes: Vec<ProofTrieNode>,
|
||||
) -> SparseStateTrieResult<()> {
|
||||
let (trie, revealed_paths) = self.storage.get_trie_and_revealed_paths_mut(account);
|
||||
let _metric_values = Self::reveal_storage_v2_proof_nodes_inner(
|
||||
account,
|
||||
nodes,
|
||||
revealed_paths,
|
||||
trie,
|
||||
self.retain_updates,
|
||||
)?;
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
{
|
||||
self.metrics.increment_total_storage_nodes(_metric_values.total_nodes as u64);
|
||||
self.metrics.increment_skipped_storage_nodes(_metric_values.skipped_nodes as u64);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reveals storage V2 proof nodes for the given address. This is an internal static function
|
||||
/// designed to handle a variety of associated public functions.
|
||||
fn reveal_storage_v2_proof_nodes_inner(
|
||||
account: B256,
|
||||
nodes: Vec<ProofTrieNode>,
|
||||
revealed_nodes: &mut HashSet<Nibbles>,
|
||||
trie: &mut SparseTrie<S>,
|
||||
retain_updates: bool,
|
||||
) -> SparseStateTrieResult<ProofNodesMetricValues> {
|
||||
let FilteredV2ProofNodes { root_node, nodes, new_nodes, metric_values } =
|
||||
filter_revealed_v2_proof_nodes(nodes, revealed_nodes)?;
|
||||
|
||||
if let Some(root_node) = root_node {
|
||||
trace!(target: "trie::sparse", ?account, ?root_node, "Revealing root storage node from V2 proof");
|
||||
let trie = trie.reveal_root(root_node.node, root_node.masks, retain_updates)?;
|
||||
|
||||
trie.reserve_nodes(new_nodes);
|
||||
|
||||
trace!(target: "trie::sparse", ?account, total_nodes = ?nodes.len(), "Revealing storage nodes from V2 proof");
|
||||
trie.reveal_nodes(nodes)?;
|
||||
}
|
||||
|
||||
Ok(metric_values)
|
||||
}
|
||||
|
||||
/// Reveals a storage multiproof for the given address.
|
||||
pub fn reveal_storage_multiproof(
|
||||
&mut self,
|
||||
@@ -1000,6 +1163,87 @@ fn filter_map_revealed_nodes(
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Result of [`filter_revealed_v2_proof_nodes`].
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
struct FilteredV2ProofNodes {
|
||||
/// Root node which was pulled out of the original node set to be handled specially.
|
||||
root_node: Option<ProofTrieNode>,
|
||||
/// Filtered proof nodes. Root node is removed.
|
||||
nodes: Vec<ProofTrieNode>,
|
||||
/// Number of new nodes that will be revealed. This includes all children of branch nodes, even
|
||||
/// if they are not in the proof.
|
||||
new_nodes: usize,
|
||||
/// Values which are being returned so they can be incremented into metrics.
|
||||
metric_values: ProofNodesMetricValues,
|
||||
}
|
||||
|
||||
/// Filters V2 proof nodes that are already revealed, separates the root node if present, and
|
||||
/// returns additional information about the number of total, skipped, and new nodes.
|
||||
///
|
||||
/// Unlike [`filter_map_revealed_nodes`], V2 proof nodes already have masks included in the
|
||||
/// `ProofTrieNode` structure, so no separate masks map is needed.
|
||||
fn filter_revealed_v2_proof_nodes(
|
||||
proof_nodes: Vec<ProofTrieNode>,
|
||||
revealed_nodes: &mut HashSet<Nibbles>,
|
||||
) -> SparseStateTrieResult<FilteredV2ProofNodes> {
|
||||
let mut result = FilteredV2ProofNodes {
|
||||
root_node: None,
|
||||
nodes: Vec::with_capacity(proof_nodes.len()),
|
||||
new_nodes: 0,
|
||||
metric_values: Default::default(),
|
||||
};
|
||||
|
||||
// Count non-EmptyRoot nodes for sanity check. When multiple proofs are extended together,
|
||||
// duplicate EmptyRoot nodes may appear (e.g., storage proofs split across chunks for an
|
||||
// account with empty storage). We only error if there's an EmptyRoot alongside real nodes.
|
||||
let non_empty_root_count =
|
||||
proof_nodes.iter().filter(|n| !matches!(n.node, TrieNode::EmptyRoot)).count();
|
||||
|
||||
for node in proof_nodes {
|
||||
result.metric_values.total_nodes += 1;
|
||||
|
||||
let is_root = node.path.is_empty();
|
||||
|
||||
// If the node is already revealed, skip it. We don't ever skip the root node, nor do we add
|
||||
// it to `revealed_nodes`.
|
||||
if !is_root && !revealed_nodes.insert(node.path) {
|
||||
result.metric_values.skipped_nodes += 1;
|
||||
continue
|
||||
}
|
||||
|
||||
result.new_nodes += 1;
|
||||
|
||||
// Count children for capacity estimation
|
||||
match &node.node {
|
||||
TrieNode::Branch(branch) => {
|
||||
result.new_nodes += branch.state_mask.count_ones() as usize;
|
||||
}
|
||||
TrieNode::Extension(_) => {
|
||||
result.new_nodes += 1;
|
||||
}
|
||||
_ => {}
|
||||
};
|
||||
|
||||
if is_root {
|
||||
// Perform sanity check: EmptyRoot is only valid if there are no other real nodes.
|
||||
if matches!(node.node, TrieNode::EmptyRoot) && non_empty_root_count > 0 {
|
||||
return Err(SparseStateTrieErrorKind::InvalidRootNode {
|
||||
path: node.path,
|
||||
node: alloy_rlp::encode(&node.node).into(),
|
||||
}
|
||||
.into())
|
||||
}
|
||||
|
||||
result.root_node = Some(node);
|
||||
continue
|
||||
}
|
||||
|
||||
result.nodes.push(node);
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -1174,6 +1418,127 @@ mod tests {
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reveal_v2_proof_nodes() {
|
||||
let provider_factory = DefaultTrieNodeProviderFactory;
|
||||
let mut sparse = SparseStateTrie::<SerialSparseTrie>::default();
|
||||
|
||||
let leaf_value = alloy_rlp::encode(TrieAccount::default());
|
||||
let leaf_1_node = TrieNode::Leaf(LeafNode::new(Nibbles::default(), leaf_value.clone()));
|
||||
let leaf_2_node = TrieNode::Leaf(LeafNode::new(Nibbles::default(), leaf_value.clone()));
|
||||
|
||||
let branch_node = TrieNode::Branch(BranchNode {
|
||||
stack: vec![
|
||||
RlpNode::from_rlp(&alloy_rlp::encode(&leaf_1_node)),
|
||||
RlpNode::from_rlp(&alloy_rlp::encode(&leaf_2_node)),
|
||||
],
|
||||
state_mask: TrieMask::new(0b11),
|
||||
});
|
||||
|
||||
// Create V2 proof nodes with masks already included
|
||||
let v2_proof_nodes = vec![
|
||||
ProofTrieNode {
|
||||
path: Nibbles::default(),
|
||||
node: branch_node,
|
||||
masks: Some(BranchNodeMasks {
|
||||
hash_mask: TrieMask::default(),
|
||||
tree_mask: TrieMask::default(),
|
||||
}),
|
||||
},
|
||||
ProofTrieNode { path: Nibbles::from_nibbles([0x0]), node: leaf_1_node, masks: None },
|
||||
ProofTrieNode { path: Nibbles::from_nibbles([0x1]), node: leaf_2_node, masks: None },
|
||||
];
|
||||
|
||||
// Reveal V2 proof nodes
|
||||
sparse.reveal_account_v2_proof_nodes(v2_proof_nodes.clone()).unwrap();
|
||||
|
||||
// Check that the state trie contains the leaf node and value
|
||||
assert!(sparse
|
||||
.state_trie_ref()
|
||||
.unwrap()
|
||||
.nodes_ref()
|
||||
.contains_key(&Nibbles::from_nibbles([0x0])));
|
||||
assert_eq!(
|
||||
sparse.state_trie_ref().unwrap().get_leaf_value(&Nibbles::from_nibbles([0x0])),
|
||||
Some(&leaf_value)
|
||||
);
|
||||
|
||||
// Remove the leaf node
|
||||
sparse.remove_account_leaf(&Nibbles::from_nibbles([0x0]), &provider_factory).unwrap();
|
||||
assert!(sparse
|
||||
.state_trie_ref()
|
||||
.unwrap()
|
||||
.get_leaf_value(&Nibbles::from_nibbles([0x0]))
|
||||
.is_none());
|
||||
|
||||
// Reveal again - should skip already revealed paths
|
||||
sparse.reveal_account_v2_proof_nodes(v2_proof_nodes).unwrap();
|
||||
assert!(sparse
|
||||
.state_trie_ref()
|
||||
.unwrap()
|
||||
.get_leaf_value(&Nibbles::from_nibbles([0x0]))
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reveal_storage_v2_proof_nodes() {
|
||||
let provider_factory = DefaultTrieNodeProviderFactory;
|
||||
let mut sparse = SparseStateTrie::<SerialSparseTrie>::default();
|
||||
|
||||
let storage_value: Vec<u8> = alloy_rlp::encode_fixed_size(&U256::from(42)).to_vec();
|
||||
let leaf_1_node = TrieNode::Leaf(LeafNode::new(Nibbles::default(), storage_value.clone()));
|
||||
let leaf_2_node = TrieNode::Leaf(LeafNode::new(Nibbles::default(), storage_value.clone()));
|
||||
|
||||
let branch_node = TrieNode::Branch(BranchNode {
|
||||
stack: vec![
|
||||
RlpNode::from_rlp(&alloy_rlp::encode(&leaf_1_node)),
|
||||
RlpNode::from_rlp(&alloy_rlp::encode(&leaf_2_node)),
|
||||
],
|
||||
state_mask: TrieMask::new(0b11),
|
||||
});
|
||||
|
||||
let v2_proof_nodes = vec![
|
||||
ProofTrieNode { path: Nibbles::default(), node: branch_node, masks: None },
|
||||
ProofTrieNode { path: Nibbles::from_nibbles([0x0]), node: leaf_1_node, masks: None },
|
||||
ProofTrieNode { path: Nibbles::from_nibbles([0x1]), node: leaf_2_node, masks: None },
|
||||
];
|
||||
|
||||
// Reveal V2 storage proof nodes for account
|
||||
sparse.reveal_storage_v2_proof_nodes(B256::ZERO, v2_proof_nodes.clone()).unwrap();
|
||||
|
||||
// Check that the storage trie contains the leaf node and value
|
||||
assert!(sparse
|
||||
.storage_trie_ref(&B256::ZERO)
|
||||
.unwrap()
|
||||
.nodes_ref()
|
||||
.contains_key(&Nibbles::from_nibbles([0x0])));
|
||||
assert_eq!(
|
||||
sparse
|
||||
.storage_trie_ref(&B256::ZERO)
|
||||
.unwrap()
|
||||
.get_leaf_value(&Nibbles::from_nibbles([0x0])),
|
||||
Some(&storage_value)
|
||||
);
|
||||
|
||||
// Remove the leaf node
|
||||
sparse
|
||||
.remove_storage_leaf(B256::ZERO, &Nibbles::from_nibbles([0x0]), &provider_factory)
|
||||
.unwrap();
|
||||
assert!(sparse
|
||||
.storage_trie_ref(&B256::ZERO)
|
||||
.unwrap()
|
||||
.get_leaf_value(&Nibbles::from_nibbles([0x0]))
|
||||
.is_none());
|
||||
|
||||
// Reveal again - should skip already revealed paths
|
||||
sparse.reveal_storage_v2_proof_nodes(B256::ZERO, v2_proof_nodes).unwrap();
|
||||
assert!(sparse
|
||||
.storage_trie_ref(&B256::ZERO)
|
||||
.unwrap()
|
||||
.get_leaf_value(&Nibbles::from_nibbles([0x0]))
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn take_trie_updates() {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
@@ -96,6 +96,13 @@ test-utils = [
|
||||
"reth-trie-sparse/test-utils",
|
||||
"reth-stages-types/test-utils",
|
||||
]
|
||||
serde-bincode-compat = [
|
||||
"alloy-consensus/serde-bincode-compat",
|
||||
"alloy-eips/serde-bincode-compat",
|
||||
"reth-ethereum-primitives/serde-bincode-compat",
|
||||
"reth-primitives-traits/serde-bincode-compat",
|
||||
"reth-trie-common/serde-bincode-compat",
|
||||
]
|
||||
|
||||
[[bench]]
|
||||
name = "hash_post_state"
|
||||
|
||||
@@ -317,14 +317,15 @@ where
|
||||
}
|
||||
|
||||
/// Returns the path of the child on top of the `child_stack`, or the root path if the stack is
|
||||
/// empty.
|
||||
fn last_child_path(&self) -> Nibbles {
|
||||
/// empty. Returns None if the current branch has not yet pushed a child (empty `state_mask`).
|
||||
fn last_child_path(&self) -> Option<Nibbles> {
|
||||
// If there is no branch under construction then the top child must be the root child.
|
||||
let Some(branch) = self.branch_stack.last() else {
|
||||
return Nibbles::new();
|
||||
return Some(Nibbles::new());
|
||||
};
|
||||
|
||||
self.child_path_at(Self::highest_set_nibble(branch.state_mask))
|
||||
(!branch.state_mask.is_empty())
|
||||
.then(|| self.child_path_at(Self::highest_set_nibble(branch.state_mask)))
|
||||
}
|
||||
|
||||
/// Calls [`Self::commit_child`] on the last child of `child_stack`, replacing it with a
|
||||
@@ -340,7 +341,9 @@ where
|
||||
&mut self,
|
||||
targets: &mut TargetsCursor<'a>,
|
||||
) -> Result<(), StateProofError> {
|
||||
let Some(child) = self.child_stack.pop() else { return Ok(()) };
|
||||
let Some(child_path) = self.last_child_path() else { return Ok(()) };
|
||||
let child =
|
||||
self.child_stack.pop().expect("child_stack can't be empty if there's a child path");
|
||||
|
||||
// If the child is already an `RlpNode` then there is nothing to do, push it back on with no
|
||||
// changes.
|
||||
@@ -349,14 +352,15 @@ where
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
let child_path = self.last_child_path();
|
||||
// TODO theoretically `commit_child` only needs to convert to an `RlpNode` if it's going to
|
||||
// retain the proof, otherwise we could leave the child as-is on the stack and convert it
|
||||
// when popping the branch, giving more time to the DeferredEncoder to do async work.
|
||||
let child_rlp_node = self.commit_child(targets, child_path, child)?;
|
||||
// Only commit immediately if retained for the proof. Otherwise, defer conversion
|
||||
// to pop_branch() to give DeferredEncoder time for async work.
|
||||
if self.should_retain(targets, &child_path, true) {
|
||||
let child_rlp_node = self.commit_child(targets, child_path, child)?;
|
||||
self.child_stack.push(ProofTrieBranchChild::RlpNode(child_rlp_node));
|
||||
} else {
|
||||
self.child_stack.push(child);
|
||||
}
|
||||
|
||||
// Replace the child on the stack
|
||||
self.child_stack.push(ProofTrieBranchChild::RlpNode(child_rlp_node));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -499,15 +503,20 @@ where
|
||||
"Stack is missing necessary children ({num_children:?})"
|
||||
);
|
||||
|
||||
// Collect children into an `RlpNode` Vec by committing and pushing each of them.
|
||||
for (idx, child) in
|
||||
self.child_stack.drain(self.child_stack.len() - num_children..).enumerate()
|
||||
{
|
||||
let ProofTrieBranchChild::RlpNode(child_rlp_node) = child else {
|
||||
panic!(
|
||||
"all branch children must have been committed, found {} at index {idx:?}",
|
||||
std::any::type_name_of_val(&child)
|
||||
);
|
||||
// Collect children into RlpNode Vec. Children are in lexicographic order.
|
||||
for child in self.child_stack.drain(self.child_stack.len() - num_children..) {
|
||||
let child_rlp_node = match child {
|
||||
ProofTrieBranchChild::RlpNode(rlp_node) => rlp_node,
|
||||
uncommitted_child => {
|
||||
// Convert uncommitted child (not retained for proof) to RlpNode now.
|
||||
self.rlp_encode_buf.clear();
|
||||
let (rlp_node, freed_buf) =
|
||||
uncommitted_child.into_rlp(&mut self.rlp_encode_buf)?;
|
||||
if let Some(buf) = freed_buf {
|
||||
self.rlp_nodes_bufs.push(buf);
|
||||
}
|
||||
rlp_node
|
||||
}
|
||||
};
|
||||
rlp_nodes_buf.push(child_rlp_node);
|
||||
}
|
||||
@@ -642,7 +651,7 @@ where
|
||||
)]
|
||||
fn calculate_key_range<'a>(
|
||||
&mut self,
|
||||
value_encoder: &VE,
|
||||
value_encoder: &mut VE,
|
||||
targets: &mut TargetsCursor<'a>,
|
||||
hashed_cursor_current: &mut Option<(Nibbles, VE::DeferredEncoder)>,
|
||||
lower_bound: Nibbles,
|
||||
@@ -651,7 +660,7 @@ where
|
||||
// A helper closure for mapping entries returned from the `hashed_cursor`, converting the
|
||||
// key to Nibbles and immediately creating the DeferredValueEncoder so that encoding of the
|
||||
// leaf value can begin ASAP.
|
||||
let map_hashed_cursor_entry = |(key_b256, val): (B256, _)| {
|
||||
let mut map_hashed_cursor_entry = |(key_b256, val): (B256, _)| {
|
||||
debug_assert_eq!(key_b256.len(), 32);
|
||||
// SAFETY: key is a B256 and so is exactly 32-bytes.
|
||||
let key = unsafe { Nibbles::unpack_unchecked(key_b256.as_slice()) };
|
||||
@@ -670,7 +679,7 @@ where
|
||||
|
||||
let lower_key = B256::right_padding_from(&lower_bound.pack());
|
||||
*hashed_cursor_current =
|
||||
self.hashed_cursor.seek(lower_key)?.map(map_hashed_cursor_entry);
|
||||
self.hashed_cursor.seek(lower_key)?.map(&mut map_hashed_cursor_entry);
|
||||
}
|
||||
|
||||
// Loop over all keys in the range, calling `push_leaf` on each.
|
||||
@@ -680,7 +689,7 @@ where
|
||||
let (key, val) =
|
||||
core::mem::take(hashed_cursor_current).expect("while-let checks for Some");
|
||||
self.push_leaf(targets, key, val)?;
|
||||
*hashed_cursor_current = self.hashed_cursor.next()?.map(map_hashed_cursor_entry);
|
||||
*hashed_cursor_current = self.hashed_cursor.next()?.map(&mut map_hashed_cursor_entry);
|
||||
}
|
||||
|
||||
trace!(target: TRACE_TARGET, "No further keys within range");
|
||||
@@ -1116,7 +1125,7 @@ where
|
||||
)]
|
||||
fn proof_subtrie<'a>(
|
||||
&mut self,
|
||||
value_encoder: &VE,
|
||||
value_encoder: &mut VE,
|
||||
trie_cursor_state: &mut TrieCursorState,
|
||||
hashed_cursor_current: &mut Option<(Nibbles, VE::DeferredEncoder)>,
|
||||
sub_trie_targets: SubTrieTargets<'a>,
|
||||
@@ -1245,7 +1254,7 @@ where
|
||||
/// See docs on [`Self::proof`] for expected behavior.
|
||||
fn proof_inner(
|
||||
&mut self,
|
||||
value_encoder: &VE,
|
||||
value_encoder: &mut VE,
|
||||
targets: &mut [Target],
|
||||
) -> Result<Vec<ProofTrieNode>, StateProofError> {
|
||||
// If there are no targets then nothing could be returned, return early.
|
||||
@@ -1296,7 +1305,7 @@ where
|
||||
#[instrument(target = TRACE_TARGET, level = "trace", skip_all)]
|
||||
pub fn proof(
|
||||
&mut self,
|
||||
value_encoder: &VE,
|
||||
value_encoder: &mut VE,
|
||||
targets: &mut [Target],
|
||||
) -> Result<Vec<ProofTrieNode>, StateProofError> {
|
||||
self.trie_cursor.reset();
|
||||
@@ -1332,9 +1341,6 @@ where
|
||||
hashed_address: B256,
|
||||
targets: &mut [Target],
|
||||
) -> Result<Vec<ProofTrieNode>, StateProofError> {
|
||||
/// Static storage value encoder instance used by all storage proofs.
|
||||
static STORAGE_VALUE_ENCODER: StorageValueEncoder = StorageValueEncoder;
|
||||
|
||||
self.hashed_cursor.set_hashed_address(hashed_address);
|
||||
|
||||
// Shortcut: check if storage is empty
|
||||
@@ -1351,8 +1357,9 @@ where
|
||||
// been checked.
|
||||
self.trie_cursor.set_hashed_address(hashed_address);
|
||||
|
||||
// Use the static StorageValueEncoder and pass it to proof_inner
|
||||
self.proof_inner(&STORAGE_VALUE_ENCODER, targets)
|
||||
// Create a mutable storage value encoder
|
||||
let mut storage_value_encoder = StorageValueEncoder;
|
||||
self.proof_inner(&mut storage_value_encoder, targets)
|
||||
}
|
||||
|
||||
/// Computes the root hash from a set of proof nodes.
|
||||
@@ -1630,13 +1637,13 @@ mod tests {
|
||||
InstrumentedHashedCursor::new(hashed_cursor, &mut hashed_cursor_metrics);
|
||||
|
||||
// Call ProofCalculator::proof with account targets
|
||||
let value_encoder = SyncAccountValueEncoder::new(
|
||||
let mut value_encoder = SyncAccountValueEncoder::new(
|
||||
self.trie_cursor_factory.clone(),
|
||||
self.hashed_cursor_factory.clone(),
|
||||
);
|
||||
let mut proof_calculator = ProofCalculator::new(trie_cursor, hashed_cursor);
|
||||
let proof_v2_result =
|
||||
proof_calculator.proof(&value_encoder, &mut targets_vec.clone())?;
|
||||
proof_calculator.proof(&mut value_encoder, &mut targets_vec.clone())?;
|
||||
|
||||
// Output metrics
|
||||
trace!(target: TRACE_TARGET, ?trie_cursor_metrics, "V2 trie cursor metrics");
|
||||
|
||||
@@ -19,6 +19,11 @@ impl Target {
|
||||
Self { key, min_len: 0 }
|
||||
}
|
||||
|
||||
/// Returns the key the target was initialized with.
|
||||
pub fn key(&self) -> B256 {
|
||||
B256::from_slice(&self.key.pack())
|
||||
}
|
||||
|
||||
/// Only match trie nodes whose path is at least this long.
|
||||
///
|
||||
/// # Panics
|
||||
|
||||
@@ -44,7 +44,7 @@ pub trait LeafValueEncoder {
|
||||
///
|
||||
/// The returned deferred encoder will be called as late as possible in the algorithm to
|
||||
/// maximize the time available for parallel computation (e.g., storage root calculation).
|
||||
fn deferred_encoder(&self, key: B256, value: Self::Value) -> Self::DeferredEncoder;
|
||||
fn deferred_encoder(&mut self, key: B256, value: Self::Value) -> Self::DeferredEncoder;
|
||||
}
|
||||
|
||||
/// An encoder for storage slot values.
|
||||
@@ -68,7 +68,7 @@ impl LeafValueEncoder for StorageValueEncoder {
|
||||
type Value = U256;
|
||||
type DeferredEncoder = StorageDeferredValueEncoder;
|
||||
|
||||
fn deferred_encoder(&self, _key: B256, value: Self::Value) -> Self::DeferredEncoder {
|
||||
fn deferred_encoder(&mut self, _key: B256, value: Self::Value) -> Self::DeferredEncoder {
|
||||
StorageDeferredValueEncoder(value)
|
||||
}
|
||||
}
|
||||
@@ -157,7 +157,7 @@ where
|
||||
type DeferredEncoder = SyncAccountDeferredValueEncoder<T, H>;
|
||||
|
||||
fn deferred_encoder(
|
||||
&self,
|
||||
&mut self,
|
||||
hashed_address: B256,
|
||||
account: Self::Value,
|
||||
) -> Self::DeferredEncoder {
|
||||
|
||||
@@ -10,6 +10,8 @@
|
||||
- [`reth db stats`](./reth/db/stats.mdx)
|
||||
- [`reth db list`](./reth/db/list.mdx)
|
||||
- [`reth db checksum`](./reth/db/checksum.mdx)
|
||||
- [`reth db checksum mdbx`](./reth/db/checksum/mdbx.mdx)
|
||||
- [`reth db checksum static-file`](./reth/db/checksum/static-file.mdx)
|
||||
- [`reth db diff`](./reth/db/diff.mdx)
|
||||
- [`reth db get`](./reth/db/get.mdx)
|
||||
- [`reth db get mdbx`](./reth/db/get/mdbx.mdx)
|
||||
@@ -66,6 +68,8 @@
|
||||
- [`op-reth db stats`](./op-reth/db/stats.mdx)
|
||||
- [`op-reth db list`](./op-reth/db/list.mdx)
|
||||
- [`op-reth db checksum`](./op-reth/db/checksum.mdx)
|
||||
- [`op-reth db checksum mdbx`](./op-reth/db/checksum/mdbx.mdx)
|
||||
- [`op-reth db checksum static-file`](./op-reth/db/checksum/static-file.mdx)
|
||||
- [`op-reth db diff`](./op-reth/db/diff.mdx)
|
||||
- [`op-reth db get`](./op-reth/db/get.mdx)
|
||||
- [`op-reth db get mdbx`](./op-reth/db/get/mdbx.mdx)
|
||||
|
||||
@@ -11,7 +11,7 @@ Usage: op-reth db [OPTIONS] <COMMAND>
|
||||
Commands:
|
||||
stats Lists all the tables, their entry count and their size
|
||||
list Lists the contents of a table
|
||||
checksum Calculates the content checksum of a table
|
||||
checksum Calculates the content checksum of a table or static file segment
|
||||
diff Create a diff between two database tables or two entire databases
|
||||
get Gets the content of a table for the given key
|
||||
drop Deletes all database entries
|
||||
@@ -124,27 +124,66 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.receipts
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
When enabled, receipts will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
--static-files.transaction-senders
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.transaction-senders <TRANSACTION_SENDERS>
|
||||
Store transaction senders in static files instead of the database.
|
||||
|
||||
When enabled, transaction senders will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
--static-files.account-change-sets
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.account-change-sets <ACCOUNT_CHANGESETS>
|
||||
Store account changesets in static files.
|
||||
|
||||
When enabled, account changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This enables `RocksDB` for `tx-hash`, `storages-history`, and `account-history` tables. Cannot be combined with individual flags set to false.
|
||||
|
||||
--rocksdb.tx-hash <TX_HASH>
|
||||
Route tx hash -> number table to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--rocksdb.storages-history <STORAGES_HISTORY>
|
||||
Route storages history tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--rocksdb.account-history <ACCOUNT_HISTORY>
|
||||
Route account history tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
Logging:
|
||||
--log.stdout.format <FORMAT>
|
||||
The format to use for logs written to stdout
|
||||
|
||||
@@ -1,27 +1,19 @@
|
||||
# op-reth db checksum
|
||||
|
||||
Calculates the content checksum of a table
|
||||
Calculates the content checksum of a table or static file segment
|
||||
|
||||
```bash
|
||||
$ op-reth db checksum --help
|
||||
```
|
||||
```txt
|
||||
Usage: op-reth db checksum [OPTIONS] <TABLE>
|
||||
Usage: op-reth db checksum [OPTIONS] <COMMAND>
|
||||
|
||||
Arguments:
|
||||
<TABLE>
|
||||
The table name
|
||||
Commands:
|
||||
mdbx Calculates the checksum of a database table
|
||||
static-file Calculates the checksum of a static file segment
|
||||
help Print this message or the help of the given subcommand(s)
|
||||
|
||||
Options:
|
||||
--start-key <START_KEY>
|
||||
The start of the range to checksum
|
||||
|
||||
--end-key <END_KEY>
|
||||
The end of the range to checksum
|
||||
|
||||
--limit <LIMIT>
|
||||
The maximum number of records that are queried and used to compute the checksum
|
||||
|
||||
-h, --help
|
||||
Print help (see a summary with '-h')
|
||||
|
||||
|
||||
179
docs/vocs/docs/pages/cli/op-reth/db/checksum/mdbx.mdx
Normal file
179
docs/vocs/docs/pages/cli/op-reth/db/checksum/mdbx.mdx
Normal file
@@ -0,0 +1,179 @@
|
||||
# op-reth db checksum mdbx
|
||||
|
||||
Calculates the checksum of a database table
|
||||
|
||||
```bash
|
||||
$ op-reth db checksum mdbx --help
|
||||
```
|
||||
```txt
|
||||
Usage: op-reth db checksum mdbx [OPTIONS] <TABLE>
|
||||
|
||||
Arguments:
|
||||
<TABLE>
|
||||
The table name
|
||||
|
||||
Options:
|
||||
--start-key <START_KEY>
|
||||
The start of the range to checksum
|
||||
|
||||
--end-key <END_KEY>
|
||||
The end of the range to checksum
|
||||
|
||||
--limit <LIMIT>
|
||||
The maximum number of records that are queried and used to compute the checksum
|
||||
|
||||
-h, --help
|
||||
Print help (see a summary with '-h')
|
||||
|
||||
Datadir:
|
||||
--chain <CHAIN_OR_PATH>
|
||||
The chain this node is running.
|
||||
Possible values are either a built-in chain or the path to a chain specification file.
|
||||
|
||||
Built-in chains:
|
||||
optimism, optimism_sepolia, optimism-sepolia, base, base_sepolia, base-sepolia, arena-z, arena-z-sepolia, automata, base-devnet-0-sepolia-dev-0, bob, boba-sepolia, boba, camp-sepolia, celo, creator-chain-testnet-sepolia, cyber, cyber-sepolia, ethernity, ethernity-sepolia, fraxtal, funki, funki-sepolia, hashkeychain, ink, ink-sepolia, lisk, lisk-sepolia, lyra, metal, metal-sepolia, mint, mode, mode-sepolia, oplabs-devnet-0-sepolia-dev-0, orderly, ozean-sepolia, pivotal-sepolia, polynomial, race, race-sepolia, radius_testnet-sepolia, redstone, rehearsal-0-bn-0-rehearsal-0-bn, rehearsal-0-bn-1-rehearsal-0-bn, settlus-mainnet, settlus-sepolia-sepolia, shape, shape-sepolia, silent-data-mainnet, snax, soneium, soneium-minato-sepolia, sseed, swan, swell, tbn, tbn-sepolia, unichain, unichain-sepolia, worldchain, worldchain-sepolia, xterio-eth, zora, zora-sepolia, dev
|
||||
|
||||
[default: optimism]
|
||||
|
||||
Logging:
|
||||
--log.stdout.format <FORMAT>
|
||||
The format to use for logs written to stdout
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.stdout.filter <FILTER>
|
||||
The filter to use for logs written to stdout
|
||||
|
||||
[default: ]
|
||||
|
||||
--log.file.format <FORMAT>
|
||||
The format to use for logs written to the log file
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.file.filter <FILTER>
|
||||
The filter to use for logs written to the log file
|
||||
|
||||
[default: debug]
|
||||
|
||||
--log.file.directory <PATH>
|
||||
The path to put log files in
|
||||
|
||||
[default: <CACHE_DIR>/logs]
|
||||
|
||||
--log.file.name <NAME>
|
||||
The prefix name of the log files
|
||||
|
||||
[default: reth.log]
|
||||
|
||||
--log.file.max-size <SIZE>
|
||||
The maximum size (in MB) of one log file
|
||||
|
||||
[default: 200]
|
||||
|
||||
--log.file.max-files <COUNT>
|
||||
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
|
||||
|
||||
[default: 5]
|
||||
|
||||
--log.journald
|
||||
Write logs to journald
|
||||
|
||||
--log.journald.filter <FILTER>
|
||||
The filter to use for logs written to journald
|
||||
|
||||
[default: error]
|
||||
|
||||
--color <COLOR>
|
||||
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
|
||||
|
||||
Possible values:
|
||||
- always: Colors on
|
||||
- auto: Auto-detect
|
||||
- never: Colors off
|
||||
|
||||
[default: always]
|
||||
|
||||
--logs-otlp[=<URL>]
|
||||
Enable `Opentelemetry` logs export to an OTLP endpoint.
|
||||
|
||||
If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/logs` - gRPC: `http://localhost:4317`
|
||||
|
||||
Example: --logs-otlp=http://collector:4318/v1/logs
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=]
|
||||
|
||||
--logs-otlp.filter <FILTER>
|
||||
Set a filter directive for the OTLP logs exporter. This controls the verbosity of logs sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable.
|
||||
|
||||
Example: --logs-otlp.filter=info,reth=debug
|
||||
|
||||
Defaults to INFO if not specified.
|
||||
|
||||
[default: info]
|
||||
|
||||
Display:
|
||||
-v, --verbosity...
|
||||
Set the minimum log level.
|
||||
|
||||
-v Errors
|
||||
-vv Warnings
|
||||
-vvv Info
|
||||
-vvvv Debug
|
||||
-vvvvv Traces (warning: very verbose!)
|
||||
|
||||
-q, --quiet
|
||||
Silence all log output
|
||||
|
||||
Tracing:
|
||||
--tracing-otlp[=<URL>]
|
||||
Enable `Opentelemetry` tracing export to an OTLP endpoint.
|
||||
|
||||
If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317`
|
||||
|
||||
Example: --tracing-otlp=http://collector:4318/v1/traces
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=]
|
||||
|
||||
--tracing-otlp-protocol <PROTOCOL>
|
||||
OTLP transport protocol to use for exporting traces and logs.
|
||||
|
||||
- `http`: expects endpoint path to end with `/v1/traces` or `/v1/logs` - `grpc`: expects endpoint without a path
|
||||
|
||||
Defaults to HTTP if not specified.
|
||||
|
||||
Possible values:
|
||||
- http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path
|
||||
- grpc: gRPC transport, port 4317
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_PROTOCOL=]
|
||||
[default: http]
|
||||
|
||||
--tracing-otlp.filter <FILTER>
|
||||
Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable.
|
||||
|
||||
Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off
|
||||
|
||||
Defaults to TRACE if not specified.
|
||||
|
||||
[default: debug]
|
||||
|
||||
--tracing-otlp.sample-ratio <RATIO>
|
||||
Trace sampling ratio to control the percentage of traces to export.
|
||||
|
||||
Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
|
||||
|
||||
Example: --tracing-otlp.sample-ratio=0.0.
|
||||
|
||||
[env: OTEL_TRACES_SAMPLER_ARG=]
|
||||
```
|
||||
186
docs/vocs/docs/pages/cli/op-reth/db/checksum/static-file.mdx
Normal file
186
docs/vocs/docs/pages/cli/op-reth/db/checksum/static-file.mdx
Normal file
@@ -0,0 +1,186 @@
|
||||
# op-reth db checksum static-file
|
||||
|
||||
Calculates the checksum of a static file segment
|
||||
|
||||
```bash
|
||||
$ op-reth db checksum static-file --help
|
||||
```
|
||||
```txt
|
||||
Usage: op-reth db checksum static-file [OPTIONS] <SEGMENT>
|
||||
|
||||
Arguments:
|
||||
<SEGMENT>
|
||||
The static file segment
|
||||
|
||||
Possible values:
|
||||
- headers: Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTerminalDifficulties` tables
|
||||
- transactions: Static File segment responsible for the `Transactions` table
|
||||
- receipts: Static File segment responsible for the `Receipts` table
|
||||
- transaction-senders: Static File segment responsible for the `TransactionSenders` table
|
||||
- account-change-sets: Static File segment responsible for the `AccountChangeSets` table
|
||||
|
||||
Options:
|
||||
--start-block <START_BLOCK>
|
||||
The block number to start from (inclusive)
|
||||
|
||||
--end-block <END_BLOCK>
|
||||
The block number to end at (inclusive)
|
||||
|
||||
--limit <LIMIT>
|
||||
The maximum number of rows to checksum
|
||||
|
||||
-h, --help
|
||||
Print help (see a summary with '-h')
|
||||
|
||||
Datadir:
|
||||
--chain <CHAIN_OR_PATH>
|
||||
The chain this node is running.
|
||||
Possible values are either a built-in chain or the path to a chain specification file.
|
||||
|
||||
Built-in chains:
|
||||
optimism, optimism_sepolia, optimism-sepolia, base, base_sepolia, base-sepolia, arena-z, arena-z-sepolia, automata, base-devnet-0-sepolia-dev-0, bob, boba-sepolia, boba, camp-sepolia, celo, creator-chain-testnet-sepolia, cyber, cyber-sepolia, ethernity, ethernity-sepolia, fraxtal, funki, funki-sepolia, hashkeychain, ink, ink-sepolia, lisk, lisk-sepolia, lyra, metal, metal-sepolia, mint, mode, mode-sepolia, oplabs-devnet-0-sepolia-dev-0, orderly, ozean-sepolia, pivotal-sepolia, polynomial, race, race-sepolia, radius_testnet-sepolia, redstone, rehearsal-0-bn-0-rehearsal-0-bn, rehearsal-0-bn-1-rehearsal-0-bn, settlus-mainnet, settlus-sepolia-sepolia, shape, shape-sepolia, silent-data-mainnet, snax, soneium, soneium-minato-sepolia, sseed, swan, swell, tbn, tbn-sepolia, unichain, unichain-sepolia, worldchain, worldchain-sepolia, xterio-eth, zora, zora-sepolia, dev
|
||||
|
||||
[default: optimism]
|
||||
|
||||
Logging:
|
||||
--log.stdout.format <FORMAT>
|
||||
The format to use for logs written to stdout
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.stdout.filter <FILTER>
|
||||
The filter to use for logs written to stdout
|
||||
|
||||
[default: ]
|
||||
|
||||
--log.file.format <FORMAT>
|
||||
The format to use for logs written to the log file
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.file.filter <FILTER>
|
||||
The filter to use for logs written to the log file
|
||||
|
||||
[default: debug]
|
||||
|
||||
--log.file.directory <PATH>
|
||||
The path to put log files in
|
||||
|
||||
[default: <CACHE_DIR>/logs]
|
||||
|
||||
--log.file.name <NAME>
|
||||
The prefix name of the log files
|
||||
|
||||
[default: reth.log]
|
||||
|
||||
--log.file.max-size <SIZE>
|
||||
The maximum size (in MB) of one log file
|
||||
|
||||
[default: 200]
|
||||
|
||||
--log.file.max-files <COUNT>
|
||||
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
|
||||
|
||||
[default: 5]
|
||||
|
||||
--log.journald
|
||||
Write logs to journald
|
||||
|
||||
--log.journald.filter <FILTER>
|
||||
The filter to use for logs written to journald
|
||||
|
||||
[default: error]
|
||||
|
||||
--color <COLOR>
|
||||
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
|
||||
|
||||
Possible values:
|
||||
- always: Colors on
|
||||
- auto: Auto-detect
|
||||
- never: Colors off
|
||||
|
||||
[default: always]
|
||||
|
||||
--logs-otlp[=<URL>]
|
||||
Enable `Opentelemetry` logs export to an OTLP endpoint.
|
||||
|
||||
If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/logs` - gRPC: `http://localhost:4317`
|
||||
|
||||
Example: --logs-otlp=http://collector:4318/v1/logs
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=]
|
||||
|
||||
--logs-otlp.filter <FILTER>
|
||||
Set a filter directive for the OTLP logs exporter. This controls the verbosity of logs sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable.
|
||||
|
||||
Example: --logs-otlp.filter=info,reth=debug
|
||||
|
||||
Defaults to INFO if not specified.
|
||||
|
||||
[default: info]
|
||||
|
||||
Display:
|
||||
-v, --verbosity...
|
||||
Set the minimum log level.
|
||||
|
||||
-v Errors
|
||||
-vv Warnings
|
||||
-vvv Info
|
||||
-vvvv Debug
|
||||
-vvvvv Traces (warning: very verbose!)
|
||||
|
||||
-q, --quiet
|
||||
Silence all log output
|
||||
|
||||
Tracing:
|
||||
--tracing-otlp[=<URL>]
|
||||
Enable `Opentelemetry` tracing export to an OTLP endpoint.
|
||||
|
||||
If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317`
|
||||
|
||||
Example: --tracing-otlp=http://collector:4318/v1/traces
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=]
|
||||
|
||||
--tracing-otlp-protocol <PROTOCOL>
|
||||
OTLP transport protocol to use for exporting traces and logs.
|
||||
|
||||
- `http`: expects endpoint path to end with `/v1/traces` or `/v1/logs` - `grpc`: expects endpoint without a path
|
||||
|
||||
Defaults to HTTP if not specified.
|
||||
|
||||
Possible values:
|
||||
- http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path
|
||||
- grpc: gRPC transport, port 4317
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_PROTOCOL=]
|
||||
[default: http]
|
||||
|
||||
--tracing-otlp.filter <FILTER>
|
||||
Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable.
|
||||
|
||||
Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off
|
||||
|
||||
Defaults to TRACE if not specified.
|
||||
|
||||
[default: debug]
|
||||
|
||||
--tracing-otlp.sample-ratio <RATIO>
|
||||
Trace sampling ratio to control the percentage of traces to export.
|
||||
|
||||
Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
|
||||
|
||||
Example: --tracing-otlp.sample-ratio=0.0.
|
||||
|
||||
[env: OTEL_TRACES_SAMPLER_ARG=]
|
||||
```
|
||||
@@ -108,27 +108,66 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.receipts
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
When enabled, receipts will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
--static-files.transaction-senders
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.transaction-senders <TRANSACTION_SENDERS>
|
||||
Store transaction senders in static files instead of the database.
|
||||
|
||||
When enabled, transaction senders will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
--static-files.account-change-sets
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.account-change-sets <ACCOUNT_CHANGESETS>
|
||||
Store account changesets in static files.
|
||||
|
||||
When enabled, account changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This enables `RocksDB` for `tx-hash`, `storages-history`, and `account-history` tables. Cannot be combined with individual flags set to false.
|
||||
|
||||
--rocksdb.tx-hash <TX_HASH>
|
||||
Route tx hash -> number table to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--rocksdb.storages-history <STORAGES_HISTORY>
|
||||
Route storages history tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--rocksdb.account-history <ACCOUNT_HISTORY>
|
||||
Route account history tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--chunk-len <CHUNK_LEN>
|
||||
Chunk byte length to read from file.
|
||||
|
||||
|
||||
@@ -108,27 +108,66 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.receipts
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
When enabled, receipts will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
--static-files.transaction-senders
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.transaction-senders <TRANSACTION_SENDERS>
|
||||
Store transaction senders in static files instead of the database.
|
||||
|
||||
When enabled, transaction senders will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
--static-files.account-change-sets
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.account-change-sets <ACCOUNT_CHANGESETS>
|
||||
Store account changesets in static files.
|
||||
|
||||
When enabled, account changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This enables `RocksDB` for `tx-hash`, `storages-history`, and `account-history` tables. Cannot be combined with individual flags set to false.
|
||||
|
||||
--rocksdb.tx-hash <TX_HASH>
|
||||
Route tx hash -> number table to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--rocksdb.storages-history <STORAGES_HISTORY>
|
||||
Route storages history tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--rocksdb.account-history <ACCOUNT_HISTORY>
|
||||
Route account history tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--chunk-len <CHUNK_LEN>
|
||||
Chunk byte length to read from file.
|
||||
|
||||
|
||||
@@ -108,27 +108,66 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.receipts
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
When enabled, receipts will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
--static-files.transaction-senders
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.transaction-senders <TRANSACTION_SENDERS>
|
||||
Store transaction senders in static files instead of the database.
|
||||
|
||||
When enabled, transaction senders will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
--static-files.account-change-sets
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.account-change-sets <ACCOUNT_CHANGESETS>
|
||||
Store account changesets in static files.
|
||||
|
||||
When enabled, account changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This enables `RocksDB` for `tx-hash`, `storages-history`, and `account-history` tables. Cannot be combined with individual flags set to false.
|
||||
|
||||
--rocksdb.tx-hash <TX_HASH>
|
||||
Route tx hash -> number table to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--rocksdb.storages-history <STORAGES_HISTORY>
|
||||
Route storages history tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--rocksdb.account-history <ACCOUNT_HISTORY>
|
||||
Route account history tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--without-evm
|
||||
Specifies whether to initialize the state without relying on EVM historical data.
|
||||
|
||||
|
||||
@@ -108,27 +108,66 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.receipts
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
When enabled, receipts will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
--static-files.transaction-senders
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.transaction-senders <TRANSACTION_SENDERS>
|
||||
Store transaction senders in static files instead of the database.
|
||||
|
||||
When enabled, transaction senders will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
--static-files.account-change-sets
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.account-change-sets <ACCOUNT_CHANGESETS>
|
||||
Store account changesets in static files.
|
||||
|
||||
When enabled, account changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This enables `RocksDB` for `tx-hash`, `storages-history`, and `account-history` tables. Cannot be combined with individual flags set to false.
|
||||
|
||||
--rocksdb.tx-hash <TX_HASH>
|
||||
Route tx hash -> number table to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--rocksdb.storages-history <STORAGES_HISTORY>
|
||||
Route storages history tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--rocksdb.account-history <ACCOUNT_HISTORY>
|
||||
Route account history tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
Logging:
|
||||
--log.stdout.format <FORMAT>
|
||||
The format to use for logs written to stdout
|
||||
|
||||
@@ -897,6 +897,36 @@ Pruning:
|
||||
--prune.bodies.before <BLOCK_NUMBER>
|
||||
Prune storage history before the specified block number. The specified block number is not pruned
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This enables `RocksDB` for `tx-hash`, `storages-history`, and `account-history` tables. Cannot be combined with individual flags set to false.
|
||||
|
||||
--rocksdb.tx-hash <TX_HASH>
|
||||
Route tx hash -> number table to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--rocksdb.storages-history <STORAGES_HISTORY>
|
||||
Route storages history tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--rocksdb.account-history <ACCOUNT_HISTORY>
|
||||
Route account history tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
Engine:
|
||||
--engine.persistence-threshold <PERSISTENCE_THRESHOLD>
|
||||
Configure persistence threshold for the engine. This determines how many canonical blocks must be in-memory, ahead of the last persisted block, before flushing canonical blocks to disk again.
|
||||
@@ -972,6 +1002,9 @@ Engine:
|
||||
--engine.enable-proof-v2
|
||||
Enable V2 storage proofs for state root calculations
|
||||
|
||||
--engine.disable-cache-metrics
|
||||
Disable cache metrics recording, which can take up to 50ms with large cached state
|
||||
|
||||
ERA:
|
||||
--era.enable
|
||||
Enable import from ERA1 files
|
||||
@@ -1003,27 +1036,36 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.receipts
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
When enabled, receipts will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
--static-files.transaction-senders
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.transaction-senders <TRANSACTION_SENDERS>
|
||||
Store transaction senders in static files instead of the database.
|
||||
|
||||
When enabled, transaction senders will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
--static-files.account-change-sets
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.account-change-sets <ACCOUNT_CHANGESETS>
|
||||
Store account changesets in static files.
|
||||
|
||||
When enabled, account changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
Rollup:
|
||||
--rollup.sequencer <SEQUENCER>
|
||||
Endpoint for the sequencer mempool (can be both HTTP and WS)
|
||||
|
||||
@@ -108,27 +108,66 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.receipts
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
When enabled, receipts will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
--static-files.transaction-senders
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.transaction-senders <TRANSACTION_SENDERS>
|
||||
Store transaction senders in static files instead of the database.
|
||||
|
||||
When enabled, transaction senders will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
--static-files.account-change-sets
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.account-change-sets <ACCOUNT_CHANGESETS>
|
||||
Store account changesets in static files.
|
||||
|
||||
When enabled, account changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This enables `RocksDB` for `tx-hash`, `storages-history`, and `account-history` tables. Cannot be combined with individual flags set to false.
|
||||
|
||||
--rocksdb.tx-hash <TX_HASH>
|
||||
Route tx hash -> number table to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--rocksdb.storages-history <STORAGES_HISTORY>
|
||||
Route storages history tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--rocksdb.account-history <ACCOUNT_HISTORY>
|
||||
Route account history tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
Logging:
|
||||
--log.stdout.format <FORMAT>
|
||||
The format to use for logs written to stdout
|
||||
|
||||
@@ -108,27 +108,66 @@ Static Files:
|
||||
--static-files.blocks-per-file.account-change-sets <BLOCKS_PER_FILE_ACCOUNT_CHANGE_SETS>
|
||||
Number of blocks per file for the account changesets segment
|
||||
|
||||
--static-files.receipts
|
||||
--static-files.receipts <RECEIPTS>
|
||||
Store receipts in static files instead of the database.
|
||||
|
||||
When enabled, receipts will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
--static-files.transaction-senders
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.transaction-senders <TRANSACTION_SENDERS>
|
||||
Store transaction senders in static files instead of the database.
|
||||
|
||||
When enabled, transaction senders will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
--static-files.account-change-sets
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--static-files.account-change-sets <ACCOUNT_CHANGESETS>
|
||||
Store account changesets in static files.
|
||||
|
||||
When enabled, account changesets will be written to static files on disk instead of the database.
|
||||
|
||||
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
RocksDB:
|
||||
--rocksdb.all
|
||||
Route all supported tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This enables `RocksDB` for `tx-hash`, `storages-history`, and `account-history` tables. Cannot be combined with individual flags set to false.
|
||||
|
||||
--rocksdb.tx-hash <TX_HASH>
|
||||
Route tx hash -> number table to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--rocksdb.storages-history <STORAGES_HISTORY>
|
||||
Route storages history tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--rocksdb.account-history <ACCOUNT_HISTORY>
|
||||
Route account history tables to `RocksDB` instead of MDBX.
|
||||
|
||||
This is a genesis-initialization-only flag: changing it after genesis requires a re-sync. Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
|
||||
[default: false]
|
||||
[possible values: true, false]
|
||||
|
||||
--from <FROM>
|
||||
The height to start at
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user