mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
75 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6be57ce9e1 | ||
|
|
b997bee71b | ||
|
|
e3c7413747 | ||
|
|
0bf2335c8c | ||
|
|
1f7a5acae1 | ||
|
|
5c3e45cd6b | ||
|
|
68fdba32d2 | ||
|
|
8f6a0a2992 | ||
|
|
ec9c7f8d3e | ||
|
|
dbdaf068f0 | ||
|
|
055bf63ee9 | ||
|
|
2305c3ebeb | ||
|
|
eb55c3c3da | ||
|
|
72e1467ba3 | ||
|
|
74edce0089 | ||
|
|
8c645d5762 | ||
|
|
b7d2ee2566 | ||
|
|
7609deddda | ||
|
|
ec50fd40b3 | ||
|
|
624ddc5779 | ||
|
|
dd72cfe23e | ||
|
|
ff8ac97e33 | ||
|
|
0974485863 | ||
|
|
274394e777 | ||
|
|
1954c91a60 | ||
|
|
9cf82c8403 | ||
|
|
f85fcba872 | ||
|
|
ebaa4bda3a | ||
|
|
04d4c9a02f | ||
|
|
3065a328f9 | ||
|
|
43a84f1231 | ||
|
|
5a5c21cc1b | ||
|
|
8a8a9126d6 | ||
|
|
6f73c2447d | ||
|
|
2cae438642 | ||
|
|
37b5db0d47 | ||
|
|
238433e146 | ||
|
|
660964a0f5 | ||
|
|
22b465dd64 | ||
|
|
3ff575b877 | ||
|
|
d12752dc8a | ||
|
|
869b5d0851 | ||
|
|
78de3d8f61 | ||
|
|
bc79cc44c9 | ||
|
|
ff8f434dcd | ||
|
|
9662dc5271 | ||
|
|
3ba37082dc | ||
|
|
7934294988 | ||
|
|
7371bd3f29 | ||
|
|
80980b8e4d | ||
|
|
2e2cd67663 | ||
|
|
4f009728e2 | ||
|
|
39d5ae73e8 | ||
|
|
5ef200eaad | ||
|
|
d002dacc13 | ||
|
|
bb39cba504 | ||
|
|
bd144a4c42 | ||
|
|
a0845bab18 | ||
|
|
346cc0da71 | ||
|
|
ea3d4663ae | ||
|
|
3667d3b5aa | ||
|
|
7cfb19c98e | ||
|
|
5a38871489 | ||
|
|
c825c8c187 | ||
|
|
8f37cd08fc | ||
|
|
c9dad4765d | ||
|
|
1d55abeef3 | ||
|
|
f7460e219c | ||
|
|
0c66315f20 | ||
|
|
6a2010e595 | ||
|
|
c2435ff6f8 | ||
|
|
52ec8e9491 | ||
|
|
a901d80ee6 | ||
|
|
915164078f | ||
|
|
be3234d848 |
43
.github/CODEOWNERS
vendored
43
.github/CODEOWNERS
vendored
@@ -1,45 +1,52 @@
|
||||
* @gakonst
|
||||
crates/blockchain-tree-api/ @rakita @mattsse @Rjected
|
||||
crates/blockchain-tree/ @rakita @mattsse @Rjected
|
||||
crates/chain-state/ @fgimenez @mattsse
|
||||
crates/chainspec/ @Rjected @joshieDo @mattsse
|
||||
crates/cli/ @mattsse
|
||||
crates/config/ @shekhirin @mattsse @Rjected
|
||||
crates/consensus/ @mattsse @Rjected
|
||||
crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez
|
||||
crates/engine/ @mattsse @Rjected @fgimenez @mediocregopher @yongkangc
|
||||
crates/era/ @mattsse @RomanHodulak
|
||||
crates/engine/ @mattsse @Rjected @mediocregopher @yongkangc
|
||||
crates/era/ @mattsse
|
||||
crates/era-downloader/ @mattsse
|
||||
crates/era-utils/ @mattsse
|
||||
crates/errors/ @mattsse
|
||||
crates/ethereum-forks/ @mattsse @Rjected
|
||||
crates/ethereum/ @mattsse @Rjected
|
||||
crates/etl/ @joshieDo @shekhirin
|
||||
crates/evm/ @rakita @mattsse @Rjected
|
||||
crates/evm/ @mattsse @Rjected @klkvr
|
||||
crates/exex/ @shekhirin
|
||||
crates/fs-util/ @mattsse
|
||||
crates/metrics/ @mattsse @Rjected
|
||||
crates/net/ @mattsse @Rjected
|
||||
crates/net/downloaders/ @Rjected
|
||||
crates/node/ @mattsse @Rjected @klkvr
|
||||
crates/optimism/ @mattsse @Rjected @fgimenez
|
||||
crates/optimism/ @mattsse @Rjected
|
||||
crates/payload/ @mattsse @Rjected
|
||||
crates/primitives-traits/ @Rjected @RomanHodulak @mattsse @klkvr
|
||||
crates/primitives-traits/ @Rjected @mattsse @klkvr
|
||||
crates/primitives/ @Rjected @mattsse @klkvr
|
||||
crates/prune/ @shekhirin @joshieDo
|
||||
crates/ress @shekhirin @Rjected
|
||||
crates/revm/ @mattsse @rakita
|
||||
crates/rpc/ @mattsse @Rjected @RomanHodulak
|
||||
crates/ress/ @shekhirin @Rjected
|
||||
crates/revm/ @mattsse
|
||||
crates/rpc/ @mattsse @Rjected
|
||||
crates/stages/ @shekhirin @mediocregopher
|
||||
crates/static-file/ @joshieDo @shekhirin
|
||||
crates/stateless/ @mattsse
|
||||
crates/storage/codecs/ @joshieDo
|
||||
crates/storage/db-api/ @joshieDo @rakita
|
||||
crates/storage/db-api/ @joshieDo
|
||||
crates/storage/db-common/ @Rjected
|
||||
crates/storage/db/ @joshieDo @rakita
|
||||
crates/storage/errors/ @rakita
|
||||
crates/storage/libmdbx-rs/ @rakita @shekhirin
|
||||
crates/storage/db/ @joshieDo
|
||||
crates/storage/errors/ @joshieDo
|
||||
crates/storage/libmdbx-rs/ @shekhirin
|
||||
crates/storage/nippy-jar/ @joshieDo @shekhirin
|
||||
crates/storage/provider/ @rakita @joshieDo @shekhirin
|
||||
crates/storage/provider/ @joshieDo @shekhirin
|
||||
crates/storage/storage-api/ @joshieDo
|
||||
crates/tasks/ @mattsse
|
||||
crates/tokio-util/ @fgimenez
|
||||
crates/tokio-util/ @mattsse
|
||||
crates/tracing/ @mattsse @shekhirin
|
||||
crates/tracing-otlp/ @mattsse @Rjected
|
||||
crates/transaction-pool/ @mattsse @yongkangc
|
||||
crates/trie/ @Rjected @shekhirin @mediocregopher
|
||||
crates/trie/ @Rjected @shekhirin @mediocregopher @yongkangc
|
||||
bin/reth/ @mattsse @shekhirin @Rjected
|
||||
bin/reth-bench/ @mattsse @Rjected @shekhirin @yongkangc
|
||||
bin/reth-bench-compare/ @mediocregopher @shekhirin @yongkangc
|
||||
etc/ @Rjected @shekhirin
|
||||
.github/ @gakonst @DaniPopes
|
||||
|
||||
14
.github/scripts/codspeed-build.sh
vendored
14
.github/scripts/codspeed-build.sh
vendored
@@ -1,14 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eo pipefail
|
||||
|
||||
# TODO: Benchmarks run WAY too slow due to excessive amount of iterations.
|
||||
|
||||
cmd=(cargo codspeed build --profile profiling)
|
||||
crates=(
|
||||
-p reth-primitives
|
||||
-p reth-trie
|
||||
-p reth-trie-common
|
||||
-p reth-trie-sparse
|
||||
)
|
||||
|
||||
"${cmd[@]}" --features test-utils "${crates[@]}"
|
||||
14
.github/workflows/bench.yml
vendored
14
.github/workflows/bench.yml
vendored
@@ -17,6 +17,16 @@ name: bench
|
||||
jobs:
|
||||
codspeed:
|
||||
runs-on: depot-ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
partition: [1, 2]
|
||||
total_partitions: [2]
|
||||
include:
|
||||
- partition: 1
|
||||
crates: "-p reth-primitives -p reth-trie-common -p reth-trie-sparse"
|
||||
- partition: 2
|
||||
crates: "-p reth-trie"
|
||||
name: codspeed (${{ matrix.partition }}/${{ matrix.total_partitions }})
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
@@ -32,10 +42,10 @@ jobs:
|
||||
with:
|
||||
tool: cargo-codspeed
|
||||
- name: Build the benchmark target(s)
|
||||
run: ./.github/scripts/codspeed-build.sh
|
||||
run: cargo codspeed build --profile profiling --features test-utils ${{ matrix.crates }}
|
||||
- name: Run the benchmarks
|
||||
uses: CodSpeedHQ/action@v4
|
||||
with:
|
||||
run: cargo codspeed run --workspace
|
||||
run: cargo codspeed run ${{ matrix.crates }}
|
||||
mode: instrumentation
|
||||
token: ${{ secrets.CODSPEED_TOKEN }}
|
||||
|
||||
66
.github/workflows/check-alloy.yml
vendored
Normal file
66
.github/workflows/check-alloy.yml
vendored
Normal file
@@ -0,0 +1,66 @@
|
||||
# Checks reth compilation against alloy branches to detect breaking changes.
|
||||
# Run on-demand via workflow_dispatch.
|
||||
|
||||
name: Check Alloy Breaking Changes
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
alloy_branch:
|
||||
description: 'Branch/rev for alloy-rs/alloy (leave empty to skip)'
|
||||
required: false
|
||||
type: string
|
||||
alloy_evm_branch:
|
||||
description: 'Branch/rev for alloy-rs/evm (alloy-evm, alloy-op-evm) (leave empty to skip)'
|
||||
required: false
|
||||
type: string
|
||||
op_alloy_branch:
|
||||
description: 'Branch/rev for alloy-rs/op-alloy (leave empty to skip)'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Check compilation with patched alloy
|
||||
runs-on: depot-ubuntu-latest-16
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
|
||||
- name: Apply alloy patches
|
||||
run: |
|
||||
ARGS=""
|
||||
if [ -n "${{ inputs.alloy_branch }}" ]; then
|
||||
ARGS="$ARGS --alloy ${{ inputs.alloy_branch }}"
|
||||
fi
|
||||
if [ -n "${{ inputs.alloy_evm_branch }}" ]; then
|
||||
ARGS="$ARGS --evm ${{ inputs.alloy_evm_branch }}"
|
||||
fi
|
||||
if [ -n "${{ inputs.op_alloy_branch }}" ]; then
|
||||
ARGS="$ARGS --op ${{ inputs.op_alloy_branch }}"
|
||||
fi
|
||||
|
||||
if [ -z "$ARGS" ]; then
|
||||
echo "No branches specified, nothing to patch"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
./scripts/patch-alloy.sh $ARGS
|
||||
|
||||
echo "=== Final patch section ==="
|
||||
tail -50 Cargo.toml
|
||||
|
||||
- name: Check workspace
|
||||
run: cargo check --workspace --all-features
|
||||
|
||||
- name: Check Optimism
|
||||
run: cargo check -p reth-optimism-node --all-features
|
||||
2
.github/workflows/dependencies.yml
vendored
2
.github/workflows/dependencies.yml
vendored
@@ -15,6 +15,6 @@ permissions:
|
||||
|
||||
jobs:
|
||||
update:
|
||||
uses: ithacaxyz/ci/.github/workflows/cargo-update-pr.yml@main
|
||||
uses: tempoxyz/ci/.github/workflows/cargo-update-pr.yml@main
|
||||
secrets:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
2
.github/workflows/lint.yml
vendored
2
.github/workflows/lint.yml
vendored
@@ -285,7 +285,7 @@ jobs:
|
||||
- run: zepter run check
|
||||
|
||||
deny:
|
||||
uses: ithacaxyz/ci/.github/workflows/deny.yml@main
|
||||
uses: tempoxyz/ci/.github/workflows/deny.yml@main
|
||||
|
||||
lint-success:
|
||||
name: lint success
|
||||
|
||||
@@ -249,7 +249,7 @@ Write comments that remain valuable after the PR is merged. Future readers won't
|
||||
unsafe impl GlobalAlloc for LimitedAllocator { ... }
|
||||
|
||||
// Binary search requires sorted input. Panics on unsorted slices.
|
||||
fn find_index(items: &[Item], target: &Item) -> Option
|
||||
fn find_index(items: &[Item], target: &Item) -> Option<usize>
|
||||
|
||||
// Timeout set to 5s to match EVM block processing limits
|
||||
const TRACER_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
706
Cargo.lock
generated
706
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
31
Cargo.toml
31
Cargo.toml
@@ -1,5 +1,5 @@
|
||||
[workspace.package]
|
||||
version = "1.10.0"
|
||||
version = "1.10.2"
|
||||
edition = "2024"
|
||||
rust-version = "1.88"
|
||||
license = "MIT OR Apache-2.0"
|
||||
@@ -797,3 +797,32 @@ ipnet = "2.11"
|
||||
|
||||
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
|
||||
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
|
||||
|
||||
# Patched by patch-alloy.sh
|
||||
alloy-consensus = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-contract = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-eips = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-genesis = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-network = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-provider = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-serde = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-signer = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-transport = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
|
||||
|
||||
@@ -163,6 +163,7 @@ impl NodeManager {
|
||||
"eth,reth".to_string(),
|
||||
"--disable-discovery".to_string(),
|
||||
"--trusted-only".to_string(),
|
||||
"--disable-tx-gossip".to_string(),
|
||||
]);
|
||||
|
||||
// Add tracing arguments if OTLP endpoint is configured
|
||||
|
||||
@@ -22,12 +22,42 @@ use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIM
|
||||
use std::{path::PathBuf, time::Instant};
|
||||
use tracing::info;
|
||||
|
||||
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
|
||||
///
|
||||
/// Examples: "30000000", "30M", "1G", "2G"
|
||||
fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
|
||||
let s = s.trim();
|
||||
if s.is_empty() {
|
||||
return Err(eyre::eyre!("empty value"));
|
||||
}
|
||||
|
||||
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
|
||||
(prefix, 1_000_000_000u64)
|
||||
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
|
||||
(prefix, 1_000_000u64)
|
||||
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
|
||||
(prefix, 1_000u64)
|
||||
} else {
|
||||
(s, 1u64)
|
||||
};
|
||||
|
||||
let base: u64 = num_str.trim().parse()?;
|
||||
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
|
||||
}
|
||||
|
||||
/// `reth benchmark gas-limit-ramp` command.
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Command {
|
||||
/// Number of blocks to generate.
|
||||
#[arg(long, value_name = "BLOCKS")]
|
||||
blocks: u64,
|
||||
/// Number of blocks to generate. Mutually exclusive with --target-gas-limit.
|
||||
#[arg(long, value_name = "BLOCKS", conflicts_with = "target_gas_limit")]
|
||||
blocks: Option<u64>,
|
||||
|
||||
/// Target gas limit to ramp up to. The benchmark will generate blocks until the gas limit
|
||||
/// reaches or exceeds this value. Mutually exclusive with --blocks.
|
||||
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 2G = 2
|
||||
/// billion).
|
||||
#[arg(long, value_name = "TARGET_GAS_LIMIT", conflicts_with = "blocks", value_parser = parse_gas_limit)]
|
||||
target_gas_limit: Option<u64>,
|
||||
|
||||
/// The Engine API RPC URL.
|
||||
#[arg(long = "engine-rpc-url", value_name = "ENGINE_RPC_URL")]
|
||||
@@ -42,12 +72,37 @@ pub struct Command {
|
||||
output: PathBuf,
|
||||
}
|
||||
|
||||
/// Mode for determining when to stop ramping.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum RampMode {
|
||||
/// Ramp for a fixed number of blocks.
|
||||
Blocks(u64),
|
||||
/// Ramp until reaching or exceeding target gas limit.
|
||||
TargetGasLimit(u64),
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `benchmark gas-limit-ramp` command.
|
||||
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
|
||||
if self.blocks == 0 {
|
||||
return Err(eyre::eyre!("--blocks must be greater than 0"));
|
||||
}
|
||||
let mode = match (self.blocks, self.target_gas_limit) {
|
||||
(Some(blocks), None) => {
|
||||
if blocks == 0 {
|
||||
return Err(eyre::eyre!("--blocks must be greater than 0"));
|
||||
}
|
||||
RampMode::Blocks(blocks)
|
||||
}
|
||||
(None, Some(target)) => {
|
||||
if target == 0 {
|
||||
return Err(eyre::eyre!("--target-gas-limit must be greater than 0"));
|
||||
}
|
||||
RampMode::TargetGasLimit(target)
|
||||
}
|
||||
_ => {
|
||||
return Err(eyre::eyre!(
|
||||
"Exactly one of --blocks or --target-gas-limit must be specified"
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
// Ensure output directory exists
|
||||
if self.output.is_file() {
|
||||
@@ -84,14 +139,31 @@ impl Command {
|
||||
|
||||
let canonical_parent = parent_header.number;
|
||||
let start_block = canonical_parent + 1;
|
||||
let end_block = start_block + self.blocks - 1;
|
||||
|
||||
info!(canonical_parent, start_block, end_block, "Starting gas limit ramp benchmark");
|
||||
match mode {
|
||||
RampMode::Blocks(blocks) => {
|
||||
info!(
|
||||
canonical_parent,
|
||||
start_block,
|
||||
end_block = start_block + blocks - 1,
|
||||
"Starting gas limit ramp benchmark (block count mode)"
|
||||
);
|
||||
}
|
||||
RampMode::TargetGasLimit(target) => {
|
||||
info!(
|
||||
canonical_parent,
|
||||
start_block,
|
||||
current_gas_limit = parent_header.gas_limit,
|
||||
target_gas_limit = target,
|
||||
"Starting gas limit ramp benchmark (target gas limit mode)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut next_block_number = start_block;
|
||||
let mut blocks_processed = 0u64;
|
||||
let total_benchmark_duration = Instant::now();
|
||||
|
||||
while next_block_number <= end_block {
|
||||
while !should_stop(mode, blocks_processed, parent_header.gas_limit) {
|
||||
let timestamp = parent_header.timestamp.saturating_add(1);
|
||||
|
||||
let request = prepare_payload_request(&chain_spec, timestamp, parent_hash);
|
||||
@@ -140,13 +212,13 @@ impl Command {
|
||||
|
||||
parent_header = block.header;
|
||||
parent_hash = block_hash;
|
||||
next_block_number += 1;
|
||||
blocks_processed += 1;
|
||||
}
|
||||
|
||||
let final_gas_limit = parent_header.gas_limit;
|
||||
info!(
|
||||
total_duration=?total_benchmark_duration.elapsed(),
|
||||
blocks_processed = self.blocks,
|
||||
blocks_processed,
|
||||
final_gas_limit,
|
||||
"Benchmark complete"
|
||||
);
|
||||
@@ -158,3 +230,57 @@ impl Command {
|
||||
const fn max_gas_limit_increase(parent_gas_limit: u64) -> u64 {
|
||||
(parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR).saturating_sub(1)
|
||||
}
|
||||
|
||||
const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u64) -> bool {
|
||||
match mode {
|
||||
RampMode::Blocks(target_blocks) => blocks_processed >= target_blocks,
|
||||
RampMode::TargetGasLimit(target) => current_gas_limit >= target,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_plain_number() {
|
||||
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
|
||||
assert_eq!(parse_gas_limit("1").unwrap(), 1);
|
||||
assert_eq!(parse_gas_limit("0").unwrap(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_k_suffix() {
|
||||
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
|
||||
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
|
||||
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_m_suffix() {
|
||||
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
|
||||
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
|
||||
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_g_suffix() {
|
||||
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
|
||||
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
|
||||
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_with_whitespace() {
|
||||
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
|
||||
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_errors() {
|
||||
assert!(parse_gas_limit("").is_err());
|
||||
assert!(parse_gas_limit("abc").is_err());
|
||||
assert!(parse_gas_limit("G").is_err());
|
||||
assert!(parse_gas_limit("-1G").is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,7 +180,7 @@ impl Command {
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| {
|
||||
e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
|
||||
e.file_name().to_string_lossy().starts_with("payload_")
|
||||
e.file_name().to_string_lossy().starts_with("payload_block_")
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -191,7 +191,7 @@ impl Command {
|
||||
let name = e.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
// Extract index from "payload_NNN.json"
|
||||
let index_str = name_str.strip_prefix("payload_")?.strip_suffix(".json")?;
|
||||
let index_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
|
||||
let index: u64 = index_str.parse().ok()?;
|
||||
Some((index, e.path()))
|
||||
})
|
||||
|
||||
@@ -41,6 +41,7 @@ derive_more.workspace = true
|
||||
metrics.workspace = true
|
||||
parking_lot.workspace = true
|
||||
pin-project.workspace = true
|
||||
rayon = { workspace = true, optional = true }
|
||||
serde = { workspace = true, optional = true }
|
||||
|
||||
# optional deps for test-utils
|
||||
@@ -84,6 +85,7 @@ test-utils = [
|
||||
"reth-trie/test-utils",
|
||||
"reth-ethereum-primitives/test-utils",
|
||||
]
|
||||
rayon = ["dep:rayon"]
|
||||
|
||||
[[bench]]
|
||||
name = "canonical_hashes_range"
|
||||
|
||||
@@ -163,14 +163,29 @@ impl DeferredTrieData {
|
||||
anchor_hash: B256,
|
||||
ancestors: &[Self],
|
||||
) -> ComputedTrieData {
|
||||
let sorted_hashed_state = match Arc::try_unwrap(hashed_state) {
|
||||
Ok(state) => state.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
};
|
||||
let sorted_trie_updates = match Arc::try_unwrap(trie_updates) {
|
||||
Ok(updates) => updates.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
};
|
||||
#[cfg(feature = "rayon")]
|
||||
let (sorted_hashed_state, sorted_trie_updates) = rayon::join(
|
||||
|| match Arc::try_unwrap(hashed_state) {
|
||||
Ok(state) => state.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
},
|
||||
|| match Arc::try_unwrap(trie_updates) {
|
||||
Ok(updates) => updates.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
},
|
||||
);
|
||||
|
||||
#[cfg(not(feature = "rayon"))]
|
||||
let (sorted_hashed_state, sorted_trie_updates) = (
|
||||
match Arc::try_unwrap(hashed_state) {
|
||||
Ok(state) => state.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
},
|
||||
match Arc::try_unwrap(trie_updates) {
|
||||
Ok(updates) => updates.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
},
|
||||
);
|
||||
|
||||
// Reuse parent's overlay if available and anchors match.
|
||||
// We can only reuse the parent's overlay if it was built on top of the same
|
||||
@@ -192,10 +207,10 @@ impl DeferredTrieData {
|
||||
);
|
||||
// Only trigger COW clone if there's actually data to add.
|
||||
if !sorted_hashed_state.is_empty() {
|
||||
Arc::make_mut(&mut overlay.state).extend_ref(&sorted_hashed_state);
|
||||
Arc::make_mut(&mut overlay.state).extend_ref_and_sort(&sorted_hashed_state);
|
||||
}
|
||||
if !sorted_trie_updates.is_empty() {
|
||||
Arc::make_mut(&mut overlay.nodes).extend_ref(&sorted_trie_updates);
|
||||
Arc::make_mut(&mut overlay.nodes).extend_ref_and_sort(&sorted_trie_updates);
|
||||
}
|
||||
overlay
|
||||
}
|
||||
@@ -228,8 +243,53 @@ impl DeferredTrieData {
|
||||
/// In normal operation, the parent always has a cached overlay and this
|
||||
/// function is never called.
|
||||
///
|
||||
/// Iterates ancestors oldest -> newest, then extends with current block's data,
|
||||
/// so later state takes precedence.
|
||||
/// When the `rayon` feature is enabled, uses parallel collection and merge:
|
||||
/// 1. Collects ancestor data in parallel (each `wait_cloned()` may compute)
|
||||
/// 2. Merges hashed state and trie updates in parallel with each other
|
||||
/// 3. Uses tree reduction within each merge for O(log n) depth
|
||||
#[cfg(feature = "rayon")]
|
||||
fn merge_ancestors_into_overlay(
|
||||
ancestors: &[Self],
|
||||
sorted_hashed_state: &HashedPostStateSorted,
|
||||
sorted_trie_updates: &TrieUpdatesSorted,
|
||||
) -> TrieInputSorted {
|
||||
// Early exit: no ancestors means just wrap current block's data
|
||||
if ancestors.is_empty() {
|
||||
return TrieInputSorted::new(
|
||||
Arc::new(sorted_trie_updates.clone()),
|
||||
Arc::new(sorted_hashed_state.clone()),
|
||||
Default::default(),
|
||||
);
|
||||
}
|
||||
|
||||
// Collect ancestor data, unzipping states and updates into Arc slices
|
||||
let (states, updates): (Vec<_>, Vec<_>) = ancestors
|
||||
.iter()
|
||||
.map(|a| {
|
||||
let data = a.wait_cloned();
|
||||
(data.hashed_state, data.trie_updates)
|
||||
})
|
||||
.unzip();
|
||||
|
||||
// Merge state and nodes in parallel with each other using tree reduction
|
||||
let (state, nodes) = rayon::join(
|
||||
|| {
|
||||
let mut merged = HashedPostStateSorted::merge_parallel(&states);
|
||||
merged.extend_ref_and_sort(sorted_hashed_state);
|
||||
merged
|
||||
},
|
||||
|| {
|
||||
let mut merged = TrieUpdatesSorted::merge_parallel(&updates);
|
||||
merged.extend_ref_and_sort(sorted_trie_updates);
|
||||
merged
|
||||
},
|
||||
);
|
||||
|
||||
TrieInputSorted::new(Arc::new(nodes), Arc::new(state), Default::default())
|
||||
}
|
||||
|
||||
/// Merge all ancestors and current block's data into a single overlay (sequential fallback).
|
||||
#[cfg(not(feature = "rayon"))]
|
||||
fn merge_ancestors_into_overlay(
|
||||
ancestors: &[Self],
|
||||
sorted_hashed_state: &HashedPostStateSorted,
|
||||
@@ -242,13 +302,13 @@ impl DeferredTrieData {
|
||||
|
||||
for ancestor in ancestors {
|
||||
let ancestor_data = ancestor.wait_cloned();
|
||||
state_mut.extend_ref(ancestor_data.hashed_state.as_ref());
|
||||
nodes_mut.extend_ref(ancestor_data.trie_updates.as_ref());
|
||||
state_mut.extend_ref_and_sort(ancestor_data.hashed_state.as_ref());
|
||||
nodes_mut.extend_ref_and_sort(ancestor_data.trie_updates.as_ref());
|
||||
}
|
||||
|
||||
// Extend with current block's sorted data last (takes precedence)
|
||||
state_mut.extend_ref(sorted_hashed_state);
|
||||
nodes_mut.extend_ref(sorted_trie_updates);
|
||||
state_mut.extend_ref_and_sort(sorted_hashed_state);
|
||||
nodes_mut.extend_ref_and_sort(sorted_trie_updates);
|
||||
|
||||
overlay
|
||||
}
|
||||
@@ -521,7 +581,7 @@ mod tests {
|
||||
let hashed_state = Arc::new(HashedPostStateSorted::new(accounts, B256Map::default()));
|
||||
let trie_updates = Arc::default();
|
||||
let mut overlay = TrieInputSorted::default();
|
||||
Arc::make_mut(&mut overlay.state).extend_ref(hashed_state.as_ref());
|
||||
Arc::make_mut(&mut overlay.state).extend_ref_and_sort(hashed_state.as_ref());
|
||||
|
||||
DeferredTrieData::ready(ComputedTrieData {
|
||||
hashed_state,
|
||||
|
||||
@@ -123,60 +123,18 @@ impl LazyOverlay {
|
||||
|
||||
/// Merge all blocks' trie data into a single [`TrieInputSorted`].
|
||||
///
|
||||
/// Blocks are ordered newest to oldest. Uses hybrid merge algorithm that
|
||||
/// switches between `extend_ref` (small batches) and k-way merge (large batches).
|
||||
/// Blocks are ordered newest to oldest.
|
||||
fn merge_blocks(blocks: &[DeferredTrieData]) -> TrieInputSorted {
|
||||
const MERGE_BATCH_THRESHOLD: usize = 64;
|
||||
|
||||
if blocks.is_empty() {
|
||||
return TrieInputSorted::default();
|
||||
}
|
||||
|
||||
// Single block: use its data directly (no allocation)
|
||||
if blocks.len() == 1 {
|
||||
let data = blocks[0].wait_cloned();
|
||||
return TrieInputSorted {
|
||||
state: data.hashed_state,
|
||||
nodes: data.trie_updates,
|
||||
prefix_sets: Default::default(),
|
||||
};
|
||||
}
|
||||
let state =
|
||||
HashedPostStateSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().hashed_state));
|
||||
let nodes =
|
||||
TrieUpdatesSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().trie_updates));
|
||||
|
||||
if blocks.len() < MERGE_BATCH_THRESHOLD {
|
||||
// Small k: extend_ref loop with Arc::make_mut is faster.
|
||||
// Uses copy-on-write - only clones inner data if Arc has multiple refs.
|
||||
// Iterate oldest->newest so newer values override older ones.
|
||||
let mut blocks_iter = blocks.iter().rev();
|
||||
let first = blocks_iter.next().expect("blocks is non-empty");
|
||||
let data = first.wait_cloned();
|
||||
|
||||
let mut state = data.hashed_state;
|
||||
let mut nodes = data.trie_updates;
|
||||
|
||||
for block in blocks_iter {
|
||||
let block_data = block.wait_cloned();
|
||||
Arc::make_mut(&mut state).extend_ref(block_data.hashed_state.as_ref());
|
||||
Arc::make_mut(&mut nodes).extend_ref(block_data.trie_updates.as_ref());
|
||||
}
|
||||
|
||||
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
|
||||
} else {
|
||||
// Large k: k-way merge is faster (O(n log k)).
|
||||
// Collect is unavoidable here - we need all data materialized for k-way merge.
|
||||
let trie_data: Vec<_> = blocks.iter().map(|b| b.wait_cloned()).collect();
|
||||
|
||||
let merged_state = HashedPostStateSorted::merge_batch(
|
||||
trie_data.iter().map(|d| d.hashed_state.as_ref()),
|
||||
);
|
||||
let merged_nodes =
|
||||
TrieUpdatesSorted::merge_batch(trie_data.iter().map(|d| d.trie_updates.as_ref()));
|
||||
|
||||
TrieInputSorted {
|
||||
state: Arc::new(merged_state),
|
||||
nodes: Arc::new(merged_nodes),
|
||||
prefix_sets: Default::default(),
|
||||
}
|
||||
}
|
||||
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -278,6 +278,7 @@ pub fn create_chain_config(
|
||||
// Check if DAO fork is supported (it has an activation block)
|
||||
let dao_fork_support = hardforks.fork(EthereumHardfork::Dao) != ForkCondition::Never;
|
||||
|
||||
#[allow(clippy::needless_update)]
|
||||
ChainConfig {
|
||||
chain_id: chain.map(|c| c.id()).unwrap_or(0),
|
||||
homestead_block: block_num(EthereumHardfork::Homestead),
|
||||
@@ -313,6 +314,7 @@ pub fn create_chain_config(
|
||||
extra_fields: Default::default(),
|
||||
deposit_contract_address,
|
||||
blob_schedule,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -131,4 +131,4 @@ arbitrary = [
|
||||
"reth-ethereum-primitives/arbitrary",
|
||||
]
|
||||
|
||||
edge = ["reth-db-common/edge", "reth-stages/rocksdb"]
|
||||
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb"]
|
||||
|
||||
@@ -19,7 +19,7 @@ use reth_node_builder::{
|
||||
Node, NodeComponents, NodeComponentsBuilder, NodeTypes, NodeTypesWithDBAdapter,
|
||||
};
|
||||
use reth_node_core::{
|
||||
args::{DatabaseArgs, DatadirArgs, StaticFilesArgs},
|
||||
args::{DatabaseArgs, DatadirArgs, RocksDbArgs, StaticFilesArgs},
|
||||
dirs::{ChainPath, DataDirPath},
|
||||
};
|
||||
use reth_provider::{
|
||||
@@ -27,7 +27,7 @@ use reth_provider::{
|
||||
BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider,
|
||||
StaticFileProviderBuilder,
|
||||
},
|
||||
ProviderFactory, StaticFileProviderFactory,
|
||||
ProviderFactory, StaticFileProviderFactory, StorageSettings,
|
||||
};
|
||||
use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget};
|
||||
use reth_static_file::StaticFileProducer;
|
||||
@@ -66,9 +66,24 @@ pub struct EnvironmentArgs<C: ChainSpecParser> {
|
||||
/// All static files related arguments
|
||||
#[command(flatten)]
|
||||
pub static_files: StaticFilesArgs,
|
||||
|
||||
/// All `RocksDB` related arguments
|
||||
#[command(flatten)]
|
||||
pub rocksdb: RocksDbArgs,
|
||||
}
|
||||
|
||||
impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
/// Returns the effective storage settings derived from static-file and `RocksDB` CLI args.
|
||||
pub fn storage_settings(&self) -> StorageSettings {
|
||||
StorageSettings::base()
|
||||
.with_receipts_in_static_files(self.static_files.receipts)
|
||||
.with_transaction_senders_in_static_files(self.static_files.transaction_senders)
|
||||
.with_account_changesets_in_static_files(self.static_files.account_changesets)
|
||||
.with_transaction_hash_numbers_in_rocksdb(self.rocksdb.all || self.rocksdb.tx_hash)
|
||||
.with_storages_history_in_rocksdb(self.rocksdb.all || self.rocksdb.storages_history)
|
||||
.with_account_history_in_rocksdb(self.rocksdb.all || self.rocksdb.account_history)
|
||||
}
|
||||
|
||||
/// Initializes environment according to [`AccessRights`] and returns an instance of
|
||||
/// [`Environment`].
|
||||
pub fn init<N: CliNodeTypes>(&self, access: AccessRights) -> eyre::Result<Environment<N>>
|
||||
@@ -121,17 +136,17 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
})
|
||||
}
|
||||
};
|
||||
// TransactionDB only support read-write mode
|
||||
let rocksdb_provider = RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level)
|
||||
.with_read_only(!access.is_read_write())
|
||||
.build()?;
|
||||
|
||||
let provider_factory =
|
||||
self.create_provider_factory(&config, db, sfp, rocksdb_provider, access)?;
|
||||
if access.is_read_write() {
|
||||
debug!(target: "reth::cli", chain=%self.chain.chain(), genesis=?self.chain.genesis_hash(), "Initializing genesis");
|
||||
init_genesis_with_settings(&provider_factory, self.static_files.to_settings())?;
|
||||
init_genesis_with_settings(&provider_factory, self.storage_settings())?;
|
||||
}
|
||||
|
||||
Ok(Environment { config, provider_factory, data_dir })
|
||||
|
||||
@@ -1,145 +0,0 @@
|
||||
use crate::{
|
||||
common::CliNodeTypes,
|
||||
db::get::{maybe_json_value_parser, table_key},
|
||||
};
|
||||
use alloy_primitives::map::foldhash::fast::FixedState;
|
||||
use clap::Parser;
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_db::DatabaseEnv;
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
|
||||
Tables,
|
||||
};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
|
||||
use reth_provider::{providers::ProviderNodeTypes, DBProvider};
|
||||
use std::{
|
||||
hash::{BuildHasher, Hasher},
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
/// The arguments for the `reth db checksum` command
|
||||
pub struct Command {
|
||||
/// The table name
|
||||
table: Tables,
|
||||
|
||||
/// The start of the range to checksum.
|
||||
#[arg(long, value_parser = maybe_json_value_parser)]
|
||||
start_key: Option<String>,
|
||||
|
||||
/// The end of the range to checksum.
|
||||
#[arg(long, value_parser = maybe_json_value_parser)]
|
||||
end_key: Option<String>,
|
||||
|
||||
/// The maximum number of records that are queried and used to compute the
|
||||
/// checksum.
|
||||
#[arg(long)]
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `db checksum` command
|
||||
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
self,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
) -> eyre::Result<()> {
|
||||
warn!("This command should be run without the node running!");
|
||||
self.table.view(&ChecksumViewer {
|
||||
tool,
|
||||
start_key: self.start_key,
|
||||
end_key: self.end_key,
|
||||
limit: self.limit,
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
|
||||
tool: &'a DbTool<N>,
|
||||
start_key: Option<String>,
|
||||
end_key: Option<String>,
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
|
||||
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
|
||||
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
|
||||
type Error = eyre::Report;
|
||||
|
||||
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
|
||||
let provider =
|
||||
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
|
||||
let tx = provider.tx_ref();
|
||||
info!(
|
||||
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
|
||||
self.start_key, self.end_key, self.limit
|
||||
);
|
||||
|
||||
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
|
||||
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
|
||||
(Some(start), Some(end)) => {
|
||||
let start_key = table_key::<T>(start).map(RawKey::new)?;
|
||||
let end_key = table_key::<T>(end).map(RawKey::new)?;
|
||||
cursor.walk_range(start_key..=end_key)?
|
||||
}
|
||||
(None, Some(end)) => {
|
||||
let end_key = table_key::<T>(end).map(RawKey::new)?;
|
||||
|
||||
cursor.walk_range(..=end_key)?
|
||||
}
|
||||
(Some(start), None) => {
|
||||
let start_key = table_key::<T>(start).map(RawKey::new)?;
|
||||
cursor.walk_range(start_key..)?
|
||||
}
|
||||
(None, None) => cursor.walk_range(..)?,
|
||||
};
|
||||
|
||||
let start_time = Instant::now();
|
||||
let mut hasher = FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher();
|
||||
let mut total = 0;
|
||||
|
||||
let limit = self.limit.unwrap_or(usize::MAX);
|
||||
let mut enumerate_start_key = None;
|
||||
let mut enumerate_end_key = None;
|
||||
for (index, entry) in walker.enumerate() {
|
||||
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
|
||||
|
||||
if index.is_multiple_of(100_000) {
|
||||
info!("Hashed {index} entries.");
|
||||
}
|
||||
|
||||
hasher.write(k.raw_key());
|
||||
hasher.write(v.raw_value());
|
||||
|
||||
if enumerate_start_key.is_none() {
|
||||
enumerate_start_key = Some(k.clone());
|
||||
}
|
||||
enumerate_end_key = Some(k);
|
||||
|
||||
total = index + 1;
|
||||
if total >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
info!("Hashed {total} entries.");
|
||||
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
|
||||
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
|
||||
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
|
||||
}
|
||||
|
||||
let checksum = hasher.finish();
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
|
||||
|
||||
Ok((checksum, elapsed))
|
||||
}
|
||||
}
|
||||
288
crates/cli/commands/src/db/checksum/mod.rs
Normal file
288
crates/cli/commands/src/db/checksum/mod.rs
Normal file
@@ -0,0 +1,288 @@
|
||||
use crate::{
|
||||
common::CliNodeTypes,
|
||||
db::get::{maybe_json_value_parser, table_key},
|
||||
};
|
||||
use alloy_primitives::map::foldhash::fast::FixedState;
|
||||
use clap::Parser;
|
||||
use itertools::Itertools;
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_db::{static_file::iter_static_files, DatabaseEnv};
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
|
||||
Tables,
|
||||
};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
|
||||
use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProviderFactory};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use std::{
|
||||
hash::{BuildHasher, Hasher},
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
#[cfg(all(unix, feature = "edge"))]
|
||||
mod rocksdb;
|
||||
|
||||
/// Interval for logging progress during checksum computation.
|
||||
const PROGRESS_LOG_INTERVAL: usize = 100_000;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
/// The arguments for the `reth db checksum` command
|
||||
pub struct Command {
|
||||
#[command(subcommand)]
|
||||
subcommand: Subcommand,
|
||||
}
|
||||
|
||||
#[derive(clap::Subcommand, Debug)]
|
||||
enum Subcommand {
|
||||
/// Calculates the checksum of a database table
|
||||
Mdbx {
|
||||
/// The table name
|
||||
table: Tables,
|
||||
|
||||
/// The start of the range to checksum.
|
||||
#[arg(long, value_parser = maybe_json_value_parser)]
|
||||
start_key: Option<String>,
|
||||
|
||||
/// The end of the range to checksum.
|
||||
#[arg(long, value_parser = maybe_json_value_parser)]
|
||||
end_key: Option<String>,
|
||||
|
||||
/// The maximum number of records that are queried and used to compute the
|
||||
/// checksum.
|
||||
#[arg(long)]
|
||||
limit: Option<usize>,
|
||||
},
|
||||
/// Calculates the checksum of a static file segment
|
||||
StaticFile {
|
||||
/// The static file segment
|
||||
#[arg(value_enum)]
|
||||
segment: StaticFileSegment,
|
||||
|
||||
/// The block number to start from (inclusive).
|
||||
#[arg(long)]
|
||||
start_block: Option<u64>,
|
||||
|
||||
/// The block number to end at (inclusive).
|
||||
#[arg(long)]
|
||||
end_block: Option<u64>,
|
||||
|
||||
/// The maximum number of rows to checksum.
|
||||
#[arg(long)]
|
||||
limit: Option<usize>,
|
||||
},
|
||||
/// Calculates the checksum of a RocksDB table
|
||||
#[cfg(all(unix, feature = "edge"))]
|
||||
Rocksdb {
|
||||
/// The RocksDB table
|
||||
#[arg(value_enum)]
|
||||
table: rocksdb::RocksDbTable,
|
||||
|
||||
/// The maximum number of records to checksum.
|
||||
#[arg(long)]
|
||||
limit: Option<usize>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `db checksum` command
|
||||
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
self,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
) -> eyre::Result<()> {
|
||||
warn!("This command should be run without the node running!");
|
||||
|
||||
match self.subcommand {
|
||||
Subcommand::Mdbx { table, start_key, end_key, limit } => {
|
||||
table.view(&ChecksumViewer { tool, start_key, end_key, limit })?;
|
||||
}
|
||||
Subcommand::StaticFile { segment, start_block, end_block, limit } => {
|
||||
checksum_static_file(tool, segment, start_block, end_block, limit)?;
|
||||
}
|
||||
#[cfg(all(unix, feature = "edge"))]
|
||||
Subcommand::Rocksdb { table, limit } => {
|
||||
rocksdb::checksum_rocksdb(tool, table, limit)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a new hasher with the standard seed used for checksum computation.
|
||||
fn checksum_hasher() -> impl Hasher {
|
||||
FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher()
|
||||
}
|
||||
|
||||
fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
segment: StaticFileSegment,
|
||||
start_block: Option<u64>,
|
||||
end_block: Option<u64>,
|
||||
limit: Option<usize>,
|
||||
) -> eyre::Result<()> {
|
||||
let static_file_provider = tool.provider_factory.static_file_provider();
|
||||
if let Err(err) = static_file_provider.check_consistency(&tool.provider_factory.provider()?) {
|
||||
warn!("Error checking consistency of static files: {err}");
|
||||
}
|
||||
|
||||
let static_files = iter_static_files(static_file_provider.directory())?;
|
||||
|
||||
let ranges = static_files
|
||||
.get(segment)
|
||||
.ok_or_else(|| eyre::eyre!("No static files found for segment: {}", segment))?;
|
||||
|
||||
let start_time = Instant::now();
|
||||
let mut hasher = checksum_hasher();
|
||||
let mut total = 0usize;
|
||||
let limit = limit.unwrap_or(usize::MAX);
|
||||
|
||||
let start_block = start_block.unwrap_or(0);
|
||||
let end_block = end_block.unwrap_or(u64::MAX);
|
||||
|
||||
info!(
|
||||
"Computing checksum for {} static files, start_block={}, end_block={}, limit={:?}",
|
||||
segment,
|
||||
start_block,
|
||||
end_block,
|
||||
if limit == usize::MAX { None } else { Some(limit) }
|
||||
);
|
||||
|
||||
'outer: for (block_range, _header) in ranges.iter().sorted_by_key(|(range, _)| range.start()) {
|
||||
if block_range.end() < start_block || block_range.start() > end_block {
|
||||
continue;
|
||||
}
|
||||
|
||||
let fixed_block_range = static_file_provider.find_fixed_range(segment, block_range.start());
|
||||
let jar_provider = static_file_provider
|
||||
.get_segment_provider_for_range(segment, || Some(fixed_block_range), None)?
|
||||
.ok_or_else(|| {
|
||||
eyre::eyre!(
|
||||
"Failed to get segment provider for segment {} at range {}",
|
||||
segment,
|
||||
block_range
|
||||
)
|
||||
})?;
|
||||
|
||||
let mut cursor = jar_provider.cursor()?;
|
||||
|
||||
while let Ok(Some(row)) = cursor.next_row() {
|
||||
for col_data in row.iter() {
|
||||
hasher.write(col_data);
|
||||
}
|
||||
|
||||
total += 1;
|
||||
|
||||
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
|
||||
info!("Hashed {total} entries.");
|
||||
}
|
||||
|
||||
if total >= limit {
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
|
||||
// Explicitly drop provider before removing from cache to avoid deadlock
|
||||
drop(jar_provider);
|
||||
static_file_provider.remove_cached_provider(segment, fixed_block_range.end());
|
||||
}
|
||||
|
||||
let checksum = hasher.finish();
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
info!(
|
||||
"Checksum for static file segment `{}`: {:#x} ({} entries, elapsed: {:?})",
|
||||
segment, checksum, total, elapsed
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
|
||||
tool: &'a DbTool<N>,
|
||||
start_key: Option<String>,
|
||||
end_key: Option<String>,
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
|
||||
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
|
||||
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
|
||||
type Error = eyre::Report;
|
||||
|
||||
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
|
||||
let provider =
|
||||
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
|
||||
let tx = provider.tx_ref();
|
||||
info!(
|
||||
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
|
||||
self.start_key, self.end_key, self.limit
|
||||
);
|
||||
|
||||
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
|
||||
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
|
||||
(Some(start), Some(end)) => {
|
||||
let start_key = table_key::<T>(start).map(RawKey::new)?;
|
||||
let end_key = table_key::<T>(end).map(RawKey::new)?;
|
||||
cursor.walk_range(start_key..=end_key)?
|
||||
}
|
||||
(None, Some(end)) => {
|
||||
let end_key = table_key::<T>(end).map(RawKey::new)?;
|
||||
|
||||
cursor.walk_range(..=end_key)?
|
||||
}
|
||||
(Some(start), None) => {
|
||||
let start_key = table_key::<T>(start).map(RawKey::new)?;
|
||||
cursor.walk_range(start_key..)?
|
||||
}
|
||||
(None, None) => cursor.walk_range(..)?,
|
||||
};
|
||||
|
||||
let start_time = Instant::now();
|
||||
let mut hasher = checksum_hasher();
|
||||
let mut total = 0;
|
||||
|
||||
let limit = self.limit.unwrap_or(usize::MAX);
|
||||
let mut enumerate_start_key = None;
|
||||
let mut enumerate_end_key = None;
|
||||
for (index, entry) in walker.enumerate() {
|
||||
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
|
||||
|
||||
if index.is_multiple_of(PROGRESS_LOG_INTERVAL) {
|
||||
info!("Hashed {index} entries.");
|
||||
}
|
||||
|
||||
hasher.write(k.raw_key());
|
||||
hasher.write(v.raw_value());
|
||||
|
||||
if enumerate_start_key.is_none() {
|
||||
enumerate_start_key = Some(k.clone());
|
||||
}
|
||||
enumerate_end_key = Some(k);
|
||||
|
||||
total = index + 1;
|
||||
if total >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
info!("Hashed {total} entries.");
|
||||
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
|
||||
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
|
||||
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
|
||||
}
|
||||
|
||||
let checksum = hasher.finish();
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
|
||||
|
||||
Ok((checksum, elapsed))
|
||||
}
|
||||
}
|
||||
106
crates/cli/commands/src/db/checksum/rocksdb.rs
Normal file
106
crates/cli/commands/src/db/checksum/rocksdb.rs
Normal file
@@ -0,0 +1,106 @@
|
||||
//! RocksDB checksum implementation.
|
||||
|
||||
use super::{checksum_hasher, PROGRESS_LOG_INTERVAL};
|
||||
use crate::common::CliNodeTypes;
|
||||
use clap::ValueEnum;
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_db::{tables, DatabaseEnv};
|
||||
use reth_db_api::table::Table;
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::NodeTypesWithDBAdapter;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use std::{hash::Hasher, sync::Arc, time::Instant};
|
||||
use tracing::info;
|
||||
|
||||
/// RocksDB tables that can be checksummed.
|
||||
#[derive(Debug, Clone, Copy, ValueEnum)]
|
||||
pub enum RocksDbTable {
|
||||
/// Transaction hash to transaction number mapping
|
||||
TransactionHashNumbers,
|
||||
/// Account history indices
|
||||
AccountsHistory,
|
||||
/// Storage history indices
|
||||
StoragesHistory,
|
||||
}
|
||||
|
||||
impl RocksDbTable {
|
||||
/// Returns the table name as a string
|
||||
const fn name(&self) -> &'static str {
|
||||
match self {
|
||||
Self::TransactionHashNumbers => tables::TransactionHashNumbers::NAME,
|
||||
Self::AccountsHistory => tables::AccountsHistory::NAME,
|
||||
Self::StoragesHistory => tables::StoragesHistory::NAME,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Computes a checksum for a RocksDB table.
|
||||
pub fn checksum_rocksdb<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
table: RocksDbTable,
|
||||
limit: Option<usize>,
|
||||
) -> eyre::Result<()> {
|
||||
let rocksdb = tool.provider_factory.rocksdb_provider();
|
||||
|
||||
let start_time = Instant::now();
|
||||
let limit = limit.unwrap_or(usize::MAX);
|
||||
|
||||
info!(
|
||||
"Computing checksum for RocksDB table `{}`, limit={:?}",
|
||||
table.name(),
|
||||
if limit == usize::MAX { None } else { Some(limit) }
|
||||
);
|
||||
|
||||
let (checksum, total) = match table {
|
||||
RocksDbTable::TransactionHashNumbers => {
|
||||
checksum_rocksdb_table::<tables::TransactionHashNumbers>(&rocksdb, limit)?
|
||||
}
|
||||
RocksDbTable::AccountsHistory => {
|
||||
checksum_rocksdb_table::<tables::AccountsHistory>(&rocksdb, limit)?
|
||||
}
|
||||
RocksDbTable::StoragesHistory => {
|
||||
checksum_rocksdb_table::<tables::StoragesHistory>(&rocksdb, limit)?
|
||||
}
|
||||
};
|
||||
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
info!(
|
||||
"Checksum for RocksDB table `{}`: {:#x} ({} entries, elapsed: {:?})",
|
||||
table.name(),
|
||||
checksum,
|
||||
total,
|
||||
elapsed
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Computes checksum for a specific RocksDB table by iterating over rows.
|
||||
fn checksum_rocksdb_table<T: Table>(
|
||||
rocksdb: &reth_provider::providers::RocksDBProvider,
|
||||
limit: usize,
|
||||
) -> eyre::Result<(u64, usize)> {
|
||||
let iter = rocksdb.raw_iter::<T>()?;
|
||||
let mut hasher = checksum_hasher();
|
||||
let mut total = 0usize;
|
||||
|
||||
for entry in iter {
|
||||
let (key_bytes, value_bytes) = entry?;
|
||||
|
||||
hasher.write(&key_bytes);
|
||||
hasher.write(&value_bytes);
|
||||
|
||||
total += 1;
|
||||
|
||||
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
|
||||
info!("Hashed {total} entries.");
|
||||
}
|
||||
|
||||
if total >= limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok((hasher.finish(), total))
|
||||
}
|
||||
@@ -39,7 +39,7 @@ pub enum Subcommands {
|
||||
Stats(stats::Command),
|
||||
/// Lists the contents of a table
|
||||
List(list::Command),
|
||||
/// Calculates the content checksum of a table
|
||||
/// Calculates the content checksum of a table or static file segment
|
||||
Checksum(checksum::Command),
|
||||
/// Create a diff between two database tables or two entire databases.
|
||||
Diff(diff::Command),
|
||||
|
||||
@@ -11,7 +11,10 @@ use reth_db_common::DbTool;
|
||||
use reth_fs_util as fs;
|
||||
use reth_node_builder::{NodePrimitives, NodeTypesWithDB, NodeTypesWithDBAdapter};
|
||||
use reth_node_core::dirs::{ChainPath, DataDirPath};
|
||||
use reth_provider::providers::{ProviderNodeTypes, StaticFileProvider};
|
||||
use reth_provider::{
|
||||
providers::{ProviderNodeTypes, StaticFileProvider},
|
||||
RocksDBProviderFactory,
|
||||
};
|
||||
use reth_static_file_types::SegmentRangeInclusive;
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
@@ -61,6 +64,11 @@ impl Command {
|
||||
let db_stats_table = self.db_stats_table(tool)?;
|
||||
println!("{db_stats_table}");
|
||||
|
||||
println!("\n");
|
||||
|
||||
let rocksdb_stats_table = self.rocksdb_stats_table(tool);
|
||||
println!("{rocksdb_stats_table}");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -148,6 +156,60 @@ impl Command {
|
||||
Ok(table)
|
||||
}
|
||||
|
||||
fn rocksdb_stats_table<N: NodeTypesWithDB>(&self, tool: &DbTool<N>) -> ComfyTable {
|
||||
let mut table = ComfyTable::new();
|
||||
table.load_preset(comfy_table::presets::ASCII_MARKDOWN);
|
||||
table.set_header([
|
||||
"RocksDB Table Name",
|
||||
"# Entries",
|
||||
"SST Size",
|
||||
"Memtable Size",
|
||||
"Total Size",
|
||||
"Pending Compaction",
|
||||
]);
|
||||
|
||||
let stats = tool.provider_factory.rocksdb_provider().table_stats();
|
||||
let mut total_sst: u64 = 0;
|
||||
let mut total_memtable: u64 = 0;
|
||||
let mut total_size: u64 = 0;
|
||||
let mut total_pending: u64 = 0;
|
||||
|
||||
for stat in &stats {
|
||||
total_sst += stat.sst_size_bytes;
|
||||
total_memtable += stat.memtable_size_bytes;
|
||||
total_size += stat.estimated_size_bytes;
|
||||
total_pending += stat.pending_compaction_bytes;
|
||||
let mut row = Row::new();
|
||||
row.add_cell(Cell::new(&stat.name))
|
||||
.add_cell(Cell::new(stat.estimated_num_keys))
|
||||
.add_cell(Cell::new(human_bytes(stat.sst_size_bytes as f64)))
|
||||
.add_cell(Cell::new(human_bytes(stat.memtable_size_bytes as f64)))
|
||||
.add_cell(Cell::new(human_bytes(stat.estimated_size_bytes as f64)))
|
||||
.add_cell(Cell::new(human_bytes(stat.pending_compaction_bytes as f64)));
|
||||
table.add_row(row);
|
||||
}
|
||||
|
||||
if !stats.is_empty() {
|
||||
let max_widths = table.column_max_content_widths();
|
||||
let mut separator = Row::new();
|
||||
for width in max_widths {
|
||||
separator.add_cell(Cell::new("-".repeat(width as usize)));
|
||||
}
|
||||
table.add_row(separator);
|
||||
|
||||
let mut row = Row::new();
|
||||
row.add_cell(Cell::new("RocksDB Total"))
|
||||
.add_cell(Cell::new(""))
|
||||
.add_cell(Cell::new(human_bytes(total_sst as f64)))
|
||||
.add_cell(Cell::new(human_bytes(total_memtable as f64)))
|
||||
.add_cell(Cell::new(human_bytes(total_size as f64)))
|
||||
.add_cell(Cell::new(human_bytes(total_pending as f64)));
|
||||
table.add_row(row);
|
||||
}
|
||||
|
||||
table
|
||||
}
|
||||
|
||||
fn static_files_stats_table<N: NodePrimitives>(
|
||||
&self,
|
||||
data_dir: ChainPath<DataDirPath>,
|
||||
|
||||
@@ -2,14 +2,15 @@ use crate::common::EnvironmentArgs;
|
||||
use clap::Parser;
|
||||
use eyre::Result;
|
||||
use lz4::Decoder;
|
||||
use reqwest::Client;
|
||||
use reqwest::{blocking::Client as BlockingClient, header::RANGE, Client, StatusCode};
|
||||
use reth_chainspec::{EthChainSpec, EthereumHardforks};
|
||||
use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_fs_util as fs;
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
io::{self, Read, Write},
|
||||
path::Path,
|
||||
fs::OpenOptions,
|
||||
io::{self, BufWriter, Read, Write},
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, OnceLock},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
@@ -327,18 +328,158 @@ fn extract_from_file(path: &Path, format: CompressionFormat, target_dir: &Path)
|
||||
extract_archive(file, total_size, format, target_dir)
|
||||
}
|
||||
|
||||
/// Fetches the snapshot from a remote URL, uncompressing it in a streaming fashion.
|
||||
const MAX_DOWNLOAD_RETRIES: u32 = 10;
|
||||
const RETRY_BACKOFF_SECS: u64 = 5;
|
||||
|
||||
/// Wrapper that tracks download progress while writing data.
|
||||
/// Used with [`io::copy`] to display progress during downloads.
|
||||
struct ProgressWriter<W> {
|
||||
inner: W,
|
||||
progress: DownloadProgress,
|
||||
}
|
||||
|
||||
impl<W: Write> Write for ProgressWriter<W> {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
let n = self.inner.write(buf)?;
|
||||
let _ = self.progress.update(n as u64);
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()
|
||||
}
|
||||
}
|
||||
|
||||
/// Downloads a file with resume support using HTTP Range requests.
|
||||
/// Automatically retries on failure, resuming from where it left off.
|
||||
/// Returns the path to the downloaded file and its total size.
|
||||
fn resumable_download(url: &str, target_dir: &Path) -> Result<(PathBuf, u64)> {
|
||||
let file_name = Url::parse(url)
|
||||
.ok()
|
||||
.and_then(|u| u.path_segments()?.next_back().map(|s| s.to_string()))
|
||||
.unwrap_or_else(|| "snapshot.tar".to_string());
|
||||
|
||||
let final_path = target_dir.join(&file_name);
|
||||
let part_path = target_dir.join(format!("{file_name}.part"));
|
||||
|
||||
let client = BlockingClient::builder().timeout(Duration::from_secs(30)).build()?;
|
||||
|
||||
let mut total_size: Option<u64> = None;
|
||||
let mut last_error: Option<eyre::Error> = None;
|
||||
|
||||
for attempt in 1..=MAX_DOWNLOAD_RETRIES {
|
||||
let existing_size = fs::metadata(&part_path).map(|m| m.len()).unwrap_or(0);
|
||||
|
||||
if let Some(total) = total_size &&
|
||||
existing_size >= total
|
||||
{
|
||||
fs::rename(&part_path, &final_path)?;
|
||||
info!(target: "reth::cli", "Download complete: {}", final_path.display());
|
||||
return Ok((final_path, total));
|
||||
}
|
||||
|
||||
if attempt > 1 {
|
||||
info!(target: "reth::cli",
|
||||
"Retry attempt {}/{} - resuming from {} bytes",
|
||||
attempt, MAX_DOWNLOAD_RETRIES, existing_size
|
||||
);
|
||||
}
|
||||
|
||||
let mut request = client.get(url);
|
||||
if existing_size > 0 {
|
||||
request = request.header(RANGE, format!("bytes={existing_size}-"));
|
||||
if attempt == 1 {
|
||||
info!(target: "reth::cli", "Resuming download from {} bytes", existing_size);
|
||||
}
|
||||
}
|
||||
|
||||
let response = match request.send().and_then(|r| r.error_for_status()) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
last_error = Some(e.into());
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
info!(target: "reth::cli",
|
||||
"Download failed, retrying in {} seconds...", RETRY_BACKOFF_SECS
|
||||
);
|
||||
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let is_partial = response.status() == StatusCode::PARTIAL_CONTENT;
|
||||
|
||||
let size = if is_partial {
|
||||
response
|
||||
.headers()
|
||||
.get("Content-Range")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.split('/').next_back())
|
||||
.and_then(|v| v.parse().ok())
|
||||
} else {
|
||||
response.content_length()
|
||||
};
|
||||
|
||||
if total_size.is_none() {
|
||||
total_size = size;
|
||||
}
|
||||
|
||||
let current_total = total_size.ok_or_else(|| {
|
||||
eyre::eyre!("Server did not provide Content-Length or Content-Range header")
|
||||
})?;
|
||||
|
||||
let file = if is_partial && existing_size > 0 {
|
||||
OpenOptions::new()
|
||||
.append(true)
|
||||
.open(&part_path)
|
||||
.map_err(|e| fs::FsPathError::open(e, &part_path))?
|
||||
} else {
|
||||
fs::create_file(&part_path)?
|
||||
};
|
||||
|
||||
let start_offset = if is_partial { existing_size } else { 0 };
|
||||
let mut progress = DownloadProgress::new(current_total);
|
||||
progress.downloaded = start_offset;
|
||||
|
||||
let mut writer = ProgressWriter { inner: BufWriter::new(file), progress };
|
||||
let mut reader = response;
|
||||
|
||||
let copy_result = io::copy(&mut reader, &mut writer);
|
||||
let flush_result = writer.inner.flush();
|
||||
println!();
|
||||
|
||||
if let Err(e) = copy_result.and(flush_result) {
|
||||
last_error = Some(e.into());
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
info!(target: "reth::cli",
|
||||
"Download interrupted, retrying in {} seconds...", RETRY_BACKOFF_SECS
|
||||
);
|
||||
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
fs::rename(&part_path, &final_path)?;
|
||||
info!(target: "reth::cli", "Download complete: {}", final_path.display());
|
||||
return Ok((final_path, current_total));
|
||||
}
|
||||
|
||||
Err(last_error
|
||||
.unwrap_or_else(|| eyre::eyre!("Download failed after {} attempts", MAX_DOWNLOAD_RETRIES)))
|
||||
}
|
||||
|
||||
/// Fetches the snapshot from a remote URL with resume support, then extracts it.
|
||||
fn download_and_extract(url: &str, format: CompressionFormat, target_dir: &Path) -> Result<()> {
|
||||
let client = reqwest::blocking::Client::builder().build()?;
|
||||
let response = client.get(url).send()?.error_for_status()?;
|
||||
let (downloaded_path, total_size) = resumable_download(url, target_dir)?;
|
||||
|
||||
let total_size = response.content_length().ok_or_else(|| {
|
||||
eyre::eyre!(
|
||||
"Server did not provide Content-Length header. This is required for snapshot downloads"
|
||||
)
|
||||
})?;
|
||||
info!(target: "reth::cli", "Extracting snapshot...");
|
||||
let file = fs::open(&downloaded_path)?;
|
||||
extract_archive(file, total_size, format, target_dir)?;
|
||||
|
||||
extract_archive(response, total_size, format, target_dir)
|
||||
fs::remove_file(&downloaded_path)?;
|
||||
info!(target: "reth::cli", "Removed downloaded archive");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Downloads and extracts a snapshot, blocking until finished.
|
||||
|
||||
@@ -10,7 +10,8 @@ use reth_node_builder::NodeBuilder;
|
||||
use reth_node_core::{
|
||||
args::{
|
||||
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, EraArgs, MetricArgs,
|
||||
NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
|
||||
NetworkArgs, PayloadBuilderArgs, PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs,
|
||||
TxPoolArgs,
|
||||
},
|
||||
node_config::NodeConfig,
|
||||
version,
|
||||
@@ -102,6 +103,10 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
|
||||
#[command(flatten)]
|
||||
pub pruning: PruningArgs,
|
||||
|
||||
/// All `RocksDB` table routing arguments
|
||||
#[command(flatten)]
|
||||
pub rocksdb: RocksDbArgs,
|
||||
|
||||
/// Engine cli arguments
|
||||
#[command(flatten, next_help_heading = "Engine")]
|
||||
pub engine: EngineArgs,
|
||||
@@ -166,12 +171,16 @@ where
|
||||
db,
|
||||
dev,
|
||||
pruning,
|
||||
rocksdb,
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
ext,
|
||||
} = self;
|
||||
|
||||
// Validate RocksDB arguments
|
||||
rocksdb.validate()?;
|
||||
|
||||
// set up node config
|
||||
let mut node_config = NodeConfig {
|
||||
datadir,
|
||||
@@ -187,6 +196,7 @@ where
|
||||
db,
|
||||
dev,
|
||||
pruning,
|
||||
rocksdb,
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
|
||||
@@ -121,7 +121,16 @@ pub fn install() {
|
||||
unsafe {
|
||||
let alt_stack_size: usize = min_sigstack_size() + 64 * 1024;
|
||||
let mut alt_stack: libc::stack_t = mem::zeroed();
|
||||
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, 1).unwrap()).cast();
|
||||
// Both SysV AMD64 ABI and aarch64 ABI require 16 bytes alignment. We are going to be
|
||||
// generous here and just use a size of a page.
|
||||
let raw_page_sz = libc::sysconf(libc::_SC_PAGESIZE);
|
||||
let page_sz = if raw_page_sz == -1 {
|
||||
// Fallback alignment in case sysconf fails.
|
||||
4096_usize
|
||||
} else {
|
||||
raw_page_sz as usize
|
||||
};
|
||||
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, page_sz).unwrap()).cast();
|
||||
alt_stack.ss_size = alt_stack_size;
|
||||
libc::sigaltstack(&raw const alt_stack, ptr::null_mut());
|
||||
|
||||
|
||||
@@ -137,6 +137,8 @@ pub struct TreeConfig {
|
||||
account_worker_count: usize,
|
||||
/// Whether to enable V2 storage proofs.
|
||||
enable_proof_v2: bool,
|
||||
/// Whether to disable cache metrics recording (can be expensive with large cached state).
|
||||
disable_cache_metrics: bool,
|
||||
}
|
||||
|
||||
impl Default for TreeConfig {
|
||||
@@ -166,6 +168,7 @@ impl Default for TreeConfig {
|
||||
storage_worker_count: default_storage_worker_count(),
|
||||
account_worker_count: default_account_worker_count(),
|
||||
enable_proof_v2: false,
|
||||
disable_cache_metrics: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -198,6 +201,7 @@ impl TreeConfig {
|
||||
storage_worker_count: usize,
|
||||
account_worker_count: usize,
|
||||
enable_proof_v2: bool,
|
||||
disable_cache_metrics: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
persistence_threshold,
|
||||
@@ -224,6 +228,7 @@ impl TreeConfig {
|
||||
storage_worker_count,
|
||||
account_worker_count,
|
||||
enable_proof_v2,
|
||||
disable_cache_metrics,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -516,4 +521,15 @@ impl TreeConfig {
|
||||
self.enable_proof_v2 = enable_proof_v2;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether cache metrics recording is disabled.
|
||||
pub const fn disable_cache_metrics(&self) -> bool {
|
||||
self.disable_cache_metrics
|
||||
}
|
||||
|
||||
/// Setter for whether to disable cache metrics recording.
|
||||
pub const fn without_cache_metrics(mut self, disable_cache_metrics: bool) -> Self {
|
||||
self.disable_cache_metrics = disable_cache_metrics;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
# reth
|
||||
reth-chain-state.workspace = true
|
||||
reth-chain-state = { workspace = true, features = ["rayon"] }
|
||||
reth-chainspec = { workspace = true, optional = true }
|
||||
reth-consensus.workspace = true
|
||||
reth-db.workspace = true
|
||||
|
||||
@@ -606,12 +606,21 @@ pub(crate) struct SavedCache {
|
||||
/// A guard to track in-flight usage of this cache.
|
||||
/// The cache is considered available if the strong count is 1.
|
||||
usage_guard: Arc<()>,
|
||||
|
||||
/// Whether to skip cache metrics recording (can be expensive with large cached state).
|
||||
disable_cache_metrics: bool,
|
||||
}
|
||||
|
||||
impl SavedCache {
|
||||
/// Creates a new instance with the internals
|
||||
pub(super) fn new(hash: B256, caches: ExecutionCache, metrics: CachedStateMetrics) -> Self {
|
||||
Self { hash, caches, metrics, usage_guard: Arc::new(()) }
|
||||
Self { hash, caches, metrics, usage_guard: Arc::new(()), disable_cache_metrics: false }
|
||||
}
|
||||
|
||||
/// Sets whether to disable cache metrics recording.
|
||||
pub(super) const fn with_disable_cache_metrics(mut self, disable: bool) -> Self {
|
||||
self.disable_cache_metrics = disable;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns the hash for this cache
|
||||
@@ -619,9 +628,9 @@ impl SavedCache {
|
||||
self.hash
|
||||
}
|
||||
|
||||
/// Splits the cache into its caches and metrics, consuming it.
|
||||
pub(crate) fn split(self) -> (ExecutionCache, CachedStateMetrics) {
|
||||
(self.caches, self.metrics)
|
||||
/// Splits the cache into its caches, metrics, and `disable_cache_metrics` flag, consuming it.
|
||||
pub(crate) fn split(self) -> (ExecutionCache, CachedStateMetrics, bool) {
|
||||
(self.caches, self.metrics, self.disable_cache_metrics)
|
||||
}
|
||||
|
||||
/// Returns true if the cache is available for use (no other tasks are currently using it).
|
||||
@@ -645,7 +654,13 @@ impl SavedCache {
|
||||
}
|
||||
|
||||
/// Updates the metrics for the [`ExecutionCache`].
|
||||
///
|
||||
/// Note: This can be expensive with large cached state as it iterates over
|
||||
/// all storage entries. Use `with_disable_cache_metrics(true)` to skip.
|
||||
pub(crate) fn update_metrics(&self) {
|
||||
if self.disable_cache_metrics {
|
||||
return;
|
||||
}
|
||||
self.metrics.storage_cache_size.set(self.caches.total_storage_slots() as f64);
|
||||
self.metrics.account_cache_size.set(self.caches.account_cache.entry_count() as f64);
|
||||
self.metrics.code_cache_size.set(self.caches.code_cache.entry_count() as f64);
|
||||
|
||||
@@ -1,25 +1,15 @@
|
||||
use crate::tree::{error::InsertBlockFatalError, MeteredStateHook, TreeOutcome};
|
||||
use alloy_consensus::transaction::TxHashRef;
|
||||
use alloy_evm::{
|
||||
block::{BlockExecutor, ExecutableTx},
|
||||
Evm,
|
||||
};
|
||||
use crate::tree::{error::InsertBlockFatalError, TreeOutcome};
|
||||
use alloy_rpc_types_engine::{PayloadStatus, PayloadStatusEnum};
|
||||
use core::borrow::BorrowMut;
|
||||
use reth_engine_primitives::{ForkchoiceStatus, OnForkChoiceUpdated};
|
||||
use reth_errors::{BlockExecutionError, ProviderError};
|
||||
use reth_evm::{metrics::ExecutorMetrics, OnStateHook};
|
||||
use reth_errors::ProviderError;
|
||||
use reth_evm::metrics::ExecutorMetrics;
|
||||
use reth_execution_types::BlockExecutionOutput;
|
||||
use reth_metrics::{
|
||||
metrics::{Counter, Gauge, Histogram},
|
||||
Metrics,
|
||||
};
|
||||
use reth_primitives_traits::SignedTransaction;
|
||||
use reth_trie::updates::TrieUpdates;
|
||||
use revm::database::{states::bundle_state::BundleRetention, State};
|
||||
use revm_primitives::Address;
|
||||
use std::time::Instant;
|
||||
use tracing::{debug_span, trace};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Metrics for the `EngineApi`.
|
||||
#[derive(Debug, Default)]
|
||||
@@ -35,114 +25,24 @@ pub(crate) struct EngineApiMetrics {
|
||||
}
|
||||
|
||||
impl EngineApiMetrics {
|
||||
/// Helper function for metered execution
|
||||
fn metered<F, R>(&self, f: F) -> R
|
||||
where
|
||||
F: FnOnce() -> (u64, R),
|
||||
{
|
||||
// Execute the block and record the elapsed time.
|
||||
let execute_start = Instant::now();
|
||||
let (gas_used, output) = f();
|
||||
let execution_duration = execute_start.elapsed().as_secs_f64();
|
||||
|
||||
// Update gas metrics.
|
||||
self.executor.gas_processed_total.increment(gas_used);
|
||||
self.executor.gas_per_second.set(gas_used as f64 / execution_duration);
|
||||
self.executor.gas_used_histogram.record(gas_used as f64);
|
||||
self.executor.execution_histogram.record(execution_duration);
|
||||
self.executor.execution_duration.set(execution_duration);
|
||||
|
||||
output
|
||||
}
|
||||
|
||||
/// Execute the given block using the provided [`BlockExecutor`] and update metrics for the
|
||||
/// execution.
|
||||
/// Records metrics for block execution.
|
||||
///
|
||||
/// This method updates metrics for execution time, gas usage, and the number
|
||||
/// of accounts, storage slots and bytecodes loaded and updated.
|
||||
///
|
||||
/// The optional `on_receipt` callback is invoked after each transaction with the receipt
|
||||
/// index and a reference to all receipts collected so far. This allows callers to stream
|
||||
/// receipts to a background task for incremental receipt root computation.
|
||||
pub(crate) fn execute_metered<E, DB, F>(
|
||||
/// of accounts, storage slots and bytecodes updated.
|
||||
pub(crate) fn record_block_execution<R>(
|
||||
&self,
|
||||
executor: E,
|
||||
mut transactions: impl Iterator<Item = Result<impl ExecutableTx<E>, BlockExecutionError>>,
|
||||
transaction_count: usize,
|
||||
state_hook: Box<dyn OnStateHook>,
|
||||
mut on_receipt: F,
|
||||
) -> Result<(BlockExecutionOutput<E::Receipt>, Vec<Address>), BlockExecutionError>
|
||||
where
|
||||
DB: alloy_evm::Database,
|
||||
E: BlockExecutor<Evm: Evm<DB: BorrowMut<State<DB>>>, Transaction: SignedTransaction>,
|
||||
F: FnMut(&[E::Receipt]),
|
||||
{
|
||||
// clone here is cheap, all the metrics are Option<Arc<_>>. additionally
|
||||
// they are globally registered so that the data recorded in the hook will
|
||||
// be accessible.
|
||||
let wrapper = MeteredStateHook { metrics: self.executor.clone(), inner_hook: state_hook };
|
||||
output: &BlockExecutionOutput<R>,
|
||||
execution_duration: Duration,
|
||||
) {
|
||||
let execution_secs = execution_duration.as_secs_f64();
|
||||
let gas_used = output.result.gas_used;
|
||||
|
||||
let mut senders = Vec::with_capacity(transaction_count);
|
||||
let mut executor = executor.with_state_hook(Some(Box::new(wrapper)));
|
||||
|
||||
let f = || {
|
||||
let start = Instant::now();
|
||||
debug_span!(target: "engine::tree", "pre execution")
|
||||
.entered()
|
||||
.in_scope(|| executor.apply_pre_execution_changes())?;
|
||||
self.executor.pre_execution_histogram.record(start.elapsed());
|
||||
|
||||
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
|
||||
loop {
|
||||
let start = Instant::now();
|
||||
let Some(tx) = transactions.next() else { break };
|
||||
self.executor.transaction_wait_histogram.record(start.elapsed());
|
||||
|
||||
let tx = tx?;
|
||||
senders.push(*tx.signer());
|
||||
|
||||
let span = debug_span!(
|
||||
target: "engine::tree",
|
||||
"execute tx",
|
||||
tx_hash = ?tx.tx().tx_hash(),
|
||||
gas_used = tracing::field::Empty,
|
||||
);
|
||||
let enter = span.entered();
|
||||
trace!(target: "engine::tree", "Executing transaction");
|
||||
let start = Instant::now();
|
||||
let gas_used = executor.execute_transaction(tx)?;
|
||||
self.executor.transaction_execution_histogram.record(start.elapsed());
|
||||
|
||||
// Invoke callback with the latest receipt
|
||||
on_receipt(executor.receipts());
|
||||
|
||||
// record the tx gas used
|
||||
enter.record("gas_used", gas_used);
|
||||
}
|
||||
drop(exec_span);
|
||||
|
||||
let start = Instant::now();
|
||||
let result = debug_span!(target: "engine::tree", "finish")
|
||||
.entered()
|
||||
.in_scope(|| executor.finish())
|
||||
.map(|(evm, result)| (evm.into_db(), result));
|
||||
self.executor.post_execution_histogram.record(start.elapsed());
|
||||
|
||||
result
|
||||
};
|
||||
|
||||
// Use metered to execute and track timing/gas metrics
|
||||
let (mut db, result) = self.metered(|| {
|
||||
let res = f();
|
||||
let gas_used = res.as_ref().map(|r| r.1.gas_used).unwrap_or(0);
|
||||
(gas_used, res)
|
||||
})?;
|
||||
|
||||
// merge transitions into bundle state
|
||||
debug_span!(target: "engine::tree", "merge transitions")
|
||||
.entered()
|
||||
.in_scope(|| db.borrow_mut().merge_transitions(BundleRetention::Reverts));
|
||||
let output = BlockExecutionOutput { result, state: db.borrow_mut().take_bundle() };
|
||||
// Update gas metrics
|
||||
self.executor.gas_processed_total.increment(gas_used);
|
||||
self.executor.gas_per_second.set(gas_used as f64 / execution_secs);
|
||||
self.executor.gas_used_histogram.record(gas_used as f64);
|
||||
self.executor.execution_histogram.record(execution_secs);
|
||||
self.executor.execution_duration.set(execution_secs);
|
||||
|
||||
// Update the metrics for the number of accounts, storage slots and bytecodes updated
|
||||
let accounts = output.state.state.len();
|
||||
@@ -153,8 +53,31 @@ impl EngineApiMetrics {
|
||||
self.executor.accounts_updated_histogram.record(accounts as f64);
|
||||
self.executor.storage_slots_updated_histogram.record(storage_slots as f64);
|
||||
self.executor.bytecodes_updated_histogram.record(bytecodes as f64);
|
||||
}
|
||||
|
||||
Ok((output, senders))
|
||||
/// Returns a reference to the executor metrics for use in state hooks.
|
||||
pub(crate) const fn executor_metrics(&self) -> &ExecutorMetrics {
|
||||
&self.executor
|
||||
}
|
||||
|
||||
/// Records the duration of block pre-execution changes (e.g., beacon root update).
|
||||
pub(crate) fn record_pre_execution(&self, elapsed: Duration) {
|
||||
self.executor.pre_execution_histogram.record(elapsed);
|
||||
}
|
||||
|
||||
/// Records the duration of block post-execution changes (e.g., finalization).
|
||||
pub(crate) fn record_post_execution(&self, elapsed: Duration) {
|
||||
self.executor.post_execution_histogram.record(elapsed);
|
||||
}
|
||||
|
||||
/// Records the time spent waiting for the next transaction from the iterator.
|
||||
pub(crate) fn record_transaction_wait(&self, elapsed: Duration) {
|
||||
self.executor.transaction_wait_histogram.record(elapsed);
|
||||
}
|
||||
|
||||
/// Records the duration of a single transaction execution.
|
||||
pub(crate) fn record_transaction_execution(&self, elapsed: Duration) {
|
||||
self.executor.transaction_execution_histogram.record(elapsed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -210,6 +133,12 @@ pub(crate) struct EngineMetrics {
|
||||
#[derive(Metrics)]
|
||||
#[metrics(scope = "consensus.engine.beacon")]
|
||||
pub(crate) struct ForkchoiceUpdatedMetrics {
|
||||
/// Finish time of the latest forkchoice updated call.
|
||||
#[metric(skip)]
|
||||
pub(crate) latest_finish_at: Option<Instant>,
|
||||
/// Start time of the latest forkchoice updated call.
|
||||
#[metric(skip)]
|
||||
pub(crate) latest_start_at: Option<Instant>,
|
||||
/// The total count of forkchoice updated messages received.
|
||||
pub(crate) forkchoice_updated_messages: Counter,
|
||||
/// The total count of forkchoice updated messages with payload received.
|
||||
@@ -232,18 +161,35 @@ pub(crate) struct ForkchoiceUpdatedMetrics {
|
||||
pub(crate) forkchoice_updated_last: Gauge,
|
||||
/// Time diff between new payload call response and the next forkchoice updated call request.
|
||||
pub(crate) new_payload_forkchoice_updated_time_diff: Histogram,
|
||||
/// Time from previous forkchoice updated finish to current forkchoice updated start (idle
|
||||
/// time).
|
||||
pub(crate) time_between_forkchoice_updated: Histogram,
|
||||
/// Time from previous forkchoice updated start to current forkchoice updated start (total
|
||||
/// interval).
|
||||
pub(crate) forkchoice_updated_interval: Histogram,
|
||||
}
|
||||
|
||||
impl ForkchoiceUpdatedMetrics {
|
||||
/// Increment the forkchoiceUpdated counter based on the given result
|
||||
pub(crate) fn update_response_metrics(
|
||||
&self,
|
||||
&mut self,
|
||||
start: Instant,
|
||||
latest_new_payload_at: &mut Option<Instant>,
|
||||
has_attrs: bool,
|
||||
result: &Result<TreeOutcome<OnForkChoiceUpdated>, ProviderError>,
|
||||
) {
|
||||
let elapsed = start.elapsed();
|
||||
let finish = Instant::now();
|
||||
let elapsed = finish - start;
|
||||
|
||||
if let Some(prev_finish) = self.latest_finish_at {
|
||||
self.time_between_forkchoice_updated.record(start - prev_finish);
|
||||
}
|
||||
if let Some(prev_start) = self.latest_start_at {
|
||||
self.forkchoice_updated_interval.record(start - prev_start);
|
||||
}
|
||||
self.latest_finish_at = Some(finish);
|
||||
self.latest_start_at = Some(start);
|
||||
|
||||
match result {
|
||||
Ok(outcome) => match outcome.outcome.forkchoice_status() {
|
||||
ForkchoiceStatus::Valid => self.forkchoice_updated_valid.increment(1),
|
||||
@@ -410,138 +356,10 @@ pub(crate) struct BlockBufferMetrics {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_eips::eip7685::Requests;
|
||||
use alloy_evm::block::StateChangeSource;
|
||||
use alloy_primitives::{B256, U256};
|
||||
use metrics_util::debugging::{DebuggingRecorder, Snapshotter};
|
||||
use reth_ethereum_primitives::{Receipt, TransactionSigned};
|
||||
use reth_evm_ethereum::EthEvm;
|
||||
use reth_ethereum_primitives::Receipt;
|
||||
use reth_execution_types::BlockExecutionResult;
|
||||
use reth_primitives_traits::RecoveredBlock;
|
||||
use revm::{
|
||||
context::result::{ExecutionResult, Output, ResultAndState, SuccessReason},
|
||||
database::State,
|
||||
database_interface::EmptyDB,
|
||||
inspector::NoOpInspector,
|
||||
state::{Account, AccountInfo, AccountStatus, EvmState, EvmStorage, EvmStorageSlot},
|
||||
Context, MainBuilder, MainContext,
|
||||
};
|
||||
use revm_primitives::Bytes;
|
||||
use std::sync::mpsc;
|
||||
|
||||
/// A simple mock executor for testing that doesn't require complex EVM setup
|
||||
struct MockExecutor {
|
||||
state: EvmState,
|
||||
receipts: Vec<Receipt>,
|
||||
hook: Option<Box<dyn OnStateHook>>,
|
||||
}
|
||||
|
||||
impl MockExecutor {
|
||||
fn new(state: EvmState) -> Self {
|
||||
Self { state, receipts: vec![], hook: None }
|
||||
}
|
||||
}
|
||||
|
||||
// Mock Evm type for testing
|
||||
type MockEvm = EthEvm<State<EmptyDB>, NoOpInspector>;
|
||||
|
||||
impl BlockExecutor for MockExecutor {
|
||||
type Transaction = TransactionSigned;
|
||||
type Receipt = Receipt;
|
||||
type Evm = MockEvm;
|
||||
|
||||
fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn execute_transaction_without_commit(
|
||||
&mut self,
|
||||
_tx: impl ExecutableTx<Self>,
|
||||
) -> Result<ResultAndState<<Self::Evm as Evm>::HaltReason>, BlockExecutionError> {
|
||||
// Call hook with our mock state for each transaction
|
||||
if let Some(hook) = self.hook.as_mut() {
|
||||
hook.on_state(StateChangeSource::Transaction(0), &self.state);
|
||||
}
|
||||
|
||||
Ok(ResultAndState::new(
|
||||
ExecutionResult::Success {
|
||||
reason: SuccessReason::Return,
|
||||
gas_used: 1000, // Mock gas used
|
||||
gas_refunded: 0,
|
||||
logs: vec![],
|
||||
output: Output::Call(Bytes::from(vec![])),
|
||||
},
|
||||
Default::default(),
|
||||
))
|
||||
}
|
||||
|
||||
fn commit_transaction(
|
||||
&mut self,
|
||||
_output: ResultAndState<<Self::Evm as Evm>::HaltReason>,
|
||||
_tx: impl ExecutableTx<Self>,
|
||||
) -> Result<u64, BlockExecutionError> {
|
||||
Ok(1000)
|
||||
}
|
||||
|
||||
fn finish(
|
||||
self,
|
||||
) -> Result<(Self::Evm, BlockExecutionResult<Self::Receipt>), BlockExecutionError> {
|
||||
let Self { hook, state, .. } = self;
|
||||
|
||||
// Call hook with our mock state
|
||||
if let Some(mut hook) = hook {
|
||||
hook.on_state(StateChangeSource::Transaction(0), &state);
|
||||
}
|
||||
|
||||
// Create a mock EVM
|
||||
let db = State::builder()
|
||||
.with_database(EmptyDB::default())
|
||||
.with_bundle_update()
|
||||
.without_state_clear()
|
||||
.build();
|
||||
let evm = EthEvm::new(
|
||||
Context::mainnet().with_db(db).build_mainnet_with_inspector(NoOpInspector {}),
|
||||
false,
|
||||
);
|
||||
|
||||
// Return successful result like the original tests
|
||||
Ok((
|
||||
evm,
|
||||
BlockExecutionResult {
|
||||
receipts: vec![],
|
||||
requests: Requests::default(),
|
||||
gas_used: 1000,
|
||||
blob_gas_used: 0,
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
fn set_state_hook(&mut self, hook: Option<Box<dyn OnStateHook>>) {
|
||||
self.hook = hook;
|
||||
}
|
||||
|
||||
fn evm_mut(&mut self) -> &mut Self::Evm {
|
||||
panic!("Mock executor evm_mut() not implemented")
|
||||
}
|
||||
|
||||
fn evm(&self) -> &Self::Evm {
|
||||
panic!("Mock executor evm() not implemented")
|
||||
}
|
||||
|
||||
fn receipts(&self) -> &[Self::Receipt] {
|
||||
&self.receipts
|
||||
}
|
||||
}
|
||||
|
||||
struct ChannelStateHook {
|
||||
output: i32,
|
||||
sender: mpsc::Sender<i32>,
|
||||
}
|
||||
|
||||
impl OnStateHook for ChannelStateHook {
|
||||
fn on_state(&mut self, _source: StateChangeSource, _state: &EvmState) {
|
||||
let _ = self.sender.send(self.output);
|
||||
}
|
||||
}
|
||||
use reth_revm::db::BundleState;
|
||||
|
||||
fn setup_test_recorder() -> Snapshotter {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
@@ -551,38 +369,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_executor_metrics_hook_called() {
|
||||
let metrics = EngineApiMetrics::default();
|
||||
let input = RecoveredBlock::<reth_ethereum_primitives::Block>::default();
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let expected_output = 42;
|
||||
let state_hook = Box::new(ChannelStateHook { sender: tx, output: expected_output });
|
||||
|
||||
let state = EvmState::default();
|
||||
let executor = MockExecutor::new(state);
|
||||
|
||||
// This will fail to create the EVM but should still call the hook
|
||||
let _result = metrics.execute_metered::<_, EmptyDB, _>(
|
||||
executor,
|
||||
input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>),
|
||||
input.transaction_count(),
|
||||
state_hook,
|
||||
|_| {},
|
||||
);
|
||||
|
||||
// Check if hook was called (it might not be if finish() fails early)
|
||||
match rx.try_recv() {
|
||||
Ok(actual_output) => assert_eq!(actual_output, expected_output),
|
||||
Err(_) => {
|
||||
// Hook wasn't called, which is expected if the mock fails early
|
||||
// The test still validates that the code compiles and runs
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_executor_metrics_hook_metrics_recorded() {
|
||||
fn test_record_block_execution_metrics() {
|
||||
let snapshotter = setup_test_recorder();
|
||||
let metrics = EngineApiMetrics::default();
|
||||
|
||||
@@ -591,45 +378,17 @@ mod tests {
|
||||
metrics.executor.gas_per_second.set(0.0);
|
||||
metrics.executor.gas_used_histogram.record(0.0);
|
||||
|
||||
let input = RecoveredBlock::<reth_ethereum_primitives::Block>::default();
|
||||
|
||||
let (tx, _rx) = mpsc::channel();
|
||||
let state_hook = Box::new(ChannelStateHook { sender: tx, output: 42 });
|
||||
|
||||
// Create a state with some data
|
||||
let state = {
|
||||
let mut state = EvmState::default();
|
||||
let storage =
|
||||
EvmStorage::from_iter([(U256::from(1), EvmStorageSlot::new(U256::from(2), 0))]);
|
||||
state.insert(
|
||||
Default::default(),
|
||||
Account {
|
||||
info: AccountInfo {
|
||||
balance: U256::from(100),
|
||||
nonce: 10,
|
||||
code_hash: B256::random(),
|
||||
code: Default::default(),
|
||||
account_id: None,
|
||||
},
|
||||
original_info: Box::new(AccountInfo::default()),
|
||||
storage,
|
||||
status: AccountStatus::default(),
|
||||
transaction_id: 0,
|
||||
},
|
||||
);
|
||||
state
|
||||
let output = BlockExecutionOutput::<Receipt> {
|
||||
state: BundleState::default(),
|
||||
result: BlockExecutionResult {
|
||||
receipts: vec![],
|
||||
requests: Requests::default(),
|
||||
gas_used: 21000,
|
||||
blob_gas_used: 0,
|
||||
},
|
||||
};
|
||||
|
||||
let executor = MockExecutor::new(state);
|
||||
|
||||
// Execute (will fail but should still update some metrics)
|
||||
let _result = metrics.execute_metered::<_, EmptyDB, _>(
|
||||
executor,
|
||||
input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>),
|
||||
input.transaction_count(),
|
||||
state_hook,
|
||||
|_| {},
|
||||
);
|
||||
metrics.record_block_execution(&output, Duration::from_millis(100));
|
||||
|
||||
let snapshot = snapshotter.snapshot().into_vec();
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ use alloy_evm::{block::StateChangeSource, ToTxEnv};
|
||||
use alloy_primitives::B256;
|
||||
use crossbeam_channel::Sender as CrossbeamSender;
|
||||
use executor::WorkloadExecutor;
|
||||
use metrics::Counter;
|
||||
use multiproof::{SparseTrieUpdate, *};
|
||||
use parking_lot::RwLock;
|
||||
use prewarm::PrewarmMetrics;
|
||||
@@ -28,6 +29,7 @@ use reth_evm::{
|
||||
ConfigureEvm, EvmEnvFor, ExecutableTxIterator, ExecutableTxTuple, OnStateHook, SpecFor,
|
||||
TxEnvFor,
|
||||
};
|
||||
use reth_metrics::Metrics;
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_provider::{
|
||||
BlockExecutionOutput, BlockReader, DatabaseProviderROFactory, StateProvider,
|
||||
@@ -139,6 +141,8 @@ where
|
||||
disable_parallel_sparse_trie: bool,
|
||||
/// Maximum concurrency for prewarm task.
|
||||
prewarm_max_concurrency: usize,
|
||||
/// Whether to disable cache metrics recording.
|
||||
disable_cache_metrics: bool,
|
||||
}
|
||||
|
||||
impl<N, Evm> PayloadProcessor<Evm>
|
||||
@@ -171,6 +175,7 @@ where
|
||||
sparse_state_trie: Arc::default(),
|
||||
disable_parallel_sparse_trie: config.disable_parallel_sparse_trie(),
|
||||
prewarm_max_concurrency: config.prewarm_max_concurrency(),
|
||||
disable_cache_metrics: config.disable_cache_metrics(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -300,7 +305,7 @@ where
|
||||
// Build a state provider for the multiproof task
|
||||
let provider = provider_builder.build().expect("failed to build provider");
|
||||
let provider = if let Some(saved_cache) = saved_cache {
|
||||
let (cache, metrics) = saved_cache.split();
|
||||
let (cache, metrics, _) = saved_cache.split();
|
||||
Box::new(CachedStateProvider::new(provider, cache, metrics))
|
||||
as Box<dyn StateProvider>
|
||||
} else {
|
||||
@@ -477,6 +482,7 @@ where
|
||||
debug!("creating new execution cache on cache miss");
|
||||
let cache = ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size);
|
||||
SavedCache::new(parent_hash, cache, CachedStateMetrics::zeroed())
|
||||
.with_disable_cache_metrics(self.disable_cache_metrics)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -558,6 +564,7 @@ where
|
||||
block_with_parent: BlockWithParent,
|
||||
bundle_state: &BundleState,
|
||||
) {
|
||||
let disable_cache_metrics = self.disable_cache_metrics;
|
||||
self.execution_cache.update_with_guard(|cached| {
|
||||
if cached.as_ref().is_some_and(|c| c.executed_block_hash() != block_with_parent.parent) {
|
||||
debug!(
|
||||
@@ -571,7 +578,8 @@ where
|
||||
// Take existing cache (if any) or create fresh caches
|
||||
let (caches, cache_metrics) = match cached.take() {
|
||||
Some(existing) => {
|
||||
existing.split()
|
||||
let (c, m, _) = existing.split();
|
||||
(c, m)
|
||||
}
|
||||
None => (
|
||||
ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size),
|
||||
@@ -580,7 +588,8 @@ where
|
||||
};
|
||||
|
||||
// Insert the block's bundle state into cache
|
||||
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics);
|
||||
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics)
|
||||
.with_disable_cache_metrics(disable_cache_metrics);
|
||||
if new_cache.cache().insert_state(bundle_state).is_err() {
|
||||
*cached = None;
|
||||
debug!(target: "engine::caching", "cleared execution cache on update error");
|
||||
@@ -770,6 +779,8 @@ impl<R> Drop for CacheTaskHandle<R> {
|
||||
struct ExecutionCache {
|
||||
/// Guarded cloneable cache identified by a block hash.
|
||||
inner: Arc<RwLock<Option<SavedCache>>>,
|
||||
/// Metrics for cache operations.
|
||||
metrics: ExecutionCacheMetrics,
|
||||
}
|
||||
|
||||
impl ExecutionCache {
|
||||
@@ -811,6 +822,10 @@ impl ExecutionCache {
|
||||
if hash_matches && available {
|
||||
return Some(c.clone());
|
||||
}
|
||||
|
||||
if hash_matches && !available {
|
||||
self.metrics.execution_cache_in_use.increment(1);
|
||||
}
|
||||
} else {
|
||||
debug!(target: "engine::caching", %parent_hash, "No cache found");
|
||||
}
|
||||
@@ -846,6 +861,15 @@ impl ExecutionCache {
|
||||
}
|
||||
}
|
||||
|
||||
/// Metrics for execution cache operations.
|
||||
#[derive(Metrics, Clone)]
|
||||
#[metrics(scope = "consensus.engine.beacon")]
|
||||
pub(crate) struct ExecutionCacheMetrics {
|
||||
/// Counter for when the execution cache was unavailable because other threads
|
||||
/// (e.g., prewarming) are still using it.
|
||||
pub(crate) execution_cache_in_use: Counter,
|
||||
}
|
||||
|
||||
/// EVM context required to execute a block.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ExecutionEnv<Evm: ConfigureEvm> {
|
||||
|
||||
@@ -278,8 +278,9 @@ where
|
||||
execution_cache.update_with_guard(|cached| {
|
||||
// consumes the `SavedCache` held by the prewarming task, which releases its usage
|
||||
// guard
|
||||
let (caches, cache_metrics) = saved_cache.split();
|
||||
let new_cache = SavedCache::new(hash, caches, cache_metrics);
|
||||
let (caches, cache_metrics, disable_cache_metrics) = saved_cache.split();
|
||||
let new_cache = SavedCache::new(hash, caches, cache_metrics)
|
||||
.with_disable_cache_metrics(disable_cache_metrics);
|
||||
|
||||
// Insert state into cache while holding the lock
|
||||
// Access the BundleState through the shared ExecutionOutcome
|
||||
@@ -296,6 +297,11 @@ where
|
||||
// Replace the shared cache with the new one; the previous cache (if any) is
|
||||
// dropped.
|
||||
*cached = Some(new_cache);
|
||||
} else {
|
||||
// Block was invalid; caches were already mutated by insert_state above,
|
||||
// so we must clear to prevent using polluted state
|
||||
*cached = None;
|
||||
debug!(target: "engine::caching", "cleared execution cache on invalid block");
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -64,14 +64,11 @@ impl<R: Receipt> ReceiptRootTaskHandle<R> {
|
||||
///
|
||||
/// * `receipts_len` - The total number of receipts expected. This is needed to correctly order
|
||||
/// the trie keys according to RLP encoding rules.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if the number of receipts received doesn't match `receipts_len`.
|
||||
pub fn run(self, receipts_len: usize) {
|
||||
let mut builder = OrderedTrieRootEncodedBuilder::new(receipts_len);
|
||||
let mut aggregated_bloom = Bloom::ZERO;
|
||||
let mut encode_buf = Vec::new();
|
||||
let mut received_count = 0usize;
|
||||
|
||||
for indexed_receipt in self.receipt_rx {
|
||||
let receipt_with_bloom = indexed_receipt.receipt.with_bloom_ref();
|
||||
@@ -81,9 +78,21 @@ impl<R: Receipt> ReceiptRootTaskHandle<R> {
|
||||
|
||||
aggregated_bloom |= *receipt_with_bloom.bloom_ref();
|
||||
builder.push_unchecked(indexed_receipt.index, &encode_buf);
|
||||
received_count += 1;
|
||||
}
|
||||
|
||||
let root = builder.finalize().expect("receipt root builder incomplete");
|
||||
let Ok(root) = builder.finalize() else {
|
||||
// Finalize fails if we didn't receive exactly `receipts_len` receipts. This can
|
||||
// happen if execution was aborted early (e.g., invalid transaction encountered).
|
||||
// We return without sending a result, allowing the caller to handle the abort.
|
||||
tracing::error!(
|
||||
target: "engine::tree::payload_processor",
|
||||
expected = receipts_len,
|
||||
received = received_count,
|
||||
"Receipt root task received incomplete receipts, execution likely aborted"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let _ = self.result_tx.send((root, aggregated_bloom));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,10 +7,10 @@ use crate::tree::{
|
||||
payload_processor::{executor::WorkloadExecutor, PayloadProcessor},
|
||||
precompile_cache::{CachedPrecompile, CachedPrecompileMetrics, PrecompileCacheMap},
|
||||
sparse_trie::StateRootComputeOutcome,
|
||||
EngineApiMetrics, EngineApiTreeState, ExecutionEnv, PayloadHandle, StateProviderBuilder,
|
||||
StateProviderDatabase, TreeConfig,
|
||||
EngineApiMetrics, EngineApiTreeState, ExecutionEnv, MeteredStateHook, PayloadHandle,
|
||||
StateProviderBuilder, StateProviderDatabase, TreeConfig,
|
||||
};
|
||||
use alloy_consensus::transaction::Either;
|
||||
use alloy_consensus::transaction::{Either, TxHashRef};
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::{eip1898::BlockWithParent, NumHash};
|
||||
use alloy_evm::Evm;
|
||||
@@ -41,7 +41,7 @@ use reth_provider::{
|
||||
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
|
||||
StateProviderFactory, StateReader,
|
||||
};
|
||||
use reth_revm::db::State;
|
||||
use reth_revm::db::{states::bundle_state::BundleRetention, State};
|
||||
use reth_trie::{updates::TrieUpdates, HashedPostState, StateRoot};
|
||||
use reth_trie_db::ChangesetCache;
|
||||
use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError};
|
||||
@@ -479,11 +479,15 @@ where
|
||||
let block = self.convert_to_block(input)?.with_senders(senders);
|
||||
|
||||
// Wait for the receipt root computation to complete.
|
||||
let receipt_root_bloom = Some(
|
||||
receipt_root_rx
|
||||
.blocking_recv()
|
||||
.expect("receipt root task dropped sender without result"),
|
||||
);
|
||||
let receipt_root_bloom = receipt_root_rx
|
||||
.blocking_recv()
|
||||
.inspect_err(|_| {
|
||||
tracing::error!(
|
||||
target: "engine::tree::payload_validator",
|
||||
"Receipt root task dropped sender without result, receipt root calculation likely aborted"
|
||||
);
|
||||
})
|
||||
.ok();
|
||||
|
||||
let hashed_state = ensure_ok_post_block!(
|
||||
self.validate_post_execution(
|
||||
@@ -638,7 +642,13 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Executes a block with the given state provider
|
||||
/// Executes a block with the given state provider.
|
||||
///
|
||||
/// This method orchestrates block execution:
|
||||
/// 1. Sets up the EVM with state database and precompile caching
|
||||
/// 2. Spawns a background task for incremental receipt root computation
|
||||
/// 3. Executes transactions with metrics collection via state hooks
|
||||
/// 4. Merges state transitions and records execution metrics
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)]
|
||||
#[expect(clippy::type_complexity)]
|
||||
fn execute_block<S, Err, T>(
|
||||
@@ -701,31 +711,117 @@ where
|
||||
let task_handle = ReceiptRootTaskHandle::new(receipt_rx, result_tx);
|
||||
self.payload_processor.executor().spawn_blocking(move || task_handle.run(receipts_len));
|
||||
|
||||
// Wrap the state hook with metrics collection
|
||||
let inner_hook = Box::new(handle.state_hook());
|
||||
let state_hook =
|
||||
MeteredStateHook { metrics: self.metrics.executor_metrics().clone(), inner_hook };
|
||||
|
||||
let transaction_count = input.transaction_count();
|
||||
let executor = executor.with_state_hook(Some(Box::new(state_hook)));
|
||||
|
||||
let execution_start = Instant::now();
|
||||
let state_hook = Box::new(handle.state_hook());
|
||||
let (output, senders) = self.metrics.execute_metered(
|
||||
|
||||
// Execute all transactions and finalize
|
||||
let (executor, senders) = self.execute_transactions(
|
||||
executor,
|
||||
handle.iter_transactions().map(|res| res.map_err(BlockExecutionError::other)),
|
||||
input.transaction_count(),
|
||||
state_hook,
|
||||
|receipts| {
|
||||
// Send the latest receipt to the background task for incremental root computation.
|
||||
// The receipt is cloned here; encoding happens in the background thread.
|
||||
if let Some(receipt) = receipts.last() {
|
||||
// Infer tx_index from the number of receipts collected so far
|
||||
let tx_index = receipts.len() - 1;
|
||||
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
|
||||
}
|
||||
},
|
||||
transaction_count,
|
||||
handle.iter_transactions(),
|
||||
&receipt_tx,
|
||||
)?;
|
||||
drop(receipt_tx);
|
||||
|
||||
let execution_finish = Instant::now();
|
||||
let execution_time = execution_finish.duration_since(execution_start);
|
||||
debug!(target: "engine::tree::payload_validator", elapsed = ?execution_time, "Executed block");
|
||||
// Finish execution and get the result
|
||||
let post_exec_start = Instant::now();
|
||||
let (_evm, result) = debug_span!(target: "engine::tree", "finish")
|
||||
.in_scope(|| executor.finish())
|
||||
.map(|(evm, result)| (evm.into_db(), result))?;
|
||||
self.metrics.record_post_execution(post_exec_start.elapsed());
|
||||
|
||||
// Merge transitions into bundle state
|
||||
debug_span!(target: "engine::tree", "merge transitions")
|
||||
.in_scope(|| db.merge_transitions(BundleRetention::Reverts));
|
||||
|
||||
let output = BlockExecutionOutput { result, state: db.take_bundle() };
|
||||
|
||||
let execution_duration = execution_start.elapsed();
|
||||
self.metrics.record_block_execution(&output, execution_duration);
|
||||
|
||||
debug!(target: "engine::tree::payload_validator", elapsed = ?execution_duration, "Executed block");
|
||||
Ok((output, senders, result_rx))
|
||||
}
|
||||
|
||||
/// Executes transactions and collects senders, streaming receipts to a background task.
|
||||
///
|
||||
/// This method handles:
|
||||
/// - Applying pre-execution changes (e.g., beacon root updates)
|
||||
/// - Executing each transaction with timing metrics
|
||||
/// - Streaming receipts to the receipt root computation task
|
||||
/// - Collecting transaction senders for later use
|
||||
///
|
||||
/// Returns the executor (for finalization) and the collected senders.
|
||||
fn execute_transactions<E, Tx, InnerTx, Err>(
|
||||
&self,
|
||||
mut executor: E,
|
||||
transaction_count: usize,
|
||||
transactions: impl Iterator<Item = Result<Tx, Err>>,
|
||||
receipt_tx: &crossbeam_channel::Sender<IndexedReceipt<N::Receipt>>,
|
||||
) -> Result<(E, Vec<Address>), BlockExecutionError>
|
||||
where
|
||||
E: BlockExecutor<Receipt = N::Receipt>,
|
||||
Tx: alloy_evm::block::ExecutableTx<E> + alloy_evm::RecoveredTx<InnerTx>,
|
||||
InnerTx: TxHashRef,
|
||||
Err: core::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
let mut senders = Vec::with_capacity(transaction_count);
|
||||
|
||||
// Apply pre-execution changes (e.g., beacon root update)
|
||||
let pre_exec_start = Instant::now();
|
||||
debug_span!(target: "engine::tree", "pre execution")
|
||||
.in_scope(|| executor.apply_pre_execution_changes())?;
|
||||
self.metrics.record_pre_execution(pre_exec_start.elapsed());
|
||||
|
||||
// Execute transactions
|
||||
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
|
||||
let mut transactions = transactions.into_iter();
|
||||
loop {
|
||||
// Measure time spent waiting for next transaction from iterator
|
||||
// (e.g., parallel signature recovery)
|
||||
let wait_start = Instant::now();
|
||||
let Some(tx_result) = transactions.next() else { break };
|
||||
self.metrics.record_transaction_wait(wait_start.elapsed());
|
||||
|
||||
let tx = tx_result.map_err(BlockExecutionError::other)?;
|
||||
let tx_signer = *<Tx as alloy_evm::RecoveredTx<InnerTx>>::signer(&tx);
|
||||
let tx_hash = <Tx as alloy_evm::RecoveredTx<InnerTx>>::tx(&tx).tx_hash();
|
||||
|
||||
senders.push(tx_signer);
|
||||
|
||||
let span = debug_span!(
|
||||
target: "engine::tree",
|
||||
"execute tx",
|
||||
?tx_hash,
|
||||
gas_used = tracing::field::Empty,
|
||||
);
|
||||
let enter = span.entered();
|
||||
trace!(target: "engine::tree", "Executing transaction");
|
||||
|
||||
let tx_start = Instant::now();
|
||||
let gas_used = executor.execute_transaction(tx)?;
|
||||
self.metrics.record_transaction_execution(tx_start.elapsed());
|
||||
|
||||
// Send the latest receipt to the background task for incremental root computation
|
||||
if let Some(receipt) = executor.receipts().last() {
|
||||
let tx_index = executor.receipts().len() - 1;
|
||||
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
|
||||
}
|
||||
|
||||
enter.record("gas_used", gas_used);
|
||||
}
|
||||
drop(exec_span);
|
||||
|
||||
Ok((executor, senders))
|
||||
}
|
||||
|
||||
/// Compute state root for the given hashed post state in parallel.
|
||||
///
|
||||
/// Uses an overlay factory which provides the state of the parent block, along with the
|
||||
|
||||
@@ -153,7 +153,7 @@ async fn maintain_txpool_reorg() -> eyre::Result<()> {
|
||||
w1.address(),
|
||||
);
|
||||
let pooled_tx1 = EthPooledTransaction::new(tx1.clone(), 200);
|
||||
let tx_hash1 = *pooled_tx1.clone().hash();
|
||||
let tx_hash1 = *pooled_tx1.hash();
|
||||
|
||||
// build tx2 from wallet2
|
||||
let envelop2 = TransactionTestContext::transfer_tx(1, w2.clone()).await;
|
||||
@@ -162,7 +162,7 @@ async fn maintain_txpool_reorg() -> eyre::Result<()> {
|
||||
w2.address(),
|
||||
);
|
||||
let pooled_tx2 = EthPooledTransaction::new(tx2.clone(), 200);
|
||||
let tx_hash2 = *pooled_tx2.clone().hash();
|
||||
let tx_hash2 = *pooled_tx2.hash();
|
||||
|
||||
let block_info = BlockInfo {
|
||||
block_gas_limit: ETHEREUM_BLOCK_GAS_LIMIT_30M,
|
||||
|
||||
@@ -236,7 +236,7 @@ impl reth_codecs::Compact for Transaction {
|
||||
// # Panics
|
||||
//
|
||||
// A panic will be triggered if an identifier larger than 3 is passed from the database. For
|
||||
// optimism a identifier with value [`DEPOSIT_TX_TYPE_ID`] is allowed.
|
||||
// optimism an identifier with value [`DEPOSIT_TX_TYPE_ID`] is allowed.
|
||||
fn from_compact(buf: &[u8], identifier: usize) -> (Self, &[u8]) {
|
||||
let (tx_type, buf) = TxType::from_compact(buf, identifier);
|
||||
|
||||
|
||||
@@ -148,20 +148,6 @@ pub trait Executor<DB: Database>: Sized {
|
||||
fn size_hint(&self) -> usize;
|
||||
}
|
||||
|
||||
/// Helper type for the output of executing a block.
|
||||
///
|
||||
/// Deprecated: this type is unused within reth and will be removed in the next
|
||||
/// major release. Use `reth_execution_types::BlockExecutionResult` or
|
||||
/// `reth_execution_types::BlockExecutionOutput`.
|
||||
#[deprecated(note = "Use reth_execution_types::BlockExecutionResult or BlockExecutionOutput")]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ExecuteOutput<R> {
|
||||
/// Receipts obtained after executing a block.
|
||||
pub receipts: Vec<R>,
|
||||
/// Cumulative gas used in the block execution.
|
||||
pub gas_used: u64,
|
||||
}
|
||||
|
||||
/// Input for block building. Consumed by [`BlockAssembler`].
|
||||
///
|
||||
/// This struct contains all the data needed by the [`BlockAssembler`] to create
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
use crate::ExecutionOutcome;
|
||||
use alloc::{borrow::Cow, collections::BTreeMap, vec::Vec};
|
||||
use alloy_consensus::{transaction::Recovered, BlockHeader};
|
||||
use alloy_consensus::{transaction::Recovered, BlockHeader, TxReceipt};
|
||||
use alloy_eips::{eip1898::ForkBlock, eip2718::Encodable2718, BlockNumHash};
|
||||
use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash};
|
||||
use alloy_primitives::{Address, BlockHash, BlockNumber, Log, TxHash};
|
||||
use core::{fmt, ops::RangeInclusive};
|
||||
use reth_primitives_traits::{
|
||||
transaction::signed::SignedTransaction, Block, BlockBody, IndexedTx, NodePrimitives,
|
||||
@@ -184,6 +184,19 @@ impl<N: NodePrimitives> Chain<N> {
|
||||
self.execution_outcome.receipts().iter()
|
||||
}
|
||||
|
||||
/// Returns an iterator over all receipts in the chain.
|
||||
pub fn receipts_iter(&self) -> impl Iterator<Item = &N::Receipt> + '_ {
|
||||
self.block_receipts_iter().flatten()
|
||||
}
|
||||
|
||||
/// Returns an iterator over all logs in the chain.
|
||||
pub fn logs_iter(&self) -> impl Iterator<Item = &Log> + '_
|
||||
where
|
||||
N::Receipt: TxReceipt<Log = Log>,
|
||||
{
|
||||
self.receipts_iter().flat_map(|receipt| receipt.logs())
|
||||
}
|
||||
|
||||
/// Returns an iterator over all blocks in the chain with increasing block number.
|
||||
pub fn blocks_iter(&self) -> impl Iterator<Item = &RecoveredBlock<N::Block>> + '_ {
|
||||
self.blocks().iter().map(|block| block.1)
|
||||
|
||||
@@ -106,7 +106,7 @@ impl BanList {
|
||||
self.banned_ips.contains_key(ip)
|
||||
}
|
||||
|
||||
/// checks the ban list to see if it contains the given ip
|
||||
/// checks the ban list to see if it contains the given peer
|
||||
#[inline]
|
||||
pub fn is_banned_peer(&self, peer_id: &PeerId) -> bool {
|
||||
self.banned_peers.contains_key(peer_id)
|
||||
@@ -117,7 +117,7 @@ impl BanList {
|
||||
self.banned_ips.remove(ip);
|
||||
}
|
||||
|
||||
/// Unbans the ip address
|
||||
/// Unbans the peer
|
||||
pub fn unban_peer(&mut self, peer_id: &PeerId) {
|
||||
self.banned_peers.remove(peer_id);
|
||||
}
|
||||
|
||||
@@ -676,19 +676,13 @@ where
|
||||
|
||||
/// Convenience function to [`Self::init_genesis`]
|
||||
pub fn with_genesis(self) -> Result<Self, InitStorageError> {
|
||||
init_genesis_with_settings(
|
||||
self.provider_factory(),
|
||||
self.node_config().static_files.to_settings(),
|
||||
)?;
|
||||
init_genesis_with_settings(self.provider_factory(), self.node_config().storage_settings())?;
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
/// Write the genesis block and state if it has not already been written
|
||||
pub fn init_genesis(&self) -> Result<B256, InitStorageError> {
|
||||
init_genesis_with_settings(
|
||||
self.provider_factory(),
|
||||
self.node_config().static_files.to_settings(),
|
||||
)
|
||||
init_genesis_with_settings(self.provider_factory(), self.node_config().storage_settings())
|
||||
}
|
||||
|
||||
/// Creates a new `WithMeteredProvider` container and attaches it to the
|
||||
@@ -1283,6 +1277,10 @@ pub fn metrics_hooks<N: NodeTypesWithDB>(provider_factory: &ProviderFactory<N>)
|
||||
})
|
||||
}
|
||||
})
|
||||
.with_hook({
|
||||
let rocksdb = provider_factory.rocksdb_provider();
|
||||
move || throttle!(Duration::from_secs(5 * 60), || rocksdb.report_metrics())
|
||||
})
|
||||
.build()
|
||||
}
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ use reth_node_core::{
|
||||
use reth_node_events::node;
|
||||
use reth_provider::{
|
||||
providers::{BlockchainProvider, NodeTypesForProvider},
|
||||
BlockNumReader, MetadataProvider,
|
||||
BlockNumReader, StorageSettingsCache,
|
||||
};
|
||||
use reth_tasks::TaskExecutor;
|
||||
use reth_tokio_util::EventSender;
|
||||
@@ -41,7 +41,6 @@ use reth_trie_db::ChangesetCache;
|
||||
use std::{future::Future, pin::Pin, sync::Arc};
|
||||
use tokio::sync::{mpsc::unbounded_channel, oneshot};
|
||||
use tokio_stream::wrappers::UnboundedReceiverStream;
|
||||
use tracing::warn;
|
||||
|
||||
/// The engine node launcher.
|
||||
#[derive(Debug)]
|
||||
@@ -104,24 +103,8 @@ impl EngineNodeLauncher {
|
||||
.with_adjusted_configs()
|
||||
// Create the provider factory with changeset cache
|
||||
.with_provider_factory::<_, <CB::Components as NodeComponents<T>>::Evm>(changeset_cache.clone()).await?
|
||||
.inspect(|ctx| {
|
||||
.inspect(|_| {
|
||||
info!(target: "reth::cli", "Database opened");
|
||||
match ctx.provider_factory().storage_settings() {
|
||||
Ok(settings) => {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
?settings,
|
||||
"Storage settings"
|
||||
);
|
||||
},
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "reth::cli",
|
||||
?err,
|
||||
"Failed to get storage settings"
|
||||
);
|
||||
},
|
||||
}
|
||||
})
|
||||
.with_prometheus_server().await?
|
||||
.inspect(|this| {
|
||||
@@ -130,6 +113,8 @@ impl EngineNodeLauncher {
|
||||
.with_genesis()?
|
||||
.inspect(|this: &LaunchContextWith<Attached<WithConfigs<<T::Types as NodeTypes>::ChainSpec>, _>>| {
|
||||
info!(target: "reth::cli", "\n{}", this.chain_spec().display_hardforks());
|
||||
let settings = this.provider_factory().cached_storage_settings();
|
||||
info!(target: "reth::cli", ?settings, "Loaded storage settings");
|
||||
})
|
||||
.with_metrics_task()
|
||||
// passing FullNodeTypes as type parameter here so that we can build
|
||||
|
||||
@@ -19,7 +19,6 @@ reth-cli-util.workspace = true
|
||||
reth-db = { workspace = true, features = ["mdbx"] }
|
||||
reth-storage-errors.workspace = true
|
||||
reth-storage-api = { workspace = true, features = ["std", "db-api"] }
|
||||
reth-provider.workspace = true
|
||||
reth-network = { workspace = true, features = ["serde"] }
|
||||
reth-network-p2p.workspace = true
|
||||
reth-rpc-eth-types.workspace = true
|
||||
@@ -92,7 +91,7 @@ min-debug-logs = ["tracing/release_max_level_debug"]
|
||||
min-trace-logs = ["tracing/release_max_level_trace"]
|
||||
|
||||
# Marker feature for edge/unstable builds - captured by vergen in build.rs
|
||||
edge = []
|
||||
edge = ["reth-storage-api/edge"]
|
||||
|
||||
[build-dependencies]
|
||||
vergen = { workspace = true, features = ["build", "cargo", "emit_and_set"] }
|
||||
|
||||
@@ -37,6 +37,7 @@ pub struct DefaultEngineValues {
|
||||
storage_worker_count: Option<usize>,
|
||||
account_worker_count: Option<usize>,
|
||||
enable_proof_v2: bool,
|
||||
cache_metrics_disabled: bool,
|
||||
}
|
||||
|
||||
impl DefaultEngineValues {
|
||||
@@ -172,6 +173,12 @@ impl DefaultEngineValues {
|
||||
self.enable_proof_v2 = v;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether to disable cache metrics by default
|
||||
pub const fn with_cache_metrics_disabled(mut self, v: bool) -> Self {
|
||||
self.cache_metrics_disabled = v;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DefaultEngineValues {
|
||||
@@ -197,6 +204,7 @@ impl Default for DefaultEngineValues {
|
||||
storage_worker_count: None,
|
||||
account_worker_count: None,
|
||||
enable_proof_v2: false,
|
||||
cache_metrics_disabled: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -320,6 +328,10 @@ pub struct EngineArgs {
|
||||
/// Enable V2 storage proofs for state root calculations
|
||||
#[arg(long = "engine.enable-proof-v2", default_value_t = DefaultEngineValues::get_global().enable_proof_v2)]
|
||||
pub enable_proof_v2: bool,
|
||||
|
||||
/// Disable cache metrics recording, which can take up to 50ms with large cached state.
|
||||
#[arg(long = "engine.disable-cache-metrics", default_value_t = DefaultEngineValues::get_global().cache_metrics_disabled)]
|
||||
pub cache_metrics_disabled: bool,
|
||||
}
|
||||
|
||||
#[allow(deprecated)]
|
||||
@@ -346,6 +358,7 @@ impl Default for EngineArgs {
|
||||
storage_worker_count,
|
||||
account_worker_count,
|
||||
enable_proof_v2,
|
||||
cache_metrics_disabled,
|
||||
} = DefaultEngineValues::get_global().clone();
|
||||
Self {
|
||||
persistence_threshold,
|
||||
@@ -371,6 +384,7 @@ impl Default for EngineArgs {
|
||||
storage_worker_count,
|
||||
account_worker_count,
|
||||
enable_proof_v2,
|
||||
cache_metrics_disabled,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -407,6 +421,7 @@ impl EngineArgs {
|
||||
}
|
||||
|
||||
config = config.with_enable_proof_v2(self.enable_proof_v2);
|
||||
config = config.without_cache_metrics(self.cache_metrics_disabled);
|
||||
|
||||
config
|
||||
}
|
||||
@@ -458,6 +473,7 @@ mod tests {
|
||||
storage_worker_count: Some(16),
|
||||
account_worker_count: Some(8),
|
||||
enable_proof_v2: false,
|
||||
cache_metrics_disabled: true,
|
||||
};
|
||||
|
||||
let parsed_args = CommandParser::<EngineArgs>::parse_from([
|
||||
@@ -488,6 +504,7 @@ mod tests {
|
||||
"16",
|
||||
"--engine.account-worker-count",
|
||||
"8",
|
||||
"--engine.disable-cache-metrics",
|
||||
])
|
||||
.args;
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ pub use dev::DevArgs;
|
||||
|
||||
/// PruneArgs for configuring the pruning and full node
|
||||
mod pruning;
|
||||
pub use pruning::PruningArgs;
|
||||
pub use pruning::{DefaultPruningValues, PruningArgs};
|
||||
|
||||
/// DatadirArgs for configuring data storage paths
|
||||
mod datadir_args;
|
||||
@@ -80,5 +80,9 @@ pub use era::{DefaultEraHost, EraArgs, EraSourceArgs};
|
||||
mod static_files;
|
||||
pub use static_files::{StaticFilesArgs, MINIMAL_BLOCKS_PER_FILE};
|
||||
|
||||
/// `RocksDbArgs` for configuring RocksDB table routing.
|
||||
mod rocksdb;
|
||||
pub use rocksdb::{RocksDbArgs, RocksDbArgsError};
|
||||
|
||||
mod error;
|
||||
pub mod types;
|
||||
|
||||
@@ -6,7 +6,88 @@ use clap::{builder::RangedU64ValueParser, Args};
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_config::config::PruneConfig;
|
||||
use reth_prune_types::{PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_PRUNING_DISTANCE};
|
||||
use std::{collections::BTreeMap, ops::Not};
|
||||
use std::{collections::BTreeMap, ops::Not, sync::OnceLock};
|
||||
|
||||
/// Global static pruning defaults
|
||||
static PRUNING_DEFAULTS: OnceLock<DefaultPruningValues> = OnceLock::new();
|
||||
|
||||
/// Default values for `--full` and `--minimal` pruning modes that can be customized.
|
||||
///
|
||||
/// Global defaults can be set via [`DefaultPruningValues::try_init`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DefaultPruningValues {
|
||||
/// Prune modes for `--full` flag.
|
||||
///
|
||||
/// Note: `bodies_history` is ignored when `full_bodies_history_use_pre_merge` is `true`.
|
||||
pub full_prune_modes: PruneModes,
|
||||
/// If `true`, `--full` will set `bodies_history` to prune everything before the merge block
|
||||
/// (Paris hardfork). If `false`, uses `full_prune_modes.bodies_history` directly.
|
||||
pub full_bodies_history_use_pre_merge: bool,
|
||||
/// Prune modes for `--minimal` flag.
|
||||
pub minimal_prune_modes: PruneModes,
|
||||
}
|
||||
|
||||
impl DefaultPruningValues {
|
||||
/// Initialize the global pruning defaults with this configuration.
|
||||
///
|
||||
/// Returns `Err(self)` if already initialized.
|
||||
pub fn try_init(self) -> Result<(), Self> {
|
||||
PRUNING_DEFAULTS.set(self)
|
||||
}
|
||||
|
||||
/// Get a reference to the global pruning defaults.
|
||||
pub fn get_global() -> &'static Self {
|
||||
PRUNING_DEFAULTS.get_or_init(Self::default)
|
||||
}
|
||||
|
||||
/// Set the prune modes for `--full` flag.
|
||||
pub fn with_full_prune_modes(mut self, modes: PruneModes) -> Self {
|
||||
self.full_prune_modes = modes;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether `--full` should use pre-merge pruning for bodies history.
|
||||
///
|
||||
/// When `true` (default), bodies are pruned before the Paris hardfork block.
|
||||
/// When `false`, uses `full_prune_modes.bodies_history` directly.
|
||||
pub const fn with_full_bodies_history_use_pre_merge(mut self, use_pre_merge: bool) -> Self {
|
||||
self.full_bodies_history_use_pre_merge = use_pre_merge;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the prune modes for `--minimal` flag.
|
||||
pub fn with_minimal_prune_modes(mut self, modes: PruneModes) -> Self {
|
||||
self.minimal_prune_modes = modes;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DefaultPruningValues {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
full_prune_modes: PruneModes {
|
||||
sender_recovery: Some(PruneMode::Full),
|
||||
transaction_lookup: None,
|
||||
receipts: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
account_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
storage_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
// This field is ignored when full_bodies_history_use_pre_merge is true
|
||||
bodies_history: None,
|
||||
receipts_log_filter: Default::default(),
|
||||
},
|
||||
full_bodies_history_use_pre_merge: true,
|
||||
minimal_prune_modes: PruneModes {
|
||||
sender_recovery: Some(PruneMode::Full),
|
||||
transaction_lookup: Some(PruneMode::Full),
|
||||
receipts: Some(PruneMode::Full),
|
||||
account_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
storage_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
bodies_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
receipts_log_filter: Default::default(),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parameters for pruning and full node
|
||||
#[derive(Debug, Clone, Args, PartialEq, Eq, Default)]
|
||||
@@ -128,36 +209,22 @@ impl PruningArgs {
|
||||
|
||||
// If --full is set, use full node defaults.
|
||||
if self.full {
|
||||
config = PruneConfig {
|
||||
block_interval: config.block_interval,
|
||||
segments: PruneModes {
|
||||
sender_recovery: Some(PruneMode::Full),
|
||||
transaction_lookup: None,
|
||||
receipts: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
account_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
storage_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
|
||||
bodies_history: chain_spec
|
||||
.ethereum_fork_activation(EthereumHardfork::Paris)
|
||||
.block_number()
|
||||
.map(PruneMode::Before),
|
||||
receipts_log_filter: Default::default(),
|
||||
},
|
||||
let defaults = DefaultPruningValues::get_global();
|
||||
let mut segments = defaults.full_prune_modes.clone();
|
||||
if defaults.full_bodies_history_use_pre_merge {
|
||||
segments.bodies_history = chain_spec
|
||||
.ethereum_fork_activation(EthereumHardfork::Paris)
|
||||
.block_number()
|
||||
.map(PruneMode::Before);
|
||||
}
|
||||
config = PruneConfig { block_interval: config.block_interval, segments }
|
||||
}
|
||||
|
||||
// If --minimal is set, use minimal storage mode with aggressive pruning.
|
||||
if self.minimal {
|
||||
config = PruneConfig {
|
||||
block_interval: config.block_interval,
|
||||
segments: PruneModes {
|
||||
sender_recovery: Some(PruneMode::Full),
|
||||
transaction_lookup: Some(PruneMode::Full),
|
||||
receipts: Some(PruneMode::Full),
|
||||
account_history: Some(PruneMode::Distance(10064)),
|
||||
storage_history: Some(PruneMode::Distance(10064)),
|
||||
bodies_history: Some(PruneMode::Distance(10064)),
|
||||
receipts_log_filter: Default::default(),
|
||||
},
|
||||
segments: DefaultPruningValues::get_global().minimal_prune_modes.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
160
crates/node/core/src/args/rocksdb.rs
Normal file
160
crates/node/core/src/args/rocksdb.rs
Normal file
@@ -0,0 +1,160 @@
|
||||
//! clap [Args](clap::Args) for `RocksDB` table routing configuration
|
||||
|
||||
use clap::{ArgAction, Args};
|
||||
|
||||
/// Default value for `RocksDB` routing flags.
|
||||
///
|
||||
/// When the `edge` feature is enabled, defaults to `true` to enable edge storage features.
|
||||
/// Otherwise defaults to `false` for legacy behavior.
|
||||
const fn default_rocksdb_flag() -> bool {
|
||||
cfg!(feature = "edge")
|
||||
}
|
||||
|
||||
/// Parameters for `RocksDB` table routing configuration.
|
||||
///
|
||||
/// These flags control which database tables are stored in `RocksDB` instead of MDBX.
|
||||
/// All flags are genesis-initialization-only: changing them after genesis requires a re-sync.
|
||||
#[derive(Debug, Args, PartialEq, Eq, Clone, Copy)]
|
||||
#[command(next_help_heading = "RocksDB")]
|
||||
pub struct RocksDbArgs {
|
||||
/// Route all supported tables to `RocksDB` instead of MDBX.
|
||||
///
|
||||
/// This enables `RocksDB` for `tx-hash`, `storages-history`, and `account-history` tables.
|
||||
/// Cannot be combined with individual flags set to false.
|
||||
#[arg(long = "rocksdb.all", action = ArgAction::SetTrue)]
|
||||
pub all: bool,
|
||||
|
||||
/// Route tx hash -> number table to `RocksDB` instead of MDBX.
|
||||
///
|
||||
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
|
||||
/// Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
#[arg(long = "rocksdb.tx-hash", default_value_t = default_rocksdb_flag(), action = ArgAction::Set)]
|
||||
pub tx_hash: bool,
|
||||
|
||||
/// Route storages history tables to `RocksDB` instead of MDBX.
|
||||
///
|
||||
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
|
||||
/// Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
#[arg(long = "rocksdb.storages-history", default_value_t = default_rocksdb_flag(), action = ArgAction::Set)]
|
||||
pub storages_history: bool,
|
||||
|
||||
/// Route account history tables to `RocksDB` instead of MDBX.
|
||||
///
|
||||
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
|
||||
/// Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
|
||||
#[arg(long = "rocksdb.account-history", default_value_t = default_rocksdb_flag(), action = ArgAction::Set)]
|
||||
pub account_history: bool,
|
||||
}
|
||||
|
||||
impl Default for RocksDbArgs {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
all: false,
|
||||
tx_hash: default_rocksdb_flag(),
|
||||
storages_history: default_rocksdb_flag(),
|
||||
account_history: default_rocksdb_flag(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RocksDbArgs {
|
||||
/// Validates the `RocksDB` arguments.
|
||||
///
|
||||
/// Returns an error if `--rocksdb.all` is used with any individual flag set to `false`.
|
||||
pub const fn validate(&self) -> Result<(), RocksDbArgsError> {
|
||||
if self.all {
|
||||
if !self.tx_hash {
|
||||
return Err(RocksDbArgsError::ConflictingFlags("tx-hash"));
|
||||
}
|
||||
if !self.storages_history {
|
||||
return Err(RocksDbArgsError::ConflictingFlags("storages-history"));
|
||||
}
|
||||
if !self.account_history {
|
||||
return Err(RocksDbArgsError::ConflictingFlags("account-history"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Error type for `RocksDB` argument validation.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum RocksDbArgsError {
|
||||
/// `--rocksdb.all` cannot be combined with an individual flag set to false.
|
||||
#[error("--rocksdb.all cannot be combined with --rocksdb.{0}=false")]
|
||||
ConflictingFlags(&'static str),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use clap::Parser;
|
||||
|
||||
#[derive(Parser)]
|
||||
struct CommandParser<T: Args> {
|
||||
#[command(flatten)]
|
||||
args: T,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_default_rocksdb_args() {
|
||||
let args = CommandParser::<RocksDbArgs>::parse_from(["reth"]).args;
|
||||
assert_eq!(args, RocksDbArgs::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_all_flag() {
|
||||
let args = CommandParser::<RocksDbArgs>::parse_from(["reth", "--rocksdb.all"]).args;
|
||||
assert!(args.all);
|
||||
assert_eq!(args.tx_hash, default_rocksdb_flag());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_individual_flags() {
|
||||
let args = CommandParser::<RocksDbArgs>::parse_from([
|
||||
"reth",
|
||||
"--rocksdb.tx-hash=true",
|
||||
"--rocksdb.storages-history=false",
|
||||
"--rocksdb.account-history=true",
|
||||
])
|
||||
.args;
|
||||
assert!(!args.all);
|
||||
assert!(args.tx_hash);
|
||||
assert!(!args.storages_history);
|
||||
assert!(args.account_history);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_all_with_true_ok() {
|
||||
let args =
|
||||
RocksDbArgs { all: true, tx_hash: true, storages_history: true, account_history: true };
|
||||
assert!(args.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_all_with_false_errors() {
|
||||
let args = RocksDbArgs {
|
||||
all: true,
|
||||
tx_hash: false,
|
||||
storages_history: true,
|
||||
account_history: true,
|
||||
};
|
||||
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("tx-hash")));
|
||||
|
||||
let args = RocksDbArgs {
|
||||
all: true,
|
||||
tx_hash: true,
|
||||
storages_history: false,
|
||||
account_history: true,
|
||||
};
|
||||
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("storages-history")));
|
||||
|
||||
let args = RocksDbArgs {
|
||||
all: true,
|
||||
tx_hash: true,
|
||||
storages_history: true,
|
||||
account_history: false,
|
||||
};
|
||||
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("account-history")));
|
||||
}
|
||||
}
|
||||
@@ -2,15 +2,22 @@
|
||||
|
||||
use clap::Args;
|
||||
use reth_config::config::{BlocksPerFileConfig, StaticFilesConfig};
|
||||
use reth_provider::StorageSettings;
|
||||
|
||||
/// Blocks per static file when running in `--minimal` node.
|
||||
///
|
||||
/// 10000 blocks per static file allows us to prune all history every 10k blocks.
|
||||
pub const MINIMAL_BLOCKS_PER_FILE: u64 = 10000;
|
||||
|
||||
/// Default value for static file storage flags.
|
||||
///
|
||||
/// When the `edge` feature is enabled, defaults to `true` to enable edge storage features.
|
||||
/// Otherwise defaults to `false` for legacy behavior.
|
||||
const fn default_static_file_flag() -> bool {
|
||||
cfg!(feature = "edge")
|
||||
}
|
||||
|
||||
/// Parameters for static files configuration
|
||||
#[derive(Debug, Args, PartialEq, Eq, Default, Clone, Copy)]
|
||||
#[derive(Debug, Args, PartialEq, Eq, Clone, Copy)]
|
||||
#[command(next_help_heading = "Static Files")]
|
||||
pub struct StaticFilesArgs {
|
||||
/// Number of blocks per file for the headers segment.
|
||||
@@ -39,7 +46,7 @@ pub struct StaticFilesArgs {
|
||||
///
|
||||
/// Note: This setting can only be configured at genesis initialization. Once
|
||||
/// the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
#[arg(long = "static-files.receipts")]
|
||||
#[arg(long = "static-files.receipts", default_value_t = default_static_file_flag(), action = clap::ArgAction::Set)]
|
||||
pub receipts: bool,
|
||||
|
||||
/// Store transaction senders in static files instead of the database.
|
||||
@@ -49,7 +56,7 @@ pub struct StaticFilesArgs {
|
||||
///
|
||||
/// Note: This setting can only be configured at genesis initialization. Once
|
||||
/// the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
#[arg(long = "static-files.transaction-senders")]
|
||||
#[arg(long = "static-files.transaction-senders", default_value_t = default_static_file_flag(), action = clap::ArgAction::Set)]
|
||||
pub transaction_senders: bool,
|
||||
|
||||
/// Store account changesets in static files.
|
||||
@@ -59,7 +66,7 @@ pub struct StaticFilesArgs {
|
||||
///
|
||||
/// Note: This setting can only be configured at genesis initialization. Once
|
||||
/// the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
#[arg(long = "static-files.account-change-sets")]
|
||||
#[arg(long = "static-files.account-change-sets", default_value_t = default_static_file_flag(), action = clap::ArgAction::Set)]
|
||||
pub account_changesets: bool,
|
||||
}
|
||||
|
||||
@@ -94,12 +101,19 @@ impl StaticFilesArgs {
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts the static files arguments into [`StorageSettings`].
|
||||
pub const fn to_settings(&self) -> StorageSettings {
|
||||
StorageSettings::legacy()
|
||||
.with_receipts_in_static_files(self.receipts)
|
||||
.with_transaction_senders_in_static_files(self.transaction_senders)
|
||||
.with_account_changesets_in_static_files(self.account_changesets)
|
||||
impl Default for StaticFilesArgs {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
blocks_per_file_headers: None,
|
||||
blocks_per_file_transactions: None,
|
||||
blocks_per_file_receipts: None,
|
||||
blocks_per_file_transaction_senders: None,
|
||||
blocks_per_file_account_change_sets: None,
|
||||
receipts: default_static_file_flag(),
|
||||
transaction_senders: default_static_file_flag(),
|
||||
account_changesets: default_static_file_flag(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use crate::{
|
||||
args::{
|
||||
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, NetworkArgs, PayloadBuilderArgs,
|
||||
PruningArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
|
||||
PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
|
||||
},
|
||||
dirs::{ChainPath, DataDirPath},
|
||||
utils::get_single_header,
|
||||
@@ -21,6 +21,7 @@ use reth_primitives_traits::SealedHeader;
|
||||
use reth_stages_types::StageId;
|
||||
use reth_storage_api::{
|
||||
BlockHashReader, DatabaseProviderFactory, HeaderProvider, StageCheckpointReader,
|
||||
StorageSettings,
|
||||
};
|
||||
use reth_storage_errors::provider::ProviderResult;
|
||||
use reth_transaction_pool::TransactionPool;
|
||||
@@ -150,6 +151,9 @@ pub struct NodeConfig<ChainSpec> {
|
||||
|
||||
/// All static files related arguments
|
||||
pub static_files: StaticFilesArgs,
|
||||
|
||||
/// All `RocksDB` table routing arguments
|
||||
pub rocksdb: RocksDbArgs,
|
||||
}
|
||||
|
||||
impl NodeConfig<ChainSpec> {
|
||||
@@ -181,6 +185,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
engine: EngineArgs::default(),
|
||||
era: EraArgs::default(),
|
||||
static_files: StaticFilesArgs::default(),
|
||||
rocksdb: RocksDbArgs::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -255,6 +260,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
rocksdb,
|
||||
..
|
||||
} = self;
|
||||
NodeConfig {
|
||||
@@ -274,6 +280,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
rocksdb,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -350,6 +357,17 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
self.pruning.prune_config(&self.chain)
|
||||
}
|
||||
|
||||
/// Returns the effective storage settings derived from static-file and `RocksDB` CLI args.
|
||||
pub const fn storage_settings(&self) -> StorageSettings {
|
||||
StorageSettings::base()
|
||||
.with_receipts_in_static_files(self.static_files.receipts)
|
||||
.with_transaction_senders_in_static_files(self.static_files.transaction_senders)
|
||||
.with_account_changesets_in_static_files(self.static_files.account_changesets)
|
||||
.with_transaction_hash_numbers_in_rocksdb(self.rocksdb.all || self.rocksdb.tx_hash)
|
||||
.with_storages_history_in_rocksdb(self.rocksdb.all || self.rocksdb.storages_history)
|
||||
.with_account_history_in_rocksdb(self.rocksdb.all || self.rocksdb.account_history)
|
||||
}
|
||||
|
||||
/// Returns the max block that the node should run to, looking it up from the network if
|
||||
/// necessary
|
||||
pub async fn max_block<Provider, Client>(
|
||||
@@ -544,6 +562,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
engine: self.engine,
|
||||
era: self.era,
|
||||
static_files: self.static_files,
|
||||
rocksdb: self.rocksdb,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -585,6 +604,7 @@ impl<ChainSpec> Clone for NodeConfig<ChainSpec> {
|
||||
engine: self.engine.clone(),
|
||||
era: self.era.clone(),
|
||||
static_files: self.static_files,
|
||||
rocksdb: self.rocksdb,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,6 +106,7 @@ impl MetricServer {
|
||||
// Describe metrics after recorder installation
|
||||
describe_db_metrics();
|
||||
describe_static_file_metrics();
|
||||
describe_rocksdb_metrics();
|
||||
Collector::default().describe();
|
||||
describe_memory_stats();
|
||||
describe_io_stats();
|
||||
@@ -238,6 +239,26 @@ fn describe_static_file_metrics() {
|
||||
);
|
||||
}
|
||||
|
||||
fn describe_rocksdb_metrics() {
|
||||
describe_gauge!(
|
||||
"rocksdb.table_size",
|
||||
Unit::Bytes,
|
||||
"The estimated size of a RocksDB table (SST + memtable)"
|
||||
);
|
||||
describe_gauge!("rocksdb.table_entries", "The estimated number of keys in a RocksDB table");
|
||||
describe_gauge!(
|
||||
"rocksdb.pending_compaction_bytes",
|
||||
Unit::Bytes,
|
||||
"Bytes pending compaction for a RocksDB table"
|
||||
);
|
||||
describe_gauge!("rocksdb.sst_size", Unit::Bytes, "The size of SST files for a RocksDB table");
|
||||
describe_gauge!(
|
||||
"rocksdb.memtable_size",
|
||||
Unit::Bytes,
|
||||
"The size of memtables for a RocksDB table"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "jemalloc", unix))]
|
||||
fn describe_memory_stats() {
|
||||
describe_gauge!(
|
||||
|
||||
@@ -12,10 +12,18 @@ use reth_primitives_traits::{AlloyBlockHeader, BlockTy, HeaderTy, NodePrimitives
|
||||
use reth_revm::cached::CachedReads;
|
||||
use reth_storage_api::{BlockReaderIdExt, StateProviderFactory};
|
||||
use reth_tasks::TaskExecutor;
|
||||
use std::{sync::Arc, time::Instant};
|
||||
use tokio::sync::{oneshot, watch};
|
||||
use std::{
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tokio::{
|
||||
sync::{oneshot, watch},
|
||||
time::sleep,
|
||||
};
|
||||
use tracing::*;
|
||||
|
||||
const CONNECTION_BACKOUT_PERIOD: Duration = Duration::from_secs(5);
|
||||
|
||||
/// The `FlashBlockService` maintains an in-memory [`PendingFlashBlock`] built out of a sequence of
|
||||
/// [`FlashBlock`]s.
|
||||
#[derive(Debug)]
|
||||
@@ -167,7 +175,13 @@ where
|
||||
self.try_start_build_job();
|
||||
}
|
||||
Some(Err(err)) => {
|
||||
warn!(target: "flashblocks", %err, "Error receiving flashblock");
|
||||
warn!(
|
||||
target: "flashblocks",
|
||||
%err,
|
||||
retry_period = CONNECTION_BACKOUT_PERIOD.as_secs(),
|
||||
"Error receiving flashblock"
|
||||
);
|
||||
sleep(CONNECTION_BACKOUT_PERIOD).await;
|
||||
}
|
||||
None => {
|
||||
warn!(target: "flashblocks", "Flashblock stream ended");
|
||||
|
||||
@@ -510,27 +510,12 @@ mod tests {
|
||||
fn test_sealed_block_rlp_roundtrip() {
|
||||
// Create a sample block using alloy_consensus::Block
|
||||
let header = alloy_consensus::Header {
|
||||
parent_hash: B256::ZERO,
|
||||
ommers_hash: B256::ZERO,
|
||||
beneficiary: Address::ZERO,
|
||||
state_root: B256::ZERO,
|
||||
transactions_root: B256::ZERO,
|
||||
receipts_root: B256::ZERO,
|
||||
logs_bloom: Default::default(),
|
||||
difficulty: Default::default(),
|
||||
number: 42,
|
||||
gas_limit: 30_000_000,
|
||||
gas_used: 21_000,
|
||||
timestamp: 1_000_000,
|
||||
extra_data: Default::default(),
|
||||
mix_hash: B256::ZERO,
|
||||
nonce: Default::default(),
|
||||
base_fee_per_gas: Some(1_000_000_000),
|
||||
withdrawals_root: None,
|
||||
blob_gas_used: None,
|
||||
excess_blob_gas: None,
|
||||
parent_beacon_block_root: None,
|
||||
requests_hash: None,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Create a simple transaction
|
||||
@@ -585,27 +570,12 @@ mod tests {
|
||||
fn test_decode_sealed_produces_correct_hash() {
|
||||
// Create a sample block using alloy_consensus::Block
|
||||
let header = alloy_consensus::Header {
|
||||
parent_hash: B256::ZERO,
|
||||
ommers_hash: B256::ZERO,
|
||||
beneficiary: Address::ZERO,
|
||||
state_root: B256::ZERO,
|
||||
transactions_root: B256::ZERO,
|
||||
receipts_root: B256::ZERO,
|
||||
logs_bloom: Default::default(),
|
||||
difficulty: Default::default(),
|
||||
number: 42,
|
||||
gas_limit: 30_000_000,
|
||||
gas_used: 21_000,
|
||||
timestamp: 1_000_000,
|
||||
extra_data: Default::default(),
|
||||
mix_hash: B256::ZERO,
|
||||
nonce: Default::default(),
|
||||
base_fee_per_gas: Some(1_000_000_000),
|
||||
withdrawals_root: None,
|
||||
blob_gas_used: None,
|
||||
excess_blob_gas: None,
|
||||
parent_beacon_block_root: None,
|
||||
requests_hash: None,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Create a simple transaction
|
||||
|
||||
@@ -84,7 +84,14 @@ where
|
||||
.into_inner();
|
||||
let tx_range = start..=
|
||||
Some(end)
|
||||
.min(input.limiter.deleted_entries_limit_left().map(|left| start + left as u64 - 1))
|
||||
.min(
|
||||
input
|
||||
.limiter
|
||||
.deleted_entries_limit_left()
|
||||
// Use saturating addition here to avoid panicking on
|
||||
// `deleted_entries_limit == usize::MAX`
|
||||
.map(|left| start.saturating_add(left as u64) - 1),
|
||||
)
|
||||
.unwrap();
|
||||
let tx_range_end = *tx_range.end();
|
||||
|
||||
|
||||
@@ -252,6 +252,18 @@ pub trait EngineApi<Engine: EngineTypes> {
|
||||
&self,
|
||||
versioned_hashes: Vec<B256>,
|
||||
) -> RpcResult<Option<Vec<Option<BlobAndProofV2>>>>;
|
||||
|
||||
/// Returns the Block Access Lists for the given block hashes.
|
||||
///
|
||||
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
|
||||
#[method(name = "getBALsByHashV1")]
|
||||
async fn get_bals_by_hash_v1(&self, block_hashes: Vec<BlockHash>) -> RpcResult<Vec<Bytes>>;
|
||||
|
||||
/// Returns the Block Access Lists for the given block range.
|
||||
///
|
||||
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
|
||||
#[method(name = "getBALsByRangeV1")]
|
||||
async fn get_bals_by_range_v1(&self, start: U64, count: U64) -> RpcResult<Vec<Bytes>>;
|
||||
}
|
||||
|
||||
/// A subset of the ETH rpc interface: <https://ethereum.github.io/execution-apis/api-documentation>
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
use std::collections::HashSet;
|
||||
//! Engine API capabilities.
|
||||
|
||||
/// The list of all supported Engine capabilities available over the engine endpoint.
|
||||
use std::collections::HashSet;
|
||||
use tracing::warn;
|
||||
|
||||
/// All Engine API capabilities supported by Reth (Ethereum mainnet).
|
||||
///
|
||||
/// See <https://github.com/ethereum/execution-apis/tree/main/src/engine> for updates.
|
||||
pub const CAPABILITIES: &[&str] = &[
|
||||
"engine_forkchoiceUpdatedV1",
|
||||
"engine_forkchoiceUpdatedV2",
|
||||
@@ -22,43 +27,150 @@ pub const CAPABILITIES: &[&str] = &[
|
||||
"engine_getBlobsV3",
|
||||
];
|
||||
|
||||
// The list of all supported Engine capabilities available over the engine endpoint.
|
||||
///
|
||||
/// Latest spec: Prague
|
||||
/// Engine API capabilities set.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EngineCapabilities {
|
||||
inner: HashSet<String>,
|
||||
}
|
||||
|
||||
impl EngineCapabilities {
|
||||
/// Creates a new `EngineCapabilities` instance with the given capabilities.
|
||||
pub fn new(capabilities: impl IntoIterator<Item: Into<String>>) -> Self {
|
||||
/// Creates from an iterator of capability strings.
|
||||
pub fn new(capabilities: impl IntoIterator<Item = impl Into<String>>) -> Self {
|
||||
Self { inner: capabilities.into_iter().map(Into::into).collect() }
|
||||
}
|
||||
|
||||
/// Returns the list of all supported Engine capabilities for Prague spec.
|
||||
fn prague() -> Self {
|
||||
Self { inner: CAPABILITIES.iter().copied().map(str::to_owned).collect() }
|
||||
}
|
||||
|
||||
/// Returns the list of all supported Engine capabilities.
|
||||
/// Returns the capabilities as a list of strings.
|
||||
pub fn list(&self) -> Vec<String> {
|
||||
self.inner.iter().cloned().collect()
|
||||
}
|
||||
|
||||
/// Inserts a new capability.
|
||||
pub fn add_capability(&mut self, capability: impl Into<String>) {
|
||||
self.inner.insert(capability.into());
|
||||
/// Returns a reference to the inner set.
|
||||
pub const fn as_set(&self) -> &HashSet<String> {
|
||||
&self.inner
|
||||
}
|
||||
|
||||
/// Removes a capability.
|
||||
pub fn remove_capability(&mut self, capability: &str) -> Option<String> {
|
||||
self.inner.take(capability)
|
||||
/// Compares CL capabilities with this EL's capabilities and returns any mismatches.
|
||||
///
|
||||
/// Called during `engine_exchangeCapabilities` to detect version mismatches
|
||||
/// between the consensus layer and execution layer.
|
||||
pub fn get_capability_mismatches(&self, cl_capabilities: &[String]) -> CapabilityMismatches {
|
||||
let cl_set: HashSet<&str> = cl_capabilities.iter().map(String::as_str).collect();
|
||||
|
||||
// CL has methods EL doesn't support
|
||||
let mut missing_in_el: Vec<_> = cl_capabilities
|
||||
.iter()
|
||||
.filter(|cap| !self.inner.contains(cap.as_str()))
|
||||
.cloned()
|
||||
.collect();
|
||||
missing_in_el.sort();
|
||||
|
||||
// EL has methods CL doesn't support
|
||||
let mut missing_in_cl: Vec<_> =
|
||||
self.inner.iter().filter(|cap| !cl_set.contains(cap.as_str())).cloned().collect();
|
||||
missing_in_cl.sort();
|
||||
|
||||
CapabilityMismatches { missing_in_el, missing_in_cl }
|
||||
}
|
||||
|
||||
/// Logs warnings if CL and EL capabilities don't match.
|
||||
///
|
||||
/// Called during `engine_exchangeCapabilities` to warn operators about
|
||||
/// version mismatches between the consensus layer and execution layer.
|
||||
pub fn log_capability_mismatches(&self, cl_capabilities: &[String]) {
|
||||
let mismatches = self.get_capability_mismatches(cl_capabilities);
|
||||
|
||||
if !mismatches.missing_in_el.is_empty() {
|
||||
warn!(
|
||||
target: "rpc::engine",
|
||||
missing = ?mismatches.missing_in_el,
|
||||
"CL supports Engine API methods that Reth doesn't. Consider upgrading Reth."
|
||||
);
|
||||
}
|
||||
|
||||
if !mismatches.missing_in_cl.is_empty() {
|
||||
warn!(
|
||||
target: "rpc::engine",
|
||||
missing = ?mismatches.missing_in_cl,
|
||||
"Reth supports Engine API methods that CL doesn't. Consider upgrading your consensus client."
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for EngineCapabilities {
|
||||
fn default() -> Self {
|
||||
Self::prague()
|
||||
Self::new(CAPABILITIES.iter().copied())
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of comparing CL and EL capabilities.
|
||||
#[derive(Debug, Default, PartialEq, Eq)]
|
||||
pub struct CapabilityMismatches {
|
||||
/// Methods supported by CL but not by EL (Reth).
|
||||
/// Operators should consider upgrading Reth.
|
||||
pub missing_in_el: Vec<String>,
|
||||
/// Methods supported by EL (Reth) but not by CL.
|
||||
/// Operators should consider upgrading their consensus client.
|
||||
pub missing_in_cl: Vec<String>,
|
||||
}
|
||||
|
||||
impl CapabilityMismatches {
|
||||
/// Returns `true` if there are no mismatches.
|
||||
pub const fn is_empty(&self) -> bool {
|
||||
self.missing_in_el.is_empty() && self.missing_in_cl.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_no_mismatches() {
|
||||
let el = EngineCapabilities::new(["method_a", "method_b"]);
|
||||
let cl = vec!["method_a".to_string(), "method_b".to_string()];
|
||||
|
||||
let result = el.get_capability_mismatches(&cl);
|
||||
assert!(result.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cl_has_extra_methods() {
|
||||
let el = EngineCapabilities::new(["method_a"]);
|
||||
let cl = vec!["method_a".to_string(), "method_b".to_string()];
|
||||
|
||||
let result = el.get_capability_mismatches(&cl);
|
||||
assert_eq!(result.missing_in_el, vec!["method_b"]);
|
||||
assert!(result.missing_in_cl.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_el_has_extra_methods() {
|
||||
let el = EngineCapabilities::new(["method_a", "method_b"]);
|
||||
let cl = vec!["method_a".to_string()];
|
||||
|
||||
let result = el.get_capability_mismatches(&cl);
|
||||
assert!(result.missing_in_el.is_empty());
|
||||
assert_eq!(result.missing_in_cl, vec!["method_b"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_both_have_extra_methods() {
|
||||
let el = EngineCapabilities::new(["method_a", "method_c"]);
|
||||
let cl = vec!["method_a".to_string(), "method_b".to_string()];
|
||||
|
||||
let result = el.get_capability_mismatches(&cl);
|
||||
assert_eq!(result.missing_in_el, vec!["method_b"]);
|
||||
assert_eq!(result.missing_in_cl, vec!["method_c"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_results_are_sorted() {
|
||||
let el = EngineCapabilities::new(["z_method", "a_method"]);
|
||||
let cl = vec!["z_other".to_string(), "a_other".to_string()];
|
||||
|
||||
let result = el.get_capability_mismatches(&cl);
|
||||
assert_eq!(result.missing_in_el, vec!["a_other", "z_other"]);
|
||||
assert_eq!(result.missing_in_cl, vec!["a_method", "z_method"]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1134,8 +1134,13 @@ where
|
||||
|
||||
/// Handler for `engine_exchangeCapabilitiesV1`
|
||||
/// See also <https://github.com/ethereum/execution-apis/blob/6452a6b194d7db269bf1dbd087a267251d3cc7f8/src/engine/common.md#capabilities>
|
||||
async fn exchange_capabilities(&self, _capabilities: Vec<String>) -> RpcResult<Vec<String>> {
|
||||
Ok(self.capabilities().list())
|
||||
async fn exchange_capabilities(&self, capabilities: Vec<String>) -> RpcResult<Vec<String>> {
|
||||
trace!(target: "rpc::engine", "Serving engine_exchangeCapabilities");
|
||||
|
||||
let el_caps = self.capabilities();
|
||||
el_caps.log_capability_mismatches(&capabilities);
|
||||
|
||||
Ok(el_caps.list())
|
||||
}
|
||||
|
||||
async fn get_blobs_v1(
|
||||
@@ -1161,6 +1166,33 @@ where
|
||||
trace!(target: "rpc::engine", "Serving engine_getBlobsV3");
|
||||
Ok(self.get_blobs_v3_metered(versioned_hashes)?)
|
||||
}
|
||||
|
||||
/// Handler for `engine_getBALsByHashV1`
|
||||
///
|
||||
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
|
||||
async fn get_bals_by_hash_v1(
|
||||
&self,
|
||||
_block_hashes: Vec<BlockHash>,
|
||||
) -> RpcResult<Vec<alloy_primitives::Bytes>> {
|
||||
trace!(target: "rpc::engine", "Serving engine_getBALsByHashV1");
|
||||
Err(EngineApiError::EngineObjectValidationError(
|
||||
reth_payload_primitives::EngineObjectValidationError::UnsupportedFork,
|
||||
))?
|
||||
}
|
||||
|
||||
/// Handler for `engine_getBALsByRangeV1`
|
||||
///
|
||||
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
|
||||
async fn get_bals_by_range_v1(
|
||||
&self,
|
||||
_start: U64,
|
||||
_count: U64,
|
||||
) -> RpcResult<Vec<alloy_primitives::Bytes>> {
|
||||
trace!(target: "rpc::engine", "Serving engine_getBALsByRangeV1");
|
||||
Err(EngineApiError::EngineObjectValidationError(
|
||||
reth_payload_primitives::EngineObjectValidationError::UnsupportedFork,
|
||||
))?
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider, EngineT, Pool, Validator, ChainSpec> IntoEngineApiRpcModule
|
||||
|
||||
@@ -507,6 +507,7 @@ impl From<reth_errors::ProviderError> for EthApiError {
|
||||
ProviderError::BlockNumberForTransactionIndexNotFound => Self::UnknownBlockOrTxIndex,
|
||||
ProviderError::FinalizedBlockNotFound => Self::HeaderNotFound(BlockId::finalized()),
|
||||
ProviderError::SafeBlockNotFound => Self::HeaderNotFound(BlockId::safe()),
|
||||
ProviderError::BlockExpired { .. } => Self::PrunedHistoryUnavailable,
|
||||
err => Self::Internal(err.into()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,6 +86,14 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Clear all ETL state. Called on error paths to prevent buffer pollution on retry.
|
||||
fn clear_etl_state(&mut self) {
|
||||
self.sync_gap = None;
|
||||
self.hash_collector.clear();
|
||||
self.header_collector.clear();
|
||||
self.is_etl_ready = false;
|
||||
}
|
||||
|
||||
/// Write downloaded headers to storage from ETL.
|
||||
///
|
||||
/// Writes to static files ( `Header | HeaderTD | HeaderHash` ) and [`tables::HeaderNumbers`]
|
||||
@@ -258,7 +266,7 @@ where
|
||||
}
|
||||
Some(Err(HeadersDownloaderError::DetachedHead { local_head, header, error })) => {
|
||||
error!(target: "sync::stages::headers", %error, "Cannot attach header to head");
|
||||
self.sync_gap = None;
|
||||
self.clear_etl_state();
|
||||
return Poll::Ready(Err(StageError::DetachedHead {
|
||||
local_head: Box::new(local_head.block_with_parent()),
|
||||
header: Box::new(header.block_with_parent()),
|
||||
@@ -266,7 +274,7 @@ where
|
||||
}))
|
||||
}
|
||||
None => {
|
||||
self.sync_gap = None;
|
||||
self.clear_etl_state();
|
||||
return Poll::Ready(Err(StageError::ChannelClosed))
|
||||
}
|
||||
}
|
||||
@@ -324,7 +332,7 @@ where
|
||||
provider: &Provider,
|
||||
input: UnwindInput,
|
||||
) -> Result<UnwindOutput, StageError> {
|
||||
self.sync_gap.take();
|
||||
self.clear_etl_state();
|
||||
|
||||
// First unwind the db tables, until the unwind_to block number. use the walker to unwind
|
||||
// HeaderNumbers based on the index in CanonicalHeaders
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
use crate::stages::utils::collect_history_indices;
|
||||
|
||||
use super::{collect_account_history_indices, load_history_indices};
|
||||
use alloy_primitives::Address;
|
||||
use super::collect_account_history_indices;
|
||||
use crate::stages::utils::{collect_history_indices, load_account_history};
|
||||
use reth_config::config::{EtlConfig, IndexHistoryConfig};
|
||||
use reth_db_api::{models::ShardedKey, table::Decode, tables, transaction::DbTxMut};
|
||||
use reth_db_api::{models::ShardedKey, tables, transaction::DbTxMut};
|
||||
use reth_provider::{
|
||||
DBProvider, HistoryWriter, PruneCheckpointReader, PruneCheckpointWriter, StorageSettingsCache,
|
||||
DBProvider, EitherWriter, HistoryWriter, PruneCheckpointReader, PruneCheckpointWriter,
|
||||
RocksDBProviderFactory, StorageSettingsCache,
|
||||
};
|
||||
use reth_prune_types::{PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment};
|
||||
use reth_stages_api::{
|
||||
@@ -53,7 +52,8 @@ where
|
||||
+ PruneCheckpointWriter
|
||||
+ reth_storage_api::ChangeSetReader
|
||||
+ reth_provider::StaticFileProviderFactory
|
||||
+ StorageSettingsCache,
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory,
|
||||
{
|
||||
/// Return the id of the stage
|
||||
fn id(&self) -> StageId {
|
||||
@@ -101,15 +101,25 @@ where
|
||||
|
||||
let mut range = input.next_block_range();
|
||||
let first_sync = input.checkpoint().block_number == 0;
|
||||
let use_rocksdb = provider.cached_storage_settings().account_history_in_rocksdb;
|
||||
|
||||
// On first sync we might have history coming from genesis. We clear the table since it's
|
||||
// faster to rebuild from scratch.
|
||||
if first_sync {
|
||||
provider.tx_ref().clear::<tables::AccountsHistory>()?;
|
||||
if use_rocksdb {
|
||||
// Note: RocksDB clear() executes immediately (not deferred to commit like MDBX),
|
||||
// but this is safe for first_sync because if we crash before commit, the
|
||||
// checkpoint stays at 0 and we'll just clear and rebuild again on restart. The
|
||||
// source data (changesets) is intact.
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
provider.rocksdb_provider().clear::<tables::AccountsHistory>()?;
|
||||
} else {
|
||||
provider.tx_ref().clear::<tables::AccountsHistory>()?;
|
||||
}
|
||||
range = 0..=*input.next_block_range().end();
|
||||
}
|
||||
|
||||
info!(target: "sync::stages::index_account_history::exec", ?first_sync, "Collecting indices");
|
||||
info!(target: "sync::stages::index_account_history::exec", ?first_sync, ?use_rocksdb, "Collecting indices");
|
||||
|
||||
let collector = if provider.cached_storage_settings().account_changesets_in_static_files {
|
||||
// Use the provider-based collection that can read from static files.
|
||||
@@ -125,14 +135,13 @@ where
|
||||
};
|
||||
|
||||
info!(target: "sync::stages::index_account_history::exec", "Loading indices into database");
|
||||
load_history_indices::<_, tables::AccountsHistory, _>(
|
||||
provider,
|
||||
collector,
|
||||
first_sync,
|
||||
ShardedKey::new,
|
||||
ShardedKey::<Address>::decode_owned,
|
||||
|key| key.key,
|
||||
)?;
|
||||
|
||||
provider.with_rocksdb_batch(|rocksdb_batch| {
|
||||
let mut writer = EitherWriter::new_accounts_history(provider, rocksdb_batch)?;
|
||||
load_account_history(collector, first_sync, &mut writer)
|
||||
.map_err(|e| reth_provider::ProviderError::other(Box::new(e)))?;
|
||||
Ok(((), writer.into_raw_rocksdb_batch()))
|
||||
})?;
|
||||
|
||||
Ok(ExecOutput { checkpoint: StageCheckpoint::new(*range.end()), done: true })
|
||||
}
|
||||
@@ -160,7 +169,7 @@ mod tests {
|
||||
stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, TestRunnerError,
|
||||
TestStageDB, UnwindStageTestRunner,
|
||||
};
|
||||
use alloy_primitives::{address, BlockNumber, B256};
|
||||
use alloy_primitives::{address, Address, BlockNumber, B256};
|
||||
use itertools::Itertools;
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO,
|
||||
@@ -646,4 +655,169 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
mod rocksdb_tests {
|
||||
use super::*;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use reth_storage_api::StorageSettings;
|
||||
|
||||
/// Test that when `account_history_in_rocksdb` is enabled, the stage
|
||||
/// writes account history indices to `RocksDB` instead of MDBX.
|
||||
#[tokio::test]
|
||||
async fn execute_writes_to_rocksdb_when_enabled() {
|
||||
// init
|
||||
let db = TestStageDB::default();
|
||||
|
||||
// Enable RocksDB for account history
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_account_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), ..Default::default() };
|
||||
let mut stage = IndexAccountHistoryStage::default();
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
// Verify MDBX table is empty (data should be in RocksDB)
|
||||
let mdbx_table = db.table::<tables::AccountsHistory>().unwrap();
|
||||
assert!(
|
||||
mdbx_table.is_empty(),
|
||||
"MDBX AccountsHistory should be empty when RocksDB is enabled"
|
||||
);
|
||||
|
||||
// Verify RocksDB has the data
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should contain account history");
|
||||
|
||||
let block_list = result.unwrap();
|
||||
let blocks: Vec<u64> = block_list.iter().collect();
|
||||
assert_eq!(blocks, (0..=10).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
/// Test that unwind works correctly when `account_history_in_rocksdb` is enabled.
|
||||
#[tokio::test]
|
||||
async fn unwind_works_when_rocksdb_enabled() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_account_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), ..Default::default() };
|
||||
let mut stage = IndexAccountHistoryStage::default();
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
// Verify RocksDB has blocks 0-10 before unwind
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should have data before unwind");
|
||||
let blocks_before: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks_before, (0..=10).collect::<Vec<_>>());
|
||||
|
||||
// Unwind to block 5 (remove blocks 6-10)
|
||||
let unwind_input =
|
||||
UnwindInput { checkpoint: StageCheckpoint::new(10), unwind_to: 5, bad_block: None };
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.unwind(&provider, unwind_input).unwrap();
|
||||
assert_eq!(out, UnwindOutput { checkpoint: StageCheckpoint::new(5) });
|
||||
provider.commit().unwrap();
|
||||
|
||||
// Verify RocksDB now only has blocks 0-5 (blocks 6-10 removed)
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should still have data after unwind");
|
||||
let blocks_after: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks_after, (0..=5).collect::<Vec<_>>(), "Should only have blocks 0-5");
|
||||
}
|
||||
|
||||
/// Test incremental sync merges new data with existing shards.
|
||||
#[tokio::test]
|
||||
async fn execute_incremental_sync() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_account_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=5 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(5), ..Default::default() };
|
||||
let mut stage = IndexAccountHistoryStage::default();
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(5), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some());
|
||||
let blocks: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks, (0..=5).collect::<Vec<_>>());
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 6..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), checkpoint: Some(StageCheckpoint::new(5)) };
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should have merged data");
|
||||
let blocks: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks, (0..=10).collect::<Vec<_>>());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,19 +1,21 @@
|
||||
use super::{collect_history_indices, load_history_indices};
|
||||
use crate::{StageCheckpoint, StageId};
|
||||
use super::collect_history_indices;
|
||||
use crate::{stages::utils::load_storage_history, StageCheckpoint, StageId};
|
||||
use reth_config::config::{EtlConfig, IndexHistoryConfig};
|
||||
use reth_db_api::{
|
||||
models::{storage_sharded_key::StorageShardedKey, AddressStorageKey, BlockNumberAddress},
|
||||
table::Decode,
|
||||
tables,
|
||||
transaction::DbTxMut,
|
||||
};
|
||||
use reth_provider::{DBProvider, HistoryWriter, PruneCheckpointReader, PruneCheckpointWriter};
|
||||
use reth_provider::{
|
||||
DBProvider, EitherWriter, HistoryWriter, PruneCheckpointReader, PruneCheckpointWriter,
|
||||
RocksDBProviderFactory, StorageSettingsCache,
|
||||
};
|
||||
use reth_prune_types::{PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment};
|
||||
use reth_stages_api::{ExecInput, ExecOutput, Stage, StageError, UnwindInput, UnwindOutput};
|
||||
use std::fmt::Debug;
|
||||
use tracing::info;
|
||||
|
||||
/// Stage is indexing history the account changesets generated in
|
||||
/// Stage is indexing history the storage changesets generated in
|
||||
/// [`ExecutionStage`][crate::stages::ExecutionStage]. For more information
|
||||
/// on index sharding take a look at [`tables::StoragesHistory`].
|
||||
#[derive(Debug)]
|
||||
@@ -34,7 +36,7 @@ impl IndexStorageHistoryStage {
|
||||
etl_config: EtlConfig,
|
||||
prune_mode: Option<PruneMode>,
|
||||
) -> Self {
|
||||
Self { commit_threshold: config.commit_threshold, prune_mode, etl_config }
|
||||
Self { commit_threshold: config.commit_threshold, etl_config, prune_mode }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,8 +48,13 @@ impl Default for IndexStorageHistoryStage {
|
||||
|
||||
impl<Provider> Stage<Provider> for IndexStorageHistoryStage
|
||||
where
|
||||
Provider:
|
||||
DBProvider<Tx: DbTxMut> + PruneCheckpointWriter + HistoryWriter + PruneCheckpointReader,
|
||||
Provider: DBProvider<Tx: DbTxMut>
|
||||
+ HistoryWriter
|
||||
+ PruneCheckpointReader
|
||||
+ PruneCheckpointWriter
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory
|
||||
+ reth_provider::NodePrimitivesProvider,
|
||||
{
|
||||
/// Return the id of the stage
|
||||
fn id(&self) -> StageId {
|
||||
@@ -95,15 +102,25 @@ where
|
||||
|
||||
let mut range = input.next_block_range();
|
||||
let first_sync = input.checkpoint().block_number == 0;
|
||||
let use_rocksdb = provider.cached_storage_settings().storages_history_in_rocksdb;
|
||||
|
||||
// On first sync we might have history coming from genesis. We clear the table since it's
|
||||
// faster to rebuild from scratch.
|
||||
if first_sync {
|
||||
provider.tx_ref().clear::<tables::StoragesHistory>()?;
|
||||
if use_rocksdb {
|
||||
// Note: RocksDB clear() executes immediately (not deferred to commit like MDBX),
|
||||
// but this is safe for first_sync because if we crash before commit, the
|
||||
// checkpoint stays at 0 and we'll just clear and rebuild again on restart. The
|
||||
// source data (changesets) is intact.
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
provider.rocksdb_provider().clear::<tables::StoragesHistory>()?;
|
||||
} else {
|
||||
provider.tx_ref().clear::<tables::StoragesHistory>()?;
|
||||
}
|
||||
range = 0..=*input.next_block_range().end();
|
||||
}
|
||||
|
||||
info!(target: "sync::stages::index_storage_history::exec", ?first_sync, "Collecting indices");
|
||||
info!(target: "sync::stages::index_storage_history::exec", ?first_sync, ?use_rocksdb, "Collecting indices");
|
||||
let collector =
|
||||
collect_history_indices::<_, tables::StorageChangeSets, tables::StoragesHistory, _>(
|
||||
provider,
|
||||
@@ -116,16 +133,13 @@ where
|
||||
)?;
|
||||
|
||||
info!(target: "sync::stages::index_storage_history::exec", "Loading indices into database");
|
||||
load_history_indices::<_, tables::StoragesHistory, _>(
|
||||
provider,
|
||||
collector,
|
||||
first_sync,
|
||||
|AddressStorageKey((address, storage_key)), highest_block_number| {
|
||||
StorageShardedKey::new(address, storage_key, highest_block_number)
|
||||
},
|
||||
StorageShardedKey::decode_owned,
|
||||
|key| AddressStorageKey((key.address, key.sharded_key.key)),
|
||||
)?;
|
||||
|
||||
provider.with_rocksdb_batch(|rocksdb_batch| {
|
||||
let mut writer = EitherWriter::new_storages_history(provider, rocksdb_batch)?;
|
||||
load_storage_history(collector, first_sync, &mut writer)
|
||||
.map_err(|e| reth_provider::ProviderError::other(Box::new(e)))?;
|
||||
Ok(((), writer.into_raw_rocksdb_batch()))
|
||||
})?;
|
||||
|
||||
Ok(ExecOutput { checkpoint: StageCheckpoint::new(*range.end()), done: true })
|
||||
}
|
||||
@@ -382,12 +396,12 @@ mod tests {
|
||||
async fn insert_index_second_half_shard() {
|
||||
// init
|
||||
let db = TestStageDB::default();
|
||||
let mut close_full_list = (1..=LAST_BLOCK_IN_FULL_SHARD - 1).collect::<Vec<_>>();
|
||||
let mut almost_full_list = (1..=LAST_BLOCK_IN_FULL_SHARD - 1).collect::<Vec<_>>();
|
||||
|
||||
// setup
|
||||
partial_setup(&db);
|
||||
db.commit(|tx| {
|
||||
tx.put::<tables::StoragesHistory>(shard(u64::MAX), list(&close_full_list)).unwrap();
|
||||
tx.put::<tables::StoragesHistory>(shard(u64::MAX), list(&almost_full_list)).unwrap();
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
@@ -396,12 +410,12 @@ mod tests {
|
||||
run(&db, LAST_BLOCK_IN_FULL_SHARD + 1, Some(LAST_BLOCK_IN_FULL_SHARD - 1));
|
||||
|
||||
// verify
|
||||
close_full_list.push(LAST_BLOCK_IN_FULL_SHARD);
|
||||
almost_full_list.push(LAST_BLOCK_IN_FULL_SHARD);
|
||||
let table = cast(db.table::<tables::StoragesHistory>().unwrap());
|
||||
assert_eq!(
|
||||
table,
|
||||
BTreeMap::from([
|
||||
(shard(LAST_BLOCK_IN_FULL_SHARD), close_full_list.clone()),
|
||||
(shard(LAST_BLOCK_IN_FULL_SHARD), almost_full_list.clone()),
|
||||
(shard(u64::MAX), vec![LAST_BLOCK_IN_FULL_SHARD + 1])
|
||||
])
|
||||
);
|
||||
@@ -410,9 +424,9 @@ mod tests {
|
||||
unwind(&db, LAST_BLOCK_IN_FULL_SHARD, LAST_BLOCK_IN_FULL_SHARD - 1);
|
||||
|
||||
// verify initial state
|
||||
close_full_list.pop();
|
||||
almost_full_list.pop();
|
||||
let table = cast(db.table::<tables::StoragesHistory>().unwrap());
|
||||
assert_eq!(table, BTreeMap::from([(shard(u64::MAX), close_full_list)]));
|
||||
assert_eq!(table, BTreeMap::from([(shard(u64::MAX), almost_full_list)]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -663,4 +677,294 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
mod rocksdb_tests {
|
||||
use super::*;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use reth_storage_api::StorageSettings;
|
||||
|
||||
/// Test that when `storages_history_in_rocksdb` is enabled, the stage
|
||||
/// writes storage history indices to `RocksDB` instead of MDBX.
|
||||
#[tokio::test]
|
||||
async fn execute_writes_to_rocksdb_when_enabled() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let mdbx_table = db.table::<tables::StoragesHistory>().unwrap();
|
||||
assert!(
|
||||
mdbx_table.is_empty(),
|
||||
"MDBX StoragesHistory should be empty when RocksDB is enabled"
|
||||
);
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should contain storage history");
|
||||
|
||||
let block_list = result.unwrap();
|
||||
let blocks: Vec<u64> = block_list.iter().collect();
|
||||
assert_eq!(blocks, (0..=10).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
/// Test that unwind works correctly when `storages_history_in_rocksdb` is enabled.
|
||||
#[tokio::test]
|
||||
async fn unwind_works_when_rocksdb_enabled() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should have data before unwind");
|
||||
let blocks_before: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks_before, (0..=10).collect::<Vec<_>>());
|
||||
|
||||
let unwind_input =
|
||||
UnwindInput { checkpoint: StageCheckpoint::new(10), unwind_to: 5, bad_block: None };
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.unwind(&provider, unwind_input).unwrap();
|
||||
assert_eq!(out, UnwindOutput { checkpoint: StageCheckpoint::new(5) });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should still have data after partial unwind");
|
||||
let blocks_after: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(
|
||||
blocks_after,
|
||||
(0..=5).collect::<Vec<_>>(),
|
||||
"Should only have blocks 0-5 after unwind to block 5"
|
||||
);
|
||||
}
|
||||
|
||||
/// Test that unwind to block 0 keeps only block 0's history.
|
||||
#[tokio::test]
|
||||
async fn unwind_to_zero_keeps_block_zero() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=5 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(5), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(5), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should have data before unwind");
|
||||
|
||||
let unwind_input =
|
||||
UnwindInput { checkpoint: StageCheckpoint::new(5), unwind_to: 0, bad_block: None };
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.unwind(&provider, unwind_input).unwrap();
|
||||
assert_eq!(out, UnwindOutput { checkpoint: StageCheckpoint::new(0) });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should still have block 0 history");
|
||||
let blocks_after: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks_after, vec![0], "Should only have block 0 after unwinding to 0");
|
||||
}
|
||||
|
||||
/// Test incremental sync merges new data with existing shards.
|
||||
#[tokio::test]
|
||||
async fn execute_incremental_sync() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=5 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(5), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(5), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some());
|
||||
let blocks: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks, (0..=5).collect::<Vec<_>>());
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 6..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), checkpoint: Some(StageCheckpoint::new(5)) };
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
|
||||
assert!(result.is_some(), "RocksDB should have merged data");
|
||||
let blocks: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks, (0..=10).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
/// Test multi-shard unwind correctly handles shards that span across unwind boundary.
|
||||
#[tokio::test]
|
||||
async fn unwind_multi_shard() {
|
||||
use reth_db_api::models::sharded_key::NUM_OF_INDICES_IN_SHARD;
|
||||
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let num_blocks = (NUM_OF_INDICES_IN_SHARD * 2 + 100) as u64;
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..num_blocks {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(num_blocks - 1), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
assert_eq!(
|
||||
out,
|
||||
ExecOutput { checkpoint: StageCheckpoint::new(num_blocks - 1), done: true }
|
||||
);
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let shards = rocksdb.storage_history_shards(ADDRESS, STORAGE_KEY).unwrap();
|
||||
assert!(shards.len() >= 2, "Should have at least 2 shards for {} blocks", num_blocks);
|
||||
|
||||
let unwind_to = NUM_OF_INDICES_IN_SHARD as u64 + 50;
|
||||
let unwind_input = UnwindInput {
|
||||
checkpoint: StageCheckpoint::new(num_blocks - 1),
|
||||
unwind_to,
|
||||
bad_block: None,
|
||||
};
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.unwind(&provider, unwind_input).unwrap();
|
||||
assert_eq!(out, UnwindOutput { checkpoint: StageCheckpoint::new(unwind_to) });
|
||||
provider.commit().unwrap();
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let shards_after = rocksdb.storage_history_shards(ADDRESS, STORAGE_KEY).unwrap();
|
||||
assert!(!shards_after.is_empty(), "Should still have shards after unwind");
|
||||
|
||||
let all_blocks: Vec<u64> =
|
||||
shards_after.iter().flat_map(|(_, list)| list.iter()).collect();
|
||||
assert_eq!(
|
||||
all_blocks,
|
||||
(0..=unwind_to).collect::<Vec<_>>(),
|
||||
"Should only have blocks 0 to {} after unwind",
|
||||
unwind_to
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -217,7 +217,7 @@ impl MerkleChangeSets {
|
||||
let compute_cumulative_state_revert = |block_number: BlockNumber| -> HashedPostStateSorted {
|
||||
let mut cumulative_revert = HashedPostStateSorted::default();
|
||||
for n in (block_number..target_end).rev() {
|
||||
cumulative_revert.extend_ref(get_block_state_revert(n))
|
||||
cumulative_revert.extend_ref_and_sort(get_block_state_revert(n))
|
||||
}
|
||||
cumulative_revert
|
||||
};
|
||||
@@ -270,7 +270,7 @@ impl MerkleChangeSets {
|
||||
|
||||
let trie_overlay = Arc::clone(&nodes);
|
||||
let mut nodes_mut = Arc::unwrap_or_clone(nodes);
|
||||
nodes_mut.extend_ref(&this_trie_updates);
|
||||
nodes_mut.extend_ref_and_sort(&this_trie_updates);
|
||||
nodes = Arc::new(nodes_mut);
|
||||
|
||||
// Write the changesets to the DB using the trie updates produced by the block, and the
|
||||
|
||||
@@ -1,16 +1,20 @@
|
||||
//! Utils for `stages`.
|
||||
use alloy_primitives::{Address, BlockNumber, TxNumber};
|
||||
use alloy_primitives::{Address, BlockNumber, TxNumber, B256};
|
||||
use reth_config::config::EtlConfig;
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbCursorRW},
|
||||
models::{sharded_key::NUM_OF_INDICES_IN_SHARD, AccountBeforeTx, ShardedKey},
|
||||
table::{Decompress, Table},
|
||||
transaction::{DbTx, DbTxMut},
|
||||
BlockNumberList, DatabaseError,
|
||||
models::{
|
||||
sharded_key::NUM_OF_INDICES_IN_SHARD, storage_sharded_key::StorageShardedKey,
|
||||
AccountBeforeTx, ShardedKey,
|
||||
},
|
||||
table::{Decode, Decompress, Table},
|
||||
transaction::DbTx,
|
||||
BlockNumberList,
|
||||
};
|
||||
use reth_etl::Collector;
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_provider::{
|
||||
providers::StaticFileProvider, to_range, BlockReader, DBProvider, ProviderError,
|
||||
providers::StaticFileProvider, to_range, BlockReader, DBProvider, EitherWriter, ProviderError,
|
||||
StaticFileProviderFactory,
|
||||
};
|
||||
use reth_stages_api::StageError;
|
||||
@@ -57,12 +61,12 @@ where
|
||||
let mut collector = Collector::new(etl_config.file_size, etl_config.dir.clone());
|
||||
let mut cache: HashMap<P, Vec<u64>> = HashMap::default();
|
||||
|
||||
let mut collect = |cache: &HashMap<P, Vec<u64>>| {
|
||||
for (key, indices) in cache {
|
||||
let last = indices.last().expect("qed");
|
||||
let mut collect = |cache: &mut HashMap<P, Vec<u64>>| {
|
||||
for (key, indices) in cache.drain() {
|
||||
let last = *indices.last().expect("qed");
|
||||
collector.insert(
|
||||
sharded_key_factory(*key, *last),
|
||||
BlockNumberList::new_pre_sorted(indices.iter().copied()),
|
||||
sharded_key_factory(key, last),
|
||||
BlockNumberList::new_pre_sorted(indices.into_iter()),
|
||||
)?;
|
||||
}
|
||||
Ok::<(), StageError>(())
|
||||
@@ -87,13 +91,12 @@ where
|
||||
current_block_number = block_number;
|
||||
flush_counter += 1;
|
||||
if flush_counter > DEFAULT_CACHE_THRESHOLD {
|
||||
collect(&cache)?;
|
||||
cache.clear();
|
||||
collect(&mut cache)?;
|
||||
flush_counter = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
collect(&cache)?;
|
||||
collect(&mut cache)?;
|
||||
|
||||
Ok(collector)
|
||||
}
|
||||
@@ -109,7 +112,7 @@ where
|
||||
for (address, indices) in cache {
|
||||
insert_fn(address, indices)?
|
||||
}
|
||||
Ok::<(), StageError>(())
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Collects account history indices using a provider that implements `ChangeSetReader`.
|
||||
@@ -125,12 +128,12 @@ where
|
||||
let mut cache: HashMap<Address, Vec<u64>> = HashMap::default();
|
||||
|
||||
let mut insert_fn = |address: Address, indices: Vec<u64>| {
|
||||
let last = indices.last().expect("qed");
|
||||
let last = indices.last().expect("indices is non-empty");
|
||||
collector.insert(
|
||||
ShardedKey::new(address, *last),
|
||||
BlockNumberList::new_pre_sorted(indices.into_iter()),
|
||||
)?;
|
||||
Ok::<(), StageError>(())
|
||||
Ok(())
|
||||
};
|
||||
|
||||
// Convert range bounds to concrete range
|
||||
@@ -171,154 +174,176 @@ where
|
||||
Ok(collector)
|
||||
}
|
||||
|
||||
/// Given a [`Collector`] created by [`collect_history_indices`] it iterates all entries, loading
|
||||
/// the indices into the database in shards.
|
||||
/// Loads account history indices into the database via `EitherWriter`.
|
||||
///
|
||||
/// ## Process
|
||||
/// Iterates over elements, grouping indices by their partial keys (e.g., `Address` or
|
||||
/// `Address.StorageKey`). It flushes indices to disk when reaching a shard's max length
|
||||
/// (`NUM_OF_INDICES_IN_SHARD`) or when the partial key changes, ensuring the last previous partial
|
||||
/// key shard is stored.
|
||||
pub(crate) fn load_history_indices<Provider, H, P>(
|
||||
provider: &Provider,
|
||||
mut collector: Collector<H::Key, H::Value>,
|
||||
/// Works with [`EitherWriter`] to support both MDBX and `RocksDB` backends.
|
||||
///
|
||||
/// ## Process
|
||||
/// Iterates over elements, grouping indices by their address. It flushes indices to disk
|
||||
/// when reaching a shard's max length (`NUM_OF_INDICES_IN_SHARD`) or when the address changes,
|
||||
/// ensuring the last previous address shard is stored.
|
||||
///
|
||||
/// Uses `Option<Address>` instead of `Address::default()` as the sentinel to avoid
|
||||
/// incorrectly treating `Address::ZERO` as "no previous address".
|
||||
pub(crate) fn load_account_history<N, CURSOR>(
|
||||
mut collector: Collector<ShardedKey<Address>, BlockNumberList>,
|
||||
append_only: bool,
|
||||
sharded_key_factory: impl Clone + Fn(P, u64) -> <H as Table>::Key,
|
||||
decode_key: impl Fn(Vec<u8>) -> Result<<H as Table>::Key, DatabaseError>,
|
||||
get_partial: impl Fn(<H as Table>::Key) -> P,
|
||||
writer: &mut EitherWriter<'_, CURSOR, N>,
|
||||
) -> Result<(), StageError>
|
||||
where
|
||||
Provider: DBProvider<Tx: DbTxMut>,
|
||||
H: Table<Value = BlockNumberList>,
|
||||
P: Copy + Default + Eq,
|
||||
N: NodePrimitives,
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::AccountsHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::AccountsHistory>,
|
||||
{
|
||||
let mut write_cursor = provider.tx_ref().cursor_write::<H>()?;
|
||||
let mut current_partial = P::default();
|
||||
let mut current_address: Option<Address> = None;
|
||||
// Accumulator for block numbers where the current address changed.
|
||||
let mut current_list = Vec::<u64>::new();
|
||||
|
||||
// observability
|
||||
let total_entries = collector.len();
|
||||
let interval = (total_entries / 10).max(1);
|
||||
|
||||
for (index, element) in collector.iter()?.enumerate() {
|
||||
let (k, v) = element?;
|
||||
let sharded_key = decode_key(k)?;
|
||||
let sharded_key = ShardedKey::<Address>::decode_owned(k)?;
|
||||
let new_list = BlockNumberList::decompress_owned(v)?;
|
||||
|
||||
if index > 0 && index.is_multiple_of(interval) && total_entries > 10 {
|
||||
info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices");
|
||||
}
|
||||
|
||||
// AccountsHistory: `Address`.
|
||||
// StorageHistory: `Address.StorageKey`.
|
||||
let partial_key = get_partial(sharded_key);
|
||||
let address = sharded_key.key;
|
||||
|
||||
if current_partial != partial_key {
|
||||
// We have reached the end of this subset of keys so
|
||||
// we need to flush its last indice shard.
|
||||
load_indices(
|
||||
&mut write_cursor,
|
||||
current_partial,
|
||||
&mut current_list,
|
||||
&sharded_key_factory,
|
||||
append_only,
|
||||
LoadMode::Flush,
|
||||
)?;
|
||||
// When address changes, flush the previous address's shards and start fresh.
|
||||
if current_address != Some(address) {
|
||||
// Flush all remaining shards for the previous address (uses u64::MAX for last shard).
|
||||
if let Some(prev_addr) = current_address {
|
||||
flush_account_history_shards(prev_addr, &mut current_list, append_only, writer)?;
|
||||
}
|
||||
|
||||
current_partial = partial_key;
|
||||
current_address = Some(address);
|
||||
current_list.clear();
|
||||
|
||||
// If it's not the first sync, there might an existing shard already, so we need to
|
||||
// merge it with the one coming from the collector
|
||||
// On incremental sync, merge with the existing last shard from the database.
|
||||
// The last shard is stored with key (address, u64::MAX) so we can find it.
|
||||
if !append_only &&
|
||||
let Some((_, last_database_shard)) =
|
||||
write_cursor.seek_exact(sharded_key_factory(current_partial, u64::MAX))?
|
||||
let Some(last_shard) = writer.get_last_account_history_shard(address)?
|
||||
{
|
||||
current_list.extend(last_database_shard.iter());
|
||||
current_list.extend(last_shard.iter());
|
||||
}
|
||||
}
|
||||
|
||||
// Append new block numbers to the accumulator.
|
||||
current_list.extend(new_list.iter());
|
||||
load_indices(
|
||||
&mut write_cursor,
|
||||
current_partial,
|
||||
&mut current_list,
|
||||
&sharded_key_factory,
|
||||
append_only,
|
||||
LoadMode::KeepLast,
|
||||
)?;
|
||||
|
||||
// Flush complete shards, keeping the last (partial) shard buffered.
|
||||
flush_account_history_shards_partial(address, &mut current_list, append_only, writer)?;
|
||||
}
|
||||
|
||||
// There will be one remaining shard that needs to be flushed to DB.
|
||||
load_indices(
|
||||
&mut write_cursor,
|
||||
current_partial,
|
||||
&mut current_list,
|
||||
&sharded_key_factory,
|
||||
append_only,
|
||||
LoadMode::Flush,
|
||||
)?;
|
||||
// Flush the final address's remaining shard.
|
||||
if let Some(addr) = current_address {
|
||||
flush_account_history_shards(addr, &mut current_list, append_only, writer)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Shard and insert the indices list according to [`LoadMode`] and its length.
|
||||
pub(crate) fn load_indices<H, C, P>(
|
||||
cursor: &mut C,
|
||||
partial_key: P,
|
||||
list: &mut Vec<BlockNumber>,
|
||||
sharded_key_factory: &impl Fn(P, BlockNumber) -> <H as Table>::Key,
|
||||
/// Flushes complete shards for account history, keeping the trailing partial shard buffered.
|
||||
///
|
||||
/// Only flushes when we have more than one shard's worth of data, keeping the last
|
||||
/// (possibly partial) shard for continued accumulation. This avoids writing a shard
|
||||
/// that may need to be updated when more indices arrive.
|
||||
fn flush_account_history_shards_partial<N, CURSOR>(
|
||||
address: Address,
|
||||
list: &mut Vec<u64>,
|
||||
append_only: bool,
|
||||
mode: LoadMode,
|
||||
writer: &mut EitherWriter<'_, CURSOR, N>,
|
||||
) -> Result<(), StageError>
|
||||
where
|
||||
C: DbCursorRO<H> + DbCursorRW<H>,
|
||||
H: Table<Value = BlockNumberList>,
|
||||
P: Copy,
|
||||
N: NodePrimitives,
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::AccountsHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::AccountsHistory>,
|
||||
{
|
||||
if list.len() > NUM_OF_INDICES_IN_SHARD || mode.is_flush() {
|
||||
let chunks = list
|
||||
.chunks(NUM_OF_INDICES_IN_SHARD)
|
||||
.map(|chunks| chunks.to_vec())
|
||||
.collect::<Vec<Vec<u64>>>();
|
||||
// Nothing to flush if we haven't filled a complete shard yet.
|
||||
if list.len() <= NUM_OF_INDICES_IN_SHARD {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut iter = chunks.into_iter().peekable();
|
||||
while let Some(chunk) = iter.next() {
|
||||
let mut highest = *chunk.last().expect("at least one index");
|
||||
let num_full_shards = list.len() / NUM_OF_INDICES_IN_SHARD;
|
||||
|
||||
if !mode.is_flush() && iter.peek().is_none() {
|
||||
*list = chunk;
|
||||
} else {
|
||||
if iter.peek().is_none() {
|
||||
highest = u64::MAX;
|
||||
}
|
||||
let key = sharded_key_factory(partial_key, highest);
|
||||
let value = BlockNumberList::new_pre_sorted(chunk);
|
||||
// Always keep at least one shard buffered for continued accumulation.
|
||||
// If len is exact multiple of shard size, keep the last full shard.
|
||||
let shards_to_flush = if list.len().is_multiple_of(NUM_OF_INDICES_IN_SHARD) {
|
||||
num_full_shards - 1
|
||||
} else {
|
||||
num_full_shards
|
||||
};
|
||||
|
||||
if append_only {
|
||||
cursor.append(key, &value)?;
|
||||
} else {
|
||||
cursor.upsert(key, &value)?;
|
||||
}
|
||||
}
|
||||
if shards_to_flush == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Split: flush the first N shards, keep the remainder buffered.
|
||||
let flush_len = shards_to_flush * NUM_OF_INDICES_IN_SHARD;
|
||||
let remainder = list.split_off(flush_len);
|
||||
|
||||
// Write each complete shard with its highest block number as the key.
|
||||
for chunk in list.chunks(NUM_OF_INDICES_IN_SHARD) {
|
||||
let highest = *chunk.last().expect("chunk is non-empty");
|
||||
let key = ShardedKey::new(address, highest);
|
||||
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
|
||||
|
||||
if append_only {
|
||||
writer.append_account_history(key, &value)?;
|
||||
} else {
|
||||
writer.upsert_account_history(key, &value)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the remaining indices for the next iteration.
|
||||
*list = remainder;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Mode on how to load index shards into the database.
|
||||
pub(crate) enum LoadMode {
|
||||
/// Keep the last shard in memory and don't flush it to the database.
|
||||
KeepLast,
|
||||
/// Flush all shards into the database.
|
||||
Flush,
|
||||
}
|
||||
|
||||
impl LoadMode {
|
||||
const fn is_flush(&self) -> bool {
|
||||
matches!(self, Self::Flush)
|
||||
/// Flushes all remaining shards for account history, using `u64::MAX` for the last shard.
|
||||
///
|
||||
/// The `u64::MAX` key for the final shard is an invariant that allows `seek_exact(address,
|
||||
/// u64::MAX)` to find the last shard during incremental sync for merging with new indices.
|
||||
fn flush_account_history_shards<N, CURSOR>(
|
||||
address: Address,
|
||||
list: &mut Vec<u64>,
|
||||
append_only: bool,
|
||||
writer: &mut EitherWriter<'_, CURSOR, N>,
|
||||
) -> Result<(), StageError>
|
||||
where
|
||||
N: NodePrimitives,
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::AccountsHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::AccountsHistory>,
|
||||
{
|
||||
if list.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let num_chunks = list.len().div_ceil(NUM_OF_INDICES_IN_SHARD);
|
||||
|
||||
for (i, chunk) in list.chunks(NUM_OF_INDICES_IN_SHARD).enumerate() {
|
||||
let is_last = i == num_chunks - 1;
|
||||
|
||||
// Use u64::MAX for the final shard's key. This invariant allows incremental sync
|
||||
// to find the last shard via seek_exact(address, u64::MAX) for merging.
|
||||
let highest = if is_last { u64::MAX } else { *chunk.last().expect("chunk is non-empty") };
|
||||
|
||||
let key = ShardedKey::new(address, highest);
|
||||
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
|
||||
|
||||
if append_only {
|
||||
writer.append_account_history(key, &value)?;
|
||||
} else {
|
||||
writer.upsert_account_history(key, &value)?;
|
||||
}
|
||||
}
|
||||
|
||||
list.clear();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Called when database is ahead of static files. Attempts to find the first block we are missing
|
||||
@@ -356,3 +381,191 @@ where
|
||||
segment,
|
||||
})
|
||||
}
|
||||
|
||||
/// Loads storage history indices into the database via `EitherWriter`.
|
||||
///
|
||||
/// Works with [`EitherWriter`] to support both MDBX and `RocksDB` backends.
|
||||
///
|
||||
/// ## Process
|
||||
/// Iterates over elements, grouping indices by their (address, `storage_key`) pairs. It flushes
|
||||
/// indices to disk when reaching a shard's max length (`NUM_OF_INDICES_IN_SHARD`) or when the
|
||||
/// (address, `storage_key`) pair changes, ensuring the last previous shard is stored.
|
||||
///
|
||||
/// Uses `Option<(Address, B256)>` instead of default values as the sentinel to avoid
|
||||
/// incorrectly treating `(Address::ZERO, B256::ZERO)` as "no previous key".
|
||||
pub(crate) fn load_storage_history<N, CURSOR>(
|
||||
mut collector: Collector<StorageShardedKey, BlockNumberList>,
|
||||
append_only: bool,
|
||||
writer: &mut EitherWriter<'_, CURSOR, N>,
|
||||
) -> Result<(), StageError>
|
||||
where
|
||||
N: NodePrimitives,
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::StoragesHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::StoragesHistory>,
|
||||
{
|
||||
let mut current_key: Option<(Address, B256)> = None;
|
||||
// Accumulator for block numbers where the current (address, storage_key) changed.
|
||||
let mut current_list = Vec::<u64>::new();
|
||||
|
||||
let total_entries = collector.len();
|
||||
let interval = (total_entries / 10).max(1);
|
||||
|
||||
for (index, element) in collector.iter()?.enumerate() {
|
||||
let (k, v) = element?;
|
||||
let sharded_key = StorageShardedKey::decode_owned(k)?;
|
||||
let new_list = BlockNumberList::decompress_owned(v)?;
|
||||
|
||||
if index > 0 && index.is_multiple_of(interval) && total_entries > 10 {
|
||||
info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices");
|
||||
}
|
||||
|
||||
let partial_key = (sharded_key.address, sharded_key.sharded_key.key);
|
||||
|
||||
// When (address, storage_key) changes, flush the previous key's shards and start fresh.
|
||||
if current_key != Some(partial_key) {
|
||||
// Flush all remaining shards for the previous key (uses u64::MAX for last shard).
|
||||
if let Some((prev_addr, prev_storage_key)) = current_key {
|
||||
flush_storage_history_shards(
|
||||
prev_addr,
|
||||
prev_storage_key,
|
||||
&mut current_list,
|
||||
append_only,
|
||||
writer,
|
||||
)?;
|
||||
}
|
||||
|
||||
current_key = Some(partial_key);
|
||||
current_list.clear();
|
||||
|
||||
// On incremental sync, merge with the existing last shard from the database.
|
||||
// The last shard is stored with key (address, storage_key, u64::MAX) so we can find it.
|
||||
if !append_only &&
|
||||
let Some(last_shard) =
|
||||
writer.get_last_storage_history_shard(partial_key.0, partial_key.1)?
|
||||
{
|
||||
current_list.extend(last_shard.iter());
|
||||
}
|
||||
}
|
||||
|
||||
// Append new block numbers to the accumulator.
|
||||
current_list.extend(new_list.iter());
|
||||
|
||||
// Flush complete shards, keeping the last (partial) shard buffered.
|
||||
flush_storage_history_shards_partial(
|
||||
partial_key.0,
|
||||
partial_key.1,
|
||||
&mut current_list,
|
||||
append_only,
|
||||
writer,
|
||||
)?;
|
||||
}
|
||||
|
||||
// Flush the final key's remaining shard.
|
||||
if let Some((addr, storage_key)) = current_key {
|
||||
flush_storage_history_shards(addr, storage_key, &mut current_list, append_only, writer)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flushes complete shards for storage history, keeping the trailing partial shard buffered.
|
||||
///
|
||||
/// Only flushes when we have more than one shard's worth of data, keeping the last
|
||||
/// (possibly partial) shard for continued accumulation. This avoids writing a shard
|
||||
/// that may need to be updated when more indices arrive.
|
||||
fn flush_storage_history_shards_partial<N, CURSOR>(
|
||||
address: Address,
|
||||
storage_key: B256,
|
||||
list: &mut Vec<u64>,
|
||||
append_only: bool,
|
||||
writer: &mut EitherWriter<'_, CURSOR, N>,
|
||||
) -> Result<(), StageError>
|
||||
where
|
||||
N: NodePrimitives,
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::StoragesHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::StoragesHistory>,
|
||||
{
|
||||
// Nothing to flush if we haven't filled a complete shard yet.
|
||||
if list.len() <= NUM_OF_INDICES_IN_SHARD {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let num_full_shards = list.len() / NUM_OF_INDICES_IN_SHARD;
|
||||
|
||||
// Always keep at least one shard buffered for continued accumulation.
|
||||
// If len is exact multiple of shard size, keep the last full shard.
|
||||
let shards_to_flush = if list.len().is_multiple_of(NUM_OF_INDICES_IN_SHARD) {
|
||||
num_full_shards - 1
|
||||
} else {
|
||||
num_full_shards
|
||||
};
|
||||
|
||||
if shards_to_flush == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Split: flush the first N shards, keep the remainder buffered.
|
||||
let flush_len = shards_to_flush * NUM_OF_INDICES_IN_SHARD;
|
||||
let remainder = list.split_off(flush_len);
|
||||
|
||||
// Write each complete shard with its highest block number as the key.
|
||||
for chunk in list.chunks(NUM_OF_INDICES_IN_SHARD) {
|
||||
let highest = *chunk.last().expect("chunk is non-empty");
|
||||
let key = StorageShardedKey::new(address, storage_key, highest);
|
||||
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
|
||||
|
||||
if append_only {
|
||||
writer.append_storage_history(key, &value)?;
|
||||
} else {
|
||||
writer.upsert_storage_history(key, &value)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the remaining indices for the next iteration.
|
||||
*list = remainder;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flushes all remaining shards for storage history, using `u64::MAX` for the last shard.
|
||||
///
|
||||
/// The `u64::MAX` key for the final shard is an invariant that allows
|
||||
/// `seek_exact(address, storage_key, u64::MAX)` to find the last shard during incremental
|
||||
/// sync for merging with new indices.
|
||||
fn flush_storage_history_shards<N, CURSOR>(
|
||||
address: Address,
|
||||
storage_key: B256,
|
||||
list: &mut Vec<u64>,
|
||||
append_only: bool,
|
||||
writer: &mut EitherWriter<'_, CURSOR, N>,
|
||||
) -> Result<(), StageError>
|
||||
where
|
||||
N: NodePrimitives,
|
||||
CURSOR: DbCursorRW<reth_db_api::tables::StoragesHistory>
|
||||
+ DbCursorRO<reth_db_api::tables::StoragesHistory>,
|
||||
{
|
||||
if list.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let num_chunks = list.len().div_ceil(NUM_OF_INDICES_IN_SHARD);
|
||||
|
||||
for (i, chunk) in list.chunks(NUM_OF_INDICES_IN_SHARD).enumerate() {
|
||||
let is_last = i == num_chunks - 1;
|
||||
|
||||
// Use u64::MAX for the final shard's key. This invariant allows incremental sync
|
||||
// to find the last shard via seek_exact(address, storage_key, u64::MAX) for merging.
|
||||
let highest = if is_last { u64::MAX } else { *chunk.last().expect("chunk is non-empty") };
|
||||
|
||||
let key = StorageShardedKey::new(address, storage_key, highest);
|
||||
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
|
||||
|
||||
if append_only {
|
||||
writer.append_storage_history(key, &value)?;
|
||||
} else {
|
||||
writer.upsert_storage_history(key, &value)?;
|
||||
}
|
||||
}
|
||||
|
||||
list.clear();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -69,7 +69,7 @@ pub fn generate_from_to(
|
||||
}
|
||||
}
|
||||
|
||||
/// Generates code to implement the `Compact` trait method `to_compact`.
|
||||
/// Generates code to implement the `Compact` trait method `from_compact`.
|
||||
fn generate_from_compact(
|
||||
fields: &FieldList,
|
||||
ident: &Ident,
|
||||
@@ -155,7 +155,7 @@ fn generate_from_compact(
|
||||
}
|
||||
}
|
||||
|
||||
/// Generates code to implement the `Compact` trait method `from_compact`.
|
||||
/// Generates code to implement the `Compact` trait method `to_compact`.
|
||||
fn generate_to_compact(
|
||||
fields: &FieldList,
|
||||
ident: &Ident,
|
||||
|
||||
@@ -175,7 +175,7 @@ fn should_use_alt_impl(ftype: &str, segment: &syn::PathSegment) -> bool {
|
||||
let syn::PathArguments::AngleBracketed(ref args) = segment.arguments &&
|
||||
let Some(syn::GenericArgument::Type(syn::Type::Path(arg_path))) = args.args.last() &&
|
||||
let (Some(path), 1) = (arg_path.path.segments.first(), arg_path.path.segments.len()) &&
|
||||
["B256", "Address", "Address", "Bloom", "TxHash", "BlockHash", "CompactPlaceholder"]
|
||||
["B256", "Address", "Bloom", "TxHash", "BlockHash", "CompactPlaceholder"]
|
||||
.iter()
|
||||
.any(|&s| path.ident == s)
|
||||
{
|
||||
|
||||
@@ -40,6 +40,7 @@ serde = { workspace = true, default-features = false }
|
||||
metrics.workspace = true
|
||||
|
||||
# misc
|
||||
arrayvec.workspace = true
|
||||
derive_more.workspace = true
|
||||
bytes.workspace = true
|
||||
|
||||
|
||||
@@ -34,6 +34,21 @@ pub struct StorageSettings {
|
||||
}
|
||||
|
||||
impl StorageSettings {
|
||||
/// Returns the default base `StorageSettings` for this build.
|
||||
///
|
||||
/// When the `edge` feature is enabled, returns [`Self::edge()`].
|
||||
/// Otherwise, returns [`Self::legacy()`].
|
||||
pub const fn base() -> Self {
|
||||
#[cfg(feature = "edge")]
|
||||
{
|
||||
Self::edge()
|
||||
}
|
||||
#[cfg(not(feature = "edge"))]
|
||||
{
|
||||
Self::legacy()
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates `StorageSettings` for edge nodes with all storage features enabled:
|
||||
/// - Receipts and transaction senders in static files
|
||||
/// - History indices in `RocksDB` (storages, accounts, transaction hashes)
|
||||
@@ -45,7 +60,7 @@ impl StorageSettings {
|
||||
transaction_senders_in_static_files: true,
|
||||
account_changesets_in_static_files: true,
|
||||
storages_history_in_rocksdb: false,
|
||||
transaction_hash_numbers_in_rocksdb: false,
|
||||
transaction_hash_numbers_in_rocksdb: true,
|
||||
account_history_in_rocksdb: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,13 +126,10 @@ impl Decode for String {
|
||||
}
|
||||
|
||||
impl Encode for StoredNibbles {
|
||||
type Encoded = Vec<u8>;
|
||||
type Encoded = arrayvec::ArrayVec<u8, 64>;
|
||||
|
||||
// Delegate to the Compact implementation
|
||||
fn encode(self) -> Self::Encoded {
|
||||
// NOTE: This used to be `to_compact`, but all it does is append the bytes to the buffer,
|
||||
// so we can just use the implementation of `Into<Vec<u8>>` to reuse the buffer.
|
||||
self.0.to_vec()
|
||||
self.0.iter().collect()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,13 +3,16 @@ use crate::{
|
||||
table::{Decode, Encode},
|
||||
DatabaseError,
|
||||
};
|
||||
use alloy_primitives::BlockNumber;
|
||||
use alloy_primitives::{Address, BlockNumber};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::hash::Hash;
|
||||
|
||||
/// Number of indices in one shard.
|
||||
pub const NUM_OF_INDICES_IN_SHARD: usize = 2_000;
|
||||
|
||||
/// Size of `BlockNumber` in bytes (u64 = 8 bytes).
|
||||
const BLOCK_NUMBER_SIZE: usize = std::mem::size_of::<BlockNumber>();
|
||||
|
||||
/// Sometimes data can be too big to be saved for a single key. This helps out by dividing the data
|
||||
/// into different shards. Example:
|
||||
///
|
||||
@@ -43,21 +46,68 @@ impl<T> ShardedKey<T> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Encode> Encode for ShardedKey<T> {
|
||||
type Encoded = Vec<u8>;
|
||||
/// Stack-allocated encoded key for `ShardedKey<Address>`.
|
||||
///
|
||||
/// This avoids heap allocation in hot database paths. The key layout is:
|
||||
/// - 20 bytes: `Address`
|
||||
/// - 8 bytes: `BlockNumber` (big-endian)
|
||||
pub type ShardedKeyAddressEncoded = [u8; 20 + BLOCK_NUMBER_SIZE];
|
||||
|
||||
impl Encode for ShardedKey<Address> {
|
||||
type Encoded = ShardedKeyAddressEncoded;
|
||||
|
||||
#[inline]
|
||||
fn encode(self) -> Self::Encoded {
|
||||
let mut buf: Vec<u8> = Encode::encode(self.key).into();
|
||||
buf.extend_from_slice(&self.highest_block_number.to_be_bytes());
|
||||
let mut buf = [0u8; 20 + BLOCK_NUMBER_SIZE];
|
||||
buf[..20].copy_from_slice(self.key.as_slice());
|
||||
buf[20..].copy_from_slice(&self.highest_block_number.to_be_bytes());
|
||||
buf
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Decode> Decode for ShardedKey<T> {
|
||||
impl Decode for ShardedKey<Address> {
|
||||
fn decode(value: &[u8]) -> Result<Self, DatabaseError> {
|
||||
let (key, highest_tx_number) = value.split_last_chunk().ok_or(DatabaseError::Decode)?;
|
||||
let key = T::decode(key)?;
|
||||
let highest_tx_number = u64::from_be_bytes(*highest_tx_number);
|
||||
Ok(Self::new(key, highest_tx_number))
|
||||
if value.len() != 20 + BLOCK_NUMBER_SIZE {
|
||||
return Err(DatabaseError::Decode);
|
||||
}
|
||||
let key = Address::from_slice(&value[..20]);
|
||||
let highest_block_number =
|
||||
u64::from_be_bytes(value[20..].try_into().map_err(|_| DatabaseError::Decode)?);
|
||||
Ok(Self::new(key, highest_block_number))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_primitives::address;
|
||||
|
||||
#[test]
|
||||
fn sharded_key_address_encode_decode_roundtrip() {
|
||||
let addr = address!("0102030405060708091011121314151617181920");
|
||||
let block_num = 0x123456789ABCDEF0u64;
|
||||
let key = ShardedKey::new(addr, block_num);
|
||||
|
||||
let encoded = key.encode();
|
||||
|
||||
// Verify it's stack-allocated (28 bytes)
|
||||
assert_eq!(encoded.len(), 28);
|
||||
assert_eq!(std::mem::size_of_val(&encoded), 28);
|
||||
|
||||
// Verify roundtrip (check against expected values since key was consumed)
|
||||
let decoded = ShardedKey::<Address>::decode(&encoded).unwrap();
|
||||
assert_eq!(decoded.key, address!("0102030405060708091011121314151617181920"));
|
||||
assert_eq!(decoded.highest_block_number, 0x123456789ABCDEF0u64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sharded_key_last_works() {
|
||||
let addr = address!("0102030405060708091011121314151617181920");
|
||||
let key = ShardedKey::<Address>::last(addr);
|
||||
assert_eq!(key.highest_block_number, u64::MAX);
|
||||
|
||||
let encoded = key.encode();
|
||||
let decoded = ShardedKey::<Address>::decode(&encoded).unwrap();
|
||||
assert_eq!(decoded.highest_block_number, u64::MAX);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,14 @@ pub const NUM_OF_INDICES_IN_SHARD: usize = 2_000;
|
||||
/// The fields are: 20-byte address, 32-byte key, and 8-byte block number
|
||||
const STORAGE_SHARD_KEY_BYTES_SIZE: usize = 20 + 32 + 8;
|
||||
|
||||
/// Stack-allocated encoded key for `StorageShardedKey`.
|
||||
///
|
||||
/// This avoids heap allocation in hot database paths. The key layout is:
|
||||
/// - 20 bytes: `Address`
|
||||
/// - 32 bytes: `B256` storage key
|
||||
/// - 8 bytes: `BlockNumber` (big-endian)
|
||||
pub type StorageShardedKeyEncoded = [u8; STORAGE_SHARD_KEY_BYTES_SIZE];
|
||||
|
||||
/// Sometimes data can be too big to be saved for a single key. This helps out by dividing the data
|
||||
/// into different shards. Example:
|
||||
///
|
||||
@@ -54,13 +62,14 @@ impl StorageShardedKey {
|
||||
}
|
||||
|
||||
impl Encode for StorageShardedKey {
|
||||
type Encoded = Vec<u8>;
|
||||
type Encoded = StorageShardedKeyEncoded;
|
||||
|
||||
#[inline]
|
||||
fn encode(self) -> Self::Encoded {
|
||||
let mut buf: Vec<u8> = Vec::with_capacity(STORAGE_SHARD_KEY_BYTES_SIZE);
|
||||
buf.extend_from_slice(&Encode::encode(self.address));
|
||||
buf.extend_from_slice(&Encode::encode(self.sharded_key.key));
|
||||
buf.extend_from_slice(&self.sharded_key.highest_block_number.to_be_bytes());
|
||||
let mut buf = [0u8; STORAGE_SHARD_KEY_BYTES_SIZE];
|
||||
buf[..20].copy_from_slice(self.address.as_slice());
|
||||
buf[20..52].copy_from_slice(self.sharded_key.key.as_slice());
|
||||
buf[52..].copy_from_slice(&self.sharded_key.highest_block_number.to_be_bytes());
|
||||
buf
|
||||
}
|
||||
}
|
||||
@@ -81,3 +90,44 @@ impl Decode for StorageShardedKey {
|
||||
Ok(Self { address, sharded_key: ShardedKey::new(storage_key, highest_block_number) })
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_primitives::{address, b256};
|
||||
|
||||
#[test]
|
||||
fn storage_sharded_key_encode_decode_roundtrip() {
|
||||
let addr = address!("0102030405060708091011121314151617181920");
|
||||
let storage_key = b256!("0001020304050607080910111213141516171819202122232425262728293031");
|
||||
let block_num = 0x123456789ABCDEFu64;
|
||||
let key = StorageShardedKey::new(addr, storage_key, block_num);
|
||||
|
||||
let encoded = key.encode();
|
||||
|
||||
// Verify it's stack-allocated (60 bytes)
|
||||
assert_eq!(encoded.len(), 60);
|
||||
assert_eq!(std::mem::size_of_val(&encoded), 60);
|
||||
|
||||
// Verify roundtrip (check against expected values since key was consumed)
|
||||
let decoded = StorageShardedKey::decode(&encoded).unwrap();
|
||||
assert_eq!(decoded.address, address!("0102030405060708091011121314151617181920"));
|
||||
assert_eq!(
|
||||
decoded.sharded_key.key,
|
||||
b256!("0001020304050607080910111213141516171819202122232425262728293031")
|
||||
);
|
||||
assert_eq!(decoded.sharded_key.highest_block_number, 0x123456789ABCDEFu64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn storage_sharded_key_last_works() {
|
||||
let addr = address!("0102030405060708091011121314151617181920");
|
||||
let storage_key = b256!("0001020304050607080910111213141516171819202122232425262728293031");
|
||||
let key = StorageShardedKey::last(addr, storage_key);
|
||||
assert_eq!(key.sharded_key.highest_block_number, u64::MAX);
|
||||
|
||||
let encoded = key.encode();
|
||||
let decoded = StorageShardedKey::decode(&encoded).unwrap();
|
||||
assert_eq!(decoded.sharded_key.highest_block_number, u64::MAX);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,10 +46,40 @@ pub trait Decompress: Send + Sync + Sized + Debug {
|
||||
}
|
||||
}
|
||||
|
||||
/// Trait for converting encoded types to `Vec<u8>`.
|
||||
///
|
||||
/// This is implemented for all `AsRef<[u8]>` types. For `Vec<u8>` this is a no-op,
|
||||
/// for other types like `ArrayVec` or fixed arrays it performs a copy.
|
||||
pub trait IntoVec: AsRef<[u8]> {
|
||||
/// Convert to a `Vec<u8>`.
|
||||
fn into_vec(self) -> Vec<u8>;
|
||||
}
|
||||
|
||||
impl IntoVec for Vec<u8> {
|
||||
#[inline]
|
||||
fn into_vec(self) -> Vec<u8> {
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl<const N: usize> IntoVec for [u8; N] {
|
||||
#[inline]
|
||||
fn into_vec(self) -> Vec<u8> {
|
||||
self.to_vec()
|
||||
}
|
||||
}
|
||||
|
||||
impl<const N: usize> IntoVec for arrayvec::ArrayVec<u8, N> {
|
||||
#[inline]
|
||||
fn into_vec(self) -> Vec<u8> {
|
||||
self.to_vec()
|
||||
}
|
||||
}
|
||||
|
||||
/// Trait that will transform the data to be saved in the DB.
|
||||
pub trait Encode: Send + Sync + Sized + Debug {
|
||||
/// Encoded type.
|
||||
type Encoded: AsRef<[u8]> + Into<Vec<u8>> + Send + Sync + Ord + Debug;
|
||||
type Encoded: AsRef<[u8]> + IntoVec + Send + Sync + Ord + Debug;
|
||||
|
||||
/// Encodes data going into the database.
|
||||
fn encode(self) -> Self::Encoded;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use crate::{
|
||||
table::{Compress, Decode, Decompress, DupSort, Encode, Key, Table, Value},
|
||||
table::{Compress, Decode, Decompress, DupSort, Encode, IntoVec, Key, Table, Value},
|
||||
DatabaseError,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -52,7 +52,7 @@ pub struct RawKey<K: Key> {
|
||||
impl<K: Key> RawKey<K> {
|
||||
/// Create new raw key.
|
||||
pub fn new(key: K) -> Self {
|
||||
Self { key: K::encode(key).into(), _phantom: std::marker::PhantomData }
|
||||
Self { key: K::encode(key).into_vec(), _phantom: std::marker::PhantomData }
|
||||
}
|
||||
|
||||
/// Creates a raw key from an existing `Vec`. Useful when we already have the encoded
|
||||
|
||||
@@ -11,7 +11,7 @@ use reth_db_api::{
|
||||
DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW, DupWalker, RangeWalker,
|
||||
ReverseWalker, Walker,
|
||||
},
|
||||
table::{Compress, Decode, Decompress, DupSort, Encode, Table},
|
||||
table::{Compress, Decode, Decompress, DupSort, Encode, IntoVec, Table},
|
||||
};
|
||||
use reth_libmdbx::{Error as MDBXError, TransactionKind, WriteFlags, RO, RW};
|
||||
use reth_storage_errors::db::{DatabaseErrorInfo, DatabaseWriteError, DatabaseWriteOperation};
|
||||
@@ -215,27 +215,26 @@ impl<K: TransactionKind, T: DupSort> DbDupCursorRO<T> for Cursor<K, T> {
|
||||
) -> Result<DupWalker<'_, T, Self>, DatabaseError> {
|
||||
let start = match (key, subkey) {
|
||||
(Some(key), Some(subkey)) => {
|
||||
// encode key and decode it after.
|
||||
let key: Vec<u8> = key.encode().into();
|
||||
let encoded_key = key.encode();
|
||||
self.inner
|
||||
.get_both_range(key.as_ref(), subkey.encode().as_ref())
|
||||
.get_both_range(encoded_key.as_ref(), subkey.encode().as_ref())
|
||||
.map_err(|e| DatabaseError::Read(e.into()))?
|
||||
.map(|val| decoder::<T>((Cow::Owned(key), val)))
|
||||
.map(|val| decoder::<T>((Cow::Borrowed(encoded_key.as_ref()), val)))
|
||||
}
|
||||
(Some(key), None) => {
|
||||
let key: Vec<u8> = key.encode().into();
|
||||
let encoded_key = key.encode();
|
||||
self.inner
|
||||
.set(key.as_ref())
|
||||
.set(encoded_key.as_ref())
|
||||
.map_err(|e| DatabaseError::Read(e.into()))?
|
||||
.map(|val| decoder::<T>((Cow::Owned(key), val)))
|
||||
.map(|val| decoder::<T>((Cow::Borrowed(encoded_key.as_ref()), val)))
|
||||
}
|
||||
(None, Some(subkey)) => {
|
||||
if let Some((key, _)) = self.first()? {
|
||||
let key: Vec<u8> = key.encode().into();
|
||||
let encoded_key = key.encode();
|
||||
self.inner
|
||||
.get_both_range(key.as_ref(), subkey.encode().as_ref())
|
||||
.get_both_range(encoded_key.as_ref(), subkey.encode().as_ref())
|
||||
.map_err(|e| DatabaseError::Read(e.into()))?
|
||||
.map(|val| decoder::<T>((Cow::Owned(key), val)))
|
||||
.map(|val| decoder::<T>((Cow::Borrowed(encoded_key.as_ref()), val)))
|
||||
} else {
|
||||
Some(Err(DatabaseError::Read(MDBXError::NotFound.into())))
|
||||
}
|
||||
@@ -269,7 +268,7 @@ impl<T: Table> DbCursorRW<T> for Cursor<RW, T> {
|
||||
info: e.into(),
|
||||
operation: DatabaseWriteOperation::CursorUpsert,
|
||||
table_name: T::NAME,
|
||||
key: key.into(),
|
||||
key: key.into_vec(),
|
||||
}
|
||||
.into()
|
||||
})
|
||||
@@ -291,7 +290,7 @@ impl<T: Table> DbCursorRW<T> for Cursor<RW, T> {
|
||||
info: e.into(),
|
||||
operation: DatabaseWriteOperation::CursorInsert,
|
||||
table_name: T::NAME,
|
||||
key: key.into(),
|
||||
key: key.into_vec(),
|
||||
}
|
||||
.into()
|
||||
})
|
||||
@@ -315,7 +314,7 @@ impl<T: Table> DbCursorRW<T> for Cursor<RW, T> {
|
||||
info: e.into(),
|
||||
operation: DatabaseWriteOperation::CursorAppend,
|
||||
table_name: T::NAME,
|
||||
key: key.into(),
|
||||
key: key.into_vec(),
|
||||
}
|
||||
.into()
|
||||
})
|
||||
@@ -351,7 +350,7 @@ impl<T: DupSort> DbDupCursorRW<T> for Cursor<RW, T> {
|
||||
info: e.into(),
|
||||
operation: DatabaseWriteOperation::CursorAppendDup,
|
||||
table_name: T::NAME,
|
||||
key: key.into(),
|
||||
key: key.into_vec(),
|
||||
}
|
||||
.into()
|
||||
})
|
||||
|
||||
@@ -6,7 +6,7 @@ use crate::{
|
||||
DatabaseError,
|
||||
};
|
||||
use reth_db_api::{
|
||||
table::{Compress, DupSort, Encode, Table, TableImporter},
|
||||
table::{Compress, DupSort, Encode, IntoVec, Table, TableImporter},
|
||||
transaction::{DbTx, DbTxMut},
|
||||
};
|
||||
use reth_libmdbx::{ffi::MDBX_dbi, CommitLatency, Transaction, TransactionKind, WriteFlags, RW};
|
||||
@@ -387,7 +387,7 @@ impl Tx<RW> {
|
||||
info: e.into(),
|
||||
operation: write_operation,
|
||||
table_name: T::NAME,
|
||||
key: key.into(),
|
||||
key: key.into_vec(),
|
||||
}
|
||||
.into()
|
||||
})
|
||||
|
||||
@@ -104,6 +104,16 @@ pub enum ProviderError {
|
||||
/// State is not available for the given block number because it is pruned.
|
||||
#[error("state at block #{_0} is pruned")]
|
||||
StateAtBlockPruned(BlockNumber),
|
||||
/// Block data is not available because history has expired.
|
||||
///
|
||||
/// The requested block number is below the earliest available block.
|
||||
#[error("block #{requested} is not available, history has expired (earliest available: #{earliest_available})")]
|
||||
BlockExpired {
|
||||
/// The block number that was requested.
|
||||
requested: BlockNumber,
|
||||
/// The earliest available block number.
|
||||
earliest_available: BlockNumber,
|
||||
},
|
||||
/// Provider does not support this particular request.
|
||||
#[error("this provider does not support this request")]
|
||||
UnsupportedProvider,
|
||||
|
||||
@@ -312,7 +312,7 @@ where
|
||||
}
|
||||
|
||||
/// Position at first key-value pair greater than or equal to specified, return both key and
|
||||
/// data, and the return code depends on a exact match.
|
||||
/// data, and the return code depends on an exact match.
|
||||
///
|
||||
/// For non DupSort-ed collections this works the same as [`Self::set_range()`], but returns
|
||||
/// [false] if key found exactly and [true] if greater key was found.
|
||||
|
||||
@@ -223,7 +223,11 @@ where
|
||||
let mut flags: c_uint = 0;
|
||||
unsafe {
|
||||
self.txn_execute(|txn| {
|
||||
mdbx_result(ffi::mdbx_dbi_flags_ex(txn, dbi, &mut flags, ptr::null_mut()))
|
||||
// `mdbx_dbi_flags_ex` requires `status` to be a non-NULL ptr, otherwise it will
|
||||
// return an EINVAL and panic below, so we just provide a placeholder variable
|
||||
// which we discard immediately.
|
||||
let mut _status: c_uint = 0;
|
||||
mdbx_result(ffi::mdbx_dbi_flags_ex(txn, dbi, &mut flags, &mut _status))
|
||||
})??;
|
||||
}
|
||||
|
||||
|
||||
@@ -37,12 +37,12 @@ impl<H: NippyJarHeader> NippyJarChecker<H> {
|
||||
Self { jar, data_file: None, offsets_file: None }
|
||||
}
|
||||
|
||||
/// It will throw an error if the [`NippyJar`] is in a inconsistent state.
|
||||
/// It will throw an error if the [`NippyJar`] is in an inconsistent state.
|
||||
pub fn check_consistency(&mut self) -> Result<(), NippyJarError> {
|
||||
self.handle_consistency(ConsistencyFailStrategy::ThrowError)
|
||||
}
|
||||
|
||||
/// It will attempt to heal if the [`NippyJar`] is in a inconsistent state.
|
||||
/// It will attempt to heal if the [`NippyJar`] is in an inconsistent state.
|
||||
///
|
||||
/// **ATTENTION**: disk commit should be handled externally by consuming `Self`
|
||||
pub fn ensure_consistency(&mut self) -> Result<(), NippyJarError> {
|
||||
|
||||
@@ -85,6 +85,7 @@ rand.workspace = true
|
||||
tokio = { workspace = true, features = ["sync", "macros", "rt-multi-thread"] }
|
||||
|
||||
[features]
|
||||
edge = ["reth-storage-api/edge", "rocksdb"]
|
||||
rocksdb = ["dep:rocksdb"]
|
||||
test-utils = [
|
||||
"reth-db/test-utils",
|
||||
|
||||
@@ -13,7 +13,7 @@ use crate::{
|
||||
providers::{history_info, HistoryInfo, StaticFileProvider, StaticFileProviderRWRefMut},
|
||||
StaticFileProviderFactory,
|
||||
};
|
||||
use alloy_primitives::{map::HashMap, Address, BlockNumber, TxHash, TxNumber};
|
||||
use alloy_primitives::{map::HashMap, Address, BlockNumber, TxHash, TxNumber, B256};
|
||||
use rayon::slice::ParallelSliceMut;
|
||||
use reth_db::{
|
||||
cursor::{DbCursorRO, DbDupCursorRW},
|
||||
@@ -512,14 +512,71 @@ where
|
||||
Self::RocksDB(batch) => batch.delete::<tables::StoragesHistory>(key),
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends a storage history entry (for first sync - more efficient).
|
||||
pub fn append_storage_history(
|
||||
&mut self,
|
||||
key: StorageShardedKey,
|
||||
value: &BlockNumberList,
|
||||
) -> ProviderResult<()> {
|
||||
match self {
|
||||
Self::Database(cursor) => Ok(cursor.append(key, value)?),
|
||||
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(batch) => batch.put::<tables::StoragesHistory>(key, value),
|
||||
}
|
||||
}
|
||||
|
||||
/// Upserts a storage history entry (for incremental sync).
|
||||
pub fn upsert_storage_history(
|
||||
&mut self,
|
||||
key: StorageShardedKey,
|
||||
value: &BlockNumberList,
|
||||
) -> ProviderResult<()> {
|
||||
match self {
|
||||
Self::Database(cursor) => Ok(cursor.upsert(key, value)?),
|
||||
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(batch) => batch.put::<tables::StoragesHistory>(key, value),
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets the last shard for an address and storage key (keyed with `u64::MAX`).
|
||||
pub fn get_last_storage_history_shard(
|
||||
&mut self,
|
||||
address: Address,
|
||||
storage_key: B256,
|
||||
) -> ProviderResult<Option<BlockNumberList>> {
|
||||
let key = StorageShardedKey::last(address, storage_key);
|
||||
match self {
|
||||
Self::Database(cursor) => Ok(cursor.seek_exact(key)?.map(|(_, v)| v)),
|
||||
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(batch) => batch.get::<tables::StoragesHistory>(key),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N>
|
||||
where
|
||||
CURSOR: DbCursorRW<tables::AccountsHistory> + DbCursorRO<tables::AccountsHistory>,
|
||||
{
|
||||
/// Puts an account history entry.
|
||||
pub fn put_account_history(
|
||||
/// Appends an account history entry (for first sync - more efficient).
|
||||
pub fn append_account_history(
|
||||
&mut self,
|
||||
key: ShardedKey<Address>,
|
||||
value: &BlockNumberList,
|
||||
) -> ProviderResult<()> {
|
||||
match self {
|
||||
Self::Database(cursor) => Ok(cursor.append(key, value)?),
|
||||
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(batch) => batch.put::<tables::AccountsHistory>(key, value),
|
||||
}
|
||||
}
|
||||
|
||||
/// Upserts an account history entry (for incremental sync).
|
||||
pub fn upsert_account_history(
|
||||
&mut self,
|
||||
key: ShardedKey<Address>,
|
||||
value: &BlockNumberList,
|
||||
@@ -532,6 +589,21 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets the last shard for an address (keyed with `u64::MAX`).
|
||||
pub fn get_last_account_history_shard(
|
||||
&mut self,
|
||||
address: Address,
|
||||
) -> ProviderResult<Option<BlockNumberList>> {
|
||||
match self {
|
||||
Self::Database(cursor) => {
|
||||
Ok(cursor.seek_exact(ShardedKey::last(address))?.map(|(_, v)| v))
|
||||
}
|
||||
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(batch) => batch.get::<tables::AccountsHistory>(ShardedKey::last(address)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Deletes an account history entry.
|
||||
pub fn delete_account_history(&mut self, key: ShardedKey<Address>) -> ProviderResult<()> {
|
||||
match self {
|
||||
@@ -1266,8 +1338,8 @@ mod rocksdb_tests {
|
||||
for (highest_block, blocks) in shards {
|
||||
let key = ShardedKey::new(address, *highest_block);
|
||||
let value = IntegerList::new(blocks.clone()).unwrap();
|
||||
mdbx_writer.put_account_history(key.clone(), &value).unwrap();
|
||||
rocks_writer.put_account_history(key, &value).unwrap();
|
||||
mdbx_writer.upsert_account_history(key.clone(), &value).unwrap();
|
||||
rocks_writer.upsert_account_history(key, &value).unwrap();
|
||||
}
|
||||
|
||||
// Commit both backends
|
||||
@@ -1279,7 +1351,8 @@ mod rocksdb_tests {
|
||||
|
||||
// Run queries against both backends using EitherReader
|
||||
let mdbx_ro = factory.database_provider_ro().unwrap();
|
||||
let rocks_tx = rocks_provider.tx();
|
||||
// Use `with_assume_history_complete()` since both backends have identical data
|
||||
let rocks_tx = rocks_provider.tx().with_assume_history_complete();
|
||||
|
||||
for (i, query) in queries.iter().enumerate() {
|
||||
// MDBX query via EitherReader
|
||||
@@ -1371,7 +1444,8 @@ mod rocksdb_tests {
|
||||
|
||||
// Run queries against both backends using EitherReader
|
||||
let mdbx_ro = factory.database_provider_ro().unwrap();
|
||||
let rocks_tx = rocks_provider.tx();
|
||||
// Use `with_assume_history_complete()` since both backends have identical data
|
||||
let rocks_tx = rocks_provider.tx().with_assume_history_complete();
|
||||
|
||||
for (i, query) in queries.iter().enumerate() {
|
||||
// MDBX query via EitherReader
|
||||
|
||||
@@ -2,7 +2,7 @@ use crate::{
|
||||
changesets_utils::StorageRevertsIter,
|
||||
providers::{
|
||||
database::{chain::ChainStorage, metrics},
|
||||
rocksdb::{PendingRocksDBBatches, RocksDBProvider, RocksDBWriteCtx},
|
||||
rocksdb::{PendingHistory, PendingRocksDBBatches, RocksDBProvider, RocksDBWriteCtx},
|
||||
static_file::{StaticFileWriteCtx, StaticFileWriter},
|
||||
NodeTypesForProvider, StaticFileProvider,
|
||||
},
|
||||
@@ -186,6 +186,10 @@ pub struct DatabaseProvider<TX, N: NodeTypes> {
|
||||
/// Pending `RocksDB` batches to be committed at provider commit time.
|
||||
#[cfg_attr(not(all(unix, feature = "rocksdb")), allow(dead_code))]
|
||||
pending_rocksdb_batches: PendingRocksDBBatches,
|
||||
/// Pending history writes accumulated across `save_blocks()` calls.
|
||||
/// Materialized into `RocksDB` shards at commit time after MDBX commits.
|
||||
#[cfg_attr(not(all(unix, feature = "rocksdb")), allow(dead_code))]
|
||||
pending_rocksdb_history: PendingHistory,
|
||||
/// Minimum distance from tip required for pruning
|
||||
minimum_pruning_distance: u64,
|
||||
/// Database provider metrics
|
||||
@@ -204,6 +208,7 @@ impl<TX: Debug, N: NodeTypes> Debug for DatabaseProvider<TX, N> {
|
||||
.field("rocksdb_provider", &self.rocksdb_provider)
|
||||
.field("changeset_cache", &self.changeset_cache)
|
||||
.field("pending_rocksdb_batches", &"<pending batches>")
|
||||
.field("pending_rocksdb_history", &"<pending history>")
|
||||
.field("minimum_pruning_distance", &self.minimum_pruning_distance)
|
||||
.finish()
|
||||
}
|
||||
@@ -337,6 +342,7 @@ impl<TX: DbTxMut, N: NodeTypes> DatabaseProvider<TX, N> {
|
||||
rocksdb_provider,
|
||||
changeset_cache,
|
||||
pending_rocksdb_batches: Default::default(),
|
||||
pending_rocksdb_history: Default::default(),
|
||||
minimum_pruning_distance: MINIMUM_PRUNING_DISTANCE,
|
||||
metrics: metrics::DatabaseProviderMetrics::default(),
|
||||
}
|
||||
@@ -410,6 +416,7 @@ impl<TX: DbTx + DbTxMut + 'static, N: NodeTypesForProvider> DatabaseProvider<TX,
|
||||
prune_tx_lookup: self.prune_modes.transaction_lookup,
|
||||
storage_settings: self.cached_storage_settings(),
|
||||
pending_batches: self.pending_rocksdb_batches.clone(),
|
||||
pending_history: self.pending_rocksdb_history.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -560,42 +567,12 @@ impl<TX: DbTx + DbTxMut + 'static, N: NodeTypesForProvider> DatabaseProvider<TX,
|
||||
|
||||
// Write all trie updates in a single batch.
|
||||
// This reduces cursor open/close overhead from N calls to 1.
|
||||
// Uses hybrid algorithm: extend_ref for small batches, k-way merge for large.
|
||||
if save_mode.with_state() {
|
||||
const MERGE_BATCH_THRESHOLD: usize = 30;
|
||||
|
||||
let start = Instant::now();
|
||||
let num_blocks = blocks.len();
|
||||
|
||||
let merged = if num_blocks == 0 {
|
||||
TrieUpdatesSorted::default()
|
||||
} else if num_blocks == 1 {
|
||||
// Single block: use directly (Arc::try_unwrap avoids clone if refcount is 1)
|
||||
match Arc::try_unwrap(blocks[0].trie_updates()) {
|
||||
Ok(owned) => owned,
|
||||
Err(arc) => (*arc).clone(),
|
||||
}
|
||||
} else if num_blocks < MERGE_BATCH_THRESHOLD {
|
||||
// Small k: extend_ref with Arc::make_mut (copy-on-write).
|
||||
// Blocks are oldest-to-newest, iterate forward so newest overrides.
|
||||
let mut blocks_iter = blocks.iter();
|
||||
let mut result = blocks_iter.next().expect("non-empty").trie_updates();
|
||||
|
||||
for block in blocks_iter {
|
||||
Arc::make_mut(&mut result).extend_ref(block.trie_updates().as_ref());
|
||||
}
|
||||
|
||||
match Arc::try_unwrap(result) {
|
||||
Ok(owned) => owned,
|
||||
Err(arc) => (*arc).clone(),
|
||||
}
|
||||
} else {
|
||||
// Large k: k-way merge is faster (O(n log k)).
|
||||
// Collect Arcs first to extend lifetime, then pass refs.
|
||||
// Blocks are oldest-to-newest, merge_batch expects newest-to-oldest.
|
||||
let arcs: Vec<_> = blocks.iter().rev().map(|b| b.trie_updates()).collect();
|
||||
TrieUpdatesSorted::merge_batch(arcs.iter().map(|arc| arc.as_ref()))
|
||||
};
|
||||
// Blocks are oldest-to-newest, merge_batch expects newest-to-oldest.
|
||||
let merged =
|
||||
TrieUpdatesSorted::merge_batch(blocks.iter().rev().map(|b| b.trie_updates()));
|
||||
|
||||
if !merged.is_empty() {
|
||||
self.write_trie_updates_sorted(&merged)?;
|
||||
@@ -906,6 +883,7 @@ impl<TX: DbTx + 'static, N: NodeTypesForProvider> DatabaseProvider<TX, N> {
|
||||
rocksdb_provider,
|
||||
changeset_cache,
|
||||
pending_rocksdb_batches: Default::default(),
|
||||
pending_rocksdb_history: Default::default(),
|
||||
minimum_pruning_distance: MINIMUM_PRUNING_DISTANCE,
|
||||
metrics: metrics::DatabaseProviderMetrics::default(),
|
||||
}
|
||||
@@ -1534,10 +1512,17 @@ impl<TX: DbTx + 'static, N: NodeTypesForProvider> BlockReader for DatabaseProvid
|
||||
/// If the header for this block is not found, this returns `None`.
|
||||
/// If the header is found, but the transactions either do not exist, or are not indexed, this
|
||||
/// will return None.
|
||||
///
|
||||
/// Returns an error if the requested block is below the earliest available history.
|
||||
fn block(&self, id: BlockHashOrNumber) -> ProviderResult<Option<Self::Block>> {
|
||||
if let Some(number) = self.convert_hash_or_number(id)? &&
|
||||
let Some(header) = self.header_by_number(number)?
|
||||
{
|
||||
if let Some(number) = self.convert_hash_or_number(id)? {
|
||||
let earliest_available = self.static_file_provider.earliest_history_height();
|
||||
if number < earliest_available {
|
||||
return Err(ProviderError::BlockExpired { requested: number, earliest_available })
|
||||
}
|
||||
|
||||
let Some(header) = self.header_by_number(number)? else { return Ok(None) };
|
||||
|
||||
// If the body indices are not found, this means that the transactions either do not
|
||||
// exist in the database yet, or they do exit but are not indexed.
|
||||
// If they exist but are not indexed, we don't have enough
|
||||
@@ -2969,25 +2954,33 @@ impl<TX: DbTxMut + DbTx + 'static, N: NodeTypes> HistoryWriter for DatabaseProvi
|
||||
.into_iter()
|
||||
.map(|(index, account)| (account.address, *index))
|
||||
.collect::<Vec<_>>();
|
||||
last_indices.sort_by_key(|(a, _)| *a);
|
||||
last_indices.sort_unstable_by_key(|(a, _)| *a);
|
||||
|
||||
// Unwind the account history index.
|
||||
let mut cursor = self.tx.cursor_write::<tables::AccountsHistory>()?;
|
||||
for &(address, rem_index) in &last_indices {
|
||||
let partial_shard = unwind_history_shards::<_, tables::AccountsHistory, _>(
|
||||
&mut cursor,
|
||||
ShardedKey::last(address),
|
||||
rem_index,
|
||||
|sharded_key| sharded_key.key == address,
|
||||
)?;
|
||||
|
||||
// Check the last returned partial shard.
|
||||
// If it's not empty, the shard needs to be reinserted.
|
||||
if !partial_shard.is_empty() {
|
||||
cursor.insert(
|
||||
if self.cached_storage_settings().account_history_in_rocksdb {
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
{
|
||||
let batch = self.rocksdb_provider.unwind_account_history_indices(&last_indices)?;
|
||||
self.pending_rocksdb_batches.lock().push(batch);
|
||||
}
|
||||
} else {
|
||||
// Unwind the account history index in MDBX.
|
||||
let mut cursor = self.tx.cursor_write::<tables::AccountsHistory>()?;
|
||||
for &(address, rem_index) in &last_indices {
|
||||
let partial_shard = unwind_history_shards::<_, tables::AccountsHistory, _>(
|
||||
&mut cursor,
|
||||
ShardedKey::last(address),
|
||||
&BlockNumberList::new_pre_sorted(partial_shard),
|
||||
rem_index,
|
||||
|sharded_key| sharded_key.key == address,
|
||||
)?;
|
||||
|
||||
// Check the last returned partial shard.
|
||||
// If it's not empty, the shard needs to be reinserted.
|
||||
if !partial_shard.is_empty() {
|
||||
cursor.insert(
|
||||
ShardedKey::last(address),
|
||||
&BlockNumberList::new_pre_sorted(partial_shard),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3027,25 +3020,35 @@ impl<TX: DbTxMut + DbTx + 'static, N: NodeTypes> HistoryWriter for DatabaseProvi
|
||||
.collect::<Vec<_>>();
|
||||
storage_changesets.sort_by_key(|(address, key, _)| (*address, *key));
|
||||
|
||||
let mut cursor = self.tx.cursor_write::<tables::StoragesHistory>()?;
|
||||
for &(address, storage_key, rem_index) in &storage_changesets {
|
||||
let partial_shard = unwind_history_shards::<_, tables::StoragesHistory, _>(
|
||||
&mut cursor,
|
||||
StorageShardedKey::last(address, storage_key),
|
||||
rem_index,
|
||||
|storage_sharded_key| {
|
||||
storage_sharded_key.address == address &&
|
||||
storage_sharded_key.sharded_key.key == storage_key
|
||||
},
|
||||
)?;
|
||||
|
||||
// Check the last returned partial shard.
|
||||
// If it's not empty, the shard needs to be reinserted.
|
||||
if !partial_shard.is_empty() {
|
||||
cursor.insert(
|
||||
if self.cached_storage_settings().storages_history_in_rocksdb {
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
{
|
||||
let batch =
|
||||
self.rocksdb_provider.unwind_storage_history_indices(&storage_changesets)?;
|
||||
self.pending_rocksdb_batches.lock().push(batch);
|
||||
}
|
||||
} else {
|
||||
// Unwind the storage history index in MDBX.
|
||||
let mut cursor = self.tx.cursor_write::<tables::StoragesHistory>()?;
|
||||
for &(address, storage_key, rem_index) in &storage_changesets {
|
||||
let partial_shard = unwind_history_shards::<_, tables::StoragesHistory, _>(
|
||||
&mut cursor,
|
||||
StorageShardedKey::last(address, storage_key),
|
||||
&BlockNumberList::new_pre_sorted(partial_shard),
|
||||
rem_index,
|
||||
|storage_sharded_key| {
|
||||
storage_sharded_key.address == address &&
|
||||
storage_sharded_key.sharded_key.key == storage_key
|
||||
},
|
||||
)?;
|
||||
|
||||
// Check the last returned partial shard.
|
||||
// If it's not empty, the shard needs to be reinserted.
|
||||
if !partial_shard.is_empty() {
|
||||
cursor.insert(
|
||||
StorageShardedKey::last(address, storage_key),
|
||||
&BlockNumberList::new_pre_sorted(partial_shard),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3483,6 +3486,19 @@ impl<TX: DbTx + 'static, N: NodeTypes + 'static> DBProvider for DatabaseProvider
|
||||
}
|
||||
|
||||
/// Commit database transaction, static files, and pending `RocksDB` batches.
|
||||
///
|
||||
/// # Commit Order (Critical for Correctness)
|
||||
///
|
||||
/// `RocksDB` history indices must commit AFTER MDBX changesets to prevent a race
|
||||
/// condition where history indices point to non-existent changesets:
|
||||
///
|
||||
/// 1. Static files finalize (headers, transactions, receipts)
|
||||
/// 2. MDBX commits (changesets become visible)
|
||||
/// 3. `RocksDB` batches commit (non-history data like tx hashes)
|
||||
/// 4. `RocksDB` history commits (account/storage history indices)
|
||||
///
|
||||
/// This ordering ensures that when a reader follows a `RocksDB` history index
|
||||
/// to a changeset, the changeset exists in MDBX.
|
||||
fn commit(self) -> ProviderResult<()> {
|
||||
// For unwinding it makes more sense to commit the database first, since if
|
||||
// it is interrupted before the static files commit, we can just
|
||||
@@ -3493,10 +3509,15 @@ impl<TX: DbTx + 'static, N: NodeTypes + 'static> DBProvider for DatabaseProvider
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
{
|
||||
// Commit non-history batches
|
||||
let batches = std::mem::take(&mut *self.pending_rocksdb_batches.lock());
|
||||
for batch in batches {
|
||||
self.rocksdb_provider.commit_batch(batch)?;
|
||||
}
|
||||
|
||||
// Commit pending history (after MDBX, so changesets exist)
|
||||
let history = std::mem::take(&mut *self.pending_rocksdb_history.lock());
|
||||
self.rocksdb_provider.commit_pending_history(history)?;
|
||||
}
|
||||
|
||||
self.static_file_provider.commit()?;
|
||||
@@ -3508,20 +3529,30 @@ impl<TX: DbTx + 'static, N: NodeTypes + 'static> DBProvider for DatabaseProvider
|
||||
self.static_file_provider.finalize()?;
|
||||
timings.sf = start.elapsed();
|
||||
|
||||
// CRITICAL: MDBX must commit BEFORE RocksDB history indices.
|
||||
// This prevents a race where history says "look at changeset N" but
|
||||
// the changeset doesn't exist yet.
|
||||
let start = Instant::now();
|
||||
self.tx.commit()?;
|
||||
timings.mdbx = start.elapsed();
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
{
|
||||
let start = Instant::now();
|
||||
|
||||
// Commit non-history batches (tx hashes, etc.)
|
||||
let batches = std::mem::take(&mut *self.pending_rocksdb_batches.lock());
|
||||
for batch in batches {
|
||||
self.rocksdb_provider.commit_batch(batch)?;
|
||||
}
|
||||
|
||||
// Commit pending history (after MDBX, so changesets exist)
|
||||
let history = std::mem::take(&mut *self.pending_rocksdb_history.lock());
|
||||
self.rocksdb_provider.commit_pending_history(history)?;
|
||||
|
||||
timings.rocksdb = start.elapsed();
|
||||
}
|
||||
|
||||
let start = Instant::now();
|
||||
self.tx.commit()?;
|
||||
timings.mdbx = start.elapsed();
|
||||
|
||||
self.metrics.record_commit(&timings);
|
||||
}
|
||||
|
||||
|
||||
@@ -38,7 +38,9 @@ pub use consistent::ConsistentProvider;
|
||||
#[cfg_attr(not(all(unix, feature = "rocksdb")), path = "rocksdb_stub.rs")]
|
||||
pub(crate) mod rocksdb;
|
||||
|
||||
pub use rocksdb::{RocksDBBatch, RocksDBBuilder, RocksDBProvider, RocksTx};
|
||||
pub use rocksdb::{
|
||||
RocksDBBatch, RocksDBBuilder, RocksDBProvider, RocksDBRawIter, RocksDBTableStats, RocksTx,
|
||||
};
|
||||
|
||||
/// Helper trait to bound [`NodeTypes`] so that combined with database they satisfy
|
||||
/// [`ProviderNodeTypes`].
|
||||
|
||||
@@ -253,14 +253,21 @@ impl RocksDBProvider {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// Find the max highest_block_number (excluding u64::MAX sentinel) across all
|
||||
// entries. Also track if we found any non-sentinel entries.
|
||||
// Find the max block number across all entries, including sentinel shards.
|
||||
let mut max_highest_block = 0u64;
|
||||
let mut max_sentinel_block = 0u64;
|
||||
let mut found_non_sentinel = false;
|
||||
|
||||
for result in self.iter::<tables::StoragesHistory>()? {
|
||||
let (key, _) = result?;
|
||||
let (key, value) = result?;
|
||||
let highest = key.sharded_key.highest_block_number;
|
||||
if highest != u64::MAX {
|
||||
if highest == u64::MAX {
|
||||
if let Some(max_in_shard) = value.iter().max() &&
|
||||
max_in_shard > max_sentinel_block
|
||||
{
|
||||
max_sentinel_block = max_in_shard;
|
||||
}
|
||||
} else {
|
||||
found_non_sentinel = true;
|
||||
if highest > max_highest_block {
|
||||
max_highest_block = highest;
|
||||
@@ -268,31 +275,34 @@ impl RocksDBProvider {
|
||||
}
|
||||
}
|
||||
|
||||
// If all entries are sentinel entries (u64::MAX), treat as first-run scenario.
|
||||
// This means no completed shards exist (only sentinel shards with
|
||||
// highest_block_number=u64::MAX), so no actual history has been indexed.
|
||||
if !found_non_sentinel {
|
||||
let effective_max = max_highest_block.max(max_sentinel_block);
|
||||
|
||||
// If we only have sentinel shards and they're all empty, we're consistent
|
||||
if !found_non_sentinel && max_sentinel_block == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// If any entry has highest_block > checkpoint, prune excess
|
||||
if max_highest_block > checkpoint {
|
||||
// If any entry has data > checkpoint, prune excess
|
||||
if effective_max > checkpoint {
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
rocks_highest = max_highest_block,
|
||||
rocks_highest = effective_max,
|
||||
max_non_sentinel = max_highest_block,
|
||||
max_sentinel = max_sentinel_block,
|
||||
checkpoint,
|
||||
"StoragesHistory ahead of checkpoint, pruning excess data"
|
||||
);
|
||||
self.prune_storages_history_above(checkpoint)?;
|
||||
} else if max_highest_block < checkpoint {
|
||||
return Ok(None);
|
||||
} else if effective_max < checkpoint {
|
||||
// RocksDB is behind checkpoint, return highest block to signal unwind needed
|
||||
tracing::warn!(
|
||||
target: "reth::providers::rocksdb",
|
||||
rocks_highest = max_highest_block,
|
||||
rocks_highest = effective_max,
|
||||
checkpoint,
|
||||
"StoragesHistory behind checkpoint, unwind needed"
|
||||
);
|
||||
return Ok(Some(max_highest_block));
|
||||
return Ok(Some(effective_max));
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
@@ -309,25 +319,42 @@ impl RocksDBProvider {
|
||||
/// For `StoragesHistory`, the key contains `highest_block_number`, so we can iterate
|
||||
/// and delete entries where `key.sharded_key.highest_block_number > max_block`.
|
||||
///
|
||||
/// Sentinel shards (with `highest_block_number = u64::MAX`) require special handling:
|
||||
/// we must read their contents and filter out block numbers > `max_block`.
|
||||
///
|
||||
/// TODO(<https://github.com/paradigmxyz/reth/issues/20417>): this iterates the whole table,
|
||||
/// which is inefficient. Use changeset-based pruning instead.
|
||||
fn prune_storages_history_above(&self, max_block: BlockNumber) -> ProviderResult<()> {
|
||||
use reth_db_api::models::storage_sharded_key::StorageShardedKey;
|
||||
use reth_db_api::{models::storage_sharded_key::StorageShardedKey, BlockNumberList};
|
||||
|
||||
let mut to_delete: Vec<StorageShardedKey> = Vec::new();
|
||||
let mut to_rewrite: Vec<(StorageShardedKey, BlockNumberList)> = Vec::new();
|
||||
|
||||
for result in self.iter::<tables::StoragesHistory>()? {
|
||||
let (key, _) = result?;
|
||||
let (key, value) = result?;
|
||||
let highest_block = key.sharded_key.highest_block_number;
|
||||
if max_block == 0 || (highest_block != u64::MAX && highest_block > max_block) {
|
||||
|
||||
if max_block == 0 {
|
||||
to_delete.push(key);
|
||||
} else if highest_block == u64::MAX {
|
||||
let filtered: Vec<u64> = value.iter().filter(|&bn| bn <= max_block).collect();
|
||||
if filtered.is_empty() {
|
||||
to_delete.push(key);
|
||||
} else if filtered.len() != value.len() as usize {
|
||||
to_rewrite.push((key, BlockNumberList::new_pre_sorted(filtered)));
|
||||
}
|
||||
} else if highest_block > max_block {
|
||||
to_delete.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
let deleted = to_delete.len();
|
||||
if deleted > 0 {
|
||||
let rewritten = to_rewrite.len();
|
||||
if deleted > 0 || rewritten > 0 {
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
deleted_count = deleted,
|
||||
rewritten_count = rewritten,
|
||||
max_block,
|
||||
"Pruning StoragesHistory entries"
|
||||
);
|
||||
@@ -336,6 +363,9 @@ impl RocksDBProvider {
|
||||
for key in to_delete {
|
||||
batch.delete::<tables::StoragesHistory>(key)?;
|
||||
}
|
||||
for (key, value) in to_rewrite {
|
||||
batch.put::<tables::StoragesHistory>(key, &value)?;
|
||||
}
|
||||
batch.commit()?;
|
||||
}
|
||||
|
||||
@@ -374,14 +404,24 @@ impl RocksDBProvider {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// Find the max highest_block_number (excluding u64::MAX sentinel) across all
|
||||
// entries. Also track if we found any non-sentinel entries.
|
||||
// Find the max block number across all entries, including sentinel shards.
|
||||
// For sentinel shards (highest_block_number == u64::MAX), we need to look
|
||||
// at the actual block numbers stored in the shard.
|
||||
let mut max_highest_block = 0u64;
|
||||
let mut max_sentinel_block = 0u64;
|
||||
let mut found_non_sentinel = false;
|
||||
|
||||
for result in self.iter::<tables::AccountsHistory>()? {
|
||||
let (key, _) = result?;
|
||||
let (key, value) = result?;
|
||||
let highest = key.highest_block_number;
|
||||
if highest != u64::MAX {
|
||||
if highest == u64::MAX {
|
||||
// Sentinel shard: check the actual max block number in the shard
|
||||
if let Some(max_in_shard) = value.iter().max() &&
|
||||
max_in_shard > max_sentinel_block
|
||||
{
|
||||
max_sentinel_block = max_in_shard;
|
||||
}
|
||||
} else {
|
||||
found_non_sentinel = true;
|
||||
if highest > max_highest_block {
|
||||
max_highest_block = highest;
|
||||
@@ -389,18 +429,22 @@ impl RocksDBProvider {
|
||||
}
|
||||
}
|
||||
|
||||
// If all entries are sentinel entries (u64::MAX), treat as first-run scenario.
|
||||
// This means no completed shards exist (only sentinel shards with
|
||||
// highest_block_number=u64::MAX), so no actual history has been indexed.
|
||||
if !found_non_sentinel {
|
||||
// Use the higher of the two maxes
|
||||
let effective_max = max_highest_block.max(max_sentinel_block);
|
||||
|
||||
// If we only have sentinel shards and they're all within checkpoint,
|
||||
// we're consistent
|
||||
if !found_non_sentinel && max_sentinel_block == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// If any entry has highest_block > checkpoint, prune excess
|
||||
if max_highest_block > checkpoint {
|
||||
// If any entry has data > checkpoint, prune excess
|
||||
if effective_max > checkpoint {
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
rocks_highest = max_highest_block,
|
||||
rocks_highest = effective_max,
|
||||
max_non_sentinel = max_highest_block,
|
||||
max_sentinel = max_sentinel_block,
|
||||
checkpoint,
|
||||
"AccountsHistory ahead of checkpoint, pruning excess data"
|
||||
);
|
||||
@@ -409,14 +453,14 @@ impl RocksDBProvider {
|
||||
}
|
||||
|
||||
// If RocksDB is behind the checkpoint, request an unwind to rebuild.
|
||||
if max_highest_block < checkpoint {
|
||||
if effective_max < checkpoint {
|
||||
tracing::warn!(
|
||||
target: "reth::providers::rocksdb",
|
||||
rocks_highest = max_highest_block,
|
||||
rocks_highest = effective_max,
|
||||
checkpoint,
|
||||
"AccountsHistory behind checkpoint, unwind needed"
|
||||
);
|
||||
return Ok(Some(max_highest_block));
|
||||
return Ok(Some(effective_max));
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
@@ -434,26 +478,47 @@ impl RocksDBProvider {
|
||||
/// `highest_block_number`, so we can iterate and delete entries where
|
||||
/// `key.highest_block_number > max_block`.
|
||||
///
|
||||
/// Sentinel shards (with `highest_block_number = u64::MAX`) require special handling:
|
||||
/// we must read their contents and filter out block numbers > `max_block`.
|
||||
///
|
||||
/// TODO(<https://github.com/paradigmxyz/reth/issues/20417>): this iterates the whole table,
|
||||
/// which is inefficient. Use changeset-based pruning instead.
|
||||
fn prune_accounts_history_above(&self, max_block: BlockNumber) -> ProviderResult<()> {
|
||||
use alloy_primitives::Address;
|
||||
use reth_db_api::models::ShardedKey;
|
||||
use reth_db_api::{models::ShardedKey, BlockNumberList};
|
||||
|
||||
let mut to_delete: Vec<ShardedKey<Address>> = Vec::new();
|
||||
let mut to_rewrite: Vec<(ShardedKey<Address>, BlockNumberList)> = Vec::new();
|
||||
|
||||
for result in self.iter::<tables::AccountsHistory>()? {
|
||||
let (key, _) = result?;
|
||||
let (key, value) = result?;
|
||||
let highest_block = key.highest_block_number;
|
||||
if max_block == 0 || (highest_block != u64::MAX && highest_block > max_block) {
|
||||
|
||||
if max_block == 0 {
|
||||
// Clear everything
|
||||
to_delete.push(key);
|
||||
} else if highest_block == u64::MAX {
|
||||
// Sentinel shard: filter out block numbers > max_block
|
||||
let filtered: Vec<u64> = value.iter().filter(|&bn| bn <= max_block).collect();
|
||||
if filtered.is_empty() {
|
||||
to_delete.push(key);
|
||||
} else if filtered.len() != value.len() as usize {
|
||||
// Some entries were filtered out, rewrite the shard
|
||||
to_rewrite.push((key, BlockNumberList::new_pre_sorted(filtered)));
|
||||
}
|
||||
} else if highest_block > max_block {
|
||||
// Non-sentinel shard above max_block: delete
|
||||
to_delete.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
let deleted = to_delete.len();
|
||||
if deleted > 0 {
|
||||
let rewritten = to_rewrite.len();
|
||||
if deleted > 0 || rewritten > 0 {
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
deleted_count = deleted,
|
||||
rewritten_count = rewritten,
|
||||
max_block,
|
||||
"Pruning AccountsHistory entries"
|
||||
);
|
||||
@@ -462,6 +527,9 @@ impl RocksDBProvider {
|
||||
for key in to_delete {
|
||||
batch.delete::<tables::AccountsHistory>(key)?;
|
||||
}
|
||||
for (key, value) in to_rewrite {
|
||||
batch.put::<tables::AccountsHistory>(key, &value)?;
|
||||
}
|
||||
batch.commit()?;
|
||||
}
|
||||
|
||||
@@ -996,6 +1064,7 @@ mod tests {
|
||||
// This simulates a scenario where history tracking started but no shards were completed
|
||||
let key_sentinel_1 = StorageShardedKey::new(Address::ZERO, B256::ZERO, u64::MAX);
|
||||
let key_sentinel_2 = StorageShardedKey::new(Address::random(), B256::random(), u64::MAX);
|
||||
// Sentinel shards contain blocks [10, 20, 30], so max block in shard = 30
|
||||
let block_list = BlockNumberList::new_pre_sorted([10, 20, 30]);
|
||||
rocksdb.put::<tables::StoragesHistory>(key_sentinel_1, &block_list).unwrap();
|
||||
rocksdb.put::<tables::StoragesHistory>(key_sentinel_2, &block_list).unwrap();
|
||||
@@ -1009,24 +1078,21 @@ mod tests {
|
||||
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Set a checkpoint indicating we should have processed up to block 100
|
||||
// Checkpoint = 30 matches sentinel shard max block, simulating "consistent" state
|
||||
// where indexing is complete through the checkpoint block
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
provider
|
||||
.save_stage_checkpoint(StageId::IndexStorageHistory, StageCheckpoint::new(100))
|
||||
.save_stage_checkpoint(StageId::IndexStorageHistory, StageCheckpoint::new(30))
|
||||
.unwrap();
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// RocksDB has only sentinel entries (no completed shards) but checkpoint is set.
|
||||
// This is treated as a first-run/migration scenario - no unwind needed.
|
||||
// RocksDB has sentinel entries matching the checkpoint - consistent state
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(
|
||||
result, None,
|
||||
"Sentinel-only entries with checkpoint should be treated as first run"
|
||||
);
|
||||
assert_eq!(result, None, "Sentinel-only entries matching checkpoint should be consistent");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1042,6 +1108,7 @@ mod tests {
|
||||
// Insert ONLY sentinel entries (highest_block_number = u64::MAX)
|
||||
let key_sentinel_1 = ShardedKey::new(Address::ZERO, u64::MAX);
|
||||
let key_sentinel_2 = ShardedKey::new(Address::random(), u64::MAX);
|
||||
// Sentinel shards contain blocks [10, 20, 30], so max block in shard = 30
|
||||
let block_list = BlockNumberList::new_pre_sorted([10, 20, 30]);
|
||||
rocksdb.put::<tables::AccountsHistory>(key_sentinel_1, &block_list).unwrap();
|
||||
rocksdb.put::<tables::AccountsHistory>(key_sentinel_2, &block_list).unwrap();
|
||||
@@ -1055,24 +1122,21 @@ mod tests {
|
||||
StorageSettings::legacy().with_account_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Set a checkpoint indicating we should have processed up to block 100
|
||||
// Checkpoint = 30 matches sentinel shard max block, simulating "consistent" state
|
||||
// where indexing is complete through the checkpoint block
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
provider
|
||||
.save_stage_checkpoint(StageId::IndexAccountHistory, StageCheckpoint::new(100))
|
||||
.save_stage_checkpoint(StageId::IndexAccountHistory, StageCheckpoint::new(30))
|
||||
.unwrap();
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// RocksDB has only sentinel entries (no completed shards) but checkpoint is set.
|
||||
// This is treated as a first-run/migration scenario - no unwind needed.
|
||||
// RocksDB has sentinel entries matching the checkpoint - consistent state
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(
|
||||
result, None,
|
||||
"Sentinel-only entries with checkpoint should be treated as first run"
|
||||
);
|
||||
assert_eq!(result, None, "Sentinel-only entries matching checkpoint should be consistent");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -6,7 +6,7 @@ use reth_db::Tables;
|
||||
use reth_metrics::Metrics;
|
||||
use strum::{EnumIter, IntoEnumIterator};
|
||||
|
||||
const ROCKSDB_TABLES: &[&str] = &[
|
||||
pub(super) const ROCKSDB_TABLES: &[&str] = &[
|
||||
Tables::TransactionHashNumbers.name(),
|
||||
Tables::StoragesHistory.name(),
|
||||
Tables::AccountsHistory.name(),
|
||||
|
||||
@@ -4,5 +4,10 @@ mod invariants;
|
||||
mod metrics;
|
||||
mod provider;
|
||||
|
||||
pub(crate) use provider::{PendingRocksDBBatches, RocksDBWriteCtx};
|
||||
pub use provider::{RocksDBBatch, RocksDBBuilder, RocksDBProvider, RocksTx};
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) use provider::{
|
||||
PendingHistory, PendingHistoryWrites, PendingRocksDBBatches, RocksDBWriteCtx,
|
||||
};
|
||||
pub use provider::{
|
||||
RocksDBBatch, RocksDBBuilder, RocksDBProvider, RocksDBRawIter, RocksDBTableStats, RocksTx,
|
||||
};
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,8 +5,9 @@
|
||||
//! All method calls are cfg-guarded in the calling code, so only type definitions are needed here.
|
||||
|
||||
use alloy_primitives::BlockNumber;
|
||||
use metrics::Label;
|
||||
use parking_lot::Mutex;
|
||||
use reth_db_api::models::StorageSettings;
|
||||
use reth_db_api::{database_metrics::DatabaseMetrics, models::StorageSettings};
|
||||
use reth_prune_types::PruneMode;
|
||||
use reth_storage_errors::{db::LogLevel, provider::ProviderResult};
|
||||
use std::{path::Path, sync::Arc};
|
||||
@@ -14,6 +15,30 @@ use std::{path::Path, sync::Arc};
|
||||
/// Pending `RocksDB` batches type alias (stub - uses unit type).
|
||||
pub(crate) type PendingRocksDBBatches = Arc<Mutex<Vec<()>>>;
|
||||
|
||||
/// Statistics for a single `RocksDB` table (column family) - stub.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RocksDBTableStats {
|
||||
/// Size of SST files on disk in bytes.
|
||||
pub sst_size_bytes: u64,
|
||||
/// Size of memtables in memory in bytes.
|
||||
pub memtable_size_bytes: u64,
|
||||
/// Name of the table/column family.
|
||||
pub name: String,
|
||||
/// Estimated number of keys in the table.
|
||||
pub estimated_num_keys: u64,
|
||||
/// Estimated size of live data in bytes (SST files + memtables).
|
||||
pub estimated_size_bytes: u64,
|
||||
/// Estimated bytes pending compaction (reclaimable space).
|
||||
pub pending_compaction_bytes: u64,
|
||||
}
|
||||
|
||||
/// Pending history writes (stub - empty struct).
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct PendingHistoryWrites;
|
||||
|
||||
/// Pending history writes type alias (stub).
|
||||
pub(crate) type PendingHistory = Arc<Mutex<PendingHistoryWrites>>;
|
||||
|
||||
/// Context for `RocksDB` block writes (stub).
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
@@ -26,6 +51,8 @@ pub(crate) struct RocksDBWriteCtx {
|
||||
pub storage_settings: StorageSettings,
|
||||
/// Pending batches (stub - unused).
|
||||
pub pending_batches: PendingRocksDBBatches,
|
||||
/// Pending history writes (stub - unused).
|
||||
pub pending_history: PendingHistory,
|
||||
}
|
||||
|
||||
/// A stub `RocksDB` provider.
|
||||
@@ -56,6 +83,19 @@ impl RocksDBProvider {
|
||||
) -> ProviderResult<Option<BlockNumber>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Returns statistics for all column families in the database (stub implementation).
|
||||
///
|
||||
/// Returns an empty vector since there is no `RocksDB` when the feature is disabled.
|
||||
pub const fn table_stats(&self) -> Vec<RocksDBTableStats> {
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl DatabaseMetrics for RocksDBProvider {
|
||||
fn gauge_metrics(&self) -> Vec<(&'static str, f64, Vec<Label>)> {
|
||||
vec![]
|
||||
}
|
||||
}
|
||||
|
||||
/// A stub batch writer for `RocksDB`.
|
||||
@@ -102,6 +142,11 @@ impl RocksDBBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets read-only mode (stub implementation).
|
||||
pub const fn with_read_only(self, _read_only: bool) -> Self {
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the `RocksDB` provider (stub implementation).
|
||||
pub const fn build(self) -> ProviderResult<RocksDBProvider> {
|
||||
Ok(RocksDBProvider)
|
||||
@@ -111,3 +156,7 @@ impl RocksDBBuilder {
|
||||
/// A stub transaction for `RocksDB`.
|
||||
#[derive(Debug)]
|
||||
pub struct RocksTx;
|
||||
|
||||
/// A stub raw iterator for `RocksDB`.
|
||||
#[derive(Debug)]
|
||||
pub struct RocksDBRawIter;
|
||||
|
||||
@@ -291,7 +291,7 @@ impl<Provider: DBProvider + ChangeSetReader + BlockNumReader> StateRootProvider
|
||||
fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult<B256> {
|
||||
let mut revert_state = self.revert_state()?;
|
||||
let hashed_state_sorted = hashed_state.into_sorted();
|
||||
revert_state.extend_ref(&hashed_state_sorted);
|
||||
revert_state.extend_ref_and_sort(&hashed_state_sorted);
|
||||
Ok(StateRoot::overlay_root(self.tx(), &revert_state)?)
|
||||
}
|
||||
|
||||
@@ -306,7 +306,7 @@ impl<Provider: DBProvider + ChangeSetReader + BlockNumReader> StateRootProvider
|
||||
) -> ProviderResult<(B256, TrieUpdates)> {
|
||||
let mut revert_state = self.revert_state()?;
|
||||
let hashed_state_sorted = hashed_state.into_sorted();
|
||||
revert_state.extend_ref(&hashed_state_sorted);
|
||||
revert_state.extend_ref_and_sort(&hashed_state_sorted);
|
||||
Ok(StateRoot::overlay_root_with_updates(self.tx(), &revert_state)?)
|
||||
}
|
||||
|
||||
|
||||
@@ -129,6 +129,8 @@ impl<F> OverlayStateProviderFactory<F> {
|
||||
/// This overlay will be applied on top of any reverts applied via `with_block_hash`.
|
||||
pub fn with_overlay_source(mut self, source: Option<OverlaySource>) -> Self {
|
||||
self.overlay_source = source;
|
||||
// Clear the overlay cache since we've updated the source.
|
||||
self.overlay_cache = Default::default();
|
||||
self
|
||||
}
|
||||
|
||||
@@ -137,6 +139,8 @@ impl<F> OverlayStateProviderFactory<F> {
|
||||
/// Convenience method that wraps the lazy overlay in `OverlaySource::Lazy`.
|
||||
pub fn with_lazy_overlay(mut self, lazy_overlay: Option<LazyOverlay>) -> Self {
|
||||
self.overlay_source = lazy_overlay.map(OverlaySource::Lazy);
|
||||
// Clear the overlay cache since we've updated the source.
|
||||
self.overlay_cache = Default::default();
|
||||
self
|
||||
}
|
||||
|
||||
@@ -152,6 +156,8 @@ impl<F> OverlayStateProviderFactory<F> {
|
||||
trie: Arc::new(TrieUpdatesSorted::default()),
|
||||
state,
|
||||
});
|
||||
// Clear the overlay cache since we've updated the source.
|
||||
self.overlay_cache = Default::default();
|
||||
}
|
||||
self
|
||||
}
|
||||
@@ -163,12 +169,12 @@ impl<F> OverlayStateProviderFactory<F> {
|
||||
pub fn with_extended_hashed_state_overlay(mut self, other: HashedPostStateSorted) -> Self {
|
||||
match &mut self.overlay_source {
|
||||
Some(OverlaySource::Immediate { state, .. }) => {
|
||||
Arc::make_mut(state).extend_ref(&other);
|
||||
Arc::make_mut(state).extend_ref_and_sort(&other);
|
||||
}
|
||||
Some(OverlaySource::Lazy(lazy)) => {
|
||||
// Resolve lazy overlay and convert to immediate with extension
|
||||
let (trie, mut state) = lazy.as_overlay();
|
||||
Arc::make_mut(&mut state).extend_ref(&other);
|
||||
Arc::make_mut(&mut state).extend_ref_and_sort(&other);
|
||||
self.overlay_source = Some(OverlaySource::Immediate { trie, state });
|
||||
}
|
||||
None => {
|
||||
@@ -178,6 +184,8 @@ impl<F> OverlayStateProviderFactory<F> {
|
||||
});
|
||||
}
|
||||
}
|
||||
// Clear the overlay cache since we've updated the source.
|
||||
self.overlay_cache = Default::default();
|
||||
self
|
||||
}
|
||||
}
|
||||
@@ -342,7 +350,7 @@ where
|
||||
let trie_updates = if trie_reverts.is_empty() {
|
||||
overlay_trie
|
||||
} else if !overlay_trie.is_empty() {
|
||||
trie_reverts.extend_ref(&overlay_trie);
|
||||
trie_reverts.extend_ref_and_sort(&overlay_trie);
|
||||
Arc::new(trie_reverts)
|
||||
} else {
|
||||
Arc::new(trie_reverts)
|
||||
@@ -351,7 +359,7 @@ where
|
||||
let hashed_state_updates = if hashed_state_reverts.is_empty() {
|
||||
overlay_state
|
||||
} else if !overlay_state.is_empty() {
|
||||
hashed_state_reverts.extend_ref(&overlay_state);
|
||||
hashed_state_reverts.extend_ref_and_sort(&overlay_state);
|
||||
Arc::new(hashed_state_reverts)
|
||||
} else {
|
||||
Arc::new(hashed_state_reverts)
|
||||
|
||||
@@ -528,7 +528,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
}
|
||||
|
||||
/// Writes headers for all blocks to the static file segment.
|
||||
#[instrument(level = "debug", target = "providers::db", skip_all)]
|
||||
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
|
||||
fn write_headers(
|
||||
w: &mut StaticFileProviderRWRefMut<'_, N>,
|
||||
blocks: &[ExecutedBlock<N>],
|
||||
@@ -541,7 +541,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
}
|
||||
|
||||
/// Writes transactions for all blocks to the static file segment.
|
||||
#[instrument(level = "debug", target = "providers::db", skip_all)]
|
||||
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
|
||||
fn write_transactions(
|
||||
w: &mut StaticFileProviderRWRefMut<'_, N>,
|
||||
blocks: &[ExecutedBlock<N>],
|
||||
@@ -558,7 +558,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
}
|
||||
|
||||
/// Writes transaction senders for all blocks to the static file segment.
|
||||
#[instrument(level = "debug", target = "providers::db", skip_all)]
|
||||
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
|
||||
fn write_transaction_senders(
|
||||
w: &mut StaticFileProviderRWRefMut<'_, N>,
|
||||
blocks: &[ExecutedBlock<N>],
|
||||
@@ -575,7 +575,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
}
|
||||
|
||||
/// Writes receipts for all blocks to the static file segment.
|
||||
#[instrument(level = "debug", target = "providers::db", skip_all)]
|
||||
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
|
||||
fn write_receipts(
|
||||
w: &mut StaticFileProviderRWRefMut<'_, N>,
|
||||
blocks: &[ExecutedBlock<N>],
|
||||
@@ -602,7 +602,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
}
|
||||
|
||||
/// Writes account changesets for all blocks to the static file segment.
|
||||
#[instrument(level = "debug", target = "providers::db", skip_all)]
|
||||
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
|
||||
fn write_account_changesets(
|
||||
w: &mut StaticFileProviderRWRefMut<'_, N>,
|
||||
blocks: &[ExecutedBlock<N>],
|
||||
@@ -647,7 +647,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
///
|
||||
/// This spawns separate threads for each segment type and each thread calls `sync_all()` on its
|
||||
/// writer when done.
|
||||
#[instrument(level = "debug", target = "providers::db", skip_all)]
|
||||
#[instrument(level = "debug", target = "providers::static_file", skip_all)]
|
||||
pub fn write_blocks_data(
|
||||
&self,
|
||||
blocks: &[ExecutedBlock<N>],
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
use crate::{either_writer::RocksTxRefArg, providers::RocksDBProvider};
|
||||
use crate::{
|
||||
either_writer::{RawRocksDBBatch, RocksBatchArg, RocksTxRefArg},
|
||||
providers::RocksDBProvider,
|
||||
};
|
||||
use reth_storage_errors::provider::ProviderResult;
|
||||
|
||||
/// `RocksDB` provider factory.
|
||||
@@ -31,4 +34,28 @@ pub trait RocksDBProviderFactory {
|
||||
#[cfg(not(all(unix, feature = "rocksdb")))]
|
||||
f(())
|
||||
}
|
||||
|
||||
/// Executes a closure with a `RocksDB` batch, automatically registering it for commit.
|
||||
///
|
||||
/// This helper encapsulates all the cfg-gated `RocksDB` batch handling.
|
||||
fn with_rocksdb_batch<F, R>(&self, f: F) -> ProviderResult<R>
|
||||
where
|
||||
F: FnOnce(RocksBatchArg<'_>) -> ProviderResult<(R, Option<RawRocksDBBatch>)>,
|
||||
{
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
{
|
||||
let rocksdb = self.rocksdb_provider();
|
||||
let batch = rocksdb.batch();
|
||||
let (result, raw_batch) = f(batch)?;
|
||||
if let Some(b) = raw_batch {
|
||||
self.set_pending_rocksdb_batch(b);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
#[cfg(not(all(unix, feature = "rocksdb")))]
|
||||
{
|
||||
let (result, _) = f(())?;
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ serde_json = { workspace = true, optional = true }
|
||||
|
||||
[features]
|
||||
default = ["std"]
|
||||
edge = ["reth-db-api/edge"]
|
||||
std = [
|
||||
"reth-chainspec/std",
|
||||
"alloy-consensus/std",
|
||||
|
||||
@@ -746,6 +746,18 @@ impl<T: PoolTransaction> Default for AllPoolTransactions<T> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: PoolTransaction> IntoIterator for AllPoolTransactions<T> {
|
||||
type Item = Arc<ValidPoolTransaction<T>>;
|
||||
type IntoIter = std::iter::Chain<
|
||||
std::vec::IntoIter<Arc<ValidPoolTransaction<T>>>,
|
||||
std::vec::IntoIter<Arc<ValidPoolTransaction<T>>>,
|
||||
>;
|
||||
|
||||
fn into_iter(self) -> Self::IntoIter {
|
||||
self.pending.into_iter().chain(self.queued)
|
||||
}
|
||||
}
|
||||
|
||||
/// Represents transactions that were propagated over the network.
|
||||
#[derive(Debug, Clone, Eq, PartialEq, Default)]
|
||||
pub struct PropagatedTransactions(pub HashMap<TxHash, Vec<PropagateKind>>);
|
||||
|
||||
@@ -6,7 +6,7 @@ use crate::{
|
||||
utils::{extend_sorted_vec, kway_merge_sorted},
|
||||
KeyHasher, MultiProofTargets, Nibbles,
|
||||
};
|
||||
use alloc::{borrow::Cow, vec::Vec};
|
||||
use alloc::{borrow::Cow, sync::Arc, vec::Vec};
|
||||
use alloy_primitives::{
|
||||
keccak256,
|
||||
map::{hash_map, B256Map, HashMap, HashSet},
|
||||
@@ -621,7 +621,9 @@ impl HashedPostStateSorted {
|
||||
|
||||
/// Extends this state with contents of another sorted state.
|
||||
/// Entries in `other` take precedence for duplicate keys.
|
||||
pub fn extend_ref(&mut self, other: &Self) {
|
||||
///
|
||||
/// Sorts the accounts after extending. Sorts the storage after extending, for each account.
|
||||
pub fn extend_ref_and_sort(&mut self, other: &Self) {
|
||||
// Extend accounts
|
||||
extend_sorted_vec(&mut self.accounts, &other.accounts);
|
||||
|
||||
@@ -636,31 +638,44 @@ impl HashedPostStateSorted {
|
||||
|
||||
/// Batch-merge sorted hashed post states. Iterator yields **newest to oldest**.
|
||||
///
|
||||
/// Uses k-way merge for O(n log k) complexity and one-pass accumulation for storages.
|
||||
pub fn merge_batch<'a>(states: impl IntoIterator<Item = &'a Self>) -> Self {
|
||||
let states: Vec<_> = states.into_iter().collect();
|
||||
if states.is_empty() {
|
||||
return Self::default();
|
||||
/// For small batches, uses `extend_ref_and_sort` loop.
|
||||
/// For large batches, uses k-way merge for O(n log k) complexity.
|
||||
pub fn merge_batch<T: AsRef<Self> + From<Self>>(iter: impl IntoIterator<Item = T>) -> T {
|
||||
const THRESHOLD: usize = 30;
|
||||
|
||||
let items: alloc::vec::Vec<_> = iter.into_iter().collect();
|
||||
let k = items.len();
|
||||
|
||||
if k == 0 {
|
||||
return Self::default().into();
|
||||
}
|
||||
if k == 1 {
|
||||
return items.into_iter().next().expect("k == 1");
|
||||
}
|
||||
|
||||
let accounts = kway_merge_sorted(states.iter().map(|s| s.accounts.as_slice()));
|
||||
if k < THRESHOLD {
|
||||
// Small k: extend loop, oldest-to-newest so newer overrides older.
|
||||
let mut iter = items.iter().rev();
|
||||
let mut acc = iter.next().expect("k > 0").as_ref().clone();
|
||||
for next in iter {
|
||||
acc.extend_ref_and_sort(next.as_ref());
|
||||
}
|
||||
return acc.into();
|
||||
}
|
||||
|
||||
// Large k: k-way merge.
|
||||
let accounts = kway_merge_sorted(items.iter().map(|i| i.as_ref().accounts.as_slice()));
|
||||
|
||||
struct StorageAcc<'a> {
|
||||
/// Account storage was cleared (e.g., SELFDESTRUCT).
|
||||
wiped: bool,
|
||||
/// Stop collecting older slices after seeing a wipe.
|
||||
sealed: bool,
|
||||
/// Storage slot slices to merge, ordered newest to oldest.
|
||||
slices: Vec<&'a [(B256, U256)]>,
|
||||
}
|
||||
|
||||
let mut acc: B256Map<StorageAcc<'_>> = B256Map::default();
|
||||
|
||||
// Accumulate storage slices per address from newest to oldest state.
|
||||
// Once we see a `wiped` flag, the account was cleared at that point,
|
||||
// so older storage slots are irrelevant - we "seal" and stop collecting.
|
||||
for state in &states {
|
||||
for (addr, storage) in &state.storages {
|
||||
for item in &items {
|
||||
for (addr, storage) in &item.as_ref().storages {
|
||||
let entry = acc.entry(*addr).or_insert_with(|| StorageAcc {
|
||||
wiped: false,
|
||||
sealed: false,
|
||||
@@ -687,7 +702,7 @@ impl HashedPostStateSorted {
|
||||
})
|
||||
.collect();
|
||||
|
||||
Self { accounts, storages }
|
||||
Self { accounts, storages }.into()
|
||||
}
|
||||
|
||||
/// Clears all accounts and storage data.
|
||||
@@ -695,6 +710,36 @@ impl HashedPostStateSorted {
|
||||
self.accounts.clear();
|
||||
self.storages.clear();
|
||||
}
|
||||
|
||||
/// Parallel batch-merge sorted hashed post states. Slice is **oldest to newest**.
|
||||
///
|
||||
/// This is more efficient than sequential `extend_ref` calls when merging many states,
|
||||
/// as it processes all states in parallel with tree reduction using divide-and-conquer.
|
||||
#[cfg(feature = "rayon")]
|
||||
pub fn merge_parallel(states: &[Arc<Self>]) -> Self {
|
||||
fn parallel_merge_tree(states: &[Arc<HashedPostStateSorted>]) -> HashedPostStateSorted {
|
||||
match states.len() {
|
||||
0 => HashedPostStateSorted::default(),
|
||||
1 => states[0].as_ref().clone(),
|
||||
2 => {
|
||||
let mut acc = states[0].as_ref().clone();
|
||||
acc.extend_ref_and_sort(&states[1]);
|
||||
acc
|
||||
}
|
||||
n => {
|
||||
let mid = n / 2;
|
||||
let (mut left, right) = rayon::join(
|
||||
|| parallel_merge_tree(&states[..mid]),
|
||||
|| parallel_merge_tree(&states[mid..]),
|
||||
);
|
||||
left.extend_ref_and_sort(&right);
|
||||
left
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
parallel_merge_tree(states)
|
||||
}
|
||||
}
|
||||
|
||||
impl AsRef<Self> for HashedPostStateSorted {
|
||||
@@ -1416,7 +1461,7 @@ mod tests {
|
||||
storages: B256Map::default(),
|
||||
};
|
||||
|
||||
state1.extend_ref(&state2);
|
||||
state1.extend_ref_and_sort(&state2);
|
||||
|
||||
// Check accounts are merged and sorted
|
||||
assert_eq!(state1.accounts.len(), 6);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Merkle trie proofs.
|
||||
|
||||
use crate::{BranchNodeMasksMap, Nibbles, TrieAccount};
|
||||
use crate::{BranchNodeMasksMap, Nibbles, ProofTrieNode, TrieAccount};
|
||||
use alloc::{borrow::Cow, vec::Vec};
|
||||
use alloy_consensus::constants::KECCAK_EMPTY;
|
||||
use alloy_primitives::{
|
||||
@@ -267,6 +267,12 @@ impl MultiProof {
|
||||
self.account_subtree.extend_from(other.account_subtree);
|
||||
self.branch_node_masks.extend(other.branch_node_masks);
|
||||
|
||||
let reserve = if self.storages.is_empty() {
|
||||
other.storages.len()
|
||||
} else {
|
||||
other.storages.len().div_ceil(2)
|
||||
};
|
||||
self.storages.reserve(reserve);
|
||||
for (hashed_address, storage) in other.storages {
|
||||
match self.storages.entry(hashed_address) {
|
||||
hash_map::Entry::Occupied(mut entry) => {
|
||||
@@ -390,6 +396,12 @@ impl DecodedMultiProof {
|
||||
self.account_subtree.extend_from(other.account_subtree);
|
||||
self.branch_node_masks.extend(other.branch_node_masks);
|
||||
|
||||
let reserve = if self.storages.is_empty() {
|
||||
other.storages.len()
|
||||
} else {
|
||||
other.storages.len().div_ceil(2)
|
||||
};
|
||||
self.storages.reserve(reserve);
|
||||
for (hashed_address, storage) in other.storages {
|
||||
match self.storages.entry(hashed_address) {
|
||||
hash_map::Entry::Occupied(mut entry) => {
|
||||
@@ -431,6 +443,33 @@ impl TryFrom<MultiProof> for DecodedMultiProof {
|
||||
}
|
||||
}
|
||||
|
||||
/// V2 decoded multiproof which contains the results of both account and storage V2 proof
|
||||
/// calculations.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Default)]
|
||||
pub struct DecodedMultiProofV2 {
|
||||
/// Account trie proof nodes
|
||||
pub account_proofs: Vec<ProofTrieNode>,
|
||||
/// Storage trie proof nodes indexed by account
|
||||
pub storage_proofs: B256Map<Vec<ProofTrieNode>>,
|
||||
}
|
||||
|
||||
impl DecodedMultiProofV2 {
|
||||
/// Returns true if there are no proofs
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.account_proofs.is_empty() && self.storage_proofs.is_empty()
|
||||
}
|
||||
|
||||
/// Appends the given multiproof's data to this one.
|
||||
///
|
||||
/// This implementation does not deduplicate redundant proofs.
|
||||
pub fn extend(&mut self, other: Self) {
|
||||
self.account_proofs.extend(other.account_proofs);
|
||||
for (hashed_address, other_storage_proofs) in other.storage_proofs {
|
||||
self.storage_proofs.entry(hashed_address).or_default().extend(other_storage_proofs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The merkle multiproof of storage trie.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct StorageMultiProof {
|
||||
|
||||
@@ -4,6 +4,7 @@ use crate::{
|
||||
};
|
||||
use alloc::{
|
||||
collections::{btree_map::BTreeMap, btree_set::BTreeSet},
|
||||
sync::Arc,
|
||||
vec::Vec,
|
||||
};
|
||||
use alloy_primitives::{
|
||||
@@ -206,7 +207,7 @@ impl TrieUpdates {
|
||||
}
|
||||
|
||||
/// Converts trie updates into [`TrieUpdatesSortedRef`].
|
||||
pub fn into_sorted_ref<'a>(&'a self) -> TrieUpdatesSortedRef<'a> {
|
||||
pub fn into_sorted_ref(&self) -> TrieUpdatesSortedRef<'_> {
|
||||
let mut account_nodes = self.account_nodes.iter().collect::<Vec<_>>();
|
||||
account_nodes.sort_unstable_by(|a, b| a.0.cmp(b.0));
|
||||
|
||||
@@ -216,7 +217,7 @@ impl TrieUpdates {
|
||||
storage_tries: self
|
||||
.storage_tries
|
||||
.iter()
|
||||
.map(|m| (*m.0, m.1.into_sorted_ref().clone()))
|
||||
.map(|m| (*m.0, m.1.into_sorted_ref()))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
@@ -605,7 +606,10 @@ impl TrieUpdatesSorted {
|
||||
/// This merges the account nodes and storage tries from `other` into `self`.
|
||||
/// Account nodes are merged and re-sorted, with `other`'s values taking precedence
|
||||
/// for duplicate keys.
|
||||
pub fn extend_ref(&mut self, other: &Self) {
|
||||
///
|
||||
/// Sorts the account nodes after extending. Sorts the storage tries after extending, for each
|
||||
/// storage trie.
|
||||
pub fn extend_ref_and_sort(&mut self, other: &Self) {
|
||||
// Extend account nodes
|
||||
extend_sorted_vec(&mut self.account_nodes, &other.account_nodes);
|
||||
|
||||
@@ -626,48 +630,57 @@ impl TrieUpdatesSorted {
|
||||
|
||||
/// Batch-merge sorted trie updates. Iterator yields **newest to oldest**.
|
||||
///
|
||||
/// This is more efficient than repeated `extend_ref` calls for large batches,
|
||||
/// using k-way merge for O(n log k) complexity instead of O(n * k).
|
||||
pub fn merge_batch<'a>(updates: impl IntoIterator<Item = &'a Self>) -> Self {
|
||||
let updates: Vec<_> = updates.into_iter().collect();
|
||||
if updates.is_empty() {
|
||||
return Self::default();
|
||||
/// For small batches, uses `extend_ref_and_sort` loop.
|
||||
/// For large batches, uses k-way merge for O(n log k) complexity.
|
||||
pub fn merge_batch<T: AsRef<Self> + From<Self>>(iter: impl IntoIterator<Item = T>) -> T {
|
||||
const THRESHOLD: usize = 30;
|
||||
|
||||
let items: alloc::vec::Vec<_> = iter.into_iter().collect();
|
||||
let k = items.len();
|
||||
|
||||
if k == 0 {
|
||||
return Self::default().into();
|
||||
}
|
||||
if k == 1 {
|
||||
return items.into_iter().next().expect("k == 1");
|
||||
}
|
||||
|
||||
// Merge account nodes using k-way merge. Newest (index 0) takes precedence.
|
||||
let account_nodes = kway_merge_sorted(updates.iter().map(|u| u.account_nodes.as_slice()));
|
||||
if k < THRESHOLD {
|
||||
// Small k: extend loop, oldest-to-newest so newer overrides older.
|
||||
let mut iter = items.iter().rev();
|
||||
let mut acc = iter.next().expect("k > 0").as_ref().clone();
|
||||
for next in iter {
|
||||
acc.extend_ref_and_sort(next.as_ref());
|
||||
}
|
||||
return acc.into();
|
||||
}
|
||||
|
||||
// Large k: k-way merge.
|
||||
let account_nodes =
|
||||
kway_merge_sorted(items.iter().map(|i| i.as_ref().account_nodes.as_slice()));
|
||||
|
||||
// Accumulator for collecting storage trie slices per address.
|
||||
// We process updates newest-to-oldest and stop collecting for an address
|
||||
// once we hit a "deleted" storage (sealed=true), since older data is irrelevant.
|
||||
struct StorageAcc<'a> {
|
||||
/// Storage trie was deleted (account removed or cleared).
|
||||
is_deleted: bool,
|
||||
/// Stop collecting older slices after seeing a deletion.
|
||||
sealed: bool,
|
||||
/// Storage trie node slices to merge, ordered newest to oldest.
|
||||
slices: Vec<&'a [(Nibbles, Option<BranchNodeCompact>)]>,
|
||||
}
|
||||
|
||||
let mut acc: B256Map<StorageAcc<'_>> = B256Map::default();
|
||||
|
||||
// Collect storage slices per address, respecting deletion boundaries
|
||||
for update in &updates {
|
||||
for (addr, storage) in &update.storage_tries {
|
||||
for item in &items {
|
||||
for (addr, storage) in &item.as_ref().storage_tries {
|
||||
let entry = acc.entry(*addr).or_insert_with(|| StorageAcc {
|
||||
is_deleted: false,
|
||||
sealed: false,
|
||||
slices: Vec::new(),
|
||||
});
|
||||
|
||||
// Skip if we already hit a deletion for this address (older data is irrelevant)
|
||||
if entry.sealed {
|
||||
continue;
|
||||
}
|
||||
|
||||
entry.slices.push(storage.storage_nodes.as_slice());
|
||||
|
||||
// If this storage was deleted, mark as deleted and seal to ignore older updates
|
||||
if storage.is_deleted {
|
||||
entry.is_deleted = true;
|
||||
entry.sealed = true;
|
||||
@@ -675,7 +688,6 @@ impl TrieUpdatesSorted {
|
||||
}
|
||||
}
|
||||
|
||||
// Merge each address's storage slices using k-way merge
|
||||
let storage_tries = acc
|
||||
.into_iter()
|
||||
.map(|(addr, entry)| {
|
||||
@@ -684,7 +696,37 @@ impl TrieUpdatesSorted {
|
||||
})
|
||||
.collect();
|
||||
|
||||
Self { account_nodes, storage_tries }
|
||||
Self { account_nodes, storage_tries }.into()
|
||||
}
|
||||
|
||||
/// Parallel batch-merge sorted trie updates. Slice is **oldest to newest**.
|
||||
///
|
||||
/// This is more efficient than sequential `extend_ref` calls when merging many updates,
|
||||
/// as it processes all updates in parallel with tree reduction using divide-and-conquer.
|
||||
#[cfg(feature = "rayon")]
|
||||
pub fn merge_parallel(updates: &[Arc<Self>]) -> Self {
|
||||
fn parallel_merge_tree(updates: &[Arc<TrieUpdatesSorted>]) -> TrieUpdatesSorted {
|
||||
match updates.len() {
|
||||
0 => TrieUpdatesSorted::default(),
|
||||
1 => updates[0].as_ref().clone(),
|
||||
2 => {
|
||||
let mut acc = updates[0].as_ref().clone();
|
||||
acc.extend_ref_and_sort(&updates[1]);
|
||||
acc
|
||||
}
|
||||
n => {
|
||||
let mid = n / 2;
|
||||
let (mut left, right) = rayon::join(
|
||||
|| parallel_merge_tree(&updates[..mid]),
|
||||
|| parallel_merge_tree(&updates[mid..]),
|
||||
);
|
||||
left.extend_ref_and_sort(&right);
|
||||
left
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
parallel_merge_tree(updates)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -834,7 +876,7 @@ mod tests {
|
||||
// Test extending with empty updates
|
||||
let mut updates1 = TrieUpdatesSorted::default();
|
||||
let updates2 = TrieUpdatesSorted::default();
|
||||
updates1.extend_ref(&updates2);
|
||||
updates1.extend_ref_and_sort(&updates2);
|
||||
assert_eq!(updates1.account_nodes.len(), 0);
|
||||
assert_eq!(updates1.storage_tries.len(), 0);
|
||||
|
||||
@@ -853,7 +895,7 @@ mod tests {
|
||||
],
|
||||
storage_tries: B256Map::default(),
|
||||
};
|
||||
updates1.extend_ref(&updates2);
|
||||
updates1.extend_ref_and_sort(&updates2);
|
||||
assert_eq!(updates1.account_nodes.len(), 3);
|
||||
// Should be sorted: 0x01, 0x02, 0x03
|
||||
assert_eq!(updates1.account_nodes[0].0, Nibbles::from_nibbles_unchecked([0x01]));
|
||||
@@ -889,7 +931,7 @@ mod tests {
|
||||
(hashed_address2, storage_trie1),
|
||||
]),
|
||||
};
|
||||
updates1.extend_ref(&updates2);
|
||||
updates1.extend_ref_and_sort(&updates2);
|
||||
assert_eq!(updates1.storage_tries.len(), 2);
|
||||
assert!(updates1.storage_tries.contains_key(&hashed_address1));
|
||||
assert!(updates1.storage_tries.contains_key(&hashed_address2));
|
||||
|
||||
@@ -87,7 +87,7 @@ where
|
||||
|
||||
// This reverts all changes from db tip back to just after block-1 was processed
|
||||
let mut cumulative_state_revert_prev = cumulative_state_revert.clone();
|
||||
cumulative_state_revert_prev.extend_ref(&individual_state_revert);
|
||||
cumulative_state_revert_prev.extend_ref_and_sort(&individual_state_revert);
|
||||
|
||||
// Step 2: Calculate cumulative trie updates revert for block-1
|
||||
// This gives us the trie state as it was after block-1 was processed
|
||||
@@ -469,7 +469,7 @@ impl ChangesetCache {
|
||||
// Since we iterate newest to oldest, older values are added last
|
||||
// and overwrite any conflicting newer values (oldest changeset values take
|
||||
// precedence).
|
||||
accumulated_reverts.extend_ref(&changesets);
|
||||
accumulated_reverts.extend_ref_and_sort(&changesets);
|
||||
}
|
||||
|
||||
let elapsed = timer.elapsed();
|
||||
|
||||
@@ -14,6 +14,7 @@ workspace = true
|
||||
[dependencies]
|
||||
# reth
|
||||
reth-execution-errors.workspace = true
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-storage-errors.workspace = true
|
||||
reth-trie-common.workspace = true
|
||||
|
||||
@@ -22,6 +22,12 @@ pub mod proof;
|
||||
|
||||
pub mod proof_task;
|
||||
|
||||
/// Async value encoder for V2 proofs.
|
||||
pub(crate) mod value_encoder;
|
||||
|
||||
/// V2 multiproof targets and chunking.
|
||||
pub mod targets_v2;
|
||||
|
||||
/// Parallel state root metrics.
|
||||
#[cfg(feature = "metrics")]
|
||||
pub mod metrics;
|
||||
|
||||
@@ -41,7 +41,7 @@ use alloy_primitives::{
|
||||
use alloy_rlp::{BufMut, Encodable};
|
||||
use crossbeam_channel::{unbounded, Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
|
||||
use dashmap::DashMap;
|
||||
use reth_execution_errors::{SparseTrieError, SparseTrieErrorKind};
|
||||
use reth_execution_errors::{SparseTrieError, SparseTrieErrorKind, StateProofError};
|
||||
use reth_provider::{DatabaseProviderROFactory, ProviderError, ProviderResult};
|
||||
use reth_storage_errors::db::DatabaseError;
|
||||
use reth_trie::{
|
||||
@@ -305,17 +305,17 @@ impl ProofWorkerHandle {
|
||||
self.storage_work_tx
|
||||
.send(StorageWorkerJob::StorageProof { input, proof_result_sender })
|
||||
.map_err(|err| {
|
||||
let error =
|
||||
ProviderError::other(std::io::Error::other("storage workers unavailable"));
|
||||
|
||||
if let StorageWorkerJob::StorageProof { proof_result_sender, .. } = err.0 {
|
||||
let _ = proof_result_sender.send(StorageProofResultMessage {
|
||||
hashed_address,
|
||||
result: Err(ParallelStateRootError::Provider(error.clone())),
|
||||
result: Err(DatabaseError::Other(
|
||||
"storage workers unavailable".to_string(),
|
||||
)
|
||||
.into()),
|
||||
});
|
||||
}
|
||||
|
||||
error
|
||||
ProviderError::other(std::io::Error::other("storage workers unavailable"))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -432,7 +432,7 @@ where
|
||||
input: StorageProofInput,
|
||||
trie_cursor_metrics: &mut TrieCursorMetricsCache,
|
||||
hashed_cursor_metrics: &mut HashedCursorMetricsCache,
|
||||
) -> Result<StorageProofResult, ParallelStateRootError> {
|
||||
) -> Result<StorageProofResult, StateProofError> {
|
||||
// Consume the input so we can move large collections (e.g. target slots) without cloning.
|
||||
let StorageProofInput::Legacy {
|
||||
hashed_address,
|
||||
@@ -469,20 +469,13 @@ where
|
||||
.with_added_removed_keys(added_removed_keys)
|
||||
.with_trie_cursor_metrics(trie_cursor_metrics)
|
||||
.with_hashed_cursor_metrics(hashed_cursor_metrics)
|
||||
.storage_multiproof(target_slots)
|
||||
.map_err(|e| ParallelStateRootError::Other(e.to_string()));
|
||||
.storage_multiproof(target_slots);
|
||||
trie_cursor_metrics.record_span("trie_cursor");
|
||||
hashed_cursor_metrics.record_span("hashed_cursor");
|
||||
|
||||
// Decode proof into DecodedStorageMultiProof
|
||||
let decoded_result = raw_proof_result.and_then(|raw_proof| {
|
||||
raw_proof.try_into().map_err(|e: alloy_rlp::Error| {
|
||||
ParallelStateRootError::Other(format!(
|
||||
"Failed to decode storage proof for {}: {}",
|
||||
hashed_address, e
|
||||
))
|
||||
})
|
||||
})?;
|
||||
let decoded_result =
|
||||
raw_proof_result.and_then(|raw_proof| raw_proof.try_into().map_err(Into::into))?;
|
||||
|
||||
trace!(
|
||||
target: "trie::proof_task",
|
||||
@@ -502,7 +495,7 @@ where
|
||||
<Provider as TrieCursorFactory>::StorageTrieCursor<'_>,
|
||||
<Provider as HashedCursorFactory>::StorageCursor<'_>,
|
||||
>,
|
||||
) -> Result<StorageProofResult, ParallelStateRootError> {
|
||||
) -> Result<StorageProofResult, StateProofError> {
|
||||
let StorageProofInput::V2 { hashed_address, mut targets } = input else {
|
||||
panic!("compute_v2_storage_proof only accepts StorageProofInput::V2")
|
||||
};
|
||||
@@ -717,12 +710,12 @@ pub struct StorageProofResultMessage {
|
||||
/// The hashed address this storage proof belongs to
|
||||
pub(crate) hashed_address: B256,
|
||||
/// The storage proof calculation result
|
||||
pub(crate) result: Result<StorageProofResult, ParallelStateRootError>,
|
||||
pub(crate) result: Result<StorageProofResult, StateProofError>,
|
||||
}
|
||||
|
||||
/// Internal message for storage workers.
|
||||
#[derive(Debug)]
|
||||
enum StorageWorkerJob {
|
||||
pub(crate) enum StorageWorkerJob {
|
||||
/// Storage proof computation request
|
||||
StorageProof {
|
||||
/// Storage proof input parameters
|
||||
@@ -1562,8 +1555,11 @@ fn dispatch_storage_proofs(
|
||||
let mut storage_proof_receivers =
|
||||
B256Map::with_capacity_and_hasher(targets.len(), Default::default());
|
||||
|
||||
let mut sorted_targets: Vec<_> = targets.iter().collect();
|
||||
sorted_targets.sort_unstable_by_key(|(addr, _)| *addr);
|
||||
|
||||
// Dispatch all storage proofs to worker pool
|
||||
for (hashed_address, target_slots) in targets.iter() {
|
||||
for (hashed_address, target_slots) in sorted_targets {
|
||||
// Create channel for receiving ProofResultMessage
|
||||
let (result_tx, result_rx) = crossbeam_channel::unbounded();
|
||||
|
||||
|
||||
148
crates/trie/parallel/src/targets_v2.rs
Normal file
148
crates/trie/parallel/src/targets_v2.rs
Normal file
@@ -0,0 +1,148 @@
|
||||
//! V2 multiproof targets and chunking.
|
||||
|
||||
use alloy_primitives::{map::B256Map, B256};
|
||||
use reth_trie::proof_v2;
|
||||
|
||||
/// A set of account and storage V2 proof targets. The account and storage targets do not need to
|
||||
/// necessarily overlap.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct MultiProofTargetsV2 {
|
||||
/// The set of account proof targets to generate proofs for.
|
||||
pub account_targets: Vec<proof_v2::Target>,
|
||||
/// The sets of storage proof targets to generate proofs for.
|
||||
pub storage_targets: B256Map<Vec<proof_v2::Target>>,
|
||||
}
|
||||
|
||||
impl MultiProofTargetsV2 {
|
||||
/// Returns true is there are no account or storage targets.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.account_targets.is_empty() && self.storage_targets.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// An iterator that yields chunks of V2 proof targets of at most `size` account and storage
|
||||
/// targets.
|
||||
///
|
||||
/// Unlike legacy chunking, V2 preserves account targets exactly as they were (with their `min_len`
|
||||
/// metadata). Account targets must appear in a chunk. Storage targets for those accounts are
|
||||
/// chunked together, but if they exceed the chunk size, subsequent chunks contain only the
|
||||
/// remaining storage targets without repeating the account target.
|
||||
#[derive(Debug)]
|
||||
pub struct ChunkedMultiProofTargetsV2 {
|
||||
/// Remaining account targets to process
|
||||
account_targets: std::vec::IntoIter<proof_v2::Target>,
|
||||
/// Storage targets by account address
|
||||
storage_targets: B256Map<Vec<proof_v2::Target>>,
|
||||
/// Current account being processed (if any storage slots remain)
|
||||
current_account_storage: Option<(B256, std::vec::IntoIter<proof_v2::Target>)>,
|
||||
/// Chunk size
|
||||
size: usize,
|
||||
}
|
||||
|
||||
impl ChunkedMultiProofTargetsV2 {
|
||||
/// Creates a new chunked iterator for the given targets.
|
||||
pub fn new(targets: MultiProofTargetsV2, size: usize) -> Self {
|
||||
Self {
|
||||
account_targets: targets.account_targets.into_iter(),
|
||||
storage_targets: targets.storage_targets,
|
||||
current_account_storage: None,
|
||||
size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Iterator for ChunkedMultiProofTargetsV2 {
|
||||
type Item = MultiProofTargetsV2;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
let mut chunk = MultiProofTargetsV2::default();
|
||||
let mut count = 0;
|
||||
|
||||
// First, finish any remaining storage slots from previous account
|
||||
if let Some((account_addr, ref mut storage_iter)) = self.current_account_storage {
|
||||
let remaining_capacity = self.size - count;
|
||||
let slots: Vec<_> = storage_iter.by_ref().take(remaining_capacity).collect();
|
||||
|
||||
count += slots.len();
|
||||
chunk.storage_targets.insert(account_addr, slots);
|
||||
|
||||
// If iterator is exhausted, clear current_account_storage
|
||||
if storage_iter.len() == 0 {
|
||||
self.current_account_storage = None;
|
||||
}
|
||||
}
|
||||
|
||||
// Process account targets and their storage
|
||||
while count < self.size {
|
||||
let Some(account_target) = self.account_targets.next() else {
|
||||
break;
|
||||
};
|
||||
|
||||
// Add the account target
|
||||
chunk.account_targets.push(account_target);
|
||||
count += 1;
|
||||
|
||||
// Check if this account has storage targets
|
||||
let account_addr = account_target.key();
|
||||
if let Some(storage_slots) = self.storage_targets.remove(&account_addr) {
|
||||
let remaining_capacity = self.size - count;
|
||||
|
||||
if storage_slots.len() <= remaining_capacity {
|
||||
// Optimization: We can take all slots, just move the vec
|
||||
count += storage_slots.len();
|
||||
chunk.storage_targets.insert(account_addr, storage_slots);
|
||||
} else {
|
||||
// We need to split the storage slots
|
||||
let mut storage_iter = storage_slots.into_iter();
|
||||
let slots_in_chunk: Vec<_> =
|
||||
storage_iter.by_ref().take(remaining_capacity).collect();
|
||||
count += slots_in_chunk.len();
|
||||
|
||||
chunk.storage_targets.insert(account_addr, slots_in_chunk);
|
||||
|
||||
// Save remaining storage slots for next chunk
|
||||
self.current_account_storage = Some((account_addr, storage_iter));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process any remaining storage-only entries (accounts not in account_targets)
|
||||
while let Some((account_addr, storage_slots)) = self.storage_targets.iter_mut().next() &&
|
||||
count < self.size
|
||||
{
|
||||
let account_addr = *account_addr;
|
||||
let storage_slots = std::mem::take(storage_slots);
|
||||
let remaining_capacity = self.size - count;
|
||||
|
||||
// Always remove from the map - if there are remaining slots they go to
|
||||
// current_account_storage
|
||||
self.storage_targets.remove(&account_addr);
|
||||
|
||||
if storage_slots.len() <= remaining_capacity {
|
||||
// Optimization: We can take all slots, just move the vec
|
||||
count += storage_slots.len();
|
||||
chunk.storage_targets.insert(account_addr, storage_slots);
|
||||
} else {
|
||||
// We need to split the storage slots
|
||||
let mut storage_iter = storage_slots.into_iter();
|
||||
let slots_in_chunk: Vec<_> =
|
||||
storage_iter.by_ref().take(remaining_capacity).collect();
|
||||
|
||||
chunk.storage_targets.insert(account_addr, slots_in_chunk);
|
||||
|
||||
// Save remaining storage slots for next chunk
|
||||
if storage_iter.len() > 0 {
|
||||
self.current_account_storage = Some((account_addr, storage_iter));
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if chunk.account_targets.is_empty() && chunk.storage_targets.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(chunk)
|
||||
}
|
||||
}
|
||||
}
|
||||
185
crates/trie/parallel/src/value_encoder.rs
Normal file
185
crates/trie/parallel/src/value_encoder.rs
Normal file
@@ -0,0 +1,185 @@
|
||||
use crate::proof_task::{
|
||||
StorageProofInput, StorageProofResult, StorageProofResultMessage, StorageWorkerJob,
|
||||
};
|
||||
use alloy_primitives::{map::B256Map, B256};
|
||||
use alloy_rlp::Encodable;
|
||||
use core::cell::RefCell;
|
||||
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
|
||||
use dashmap::DashMap;
|
||||
use reth_execution_errors::trie::StateProofError;
|
||||
use reth_primitives_traits::Account;
|
||||
use reth_storage_errors::db::DatabaseError;
|
||||
use reth_trie::{
|
||||
proof_v2::{DeferredValueEncoder, LeafValueEncoder, Target},
|
||||
ProofTrieNode,
|
||||
};
|
||||
use std::{rc::Rc, sync::Arc};
|
||||
|
||||
/// Returned from [`AsyncAccountValueEncoder`], used to track an async storage root calculation.
|
||||
pub(crate) enum AsyncAccountDeferredValueEncoder {
|
||||
Dispatched {
|
||||
hashed_address: B256,
|
||||
account: Account,
|
||||
proof_result_rx: Result<CrossbeamReceiver<StorageProofResultMessage>, DatabaseError>,
|
||||
// None if results shouldn't be retained for this dispatched proof.
|
||||
storage_proof_results: Option<Rc<RefCell<B256Map<Vec<ProofTrieNode>>>>>,
|
||||
},
|
||||
FromCache {
|
||||
account: Account,
|
||||
root: B256,
|
||||
},
|
||||
}
|
||||
|
||||
impl DeferredValueEncoder for AsyncAccountDeferredValueEncoder {
|
||||
fn encode(self, buf: &mut Vec<u8>) -> Result<(), StateProofError> {
|
||||
let (account, root) = match self {
|
||||
Self::Dispatched {
|
||||
hashed_address,
|
||||
account,
|
||||
proof_result_rx,
|
||||
storage_proof_results,
|
||||
} => {
|
||||
let result = proof_result_rx?
|
||||
.recv()
|
||||
.map_err(|_| {
|
||||
StateProofError::Database(DatabaseError::Other(format!(
|
||||
"Storage proof channel closed for {hashed_address:?}",
|
||||
)))
|
||||
})?
|
||||
.result?;
|
||||
|
||||
let StorageProofResult::V2 { root: Some(root), proof } = result else {
|
||||
panic!("StorageProofResult is not V2 with root: {result:?}")
|
||||
};
|
||||
|
||||
if let Some(storage_proof_results) = storage_proof_results.as_ref() {
|
||||
storage_proof_results.borrow_mut().insert(hashed_address, proof);
|
||||
}
|
||||
|
||||
(account, root)
|
||||
}
|
||||
Self::FromCache { account, root } => (account, root),
|
||||
};
|
||||
|
||||
let account = account.into_trie_account(root);
|
||||
account.encode(buf);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Implements the [`LeafValueEncoder`] trait for accounts using a [`CrossbeamSender`] to dispatch
|
||||
/// and compute storage roots asynchronously. Can also accept a set of already dispatched account
|
||||
/// storage proofs, for cases where it's possible to determine some necessary accounts ahead of
|
||||
/// time.
|
||||
pub(crate) struct AsyncAccountValueEncoder {
|
||||
storage_work_tx: CrossbeamSender<StorageWorkerJob>,
|
||||
/// Storage proof jobs which were dispatched ahead of time.
|
||||
dispatched: B256Map<CrossbeamReceiver<StorageProofResultMessage>>,
|
||||
/// Storage roots which have already been computed. This can be used only if a storage proof
|
||||
/// wasn't dispatched for an account, otherwise we must consume the proof result.
|
||||
cached_storage_roots: Arc<DashMap<B256, B256>>,
|
||||
/// Tracks storage proof results received from the storage workers. [`Rc`] + [`RefCell`] is
|
||||
/// required because [`DeferredValueEncoder`] cannot have a lifetime.
|
||||
storage_proof_results: Rc<RefCell<B256Map<Vec<ProofTrieNode>>>>,
|
||||
}
|
||||
|
||||
impl AsyncAccountValueEncoder {
|
||||
/// Initializes a [`Self`] using a `ProofWorkerHandle` which will be used to calculate storage
|
||||
/// roots asynchronously.
|
||||
#[expect(dead_code)]
|
||||
pub(crate) fn new(
|
||||
storage_work_tx: CrossbeamSender<StorageWorkerJob>,
|
||||
dispatched: B256Map<CrossbeamReceiver<StorageProofResultMessage>>,
|
||||
cached_storage_roots: Arc<DashMap<B256, B256>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
storage_work_tx,
|
||||
dispatched,
|
||||
cached_storage_roots,
|
||||
storage_proof_results: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Consume [`Self`] and return all collected storage proofs which had been dispatched.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// This method panics if any deferred encoders produced by [`Self::deferred_encoder`] have not
|
||||
/// been dropped.
|
||||
#[expect(dead_code)]
|
||||
pub(crate) fn into_storage_proofs(
|
||||
self,
|
||||
) -> Result<B256Map<Vec<ProofTrieNode>>, StateProofError> {
|
||||
let mut storage_proof_results = Rc::into_inner(self.storage_proof_results)
|
||||
.expect("no deferred encoders are still allocated")
|
||||
.into_inner();
|
||||
|
||||
// Any remaining dispatched proofs need to have their results collected
|
||||
for (hashed_address, rx) in &self.dispatched {
|
||||
let result = rx
|
||||
.recv()
|
||||
.map_err(|_| {
|
||||
StateProofError::Database(DatabaseError::Other(format!(
|
||||
"Storage proof channel closed for {hashed_address:?}",
|
||||
)))
|
||||
})?
|
||||
.result?;
|
||||
|
||||
let StorageProofResult::V2 { proof, .. } = result else {
|
||||
panic!("StorageProofResult is not V2: {result:?}")
|
||||
};
|
||||
|
||||
storage_proof_results.insert(*hashed_address, proof);
|
||||
}
|
||||
|
||||
Ok(storage_proof_results)
|
||||
}
|
||||
}
|
||||
|
||||
impl LeafValueEncoder for AsyncAccountValueEncoder {
|
||||
type Value = Account;
|
||||
type DeferredEncoder = AsyncAccountDeferredValueEncoder;
|
||||
|
||||
fn deferred_encoder(
|
||||
&mut self,
|
||||
hashed_address: B256,
|
||||
account: Self::Value,
|
||||
) -> Self::DeferredEncoder {
|
||||
// If the proof job has already been dispatched for this account then it's not necessary to
|
||||
// dispatch another.
|
||||
if let Some(rx) = self.dispatched.remove(&hashed_address) {
|
||||
return AsyncAccountDeferredValueEncoder::Dispatched {
|
||||
hashed_address,
|
||||
account,
|
||||
proof_result_rx: Ok(rx),
|
||||
storage_proof_results: Some(self.storage_proof_results.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
// If the address didn't have a job dispatched for it then we can assume it has no targets,
|
||||
// and we only need its root.
|
||||
|
||||
// If the root is already calculated then just use it directly
|
||||
if let Some(root) = self.cached_storage_roots.get(&hashed_address) {
|
||||
return AsyncAccountDeferredValueEncoder::FromCache { account, root: *root }
|
||||
}
|
||||
|
||||
// Create a proof input which targets a bogus key, so that we calculate the root as a
|
||||
// side-effect.
|
||||
let input = StorageProofInput::new(hashed_address, vec![Target::new(B256::ZERO)]);
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
let proof_result_rx = self
|
||||
.storage_work_tx
|
||||
.send(StorageWorkerJob::StorageProof { input, proof_result_sender: tx })
|
||||
.map_err(|_| DatabaseError::Other("storage workers unavailable".to_string()))
|
||||
.map(|_| rx);
|
||||
|
||||
AsyncAccountDeferredValueEncoder::Dispatched {
|
||||
hashed_address,
|
||||
account,
|
||||
proof_result_rx,
|
||||
storage_proof_results: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user