Compare commits

..

40 Commits

Author SHA1 Message Date
Georgios Konstantopoulos
d701d70dc1 perf(db): disable prefault_write to avoid mincore() syscall overhead
In WRITEMAP mode, MDBX by default calls mincore() before each page write to check if
the page is resident in memory. Profiling shows this accounts for ~20-22% of
persistence time, yet pages are usually already warm from Engine Task reads.

Disabling prefault_write skips the mincore() syscall and lets the kernel handle
page faults directly. Since pages are likely resident, this trades potential
page faults (which rarely occur due to cache warmth) for syscall overhead
elimination.

Amp-Thread-ID: https://ampcode.com/threads/T-019c233a-3fa9-733b-8985-991c393eb610
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 11:23:57 +00:00
Arsenii Kulikov
53f922927a feat: reintroduce --engine.state-root-task-compare-updates (#21717) 2026-02-02 23:48:54 +00:00
Dan Cline
f1f3980d29 fix(cli): actually enable reth-prune rocksdb feature in cli (#21715) 2026-02-02 23:39:04 +00:00
Dan Cline
6946f26d77 fix(cli): delete all static files when PruneModes::Full is configured (#21647) 2026-02-02 17:30:21 +00:00
Arsenii Kulikov
f663d1d110 fix: properly drain pending account updates (#21709) 2026-02-02 17:29:43 +00:00
Huber
f4943abf73 chore(ci): add consts to typos allowlist (#21708) 2026-02-02 17:02:16 +00:00
Matthias Seitz
102a6944ba perf(trie): avoid clearing already-cached sparse trie (#21702)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-02 13:03:07 +00:00
Alexey Shekhirin
1592e51d34 feat(engine): add CLI args for sparse trie pruning configuration (#21703) 2026-02-02 12:52:31 +00:00
Arsenii Kulikov
4280ccf470 fix: short-circuit in reveal_account_v2_proof_nodes on empty nodes (#21701) 2026-02-02 12:18:45 +00:00
Alexey Shekhirin
05ab98107c fix(reth-bench): gracefully stop when transaction source exhausted (#21700) 2026-02-02 11:10:58 +00:00
Brian Picciano
49128ed28f fix(trie): Return full_key from update_leaves unless it is not a child of the missing path (#21699)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-02-02 11:07:56 +00:00
Huber
f74e594292 perf(trie): dispatch V2 storage proofs in lexicographical order (#21684) 2026-02-02 09:31:47 +00:00
Georgios Konstantopoulos
e7d4a05e36 perf(trie): fix allocation hot paths with capacity hints and buffer reuse (#21466)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
2026-02-02 06:58:45 +00:00
Georgios Konstantopoulos
9382a4c713 fix(prune): use batched pruning loop with edge feature to prevent OOM (#21649)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-02 02:38:00 +00:00
DaniPopes
28409558f9 perf: add ParallelBridgeBuffered trait to replace par_bridge (#21674) 2026-02-02 00:58:43 +00:00
DaniPopes
5ef32726db refactor: add with_* compressor utility methods (#21680) 2026-02-01 20:43:25 +00:00
Snezhkko
60c3bef1e8 fix(zstd): use transaction dictionary for tx compressor (#21382) 2026-02-01 20:12:51 +00:00
iPLAY888
af96eeae56 refactor(provider): deduplicate segment-to-stage mapping in static file manager (#21670) 2026-02-01 20:09:32 +00:00
Georgios Konstantopoulos
5528aae8f6 fix(engine): wait for persistence service thread before RocksDB drop (#21640)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-02-01 19:55:45 +00:00
Georgios Konstantopoulos
83364aa2d6 fix(prune): migrate invalid receipts prune config to Distance(64) (#21677)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 19:44:14 +00:00
DaniPopes
749a742bcf chore(deps): update metrics-derive 0.1.1 (#21673) 2026-02-01 19:38:38 +00:00
ethfanWilliam
2970624413 chore: avoid eager evaluation in base_fee_params_at_timestamp (#21536) 2026-02-01 19:04:42 +00:00
Matthias Seitz
7e18aa4be8 fix(rpc): change debug_set_head number parameter to U64 (#21678)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 18:59:22 +00:00
YK
9f8c22e2c3 feat(prune): prune rocksdb account and storage history indices (#21331)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-02-01 18:42:17 +00:00
Georgios Konstantopoulos
3d699ac9c6 perf(trie): reuse account RLP buffer in SparseTrieCacheTask (#21644)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 15:20:11 +00:00
かりんとう
9be31d504d fix(trie): silence unused param warnings in sparse-parallel no_std build (#21657) 2026-02-01 13:05:39 +00:00
github-actions[bot]
34cc65cfe6 chore(deps): weekly cargo update (#21660)
Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com>
2026-02-01 13:03:13 +00:00
Matthias Seitz
6e161f0fc9 perf: batch finalized/safe block commits with SaveBlocks (#21663)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 13:02:59 +00:00
iPLAY888
63a3e18404 fix: remove unnecessary alloc (#21665) 2026-02-01 13:01:11 +00:00
Matthias Seitz
7d10e791b2 refactor(engine): improve payload processor tx iterator (#21658)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 12:44:10 +00:00
Georgios Konstantopoulos
a9b2c1d454 feat(rpc): make blob sidecar upcasting opt-in (#21624)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 12:25:46 +00:00
CPerezz
9127563914 fix: cleanup entire temp directory when using testing_node (#18399)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-31 16:46:11 +00:00
Georgios Konstantopoulos
a500fb22ba fix(metrics): rename save_blocks_block_count to save_blocks_batch_size (#21654)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-31 12:59:09 +00:00
Matthias Seitz
e869cd4670 perf(engine): skip DB lookup for new blocks in insert_block_or_payload (#21650)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-31 03:35:20 +00:00
DaniPopes
de69654b73 chore(deps): breaking bumps (#21584)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-31 00:44:09 +00:00
DaniPopes
8d28c4c8f2 chore(trie): add set_* methods alongside with_* builders (#21639)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 22:42:57 +00:00
Georgios Konstantopoulos
bfe778ab51 perf(trie): use Entry API to avoid empty Vec allocation in extend (#21645)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 22:29:21 +00:00
DaniPopes
e523a76fb8 chore(trie): clear RevealableSparseTrie in place (#21638)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 22:27:43 +00:00
DaniPopes
cd12ae58f2 docs(CLAUDE.md): tweaks (#21646) 2026-01-30 22:26:34 +00:00
Georgios Konstantopoulos
370a548f34 refactor(db): derive Clone for DatabaseEnv (#21641)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 21:54:50 +00:00
128 changed files with 3010 additions and 1911 deletions

View File

@@ -38,7 +38,7 @@ Reth is a high-performance Ethereum execution client written in Rust, focusing o
2. **Linting**: Run clippy with all features
```bash
RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features --locked
cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features
```
3. **Testing**: Use nextest for faster test execution
@@ -169,12 +169,11 @@ Based on PR patterns, avoid:
Before submitting changes, ensure:
1. **Format Check**: `cargo +nightly fmt --all --check`
2. **Clippy**: No warnings with `RUSTFLAGS="-D warnings"`
2. **Clippy**: No warnings
3. **Tests Pass**: All unit and integration tests
4. **Documentation**: Update relevant docs and add doc comments with `cargo docs --document-private-items`
5. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
### Opening PRs against <https://github.com/paradigmxyz/reth>
Label PRs appropriately, first check the available labels and then apply the relevant ones:
@@ -349,10 +348,10 @@ Let's say you want to fix a bug where external IP resolution fails on startup:
}
```
5. **Run checks**:
5. **Run checks** (IMPORTANT!):
```bash
cargo +nightly fmt --all
cargo clippy --all-features
cargo clippy --workspace --all-features # Make sure WHOLE WORKSPACE compiles!
cargo test -p reth-discv4
```
@@ -374,7 +373,7 @@ Let's say you want to fix a bug where external IP resolution fails on startup:
cargo +nightly fmt --all
# Run lints
RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --all-features --locked
cargo +nightly clippy --workspace --all-features
# Run tests
cargo nextest run --workspace

631
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -560,7 +560,7 @@ humantime-serde = "1.1"
itertools = { version = "0.14", default-features = false }
linked_hash_set = "0.1"
lz4 = "1.28.1"
modular-bitfield = "0.11.2"
modular-bitfield = "0.13.1"
notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] }
nybbles = { version = "0.4.2", default-features = false }
once_cell = { version = "1.19", default-features = false, features = ["critical-section"] }
@@ -589,13 +589,13 @@ zstd = "0.13"
byteorder = "1"
fixed-cache = { version = "0.1.7", features = ["stats"] }
moka = "0.12"
tar-no-std = { version = "0.3.2", default-features = false }
miniz_oxide = { version = "0.8.4", default-features = false }
tar-no-std = { version = "0.4.2", default-features = false }
miniz_oxide = { version = "0.9.0", default-features = false }
chrono = "0.4.41"
# metrics
metrics = "0.24.0"
metrics-derive = "0.1"
metrics-derive = "0.1.1"
metrics-exporter-prometheus = { version = "0.18.0", default-features = false }
metrics-process = "2.1.0"
metrics-util = { default-features = false, version = "0.20.0" }
@@ -607,7 +607,7 @@ quote = "1.0"
# tokio
tokio = { version = "1.44.2", default-features = false }
tokio-stream = "0.1.11"
tokio-tungstenite = "0.26.2"
tokio-tungstenite = "0.28.0"
tokio-util = { version = "0.7.4", features = ["codec"] }
# async
@@ -620,7 +620,7 @@ futures-util = { version = "0.3", default-features = false }
hyper = "1.3"
hyper-util = "0.1.5"
pin-project = "1.0.12"
reqwest = { version = "0.12", default-features = false }
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "rustls-tls-native-roots", "stream"] }
tracing-futures = "0.2"
tower = "0.5"
tower-http = "0.6"
@@ -640,7 +640,6 @@ jsonrpsee-types = "0.26.0"
http = "1.0"
http-body = "1.0"
http-body-util = "0.1.2"
jsonwebtoken = "9"
proptest-arbitrary-interop = "0.1.0"
# crypto
@@ -654,7 +653,7 @@ rand_08 = { package = "rand", version = "0.8" }
c-kzg = "2.1.5"
# config
toml = "0.8"
toml = "0.9"
# rocksdb
rocksdb = { version = "0.24" }
@@ -673,16 +672,16 @@ assert_matches = "1.5.0"
criterion = { package = "codspeed-criterion-compat", version = "4.3" }
insta = "1.41"
proptest = "1.7"
proptest-derive = "0.5"
proptest-derive = "0.7"
similar-asserts = { version = "1.5.0", features = ["serde"] }
tempfile = "3.20"
test-fuzz = "7"
rstest = "0.24.0"
rstest = "0.26.1"
test-case = "3"
# ssz encoding
ethereum_ssz = "0.9.0"
ethereum_ssz_derive = "0.9.0"
ethereum_ssz = "0.10.1"
ethereum_ssz_derive = "0.10.1"
# allocators
jemalloc_pprof = { version = "0.8", default-features = false }
@@ -694,14 +693,14 @@ snmalloc-rs = { version = "0.3.7", features = ["build_cc"] }
aes = "0.8.1"
ahash = "0.8"
anyhow = "1.0"
bindgen = { version = "0.71", default-features = false }
block-padding = "0.3.2"
bindgen = { version = "0.72", default-features = false }
block-padding = "0.3"
cc = "1.2.15"
cipher = "0.4.3"
comfy-table = "7.0"
concat-kdf = "0.1.0"
crossbeam-channel = "0.5.13"
crossterm = "0.28.0"
crossterm = "0.29.0"
csv = "1.3.0"
ctrlc = "3.4"
ctr = "0.9.2"
@@ -714,7 +713,7 @@ hmac = "0.12.1"
human_bytes = "0.4.1"
indexmap = "2"
interprocess = "2.2.0"
lz4_flex = { version = "0.11", default-features = false }
lz4_flex = { version = "0.12", default-features = false }
memmap2 = "0.9.4"
mev-share-sse = { version = "0.5.0", default-features = false }
num-traits = "0.2.15"
@@ -722,15 +721,15 @@ page_size = "0.6.0"
parity-scale-codec = "3.2.1"
plain_hasher = "0.2"
pretty_assertions = "1.4"
ratatui = { version = "0.29", default-features = false }
ringbuffer = "0.15.0"
ratatui = { version = "0.30", default-features = false }
ringbuffer = "0.16.0"
rmp-serde = "1.3"
roaring = "0.10.2"
roaring = "0.11.3"
rolling-file = "0.2.0"
sha3 = "0.10.5"
snap = "1.1.1"
socket2 = { version = "0.5", default-features = false }
sysinfo = { version = "0.33", default-features = false }
socket2 = { version = "0.6", default-features = false }
sysinfo = { version = "0.38", default-features = false }
tracing-journald = "0.3"
tracing-logfmt = "=0.3.5"
tracing-samply = "0.1"

View File

@@ -56,7 +56,7 @@ ctrlc.workspace = true
shlex.workspace = true
[target.'cfg(unix)'.dependencies]
nix = { version = "0.29", features = ["signal", "process"] }
nix = { version = "0.31", features = ["signal", "process"] }
[features]
default = ["jemalloc"]

View File

@@ -45,7 +45,7 @@ op-alloy-consensus = { workspace = true, features = ["alloy-compat"] }
op-alloy-rpc-types-engine = { workspace = true, features = ["serde"] }
# reqwest
reqwest = { workspace = true, default-features = false, features = ["rustls-tls-native-roots"] }
reqwest.workspace = true
# tower
tower.workspace = true

View File

@@ -572,13 +572,22 @@ impl Command {
for i in 0..self.count {
// Get initial batch of transactions for this payload
let mut result = tx_buffer
.take_batch()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
let Some(mut result) = tx_buffer.take_batch().await else {
info!(
payloads_built = i,
payloads_requested = self.count,
"Transaction source exhausted, stopping"
);
break;
};
if result.transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 1));
info!(
payloads_built = i,
payloads_requested = self.count,
"No more transactions available, stopping"
);
break;
}
// Build with retry - may need to request more transactions

View File

@@ -541,7 +541,7 @@ impl<H: BlockHeader> ChainSpec<H> {
}
}
bf_params.first().map(|(_, params)| *params).unwrap_or(BaseFeeParams::ethereum())
bf_params.first().map(|(_, params)| *params).unwrap_or_else(BaseFeeParams::ethereum)
}
}
}

View File

@@ -133,4 +133,4 @@ arbitrary = [
"reth-ethereum-primitives/arbitrary",
]
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb"]
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb", "reth-prune/rocksdb"]

View File

@@ -121,13 +121,13 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
let genesis_block_number = self.chain.genesis().number.unwrap_or_default();
let (db, sfp) = match access {
AccessRights::RW => (
Arc::new(init_db(db_path, self.db.database_args())?),
init_db(db_path, self.db.database_args())?,
StaticFileProviderBuilder::read_write(sf_path)
.with_genesis_block_number(genesis_block_number)
.build()?,
),
AccessRights::RO | AccessRights::RoInconsistent => {
(Arc::new(open_db_read_only(&db_path, self.db.database_args())?), {
(open_db_read_only(&db_path, self.db.database_args())?, {
let provider = StaticFileProviderBuilder::read_only(sf_path)
.with_genesis_block_number(genesis_block_number)
.build()?;
@@ -160,16 +160,16 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
fn create_provider_factory<N: CliNodeTypes>(
&self,
config: &Config,
db: Arc<DatabaseEnv>,
db: DatabaseEnv,
static_file_provider: StaticFileProvider<N::Primitives>,
rocksdb_provider: RocksDBProvider,
access: AccessRights,
) -> eyre::Result<ProviderFactory<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>>
) -> eyre::Result<ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>>
where
C: ChainSpecParser<ChainSpec = N::ChainSpec>,
{
let prune_modes = config.prune.segments.clone();
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>::new(
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::new(
db,
self.chain.clone(),
static_file_provider,
@@ -200,7 +200,7 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
let (_tip_tx, tip_rx) = watch::channel(B256::ZERO);
// Builds and executes an unwind-only pipeline
let mut pipeline = Pipeline::<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>::builder()
let mut pipeline = Pipeline::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::builder()
.add_stages(DefaultStages::new(
factory.clone(),
tip_rx,
@@ -229,7 +229,7 @@ pub struct Environment<N: NodeTypes> {
/// Configuration for reth node
pub config: Config,
/// Provider factory.
pub provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
pub provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
/// Datadir path.
pub data_dir: ChainPath<DataDirPath>,
}
@@ -261,8 +261,8 @@ impl AccessRights {
/// Helper alias to satisfy `FullNodeTypes` bound on [`Node`] trait generic.
type FullTypesAdapter<T> = FullNodeTypesAdapter<
T,
Arc<DatabaseEnv>,
BlockchainProvider<NodeTypesWithDBAdapter<T, Arc<DatabaseEnv>>>,
DatabaseEnv,
BlockchainProvider<NodeTypesWithDBAdapter<T, DatabaseEnv>>,
>;
/// Helper trait with a common set of requirements for the

View File

@@ -17,7 +17,6 @@ use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProvider
use reth_static_file_types::StaticFileSegment;
use std::{
hash::{BuildHasher, Hasher},
sync::Arc,
time::{Duration, Instant},
};
use tracing::{info, warn};
@@ -90,7 +89,7 @@ impl Command {
/// Execute `db checksum` command
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
) -> eyre::Result<()> {
warn!("This command should be run without the node running!");
@@ -117,7 +116,7 @@ fn checksum_hasher() -> impl Hasher {
}
fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
segment: StaticFileSegment,
start_block: Option<u64>,
end_block: Option<u64>,

View File

@@ -9,7 +9,7 @@ use reth_db_api::table::Table;
use reth_db_common::DbTool;
use reth_node_builder::NodeTypesWithDBAdapter;
use reth_provider::RocksDBProviderFactory;
use std::{hash::Hasher, sync::Arc, time::Instant};
use std::{hash::Hasher, time::Instant};
use tracing::info;
/// RocksDB tables that can be checksummed.
@@ -36,7 +36,7 @@ impl RocksDbTable {
/// Computes a checksum for a RocksDB table.
pub fn checksum_rocksdb<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
table: RocksDbTable,
limit: Option<usize>,
) -> eyre::Result<()> {

View File

@@ -16,7 +16,6 @@ use std::{
hash::Hash,
io::Write,
path::{Path, PathBuf},
sync::Arc,
};
use tracing::{info, warn};
@@ -56,7 +55,7 @@ impl Command {
/// then written to a file in the output directory.
pub fn execute<T: NodeTypes>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<T, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<T, DatabaseEnv>>,
) -> eyre::Result<()> {
warn!("Make sure the node is not running when running `reth db diff`!");
// open second db

View File

@@ -7,7 +7,7 @@ use reth_db::{transaction::DbTx, DatabaseEnv};
use reth_db_api::{database::Database, table::Table, RawValue, TableViewer, Tables};
use reth_db_common::{DbTool, ListFilter};
use reth_node_builder::{NodeTypes, NodeTypesWithDBAdapter};
use std::{cell::RefCell, sync::Arc};
use std::cell::RefCell;
use tracing::error;
#[derive(Parser, Debug)]
@@ -55,7 +55,7 @@ impl Command {
/// Execute `db list` command
pub fn execute<N: NodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
) -> eyre::Result<()> {
self.table.view(&ListTableViewer { tool, args: &self })
}
@@ -89,7 +89,7 @@ impl Command {
}
struct ListTableViewer<'a, N: NodeTypes> {
tool: &'a DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &'a DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
args: &'a Command,
}

View File

@@ -16,7 +16,7 @@ use reth_provider::{
RocksDBProviderFactory,
};
use reth_static_file_types::SegmentRangeInclusive;
use std::{sync::Arc, time::Duration};
use std::time::Duration;
#[derive(Parser, Debug)]
/// The arguments for the `reth db stats` command
@@ -48,7 +48,7 @@ impl Command {
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
data_dir: ChainPath<DataDirPath>,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
) -> eyre::Result<()> {
if self.checksum {
let checksum_report = self.checksum_report(tool)?;
@@ -72,7 +72,7 @@ impl Command {
Ok(())
}
fn db_stats_table<N: NodeTypesWithDB<DB = Arc<DatabaseEnv>>>(
fn db_stats_table<N: NodeTypesWithDB<DB = DatabaseEnv>>(
&self,
tool: &DbTool<N>,
) -> eyre::Result<ComfyTable> {

View File

@@ -227,8 +227,9 @@ where
// Handle errors
if let Err(err) = res {
error!("{:?}", err)
error!("{err}");
}
Ok(())
}
}
@@ -241,6 +242,7 @@ fn event_loop<B: Backend, F, T: Table>(
) -> io::Result<()>
where
F: FnMut(usize, usize) -> Vec<TableRow<T>>,
io::Error: From<B::Error>,
{
let mut last_tick = Instant::now();
let mut running = true;

View File

@@ -2,7 +2,7 @@ use futures::Future;
use reth_cli::chainspec::ChainSpecParser;
use reth_db::DatabaseEnv;
use reth_node_builder::{NodeBuilder, WithLaunchContext};
use std::{fmt, sync::Arc};
use std::fmt;
/// A trait for launching a reth node with custom configuration strategies.
///
@@ -30,7 +30,7 @@ where
/// * `builder_args` - Extension arguments for configuration
fn entrypoint(
self,
builder: WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
builder: WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
builder_args: Ext,
) -> impl Future<Output = eyre::Result<()>>;
}
@@ -58,7 +58,7 @@ impl<F> FnLauncher<F> {
where
C: ChainSpecParser,
F: AsyncFnOnce(
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
Ext,
) -> eyre::Result<()>,
{
@@ -77,13 +77,13 @@ where
C: ChainSpecParser,
Ext: clap::Args + fmt::Debug,
F: AsyncFnOnce(
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
Ext,
) -> eyre::Result<()>,
{
fn entrypoint(
self,
builder: WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
builder: WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
builder_args: Ext,
) -> impl Future<Output = eyre::Result<()>> {
(self.func)(builder, builder_args)

View File

@@ -206,7 +206,7 @@ where
let db_path = data_dir.db();
tracing::info!(target: "reth::cli", path = ?db_path, "Opening database");
let database = Arc::new(init_db(db_path.clone(), self.db.database_args())?.with_metrics());
let database = init_db(db_path.clone(), self.db.database_args())?.with_metrics();
if with_unused_ports {
node_config = node_config.with_unused_ports();

View File

@@ -1,4 +1,4 @@
//! Command that runs pruning without any limits.
//! Command that runs pruning.
use crate::common::{AccessRights, CliNodeTypes, EnvironmentArgs};
use clap::Parser;
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks};
@@ -16,7 +16,7 @@ use reth_static_file::StaticFileProducer;
use std::sync::Arc;
use tracing::info;
/// Prunes according to the configuration without any limits
/// Prunes according to the configuration
#[derive(Debug, Parser)]
pub struct PruneCommand<C: ChainSpecParser> {
#[command(flatten)]
@@ -69,13 +69,43 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
// Delete data which has been copied to static files.
if let Some(prune_tip) = lowest_static_file_height {
info!(target: "reth::cli", ?prune_tip, ?config, "Pruning data from database...");
// Run the pruner according to the configuration, and don't enforce any limits on it
// Use batched pruning with a limit to bound memory, running in a loop until complete.
const DELETE_LIMIT: usize = 200_000;
let mut pruner = PrunerBuilder::new(config)
.delete_limit(usize::MAX)
.delete_limit(DELETE_LIMIT)
.build_with_provider_factory(provider_factory);
pruner.run(prune_tip)?;
info!(target: "reth::cli", "Pruned data from database");
let mut total_pruned = 0usize;
loop {
let output = pruner.run(prune_tip)?;
let batch_pruned: usize = output.segments.iter().map(|(_, seg)| seg.pruned).sum();
total_pruned = total_pruned.saturating_add(batch_pruned);
// Check if all segments are finished (not just the overall progress,
// since the pruner sets overall progress from the last segment only)
let all_segments_finished =
output.segments.iter().all(|(_, seg)| seg.progress.is_finished());
if all_segments_finished {
break;
}
if batch_pruned == 0 {
return Err(eyre::eyre!(
"pruner made no progress but reported more data remaining; \
aborting to prevent infinite loop"
));
}
info!(
target: "reth::cli",
batch_pruned,
total_pruned,
"Pruning batch complete, continuing..."
);
}
info!(target: "reth::cli", total_pruned, "Pruned data from database");
}
Ok(())

View File

@@ -26,7 +26,7 @@ pub(crate) async fn dump_execution_stage<N, E, C>(
consensus: C,
) -> eyre::Result<()>
where
N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>,
N: ProviderNodeTypes<DB = DatabaseEnv>,
E: ConfigureEvm<Primitives = N::Primitives> + 'static,
C: FullConsensus<E::Primitives> + 'static,
{
@@ -39,7 +39,7 @@ where
if should_run {
dry_run(
ProviderFactory::<N>::new(
Arc::new(output_db),
output_db,
db_tool.chain(),
StaticFileProvider::read_write(output_datadir.static_files())?,
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,

View File

@@ -10,10 +10,9 @@ use reth_provider::{
DatabaseProviderFactory, ProviderFactory,
};
use reth_stages::{stages::AccountHashingStage, Stage, StageCheckpoint, UnwindInput};
use std::sync::Arc;
use tracing::info;
pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = DatabaseEnv>>(
db_tool: &DbTool<N>,
from: BlockNumber,
to: BlockNumber,
@@ -36,7 +35,7 @@ pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = Arc<Dat
if should_run {
dry_run(
ProviderFactory::<N>::new(
Arc::new(output_db),
output_db,
db_tool.chain(),
StaticFileProvider::read_write(output_datadir.static_files())?,
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,

View File

@@ -9,10 +9,9 @@ use reth_provider::{
DatabaseProviderFactory, ProviderFactory,
};
use reth_stages::{stages::StorageHashingStage, Stage, StageCheckpoint, UnwindInput};
use std::sync::Arc;
use tracing::info;
pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = DatabaseEnv>>(
db_tool: &DbTool<N>,
from: u64,
to: u64,
@@ -26,7 +25,7 @@ pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = Arc<Dat
if should_run {
dry_run(
ProviderFactory::<N>::new(
Arc::new(output_db),
output_db,
db_tool.chain(),
StaticFileProvider::read_write(output_datadir.static_files())?,
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,

View File

@@ -34,7 +34,7 @@ pub(crate) async fn dump_merkle_stage<N>(
consensus: impl FullConsensus<N::Primitives> + 'static,
) -> Result<()>
where
N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>,
N: ProviderNodeTypes<DB = DatabaseEnv>,
{
let (output_db, tip_block_number) = setup(from, to, &output_datadir.db(), db_tool)?;
@@ -59,7 +59,7 @@ where
if should_run {
dry_run(
ProviderFactory::<N>::new(
Arc::new(output_db),
output_db,
db_tool.chain(),
StaticFileProvider::read_write(output_datadir.static_files())?,
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,

View File

@@ -158,7 +158,7 @@ enum Subcommands {
impl Subcommands {
/// Returns the block to unwind to. The returned block will stay in database.
fn unwind_target<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
fn unwind_target<N: ProviderNodeTypes<DB = DatabaseEnv>>(
&self,
factory: ProviderFactory<N>,
) -> eyre::Result<u64> {

View File

@@ -29,7 +29,7 @@ auto_impl.workspace = true
derive_more.workspace = true
futures.workspace = true
eyre.workspace = true
reqwest = { workspace = true, features = ["rustls-tls"] }
reqwest.workspace = true
serde = { workspace = true, features = ["derive"] }
tokio = { workspace = true, features = ["time"] }
serde_json.workspace = true

View File

@@ -95,7 +95,7 @@ where
let block_hash = payload.block_hash();
let block_number = payload.block_number();
previous_block_hashes.push(block_hash);
previous_block_hashes.enqueue(block_hash);
// Send new events to execution client
let _ = self.engine_handle.new_payload(payload).await;
@@ -160,7 +160,7 @@ mod tests {
// Push hashes 0..65
for i in 0..65u8 {
buffer.push(B256::with_last_byte(i));
buffer.enqueue(B256::with_last_byte(i));
}
// offset=0 should return the most recent (64)
@@ -181,7 +181,7 @@ mod tests {
let mut buffer: AllocRingBuffer<B256> = AllocRingBuffer::new(65);
// With only 1 entry, only offset=0 works
buffer.push(B256::with_last_byte(1));
buffer.enqueue(B256::with_last_byte(1));
assert_eq!(get_hash_at_offset(&buffer, 0), Some(B256::with_last_byte(1)));
assert_eq!(get_hash_at_offset(&buffer, 1), None);
assert_eq!(get_hash_at_offset(&buffer, 32), None);
@@ -189,7 +189,7 @@ mod tests {
// With 33 entries, offset=32 works but offset=64 doesn't
for i in 2..=33u8 {
buffer.push(B256::with_last_byte(i));
buffer.enqueue(B256::with_last_byte(i));
}
assert_eq!(get_hash_at_offset(&buffer, 32), Some(B256::with_last_byte(1)));
assert_eq!(get_hash_at_offset(&buffer, 64), None);

View File

@@ -114,22 +114,22 @@ pub async fn setup_engine_with_chain_import(
// Initialize the database using init_db (same as CLI import command)
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
let db_env = reth_db::init_db(&db_path, db_args)?;
let db = Arc::new(db_env);
let db = reth_db::init_db(&db_path, db_args)?;
// Create a provider factory with the initialized database (use regular DB, not
// TempDatabase) We need to specify the node types properly for the adapter
let provider_factory = ProviderFactory::<
NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>,
>::new(
db.clone(),
chain_spec.clone(),
reth_provider::providers::StaticFileProvider::read_write(static_files_path.clone())?,
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
.with_default_tables()
.build()
.unwrap(),
)?;
let provider_factory =
ProviderFactory::<NodeTypesWithDBAdapter<EthereumNode, DatabaseEnv>>::new(
db.clone(),
chain_spec.clone(),
reth_provider::providers::StaticFileProvider::read_write(
static_files_path.clone(),
)?,
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
.with_default_tables()
.build()
.unwrap(),
)?;
// Initialize genesis if needed
reth_db_common::init::init_genesis(&provider_factory)?;
@@ -320,11 +320,10 @@ mod tests {
// Import the chain
{
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
let db_env = reth_db::init_db(&db_path, db_args).unwrap();
let db = Arc::new(db_env);
let db = reth_db::init_db(&db_path, db_args).unwrap();
let provider_factory: ProviderFactory<
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, Arc<DatabaseEnv>>,
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, DatabaseEnv>,
> = ProviderFactory::new(
db.clone(),
chain_spec.clone(),
@@ -385,11 +384,10 @@ mod tests {
// Now reopen the database and verify checkpoints are still there
{
let db_env = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap();
let db = Arc::new(db_env);
let db = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap();
let provider_factory: ProviderFactory<
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, Arc<DatabaseEnv>>,
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, DatabaseEnv>,
> = ProviderFactory::new(
db,
chain_spec.clone(),

View File

@@ -528,8 +528,12 @@ impl TreeConfig {
}
/// Setter for the number of storage proof worker threads.
pub fn with_storage_worker_count(mut self, storage_worker_count: usize) -> Self {
self.storage_worker_count = storage_worker_count.max(MIN_WORKER_COUNT);
///
/// No-op if it's [`None`].
pub fn with_storage_worker_count_opt(mut self, storage_worker_count: Option<usize>) -> Self {
if let Some(count) = storage_worker_count {
self.storage_worker_count = count.max(MIN_WORKER_COUNT);
}
self
}
@@ -539,8 +543,12 @@ impl TreeConfig {
}
/// Setter for the number of account proof worker threads.
pub fn with_account_worker_count(mut self, account_worker_count: usize) -> Self {
self.account_worker_count = account_worker_count.max(MIN_WORKER_COUNT);
///
/// No-op if it's [`None`].
pub fn with_account_worker_count_opt(mut self, account_worker_count: Option<usize>) -> Self {
if let Some(count) = account_worker_count {
self.account_worker_count = count.max(MIN_WORKER_COUNT);
}
self
}

View File

@@ -17,7 +17,6 @@ reth-engine-tree.workspace = true
reth-evm.workspace = true
reth-network-p2p.workspace = true
reth-payload-builder.workspace = true
reth-ethereum-primitives.workspace = true
reth-provider.workspace = true
reth-prune.workspace = true
reth-stages-api.workspace = true

View File

@@ -14,7 +14,6 @@ pub use reth_engine_tree::{
chain::{ChainEvent, ChainOrchestrator},
engine::EngineApiEvent,
};
use reth_ethereum_primitives::EthPrimitives;
use reth_evm::ConfigureEvm;
use reth_network_p2p::BlockClient;
use reth_node_types::{BlockTy, NodeTypes};
@@ -97,7 +96,7 @@ where
let downloader = BasicBlockDownloader::new(client, consensus.clone());
let persistence_handle =
PersistenceHandle::<EthPrimitives>::spawn_service(provider, pruner, sync_metrics_tx);
PersistenceHandle::<N::Primitives>::spawn_service(provider, pruner, sync_metrics_tx);
let canonical_in_memory_state = blockchain_db.canonical_in_memory_state();

View File

@@ -23,7 +23,7 @@ reth-evm = { workspace = true, features = ["metrics"] }
reth-network-p2p.workspace = true
reth-payload-builder.workspace = true
reth-payload-primitives.workspace = true
reth-primitives-traits.workspace = true
reth-primitives-traits = { workspace = true, features = ["rayon"] }
reth-ethereum-primitives.workspace = true
reth-provider.workspace = true
reth-prune.workspace = true

View File

@@ -20,7 +20,7 @@ pub(crate) struct PersistenceMetrics {
/// How long it took for blocks to be saved
pub(crate) save_blocks_duration_seconds: Histogram,
/// How many blocks we persist at once.
pub(crate) save_blocks_block_count: Histogram,
pub(crate) save_blocks_batch_size: Histogram,
/// How long it took for blocks to be pruned
pub(crate) prune_before_duration_seconds: Histogram,
}

View File

@@ -12,7 +12,11 @@ use reth_provider::{
use reth_prune::{PrunerError, PrunerOutput, PrunerWithFactory};
use reth_stages_api::{MetricEvent, MetricEventsSender};
use std::{
sync::mpsc::{Receiver, SendError, Sender},
sync::{
mpsc::{Receiver, SendError, Sender},
Arc,
},
thread::JoinHandle,
time::Instant,
};
use thiserror::Error;
@@ -40,6 +44,12 @@ where
metrics: PersistenceMetrics,
/// Sender for sync metrics - we only submit sync metrics for persisted blocks
sync_metrics_tx: MetricEventsSender,
/// Pending finalized block number to be committed with the next block save.
/// This avoids triggering a separate fsync for each finalized block update.
pending_finalized_block: Option<u64>,
/// Pending safe block number to be committed with the next block save.
/// This avoids triggering a separate fsync for each safe block update.
pending_safe_block: Option<u64>,
}
impl<N> PersistenceService<N>
@@ -53,7 +63,15 @@ where
pruner: PrunerWithFactory<ProviderFactory<N>>,
sync_metrics_tx: MetricEventsSender,
) -> Self {
Self { provider, incoming, pruner, metrics: PersistenceMetrics::default(), sync_metrics_tx }
Self {
provider,
incoming,
pruner,
metrics: PersistenceMetrics::default(),
sync_metrics_tx,
pending_finalized_block: None,
pending_safe_block: None,
}
}
/// Prunes block data before the given block number according to the configured prune
@@ -106,14 +124,10 @@ where
}
}
PersistenceAction::SaveFinalizedBlock(finalized_block) => {
let provider = self.provider.database_provider_rw()?;
provider.save_finalized_block_number(finalized_block)?;
provider.commit()?;
self.pending_finalized_block = Some(finalized_block);
}
PersistenceAction::SaveSafeBlock(safe_block) => {
let provider = self.provider.database_provider_rw()?;
provider.save_safe_block_number(safe_block)?;
provider.commit()?;
self.pending_safe_block = Some(safe_block);
}
}
}
@@ -138,26 +152,39 @@ where
}
fn on_save_blocks(
&self,
&mut self,
blocks: Vec<ExecutedBlock<N::Primitives>>,
) -> Result<Option<BlockNumHash>, PersistenceError> {
let first_block = blocks.first().map(|b| b.recovered_block.num_hash());
let last_block = blocks.last().map(|b| b.recovered_block.num_hash());
let block_count = blocks.len();
// Take any pending finalized/safe block updates to commit together
let pending_finalized = self.pending_finalized_block.take();
let pending_safe = self.pending_safe_block.take();
debug!(target: "engine::persistence", ?block_count, first=?first_block, last=?last_block, "Saving range of blocks");
let start_time = Instant::now();
if last_block.is_some() {
let provider_rw = self.provider.database_provider_rw()?;
provider_rw.save_blocks(blocks, SaveBlocksMode::Full)?;
// Commit pending finalized/safe block updates in the same transaction
if let Some(finalized) = pending_finalized {
provider_rw.save_finalized_block_number(finalized)?;
}
if let Some(safe) = pending_safe {
provider_rw.save_safe_block_number(safe)?;
}
provider_rw.commit()?;
}
debug!(target: "engine::persistence", first=?first_block, last=?last_block, "Saved range of blocks");
self.metrics.save_blocks_block_count.record(block_count as f64);
self.metrics.save_blocks_batch_size.record(block_count as f64);
self.metrics.save_blocks_duration_seconds.record(start_time.elapsed());
Ok(last_block)
@@ -204,15 +231,25 @@ pub enum PersistenceAction<N: NodePrimitives = EthPrimitives> {
pub struct PersistenceHandle<N: NodePrimitives = EthPrimitives> {
/// The channel used to communicate with the persistence service
sender: Sender<PersistenceAction<N>>,
/// Guard that joins the service thread when all handles are dropped.
/// Uses `Arc` so the handle remains `Clone`.
_service_guard: Arc<ServiceGuard>,
}
impl<T: NodePrimitives> PersistenceHandle<T> {
/// Create a new [`PersistenceHandle`] from a [`Sender<PersistenceAction>`].
pub const fn new(sender: Sender<PersistenceAction<T>>) -> Self {
Self { sender }
///
/// This is intended for testing purposes where you want to mock the persistence service.
/// For production use, prefer [`spawn_service`](Self::spawn_service).
pub fn new(sender: Sender<PersistenceAction<T>>) -> Self {
Self { sender, _service_guard: Arc::new(ServiceGuard(None)) }
}
/// Create a new [`PersistenceHandle`], and spawn the persistence service.
///
/// The returned handle can be cloned and shared. When all clones are dropped, the service
/// thread will be joined, ensuring graceful shutdown before resources (like `RocksDB`) are
/// released.
pub fn spawn_service<N>(
provider_factory: ProviderFactory<N>,
pruner: PrunerWithFactory<ProviderFactory<N>>,
@@ -224,13 +261,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
// create the initial channels
let (db_service_tx, db_service_rx) = std::sync::mpsc::channel();
// construct persistence handle
let persistence_handle = PersistenceHandle::new(db_service_tx);
// spawn the persistence service
let db_service =
PersistenceService::new(provider_factory, db_service_rx, pruner, sync_metrics_tx);
std::thread::Builder::new()
let join_handle = std::thread::Builder::new()
.name("Persistence Service".to_string())
.spawn(|| {
if let Err(err) = db_service.run() {
@@ -239,7 +273,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
})
.unwrap();
persistence_handle
PersistenceHandle {
sender: db_service_tx,
_service_guard: Arc::new(ServiceGuard(Some(join_handle))),
}
}
/// Sends a specific [`PersistenceAction`] in the contained channel. The caller is responsible
@@ -267,7 +304,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
self.send_action(PersistenceAction::SaveBlocks(blocks, tx))
}
/// Persists the finalized block number on disk.
/// Queues the finalized block number to be persisted on disk.
///
/// The update is deferred and will be committed together with the next [`Self::save_blocks`]
/// call to avoid triggering a separate fsync for each update.
pub fn save_finalized_block_number(
&self,
finalized_block: u64,
@@ -275,7 +315,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
self.send_action(PersistenceAction::SaveFinalizedBlock(finalized_block))
}
/// Persists the safe block number on disk.
/// Queues the safe block number to be persisted on disk.
///
/// The update is deferred and will be committed together with the next [`Self::save_blocks`]
/// call to avoid triggering a separate fsync for each update.
pub fn save_safe_block_number(
&self,
safe_block: u64,
@@ -297,6 +340,27 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
}
}
/// Guard that joins the persistence service thread when dropped.
///
/// This ensures graceful shutdown - the service thread completes before resources like
/// `RocksDB` are released. Stored in an `Arc` inside [`PersistenceHandle`] so the handle
/// can be cloned while sharing the same guard.
struct ServiceGuard(Option<JoinHandle<()>>);
impl std::fmt::Debug for ServiceGuard {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_tuple("ServiceGuard").field(&self.0.as_ref().map(|_| "...")).finish()
}
}
impl Drop for ServiceGuard {
fn drop(&mut self) {
if let Some(join_handle) = self.0.take() {
let _ = join_handle.join();
}
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -323,12 +387,12 @@ mod tests {
#[test]
fn test_save_blocks_empty() {
reth_tracing::init_test_tracing();
let persistence_handle = default_persistence_handle();
let handle = default_persistence_handle();
let blocks = vec![];
let (tx, rx) = crossbeam_channel::bounded(1);
persistence_handle.save_blocks(blocks, tx).unwrap();
handle.save_blocks(blocks, tx).unwrap();
let hash = rx.recv().unwrap();
assert_eq!(hash, None);
@@ -337,7 +401,7 @@ mod tests {
#[test]
fn test_save_blocks_single_block() {
reth_tracing::init_test_tracing();
let persistence_handle = default_persistence_handle();
let handle = default_persistence_handle();
let block_number = 0;
let mut test_block_builder = TestBlockBuilder::eth();
let executed =
@@ -347,7 +411,7 @@ mod tests {
let blocks = vec![executed];
let (tx, rx) = crossbeam_channel::bounded(1);
persistence_handle.save_blocks(blocks, tx).unwrap();
handle.save_blocks(blocks, tx).unwrap();
let BlockNumHash { hash: actual_hash, number: _ } = rx
.recv_timeout(std::time::Duration::from_secs(10))
@@ -360,14 +424,14 @@ mod tests {
#[test]
fn test_save_blocks_multiple_blocks() {
reth_tracing::init_test_tracing();
let persistence_handle = default_persistence_handle();
let handle = default_persistence_handle();
let mut test_block_builder = TestBlockBuilder::eth();
let blocks = test_block_builder.get_executed_blocks(0..5).collect::<Vec<_>>();
let last_hash = blocks.last().unwrap().recovered_block().hash();
let (tx, rx) = crossbeam_channel::bounded(1);
persistence_handle.save_blocks(blocks, tx).unwrap();
handle.save_blocks(blocks, tx).unwrap();
let BlockNumHash { hash: actual_hash, number: _ } = rx.recv().unwrap().unwrap();
assert_eq!(last_hash, actual_hash);
}
@@ -375,7 +439,7 @@ mod tests {
#[test]
fn test_save_blocks_multiple_calls() {
reth_tracing::init_test_tracing();
let persistence_handle = default_persistence_handle();
let handle = default_persistence_handle();
let ranges = [0..1, 1..2, 2..4, 4..5];
let mut test_block_builder = TestBlockBuilder::eth();
@@ -384,7 +448,7 @@ mod tests {
let last_hash = blocks.last().unwrap().recovered_block().hash();
let (tx, rx) = crossbeam_channel::bounded(1);
persistence_handle.save_blocks(blocks, tx).unwrap();
handle.save_blocks(blocks, tx).unwrap();
let BlockNumHash { hash: actual_hash, number: _ } = rx.recv().unwrap().unwrap();
assert_eq!(last_hash, actual_hash);

View File

@@ -61,7 +61,6 @@ mod persistence_state;
pub mod precompile_cache;
#[cfg(test)]
mod tests;
#[expect(unused)]
mod trie_updates;
use crate::tree::error::AdvancePersistenceError;
@@ -2613,19 +2612,27 @@ where
let block_num_hash = block_id.block;
debug!(target: "engine::tree", block=?block_num_hash, parent = ?block_id.parent, "Inserting new block into tree");
match self.sealed_header_by_hash(block_num_hash.hash) {
Err(err) => {
let block = convert_to_block(self, input)?;
return Err(InsertBlockError::new(block, err.into()).into());
// Check if block already exists - first in memory, then DB only if it could be persisted
if self.state.tree_state.sealed_header_by_hash(&block_num_hash.hash).is_some() {
convert_to_block(self, input)?;
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid));
}
// Only query DB if block could be persisted (number <= last persisted block).
// New blocks from CL always have number > last persisted, so skip DB lookup for them.
if block_num_hash.number <= self.persistence_state.last_persisted_block.number {
match self.provider.sealed_header_by_hash(block_num_hash.hash) {
Err(err) => {
let block = convert_to_block(self, input)?;
return Err(InsertBlockError::new(block, err.into()).into());
}
Ok(Some(_)) => {
convert_to_block(self, input)?;
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid));
}
Ok(None) => {}
}
Ok(Some(_)) => {
// We now assume that we already have this block in the tree. However, we need to
// run the conversion to ensure that the block hash is valid.
convert_to_block(self, input)?;
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid))
}
_ => {}
};
}
// Ensure that the parent state is available.
match self.state_provider_builder(block_id.parent) {

View File

@@ -39,7 +39,7 @@ use reth_trie_parallel::{
proof_task::{ProofTaskCtx, ProofWorkerHandle},
root::ParallelStateRootError,
};
use reth_trie_sparse::{ClearedSparseStateTrie, RevealableSparseTrie, SparseStateTrie};
use reth_trie_sparse::{RevealableSparseTrie, SparseStateTrie};
use reth_trie_sparse_parallel::{ParallelSparseTrie, ParallelismThresholds};
use std::{
collections::BTreeMap,
@@ -235,8 +235,7 @@ where
+ 'static,
{
// start preparing transactions immediately
let (prewarm_rx, execution_rx, transaction_count_hint) =
self.spawn_tx_iterator(transactions);
let (prewarm_rx, execution_rx) = self.spawn_tx_iterator(transactions);
let span = Span::current();
let (to_sparse_trie, sparse_trie_rx) = channel();
@@ -260,7 +259,6 @@ where
self.spawn_caching_with(
env,
prewarm_rx,
transaction_count_hint,
provider_builder.clone(),
None, // Don't send proof targets when BAL is present
Some(bal),
@@ -271,7 +269,6 @@ where
self.spawn_caching_with(
env,
prewarm_rx,
transaction_count_hint,
provider_builder.clone(),
Some(to_multi_proof.clone()),
None,
@@ -355,10 +352,10 @@ where
where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
{
let (prewarm_rx, execution_rx, size_hint) = self.spawn_tx_iterator(transactions);
let (prewarm_rx, execution_rx) = self.spawn_tx_iterator(transactions);
// This path doesn't use multiproof, so V2 proofs flag doesn't matter
let prewarm_handle =
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal, false);
self.spawn_caching_with(env, prewarm_rx, provider_builder, None, bal, false);
PayloadHandle {
to_multi_proof: None,
prewarm_handle,
@@ -376,19 +373,15 @@ where
) -> (
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Recovered>>,
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Recovered>, I::Error>>,
usize,
) {
let (transactions, convert) = transactions.into();
let transactions = transactions.into_par_iter();
let transaction_count_hint = transactions.len();
let (ooo_tx, ooo_rx) = mpsc::channel();
let (prewarm_tx, prewarm_rx) = mpsc::channel();
let (execute_tx, execute_rx) = mpsc::channel();
// Spawn a task that `convert`s all transactions in parallel and sends them out-of-order.
self.executor.spawn_blocking(move || {
transactions.enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
rayon::spawn(move || {
let (transactions, convert) = transactions.into();
transactions.into_par_iter().enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
let tx = convert(tx);
let tx = tx.map(|tx| {
let (tx_env, tx) = tx.into_parts();
@@ -424,16 +417,14 @@ where
}
});
(prewarm_rx, execute_rx, transaction_count_hint)
(prewarm_rx, execute_rx)
}
/// Spawn prewarming optionally wired to the multiproof task for target updates.
#[expect(clippy::too_many_arguments)]
fn spawn_caching_with<P>(
&self,
env: ExecutionEnv<Evm>,
mut transactions: mpsc::Receiver<impl ExecutableTxFor<Evm> + Clone + Send + 'static>,
transaction_count_hint: usize,
provider_builder: StateProviderBuilder<N, P>,
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
bal: Option<Arc<BlockAccessList>>,
@@ -468,7 +459,6 @@ where
self.execution_cache.clone(),
prewarm_ctx,
to_multi_proof,
transaction_count_hint,
self.prewarm_max_concurrency,
);
@@ -562,11 +552,11 @@ where
sparse_state_trie,
))
} else {
SpawnedSparseTrieTask::Cached(SparseTrieCacheTask::new_with_cleared_trie(
SpawnedSparseTrieTask::Cached(SparseTrieCacheTask::new_with_trie(
from_multi_proof,
proof_worker_handle,
trie_metrics.clone(),
ClearedSparseStateTrie::from_state_trie(sparse_state_trie),
sparse_state_trie,
))
};
@@ -961,6 +951,10 @@ pub struct ExecutionEnv<Evm: ConfigureEvm> {
/// Used for sparse trie continuation: if the preserved trie's anchor matches this,
/// the trie can be reused directly.
pub parent_state_root: B256,
/// Number of transactions in the block.
/// Used to determine parallel worker count for prewarming.
/// A value of 0 indicates the count is unknown.
pub transaction_count: usize,
}
impl<Evm: ConfigureEvm> Default for ExecutionEnv<Evm>
@@ -973,6 +967,7 @@ where
hash: Default::default(),
parent_hash: Default::default(),
parent_state_root: Default::default(),
transaction_count: 0,
}
}
}

View File

@@ -84,8 +84,6 @@ where
ctx: PrewarmContext<N, P, Evm>,
/// How many transactions should be executed in parallel
max_concurrency: usize,
/// The number of transactions to be processed
transaction_count_hint: usize,
/// Sender to emit evm state outcome messages, if any.
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
/// Receiver for events produced by tx execution
@@ -106,7 +104,6 @@ where
execution_cache: PayloadExecutionCache,
ctx: PrewarmContext<N, P, Evm>,
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
transaction_count_hint: usize,
max_concurrency: usize,
) -> (Self, Sender<PrewarmTaskEvent<N::Receipt>>) {
let (actions_tx, actions_rx) = channel();
@@ -114,7 +111,7 @@ where
trace!(
target: "engine::tree::payload_processor::prewarm",
max_concurrency,
transaction_count_hint,
transaction_count = ctx.env.transaction_count,
"Initialized prewarm task"
);
@@ -124,7 +121,6 @@ where
execution_cache,
ctx,
max_concurrency,
transaction_count_hint,
to_multi_proof,
actions_rx,
parent_span: Span::current(),
@@ -148,7 +144,6 @@ where
let executor = self.executor.clone();
let ctx = self.ctx.clone();
let max_concurrency = self.max_concurrency;
let transaction_count_hint = self.transaction_count_hint;
let span = Span::current();
self.executor.spawn_blocking(move || {
@@ -156,13 +151,14 @@ where
let (done_tx, done_rx) = mpsc::channel();
// When transaction_count_hint is 0, it means the count is unknown. In this case, spawn
// When transaction_count is 0, it means the count is unknown. In this case, spawn
// max workers to handle potentially many transactions in parallel rather
// than bottlenecking on a single worker.
let workers_needed = if transaction_count_hint == 0 {
let transaction_count = ctx.env.transaction_count;
let workers_needed = if transaction_count == 0 {
max_concurrency
} else {
transaction_count_hint.min(max_concurrency)
transaction_count.min(max_concurrency)
};
// Spawn workers

View File

@@ -5,14 +5,15 @@ use crate::tree::{
payload_processor::multiproof::{MultiProofTaskMetrics, SparseTrieUpdate},
};
use alloy_primitives::B256;
use alloy_rlp::Decodable;
use alloy_rlp::{Decodable, Encodable};
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
use rayon::iter::{ParallelBridge, ParallelIterator};
use rayon::iter::ParallelIterator;
use reth_errors::ProviderError;
use reth_primitives_traits::Account;
use reth_primitives_traits::{Account, ParallelBridgeBuffered};
use reth_revm::state::EvmState;
use reth_trie::{
proof_v2::Target, updates::TrieUpdates, HashedPostState, Nibbles, TrieAccount, EMPTY_ROOT_HASH,
TRIE_ACCOUNT_RLP_MAX_SIZE,
};
use reth_trie_parallel::{
proof_task::{
@@ -25,8 +26,7 @@ use reth_trie_parallel::{
use reth_trie_sparse::{
errors::{SparseStateTrieResult, SparseTrieErrorKind},
provider::{TrieNodeProvider, TrieNodeProviderFactory},
ClearedSparseStateTrie, LeafUpdate, SerialSparseTrie, SparseStateTrie, SparseTrie,
SparseTrieExt,
LeafUpdate, SerialSparseTrie, SparseStateTrie, SparseTrie, SparseTrieExt,
};
use revm_primitives::{hash_map::Entry, B256Map};
use smallvec::SmallVec;
@@ -239,6 +239,8 @@ pub(super) struct SparseTrieCacheTask<A = SerialSparseTrie, S = SerialSparseTrie
/// Cache of storage proof targets that have already been fetched/requested from the proof
/// workers. account -> slot -> lowest `min_len` requested.
fetched_storage_targets: B256Map<B256Map<u8>>,
/// Reusable buffer for RLP encoding of accounts.
account_rlp_buf: Vec<u8>,
/// Metrics for the sparse trie.
metrics: MultiProofTaskMetrics,
}
@@ -248,12 +250,12 @@ where
A: SparseTrieExt + Default,
S: SparseTrieExt + Default + Clone,
{
/// Creates a new sparse trie, pre-populating with a [`ClearedSparseStateTrie`].
pub(super) fn new_with_cleared_trie(
/// Creates a new sparse trie, pre-populating with an existing [`SparseStateTrie`].
pub(super) fn new_with_trie(
updates: CrossbeamReceiver<MultiProofMessage>,
proof_worker_handle: ProofWorkerHandle,
metrics: MultiProofTaskMetrics,
sparse_state_trie: ClearedSparseStateTrie<A, S>,
sparse_state_trie: SparseStateTrie<A, S>,
) -> Self {
let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded();
Self {
@@ -261,12 +263,13 @@ where
proof_result_rx,
updates,
proof_worker_handle,
trie: sparse_state_trie.into_inner(),
trie: sparse_state_trie,
account_updates: Default::default(),
storage_updates: Default::default(),
pending_account_updates: Default::default(),
fetched_account_targets: Default::default(),
fetched_storage_targets: Default::default(),
account_rlp_buf: Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE),
metrics,
}
}
@@ -359,11 +362,6 @@ where
}
}
// Process any remaining pending account updates.
if !self.pending_account_updates.is_empty() {
self.process_updates()?;
}
debug!(target: "engine::root", "All proofs processed, ending calculation");
let start = Instant::now();
@@ -513,78 +511,94 @@ where
{
Vec::new()
} else {
// TODO: optimize allocation
alloy_rlp::encode(
account.unwrap_or_default().into_trie_account(storage_root),
)
self.account_rlp_buf.clear();
account
.unwrap_or_default()
.into_trie_account(storage_root)
.encode(&mut self.account_rlp_buf);
self.account_rlp_buf.clone()
};
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
}
}
}
// Now handle pending account updates that can be upgraded to a proper update.
self.pending_account_updates.retain(|addr, account| {
// If account has pending storage updates, it is still pending.
if self.storage_updates.get(addr).is_some_and(|updates| !updates.is_empty()) {
return true;
}
loop {
// Now handle pending account updates that can be upgraded to a proper update.
let account_rlp_buf = &mut self.account_rlp_buf;
self.pending_account_updates.retain(|addr, account| {
// If account has pending storage updates, it is still pending.
if self.storage_updates.get(addr).is_some_and(|updates| !updates.is_empty()) {
return true;
}
// Get the current account state either from the trie or from latest account update.
let trie_account = if let Some(LeafUpdate::Changed(encoded)) = self.account_updates.get(addr) {
Some(encoded).filter(|encoded| !encoded.is_empty())
} else if !self.account_updates.contains_key(addr) {
self.trie.get_account_value(addr)
} else {
// Needs to be revealed first
return true;
};
// Get the current account state either from the trie or from latest account update.
let trie_account = if let Some(LeafUpdate::Changed(encoded)) = self.account_updates.get(addr) {
Some(encoded).filter(|encoded| !encoded.is_empty())
} else if !self.account_updates.contains_key(addr) {
self.trie.get_account_value(addr)
} else {
// Needs to be revealed first
return true;
};
let trie_account = trie_account.map(|value| TrieAccount::decode(&mut &value[..]).expect("invalid account RLP"));
let trie_account = trie_account.map(|value| TrieAccount::decode(&mut &value[..]).expect("invalid account RLP"));
let (account, storage_root) = if let Some(account) = account.take() {
// If account is Some(_) here it means it didn't have any storage updates
// and we can fetch the storage root directly from the account trie.
//
// If it did have storage updates, we would've had processed it above when iterating over storage tries.
let storage_root = trie_account.map(|account| account.storage_root).unwrap_or(EMPTY_ROOT_HASH);
let (account, storage_root) = if let Some(account) = account.take() {
// If account is Some(_) here it means it didn't have any storage updates
// and we can fetch the storage root directly from the account trie.
//
// If it did have storage updates, we would've had processed it above when iterating over storage tries.
let storage_root = trie_account.map(|account| account.storage_root).unwrap_or(EMPTY_ROOT_HASH);
(account, storage_root)
} else {
(trie_account.map(Into::into), self.trie.storage_root(addr).expect("account had storage updates that were applied to its trie, storage root must be revealed by now"))
};
(account, storage_root)
} else {
(trie_account.map(Into::into), self.trie.storage_root(addr).expect("account had storage updates that were applied to its trie, storage root must be revealed by now"))
};
let encoded = if account.is_none_or(|account| account.is_empty()) && storage_root == EMPTY_ROOT_HASH {
Vec::new()
} else {
let account = account.unwrap_or_default().into_trie_account(storage_root);
let encoded = if account.is_none_or(|account| account.is_empty()) && storage_root == EMPTY_ROOT_HASH {
Vec::new()
} else {
account_rlp_buf.clear();
account.unwrap_or_default().into_trie_account(storage_root).encode(account_rlp_buf);
account_rlp_buf.clone()
};
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
// TODO: optimize allocation
alloy_rlp::encode(account)
};
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
false
});
false
});
let updates_len_before = self.account_updates.len();
// Process account trie updates and fill the account targets.
self.trie
.trie_mut()
.update_leaves(&mut self.account_updates, |target, min_len| {
match self.fetched_account_targets.entry(target) {
Entry::Occupied(mut entry) => {
if min_len < *entry.get() {
// Process account trie updates and fill the account targets.
self.trie
.trie_mut()
.update_leaves(&mut self.account_updates, |target, min_len| {
match self.fetched_account_targets.entry(target) {
Entry::Occupied(mut entry) => {
if min_len < *entry.get() {
entry.insert(min_len);
targets
.account_targets
.push(Target::new(target).with_min_len(min_len));
}
}
Entry::Vacant(entry) => {
entry.insert(min_len);
targets.account_targets.push(Target::new(target).with_min_len(min_len));
}
}
Entry::Vacant(entry) => {
entry.insert(min_len);
targets.account_targets.push(Target::new(target).with_min_len(min_len));
}
}
})
.map_err(ProviderError::other)?;
})
.map_err(ProviderError::other)?;
if updates_len_before == self.account_updates.len() {
// Only exit when no new updates are processed.
//
// We need to keep iterating if any updates are being drained because that might
// indicate that more pending account updates can be promoted.
break;
}
}
if !targets.is_empty() {
self.proof_worker_handle.dispatch_account_multiproof(AccountMultiproofInput::V2 {
@@ -651,7 +665,7 @@ where
.storages
.into_iter()
.map(|(address, storage)| (address, storage, trie.take_storage_trie(&address)))
.par_bridge()
.par_bridge_buffered()
.map(|(address, storage, storage_trie)| {
let _enter =
debug_span!(target: "engine::tree::payload_processor::sparse_trie", parent: &span, "storage trie", ?address)

View File

@@ -407,6 +407,7 @@ where
hash: input.hash(),
parent_hash: input.parent_hash(),
parent_state_root: parent_block.state_root(),
transaction_count: input.transaction_count(),
};
// Plan the strategy used for state root computation.
@@ -519,6 +520,14 @@ where
info!(target: "engine::tree::payload_validator", ?state_root, ?elapsed, "State root task finished");
// we double check the state root here for good measure
if state_root == block.header().state_root() {
// Compare trie updates with serial computation if configured
if self.config.always_compare_trie_updates() {
self.compare_trie_updates_with_serial(
overlay_factory.clone(),
&hashed_state,
trie_updates.clone(),
);
}
maybe_state_root = Some((state_root, trie_updates, elapsed))
} else {
warn!(
@@ -894,6 +903,62 @@ where
.root_with_updates()?)
}
/// Compares trie updates from the state root task with serial state root computation.
///
/// This is used for debugging and validating the correctness of the parallel state root
/// task implementation. When enabled via `--engine.state-root-task-compare-updates`, this
/// method runs a separate serial state root computation and compares the resulting trie
/// updates.
fn compare_trie_updates_with_serial(
&self,
overlay_factory: OverlayStateProviderFactory<P>,
hashed_state: &HashedPostState,
task_trie_updates: TrieUpdates,
) {
debug!(target: "engine::tree::payload_validator", "Comparing trie updates with serial computation");
match self.compute_state_root_serial(overlay_factory.clone(), hashed_state) {
Ok((serial_root, serial_trie_updates)) => {
debug!(
target: "engine::tree::payload_validator",
?serial_root,
"Serial state root computation finished for comparison"
);
// Get a database provider to use as trie cursor factory
match overlay_factory.database_provider_ro() {
Ok(provider) => {
if let Err(err) = super::trie_updates::compare_trie_updates(
&provider,
task_trie_updates,
serial_trie_updates,
) {
warn!(
target: "engine::tree::payload_validator",
%err,
"Error comparing trie updates"
);
}
}
Err(err) => {
warn!(
target: "engine::tree::payload_validator",
%err,
"Failed to get database provider for trie update comparison"
);
}
}
}
Err(err) => {
warn!(
target: "engine::tree::payload_validator",
%err,
"Failed to compute serial state root for comparison"
);
}
}
}
/// Validates the block after execution.
///
/// This performs:

View File

@@ -98,7 +98,7 @@ impl StorageTrieUpdatesDiff {
/// Compares the trie updates from state root task, regular state root calculation and database,
/// and logs the differences if there's any.
pub(super) fn compare_trie_updates(
pub(crate) fn compare_trie_updates(
trie_cursor_factory: impl TrieCursorFactory,
task: TrieUpdates,
regular: TrieUpdates,
@@ -186,7 +186,8 @@ fn compare_storage_trie_updates<C: TrieCursor>(
task: &mut StorageTrieUpdates,
regular: &mut StorageTrieUpdates,
) -> Result<StorageTrieUpdatesDiff, DatabaseError> {
let database_not_exists = trie_cursor()?.next()?.is_none();
// Check if the storage trie exists by seeking to the first entry
let database_not_exists = trie_cursor()?.seek(Nibbles::default())?.is_none();
let mut diff = StorageTrieUpdatesDiff {
// If the deletion is a no-op, meaning that the entry is not in the
// database, do not add it to the diff.

View File

@@ -20,8 +20,6 @@ reth-era.workspace = true
# http
bytes.workspace = true
reqwest.workspace = true
reqwest.default-features = false
reqwest.features = ["stream", "rustls-tls-native-roots"]
# async
tokio.workspace = true

View File

@@ -86,7 +86,7 @@ where
mut self,
components: impl CliComponentsBuilder<N>,
launcher: impl AsyncFnOnce(
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
Ext,
) -> Result<()>,
) -> Result<()>
@@ -132,7 +132,7 @@ pub(crate) fn run_commands_with<C, Ext, Rpc, N, SubCmd>(
runner: CliRunner,
components: impl CliComponentsBuilder<N>,
launcher: impl AsyncFnOnce(
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
Ext,
) -> Result<()>,
) -> Result<()>

View File

@@ -131,7 +131,7 @@ impl<
/// ````
pub fn run<L, Fut>(self, launcher: L) -> eyre::Result<()>
where
L: FnOnce(WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>, Ext) -> Fut,
L: FnOnce(WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>, Ext) -> Fut,
Fut: Future<Output = eyre::Result<()>>,
C: ChainSpecParser<ChainSpec = ChainSpec>,
{
@@ -148,7 +148,7 @@ impl<
self,
components: impl CliComponentsBuilder<N>,
launcher: impl AsyncFnOnce(
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
Ext,
) -> eyre::Result<()>,
) -> eyre::Result<()>
@@ -180,7 +180,7 @@ impl<
/// ```
pub fn with_runner<L, Fut>(self, runner: CliRunner, launcher: L) -> eyre::Result<()>
where
L: FnOnce(WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>, Ext) -> Fut,
L: FnOnce(WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>, Ext) -> Fut,
Fut: Future<Output = eyre::Result<()>>,
C: ChainSpecParser<ChainSpec = ChainSpec>,
{
@@ -196,7 +196,7 @@ impl<
runner: CliRunner,
components: impl CliComponentsBuilder<N>,
launcher: impl AsyncFnOnce(
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
Ext,
) -> eyre::Result<()>,
) -> eyre::Result<()>

View File

@@ -119,10 +119,9 @@ impl EthereumNode {
/// use reth_db::open_db_read_only;
/// use reth_node_ethereum::EthereumNode;
/// use reth_provider::providers::{RocksDBProvider, StaticFileProvider};
/// use std::sync::Arc;
///
/// let factory = EthereumNode::provider_factory_builder()
/// .db(Arc::new(open_db_read_only("db", Default::default()).unwrap()))
/// .db(open_db_read_only("db", Default::default()).unwrap())
/// .chainspec(ChainSpecBuilder::mainnet().build().into())
/// .static_file(StaticFileProvider::read_only("db/static_files", false).unwrap())
/// .rocksdb_provider(RocksDBProvider::builder("db/rocksdb").build().unwrap())

View File

@@ -100,10 +100,12 @@ async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> {
ChainSpecBuilder::default().chain(MAINNET.chain).genesis(genesis).osaka_activated().build(),
);
let genesis_hash = chain_spec.genesis_hash();
let node_config = NodeConfig::test()
.with_chain(chain_spec)
.with_unused_ports()
.with_rpc(RpcServerArgs::default().with_unused_ports().with_http());
let node_config = NodeConfig::test().with_chain(chain_spec).with_unused_ports().with_rpc(
RpcServerArgs::default()
.with_unused_ports()
.with_http()
.with_force_blob_sidecar_upcasting(),
);
let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone())
.testing_node(exec.clone())
.node(EthereumNode::default())
@@ -125,7 +127,7 @@ async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> {
let blob_tx_hash = node.rpc.inject_tx(blob_tx).await?;
// fetch it from rpc
let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?;
// assert that sidecar was converted to eip7594
// assert that sidecar was converted to eip7594 (force upcasting is enabled)
assert!(envelope.as_eip4844().unwrap().tx().sidecar().unwrap().is_eip7594());
// validate sidecar
TransactionTestContext::validate_sidecar(envelope);
@@ -161,10 +163,12 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> {
.build(),
);
let genesis_hash = chain_spec.genesis_hash();
let node_config = NodeConfig::test()
.with_chain(chain_spec)
.with_unused_ports()
.with_rpc(RpcServerArgs::default().with_unused_ports().with_http());
let node_config = NodeConfig::test().with_chain(chain_spec).with_unused_ports().with_rpc(
RpcServerArgs::default()
.with_unused_ports()
.with_http()
.with_force_blob_sidecar_upcasting(),
);
let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone())
.testing_node(exec.clone())
.node(EthereumNode::default())

View File

@@ -511,9 +511,8 @@ mod compact {
total_length += flags.len() + buffer.len();
buf.put_slice(&flags);
if zstd {
reth_zstd_compressors::RECEIPT_COMPRESSOR.with(|compressor| {
let compressed =
compressor.borrow_mut().compress(&buffer).expect("Failed to compress.");
reth_zstd_compressors::with_receipt_compressor(|compressor| {
let compressed = compressor.compress(&buffer).expect("Failed to compress.");
buf.put(compressed.as_slice());
});
} else {
@@ -525,8 +524,7 @@ mod compact {
fn from_compact(buf: &[u8], _len: usize) -> (Self, &[u8]) {
let (flags, mut buf) = ReceiptFlags::from(buf);
if flags.__zstd() != 0 {
reth_zstd_compressors::RECEIPT_DECOMPRESSOR.with(|decompressor| {
let decompressor = &mut decompressor.borrow_mut();
reth_zstd_compressors::with_receipt_decompressor(|decompressor| {
let decompressed = decompressor.decompress(buf);
let original_buf = buf;
let mut buf: &[u8] = decompressed;

View File

@@ -577,19 +577,11 @@ impl reth_codecs::Compact for TransactionSigned {
let tx_bits = if zstd_bit {
let mut tmp = Vec::with_capacity(256);
if cfg!(feature = "std") {
reth_zstd_compressors::TRANSACTION_COMPRESSOR.with(|compressor| {
let mut compressor = compressor.borrow_mut();
let tx_bits = self.transaction.to_compact(&mut tmp);
buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress"));
tx_bits as u8
})
} else {
let mut compressor = reth_zstd_compressors::create_tx_compressor();
reth_zstd_compressors::with_tx_compressor(|compressor| {
let tx_bits = self.transaction.to_compact(&mut tmp);
buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress"));
tx_bits as u8
}
})
} else {
self.transaction.to_compact(buf) as u8
};
@@ -611,26 +603,13 @@ impl reth_codecs::Compact for TransactionSigned {
let zstd_bit = bitflags >> 3;
let (transaction, buf) = if zstd_bit != 0 {
if cfg!(feature = "std") {
reth_zstd_compressors::TRANSACTION_DECOMPRESSOR.with(|decompressor| {
let mut decompressor = decompressor.borrow_mut();
// TODO: enforce that zstd is only present at a "top" level type
let transaction_type = (bitflags & 0b110) >> 1;
let (transaction, _) =
Transaction::from_compact(decompressor.decompress(buf), transaction_type);
(transaction, buf)
})
} else {
let mut decompressor = reth_zstd_compressors::create_tx_decompressor();
reth_zstd_compressors::with_tx_decompressor(|decompressor| {
// TODO: enforce that zstd is only present at a "top" level type
let transaction_type = (bitflags & 0b110) >> 1;
let (transaction, _) =
Transaction::from_compact(decompressor.decompress(buf), transaction_type);
(transaction, buf)
}
})
} else {
let transaction_type = bitflags >> 1;
Transaction::from_compact(buf, transaction_type)

View File

@@ -67,7 +67,6 @@ tempfile.workspace = true
[features]
default = []
edge = ["reth-provider/edge"]
serde = [
"reth-exex-types/serde",
"reth-revm/serde",

View File

@@ -251,6 +251,8 @@ impl<DB, ChainSpec: EthChainSpec> NodeBuilder<DB, ChainSpec> {
}
/// Creates a preconfigured node for testing purposes with a specific datadir.
///
/// The entire `datadir` will be cleaned up when the node is dropped.
#[cfg(feature = "test-utils")]
pub fn testing_node_with_datadir(
mut self,
@@ -268,7 +270,7 @@ impl<DB, ChainSpec: EthChainSpec> NodeBuilder<DB, ChainSpec> {
let data_dir =
path.unwrap_or_chain_default(self.config.chain.chain(), self.config.datadir.clone());
let db = reth_db::test_utils::create_test_rw_db_with_path(data_dir.db());
let db = reth_db::test_utils::create_test_rw_db_with_datadir(data_dir.data_dir());
WithLaunchContext { builder: self.with_database(db), task_executor }
}

View File

@@ -218,9 +218,9 @@ impl<Node: FullNodeComponents, AddOns: NodeAddOns<Node>> DerefMut for FullNode<N
}
/// Helper type alias to define [`FullNode`] for a given [`Node`].
pub type FullNodeFor<N, DB = Arc<DatabaseEnv>> =
pub type FullNodeFor<N, DB = DatabaseEnv> =
FullNode<NodeAdapter<RethFullAdapter<DB, N>>, <N as Node<RethFullAdapter<DB, N>>>::AddOns>;
/// Helper type alias to define [`NodeHandle`] for a given [`Node`].
pub type NodeHandleFor<N, DB = Arc<DatabaseEnv>> =
pub type NodeHandleFor<N, DB = DatabaseEnv> =
NodeHandle<NodeAdapter<RethFullAdapter<DB, N>>, <N as Node<RethFullAdapter<DB, N>>>::AddOns>;

View File

@@ -1192,6 +1192,7 @@ impl<'a, N: FullNodeComponents<Types: NodeTypes<ChainSpec: Hardforks + EthereumH
.pending_block_kind(self.config.pending_block_kind)
.raw_tx_forwarder(self.config.raw_tx_forwarder)
.evm_memory_limit(self.config.rpc_evm_memory_limit)
.force_blob_sidecar_upcasting(self.config.force_blob_sidecar_upcasting)
}
}

View File

@@ -1,7 +1,10 @@
//! clap [Args](clap::Args) for engine purposes
use clap::{builder::Resettable, Args};
use reth_engine_primitives::{TreeConfig, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE};
use reth_engine_primitives::{
TreeConfig, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE, DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES,
DEFAULT_SPARSE_TRIE_PRUNE_DEPTH,
};
use std::sync::OnceLock;
use crate::node_config::{
@@ -38,6 +41,8 @@ pub struct DefaultEngineValues {
disable_proof_v2: bool,
cache_metrics_disabled: bool,
enable_sparse_trie_as_cache: bool,
sparse_trie_prune_depth: usize,
sparse_trie_max_storage_tries: usize,
}
impl DefaultEngineValues {
@@ -179,6 +184,18 @@ impl DefaultEngineValues {
self.enable_sparse_trie_as_cache = v;
self
}
/// Set the sparse trie prune depth by default
pub const fn with_sparse_trie_prune_depth(mut self, v: usize) -> Self {
self.sparse_trie_prune_depth = v;
self
}
/// Set the maximum number of storage tries to retain after sparse trie pruning by default
pub const fn with_sparse_trie_max_storage_tries(mut self, v: usize) -> Self {
self.sparse_trie_max_storage_tries = v;
self
}
}
impl Default for DefaultEngineValues {
@@ -205,6 +222,8 @@ impl Default for DefaultEngineValues {
disable_proof_v2: false,
cache_metrics_disabled: false,
enable_sparse_trie_as_cache: false,
sparse_trie_prune_depth: DEFAULT_SPARSE_TRIE_PRUNE_DEPTH,
sparse_trie_max_storage_tries: DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES,
}
}
}
@@ -336,6 +355,14 @@ pub struct EngineArgs {
/// Enable sparse trie as cache.
#[arg(long = "engine.enable-sparse-trie-as-cache", default_value_t = DefaultEngineValues::get_global().enable_sparse_trie_as_cache, conflicts_with = "disable_proof_v2")]
pub enable_sparse_trie_as_cache: bool,
/// Sparse trie prune depth.
#[arg(long = "engine.sparse-trie-prune-depth", default_value_t = DefaultEngineValues::get_global().sparse_trie_prune_depth, requires = "enable_sparse_trie_as_cache")]
pub sparse_trie_prune_depth: usize,
/// Maximum number of storage tries to retain after sparse trie pruning.
#[arg(long = "engine.sparse-trie-max-storage-tries", default_value_t = DefaultEngineValues::get_global().sparse_trie_max_storage_tries, requires = "enable_sparse_trie_as_cache")]
pub sparse_trie_max_storage_tries: usize,
}
#[allow(deprecated)]
@@ -363,6 +390,8 @@ impl Default for EngineArgs {
disable_proof_v2,
cache_metrics_disabled,
enable_sparse_trie_as_cache,
sparse_trie_prune_depth,
sparse_trie_max_storage_tries,
} = DefaultEngineValues::get_global().clone();
Self {
persistence_threshold,
@@ -390,6 +419,8 @@ impl Default for EngineArgs {
disable_proof_v2,
cache_metrics_disabled,
enable_sparse_trie_as_cache,
sparse_trie_prune_depth,
sparse_trie_max_storage_tries,
}
}
}
@@ -397,7 +428,7 @@ impl Default for EngineArgs {
impl EngineArgs {
/// Creates a [`TreeConfig`] from the engine arguments.
pub fn tree_config(&self) -> TreeConfig {
let mut config = TreeConfig::default()
TreeConfig::default()
.with_persistence_threshold(self.persistence_threshold)
.with_memory_block_buffer_target(self.memory_block_buffer_target)
.with_legacy_state_root(self.legacy_state_root_task_enabled)
@@ -414,21 +445,14 @@ impl EngineArgs {
.with_always_process_payload_attributes_on_canonical_head(
self.always_process_payload_attributes_on_canonical_head,
)
.with_unwind_canonical_header(self.allow_unwind_canonical_header);
if let Some(count) = self.storage_worker_count {
config = config.with_storage_worker_count(count);
}
if let Some(count) = self.account_worker_count {
config = config.with_account_worker_count(count);
}
config = config.with_disable_proof_v2(self.disable_proof_v2);
config = config.without_cache_metrics(self.cache_metrics_disabled);
config = config.with_enable_sparse_trie_as_cache(self.enable_sparse_trie_as_cache);
config
.with_unwind_canonical_header(self.allow_unwind_canonical_header)
.with_storage_worker_count_opt(self.storage_worker_count)
.with_account_worker_count_opt(self.account_worker_count)
.with_disable_proof_v2(self.disable_proof_v2)
.without_cache_metrics(self.cache_metrics_disabled)
.with_enable_sparse_trie_as_cache(self.enable_sparse_trie_as_cache)
.with_sparse_trie_prune_depth(self.sparse_trie_prune_depth)
.with_sparse_trie_max_storage_tries(self.sparse_trie_max_storage_tries)
}
}
@@ -479,7 +503,9 @@ mod tests {
account_worker_count: Some(8),
disable_proof_v2: false,
cache_metrics_disabled: true,
enable_sparse_trie_as_cache: false,
enable_sparse_trie_as_cache: true,
sparse_trie_prune_depth: 10,
sparse_trie_max_storage_tries: 100,
};
let parsed_args = CommandParser::<EngineArgs>::parse_from([
@@ -510,6 +536,11 @@ mod tests {
"--engine.account-worker-count",
"8",
"--engine.disable-cache-metrics",
"--engine.enable-sparse-trie-as-cache",
"--engine.sparse-trie-prune-depth",
"10",
"--engine.sparse-trie-max-storage-tries",
"100",
])
.args;

View File

@@ -6,7 +6,7 @@ use clap::{builder::RangedU64ValueParser, Args};
use reth_chainspec::EthereumHardforks;
use reth_config::config::PruneConfig;
use reth_prune_types::{
PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_UNWIND_SAFE_DISTANCE,
PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_DISTANCE, MINIMUM_UNWIND_SAFE_DISTANCE,
};
use std::{collections::BTreeMap, ops::Not, sync::OnceLock};
@@ -81,7 +81,7 @@ impl Default for DefaultPruningValues {
minimal_prune_modes: PruneModes {
sender_recovery: Some(PruneMode::Full),
transaction_lookup: Some(PruneMode::Full),
receipts: Some(PruneMode::Full),
receipts: Some(PruneMode::Distance(MINIMUM_DISTANCE)),
account_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
storage_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
bodies_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),

View File

@@ -647,6 +647,14 @@ pub struct RpcServerArgs {
/// transactions from the same sender will also be skipped.
#[arg(long = "testing.skip-invalid-transactions", default_value_t = true)]
pub testing_skip_invalid_transactions: bool,
/// Force upcasting EIP-4844 blob sidecars to EIP-7594 format when Osaka is active.
///
/// When enabled, blob transactions submitted via `eth_sendRawTransaction` with EIP-4844
/// sidecars will be automatically converted to EIP-7594 format if the next block is Osaka.
/// By default this is disabled, meaning transactions are submitted as-is.
#[arg(long = "rpc.force-blob-sidecar-upcasting", default_value_t = false)]
pub rpc_force_blob_sidecar_upcasting: bool,
}
impl RpcServerArgs {
@@ -768,6 +776,12 @@ impl RpcServerArgs {
self.rpc_send_raw_transaction_sync_timeout = timeout;
self
}
/// Enables forced blob sidecar upcasting from EIP-4844 to EIP-7594 format.
pub const fn with_force_blob_sidecar_upcasting(mut self) -> Self {
self.rpc_force_blob_sidecar_upcasting = true;
self
}
}
impl Default for RpcServerArgs {
@@ -860,6 +874,7 @@ impl Default for RpcServerArgs {
gas_price_oracle,
rpc_send_raw_transaction_sync_timeout,
testing_skip_invalid_transactions: true,
rpc_force_blob_sidecar_upcasting: false,
}
}
}
@@ -1036,6 +1051,7 @@ mod tests {
},
rpc_send_raw_transaction_sync_timeout: std::time::Duration::from_secs(30),
testing_skip_invalid_transactions: true,
rpc_force_blob_sidecar_upcasting: false,
};
let parsed_args = CommandParser::<RpcServerArgs>::parse_from([

View File

@@ -21,7 +21,7 @@ alloy-primitives.workspace = true
alloy-consensus.workspace = true
tokio.workspace = true
tokio-tungstenite = { workspace = true, features = ["rustls-tls-native-roots"] }
tokio-tungstenite.workspace = true
futures-util.workspace = true
tokio-stream.workspace = true

View File

@@ -37,7 +37,7 @@ tempfile = { workspace = true, optional = true }
tikv-jemalloc-ctl = { workspace = true, optional = true, features = ["stats"] }
[target.'cfg(target_os = "linux")'.dependencies]
procfs = "0.17.0"
procfs = "0.18.0"
[dev-dependencies]
reqwest.workspace = true

View File

@@ -37,7 +37,7 @@ pub use commands::{import::ImportOpCommand, import_receipts::ImportReceiptsOpCom
use reth_optimism_chainspec::OpChainSpec;
use reth_rpc_server_types::{DefaultRpcModuleValidator, RpcModuleValidator};
use std::{ffi::OsString, fmt, marker::PhantomData, sync::Arc};
use std::{ffi::OsString, fmt, marker::PhantomData};
use chainspec::OpChainSpecParser;
use clap::Parser;
@@ -121,7 +121,7 @@ where
/// [`NodeCommand`](reth_cli_commands::node::NodeCommand).
pub fn run<L, Fut>(self, launcher: L) -> eyre::Result<()>
where
L: FnOnce(WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>, Ext) -> Fut,
L: FnOnce(WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>, Ext) -> Fut,
Fut: Future<Output = eyre::Result<()>>,
{
self.with_runner(CliRunner::try_default_runtime()?, launcher)
@@ -130,7 +130,7 @@ where
/// Execute the configured cli command with the provided [`CliRunner`].
pub fn with_runner<L, Fut>(self, runner: CliRunner, launcher: L) -> eyre::Result<()>
where
L: FnOnce(WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>, Ext) -> Fut,
L: FnOnce(WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>, Ext) -> Fut,
Fut: Future<Output = eyre::Result<()>>,
{
let mut this = self.configure();

View File

@@ -38,7 +38,7 @@ op-alloy-rpc-types-engine = { workspace = true, features = ["k256"] }
# io
tokio.workspace = true
tokio-tungstenite = { workspace = true, features = ["rustls-tls-native-roots"] }
tokio-tungstenite.workspace = true
serde_json.workspace = true
url.workspace = true
futures-util.workspace = true

View File

@@ -101,7 +101,7 @@ impl<T: SignedTransaction> SequenceManager<T> {
// Bundle completed sequence with its decoded transactions and push to cache
// Ring buffer automatically evicts oldest entry when full
let txs = std::mem::take(&mut self.pending_transactions);
self.completed_cache.push((completed, txs));
self.completed_cache.enqueue((completed, txs));
// ensure cache is wiped on new flashblock
let _ = self.pending.take_cached_reads();

View File

@@ -2,6 +2,7 @@ use futures_util::stream::StreamExt;
use reth_optimism_flashblocks::WsFlashBlockStream;
#[tokio::test]
#[ignore = "depends on external service availability"]
async fn test_streaming_flashblocks_from_remote_source_is_successful() {
let items = 3;
let ws_url = "wss://sepolia.flashblocks.base.org/ws".parse().unwrap();

View File

@@ -219,10 +219,9 @@ impl OpNode {
/// use reth_optimism_chainspec::OpChainSpecBuilder;
/// use reth_optimism_node::OpNode;
/// use reth_provider::providers::{RocksDBProvider, StaticFileProvider};
/// use std::sync::Arc;
///
/// let factory = OpNode::provider_factory_builder()
/// .db(Arc::new(open_db_read_only("db", Default::default()).unwrap()))
/// .db(open_db_read_only("db", Default::default()).unwrap())
/// .chainspec(OpChainSpecBuilder::base_mainnet().build().into())
/// .static_file(StaticFileProvider::read_only("db/static_files", false).unwrap())
/// .rocksdb_provider(RocksDBProvider::builder("db/rocksdb").build().unwrap())

View File

@@ -435,19 +435,11 @@ impl reth_codecs::Compact for OpTransactionSigned {
let tx_bits = if zstd_bit {
let mut tmp = Vec::with_capacity(256);
if cfg!(feature = "std") {
reth_zstd_compressors::TRANSACTION_COMPRESSOR.with(|compressor| {
let mut compressor = compressor.borrow_mut();
let tx_bits = self.transaction.to_compact(&mut tmp);
buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress"));
tx_bits as u8
})
} else {
let mut compressor = reth_zstd_compressors::create_tx_compressor();
reth_zstd_compressors::with_tx_compressor(|compressor| {
let tx_bits = self.transaction.to_compact(&mut tmp);
buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress"));
tx_bits as u8
}
})
} else {
self.transaction.to_compact(buf) as u8
};
@@ -469,29 +461,15 @@ impl reth_codecs::Compact for OpTransactionSigned {
let zstd_bit = bitflags >> 3;
let (transaction, buf) = if zstd_bit != 0 {
if cfg!(feature = "std") {
reth_zstd_compressors::TRANSACTION_DECOMPRESSOR.with(|decompressor| {
let mut decompressor = decompressor.borrow_mut();
// TODO: enforce that zstd is only present at a "top" level type
let transaction_type = (bitflags & 0b110) >> 1;
let (transaction, _) = OpTypedTransaction::from_compact(
decompressor.decompress(buf),
transaction_type,
);
(transaction, buf)
})
} else {
let mut decompressor = reth_zstd_compressors::create_tx_decompressor();
reth_zstd_compressors::with_tx_decompressor(|decompressor| {
// TODO: enforce that zstd is only present at a "top" level type
let transaction_type = (bitflags & 0b110) >> 1;
let (transaction, _) = OpTypedTransaction::from_compact(
decompressor.decompress(buf),
transaction_type,
);
(transaction, buf)
}
})
} else {
let transaction_type = bitflags >> 1;
OpTypedTransaction::from_compact(buf, transaction_type)

View File

@@ -61,7 +61,7 @@ op-revm.workspace = true
tokio.workspace = true
futures.workspace = true
tokio-stream.workspace = true
reqwest = { workspace = true, features = ["rustls-tls-native-roots"] }
reqwest.workspace = true
async-trait.workspace = true
tower.workspace = true

View File

@@ -188,6 +188,12 @@ pub mod serde_bincode_compat;
pub mod size;
pub use size::InMemorySize;
/// Rayon utilities
#[cfg(feature = "rayon")]
pub mod rayon;
#[cfg(feature = "rayon")]
pub use rayon::ParallelBridgeBuffered;
/// Node traits
pub mod node;
pub use node::{BlockTy, BodyTy, HeaderTy, NodePrimitives, ReceiptTy, TxTy};

View File

@@ -0,0 +1,32 @@
//! Rayon parallel iterator utilities.
use alloc::vec::Vec;
use rayon::iter::IntoParallelIterator;
/// Extension trait for iterators to convert them to parallel iterators via collection.
///
/// This is an alternative to [`rayon::iter::ParallelBridge`] that first collects the iterator
/// into a `Vec`, then calls [`IntoParallelIterator`] on it. This avoids the mutex contention
/// that can occur with `par_bridge` when either the iterator's `next()` method is fast or the
/// parallel tasks are fast, as `par_bridge` wraps the iterator in a mutex.
///
/// # When to use
///
/// Use `par_bridge_buffered` instead of `par_bridge` when:
/// - The iterator produces items quickly
/// - The parallel work per item is relatively light
/// - The total number of items is known to be reasonable for memory
///
/// Stick with `par_bridge` when:
/// - The iterator is slow (e.g., I/O bound) and you want to overlap iteration with processing
/// - Memory is constrained and you cannot afford to collect all items upfront
pub trait ParallelBridgeBuffered: Iterator<Item: Send> + Sized {
/// Collects this iterator into a `Vec` and returns a parallel iterator over it.
///
/// See [this trait's documentation](ParallelBridgeBuffered) for more details.
fn par_bridge_buffered(self) -> rayon::vec::IntoIter<Self::Item> {
self.collect::<Vec<_>>().into_par_iter()
}
}
impl<I: Iterator<Item: Send>> ParallelBridgeBuffered for I {}

View File

@@ -42,11 +42,15 @@ rayon.workspace = true
tokio.workspace = true
rustc-hash.workspace = true
[features]
rocksdb = ["reth-provider/rocksdb"]
[dev-dependencies]
# reth
reth-db = { workspace = true, features = ["test-utils"] }
reth-stages = { workspace = true, features = ["test-utils"] }
reth-stages = { workspace = true, features = ["test-utils", "rocksdb"] }
reth-primitives-traits = { workspace = true, features = ["arbitrary"] }
reth-storage-api.workspace = true
reth-testing-utils.workspace = true
reth-tracing.workspace = true

View File

@@ -7,10 +7,10 @@ use reth_primitives_traits::NodePrimitives;
use reth_provider::{
providers::StaticFileProvider, BlockReader, ChainStateBlockReader, DBProvider,
DatabaseProviderFactory, NodePrimitivesProvider, PruneCheckpointReader, PruneCheckpointWriter,
StageCheckpointReader, StaticFileProviderFactory, StorageSettingsCache,
RocksDBProviderFactory, StageCheckpointReader, StaticFileProviderFactory,
};
use reth_prune_types::PruneModes;
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader};
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader, StorageSettingsCache};
use std::time::Duration;
use tokio::sync::watch;
@@ -85,6 +85,7 @@ impl PrunerBuilder {
+ StageCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ RocksDBProviderFactory
+ StaticFileProviderFactory<
Primitives: NodePrimitives<SignedTx: Value, Receipt: Value, BlockHeader: Value>,
>,
@@ -121,7 +122,8 @@ impl PrunerBuilder {
+ StorageSettingsCache
+ StageCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader,
+ StorageChangeSetReader
+ RocksDBProviderFactory,
{
let segments = SegmentSet::<Provider>::from_components(static_file_provider, self.segments);

View File

@@ -54,6 +54,38 @@ where
})
}
/// Deletes ALL static file jars for a given segment.
///
/// This is used for `PruneMode::Full` where all data should be removed, including the highest jar.
/// Unlike [`prune_static_files`], this does not preserve the most recent jar.
pub(crate) fn delete_static_files_segment<Provider>(
provider: &Provider,
input: PruneInput,
segment: StaticFileSegment,
) -> Result<SegmentOutput, PrunerError>
where
Provider: StaticFileProviderFactory,
{
let deleted_headers = provider.static_file_provider().delete_segment(segment)?;
if deleted_headers.is_empty() {
return Ok(SegmentOutput::done())
}
let tx_ranges = deleted_headers.iter().filter_map(|header| header.tx_range());
let pruned = tx_ranges.clone().map(|range| range.len()).sum::<u64>() as usize;
Ok(SegmentOutput {
progress: PruneProgress::Finished,
pruned,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: Some(input.to_block),
tx_number: tx_ranges.map(|range| range.end()).max(),
}),
})
}
/// A segment represents a pruning of some portion of the data.
///
/// Segments are called from [`Pruner`](crate::Pruner) with the following lifecycle:

View File

@@ -7,10 +7,11 @@ use reth_db_api::{table::Value, transaction::DbTxMut};
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
providers::StaticFileProvider, BlockReader, ChainStateBlockReader, DBProvider,
PruneCheckpointReader, PruneCheckpointWriter, StaticFileProviderFactory, StorageSettingsCache,
PruneCheckpointReader, PruneCheckpointWriter, RocksDBProviderFactory,
StaticFileProviderFactory,
};
use reth_prune_types::PruneModes;
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader};
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader, StorageSettingsCache};
/// Collection of [`Segment`]. Thread-safe, allocated on the heap.
#[derive(Debug)]
@@ -55,7 +56,8 @@ where
+ ChainStateBlockReader
+ StorageSettingsCache
+ ChangeSetReader
+ StorageChangeSetReader,
+ StorageChangeSetReader
+ RocksDBProviderFactory,
{
/// Creates a [`SegmentSet`] from an existing components, such as [`StaticFileProvider`] and
/// [`PruneModes`].

View File

@@ -10,20 +10,20 @@ use alloy_primitives::BlockNumber;
use reth_db_api::{models::ShardedKey, tables, transaction::DbTxMut};
use reth_provider::{
changeset_walker::StaticFileAccountChangesetWalker, DBProvider, EitherWriter,
StaticFileProviderFactory, StorageSettingsCache,
RocksDBProviderFactory, StaticFileProviderFactory,
};
use reth_prune_types::{
PruneMode, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint,
};
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::ChangeSetReader;
use reth_storage_api::{ChangeSetReader, StorageSettingsCache};
use rustc_hash::FxHashMap;
use tracing::{instrument, trace};
/// Number of account history tables to prune in one step.
///
/// Account History consists of two tables: [`tables::AccountChangeSets`] and
/// [`tables::AccountsHistory`]. We want to prune them to the same block number.
/// Account History consists of two tables: [`tables::AccountChangeSets`] (either in database or
/// static files) and [`tables::AccountsHistory`]. We want to prune them to the same block number.
const ACCOUNT_HISTORY_TABLES_TO_PRUNE: usize = 2;
#[derive(Debug)]
@@ -42,7 +42,8 @@ where
Provider: DBProvider<Tx: DbTxMut>
+ StaticFileProviderFactory
+ StorageSettingsCache
+ ChangeSetReader,
+ ChangeSetReader
+ RocksDBProviderFactory,
{
fn segment(&self) -> PruneSegment {
PruneSegment::AccountHistory
@@ -67,7 +68,13 @@ where
};
let range_end = *range.end();
// Check where account changesets are stored
// Check where account history indices are stored
#[cfg(all(unix, feature = "rocksdb"))]
if provider.cached_storage_settings().account_history_in_rocksdb {
return self.prune_rocksdb(provider, input, range, range_end);
}
// Check where account changesets are stored (MDBX path)
if EitherWriter::account_changesets_destination(provider).is_static_file() {
self.prune_static_files(provider, input, range, range_end)
} else {
@@ -94,6 +101,8 @@ impl AccountHistory {
input.limiter
};
// The limiter may already be exhausted from a previous segment in the same prune run.
// Early exit avoids unnecessary iteration when no budget remains.
if limiter.is_limit_reached() {
return Ok(SegmentOutput::not_done(
limiter.interrupt_reason(),
@@ -101,11 +110,14 @@ impl AccountHistory {
))
}
// The size of this map it's limited by `prune_delete_limit * blocks_since_last_run /
// ACCOUNT_HISTORY_TABLES_TO_PRUNE`, and with the current defaults it's usually `3500 * 5 /
// 2`, so 8750 entries. Each entry is `160 bit + 64 bit`, so the total size should be up to
// ~0.25MB + some hashmap overhead. `blocks_since_last_run` is additionally limited by the
// `max_reorg_depth`, so no OOM is expected here.
// Deleted account changeset keys (account addresses) with the highest block number deleted
// for that key.
//
// The size of this map is limited by `prune_delete_limit * blocks_since_last_run /
// ACCOUNT_HISTORY_TABLES_TO_PRUNE`, and with current default it's usually `3500 * 5
// / 2`, so 8750 entries. Each entry is `160 bit + 64 bit`, so the total
// size should be up to ~0.25MB + some hashmap overhead. `blocks_since_last_run` is
// additionally limited by the `max_reorg_depth`, so no OOM is expected here.
let mut highest_deleted_accounts = FxHashMap::default();
let mut last_changeset_pruned_block = None;
let mut pruned_changesets = 0;
@@ -124,8 +136,8 @@ impl AccountHistory {
limiter.increment_deleted_entries_count();
}
// Delete static file jars below the pruned block
if let Some(last_block) = last_changeset_pruned_block {
// Delete static file jars only when fully processed
if done && let Some(last_block) = last_changeset_pruned_block {
provider
.static_file_provider()
.delete_segment_below_block(StaticFileSegment::AccountChangeSets, last_block + 1)?;
@@ -210,6 +222,106 @@ impl AccountHistory {
)
.map_err(Into::into)
}
/// Prunes account history when indices are stored in `RocksDB`.
///
/// Reads account changesets from static files and prunes the corresponding
/// `RocksDB` history shards.
#[cfg(all(unix, feature = "rocksdb"))]
fn prune_rocksdb<Provider>(
&self,
provider: &Provider,
input: PruneInput,
range: std::ops::RangeInclusive<BlockNumber>,
range_end: BlockNumber,
) -> Result<SegmentOutput, PrunerError>
where
Provider: DBProvider + StaticFileProviderFactory + ChangeSetReader + RocksDBProviderFactory,
{
use reth_provider::PruneShardOutcome;
// Unlike MDBX path, we don't divide the limit by 2 because RocksDB path only prunes
// history shards (no separate changeset table to delete from). The changesets are in
// static files which are deleted separately.
let mut limiter = input.limiter;
if limiter.is_limit_reached() {
return Ok(SegmentOutput::not_done(
limiter.interrupt_reason(),
input.previous_checkpoint.map(SegmentOutputCheckpoint::from_prune_checkpoint),
))
}
let mut highest_deleted_accounts = FxHashMap::default();
let mut last_changeset_pruned_block = None;
let mut changesets_processed = 0usize;
let mut done = true;
// Walk account changesets from static files using a streaming iterator.
// For each changeset, track the highest block number seen for each address
// to determine which history shard entries need pruning.
let walker = StaticFileAccountChangesetWalker::new(provider, range);
for result in walker {
if limiter.is_limit_reached() {
done = false;
break;
}
let (block_number, changeset) = result?;
highest_deleted_accounts.insert(changeset.address, block_number);
last_changeset_pruned_block = Some(block_number);
changesets_processed += 1;
limiter.increment_deleted_entries_count();
}
trace!(target: "pruner", processed = %changesets_processed, %done, "Scanned account changesets from static files");
let last_changeset_pruned_block = last_changeset_pruned_block
.map(|block_number| if done { block_number } else { block_number.saturating_sub(1) })
.unwrap_or(range_end);
// Prune RocksDB history shards for affected accounts
let mut deleted_shards = 0usize;
let mut updated_shards = 0usize;
// Sort by address for better RocksDB cache locality
let mut sorted_accounts: Vec<_> = highest_deleted_accounts.into_iter().collect();
sorted_accounts.sort_unstable_by_key(|(addr, _)| *addr);
provider.with_rocksdb_batch(|mut batch| {
for (address, highest_block) in &sorted_accounts {
let prune_to = (*highest_block).min(last_changeset_pruned_block);
match batch.prune_account_history_to(*address, prune_to)? {
PruneShardOutcome::Deleted => deleted_shards += 1,
PruneShardOutcome::Updated => updated_shards += 1,
PruneShardOutcome::Unchanged => {}
}
}
Ok(((), Some(batch.into_inner())))
})?;
trace!(target: "pruner", deleted = deleted_shards, updated = updated_shards, %done, "Pruned account history (RocksDB indices)");
// Delete static file jars only when fully processed. During provider.commit(), RocksDB
// batch is committed before the MDBX checkpoint. If crash occurs after RocksDB commit
// but before MDBX commit, on restart the pruner checkpoint indicates data needs
// re-pruning, but the RocksDB shards are already pruned - this is safe because pruning
// is idempotent (re-pruning already-pruned shards is a no-op).
if done {
provider.static_file_provider().delete_segment_below_block(
StaticFileSegment::AccountChangeSets,
last_changeset_pruned_block + 1,
)?;
}
let progress = limiter.progress(done);
Ok(SegmentOutput {
progress,
pruned: changesets_processed + deleted_shards + updated_shards,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: Some(last_changeset_pruned_block),
tx_number: None,
}),
})
}
}
#[cfg(test)]
@@ -539,4 +651,272 @@ mod tests {
test_prune(998, 2, (PruneProgress::Finished, 1000));
test_prune(1400, 3, (PruneProgress::Finished, 804));
}
#[cfg(all(unix, feature = "rocksdb"))]
#[test]
fn prune_rocksdb_path() {
use reth_db_api::models::ShardedKey;
use reth_provider::{RocksDBProviderFactory, StaticFileProviderFactory};
let db = TestStageDB::default();
let mut rng = generators::rng();
let blocks = random_block_range(
&mut rng,
0..=100,
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() },
);
db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
let accounts = random_eoa_accounts(&mut rng, 2).into_iter().collect::<BTreeMap<_, _>>();
let (changesets, _) = random_changeset_range(
&mut rng,
blocks.iter(),
accounts.into_iter().map(|(addr, acc)| (addr, (acc, Vec::new()))),
0..0,
0..0,
);
db.insert_changesets_to_static_files(changesets.clone(), None)
.expect("insert changesets to static files");
let mut account_blocks: BTreeMap<_, Vec<u64>> = BTreeMap::new();
for (block, changeset) in changesets.iter().enumerate() {
for (address, _, _) in changeset {
account_blocks.entry(*address).or_default().push(block as u64);
}
}
let rocksdb = db.factory.rocksdb_provider();
let mut batch = rocksdb.batch();
for (address, block_numbers) in &account_blocks {
let shard = BlockNumberList::new_pre_sorted(block_numbers.iter().copied());
batch
.put::<tables::AccountsHistory>(ShardedKey::new(*address, u64::MAX), &shard)
.unwrap();
}
batch.commit().unwrap();
for (address, expected_blocks) in &account_blocks {
let shards = rocksdb.account_history_shards(*address).unwrap();
assert_eq!(shards.len(), 1);
assert_eq!(shards[0].1.iter().collect::<Vec<_>>(), *expected_blocks);
}
let to_block: BlockNumber = 50;
let prune_mode = PruneMode::Before(to_block);
let input =
PruneInput { previous_checkpoint: None, to_block, limiter: PruneLimiter::default() };
let segment = AccountHistory::new(prune_mode);
db.factory.set_storage_settings_cache(
StorageSettings::default()
.with_account_changesets_in_static_files(true)
.with_account_history_in_rocksdb(true),
);
let provider = db.factory.database_provider_rw().unwrap();
let result = segment.prune(&provider, input).unwrap();
provider.commit().expect("commit");
assert_matches!(
result,
SegmentOutput { progress: PruneProgress::Finished, pruned, checkpoint: Some(_) }
if pruned > 0
);
for (address, original_blocks) in &account_blocks {
let shards = rocksdb.account_history_shards(*address).unwrap();
let expected_blocks: Vec<u64> =
original_blocks.iter().copied().filter(|b| *b > to_block).collect();
if expected_blocks.is_empty() {
assert!(
shards.is_empty(),
"Expected no shards for address {address:?} after pruning"
);
} else {
assert_eq!(shards.len(), 1, "Expected 1 shard for address {address:?}");
assert_eq!(
shards[0].1.iter().collect::<Vec<_>>(),
expected_blocks,
"Shard blocks mismatch for address {address:?}"
);
}
}
let static_file_provider = db.factory.static_file_provider();
let highest_block = static_file_provider.get_highest_static_file_block(
reth_static_file_types::StaticFileSegment::AccountChangeSets,
);
if let Some(block) = highest_block {
assert!(
block > to_block,
"Static files should only contain blocks above to_block ({to_block}), got {block}"
);
}
}
/// Tests that when a limiter stops mid-block (with multiple changes for the same block),
/// the checkpoint is set to `block_number - 1` to avoid dangling index entries.
#[test]
fn prune_partial_progress_mid_block() {
use alloy_primitives::{Address, U256};
use reth_primitives_traits::Account;
use reth_testing_utils::generators::ChangeSet;
let db = TestStageDB::default();
let mut rng = generators::rng();
// Create blocks 0..=10
let blocks = random_block_range(
&mut rng,
0..=10,
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() },
);
db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
// Create specific changesets where block 5 has 4 account changes
let addr1 = Address::with_last_byte(1);
let addr2 = Address::with_last_byte(2);
let addr3 = Address::with_last_byte(3);
let addr4 = Address::with_last_byte(4);
let addr5 = Address::with_last_byte(5);
let account = Account { nonce: 1, balance: U256::from(100), bytecode_hash: None };
// Build changesets: blocks 0-4 have 1 change each, block 5 has 4 changes, block 6 has 1
let changesets: Vec<ChangeSet> = vec![
vec![(addr1, account, vec![])], // block 0
vec![(addr1, account, vec![])], // block 1
vec![(addr1, account, vec![])], // block 2
vec![(addr1, account, vec![])], // block 3
vec![(addr1, account, vec![])], // block 4
// block 5: 4 different account changes (sorted by address for consistency)
vec![
(addr1, account, vec![]),
(addr2, account, vec![]),
(addr3, account, vec![]),
(addr4, account, vec![]),
],
vec![(addr5, account, vec![])], // block 6
];
db.insert_changesets(changesets.clone(), None).expect("insert changesets");
db.insert_history(changesets.clone(), None).expect("insert history");
// Total changesets: 5 (blocks 0-4) + 4 (block 5) + 1 (block 6) = 10
assert_eq!(
db.table::<tables::AccountChangeSets>().unwrap().len(),
changesets.iter().flatten().count()
);
let prune_mode = PruneMode::Before(10);
// Set limiter to stop after 7 entries (mid-block 5: 5 from blocks 0-4, then 2 of 4 from
// block 5). Due to ACCOUNT_HISTORY_TABLES_TO_PRUNE=2, actual limit is 7/2=3
// changesets. So we'll process blocks 0, 1, 2 (3 changesets), stopping before block
// 3. Actually, let's use a higher limit to reach block 5. With limit=14, we get 7
// changeset slots. Blocks 0-4 use 5 slots, leaving 2 for block 5 (which has 4), so
// we stop mid-block 5.
let deleted_entries_limit = 14; // 14/2 = 7 changeset entries before limit
let limiter = PruneLimiter::default().set_deleted_entries_limit(deleted_entries_limit);
let input = PruneInput { previous_checkpoint: None, to_block: 10, limiter };
let segment = AccountHistory::new(prune_mode);
let provider = db.factory.database_provider_rw().unwrap();
provider.set_storage_settings_cache(
StorageSettings::default().with_account_changesets_in_static_files(false),
);
let result = segment.prune(&provider, input).unwrap();
// Should report that there's more data
assert!(!result.progress.is_finished(), "Expected HasMoreData since we stopped mid-block");
// Save checkpoint and commit
segment
.save_checkpoint(&provider, result.checkpoint.unwrap().as_prune_checkpoint(prune_mode))
.unwrap();
provider.commit().expect("commit");
// Verify checkpoint is set to block 4 (not 5), since block 5 is incomplete
let checkpoint = db
.factory
.provider()
.unwrap()
.get_prune_checkpoint(PruneSegment::AccountHistory)
.unwrap()
.expect("checkpoint should exist");
assert_eq!(
checkpoint.block_number,
Some(4),
"Checkpoint should be block 4 (block before incomplete block 5)"
);
// Verify remaining changesets (block 5 and 6 should still have entries)
let remaining_changesets = db.table::<tables::AccountChangeSets>().unwrap();
// After pruning blocks 0-4, remaining should be block 5 (4 entries) + block 6 (1 entry) = 5
// But since we stopped mid-block 5, some of block 5 might be pruned
// However, checkpoint is 4, so on re-run we should re-process from block 5
assert!(
!remaining_changesets.is_empty(),
"Should have remaining changesets for blocks 5-6"
);
// Verify no dangling history indices for blocks that weren't fully pruned
// The indices for block 5 should still reference blocks <= 5 appropriately
let history = db.table::<tables::AccountsHistory>().unwrap();
for (key, _blocks) in &history {
// All blocks in the history should be > checkpoint block number
// OR the shard's highest_block_number should be > checkpoint
assert!(
key.highest_block_number > 4,
"Found stale history shard with highest_block_number {} <= checkpoint 4",
key.highest_block_number
);
}
// Run prune again to complete - should finish processing block 5 and 6
let input2 = PruneInput {
previous_checkpoint: Some(checkpoint),
to_block: 10,
limiter: PruneLimiter::default().set_deleted_entries_limit(100), // high limit
};
let provider2 = db.factory.database_provider_rw().unwrap();
provider2.set_storage_settings_cache(
StorageSettings::default().with_account_changesets_in_static_files(false),
);
let result2 = segment.prune(&provider2, input2).unwrap();
assert!(result2.progress.is_finished(), "Second run should complete");
segment
.save_checkpoint(
&provider2,
result2.checkpoint.unwrap().as_prune_checkpoint(prune_mode),
)
.unwrap();
provider2.commit().expect("commit");
// Verify final checkpoint
let final_checkpoint = db
.factory
.provider()
.unwrap()
.get_prune_checkpoint(PruneSegment::AccountHistory)
.unwrap()
.expect("checkpoint should exist");
// Should now be at block 6 (the last block with changesets)
assert_eq!(final_checkpoint.block_number, Some(6), "Final checkpoint should be at block 6");
// All changesets should be pruned
let final_changesets = db.table::<tables::AccountChangeSets>().unwrap();
assert!(final_changesets.is_empty(), "All changesets up to block 10 should be pruned");
}
}

View File

@@ -49,6 +49,16 @@ where
fn prune(&self, provider: &Provider, input: PruneInput) -> Result<SegmentOutput, PrunerError> {
if EitherWriterDestination::senders(provider).is_static_file() {
debug!(target: "pruner", "Pruning transaction senders from static files.");
if self.mode.is_full() {
debug!(target: "pruner", "PruneMode::Full: deleting all transaction senders static files.");
return segments::delete_static_files_segment(
provider,
input,
StaticFileSegment::TransactionSenders,
)
}
return segments::prune_static_files(
provider,
input,

View File

@@ -12,7 +12,7 @@ use reth_db_api::{
tables,
transaction::DbTxMut,
};
use reth_provider::{DBProvider, EitherWriter, StaticFileProviderFactory};
use reth_provider::{DBProvider, EitherWriter, RocksDBProviderFactory, StaticFileProviderFactory};
use reth_prune_types::{
PruneMode, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint,
};
@@ -43,7 +43,8 @@ where
Provider: DBProvider<Tx: DbTxMut>
+ StaticFileProviderFactory
+ StorageChangeSetReader
+ StorageSettingsCache,
+ StorageSettingsCache
+ RocksDBProviderFactory,
{
fn segment(&self) -> PruneSegment {
PruneSegment::StorageHistory
@@ -68,6 +69,13 @@ where
};
let range_end = *range.end();
// Check where storage history indices are stored
#[cfg(all(unix, feature = "rocksdb"))]
if provider.cached_storage_settings().storages_history_in_rocksdb {
return self.prune_rocksdb(provider, input, range, range_end);
}
// Check where storage changesets are stored (MDBX path)
if EitherWriter::storage_changesets_destination(provider).is_static_file() {
self.prune_static_files(provider, input, range, range_end)
} else {
@@ -94,6 +102,8 @@ impl StorageHistory {
input.limiter
};
// The limiter may already be exhausted from a previous segment in the same prune run.
// Early exit avoids unnecessary iteration when no budget remains.
if limiter.is_limit_reached() {
return Ok(SegmentOutput::not_done(
limiter.interrupt_reason(),
@@ -126,8 +136,8 @@ impl StorageHistory {
limiter.increment_deleted_entries_count();
}
// Delete static file jars below the pruned block
if let Some(last_block) = last_changeset_pruned_block {
// Delete static file jars only when fully processed
if done && let Some(last_block) = last_changeset_pruned_block {
provider
.static_file_provider()
.delete_segment_below_block(StaticFileSegment::StorageChangeSets, last_block + 1)?;
@@ -216,6 +226,107 @@ impl StorageHistory {
)
.map_err(Into::into)
}
/// Prunes storage history when indices are stored in `RocksDB`.
///
/// Reads storage changesets from static files and prunes the corresponding
/// `RocksDB` history shards.
#[cfg(all(unix, feature = "rocksdb"))]
fn prune_rocksdb<Provider>(
&self,
provider: &Provider,
input: PruneInput,
range: std::ops::RangeInclusive<BlockNumber>,
range_end: BlockNumber,
) -> Result<SegmentOutput, PrunerError>
where
Provider: DBProvider + StaticFileProviderFactory + RocksDBProviderFactory,
{
use reth_provider::PruneShardOutcome;
let mut limiter = input.limiter;
if limiter.is_limit_reached() {
return Ok(SegmentOutput::not_done(
limiter.interrupt_reason(),
input.previous_checkpoint.map(SegmentOutputCheckpoint::from_prune_checkpoint),
))
}
let mut highest_deleted_storages: FxHashMap<_, _> = FxHashMap::default();
let mut last_changeset_pruned_block = None;
let mut changesets_processed = 0usize;
let mut done = true;
// Walk storage changesets from static files using a streaming iterator.
// For each changeset, track the highest block number seen for each (address, storage_key)
// pair to determine which history shard entries need pruning.
let walker = provider.static_file_provider().walk_storage_changeset_range(range);
for result in walker {
if limiter.is_limit_reached() {
done = false;
break;
}
let (block_address, entry) = result?;
let block_number = block_address.block_number();
let address = block_address.address();
highest_deleted_storages.insert((address, entry.key), block_number);
last_changeset_pruned_block = Some(block_number);
changesets_processed += 1;
limiter.increment_deleted_entries_count();
}
trace!(target: "pruner", processed = %changesets_processed, %done, "Scanned storage changesets from static files");
let last_changeset_pruned_block = last_changeset_pruned_block
.map(|block_number| if done { block_number } else { block_number.saturating_sub(1) })
.unwrap_or(range_end);
// Prune RocksDB history shards for affected storage slots
let mut deleted_shards = 0usize;
let mut updated_shards = 0usize;
// Sort by (address, storage_key) for better RocksDB cache locality
let mut sorted_storages: Vec<_> = highest_deleted_storages.into_iter().collect();
sorted_storages.sort_unstable_by_key(|((addr, key), _)| (*addr, *key));
provider.with_rocksdb_batch(|mut batch| {
for ((address, storage_key), highest_block) in &sorted_storages {
let prune_to = (*highest_block).min(last_changeset_pruned_block);
match batch.prune_storage_history_to(*address, *storage_key, prune_to)? {
PruneShardOutcome::Deleted => deleted_shards += 1,
PruneShardOutcome::Updated => updated_shards += 1,
PruneShardOutcome::Unchanged => {}
}
}
Ok(((), Some(batch.into_inner())))
})?;
trace!(target: "pruner", deleted = deleted_shards, updated = updated_shards, %done, "Pruned storage history (RocksDB indices)");
// Delete static file jars only when fully processed. During provider.commit(), RocksDB
// batch is committed before the MDBX checkpoint. If crash occurs after RocksDB commit
// but before MDBX commit, on restart the pruner checkpoint indicates data needs
// re-pruning, but the RocksDB shards are already pruned - this is safe because pruning
// is idempotent (re-pruning already-pruned shards is a no-op).
if done {
provider.static_file_provider().delete_segment_below_block(
StaticFileSegment::StorageChangeSets,
last_changeset_pruned_block + 1,
)?;
}
let progress = limiter.progress(done);
Ok(SegmentOutput {
progress,
pruned: changesets_processed + deleted_shards + updated_shards,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: Some(last_changeset_pruned_block),
tx_number: None,
}),
})
}
}
#[cfg(test)]
@@ -553,4 +664,270 @@ mod tests {
test_prune(998, 2, (PruneProgress::Finished, 500));
test_prune(1200, 3, (PruneProgress::Finished, 202));
}
/// Tests that when a limiter stops mid-block (with multiple storage changes for the same
/// block), the checkpoint is set to `block_number - 1` to avoid dangling index entries.
#[test]
fn prune_partial_progress_mid_block() {
use alloy_primitives::{Address, U256};
use reth_primitives_traits::Account;
use reth_testing_utils::generators::ChangeSet;
let db = TestStageDB::default();
let mut rng = generators::rng();
// Create blocks 0..=10
let blocks = random_block_range(
&mut rng,
0..=10,
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() },
);
db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
// Create specific changesets where block 5 has 4 storage changes
let addr1 = Address::with_last_byte(1);
let addr2 = Address::with_last_byte(2);
let account = Account { nonce: 1, balance: U256::from(100), bytecode_hash: None };
// Create storage entries
let storage_entry = |key: u8| reth_primitives_traits::StorageEntry {
key: B256::with_last_byte(key),
value: U256::from(100),
};
// Build changesets: blocks 0-4 have 1 storage change each, block 5 has 4 changes, block 6
// has 1. Entries within each account must be sorted by key.
let changesets: Vec<ChangeSet> = vec![
vec![(addr1, account, vec![storage_entry(1)])], // block 0
vec![(addr1, account, vec![storage_entry(1)])], // block 1
vec![(addr1, account, vec![storage_entry(1)])], // block 2
vec![(addr1, account, vec![storage_entry(1)])], // block 3
vec![(addr1, account, vec![storage_entry(1)])], // block 4
// block 5: 4 different storage changes (2 addresses, each with 2 storage slots)
// Sorted by address, then by storage key within each address
vec![
(addr1, account, vec![storage_entry(1), storage_entry(2)]),
(addr2, account, vec![storage_entry(1), storage_entry(2)]),
],
vec![(addr1, account, vec![storage_entry(3)])], // block 6
];
db.insert_changesets(changesets.clone(), None).expect("insert changesets");
db.insert_history(changesets.clone(), None).expect("insert history");
// Total storage changesets
let total_storage_entries: usize =
changesets.iter().flat_map(|c| c.iter()).map(|(_, _, entries)| entries.len()).sum();
assert_eq!(db.table::<tables::StorageChangeSets>().unwrap().len(), total_storage_entries);
let prune_mode = PruneMode::Before(10);
// Set limiter to stop mid-block 5
// With STORAGE_HISTORY_TABLES_TO_PRUNE=2, limit=14 gives us 7 storage entries before limit
// Blocks 0-4 use 5 slots, leaving 2 for block 5 (which has 4), so we stop mid-block 5
let deleted_entries_limit = 14; // 14/2 = 7 storage entries before limit
let limiter = PruneLimiter::default().set_deleted_entries_limit(deleted_entries_limit);
let input = PruneInput { previous_checkpoint: None, to_block: 10, limiter };
let segment = StorageHistory::new(prune_mode);
let provider = db.factory.database_provider_rw().unwrap();
provider.set_storage_settings_cache(
StorageSettings::default().with_storage_changesets_in_static_files(false),
);
let result = segment.prune(&provider, input).unwrap();
// Should report that there's more data
assert!(!result.progress.is_finished(), "Expected HasMoreData since we stopped mid-block");
// Save checkpoint and commit
segment
.save_checkpoint(&provider, result.checkpoint.unwrap().as_prune_checkpoint(prune_mode))
.unwrap();
provider.commit().expect("commit");
// Verify checkpoint is set to block 4 (not 5), since block 5 is incomplete
let checkpoint = db
.factory
.provider()
.unwrap()
.get_prune_checkpoint(PruneSegment::StorageHistory)
.unwrap()
.expect("checkpoint should exist");
assert_eq!(
checkpoint.block_number,
Some(4),
"Checkpoint should be block 4 (block before incomplete block 5)"
);
// Verify remaining changesets
let remaining_changesets = db.table::<tables::StorageChangeSets>().unwrap();
assert!(
!remaining_changesets.is_empty(),
"Should have remaining changesets for blocks 5-6"
);
// Verify no dangling history indices for blocks that weren't fully pruned
let history = db.table::<tables::StoragesHistory>().unwrap();
for (key, _blocks) in &history {
assert!(
key.sharded_key.highest_block_number > 4,
"Found stale history shard with highest_block_number {} <= checkpoint 4",
key.sharded_key.highest_block_number
);
}
// Run prune again to complete - should finish processing block 5 and 6
let input2 = PruneInput {
previous_checkpoint: Some(checkpoint),
to_block: 10,
limiter: PruneLimiter::default().set_deleted_entries_limit(100), // high limit
};
let provider2 = db.factory.database_provider_rw().unwrap();
provider2.set_storage_settings_cache(
StorageSettings::default().with_storage_changesets_in_static_files(false),
);
let result2 = segment.prune(&provider2, input2).unwrap();
assert!(result2.progress.is_finished(), "Second run should complete");
segment
.save_checkpoint(
&provider2,
result2.checkpoint.unwrap().as_prune_checkpoint(prune_mode),
)
.unwrap();
provider2.commit().expect("commit");
// Verify final checkpoint
let final_checkpoint = db
.factory
.provider()
.unwrap()
.get_prune_checkpoint(PruneSegment::StorageHistory)
.unwrap()
.expect("checkpoint should exist");
// Should now be at block 6 (the last block with changesets)
assert_eq!(final_checkpoint.block_number, Some(6), "Final checkpoint should be at block 6");
// All changesets should be pruned
let final_changesets = db.table::<tables::StorageChangeSets>().unwrap();
assert!(final_changesets.is_empty(), "All changesets up to block 10 should be pruned");
}
#[cfg(all(unix, feature = "rocksdb"))]
#[test]
fn prune_rocksdb() {
use reth_db_api::models::storage_sharded_key::StorageShardedKey;
use reth_provider::RocksDBProviderFactory;
use reth_storage_api::StorageSettings;
let db = TestStageDB::default();
let mut rng = generators::rng();
let blocks = random_block_range(
&mut rng,
0..=100,
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() },
);
db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
let accounts = random_eoa_accounts(&mut rng, 2).into_iter().collect::<BTreeMap<_, _>>();
let (changesets, _) = random_changeset_range(
&mut rng,
blocks.iter(),
accounts.into_iter().map(|(addr, acc)| (addr, (acc, Vec::new()))),
1..2,
1..2,
);
db.insert_changesets_to_static_files(changesets.clone(), None)
.expect("insert changesets to static files");
let mut storage_indices: BTreeMap<(alloy_primitives::Address, B256), Vec<u64>> =
BTreeMap::new();
for (block, changeset) in changesets.iter().enumerate() {
for (address, _, storage_entries) in changeset {
for entry in storage_entries {
storage_indices.entry((*address, entry.key)).or_default().push(block as u64);
}
}
}
{
let rocksdb = db.factory.rocksdb_provider();
let mut batch = rocksdb.batch();
for ((address, storage_key), block_numbers) in &storage_indices {
let shard = BlockNumberList::new_pre_sorted(block_numbers.clone());
batch
.put::<tables::StoragesHistory>(
StorageShardedKey::last(*address, *storage_key),
&shard,
)
.expect("insert storage history shard");
}
batch.commit().expect("commit rocksdb batch");
}
{
let rocksdb = db.factory.rocksdb_provider();
for (address, storage_key) in storage_indices.keys() {
let shards = rocksdb.storage_history_shards(*address, *storage_key).unwrap();
assert!(!shards.is_empty(), "RocksDB should contain storage history before prune");
}
}
let to_block = 50u64;
let prune_mode = PruneMode::Before(to_block);
let input =
PruneInput { previous_checkpoint: None, to_block, limiter: PruneLimiter::default() };
let segment = StorageHistory::new(prune_mode);
let provider = db.factory.database_provider_rw().unwrap();
provider.set_storage_settings_cache(
StorageSettings::default()
.with_storage_changesets_in_static_files(true)
.with_storages_history_in_rocksdb(true),
);
let result = segment.prune(&provider, input).unwrap();
provider.commit().expect("commit");
assert_matches!(
result,
SegmentOutput { progress: PruneProgress::Finished, checkpoint: Some(_), .. }
);
{
let rocksdb = db.factory.rocksdb_provider();
for ((address, storage_key), block_numbers) in &storage_indices {
let shards = rocksdb.storage_history_shards(*address, *storage_key).unwrap();
let remaining_blocks: Vec<u64> =
block_numbers.iter().copied().filter(|&b| b > to_block).collect();
if remaining_blocks.is_empty() {
assert!(
shards.is_empty(),
"Shard for {:?}/{:?} should be deleted when all blocks pruned",
address,
storage_key
);
} else {
assert!(!shards.is_empty(), "Shard should exist with remaining blocks");
let actual_blocks: Vec<u64> =
shards.iter().flat_map(|(_, list)| list.iter()).collect();
assert_eq!(
actual_blocks, remaining_blocks,
"RocksDB shard should only contain blocks > {}",
to_block
);
}
}
}
}
}

View File

@@ -54,7 +54,7 @@ pub struct PruneModes {
pub transaction_lookup: Option<PruneMode>,
/// Receipts pruning configuration. This setting overrides `receipts_log_filter`
/// and offers improved performance.
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none",))]
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none"))]
pub receipts: Option<PruneMode>,
/// Account History pruning configuration.
#[cfg_attr(
@@ -75,7 +75,7 @@ pub struct PruneModes {
)]
pub storage_history: Option<PruneMode>,
/// Bodies History pruning configuration.
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none",))]
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none"))]
pub bodies_history: Option<PruneMode>,
/// Receipts pruning configuration by retaining only those receipts that contain logs emitted
/// by the specified addresses, discarding others. This setting is overridden by `receipts`.
@@ -112,7 +112,13 @@ impl PruneModes {
///
/// Returns `true` if any migration was performed.
pub const fn migrate(&mut self) -> bool {
false
match &self.receipts {
Some(PruneMode::Full | PruneMode::Distance(0..MINIMUM_DISTANCE)) => {
self.receipts = Some(PruneMode::Distance(MINIMUM_DISTANCE));
true
}
_ => false,
}
}
/// Returns an error if we can't unwind to the targeted block because the target block is

View File

@@ -2,7 +2,7 @@ use alloy_eip7928::BlockAccessList;
use alloy_eips::{BlockId, BlockNumberOrTag};
use alloy_genesis::ChainConfig;
use alloy_json_rpc::RpcObject;
use alloy_primitives::{Address, Bytes, B256};
use alloy_primitives::{Address, Bytes, B256, U64};
use alloy_rpc_types_debug::ExecutionWitness;
use alloy_rpc_types_eth::{Bundle, StateContext};
use alloy_rpc_types_trace::geth::{
@@ -325,7 +325,7 @@ pub trait DebugApi<TxReq: RpcObject> {
/// Sets the current head of the local chain by block number. Note, this is a destructive action
/// and may severely damage your chain. Use with extreme caution.
#[method(name = "setHead")]
async fn debug_set_head(&self, number: u64) -> RpcResult<()>;
async fn debug_set_head(&self, number: U64) -> RpcResult<()>;
/// Sets the rate of mutex profiling.
#[method(name = "setMutexProfileFraction")]

View File

@@ -107,6 +107,7 @@ impl RethRpcServerConfig for RpcServerArgs {
.pending_block_kind(self.rpc_pending_block)
.raw_tx_forwarder(self.rpc_forwarder.clone())
.rpc_evm_memory_limit(self.rpc_evm_memory_limit)
.force_blob_sidecar_upcasting(self.rpc_force_blob_sidecar_upcasting)
}
fn flashbots_config(&self) -> ValidationApiConfig {

View File

@@ -49,7 +49,7 @@ jsonrpsee-types.workspace = true
futures.workspace = true
tokio.workspace = true
tokio-stream.workspace = true
reqwest = { workspace = true, features = ["rustls-tls-native-roots"] }
reqwest.workspace = true
# metrics
metrics.workspace = true

View File

@@ -107,6 +107,11 @@ pub struct EthConfig {
pub send_raw_transaction_sync_timeout: Duration,
/// Maximum memory the EVM can allocate per RPC request.
pub rpc_evm_memory_limit: u64,
/// Whether to force upcasting EIP-4844 blob sidecars to EIP-7594 format when Osaka is active.
///
/// This is disabled by default, allowing blob transactions with EIP-4844 sidecars to be
/// submitted without automatic conversion.
pub force_blob_sidecar_upcasting: bool,
}
impl EthConfig {
@@ -140,6 +145,7 @@ impl Default for EthConfig {
raw_tx_forwarder: ForwardConfig::default(),
send_raw_transaction_sync_timeout: RPC_DEFAULT_SEND_RAW_TX_SYNC_TIMEOUT_SECS,
rpc_evm_memory_limit: (1 << 32) - 1,
force_blob_sidecar_upcasting: false,
}
}
}
@@ -242,6 +248,12 @@ impl EthConfig {
self.rpc_evm_memory_limit = memory_limit;
self
}
/// Configures whether to force upcasting EIP-4844 blob sidecars to EIP-7594 format.
pub const fn force_blob_sidecar_upcasting(mut self, force: bool) -> Self {
self.force_blob_sidecar_upcasting = force;
self
}
}
/// Config for the filter

View File

@@ -71,10 +71,6 @@ revm-primitives = { workspace = true, features = ["serde"] }
# rpc
jsonrpsee.workspace = true
http.workspace = true
http-body.workspace = true
hyper.workspace = true
jsonwebtoken.workspace = true
serde_json.workspace = true
jsonrpsee-types.workspace = true
@@ -82,7 +78,6 @@ jsonrpsee-types.workspace = true
async-trait.workspace = true
tokio = { workspace = true, features = ["sync"] }
tokio-stream.workspace = true
tower.workspace = true
pin-project.workspace = true
parking_lot.workspace = true

View File

@@ -3,7 +3,7 @@ use alloy_eip7928::BlockAccessList;
use alloy_eips::{eip2718::Encodable2718, BlockId, BlockNumberOrTag};
use alloy_evm::env::BlockEnvironment;
use alloy_genesis::ChainConfig;
use alloy_primitives::{hex::decode, uint, Address, Bytes, B256};
use alloy_primitives::{hex::decode, uint, Address, Bytes, B256, U64};
use alloy_rlp::{Decodable, Encodable};
use alloy_rpc_types::BlockTransactionsKind;
use alloy_rpc_types_debug::ExecutionWitness;
@@ -998,7 +998,7 @@ where
Ok(())
}
async fn debug_set_head(&self, _number: u64) -> RpcResult<()> {
async fn debug_set_head(&self, _number: U64) -> RpcResult<()> {
Ok(())
}

View File

@@ -47,6 +47,7 @@ pub struct EthApiBuilder<N: RpcNodeCore, Rpc, NextEnv = ()> {
raw_tx_forwarder: ForwardConfig,
send_raw_transaction_sync_timeout: Duration,
evm_memory_limit: u64,
force_blob_sidecar_upcasting: bool,
}
impl<Provider, Pool, Network, EvmConfig, ChainSpec>
@@ -99,6 +100,7 @@ impl<N: RpcNodeCore, Rpc, NextEnv> EthApiBuilder<N, Rpc, NextEnv> {
raw_tx_forwarder,
send_raw_transaction_sync_timeout,
evm_memory_limit,
force_blob_sidecar_upcasting,
} = self;
EthApiBuilder {
components,
@@ -121,6 +123,7 @@ impl<N: RpcNodeCore, Rpc, NextEnv> EthApiBuilder<N, Rpc, NextEnv> {
raw_tx_forwarder,
send_raw_transaction_sync_timeout,
evm_memory_limit,
force_blob_sidecar_upcasting,
}
}
}
@@ -154,6 +157,7 @@ where
raw_tx_forwarder: ForwardConfig::default(),
send_raw_transaction_sync_timeout: Duration::from_secs(30),
evm_memory_limit: (1 << 32) - 1,
force_blob_sidecar_upcasting: false,
}
}
}
@@ -194,6 +198,7 @@ where
raw_tx_forwarder,
send_raw_transaction_sync_timeout,
evm_memory_limit,
force_blob_sidecar_upcasting,
} = self;
EthApiBuilder {
components,
@@ -216,6 +221,7 @@ where
raw_tx_forwarder,
send_raw_transaction_sync_timeout,
evm_memory_limit,
force_blob_sidecar_upcasting,
}
}
@@ -245,6 +251,7 @@ where
raw_tx_forwarder,
send_raw_transaction_sync_timeout,
evm_memory_limit,
force_blob_sidecar_upcasting,
} = self;
EthApiBuilder {
components,
@@ -267,6 +274,7 @@ where
raw_tx_forwarder,
send_raw_transaction_sync_timeout,
evm_memory_limit,
force_blob_sidecar_upcasting,
}
}
@@ -502,6 +510,7 @@ where
raw_tx_forwarder,
send_raw_transaction_sync_timeout,
evm_memory_limit,
force_blob_sidecar_upcasting,
} = self;
let provider = components.provider().clone();
@@ -544,6 +553,7 @@ where
raw_tx_forwarder.forwarder_client(),
send_raw_transaction_sync_timeout,
evm_memory_limit,
force_blob_sidecar_upcasting,
)
}
@@ -574,4 +584,10 @@ where
self.evm_memory_limit = memory_limit;
self
}
/// Sets whether to force upcasting EIP-4844 blob sidecars to EIP-7594 format.
pub const fn force_blob_sidecar_upcasting(mut self, force: bool) -> Self {
self.force_blob_sidecar_upcasting = force;
self
}
}

View File

@@ -157,6 +157,7 @@ where
raw_tx_forwarder: ForwardConfig,
send_raw_transaction_sync_timeout: Duration,
evm_memory_limit: u64,
force_blob_sidecar_upcasting: bool,
) -> Self {
let inner = EthApiInner::new(
components,
@@ -177,6 +178,7 @@ where
raw_tx_forwarder.forwarder_client(),
send_raw_transaction_sync_timeout,
evm_memory_limit,
force_blob_sidecar_upcasting,
);
Self { inner: Arc::new(inner) }
@@ -333,6 +335,9 @@ pub struct EthApiInner<N: RpcNodeCore, Rpc: RpcConvert> {
/// Maximum memory the EVM can allocate per RPC request.
evm_memory_limit: u64,
/// Whether to force upcasting EIP-4844 blob sidecars to EIP-7594 format when Osaka is active.
force_blob_sidecar_upcasting: bool,
}
impl<N, Rpc> EthApiInner<N, Rpc>
@@ -361,6 +366,7 @@ where
raw_tx_forwarder: Option<RpcClient>,
send_raw_transaction_sync_timeout: Duration,
evm_memory_limit: u64,
force_blob_sidecar_upcasting: bool,
) -> Self {
let signers = parking_lot::RwLock::new(Default::default());
// get the block number of the latest block
@@ -405,6 +411,7 @@ where
send_raw_transaction_sync_timeout,
blob_sidecar_converter: BlobSidecarConverter::new(),
evm_memory_limit,
force_blob_sidecar_upcasting,
}
}
}
@@ -596,6 +603,12 @@ where
pub const fn blocking_io_request_semaphore(&self) -> &Arc<Semaphore> {
&self.blocking_io_request_semaphore
}
/// Returns whether to force upcasting EIP-4844 blob sidecars to EIP-7594 format.
#[inline]
pub const fn force_blob_sidecar_upcasting(&self) -> bool {
self.force_blob_sidecar_upcasting
}
}
#[cfg(test)]

View File

@@ -44,9 +44,9 @@ where
let mut pool_transaction =
<Self::Pool as TransactionPool>::Transaction::from_pooled(recovered);
// TODO: remove this after Osaka transition
// Convert legacy blob sidecars to EIP-7594 format
if pool_transaction.is_eip4844() {
// Optionally convert legacy blob sidecars to EIP-7594 format when Osaka is active
// This is opt-in via --rpc.force-blob-sidecar-upcasting
if self.inner.force_blob_sidecar_upcasting() && pool_transaction.is_eip4844() {
let EthBlobTransactionSidecar::Present(sidecar) = pool_transaction.take_blob() else {
return Err(EthApiError::PoolError(RpcPoolError::Eip4844(
Eip4844PoolTransactionError::MissingEip4844BlobSidecar,

View File

@@ -25,13 +25,6 @@
#![cfg_attr(docsrs, feature(doc_cfg))]
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
use http as _;
use http_body as _;
use hyper as _;
use jsonwebtoken as _;
use pin_project as _;
use tower as _;
mod admin;
mod aliases;
mod debug;

View File

@@ -61,7 +61,7 @@ rayon.workspace = true
num-traits.workspace = true
tempfile = { workspace = true, optional = true }
bincode.workspace = true
reqwest = { workspace = true, default-features = false, features = ["rustls-tls-native-roots", "blocking"] }
reqwest.workspace = true
eyre.workspace = true
[dev-dependencies]

View File

@@ -2,7 +2,7 @@ use reth_db_api::{table::Value, transaction::DbTxMut};
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
BlockReader, ChainStateBlockReader, DBProvider, PruneCheckpointReader, PruneCheckpointWriter,
StageCheckpointReader, StaticFileProviderFactory, StorageSettingsCache,
RocksDBProviderFactory, StageCheckpointReader, StaticFileProviderFactory,
};
use reth_prune::{
PruneMode, PruneModes, PruneSegment, PrunerBuilder, SegmentOutput, SegmentOutputCheckpoint,
@@ -10,7 +10,7 @@ use reth_prune::{
use reth_stages_api::{
ExecInput, ExecOutput, Stage, StageCheckpoint, StageError, StageId, UnwindInput, UnwindOutput,
};
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader};
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader, StorageSettingsCache};
use tracing::info;
/// The prune stage that runs the pruner with the provided prune modes.
@@ -49,7 +49,8 @@ where
Primitives: NodePrimitives<SignedTx: Value, Receipt: Value, BlockHeader: Value>,
> + StorageSettingsCache
+ ChangeSetReader
+ StorageChangeSetReader,
+ StorageChangeSetReader
+ RocksDBProviderFactory,
{
fn id(&self) -> StageId {
StageId::Prune
@@ -156,7 +157,8 @@ where
Primitives: NodePrimitives<SignedTx: Value, Receipt: Value, BlockHeader: Value>,
> + StorageSettingsCache
+ ChangeSetReader
+ StorageChangeSetReader,
+ StorageChangeSetReader
+ RocksDBProviderFactory,
{
fn id(&self) -> StageId {
StageId::PruneSenderRecovery

View File

@@ -232,11 +232,8 @@ where
let mut writer = EitherWriter::new_transaction_hash_numbers(provider, rocksdb_batch)?;
let static_file_provider = provider.static_file_provider();
let rev_walker = provider
.block_body_indices_range(range.clone())?
.into_iter()
.zip(range.collect::<Vec<_>>())
.rev();
let rev_walker =
provider.block_body_indices_range(range.clone())?.into_iter().rev().zip(range.rev());
for (body, number) in rev_walker {
if number <= unwind_to {

View File

@@ -38,4 +38,3 @@ assert_matches.workspace = true
tempfile.workspace = true
[features]
edge = ["reth-stages/edge"]

View File

@@ -13,30 +13,28 @@ workspace = true
[dependencies]
alloy-primitives.workspace = true
tracing = { workspace = true, optional = true }
clap = { workspace = true, features = ["derive"], optional = true }
fixed-map.workspace = true
derive_more.workspace = true
serde = { workspace = true, features = ["alloc", "derive"] }
reth-stages-types.workspace = true
strum = { workspace = true, features = ["derive"] }
[dev-dependencies]
reth-nippy-jar.workspace = true
serde_json.workspace = true
insta.workspace = true
tempfile.workspace = true
[features]
default = ["std"]
std = [
"alloy-primitives/std",
"derive_more/std",
"reth-stages-types/std",
"serde/std",
"strum/std",
"serde_json/std",
"fixed-map/std",
"dep:tracing",
"tracing?/std",
]
clap = ["dep:clap"]

View File

@@ -1,471 +0,0 @@
//! Changeset offset sidecar file I/O.
//!
//! Provides append-only writing and O(1) random-access reading for changeset offsets.
//! The file format is fixed-width 16-byte records: `[offset: u64 LE][num_changes: u64 LE]`.
use crate::ChangesetOffset;
use std::{
fs::{File, OpenOptions},
io::{self, Read, Seek, SeekFrom, Write},
path::Path,
};
/// Writer for appending changeset offsets to a sidecar file.
#[derive(Debug)]
pub struct ChangesetOffsetWriter {
file: File,
/// Number of records written.
records_written: u64,
}
impl ChangesetOffsetWriter {
/// Record size in bytes.
const RECORD_SIZE: usize = 16;
/// Opens or creates the changeset offset file for appending.
///
/// The file is healed to match `committed_len` (from the segment header):
/// - Partial records (from crash mid-write) are truncated to record boundary
/// - Extra complete records (from crash after sidecar sync but before header commit)
/// are truncated to match the committed length
/// - If the file has fewer records than committed, returns an error (data corruption)
///
/// This mirrors NippyJar's healing behavior where config/header is the commit boundary.
pub fn new(path: impl AsRef<Path>, committed_len: u64) -> io::Result<Self> {
let file = OpenOptions::new()
.create(true)
.truncate(false)
.read(true)
.write(true)
.open(path.as_ref())?;
let file_len = file.metadata()?.len();
let remainder = file_len % Self::RECORD_SIZE as u64;
// First, truncate any partial record from crash mid-write
let aligned_len = if remainder != 0 {
let truncated_len = file_len - remainder;
#[cfg(feature = "std")]
tracing::warn!(
target: "reth::static_file",
path = %path.as_ref().display(),
original_len = file_len,
truncated_len,
"Truncating partial changeset offset record"
);
file.set_len(truncated_len)?;
file.sync_all()?; // Sync required for crash safety
truncated_len
} else {
file_len
};
let records_in_file = aligned_len / Self::RECORD_SIZE as u64;
// Heal sidecar to match committed header length
match records_in_file.cmp(&committed_len) {
std::cmp::Ordering::Greater => {
// Sidecar has uncommitted records from a crash - truncate them
let target_len = committed_len * Self::RECORD_SIZE as u64;
#[cfg(feature = "std")]
tracing::warn!(
target: "reth::static_file",
path = %path.as_ref().display(),
sidecar_records = records_in_file,
committed_len,
"Truncating uncommitted changeset offset records after crash recovery"
);
file.set_len(target_len)?;
file.sync_all()?; // Sync required for crash safety
}
std::cmp::Ordering::Less => {
// Unlikely: sidecar is shorter than header claims - data corruption or
// incomplete prune
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"Changeset offset sidecar has {} records but header expects {}: {}",
records_in_file,
committed_len,
path.as_ref().display()
),
));
}
std::cmp::Ordering::Equal => {}
}
let records_written = committed_len;
let file = OpenOptions::new().create(true).append(true).open(path)?;
Ok(Self { file, records_written })
}
/// Appends a single changeset offset record.
pub fn append(&mut self, offset: &ChangesetOffset) -> io::Result<()> {
let mut buf = [0u8; Self::RECORD_SIZE];
buf[..8].copy_from_slice(&offset.offset().to_le_bytes());
buf[8..].copy_from_slice(&offset.num_changes().to_le_bytes());
self.file.write_all(&buf)?;
self.records_written += 1;
Ok(())
}
/// Appends multiple changeset offset records.
pub fn append_many(&mut self, offsets: &[ChangesetOffset]) -> io::Result<()> {
for offset in offsets {
self.append(offset)?;
}
Ok(())
}
/// Syncs all data to disk. Must be called before committing the header.
pub fn sync(&mut self) -> io::Result<()> {
self.file.sync_all()
}
/// Truncates the file to contain exactly `len` records and syncs to disk.
/// Used after prune operations to reclaim space.
///
/// The sync is required for crash safety - without it, a crash could
/// resurrect the old file length.
pub fn truncate(&mut self, len: u64) -> io::Result<()> {
self.file.set_len(len * Self::RECORD_SIZE as u64)?;
self.file.sync_all()?;
self.records_written = len;
Ok(())
}
/// Returns the number of records in the file.
pub const fn len(&self) -> u64 {
self.records_written
}
/// Returns true if the file is empty.
pub const fn is_empty(&self) -> bool {
self.records_written == 0
}
}
/// Reader for changeset offsets with O(1) random access.
#[derive(Debug)]
pub struct ChangesetOffsetReader {
file: File,
/// Cached file length in records.
len: u64,
}
impl ChangesetOffsetReader {
/// Record size in bytes.
const RECORD_SIZE: usize = 16;
/// Opens the changeset offset file for reading.
pub fn new(path: impl AsRef<Path>) -> io::Result<Self> {
let file = File::open(path)?;
let len = file.metadata()?.len() / Self::RECORD_SIZE as u64;
Ok(Self { file, len })
}
/// Opens with an explicit length (from header metadata).
/// Any records beyond `len` are ignored.
///
/// Returns an error if the file has fewer records than `len` (data corruption).
pub fn with_len(path: impl AsRef<Path>, len: u64) -> io::Result<Self> {
let file = File::open(&path)?;
let file_len = file.metadata()?.len();
let records_in_file = file_len / Self::RECORD_SIZE as u64;
if records_in_file < len {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"Changeset offset sidecar has {} records but expected at least {}: {}",
records_in_file,
len,
path.as_ref().display()
),
));
}
Ok(Self { file, len })
}
/// Reads a single changeset offset by block index.
/// Returns None if index is out of bounds.
pub fn get(&mut self, block_index: u64) -> io::Result<Option<ChangesetOffset>> {
if block_index >= self.len {
return Ok(None);
}
let byte_pos = block_index * Self::RECORD_SIZE as u64;
self.file.seek(SeekFrom::Start(byte_pos))?;
let mut buf = [0u8; Self::RECORD_SIZE];
self.file.read_exact(&mut buf)?;
let offset = u64::from_le_bytes(buf[..8].try_into().unwrap());
let num_changes = u64::from_le_bytes(buf[8..].try_into().unwrap());
Ok(Some(ChangesetOffset::new(offset, num_changes)))
}
/// Reads a range of changeset offsets.
pub fn get_range(&mut self, start: u64, end: u64) -> io::Result<Vec<ChangesetOffset>> {
let end = end.min(self.len);
if start >= end {
return Ok(Vec::new());
}
let count = (end - start) as usize;
let byte_pos = start * Self::RECORD_SIZE as u64;
self.file.seek(SeekFrom::Start(byte_pos))?;
let mut result = Vec::with_capacity(count);
let mut buf = [0u8; Self::RECORD_SIZE];
for _ in 0..count {
self.file.read_exact(&mut buf)?;
let offset = u64::from_le_bytes(buf[..8].try_into().unwrap());
let num_changes = u64::from_le_bytes(buf[8..].try_into().unwrap());
result.push(ChangesetOffset::new(offset, num_changes));
}
Ok(result)
}
/// Returns the number of valid records.
pub const fn len(&self) -> u64 {
self.len
}
/// Returns true if there are no records.
pub const fn is_empty(&self) -> bool {
self.len == 0
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
#[test]
fn test_write_and_read() {
let dir = tempdir().unwrap();
let path = dir.path().join("test.csoff");
// Write (new file, committed_len=0)
{
let mut writer = ChangesetOffsetWriter::new(&path, 0).unwrap();
writer.append(&ChangesetOffset::new(0, 5)).unwrap();
writer.append(&ChangesetOffset::new(5, 3)).unwrap();
writer.append(&ChangesetOffset::new(8, 10)).unwrap();
writer.sync().unwrap();
assert_eq!(writer.len(), 3);
}
// Read
{
let mut reader = ChangesetOffsetReader::new(&path).unwrap();
assert_eq!(reader.len(), 3);
let entry = reader.get(0).unwrap().unwrap();
assert_eq!(entry.offset(), 0);
assert_eq!(entry.num_changes(), 5);
let entry = reader.get(1).unwrap().unwrap();
assert_eq!(entry.offset(), 5);
assert_eq!(entry.num_changes(), 3);
let entry = reader.get(2).unwrap().unwrap();
assert_eq!(entry.offset(), 8);
assert_eq!(entry.num_changes(), 10);
assert!(reader.get(3).unwrap().is_none());
}
}
#[test]
fn test_truncate() {
let dir = tempdir().unwrap();
let path = dir.path().join("test.csoff");
let mut writer = ChangesetOffsetWriter::new(&path, 0).unwrap();
writer.append(&ChangesetOffset::new(0, 1)).unwrap();
writer.append(&ChangesetOffset::new(1, 2)).unwrap();
writer.append(&ChangesetOffset::new(3, 3)).unwrap();
writer.sync().unwrap();
writer.truncate(2).unwrap();
assert_eq!(writer.len(), 2);
let mut reader = ChangesetOffsetReader::new(&path).unwrap();
assert_eq!(reader.len(), 2);
assert!(reader.get(2).unwrap().is_none());
}
#[test]
fn test_partial_record_recovery() {
let dir = tempdir().unwrap();
let path = dir.path().join("test.csoff");
// Write 1 full record (16 bytes) + 8 trailing bytes (partial record)
{
let mut file = std::fs::File::create(&path).unwrap();
// Full record: offset=100, num_changes=5
file.write_all(&100u64.to_le_bytes()).unwrap();
file.write_all(&5u64.to_le_bytes()).unwrap();
// Partial record: only 8 bytes (incomplete)
file.write_all(&200u64.to_le_bytes()).unwrap();
file.sync_all().unwrap();
}
// Verify file has 24 bytes before opening with writer
assert_eq!(std::fs::metadata(&path).unwrap().len(), 24);
// Open with writer, committed_len=1 (header committed 1 record)
// Should truncate partial record and match committed length
let writer = ChangesetOffsetWriter::new(&path, 1).unwrap();
assert_eq!(writer.len(), 1);
// Verify file was truncated to 16 bytes
assert_eq!(std::fs::metadata(&path).unwrap().len(), 16);
// Verify the complete record is readable
let mut reader = ChangesetOffsetReader::new(&path).unwrap();
assert_eq!(reader.len(), 1);
let entry = reader.get(0).unwrap().unwrap();
assert_eq!(entry.offset(), 100);
assert_eq!(entry.num_changes(), 5);
}
#[test]
fn test_with_len_bounds_reads() {
let dir = tempdir().unwrap();
let path = dir.path().join("test.csoff");
// Write 3 records
{
let mut writer = ChangesetOffsetWriter::new(&path, 0).unwrap();
writer.append(&ChangesetOffset::new(0, 10)).unwrap();
writer.append(&ChangesetOffset::new(10, 20)).unwrap();
writer.append(&ChangesetOffset::new(30, 30)).unwrap();
writer.sync().unwrap();
assert_eq!(writer.len(), 3);
}
// Open with explicit len=2, ignoring the 3rd record
let mut reader = ChangesetOffsetReader::with_len(&path, 2).unwrap();
assert_eq!(reader.len(), 2);
// First two records should be readable
let entry0 = reader.get(0).unwrap().unwrap();
assert_eq!(entry0.offset(), 0);
assert_eq!(entry0.num_changes(), 10);
let entry1 = reader.get(1).unwrap().unwrap();
assert_eq!(entry1.offset(), 10);
assert_eq!(entry1.num_changes(), 20);
// Third record should be out of bounds (due to len=2)
assert!(reader.get(2).unwrap().is_none());
// get_range should also respect the len bound
let range = reader.get_range(0, 5).unwrap();
assert_eq!(range.len(), 2);
}
#[test]
fn test_truncate_uncommitted_records_on_open() {
// Simulates crash recovery where sidecar has more records than committed header length.
// ChangesetOffsetWriter::new() should automatically truncate to committed_len.
let dir = tempdir().unwrap();
let path = dir.path().join("test.csoff");
// Simulate: wrote 3 records, synced sidecar, but header only committed len=2
{
let mut writer = ChangesetOffsetWriter::new(&path, 0).unwrap();
writer.append(&ChangesetOffset::new(0, 5)).unwrap();
writer.append(&ChangesetOffset::new(5, 10)).unwrap();
writer.append(&ChangesetOffset::new(15, 7)).unwrap(); // uncommitted
writer.sync().unwrap();
assert_eq!(writer.len(), 3);
}
// On "restart", new() heals by truncating to committed length
let committed_len = 2u64;
{
let writer = ChangesetOffsetWriter::new(&path, committed_len).unwrap();
assert_eq!(writer.len(), 2); // Healed to committed length
}
// Verify file is now correct length and new appends go to the right place
{
let mut writer = ChangesetOffsetWriter::new(&path, 2).unwrap();
assert_eq!(writer.len(), 2);
// Append a new record - should be at index 2, not index 3
writer.append(&ChangesetOffset::new(15, 20)).unwrap();
writer.sync().unwrap();
assert_eq!(writer.len(), 3);
}
// Verify the records are correct
{
let mut reader = ChangesetOffsetReader::new(&path).unwrap();
assert_eq!(reader.len(), 3);
let entry0 = reader.get(0).unwrap().unwrap();
assert_eq!(entry0.offset(), 0);
assert_eq!(entry0.num_changes(), 5);
let entry1 = reader.get(1).unwrap().unwrap();
assert_eq!(entry1.offset(), 5);
assert_eq!(entry1.num_changes(), 10);
// This should be the NEW record, not the old uncommitted one
let entry2 = reader.get(2).unwrap().unwrap();
assert_eq!(entry2.offset(), 15);
assert_eq!(entry2.num_changes(), 20); // Not 7 from the old uncommitted record
}
}
#[test]
fn test_sidecar_shorter_than_committed_errors() {
// If sidecar has fewer records than committed, it's data corruption - should error.
let dir = tempdir().unwrap();
let path = dir.path().join("test.csoff");
// Write 1 record
{
let mut writer = ChangesetOffsetWriter::new(&path, 0).unwrap();
writer.append(&ChangesetOffset::new(0, 5)).unwrap();
writer.sync().unwrap();
}
// Try to open with committed_len=3 (header claims more than file has)
let result = ChangesetOffsetWriter::new(&path, 3);
assert!(result.is_err());
let err = result.unwrap_err();
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
}
#[test]
fn test_reader_with_len_shorter_than_file_errors() {
// If header claims more records than file has, with_len should error (data corruption).
let dir = tempdir().unwrap();
let path = dir.path().join("test.csoff");
// Write 1 record
{
let mut writer = ChangesetOffsetWriter::new(&path, 0).unwrap();
writer.append(&ChangesetOffset::new(0, 5)).unwrap();
writer.sync().unwrap();
}
// Try to open reader with len=3 (header claims more than file has)
let result = ChangesetOffsetReader::with_len(&path, 3);
assert!(result.is_err());
let err = result.unwrap_err();
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
}
}

View File

@@ -15,18 +15,11 @@ mod compression;
mod event;
mod segment;
#[cfg(feature = "std")]
mod changeset_offsets;
#[cfg(feature = "std")]
pub use changeset_offsets::{ChangesetOffsetReader, ChangesetOffsetWriter};
use alloy_primitives::BlockNumber;
pub use compression::Compression;
use core::ops::RangeInclusive;
pub use event::StaticFileProducerEvent;
pub use segment::{
ChangesetOffset, SegmentConfig, SegmentHeader, SegmentRangeInclusive, StaticFileSegment,
};
pub use segment::{SegmentConfig, SegmentHeader, SegmentRangeInclusive, StaticFileSegment};
/// Map keyed by [`StaticFileSegment`].
pub type StaticFileMap<T> = alloc::boxed::Box<fixed_map::Map<StaticFileSegment, T>>;

View File

@@ -1,10 +1,11 @@
use crate::{find_fixed_range, BlockNumber, Compression};
use alloc::{format, string::String};
use alloc::{format, string::String, vec::Vec};
use alloy_primitives::TxNumber;
use core::{
ops::{Range, RangeInclusive},
str::FromStr,
};
use reth_stages_types::StageId;
use serde::{de::Visitor, ser::SerializeStruct, Deserialize, Deserializer, Serialize, Serializer};
use strum::{EnumIs, EnumString};
@@ -198,6 +199,18 @@ impl StaticFileSegment {
pub const fn is_block_or_change_based(&self) -> bool {
self.is_block_based() || self.is_change_based()
}
/// Maps this segment to the [`StageId`] responsible for it.
pub const fn to_stage_id(&self) -> StageId {
match self {
Self::Headers => StageId::Headers,
Self::Transactions => StageId::Bodies,
Self::Receipts | Self::AccountChangeSets | Self::StorageChangeSets => {
StageId::Execution
}
Self::TransactionSenders => StageId::SenderRecovery,
}
}
}
/// A changeset offset, also with the number of elements in the offset for convenience
@@ -211,11 +224,6 @@ pub struct ChangesetOffset {
}
impl ChangesetOffset {
/// Creates a new changeset offset.
pub const fn new(offset: u64, num_changes: u64) -> Self {
Self { offset, num_changes }
}
/// Returns the start offset for the row for this block
pub const fn offset(&self) -> u64 {
self.offset
@@ -230,11 +238,6 @@ impl ChangesetOffset {
pub const fn changeset_range(&self) -> Range<u64> {
self.offset..(self.offset + self.num_changes)
}
/// Increments the number of changes by 1.
pub const fn increment_num_changes(&mut self) {
self.num_changes += 1;
}
}
/// A segment header that contains information common to all segments. Used for storage.
@@ -252,7 +255,7 @@ pub struct SegmentHeader {
/// Segment type
segment: StaticFileSegment,
/// List of offsets, for where each block's changeset starts.
changeset_offsets_len: u64,
changeset_offsets: Option<Vec<ChangesetOffset>>,
}
struct SegmentHeaderVisitor;
@@ -281,18 +284,21 @@ impl<'de> Visitor<'de> for SegmentHeaderVisitor {
let segment: StaticFileSegment =
seq.next_element()?.ok_or_else(|| serde::de::Error::invalid_length(3, &self))?;
let changeset_offsets_len = if segment.is_change_based() {
// Try to read the 5th field (changeset_offsets_len)
match seq.next_element::<u64>()? {
Some(len) => len,
let changeset_offsets = if segment.is_change_based() {
// Try to read the 5th field (changeset_offsets)
// If it doesn't exist (old format), this will return None
match seq.next_element()? {
Some(Some(offsets)) => Some(offsets),
// Changesets should have offsets
Some(None) => None,
None => {
return Err(serde::de::Error::custom(
"changeset_offsets_len should exist for changeset static files",
"changeset_offsets should exist for static files",
))
}
}
} else {
0
None
};
Ok(SegmentHeader {
@@ -300,7 +306,7 @@ impl<'de> Visitor<'de> for SegmentHeaderVisitor {
block_range,
tx_range,
segment,
changeset_offsets_len,
changeset_offsets,
})
}
}
@@ -335,7 +341,7 @@ impl Serialize for SegmentHeader {
state.serialize_field("segment", &self.segment)?;
if self.segment.is_change_based() {
state.serialize_field("changeset_offsets", &self.changeset_offsets_len)?;
state.serialize_field("changeset_offsets", &self.changeset_offsets)?;
}
state.end()
@@ -350,7 +356,7 @@ impl SegmentHeader {
tx_range: Option<SegmentRangeInclusive>,
segment: StaticFileSegment,
) -> Self {
Self { expected_block_range, block_range, tx_range, segment, changeset_offsets_len: 0 }
Self { expected_block_range, block_range, tx_range, segment, changeset_offsets: None }
}
/// Returns the static file segment kind.
@@ -373,20 +379,9 @@ impl SegmentHeader {
self.tx_range
}
/// Returns the number of changeset offset entries.
/// The actual offsets are stored in the .csoff sidecar file.
pub const fn changeset_offsets_len(&self) -> u64 {
self.changeset_offsets_len
}
/// Sets the changeset offsets length.
pub const fn set_changeset_offsets_len(&mut self, len: u64) {
self.changeset_offsets_len = len;
}
/// Increments the changeset offsets length by 1.
pub const fn increment_changeset_offsets_len(&mut self) {
self.changeset_offsets_len += 1;
/// Returns the changeset offsets.
pub const fn changeset_offsets(&self) -> Option<&Vec<ChangesetOffset>> {
self.changeset_offsets.as_ref()
}
/// The expected block start of the segment.
@@ -444,7 +439,7 @@ impl SegmentHeader {
}
/// Increments block end range depending on segment
pub const fn increment_block(&mut self) -> BlockNumber {
pub fn increment_block(&mut self) -> BlockNumber {
let block_num = if let Some(block_range) = &mut self.block_range {
block_range.end += 1;
block_range.end
@@ -456,10 +451,20 @@ impl SegmentHeader {
self.expected_block_start()
};
// For changeset segments, increment the offsets length.
// The actual offset entry is written to the sidecar file by the caller.
// For changeset segments, initialize an offset entry for the new block
if self.segment.is_change_based() {
self.changeset_offsets_len += 1;
let offsets = self.changeset_offsets.get_or_insert_default();
// Calculate the offset for the new block
let new_offset = if let Some(last_offset) = offsets.last() {
// The new block starts after the last block's changes
last_offset.offset + last_offset.num_changes
} else {
// First block starts at offset 0
0
};
// Add a new offset entry with 0 changes initially
offsets.push(ChangesetOffset { offset: new_offset, num_changes: 0 });
}
block_num
@@ -476,11 +481,23 @@ impl SegmentHeader {
}
}
/// Increments the latest block's number of changes.
pub fn increment_block_changes(&mut self) {
debug_assert!(self.segment().is_change_based());
if self.segment.is_change_based() {
let offsets = self.changeset_offsets.get_or_insert_with(Default::default);
if let Some(last_offset) = offsets.last_mut() {
last_offset.num_changes += 1;
} else {
// If offsets is empty, we are adding the first change for a block
// The offset for the first block is 0
offsets.push(ChangesetOffset { offset: 0, num_changes: 1 });
}
}
}
/// Removes `num` elements from end of tx or block range.
///
/// For changeset segments, also decrements the changeset offsets length.
/// The caller must truncate the sidecar file accordingly.
pub const fn prune(&mut self, num: u64) {
pub fn prune(&mut self, num: u64) {
// Changesets also contain a block range, but are not strictly block-based
if self.segment.is_block_or_change_based() {
if let Some(range) = &mut self.block_range {
@@ -488,18 +505,26 @@ impl SegmentHeader {
self.block_range = None;
// Clear all changeset offsets if we're clearing all blocks
if self.segment.is_change_based() {
self.changeset_offsets_len = 0;
self.changeset_offsets = None;
}
} else {
let old_end = range.end;
range.end = range.end.saturating_sub(num);
// Update changeset offsets length for changeset segments
if self.segment.is_change_based() {
// Update changeset offsets for account changesets
if self.segment.is_change_based() &&
let Some(offsets) = &mut self.changeset_offsets
{
// Calculate how many blocks we're removing
let blocks_to_remove = old_end - range.end;
self.changeset_offsets_len =
self.changeset_offsets_len.saturating_sub(blocks_to_remove);
// Remove the last `blocks_to_remove` entries from offsets
let new_len = offsets.len().saturating_sub(blocks_to_remove as usize);
offsets.truncate(new_len);
// If we removed all offsets, set to None
if offsets.is_empty() {
self.changeset_offsets = None;
}
}
}
};
@@ -522,24 +547,28 @@ impl SegmentHeader {
}
}
/// Synchronizes changeset offsets length with the current block range for changeset segments.
/// Synchronizes changeset offsets with the current block range for account changeset segments.
///
/// This should be called after modifying the block range when dealing with changeset segments
/// to ensure the offsets length matches the block range size.
/// The caller must also truncate the sidecar file accordingly.
pub const fn sync_changeset_offsets(&mut self) {
/// to ensure the offsets vector matches the block range size.
pub fn sync_changeset_offsets(&mut self) {
if !self.segment.is_change_based() {
return;
}
if let Some(block_range) = &self.block_range {
let expected_len = block_range.end - block_range.start + 1;
if self.changeset_offsets_len > expected_len {
self.changeset_offsets_len = expected_len;
if let Some(offsets) = &mut self.changeset_offsets {
let expected_len = (block_range.end - block_range.start + 1) as usize;
if offsets.len() > expected_len {
offsets.truncate(expected_len);
if offsets.is_empty() {
self.changeset_offsets = None;
}
}
}
} else {
// No block range means no offsets
self.changeset_offsets_len = 0;
self.changeset_offsets = None;
}
}
@@ -565,22 +594,21 @@ impl SegmentHeader {
self.tx_start()
}
/// Returns the index into the sidecar file for a given block's changeset offset.
/// Returns the `ChangesetOffset` corresponding for the given block, if it's in the block
/// range.
///
/// Returns `None` if the block is not in the block range.
/// The caller must read the actual offset from the sidecar file at this index.
pub fn changeset_offset_index(&self, block: BlockNumber) -> Option<u64> {
/// If it is not in the block range or the changeset list in the header does not contain a
/// value for the block, this returns `None`.
pub fn changeset_offset(&self, block: BlockNumber) -> Option<&ChangesetOffset> {
let block_range = self.block_range()?;
if !block_range.contains(block) {
return None;
return None
}
let index = block - block_range.start();
if index >= self.changeset_offsets_len {
return None;
}
let offsets = self.changeset_offsets.as_ref()?;
let index = (block - block_range.start()) as usize;
Some(index)
offsets.get(index)
}
}
@@ -739,42 +767,42 @@ mod tests {
block_range: Some(SegmentRangeInclusive::new(0, 100)),
tx_range: None,
segment: StaticFileSegment::Headers,
changeset_offsets_len: 0,
changeset_offsets: None,
},
SegmentHeader {
expected_block_range: SegmentRangeInclusive::new(0, 200),
block_range: None,
tx_range: Some(SegmentRangeInclusive::new(0, 300)),
segment: StaticFileSegment::Transactions,
changeset_offsets_len: 0,
changeset_offsets: None,
},
SegmentHeader {
expected_block_range: SegmentRangeInclusive::new(0, 200),
block_range: Some(SegmentRangeInclusive::new(0, 100)),
tx_range: Some(SegmentRangeInclusive::new(0, 300)),
segment: StaticFileSegment::Receipts,
changeset_offsets_len: 0,
changeset_offsets: None,
},
SegmentHeader {
expected_block_range: SegmentRangeInclusive::new(0, 200),
block_range: Some(SegmentRangeInclusive::new(0, 100)),
tx_range: Some(SegmentRangeInclusive::new(0, 300)),
segment: StaticFileSegment::TransactionSenders,
changeset_offsets_len: 0,
changeset_offsets: None,
},
SegmentHeader {
expected_block_range: SegmentRangeInclusive::new(0, 200),
block_range: Some(SegmentRangeInclusive::new(0, 100)),
tx_range: Some(SegmentRangeInclusive::new(0, 300)),
segment: StaticFileSegment::AccountChangeSets,
changeset_offsets_len: 100,
changeset_offsets: Some(vec![ChangesetOffset { offset: 1, num_changes: 1 }; 100]),
},
SegmentHeader {
expected_block_range: SegmentRangeInclusive::new(0, 200),
block_range: Some(SegmentRangeInclusive::new(0, 100)),
tx_range: None,
segment: StaticFileSegment::StorageChangeSets,
changeset_offsets_len: 100,
changeset_offsets: Some(vec![ChangesetOffset { offset: 1, num_changes: 1 }; 100]),
},
];
// Check that we test all segments

View File

@@ -2,4 +2,4 @@
source: crates/static-file/types/src/segment.rs
expression: "Bytes::from(serialized)"
---
0x01000000000000000000000000000000c80000000000000001000000000000000064000000000000000100000000000000002c0100000000000004000000640000000000000001000000000000000000000000000000000000000000000000
0x01000000000000000000000000000000c80000000000000001000000000000000064000000000000000100000000000000002c01000000000000040000000164000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000000000000000000000000000000000000

View File

@@ -2,4 +2,4 @@
source: crates/static-file/types/src/segment.rs
expression: "Bytes::from(serialized)"
---
0x01000000000000000000000000000000c80000000000000001000000000000000064000000000000000005000000640000000000000001000000000000000000000000000000000000000000000000
0x01000000000000000000000000000000c800000000000000010000000000000000640000000000000000050000000164000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000000000000000000000000000000000000

View File

@@ -133,8 +133,7 @@ fn generate_from_compact(
let decompressor = zstd.decompressor;
quote! {
if flags.__zstd() != 0 {
#decompressor.with(|decompressor| {
let decompressor = &mut decompressor.borrow_mut();
#decompressor(|decompressor| {
let decompressed = decompressor.decompress(buf);
let mut original_buf = buf;
@@ -203,9 +202,7 @@ fn generate_to_compact(
let compressor = zstd.compressor;
lines.push(quote! {
if zstd {
#compressor.with(|compressor| {
let mut compressor = compressor.borrow_mut();
#compressor(|compressor| {
let compressed = compressor.compress(&buffer).expect("Failed to compress.");
buf.put(compressed.as_slice());
});

View File

@@ -10,8 +10,8 @@ use reth_codecs_derive::CompactZstd;
#[derive(CompactZstd)]
#[reth_codecs(crate = "crate")]
#[reth_zstd(
compressor = reth_zstd_compressors::RECEIPT_COMPRESSOR,
decompressor = reth_zstd_compressors::RECEIPT_DECOMPRESSOR
compressor = reth_zstd_compressors::with_receipt_compressor,
decompressor = reth_zstd_compressors::with_receipt_decompressor
)]
struct CompactOpReceipt<'a> {
tx_type: OpTxType,

View File

@@ -149,20 +149,9 @@ impl<T: Envelope + ToTxCompact + Transaction + Send + Sync> CompactEnvelope for
self.to_tx_compact(&mut tx_buf);
buf.put_slice(
&{
#[cfg(feature = "std")]
{
reth_zstd_compressors::TRANSACTION_COMPRESSOR.with(|compressor| {
let mut compressor = compressor.borrow_mut();
compressor.compress(&tx_buf)
})
}
#[cfg(not(feature = "std"))]
{
let mut compressor = reth_zstd_compressors::create_tx_compressor();
compressor.compress(&tx_buf)
}
}
&reth_zstd_compressors::with_tx_compressor(|compressor| {
compressor.compress(&tx_buf)
})
.expect("Failed to compress"),
);
tx_bits
@@ -188,27 +177,12 @@ impl<T: Envelope + ToTxCompact + Transaction + Send + Sync> CompactEnvelope for
let (signature, buf) = Signature::from_compact(buf, sig_bit);
let (transaction, buf) = if zstd_bit != 0 {
#[cfg(feature = "std")]
{
reth_zstd_compressors::TRANSACTION_DECOMPRESSOR.with(|decompressor| {
let mut decompressor = decompressor.borrow_mut();
let decompressed = decompressor.decompress(buf);
let (tx_type, tx_buf) = T::TxType::from_compact(decompressed, tx_bits);
let (tx, _) = Self::from_tx_compact(tx_buf, tx_type, signature);
(tx, buf)
})
}
#[cfg(not(feature = "std"))]
{
let mut decompressor = reth_zstd_compressors::create_tx_decompressor();
reth_zstd_compressors::with_tx_decompressor(|decompressor| {
let decompressed = decompressor.decompress(buf);
let (tx_type, tx_buf) = T::TxType::from_compact(decompressed, tx_bits);
let (tx, _) = Self::from_tx_compact(tx_buf, tx_type, signature);
(tx, buf)
}
})
} else {
let (tx_type, buf) = T::TxType::from_compact(buf, tx_bits);
Self::from_tx_compact(buf, tx_type, signature)

View File

@@ -62,7 +62,7 @@ impl IntegerList {
/// Pushes a new integer to the list.
pub fn push(&mut self, value: u64) -> Result<(), IntegerListError> {
self.0.push(value).then_some(()).ok_or(IntegerListError::UnsortedInput)
self.0.try_push(value).map_err(|_| IntegerListError::UnsortedInput)
}
/// Clears the list.

View File

@@ -960,8 +960,8 @@ mod tests {
let provider = factory.provider().unwrap();
let tx = provider.tx_ref();
(
collect_table_entries::<Arc<DatabaseEnv>, tables::AccountsHistory>(tx).unwrap(),
collect_table_entries::<Arc<DatabaseEnv>, tables::StoragesHistory>(tx).unwrap(),
collect_table_entries::<DatabaseEnv, tables::AccountsHistory>(tx).unwrap(),
collect_table_entries::<DatabaseEnv, tables::StoragesHistory>(tx).unwrap(),
)
};

View File

@@ -374,10 +374,9 @@ mod tests {
transaction::{DbTx, DbTxMut},
};
use reth_primitives_traits::StorageEntry;
use std::sync::Arc;
use tempfile::TempDir;
fn create_test_db() -> Arc<DatabaseEnv> {
fn create_test_db() -> DatabaseEnv {
let path = TempDir::new().unwrap();
let mut db = DatabaseEnv::open(
path.path(),
@@ -386,7 +385,7 @@ mod tests {
)
.unwrap();
db.create_tables().unwrap();
Arc::new(db)
db
}
#[test]

View File

@@ -222,7 +222,7 @@ impl DatabaseArguments {
}
/// Wrapper for the libmdbx environment: [Environment]
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct DatabaseEnv {
/// Libmdbx-sys environment.
inner: Environment,
@@ -488,6 +488,10 @@ impl DatabaseEnv {
inner_env.set_max_read_transaction_duration(max_read_transaction_duration);
}
// Disable prefault writes: avoids mincore() syscall overhead since pages are likely
// already warm from Engine Task reads in WRITEMAP mode.
inner_env.set_prefault_write(false);
let env = Self {
inner: inner_env.open(path).map_err(|e| DatabaseError::Open(e.into()))?,
dbis: Arc::default(),
@@ -644,11 +648,8 @@ mod tests {
use tempfile::TempDir;
/// Create database for testing
fn create_test_db(kind: DatabaseEnvKind) -> Arc<DatabaseEnv> {
Arc::new(create_test_db_with_path(
kind,
&tempfile::TempDir::new().expect(ERROR_TEMPDIR).keep(),
))
fn create_test_db(kind: DatabaseEnvKind) -> DatabaseEnv {
create_test_db_with_path(kind, &tempfile::TempDir::new().expect(ERROR_TEMPDIR).keep())
}
/// Create database for testing with specified path
@@ -737,7 +738,7 @@ mod tests {
#[test]
fn db_dup_cursor_delete_first() {
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let tx = db.tx_mut().expect(ERROR_INIT_TX);
let mut dup_cursor = tx.cursor_dup_write::<PlainStorageState>().unwrap();
@@ -802,7 +803,7 @@ mod tests {
#[test]
fn db_cursor_walk_range() {
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
// PUT (0, 0), (1, 0), (2, 0), (3, 0)
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -866,7 +867,7 @@ mod tests {
#[test]
fn db_cursor_walk_range_on_dup_table() {
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let address0 = Address::ZERO;
let address1 = Address::with_last_byte(1);
@@ -926,7 +927,7 @@ mod tests {
#[expect(clippy::reversed_empty_ranges)]
#[test]
fn db_cursor_walk_range_invalid() {
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
// PUT (0, 0), (1, 0), (2, 0), (3, 0)
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -954,7 +955,7 @@ mod tests {
#[test]
fn db_walker() {
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
// PUT (0, 0), (1, 0), (3, 0)
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -984,7 +985,7 @@ mod tests {
#[test]
fn db_reverse_walker() {
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
// PUT (0, 0), (1, 0), (3, 0)
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -1014,7 +1015,7 @@ mod tests {
#[test]
fn db_walk_back() {
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
// PUT (0, 0), (1, 0), (3, 0)
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -1053,7 +1054,7 @@ mod tests {
#[test]
fn db_cursor_seek_exact_or_previous_key() {
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
// PUT
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -1077,7 +1078,7 @@ mod tests {
#[test]
fn db_cursor_insert() {
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
// PUT
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -1117,7 +1118,7 @@ mod tests {
#[test]
fn db_cursor_insert_dup() {
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let tx = db.tx_mut().expect(ERROR_INIT_TX);
let mut dup_cursor = tx.cursor_dup_write::<PlainStorageState>().unwrap();
@@ -1135,7 +1136,7 @@ mod tests {
#[test]
fn db_cursor_delete_current_non_existent() {
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let tx = db.tx_mut().expect(ERROR_INIT_TX);
let key1 = Address::with_last_byte(1);
@@ -1165,7 +1166,7 @@ mod tests {
#[test]
fn db_cursor_insert_wherever_cursor_is() {
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let tx = db.tx_mut().expect(ERROR_INIT_TX);
// PUT
@@ -1198,7 +1199,7 @@ mod tests {
#[test]
fn db_cursor_append() {
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
// PUT
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -1225,7 +1226,7 @@ mod tests {
#[test]
fn db_cursor_append_failure() {
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
// PUT
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -1260,7 +1261,7 @@ mod tests {
#[test]
fn db_cursor_upsert() {
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let tx = db.tx_mut().expect(ERROR_INIT_TX);
let mut cursor = tx.cursor_write::<PlainAccountState>().unwrap();
@@ -1295,7 +1296,7 @@ mod tests {
#[test]
fn db_cursor_dupsort_append() {
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let transition_id = 2;
@@ -1557,7 +1558,7 @@ mod tests {
#[test]
fn db_sharded_key() {
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let real_key = address!("0xa2c122be93b0074270ebee7f6b7292c7deb45047");
let shards = 5;

View File

@@ -203,6 +203,26 @@ pub mod test_utils {
Arc::new(TempDatabase::new(db, path))
}
/// Create read/write database for testing within a data directory.
///
/// The database is created at `datadir/db`, and `TempDatabase` will clean up the entire
/// `datadir` on drop.
#[track_caller]
pub fn create_test_rw_db_with_datadir<P: AsRef<Path>>(
datadir: P,
) -> Arc<TempDatabase<DatabaseEnv>> {
let datadir = datadir.as_ref().to_path_buf();
let db_path = datadir.join("db");
let emsg = format!("{ERROR_DB_CREATION}: {db_path:?}");
let db = init_db(
&db_path,
DatabaseArguments::new(ClientVersion::default())
.with_max_read_transaction_duration(Some(MaxReadTransactionDuration::Unbounded)),
)
.expect(&emsg);
Arc::new(TempDatabase::new(db, datadir))
}
/// Create read only database for testing
#[track_caller]
pub fn create_test_ro_db() -> Arc<TempDatabase<DatabaseEnv>> {
@@ -235,6 +255,24 @@ mod tests {
use std::time::Duration;
use tempfile::tempdir;
#[test]
fn test_temp_database_cleanup() {
// Test that TempDatabase properly cleans up its directory when dropped
let temp_path = {
let db = crate::test_utils::create_test_rw_db();
let path = db.path().to_path_buf();
assert!(path.exists(), "Database directory should exist while TempDatabase is alive");
path
// TempDatabase dropped here
};
// Verify the directory was cleaned up
assert!(
!temp_path.exists(),
"Database directory should be cleaned up after TempDatabase is dropped"
);
}
#[test]
fn db_version() {
let path = tempdir().unwrap();

View File

@@ -55,6 +55,7 @@ impl Environment {
handle_slow_readers: None,
#[cfg(feature = "read-tx-timeouts")]
max_read_transaction_duration: None,
prefault_write: None,
}
}
@@ -602,6 +603,9 @@ pub struct EnvironmentBuilder {
/// The maximum duration of a read transaction. If [None], but the `read-tx-timeout` feature is
/// enabled, the default value of [`DEFAULT_MAX_READ_TRANSACTION_DURATION`] is used.
max_read_transaction_duration: Option<read_transactions::MaxReadTransactionDuration>,
/// Controls whether prefault write is enabled. If [None], MDBX uses its default.
/// When disabled, avoids `mincore()` syscall overhead in WRITEMAP mode.
prefault_write: Option<bool>,
}
impl EnvironmentBuilder {
@@ -723,6 +727,14 @@ impl EnvironmentBuilder {
}
}
if let Some(prefault_write) = self.prefault_write {
mdbx_result(ffi::mdbx_env_set_option(
env,
ffi::MDBX_opt_prefault_write_enable,
prefault_write as u64,
))?;
}
Ok(())
})() {
ffi::mdbx_env_close_ex(env, false);
@@ -873,6 +885,19 @@ impl EnvironmentBuilder {
self.handle_slow_readers = Some(hsr);
self
}
/// Set whether to enable prefault writes.
///
/// In WRITEMAP mode, MDBX by default uses `mincore()` to check if pages are resident before
/// touching them. This avoids page faults but adds syscall overhead. Disabling prefault
/// writes skips the `mincore()` check and lets the kernel handle page faults directly.
///
/// This is beneficial when pages are likely already in memory (e.g., recently read by
/// other transactions), as it eliminates unnecessary syscall overhead.
pub const fn set_prefault_write(&mut self, prefault_write: bool) -> &mut Self {
self.prefault_write = Some(prefault_write);
self
}
}
#[cfg(feature = "read-tx-timeouts")]

Some files were not shown because too many files have changed in this diff Show More