mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
40 Commits
pr-21596
...
disable-pr
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d701d70dc1 | ||
|
|
53f922927a | ||
|
|
f1f3980d29 | ||
|
|
6946f26d77 | ||
|
|
f663d1d110 | ||
|
|
f4943abf73 | ||
|
|
102a6944ba | ||
|
|
1592e51d34 | ||
|
|
4280ccf470 | ||
|
|
05ab98107c | ||
|
|
49128ed28f | ||
|
|
f74e594292 | ||
|
|
e7d4a05e36 | ||
|
|
9382a4c713 | ||
|
|
28409558f9 | ||
|
|
5ef32726db | ||
|
|
60c3bef1e8 | ||
|
|
af96eeae56 | ||
|
|
5528aae8f6 | ||
|
|
83364aa2d6 | ||
|
|
749a742bcf | ||
|
|
2970624413 | ||
|
|
7e18aa4be8 | ||
|
|
9f8c22e2c3 | ||
|
|
3d699ac9c6 | ||
|
|
9be31d504d | ||
|
|
34cc65cfe6 | ||
|
|
6e161f0fc9 | ||
|
|
63a3e18404 | ||
|
|
7d10e791b2 | ||
|
|
a9b2c1d454 | ||
|
|
9127563914 | ||
|
|
a500fb22ba | ||
|
|
e869cd4670 | ||
|
|
de69654b73 | ||
|
|
8d28c4c8f2 | ||
|
|
bfe778ab51 | ||
|
|
e523a76fb8 | ||
|
|
cd12ae58f2 | ||
|
|
370a548f34 |
11
CLAUDE.md
11
CLAUDE.md
@@ -38,7 +38,7 @@ Reth is a high-performance Ethereum execution client written in Rust, focusing o
|
||||
|
||||
2. **Linting**: Run clippy with all features
|
||||
```bash
|
||||
RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features --locked
|
||||
cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features
|
||||
```
|
||||
|
||||
3. **Testing**: Use nextest for faster test execution
|
||||
@@ -169,12 +169,11 @@ Based on PR patterns, avoid:
|
||||
Before submitting changes, ensure:
|
||||
|
||||
1. **Format Check**: `cargo +nightly fmt --all --check`
|
||||
2. **Clippy**: No warnings with `RUSTFLAGS="-D warnings"`
|
||||
2. **Clippy**: No warnings
|
||||
3. **Tests Pass**: All unit and integration tests
|
||||
4. **Documentation**: Update relevant docs and add doc comments with `cargo docs --document-private-items`
|
||||
5. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
|
||||
|
||||
|
||||
### Opening PRs against <https://github.com/paradigmxyz/reth>
|
||||
|
||||
Label PRs appropriately, first check the available labels and then apply the relevant ones:
|
||||
@@ -349,10 +348,10 @@ Let's say you want to fix a bug where external IP resolution fails on startup:
|
||||
}
|
||||
```
|
||||
|
||||
5. **Run checks**:
|
||||
5. **Run checks** (IMPORTANT!):
|
||||
```bash
|
||||
cargo +nightly fmt --all
|
||||
cargo clippy --all-features
|
||||
cargo clippy --workspace --all-features # Make sure WHOLE WORKSPACE compiles!
|
||||
cargo test -p reth-discv4
|
||||
```
|
||||
|
||||
@@ -374,7 +373,7 @@ Let's say you want to fix a bug where external IP resolution fails on startup:
|
||||
cargo +nightly fmt --all
|
||||
|
||||
# Run lints
|
||||
RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --all-features --locked
|
||||
cargo +nightly clippy --workspace --all-features
|
||||
|
||||
# Run tests
|
||||
cargo nextest run --workspace
|
||||
|
||||
631
Cargo.lock
generated
631
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
41
Cargo.toml
41
Cargo.toml
@@ -560,7 +560,7 @@ humantime-serde = "1.1"
|
||||
itertools = { version = "0.14", default-features = false }
|
||||
linked_hash_set = "0.1"
|
||||
lz4 = "1.28.1"
|
||||
modular-bitfield = "0.11.2"
|
||||
modular-bitfield = "0.13.1"
|
||||
notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] }
|
||||
nybbles = { version = "0.4.2", default-features = false }
|
||||
once_cell = { version = "1.19", default-features = false, features = ["critical-section"] }
|
||||
@@ -589,13 +589,13 @@ zstd = "0.13"
|
||||
byteorder = "1"
|
||||
fixed-cache = { version = "0.1.7", features = ["stats"] }
|
||||
moka = "0.12"
|
||||
tar-no-std = { version = "0.3.2", default-features = false }
|
||||
miniz_oxide = { version = "0.8.4", default-features = false }
|
||||
tar-no-std = { version = "0.4.2", default-features = false }
|
||||
miniz_oxide = { version = "0.9.0", default-features = false }
|
||||
chrono = "0.4.41"
|
||||
|
||||
# metrics
|
||||
metrics = "0.24.0"
|
||||
metrics-derive = "0.1"
|
||||
metrics-derive = "0.1.1"
|
||||
metrics-exporter-prometheus = { version = "0.18.0", default-features = false }
|
||||
metrics-process = "2.1.0"
|
||||
metrics-util = { default-features = false, version = "0.20.0" }
|
||||
@@ -607,7 +607,7 @@ quote = "1.0"
|
||||
# tokio
|
||||
tokio = { version = "1.44.2", default-features = false }
|
||||
tokio-stream = "0.1.11"
|
||||
tokio-tungstenite = "0.26.2"
|
||||
tokio-tungstenite = "0.28.0"
|
||||
tokio-util = { version = "0.7.4", features = ["codec"] }
|
||||
|
||||
# async
|
||||
@@ -620,7 +620,7 @@ futures-util = { version = "0.3", default-features = false }
|
||||
hyper = "1.3"
|
||||
hyper-util = "0.1.5"
|
||||
pin-project = "1.0.12"
|
||||
reqwest = { version = "0.12", default-features = false }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "rustls-tls-native-roots", "stream"] }
|
||||
tracing-futures = "0.2"
|
||||
tower = "0.5"
|
||||
tower-http = "0.6"
|
||||
@@ -640,7 +640,6 @@ jsonrpsee-types = "0.26.0"
|
||||
http = "1.0"
|
||||
http-body = "1.0"
|
||||
http-body-util = "0.1.2"
|
||||
jsonwebtoken = "9"
|
||||
proptest-arbitrary-interop = "0.1.0"
|
||||
|
||||
# crypto
|
||||
@@ -654,7 +653,7 @@ rand_08 = { package = "rand", version = "0.8" }
|
||||
c-kzg = "2.1.5"
|
||||
|
||||
# config
|
||||
toml = "0.8"
|
||||
toml = "0.9"
|
||||
|
||||
# rocksdb
|
||||
rocksdb = { version = "0.24" }
|
||||
@@ -673,16 +672,16 @@ assert_matches = "1.5.0"
|
||||
criterion = { package = "codspeed-criterion-compat", version = "4.3" }
|
||||
insta = "1.41"
|
||||
proptest = "1.7"
|
||||
proptest-derive = "0.5"
|
||||
proptest-derive = "0.7"
|
||||
similar-asserts = { version = "1.5.0", features = ["serde"] }
|
||||
tempfile = "3.20"
|
||||
test-fuzz = "7"
|
||||
rstest = "0.24.0"
|
||||
rstest = "0.26.1"
|
||||
test-case = "3"
|
||||
|
||||
# ssz encoding
|
||||
ethereum_ssz = "0.9.0"
|
||||
ethereum_ssz_derive = "0.9.0"
|
||||
ethereum_ssz = "0.10.1"
|
||||
ethereum_ssz_derive = "0.10.1"
|
||||
|
||||
# allocators
|
||||
jemalloc_pprof = { version = "0.8", default-features = false }
|
||||
@@ -694,14 +693,14 @@ snmalloc-rs = { version = "0.3.7", features = ["build_cc"] }
|
||||
aes = "0.8.1"
|
||||
ahash = "0.8"
|
||||
anyhow = "1.0"
|
||||
bindgen = { version = "0.71", default-features = false }
|
||||
block-padding = "0.3.2"
|
||||
bindgen = { version = "0.72", default-features = false }
|
||||
block-padding = "0.3"
|
||||
cc = "1.2.15"
|
||||
cipher = "0.4.3"
|
||||
comfy-table = "7.0"
|
||||
concat-kdf = "0.1.0"
|
||||
crossbeam-channel = "0.5.13"
|
||||
crossterm = "0.28.0"
|
||||
crossterm = "0.29.0"
|
||||
csv = "1.3.0"
|
||||
ctrlc = "3.4"
|
||||
ctr = "0.9.2"
|
||||
@@ -714,7 +713,7 @@ hmac = "0.12.1"
|
||||
human_bytes = "0.4.1"
|
||||
indexmap = "2"
|
||||
interprocess = "2.2.0"
|
||||
lz4_flex = { version = "0.11", default-features = false }
|
||||
lz4_flex = { version = "0.12", default-features = false }
|
||||
memmap2 = "0.9.4"
|
||||
mev-share-sse = { version = "0.5.0", default-features = false }
|
||||
num-traits = "0.2.15"
|
||||
@@ -722,15 +721,15 @@ page_size = "0.6.0"
|
||||
parity-scale-codec = "3.2.1"
|
||||
plain_hasher = "0.2"
|
||||
pretty_assertions = "1.4"
|
||||
ratatui = { version = "0.29", default-features = false }
|
||||
ringbuffer = "0.15.0"
|
||||
ratatui = { version = "0.30", default-features = false }
|
||||
ringbuffer = "0.16.0"
|
||||
rmp-serde = "1.3"
|
||||
roaring = "0.10.2"
|
||||
roaring = "0.11.3"
|
||||
rolling-file = "0.2.0"
|
||||
sha3 = "0.10.5"
|
||||
snap = "1.1.1"
|
||||
socket2 = { version = "0.5", default-features = false }
|
||||
sysinfo = { version = "0.33", default-features = false }
|
||||
socket2 = { version = "0.6", default-features = false }
|
||||
sysinfo = { version = "0.38", default-features = false }
|
||||
tracing-journald = "0.3"
|
||||
tracing-logfmt = "=0.3.5"
|
||||
tracing-samply = "0.1"
|
||||
|
||||
@@ -56,7 +56,7 @@ ctrlc.workspace = true
|
||||
shlex.workspace = true
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
nix = { version = "0.29", features = ["signal", "process"] }
|
||||
nix = { version = "0.31", features = ["signal", "process"] }
|
||||
|
||||
[features]
|
||||
default = ["jemalloc"]
|
||||
|
||||
@@ -45,7 +45,7 @@ op-alloy-consensus = { workspace = true, features = ["alloy-compat"] }
|
||||
op-alloy-rpc-types-engine = { workspace = true, features = ["serde"] }
|
||||
|
||||
# reqwest
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls-tls-native-roots"] }
|
||||
reqwest.workspace = true
|
||||
|
||||
# tower
|
||||
tower.workspace = true
|
||||
|
||||
@@ -572,13 +572,22 @@ impl Command {
|
||||
|
||||
for i in 0..self.count {
|
||||
// Get initial batch of transactions for this payload
|
||||
let mut result = tx_buffer
|
||||
.take_batch()
|
||||
.await
|
||||
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
|
||||
let Some(mut result) = tx_buffer.take_batch().await else {
|
||||
info!(
|
||||
payloads_built = i,
|
||||
payloads_requested = self.count,
|
||||
"Transaction source exhausted, stopping"
|
||||
);
|
||||
break;
|
||||
};
|
||||
|
||||
if result.transactions.is_empty() {
|
||||
return Err(eyre::eyre!("No transactions collected for payload {}", i + 1));
|
||||
info!(
|
||||
payloads_built = i,
|
||||
payloads_requested = self.count,
|
||||
"No more transactions available, stopping"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
// Build with retry - may need to request more transactions
|
||||
|
||||
@@ -541,7 +541,7 @@ impl<H: BlockHeader> ChainSpec<H> {
|
||||
}
|
||||
}
|
||||
|
||||
bf_params.first().map(|(_, params)| *params).unwrap_or(BaseFeeParams::ethereum())
|
||||
bf_params.first().map(|(_, params)| *params).unwrap_or_else(BaseFeeParams::ethereum)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,4 +133,4 @@ arbitrary = [
|
||||
"reth-ethereum-primitives/arbitrary",
|
||||
]
|
||||
|
||||
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb"]
|
||||
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb", "reth-prune/rocksdb"]
|
||||
|
||||
@@ -121,13 +121,13 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
let genesis_block_number = self.chain.genesis().number.unwrap_or_default();
|
||||
let (db, sfp) = match access {
|
||||
AccessRights::RW => (
|
||||
Arc::new(init_db(db_path, self.db.database_args())?),
|
||||
init_db(db_path, self.db.database_args())?,
|
||||
StaticFileProviderBuilder::read_write(sf_path)
|
||||
.with_genesis_block_number(genesis_block_number)
|
||||
.build()?,
|
||||
),
|
||||
AccessRights::RO | AccessRights::RoInconsistent => {
|
||||
(Arc::new(open_db_read_only(&db_path, self.db.database_args())?), {
|
||||
(open_db_read_only(&db_path, self.db.database_args())?, {
|
||||
let provider = StaticFileProviderBuilder::read_only(sf_path)
|
||||
.with_genesis_block_number(genesis_block_number)
|
||||
.build()?;
|
||||
@@ -160,16 +160,16 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
fn create_provider_factory<N: CliNodeTypes>(
|
||||
&self,
|
||||
config: &Config,
|
||||
db: Arc<DatabaseEnv>,
|
||||
db: DatabaseEnv,
|
||||
static_file_provider: StaticFileProvider<N::Primitives>,
|
||||
rocksdb_provider: RocksDBProvider,
|
||||
access: AccessRights,
|
||||
) -> eyre::Result<ProviderFactory<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>>
|
||||
) -> eyre::Result<ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>>
|
||||
where
|
||||
C: ChainSpecParser<ChainSpec = N::ChainSpec>,
|
||||
{
|
||||
let prune_modes = config.prune.segments.clone();
|
||||
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>::new(
|
||||
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::new(
|
||||
db,
|
||||
self.chain.clone(),
|
||||
static_file_provider,
|
||||
@@ -200,7 +200,7 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
let (_tip_tx, tip_rx) = watch::channel(B256::ZERO);
|
||||
|
||||
// Builds and executes an unwind-only pipeline
|
||||
let mut pipeline = Pipeline::<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>::builder()
|
||||
let mut pipeline = Pipeline::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::builder()
|
||||
.add_stages(DefaultStages::new(
|
||||
factory.clone(),
|
||||
tip_rx,
|
||||
@@ -229,7 +229,7 @@ pub struct Environment<N: NodeTypes> {
|
||||
/// Configuration for reth node
|
||||
pub config: Config,
|
||||
/// Provider factory.
|
||||
pub provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
pub provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
/// Datadir path.
|
||||
pub data_dir: ChainPath<DataDirPath>,
|
||||
}
|
||||
@@ -261,8 +261,8 @@ impl AccessRights {
|
||||
/// Helper alias to satisfy `FullNodeTypes` bound on [`Node`] trait generic.
|
||||
type FullTypesAdapter<T> = FullNodeTypesAdapter<
|
||||
T,
|
||||
Arc<DatabaseEnv>,
|
||||
BlockchainProvider<NodeTypesWithDBAdapter<T, Arc<DatabaseEnv>>>,
|
||||
DatabaseEnv,
|
||||
BlockchainProvider<NodeTypesWithDBAdapter<T, DatabaseEnv>>,
|
||||
>;
|
||||
|
||||
/// Helper trait with a common set of requirements for the
|
||||
|
||||
@@ -17,7 +17,6 @@ use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProvider
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use std::{
|
||||
hash::{BuildHasher, Hasher},
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
@@ -90,7 +89,7 @@ impl Command {
|
||||
/// Execute `db checksum` command
|
||||
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
self,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
) -> eyre::Result<()> {
|
||||
warn!("This command should be run without the node running!");
|
||||
|
||||
@@ -117,7 +116,7 @@ fn checksum_hasher() -> impl Hasher {
|
||||
}
|
||||
|
||||
fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
segment: StaticFileSegment,
|
||||
start_block: Option<u64>,
|
||||
end_block: Option<u64>,
|
||||
|
||||
@@ -9,7 +9,7 @@ use reth_db_api::table::Table;
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::NodeTypesWithDBAdapter;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use std::{hash::Hasher, sync::Arc, time::Instant};
|
||||
use std::{hash::Hasher, time::Instant};
|
||||
use tracing::info;
|
||||
|
||||
/// RocksDB tables that can be checksummed.
|
||||
@@ -36,7 +36,7 @@ impl RocksDbTable {
|
||||
|
||||
/// Computes a checksum for a RocksDB table.
|
||||
pub fn checksum_rocksdb<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
table: RocksDbTable,
|
||||
limit: Option<usize>,
|
||||
) -> eyre::Result<()> {
|
||||
|
||||
@@ -16,7 +16,6 @@ use std::{
|
||||
hash::Hash,
|
||||
io::Write,
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
@@ -56,7 +55,7 @@ impl Command {
|
||||
/// then written to a file in the output directory.
|
||||
pub fn execute<T: NodeTypes>(
|
||||
self,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<T, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<T, DatabaseEnv>>,
|
||||
) -> eyre::Result<()> {
|
||||
warn!("Make sure the node is not running when running `reth db diff`!");
|
||||
// open second db
|
||||
|
||||
@@ -7,7 +7,7 @@ use reth_db::{transaction::DbTx, DatabaseEnv};
|
||||
use reth_db_api::{database::Database, table::Table, RawValue, TableViewer, Tables};
|
||||
use reth_db_common::{DbTool, ListFilter};
|
||||
use reth_node_builder::{NodeTypes, NodeTypesWithDBAdapter};
|
||||
use std::{cell::RefCell, sync::Arc};
|
||||
use std::cell::RefCell;
|
||||
use tracing::error;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
@@ -55,7 +55,7 @@ impl Command {
|
||||
/// Execute `db list` command
|
||||
pub fn execute<N: NodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
self,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
) -> eyre::Result<()> {
|
||||
self.table.view(&ListTableViewer { tool, args: &self })
|
||||
}
|
||||
@@ -89,7 +89,7 @@ impl Command {
|
||||
}
|
||||
|
||||
struct ListTableViewer<'a, N: NodeTypes> {
|
||||
tool: &'a DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &'a DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
args: &'a Command,
|
||||
}
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ use reth_provider::{
|
||||
RocksDBProviderFactory,
|
||||
};
|
||||
use reth_static_file_types::SegmentRangeInclusive;
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
/// The arguments for the `reth db stats` command
|
||||
@@ -48,7 +48,7 @@ impl Command {
|
||||
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
self,
|
||||
data_dir: ChainPath<DataDirPath>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
) -> eyre::Result<()> {
|
||||
if self.checksum {
|
||||
let checksum_report = self.checksum_report(tool)?;
|
||||
@@ -72,7 +72,7 @@ impl Command {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn db_stats_table<N: NodeTypesWithDB<DB = Arc<DatabaseEnv>>>(
|
||||
fn db_stats_table<N: NodeTypesWithDB<DB = DatabaseEnv>>(
|
||||
&self,
|
||||
tool: &DbTool<N>,
|
||||
) -> eyre::Result<ComfyTable> {
|
||||
|
||||
@@ -227,8 +227,9 @@ where
|
||||
|
||||
// Handle errors
|
||||
if let Err(err) = res {
|
||||
error!("{:?}", err)
|
||||
error!("{err}");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -241,6 +242,7 @@ fn event_loop<B: Backend, F, T: Table>(
|
||||
) -> io::Result<()>
|
||||
where
|
||||
F: FnMut(usize, usize) -> Vec<TableRow<T>>,
|
||||
io::Error: From<B::Error>,
|
||||
{
|
||||
let mut last_tick = Instant::now();
|
||||
let mut running = true;
|
||||
|
||||
@@ -2,7 +2,7 @@ use futures::Future;
|
||||
use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_db::DatabaseEnv;
|
||||
use reth_node_builder::{NodeBuilder, WithLaunchContext};
|
||||
use std::{fmt, sync::Arc};
|
||||
use std::fmt;
|
||||
|
||||
/// A trait for launching a reth node with custom configuration strategies.
|
||||
///
|
||||
@@ -30,7 +30,7 @@ where
|
||||
/// * `builder_args` - Extension arguments for configuration
|
||||
fn entrypoint(
|
||||
self,
|
||||
builder: WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
builder: WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
builder_args: Ext,
|
||||
) -> impl Future<Output = eyre::Result<()>>;
|
||||
}
|
||||
@@ -58,7 +58,7 @@ impl<F> FnLauncher<F> {
|
||||
where
|
||||
C: ChainSpecParser,
|
||||
F: AsyncFnOnce(
|
||||
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
Ext,
|
||||
) -> eyre::Result<()>,
|
||||
{
|
||||
@@ -77,13 +77,13 @@ where
|
||||
C: ChainSpecParser,
|
||||
Ext: clap::Args + fmt::Debug,
|
||||
F: AsyncFnOnce(
|
||||
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
Ext,
|
||||
) -> eyre::Result<()>,
|
||||
{
|
||||
fn entrypoint(
|
||||
self,
|
||||
builder: WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
builder: WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
builder_args: Ext,
|
||||
) -> impl Future<Output = eyre::Result<()>> {
|
||||
(self.func)(builder, builder_args)
|
||||
|
||||
@@ -206,7 +206,7 @@ where
|
||||
let db_path = data_dir.db();
|
||||
|
||||
tracing::info!(target: "reth::cli", path = ?db_path, "Opening database");
|
||||
let database = Arc::new(init_db(db_path.clone(), self.db.database_args())?.with_metrics());
|
||||
let database = init_db(db_path.clone(), self.db.database_args())?.with_metrics();
|
||||
|
||||
if with_unused_ports {
|
||||
node_config = node_config.with_unused_ports();
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! Command that runs pruning without any limits.
|
||||
//! Command that runs pruning.
|
||||
use crate::common::{AccessRights, CliNodeTypes, EnvironmentArgs};
|
||||
use clap::Parser;
|
||||
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks};
|
||||
@@ -16,7 +16,7 @@ use reth_static_file::StaticFileProducer;
|
||||
use std::sync::Arc;
|
||||
use tracing::info;
|
||||
|
||||
/// Prunes according to the configuration without any limits
|
||||
/// Prunes according to the configuration
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct PruneCommand<C: ChainSpecParser> {
|
||||
#[command(flatten)]
|
||||
@@ -69,13 +69,43 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
|
||||
// Delete data which has been copied to static files.
|
||||
if let Some(prune_tip) = lowest_static_file_height {
|
||||
info!(target: "reth::cli", ?prune_tip, ?config, "Pruning data from database...");
|
||||
// Run the pruner according to the configuration, and don't enforce any limits on it
|
||||
|
||||
// Use batched pruning with a limit to bound memory, running in a loop until complete.
|
||||
const DELETE_LIMIT: usize = 200_000;
|
||||
let mut pruner = PrunerBuilder::new(config)
|
||||
.delete_limit(usize::MAX)
|
||||
.delete_limit(DELETE_LIMIT)
|
||||
.build_with_provider_factory(provider_factory);
|
||||
|
||||
pruner.run(prune_tip)?;
|
||||
info!(target: "reth::cli", "Pruned data from database");
|
||||
let mut total_pruned = 0usize;
|
||||
loop {
|
||||
let output = pruner.run(prune_tip)?;
|
||||
let batch_pruned: usize = output.segments.iter().map(|(_, seg)| seg.pruned).sum();
|
||||
total_pruned = total_pruned.saturating_add(batch_pruned);
|
||||
|
||||
// Check if all segments are finished (not just the overall progress,
|
||||
// since the pruner sets overall progress from the last segment only)
|
||||
let all_segments_finished =
|
||||
output.segments.iter().all(|(_, seg)| seg.progress.is_finished());
|
||||
|
||||
if all_segments_finished {
|
||||
break;
|
||||
}
|
||||
|
||||
if batch_pruned == 0 {
|
||||
return Err(eyre::eyre!(
|
||||
"pruner made no progress but reported more data remaining; \
|
||||
aborting to prevent infinite loop"
|
||||
));
|
||||
}
|
||||
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
batch_pruned,
|
||||
total_pruned,
|
||||
"Pruning batch complete, continuing..."
|
||||
);
|
||||
}
|
||||
info!(target: "reth::cli", total_pruned, "Pruned data from database");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -26,7 +26,7 @@ pub(crate) async fn dump_execution_stage<N, E, C>(
|
||||
consensus: C,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>,
|
||||
N: ProviderNodeTypes<DB = DatabaseEnv>,
|
||||
E: ConfigureEvm<Primitives = N::Primitives> + 'static,
|
||||
C: FullConsensus<E::Primitives> + 'static,
|
||||
{
|
||||
@@ -39,7 +39,7 @@ where
|
||||
if should_run {
|
||||
dry_run(
|
||||
ProviderFactory::<N>::new(
|
||||
Arc::new(output_db),
|
||||
output_db,
|
||||
db_tool.chain(),
|
||||
StaticFileProvider::read_write(output_datadir.static_files())?,
|
||||
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,
|
||||
|
||||
@@ -10,10 +10,9 @@ use reth_provider::{
|
||||
DatabaseProviderFactory, ProviderFactory,
|
||||
};
|
||||
use reth_stages::{stages::AccountHashingStage, Stage, StageCheckpoint, UnwindInput};
|
||||
use std::sync::Arc;
|
||||
use tracing::info;
|
||||
|
||||
pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
|
||||
pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = DatabaseEnv>>(
|
||||
db_tool: &DbTool<N>,
|
||||
from: BlockNumber,
|
||||
to: BlockNumber,
|
||||
@@ -36,7 +35,7 @@ pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = Arc<Dat
|
||||
if should_run {
|
||||
dry_run(
|
||||
ProviderFactory::<N>::new(
|
||||
Arc::new(output_db),
|
||||
output_db,
|
||||
db_tool.chain(),
|
||||
StaticFileProvider::read_write(output_datadir.static_files())?,
|
||||
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,
|
||||
|
||||
@@ -9,10 +9,9 @@ use reth_provider::{
|
||||
DatabaseProviderFactory, ProviderFactory,
|
||||
};
|
||||
use reth_stages::{stages::StorageHashingStage, Stage, StageCheckpoint, UnwindInput};
|
||||
use std::sync::Arc;
|
||||
use tracing::info;
|
||||
|
||||
pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
|
||||
pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = DatabaseEnv>>(
|
||||
db_tool: &DbTool<N>,
|
||||
from: u64,
|
||||
to: u64,
|
||||
@@ -26,7 +25,7 @@ pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = Arc<Dat
|
||||
if should_run {
|
||||
dry_run(
|
||||
ProviderFactory::<N>::new(
|
||||
Arc::new(output_db),
|
||||
output_db,
|
||||
db_tool.chain(),
|
||||
StaticFileProvider::read_write(output_datadir.static_files())?,
|
||||
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,
|
||||
|
||||
@@ -34,7 +34,7 @@ pub(crate) async fn dump_merkle_stage<N>(
|
||||
consensus: impl FullConsensus<N::Primitives> + 'static,
|
||||
) -> Result<()>
|
||||
where
|
||||
N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>,
|
||||
N: ProviderNodeTypes<DB = DatabaseEnv>,
|
||||
{
|
||||
let (output_db, tip_block_number) = setup(from, to, &output_datadir.db(), db_tool)?;
|
||||
|
||||
@@ -59,7 +59,7 @@ where
|
||||
if should_run {
|
||||
dry_run(
|
||||
ProviderFactory::<N>::new(
|
||||
Arc::new(output_db),
|
||||
output_db,
|
||||
db_tool.chain(),
|
||||
StaticFileProvider::read_write(output_datadir.static_files())?,
|
||||
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,
|
||||
|
||||
@@ -158,7 +158,7 @@ enum Subcommands {
|
||||
|
||||
impl Subcommands {
|
||||
/// Returns the block to unwind to. The returned block will stay in database.
|
||||
fn unwind_target<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
|
||||
fn unwind_target<N: ProviderNodeTypes<DB = DatabaseEnv>>(
|
||||
&self,
|
||||
factory: ProviderFactory<N>,
|
||||
) -> eyre::Result<u64> {
|
||||
|
||||
@@ -29,7 +29,7 @@ auto_impl.workspace = true
|
||||
derive_more.workspace = true
|
||||
futures.workspace = true
|
||||
eyre.workspace = true
|
||||
reqwest = { workspace = true, features = ["rustls-tls"] }
|
||||
reqwest.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
tokio = { workspace = true, features = ["time"] }
|
||||
serde_json.workspace = true
|
||||
|
||||
@@ -95,7 +95,7 @@ where
|
||||
let block_hash = payload.block_hash();
|
||||
let block_number = payload.block_number();
|
||||
|
||||
previous_block_hashes.push(block_hash);
|
||||
previous_block_hashes.enqueue(block_hash);
|
||||
|
||||
// Send new events to execution client
|
||||
let _ = self.engine_handle.new_payload(payload).await;
|
||||
@@ -160,7 +160,7 @@ mod tests {
|
||||
|
||||
// Push hashes 0..65
|
||||
for i in 0..65u8 {
|
||||
buffer.push(B256::with_last_byte(i));
|
||||
buffer.enqueue(B256::with_last_byte(i));
|
||||
}
|
||||
|
||||
// offset=0 should return the most recent (64)
|
||||
@@ -181,7 +181,7 @@ mod tests {
|
||||
let mut buffer: AllocRingBuffer<B256> = AllocRingBuffer::new(65);
|
||||
|
||||
// With only 1 entry, only offset=0 works
|
||||
buffer.push(B256::with_last_byte(1));
|
||||
buffer.enqueue(B256::with_last_byte(1));
|
||||
assert_eq!(get_hash_at_offset(&buffer, 0), Some(B256::with_last_byte(1)));
|
||||
assert_eq!(get_hash_at_offset(&buffer, 1), None);
|
||||
assert_eq!(get_hash_at_offset(&buffer, 32), None);
|
||||
@@ -189,7 +189,7 @@ mod tests {
|
||||
|
||||
// With 33 entries, offset=32 works but offset=64 doesn't
|
||||
for i in 2..=33u8 {
|
||||
buffer.push(B256::with_last_byte(i));
|
||||
buffer.enqueue(B256::with_last_byte(i));
|
||||
}
|
||||
assert_eq!(get_hash_at_offset(&buffer, 32), Some(B256::with_last_byte(1)));
|
||||
assert_eq!(get_hash_at_offset(&buffer, 64), None);
|
||||
|
||||
@@ -114,22 +114,22 @@ pub async fn setup_engine_with_chain_import(
|
||||
|
||||
// Initialize the database using init_db (same as CLI import command)
|
||||
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
|
||||
let db_env = reth_db::init_db(&db_path, db_args)?;
|
||||
let db = Arc::new(db_env);
|
||||
let db = reth_db::init_db(&db_path, db_args)?;
|
||||
|
||||
// Create a provider factory with the initialized database (use regular DB, not
|
||||
// TempDatabase) We need to specify the node types properly for the adapter
|
||||
let provider_factory = ProviderFactory::<
|
||||
NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>,
|
||||
>::new(
|
||||
db.clone(),
|
||||
chain_spec.clone(),
|
||||
reth_provider::providers::StaticFileProvider::read_write(static_files_path.clone())?,
|
||||
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
|
||||
.with_default_tables()
|
||||
.build()
|
||||
.unwrap(),
|
||||
)?;
|
||||
let provider_factory =
|
||||
ProviderFactory::<NodeTypesWithDBAdapter<EthereumNode, DatabaseEnv>>::new(
|
||||
db.clone(),
|
||||
chain_spec.clone(),
|
||||
reth_provider::providers::StaticFileProvider::read_write(
|
||||
static_files_path.clone(),
|
||||
)?,
|
||||
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
|
||||
.with_default_tables()
|
||||
.build()
|
||||
.unwrap(),
|
||||
)?;
|
||||
|
||||
// Initialize genesis if needed
|
||||
reth_db_common::init::init_genesis(&provider_factory)?;
|
||||
@@ -320,11 +320,10 @@ mod tests {
|
||||
// Import the chain
|
||||
{
|
||||
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
|
||||
let db_env = reth_db::init_db(&db_path, db_args).unwrap();
|
||||
let db = Arc::new(db_env);
|
||||
let db = reth_db::init_db(&db_path, db_args).unwrap();
|
||||
|
||||
let provider_factory: ProviderFactory<
|
||||
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, Arc<DatabaseEnv>>,
|
||||
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, DatabaseEnv>,
|
||||
> = ProviderFactory::new(
|
||||
db.clone(),
|
||||
chain_spec.clone(),
|
||||
@@ -385,11 +384,10 @@ mod tests {
|
||||
|
||||
// Now reopen the database and verify checkpoints are still there
|
||||
{
|
||||
let db_env = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap();
|
||||
let db = Arc::new(db_env);
|
||||
let db = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap();
|
||||
|
||||
let provider_factory: ProviderFactory<
|
||||
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, Arc<DatabaseEnv>>,
|
||||
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, DatabaseEnv>,
|
||||
> = ProviderFactory::new(
|
||||
db,
|
||||
chain_spec.clone(),
|
||||
|
||||
@@ -528,8 +528,12 @@ impl TreeConfig {
|
||||
}
|
||||
|
||||
/// Setter for the number of storage proof worker threads.
|
||||
pub fn with_storage_worker_count(mut self, storage_worker_count: usize) -> Self {
|
||||
self.storage_worker_count = storage_worker_count.max(MIN_WORKER_COUNT);
|
||||
///
|
||||
/// No-op if it's [`None`].
|
||||
pub fn with_storage_worker_count_opt(mut self, storage_worker_count: Option<usize>) -> Self {
|
||||
if let Some(count) = storage_worker_count {
|
||||
self.storage_worker_count = count.max(MIN_WORKER_COUNT);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
@@ -539,8 +543,12 @@ impl TreeConfig {
|
||||
}
|
||||
|
||||
/// Setter for the number of account proof worker threads.
|
||||
pub fn with_account_worker_count(mut self, account_worker_count: usize) -> Self {
|
||||
self.account_worker_count = account_worker_count.max(MIN_WORKER_COUNT);
|
||||
///
|
||||
/// No-op if it's [`None`].
|
||||
pub fn with_account_worker_count_opt(mut self, account_worker_count: Option<usize>) -> Self {
|
||||
if let Some(count) = account_worker_count {
|
||||
self.account_worker_count = count.max(MIN_WORKER_COUNT);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
|
||||
@@ -17,7 +17,6 @@ reth-engine-tree.workspace = true
|
||||
reth-evm.workspace = true
|
||||
reth-network-p2p.workspace = true
|
||||
reth-payload-builder.workspace = true
|
||||
reth-ethereum-primitives.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-prune.workspace = true
|
||||
reth-stages-api.workspace = true
|
||||
|
||||
@@ -14,7 +14,6 @@ pub use reth_engine_tree::{
|
||||
chain::{ChainEvent, ChainOrchestrator},
|
||||
engine::EngineApiEvent,
|
||||
};
|
||||
use reth_ethereum_primitives::EthPrimitives;
|
||||
use reth_evm::ConfigureEvm;
|
||||
use reth_network_p2p::BlockClient;
|
||||
use reth_node_types::{BlockTy, NodeTypes};
|
||||
@@ -97,7 +96,7 @@ where
|
||||
let downloader = BasicBlockDownloader::new(client, consensus.clone());
|
||||
|
||||
let persistence_handle =
|
||||
PersistenceHandle::<EthPrimitives>::spawn_service(provider, pruner, sync_metrics_tx);
|
||||
PersistenceHandle::<N::Primitives>::spawn_service(provider, pruner, sync_metrics_tx);
|
||||
|
||||
let canonical_in_memory_state = blockchain_db.canonical_in_memory_state();
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ reth-evm = { workspace = true, features = ["metrics"] }
|
||||
reth-network-p2p.workspace = true
|
||||
reth-payload-builder.workspace = true
|
||||
reth-payload-primitives.workspace = true
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-primitives-traits = { workspace = true, features = ["rayon"] }
|
||||
reth-ethereum-primitives.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-prune.workspace = true
|
||||
|
||||
@@ -20,7 +20,7 @@ pub(crate) struct PersistenceMetrics {
|
||||
/// How long it took for blocks to be saved
|
||||
pub(crate) save_blocks_duration_seconds: Histogram,
|
||||
/// How many blocks we persist at once.
|
||||
pub(crate) save_blocks_block_count: Histogram,
|
||||
pub(crate) save_blocks_batch_size: Histogram,
|
||||
/// How long it took for blocks to be pruned
|
||||
pub(crate) prune_before_duration_seconds: Histogram,
|
||||
}
|
||||
|
||||
@@ -12,7 +12,11 @@ use reth_provider::{
|
||||
use reth_prune::{PrunerError, PrunerOutput, PrunerWithFactory};
|
||||
use reth_stages_api::{MetricEvent, MetricEventsSender};
|
||||
use std::{
|
||||
sync::mpsc::{Receiver, SendError, Sender},
|
||||
sync::{
|
||||
mpsc::{Receiver, SendError, Sender},
|
||||
Arc,
|
||||
},
|
||||
thread::JoinHandle,
|
||||
time::Instant,
|
||||
};
|
||||
use thiserror::Error;
|
||||
@@ -40,6 +44,12 @@ where
|
||||
metrics: PersistenceMetrics,
|
||||
/// Sender for sync metrics - we only submit sync metrics for persisted blocks
|
||||
sync_metrics_tx: MetricEventsSender,
|
||||
/// Pending finalized block number to be committed with the next block save.
|
||||
/// This avoids triggering a separate fsync for each finalized block update.
|
||||
pending_finalized_block: Option<u64>,
|
||||
/// Pending safe block number to be committed with the next block save.
|
||||
/// This avoids triggering a separate fsync for each safe block update.
|
||||
pending_safe_block: Option<u64>,
|
||||
}
|
||||
|
||||
impl<N> PersistenceService<N>
|
||||
@@ -53,7 +63,15 @@ where
|
||||
pruner: PrunerWithFactory<ProviderFactory<N>>,
|
||||
sync_metrics_tx: MetricEventsSender,
|
||||
) -> Self {
|
||||
Self { provider, incoming, pruner, metrics: PersistenceMetrics::default(), sync_metrics_tx }
|
||||
Self {
|
||||
provider,
|
||||
incoming,
|
||||
pruner,
|
||||
metrics: PersistenceMetrics::default(),
|
||||
sync_metrics_tx,
|
||||
pending_finalized_block: None,
|
||||
pending_safe_block: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Prunes block data before the given block number according to the configured prune
|
||||
@@ -106,14 +124,10 @@ where
|
||||
}
|
||||
}
|
||||
PersistenceAction::SaveFinalizedBlock(finalized_block) => {
|
||||
let provider = self.provider.database_provider_rw()?;
|
||||
provider.save_finalized_block_number(finalized_block)?;
|
||||
provider.commit()?;
|
||||
self.pending_finalized_block = Some(finalized_block);
|
||||
}
|
||||
PersistenceAction::SaveSafeBlock(safe_block) => {
|
||||
let provider = self.provider.database_provider_rw()?;
|
||||
provider.save_safe_block_number(safe_block)?;
|
||||
provider.commit()?;
|
||||
self.pending_safe_block = Some(safe_block);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -138,26 +152,39 @@ where
|
||||
}
|
||||
|
||||
fn on_save_blocks(
|
||||
&self,
|
||||
&mut self,
|
||||
blocks: Vec<ExecutedBlock<N::Primitives>>,
|
||||
) -> Result<Option<BlockNumHash>, PersistenceError> {
|
||||
let first_block = blocks.first().map(|b| b.recovered_block.num_hash());
|
||||
let last_block = blocks.last().map(|b| b.recovered_block.num_hash());
|
||||
let block_count = blocks.len();
|
||||
|
||||
// Take any pending finalized/safe block updates to commit together
|
||||
let pending_finalized = self.pending_finalized_block.take();
|
||||
let pending_safe = self.pending_safe_block.take();
|
||||
|
||||
debug!(target: "engine::persistence", ?block_count, first=?first_block, last=?last_block, "Saving range of blocks");
|
||||
|
||||
let start_time = Instant::now();
|
||||
|
||||
if last_block.is_some() {
|
||||
let provider_rw = self.provider.database_provider_rw()?;
|
||||
|
||||
provider_rw.save_blocks(blocks, SaveBlocksMode::Full)?;
|
||||
|
||||
// Commit pending finalized/safe block updates in the same transaction
|
||||
if let Some(finalized) = pending_finalized {
|
||||
provider_rw.save_finalized_block_number(finalized)?;
|
||||
}
|
||||
if let Some(safe) = pending_safe {
|
||||
provider_rw.save_safe_block_number(safe)?;
|
||||
}
|
||||
|
||||
provider_rw.commit()?;
|
||||
}
|
||||
|
||||
debug!(target: "engine::persistence", first=?first_block, last=?last_block, "Saved range of blocks");
|
||||
|
||||
self.metrics.save_blocks_block_count.record(block_count as f64);
|
||||
self.metrics.save_blocks_batch_size.record(block_count as f64);
|
||||
self.metrics.save_blocks_duration_seconds.record(start_time.elapsed());
|
||||
|
||||
Ok(last_block)
|
||||
@@ -204,15 +231,25 @@ pub enum PersistenceAction<N: NodePrimitives = EthPrimitives> {
|
||||
pub struct PersistenceHandle<N: NodePrimitives = EthPrimitives> {
|
||||
/// The channel used to communicate with the persistence service
|
||||
sender: Sender<PersistenceAction<N>>,
|
||||
/// Guard that joins the service thread when all handles are dropped.
|
||||
/// Uses `Arc` so the handle remains `Clone`.
|
||||
_service_guard: Arc<ServiceGuard>,
|
||||
}
|
||||
|
||||
impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
/// Create a new [`PersistenceHandle`] from a [`Sender<PersistenceAction>`].
|
||||
pub const fn new(sender: Sender<PersistenceAction<T>>) -> Self {
|
||||
Self { sender }
|
||||
///
|
||||
/// This is intended for testing purposes where you want to mock the persistence service.
|
||||
/// For production use, prefer [`spawn_service`](Self::spawn_service).
|
||||
pub fn new(sender: Sender<PersistenceAction<T>>) -> Self {
|
||||
Self { sender, _service_guard: Arc::new(ServiceGuard(None)) }
|
||||
}
|
||||
|
||||
/// Create a new [`PersistenceHandle`], and spawn the persistence service.
|
||||
///
|
||||
/// The returned handle can be cloned and shared. When all clones are dropped, the service
|
||||
/// thread will be joined, ensuring graceful shutdown before resources (like `RocksDB`) are
|
||||
/// released.
|
||||
pub fn spawn_service<N>(
|
||||
provider_factory: ProviderFactory<N>,
|
||||
pruner: PrunerWithFactory<ProviderFactory<N>>,
|
||||
@@ -224,13 +261,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
// create the initial channels
|
||||
let (db_service_tx, db_service_rx) = std::sync::mpsc::channel();
|
||||
|
||||
// construct persistence handle
|
||||
let persistence_handle = PersistenceHandle::new(db_service_tx);
|
||||
|
||||
// spawn the persistence service
|
||||
let db_service =
|
||||
PersistenceService::new(provider_factory, db_service_rx, pruner, sync_metrics_tx);
|
||||
std::thread::Builder::new()
|
||||
let join_handle = std::thread::Builder::new()
|
||||
.name("Persistence Service".to_string())
|
||||
.spawn(|| {
|
||||
if let Err(err) = db_service.run() {
|
||||
@@ -239,7 +273,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
persistence_handle
|
||||
PersistenceHandle {
|
||||
sender: db_service_tx,
|
||||
_service_guard: Arc::new(ServiceGuard(Some(join_handle))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sends a specific [`PersistenceAction`] in the contained channel. The caller is responsible
|
||||
@@ -267,7 +304,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
self.send_action(PersistenceAction::SaveBlocks(blocks, tx))
|
||||
}
|
||||
|
||||
/// Persists the finalized block number on disk.
|
||||
/// Queues the finalized block number to be persisted on disk.
|
||||
///
|
||||
/// The update is deferred and will be committed together with the next [`Self::save_blocks`]
|
||||
/// call to avoid triggering a separate fsync for each update.
|
||||
pub fn save_finalized_block_number(
|
||||
&self,
|
||||
finalized_block: u64,
|
||||
@@ -275,7 +315,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
self.send_action(PersistenceAction::SaveFinalizedBlock(finalized_block))
|
||||
}
|
||||
|
||||
/// Persists the safe block number on disk.
|
||||
/// Queues the safe block number to be persisted on disk.
|
||||
///
|
||||
/// The update is deferred and will be committed together with the next [`Self::save_blocks`]
|
||||
/// call to avoid triggering a separate fsync for each update.
|
||||
pub fn save_safe_block_number(
|
||||
&self,
|
||||
safe_block: u64,
|
||||
@@ -297,6 +340,27 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Guard that joins the persistence service thread when dropped.
|
||||
///
|
||||
/// This ensures graceful shutdown - the service thread completes before resources like
|
||||
/// `RocksDB` are released. Stored in an `Arc` inside [`PersistenceHandle`] so the handle
|
||||
/// can be cloned while sharing the same guard.
|
||||
struct ServiceGuard(Option<JoinHandle<()>>);
|
||||
|
||||
impl std::fmt::Debug for ServiceGuard {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_tuple("ServiceGuard").field(&self.0.as_ref().map(|_| "...")).finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ServiceGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(join_handle) = self.0.take() {
|
||||
let _ = join_handle.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -323,12 +387,12 @@ mod tests {
|
||||
#[test]
|
||||
fn test_save_blocks_empty() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let persistence_handle = default_persistence_handle();
|
||||
let handle = default_persistence_handle();
|
||||
|
||||
let blocks = vec![];
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
persistence_handle.save_blocks(blocks, tx).unwrap();
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
|
||||
let hash = rx.recv().unwrap();
|
||||
assert_eq!(hash, None);
|
||||
@@ -337,7 +401,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_save_blocks_single_block() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let persistence_handle = default_persistence_handle();
|
||||
let handle = default_persistence_handle();
|
||||
let block_number = 0;
|
||||
let mut test_block_builder = TestBlockBuilder::eth();
|
||||
let executed =
|
||||
@@ -347,7 +411,7 @@ mod tests {
|
||||
let blocks = vec![executed];
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
persistence_handle.save_blocks(blocks, tx).unwrap();
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
|
||||
let BlockNumHash { hash: actual_hash, number: _ } = rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
@@ -360,14 +424,14 @@ mod tests {
|
||||
#[test]
|
||||
fn test_save_blocks_multiple_blocks() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let persistence_handle = default_persistence_handle();
|
||||
let handle = default_persistence_handle();
|
||||
|
||||
let mut test_block_builder = TestBlockBuilder::eth();
|
||||
let blocks = test_block_builder.get_executed_blocks(0..5).collect::<Vec<_>>();
|
||||
let last_hash = blocks.last().unwrap().recovered_block().hash();
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
persistence_handle.save_blocks(blocks, tx).unwrap();
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
let BlockNumHash { hash: actual_hash, number: _ } = rx.recv().unwrap().unwrap();
|
||||
assert_eq!(last_hash, actual_hash);
|
||||
}
|
||||
@@ -375,7 +439,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_save_blocks_multiple_calls() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let persistence_handle = default_persistence_handle();
|
||||
let handle = default_persistence_handle();
|
||||
|
||||
let ranges = [0..1, 1..2, 2..4, 4..5];
|
||||
let mut test_block_builder = TestBlockBuilder::eth();
|
||||
@@ -384,7 +448,7 @@ mod tests {
|
||||
let last_hash = blocks.last().unwrap().recovered_block().hash();
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
persistence_handle.save_blocks(blocks, tx).unwrap();
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
|
||||
let BlockNumHash { hash: actual_hash, number: _ } = rx.recv().unwrap().unwrap();
|
||||
assert_eq!(last_hash, actual_hash);
|
||||
|
||||
@@ -61,7 +61,6 @@ mod persistence_state;
|
||||
pub mod precompile_cache;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
#[expect(unused)]
|
||||
mod trie_updates;
|
||||
|
||||
use crate::tree::error::AdvancePersistenceError;
|
||||
@@ -2613,19 +2612,27 @@ where
|
||||
let block_num_hash = block_id.block;
|
||||
debug!(target: "engine::tree", block=?block_num_hash, parent = ?block_id.parent, "Inserting new block into tree");
|
||||
|
||||
match self.sealed_header_by_hash(block_num_hash.hash) {
|
||||
Err(err) => {
|
||||
let block = convert_to_block(self, input)?;
|
||||
return Err(InsertBlockError::new(block, err.into()).into());
|
||||
// Check if block already exists - first in memory, then DB only if it could be persisted
|
||||
if self.state.tree_state.sealed_header_by_hash(&block_num_hash.hash).is_some() {
|
||||
convert_to_block(self, input)?;
|
||||
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid));
|
||||
}
|
||||
|
||||
// Only query DB if block could be persisted (number <= last persisted block).
|
||||
// New blocks from CL always have number > last persisted, so skip DB lookup for them.
|
||||
if block_num_hash.number <= self.persistence_state.last_persisted_block.number {
|
||||
match self.provider.sealed_header_by_hash(block_num_hash.hash) {
|
||||
Err(err) => {
|
||||
let block = convert_to_block(self, input)?;
|
||||
return Err(InsertBlockError::new(block, err.into()).into());
|
||||
}
|
||||
Ok(Some(_)) => {
|
||||
convert_to_block(self, input)?;
|
||||
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid));
|
||||
}
|
||||
Ok(None) => {}
|
||||
}
|
||||
Ok(Some(_)) => {
|
||||
// We now assume that we already have this block in the tree. However, we need to
|
||||
// run the conversion to ensure that the block hash is valid.
|
||||
convert_to_block(self, input)?;
|
||||
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid))
|
||||
}
|
||||
_ => {}
|
||||
};
|
||||
}
|
||||
|
||||
// Ensure that the parent state is available.
|
||||
match self.state_provider_builder(block_id.parent) {
|
||||
|
||||
@@ -39,7 +39,7 @@ use reth_trie_parallel::{
|
||||
proof_task::{ProofTaskCtx, ProofWorkerHandle},
|
||||
root::ParallelStateRootError,
|
||||
};
|
||||
use reth_trie_sparse::{ClearedSparseStateTrie, RevealableSparseTrie, SparseStateTrie};
|
||||
use reth_trie_sparse::{RevealableSparseTrie, SparseStateTrie};
|
||||
use reth_trie_sparse_parallel::{ParallelSparseTrie, ParallelismThresholds};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
@@ -235,8 +235,7 @@ where
|
||||
+ 'static,
|
||||
{
|
||||
// start preparing transactions immediately
|
||||
let (prewarm_rx, execution_rx, transaction_count_hint) =
|
||||
self.spawn_tx_iterator(transactions);
|
||||
let (prewarm_rx, execution_rx) = self.spawn_tx_iterator(transactions);
|
||||
|
||||
let span = Span::current();
|
||||
let (to_sparse_trie, sparse_trie_rx) = channel();
|
||||
@@ -260,7 +259,6 @@ where
|
||||
self.spawn_caching_with(
|
||||
env,
|
||||
prewarm_rx,
|
||||
transaction_count_hint,
|
||||
provider_builder.clone(),
|
||||
None, // Don't send proof targets when BAL is present
|
||||
Some(bal),
|
||||
@@ -271,7 +269,6 @@ where
|
||||
self.spawn_caching_with(
|
||||
env,
|
||||
prewarm_rx,
|
||||
transaction_count_hint,
|
||||
provider_builder.clone(),
|
||||
Some(to_multi_proof.clone()),
|
||||
None,
|
||||
@@ -355,10 +352,10 @@ where
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
{
|
||||
let (prewarm_rx, execution_rx, size_hint) = self.spawn_tx_iterator(transactions);
|
||||
let (prewarm_rx, execution_rx) = self.spawn_tx_iterator(transactions);
|
||||
// This path doesn't use multiproof, so V2 proofs flag doesn't matter
|
||||
let prewarm_handle =
|
||||
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal, false);
|
||||
self.spawn_caching_with(env, prewarm_rx, provider_builder, None, bal, false);
|
||||
PayloadHandle {
|
||||
to_multi_proof: None,
|
||||
prewarm_handle,
|
||||
@@ -376,19 +373,15 @@ where
|
||||
) -> (
|
||||
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Recovered>>,
|
||||
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Recovered>, I::Error>>,
|
||||
usize,
|
||||
) {
|
||||
let (transactions, convert) = transactions.into();
|
||||
let transactions = transactions.into_par_iter();
|
||||
let transaction_count_hint = transactions.len();
|
||||
|
||||
let (ooo_tx, ooo_rx) = mpsc::channel();
|
||||
let (prewarm_tx, prewarm_rx) = mpsc::channel();
|
||||
let (execute_tx, execute_rx) = mpsc::channel();
|
||||
|
||||
// Spawn a task that `convert`s all transactions in parallel and sends them out-of-order.
|
||||
self.executor.spawn_blocking(move || {
|
||||
transactions.enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
|
||||
rayon::spawn(move || {
|
||||
let (transactions, convert) = transactions.into();
|
||||
transactions.into_par_iter().enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
|
||||
let tx = convert(tx);
|
||||
let tx = tx.map(|tx| {
|
||||
let (tx_env, tx) = tx.into_parts();
|
||||
@@ -424,16 +417,14 @@ where
|
||||
}
|
||||
});
|
||||
|
||||
(prewarm_rx, execute_rx, transaction_count_hint)
|
||||
(prewarm_rx, execute_rx)
|
||||
}
|
||||
|
||||
/// Spawn prewarming optionally wired to the multiproof task for target updates.
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
fn spawn_caching_with<P>(
|
||||
&self,
|
||||
env: ExecutionEnv<Evm>,
|
||||
mut transactions: mpsc::Receiver<impl ExecutableTxFor<Evm> + Clone + Send + 'static>,
|
||||
transaction_count_hint: usize,
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
bal: Option<Arc<BlockAccessList>>,
|
||||
@@ -468,7 +459,6 @@ where
|
||||
self.execution_cache.clone(),
|
||||
prewarm_ctx,
|
||||
to_multi_proof,
|
||||
transaction_count_hint,
|
||||
self.prewarm_max_concurrency,
|
||||
);
|
||||
|
||||
@@ -562,11 +552,11 @@ where
|
||||
sparse_state_trie,
|
||||
))
|
||||
} else {
|
||||
SpawnedSparseTrieTask::Cached(SparseTrieCacheTask::new_with_cleared_trie(
|
||||
SpawnedSparseTrieTask::Cached(SparseTrieCacheTask::new_with_trie(
|
||||
from_multi_proof,
|
||||
proof_worker_handle,
|
||||
trie_metrics.clone(),
|
||||
ClearedSparseStateTrie::from_state_trie(sparse_state_trie),
|
||||
sparse_state_trie,
|
||||
))
|
||||
};
|
||||
|
||||
@@ -961,6 +951,10 @@ pub struct ExecutionEnv<Evm: ConfigureEvm> {
|
||||
/// Used for sparse trie continuation: if the preserved trie's anchor matches this,
|
||||
/// the trie can be reused directly.
|
||||
pub parent_state_root: B256,
|
||||
/// Number of transactions in the block.
|
||||
/// Used to determine parallel worker count for prewarming.
|
||||
/// A value of 0 indicates the count is unknown.
|
||||
pub transaction_count: usize,
|
||||
}
|
||||
|
||||
impl<Evm: ConfigureEvm> Default for ExecutionEnv<Evm>
|
||||
@@ -973,6 +967,7 @@ where
|
||||
hash: Default::default(),
|
||||
parent_hash: Default::default(),
|
||||
parent_state_root: Default::default(),
|
||||
transaction_count: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,8 +84,6 @@ where
|
||||
ctx: PrewarmContext<N, P, Evm>,
|
||||
/// How many transactions should be executed in parallel
|
||||
max_concurrency: usize,
|
||||
/// The number of transactions to be processed
|
||||
transaction_count_hint: usize,
|
||||
/// Sender to emit evm state outcome messages, if any.
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
/// Receiver for events produced by tx execution
|
||||
@@ -106,7 +104,6 @@ where
|
||||
execution_cache: PayloadExecutionCache,
|
||||
ctx: PrewarmContext<N, P, Evm>,
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
transaction_count_hint: usize,
|
||||
max_concurrency: usize,
|
||||
) -> (Self, Sender<PrewarmTaskEvent<N::Receipt>>) {
|
||||
let (actions_tx, actions_rx) = channel();
|
||||
@@ -114,7 +111,7 @@ where
|
||||
trace!(
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
max_concurrency,
|
||||
transaction_count_hint,
|
||||
transaction_count = ctx.env.transaction_count,
|
||||
"Initialized prewarm task"
|
||||
);
|
||||
|
||||
@@ -124,7 +121,6 @@ where
|
||||
execution_cache,
|
||||
ctx,
|
||||
max_concurrency,
|
||||
transaction_count_hint,
|
||||
to_multi_proof,
|
||||
actions_rx,
|
||||
parent_span: Span::current(),
|
||||
@@ -148,7 +144,6 @@ where
|
||||
let executor = self.executor.clone();
|
||||
let ctx = self.ctx.clone();
|
||||
let max_concurrency = self.max_concurrency;
|
||||
let transaction_count_hint = self.transaction_count_hint;
|
||||
let span = Span::current();
|
||||
|
||||
self.executor.spawn_blocking(move || {
|
||||
@@ -156,13 +151,14 @@ where
|
||||
|
||||
let (done_tx, done_rx) = mpsc::channel();
|
||||
|
||||
// When transaction_count_hint is 0, it means the count is unknown. In this case, spawn
|
||||
// When transaction_count is 0, it means the count is unknown. In this case, spawn
|
||||
// max workers to handle potentially many transactions in parallel rather
|
||||
// than bottlenecking on a single worker.
|
||||
let workers_needed = if transaction_count_hint == 0 {
|
||||
let transaction_count = ctx.env.transaction_count;
|
||||
let workers_needed = if transaction_count == 0 {
|
||||
max_concurrency
|
||||
} else {
|
||||
transaction_count_hint.min(max_concurrency)
|
||||
transaction_count.min(max_concurrency)
|
||||
};
|
||||
|
||||
// Spawn workers
|
||||
|
||||
@@ -5,14 +5,15 @@ use crate::tree::{
|
||||
payload_processor::multiproof::{MultiProofTaskMetrics, SparseTrieUpdate},
|
||||
};
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rlp::Decodable;
|
||||
use alloy_rlp::{Decodable, Encodable};
|
||||
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
|
||||
use rayon::iter::{ParallelBridge, ParallelIterator};
|
||||
use rayon::iter::ParallelIterator;
|
||||
use reth_errors::ProviderError;
|
||||
use reth_primitives_traits::Account;
|
||||
use reth_primitives_traits::{Account, ParallelBridgeBuffered};
|
||||
use reth_revm::state::EvmState;
|
||||
use reth_trie::{
|
||||
proof_v2::Target, updates::TrieUpdates, HashedPostState, Nibbles, TrieAccount, EMPTY_ROOT_HASH,
|
||||
TRIE_ACCOUNT_RLP_MAX_SIZE,
|
||||
};
|
||||
use reth_trie_parallel::{
|
||||
proof_task::{
|
||||
@@ -25,8 +26,7 @@ use reth_trie_parallel::{
|
||||
use reth_trie_sparse::{
|
||||
errors::{SparseStateTrieResult, SparseTrieErrorKind},
|
||||
provider::{TrieNodeProvider, TrieNodeProviderFactory},
|
||||
ClearedSparseStateTrie, LeafUpdate, SerialSparseTrie, SparseStateTrie, SparseTrie,
|
||||
SparseTrieExt,
|
||||
LeafUpdate, SerialSparseTrie, SparseStateTrie, SparseTrie, SparseTrieExt,
|
||||
};
|
||||
use revm_primitives::{hash_map::Entry, B256Map};
|
||||
use smallvec::SmallVec;
|
||||
@@ -239,6 +239,8 @@ pub(super) struct SparseTrieCacheTask<A = SerialSparseTrie, S = SerialSparseTrie
|
||||
/// Cache of storage proof targets that have already been fetched/requested from the proof
|
||||
/// workers. account -> slot -> lowest `min_len` requested.
|
||||
fetched_storage_targets: B256Map<B256Map<u8>>,
|
||||
/// Reusable buffer for RLP encoding of accounts.
|
||||
account_rlp_buf: Vec<u8>,
|
||||
/// Metrics for the sparse trie.
|
||||
metrics: MultiProofTaskMetrics,
|
||||
}
|
||||
@@ -248,12 +250,12 @@ where
|
||||
A: SparseTrieExt + Default,
|
||||
S: SparseTrieExt + Default + Clone,
|
||||
{
|
||||
/// Creates a new sparse trie, pre-populating with a [`ClearedSparseStateTrie`].
|
||||
pub(super) fn new_with_cleared_trie(
|
||||
/// Creates a new sparse trie, pre-populating with an existing [`SparseStateTrie`].
|
||||
pub(super) fn new_with_trie(
|
||||
updates: CrossbeamReceiver<MultiProofMessage>,
|
||||
proof_worker_handle: ProofWorkerHandle,
|
||||
metrics: MultiProofTaskMetrics,
|
||||
sparse_state_trie: ClearedSparseStateTrie<A, S>,
|
||||
sparse_state_trie: SparseStateTrie<A, S>,
|
||||
) -> Self {
|
||||
let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded();
|
||||
Self {
|
||||
@@ -261,12 +263,13 @@ where
|
||||
proof_result_rx,
|
||||
updates,
|
||||
proof_worker_handle,
|
||||
trie: sparse_state_trie.into_inner(),
|
||||
trie: sparse_state_trie,
|
||||
account_updates: Default::default(),
|
||||
storage_updates: Default::default(),
|
||||
pending_account_updates: Default::default(),
|
||||
fetched_account_targets: Default::default(),
|
||||
fetched_storage_targets: Default::default(),
|
||||
account_rlp_buf: Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE),
|
||||
metrics,
|
||||
}
|
||||
}
|
||||
@@ -359,11 +362,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// Process any remaining pending account updates.
|
||||
if !self.pending_account_updates.is_empty() {
|
||||
self.process_updates()?;
|
||||
}
|
||||
|
||||
debug!(target: "engine::root", "All proofs processed, ending calculation");
|
||||
|
||||
let start = Instant::now();
|
||||
@@ -513,78 +511,94 @@ where
|
||||
{
|
||||
Vec::new()
|
||||
} else {
|
||||
// TODO: optimize allocation
|
||||
alloy_rlp::encode(
|
||||
account.unwrap_or_default().into_trie_account(storage_root),
|
||||
)
|
||||
self.account_rlp_buf.clear();
|
||||
account
|
||||
.unwrap_or_default()
|
||||
.into_trie_account(storage_root)
|
||||
.encode(&mut self.account_rlp_buf);
|
||||
self.account_rlp_buf.clone()
|
||||
};
|
||||
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Now handle pending account updates that can be upgraded to a proper update.
|
||||
self.pending_account_updates.retain(|addr, account| {
|
||||
// If account has pending storage updates, it is still pending.
|
||||
if self.storage_updates.get(addr).is_some_and(|updates| !updates.is_empty()) {
|
||||
return true;
|
||||
}
|
||||
loop {
|
||||
// Now handle pending account updates that can be upgraded to a proper update.
|
||||
let account_rlp_buf = &mut self.account_rlp_buf;
|
||||
self.pending_account_updates.retain(|addr, account| {
|
||||
// If account has pending storage updates, it is still pending.
|
||||
if self.storage_updates.get(addr).is_some_and(|updates| !updates.is_empty()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Get the current account state either from the trie or from latest account update.
|
||||
let trie_account = if let Some(LeafUpdate::Changed(encoded)) = self.account_updates.get(addr) {
|
||||
Some(encoded).filter(|encoded| !encoded.is_empty())
|
||||
} else if !self.account_updates.contains_key(addr) {
|
||||
self.trie.get_account_value(addr)
|
||||
} else {
|
||||
// Needs to be revealed first
|
||||
return true;
|
||||
};
|
||||
// Get the current account state either from the trie or from latest account update.
|
||||
let trie_account = if let Some(LeafUpdate::Changed(encoded)) = self.account_updates.get(addr) {
|
||||
Some(encoded).filter(|encoded| !encoded.is_empty())
|
||||
} else if !self.account_updates.contains_key(addr) {
|
||||
self.trie.get_account_value(addr)
|
||||
} else {
|
||||
// Needs to be revealed first
|
||||
return true;
|
||||
};
|
||||
|
||||
let trie_account = trie_account.map(|value| TrieAccount::decode(&mut &value[..]).expect("invalid account RLP"));
|
||||
let trie_account = trie_account.map(|value| TrieAccount::decode(&mut &value[..]).expect("invalid account RLP"));
|
||||
|
||||
let (account, storage_root) = if let Some(account) = account.take() {
|
||||
// If account is Some(_) here it means it didn't have any storage updates
|
||||
// and we can fetch the storage root directly from the account trie.
|
||||
//
|
||||
// If it did have storage updates, we would've had processed it above when iterating over storage tries.
|
||||
let storage_root = trie_account.map(|account| account.storage_root).unwrap_or(EMPTY_ROOT_HASH);
|
||||
let (account, storage_root) = if let Some(account) = account.take() {
|
||||
// If account is Some(_) here it means it didn't have any storage updates
|
||||
// and we can fetch the storage root directly from the account trie.
|
||||
//
|
||||
// If it did have storage updates, we would've had processed it above when iterating over storage tries.
|
||||
let storage_root = trie_account.map(|account| account.storage_root).unwrap_or(EMPTY_ROOT_HASH);
|
||||
|
||||
(account, storage_root)
|
||||
} else {
|
||||
(trie_account.map(Into::into), self.trie.storage_root(addr).expect("account had storage updates that were applied to its trie, storage root must be revealed by now"))
|
||||
};
|
||||
(account, storage_root)
|
||||
} else {
|
||||
(trie_account.map(Into::into), self.trie.storage_root(addr).expect("account had storage updates that were applied to its trie, storage root must be revealed by now"))
|
||||
};
|
||||
|
||||
let encoded = if account.is_none_or(|account| account.is_empty()) && storage_root == EMPTY_ROOT_HASH {
|
||||
Vec::new()
|
||||
} else {
|
||||
let account = account.unwrap_or_default().into_trie_account(storage_root);
|
||||
let encoded = if account.is_none_or(|account| account.is_empty()) && storage_root == EMPTY_ROOT_HASH {
|
||||
Vec::new()
|
||||
} else {
|
||||
account_rlp_buf.clear();
|
||||
account.unwrap_or_default().into_trie_account(storage_root).encode(account_rlp_buf);
|
||||
account_rlp_buf.clone()
|
||||
};
|
||||
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
|
||||
|
||||
// TODO: optimize allocation
|
||||
alloy_rlp::encode(account)
|
||||
};
|
||||
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
|
||||
false
|
||||
});
|
||||
|
||||
false
|
||||
});
|
||||
let updates_len_before = self.account_updates.len();
|
||||
|
||||
// Process account trie updates and fill the account targets.
|
||||
self.trie
|
||||
.trie_mut()
|
||||
.update_leaves(&mut self.account_updates, |target, min_len| {
|
||||
match self.fetched_account_targets.entry(target) {
|
||||
Entry::Occupied(mut entry) => {
|
||||
if min_len < *entry.get() {
|
||||
// Process account trie updates and fill the account targets.
|
||||
self.trie
|
||||
.trie_mut()
|
||||
.update_leaves(&mut self.account_updates, |target, min_len| {
|
||||
match self.fetched_account_targets.entry(target) {
|
||||
Entry::Occupied(mut entry) => {
|
||||
if min_len < *entry.get() {
|
||||
entry.insert(min_len);
|
||||
targets
|
||||
.account_targets
|
||||
.push(Target::new(target).with_min_len(min_len));
|
||||
}
|
||||
}
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(min_len);
|
||||
targets.account_targets.push(Target::new(target).with_min_len(min_len));
|
||||
}
|
||||
}
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(min_len);
|
||||
targets.account_targets.push(Target::new(target).with_min_len(min_len));
|
||||
}
|
||||
}
|
||||
})
|
||||
.map_err(ProviderError::other)?;
|
||||
})
|
||||
.map_err(ProviderError::other)?;
|
||||
|
||||
if updates_len_before == self.account_updates.len() {
|
||||
// Only exit when no new updates are processed.
|
||||
//
|
||||
// We need to keep iterating if any updates are being drained because that might
|
||||
// indicate that more pending account updates can be promoted.
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if !targets.is_empty() {
|
||||
self.proof_worker_handle.dispatch_account_multiproof(AccountMultiproofInput::V2 {
|
||||
@@ -651,7 +665,7 @@ where
|
||||
.storages
|
||||
.into_iter()
|
||||
.map(|(address, storage)| (address, storage, trie.take_storage_trie(&address)))
|
||||
.par_bridge()
|
||||
.par_bridge_buffered()
|
||||
.map(|(address, storage, storage_trie)| {
|
||||
let _enter =
|
||||
debug_span!(target: "engine::tree::payload_processor::sparse_trie", parent: &span, "storage trie", ?address)
|
||||
|
||||
@@ -407,6 +407,7 @@ where
|
||||
hash: input.hash(),
|
||||
parent_hash: input.parent_hash(),
|
||||
parent_state_root: parent_block.state_root(),
|
||||
transaction_count: input.transaction_count(),
|
||||
};
|
||||
|
||||
// Plan the strategy used for state root computation.
|
||||
@@ -519,6 +520,14 @@ where
|
||||
info!(target: "engine::tree::payload_validator", ?state_root, ?elapsed, "State root task finished");
|
||||
// we double check the state root here for good measure
|
||||
if state_root == block.header().state_root() {
|
||||
// Compare trie updates with serial computation if configured
|
||||
if self.config.always_compare_trie_updates() {
|
||||
self.compare_trie_updates_with_serial(
|
||||
overlay_factory.clone(),
|
||||
&hashed_state,
|
||||
trie_updates.clone(),
|
||||
);
|
||||
}
|
||||
maybe_state_root = Some((state_root, trie_updates, elapsed))
|
||||
} else {
|
||||
warn!(
|
||||
@@ -894,6 +903,62 @@ where
|
||||
.root_with_updates()?)
|
||||
}
|
||||
|
||||
/// Compares trie updates from the state root task with serial state root computation.
|
||||
///
|
||||
/// This is used for debugging and validating the correctness of the parallel state root
|
||||
/// task implementation. When enabled via `--engine.state-root-task-compare-updates`, this
|
||||
/// method runs a separate serial state root computation and compares the resulting trie
|
||||
/// updates.
|
||||
fn compare_trie_updates_with_serial(
|
||||
&self,
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
hashed_state: &HashedPostState,
|
||||
task_trie_updates: TrieUpdates,
|
||||
) {
|
||||
debug!(target: "engine::tree::payload_validator", "Comparing trie updates with serial computation");
|
||||
|
||||
match self.compute_state_root_serial(overlay_factory.clone(), hashed_state) {
|
||||
Ok((serial_root, serial_trie_updates)) => {
|
||||
debug!(
|
||||
target: "engine::tree::payload_validator",
|
||||
?serial_root,
|
||||
"Serial state root computation finished for comparison"
|
||||
);
|
||||
|
||||
// Get a database provider to use as trie cursor factory
|
||||
match overlay_factory.database_provider_ro() {
|
||||
Ok(provider) => {
|
||||
if let Err(err) = super::trie_updates::compare_trie_updates(
|
||||
&provider,
|
||||
task_trie_updates,
|
||||
serial_trie_updates,
|
||||
) {
|
||||
warn!(
|
||||
target: "engine::tree::payload_validator",
|
||||
%err,
|
||||
"Error comparing trie updates"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "engine::tree::payload_validator",
|
||||
%err,
|
||||
"Failed to get database provider for trie update comparison"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "engine::tree::payload_validator",
|
||||
%err,
|
||||
"Failed to compute serial state root for comparison"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates the block after execution.
|
||||
///
|
||||
/// This performs:
|
||||
|
||||
@@ -98,7 +98,7 @@ impl StorageTrieUpdatesDiff {
|
||||
|
||||
/// Compares the trie updates from state root task, regular state root calculation and database,
|
||||
/// and logs the differences if there's any.
|
||||
pub(super) fn compare_trie_updates(
|
||||
pub(crate) fn compare_trie_updates(
|
||||
trie_cursor_factory: impl TrieCursorFactory,
|
||||
task: TrieUpdates,
|
||||
regular: TrieUpdates,
|
||||
@@ -186,7 +186,8 @@ fn compare_storage_trie_updates<C: TrieCursor>(
|
||||
task: &mut StorageTrieUpdates,
|
||||
regular: &mut StorageTrieUpdates,
|
||||
) -> Result<StorageTrieUpdatesDiff, DatabaseError> {
|
||||
let database_not_exists = trie_cursor()?.next()?.is_none();
|
||||
// Check if the storage trie exists by seeking to the first entry
|
||||
let database_not_exists = trie_cursor()?.seek(Nibbles::default())?.is_none();
|
||||
let mut diff = StorageTrieUpdatesDiff {
|
||||
// If the deletion is a no-op, meaning that the entry is not in the
|
||||
// database, do not add it to the diff.
|
||||
|
||||
@@ -20,8 +20,6 @@ reth-era.workspace = true
|
||||
# http
|
||||
bytes.workspace = true
|
||||
reqwest.workspace = true
|
||||
reqwest.default-features = false
|
||||
reqwest.features = ["stream", "rustls-tls-native-roots"]
|
||||
|
||||
# async
|
||||
tokio.workspace = true
|
||||
|
||||
@@ -86,7 +86,7 @@ where
|
||||
mut self,
|
||||
components: impl CliComponentsBuilder<N>,
|
||||
launcher: impl AsyncFnOnce(
|
||||
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
Ext,
|
||||
) -> Result<()>,
|
||||
) -> Result<()>
|
||||
@@ -132,7 +132,7 @@ pub(crate) fn run_commands_with<C, Ext, Rpc, N, SubCmd>(
|
||||
runner: CliRunner,
|
||||
components: impl CliComponentsBuilder<N>,
|
||||
launcher: impl AsyncFnOnce(
|
||||
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
Ext,
|
||||
) -> Result<()>,
|
||||
) -> Result<()>
|
||||
|
||||
@@ -131,7 +131,7 @@ impl<
|
||||
/// ````
|
||||
pub fn run<L, Fut>(self, launcher: L) -> eyre::Result<()>
|
||||
where
|
||||
L: FnOnce(WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>, Ext) -> Fut,
|
||||
L: FnOnce(WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>, Ext) -> Fut,
|
||||
Fut: Future<Output = eyre::Result<()>>,
|
||||
C: ChainSpecParser<ChainSpec = ChainSpec>,
|
||||
{
|
||||
@@ -148,7 +148,7 @@ impl<
|
||||
self,
|
||||
components: impl CliComponentsBuilder<N>,
|
||||
launcher: impl AsyncFnOnce(
|
||||
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
Ext,
|
||||
) -> eyre::Result<()>,
|
||||
) -> eyre::Result<()>
|
||||
@@ -180,7 +180,7 @@ impl<
|
||||
/// ```
|
||||
pub fn with_runner<L, Fut>(self, runner: CliRunner, launcher: L) -> eyre::Result<()>
|
||||
where
|
||||
L: FnOnce(WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>, Ext) -> Fut,
|
||||
L: FnOnce(WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>, Ext) -> Fut,
|
||||
Fut: Future<Output = eyre::Result<()>>,
|
||||
C: ChainSpecParser<ChainSpec = ChainSpec>,
|
||||
{
|
||||
@@ -196,7 +196,7 @@ impl<
|
||||
runner: CliRunner,
|
||||
components: impl CliComponentsBuilder<N>,
|
||||
launcher: impl AsyncFnOnce(
|
||||
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
Ext,
|
||||
) -> eyre::Result<()>,
|
||||
) -> eyre::Result<()>
|
||||
|
||||
@@ -119,10 +119,9 @@ impl EthereumNode {
|
||||
/// use reth_db::open_db_read_only;
|
||||
/// use reth_node_ethereum::EthereumNode;
|
||||
/// use reth_provider::providers::{RocksDBProvider, StaticFileProvider};
|
||||
/// use std::sync::Arc;
|
||||
///
|
||||
/// let factory = EthereumNode::provider_factory_builder()
|
||||
/// .db(Arc::new(open_db_read_only("db", Default::default()).unwrap()))
|
||||
/// .db(open_db_read_only("db", Default::default()).unwrap())
|
||||
/// .chainspec(ChainSpecBuilder::mainnet().build().into())
|
||||
/// .static_file(StaticFileProvider::read_only("db/static_files", false).unwrap())
|
||||
/// .rocksdb_provider(RocksDBProvider::builder("db/rocksdb").build().unwrap())
|
||||
|
||||
@@ -100,10 +100,12 @@ async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> {
|
||||
ChainSpecBuilder::default().chain(MAINNET.chain).genesis(genesis).osaka_activated().build(),
|
||||
);
|
||||
let genesis_hash = chain_spec.genesis_hash();
|
||||
let node_config = NodeConfig::test()
|
||||
.with_chain(chain_spec)
|
||||
.with_unused_ports()
|
||||
.with_rpc(RpcServerArgs::default().with_unused_ports().with_http());
|
||||
let node_config = NodeConfig::test().with_chain(chain_spec).with_unused_ports().with_rpc(
|
||||
RpcServerArgs::default()
|
||||
.with_unused_ports()
|
||||
.with_http()
|
||||
.with_force_blob_sidecar_upcasting(),
|
||||
);
|
||||
let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone())
|
||||
.testing_node(exec.clone())
|
||||
.node(EthereumNode::default())
|
||||
@@ -125,7 +127,7 @@ async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> {
|
||||
let blob_tx_hash = node.rpc.inject_tx(blob_tx).await?;
|
||||
// fetch it from rpc
|
||||
let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?;
|
||||
// assert that sidecar was converted to eip7594
|
||||
// assert that sidecar was converted to eip7594 (force upcasting is enabled)
|
||||
assert!(envelope.as_eip4844().unwrap().tx().sidecar().unwrap().is_eip7594());
|
||||
// validate sidecar
|
||||
TransactionTestContext::validate_sidecar(envelope);
|
||||
@@ -161,10 +163,12 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> {
|
||||
.build(),
|
||||
);
|
||||
let genesis_hash = chain_spec.genesis_hash();
|
||||
let node_config = NodeConfig::test()
|
||||
.with_chain(chain_spec)
|
||||
.with_unused_ports()
|
||||
.with_rpc(RpcServerArgs::default().with_unused_ports().with_http());
|
||||
let node_config = NodeConfig::test().with_chain(chain_spec).with_unused_ports().with_rpc(
|
||||
RpcServerArgs::default()
|
||||
.with_unused_ports()
|
||||
.with_http()
|
||||
.with_force_blob_sidecar_upcasting(),
|
||||
);
|
||||
let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone())
|
||||
.testing_node(exec.clone())
|
||||
.node(EthereumNode::default())
|
||||
|
||||
@@ -511,9 +511,8 @@ mod compact {
|
||||
total_length += flags.len() + buffer.len();
|
||||
buf.put_slice(&flags);
|
||||
if zstd {
|
||||
reth_zstd_compressors::RECEIPT_COMPRESSOR.with(|compressor| {
|
||||
let compressed =
|
||||
compressor.borrow_mut().compress(&buffer).expect("Failed to compress.");
|
||||
reth_zstd_compressors::with_receipt_compressor(|compressor| {
|
||||
let compressed = compressor.compress(&buffer).expect("Failed to compress.");
|
||||
buf.put(compressed.as_slice());
|
||||
});
|
||||
} else {
|
||||
@@ -525,8 +524,7 @@ mod compact {
|
||||
fn from_compact(buf: &[u8], _len: usize) -> (Self, &[u8]) {
|
||||
let (flags, mut buf) = ReceiptFlags::from(buf);
|
||||
if flags.__zstd() != 0 {
|
||||
reth_zstd_compressors::RECEIPT_DECOMPRESSOR.with(|decompressor| {
|
||||
let decompressor = &mut decompressor.borrow_mut();
|
||||
reth_zstd_compressors::with_receipt_decompressor(|decompressor| {
|
||||
let decompressed = decompressor.decompress(buf);
|
||||
let original_buf = buf;
|
||||
let mut buf: &[u8] = decompressed;
|
||||
|
||||
@@ -577,19 +577,11 @@ impl reth_codecs::Compact for TransactionSigned {
|
||||
|
||||
let tx_bits = if zstd_bit {
|
||||
let mut tmp = Vec::with_capacity(256);
|
||||
if cfg!(feature = "std") {
|
||||
reth_zstd_compressors::TRANSACTION_COMPRESSOR.with(|compressor| {
|
||||
let mut compressor = compressor.borrow_mut();
|
||||
let tx_bits = self.transaction.to_compact(&mut tmp);
|
||||
buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress"));
|
||||
tx_bits as u8
|
||||
})
|
||||
} else {
|
||||
let mut compressor = reth_zstd_compressors::create_tx_compressor();
|
||||
reth_zstd_compressors::with_tx_compressor(|compressor| {
|
||||
let tx_bits = self.transaction.to_compact(&mut tmp);
|
||||
buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress"));
|
||||
tx_bits as u8
|
||||
}
|
||||
})
|
||||
} else {
|
||||
self.transaction.to_compact(buf) as u8
|
||||
};
|
||||
@@ -611,26 +603,13 @@ impl reth_codecs::Compact for TransactionSigned {
|
||||
|
||||
let zstd_bit = bitflags >> 3;
|
||||
let (transaction, buf) = if zstd_bit != 0 {
|
||||
if cfg!(feature = "std") {
|
||||
reth_zstd_compressors::TRANSACTION_DECOMPRESSOR.with(|decompressor| {
|
||||
let mut decompressor = decompressor.borrow_mut();
|
||||
|
||||
// TODO: enforce that zstd is only present at a "top" level type
|
||||
|
||||
let transaction_type = (bitflags & 0b110) >> 1;
|
||||
let (transaction, _) =
|
||||
Transaction::from_compact(decompressor.decompress(buf), transaction_type);
|
||||
|
||||
(transaction, buf)
|
||||
})
|
||||
} else {
|
||||
let mut decompressor = reth_zstd_compressors::create_tx_decompressor();
|
||||
reth_zstd_compressors::with_tx_decompressor(|decompressor| {
|
||||
// TODO: enforce that zstd is only present at a "top" level type
|
||||
let transaction_type = (bitflags & 0b110) >> 1;
|
||||
let (transaction, _) =
|
||||
Transaction::from_compact(decompressor.decompress(buf), transaction_type);
|
||||
|
||||
(transaction, buf)
|
||||
}
|
||||
})
|
||||
} else {
|
||||
let transaction_type = bitflags >> 1;
|
||||
Transaction::from_compact(buf, transaction_type)
|
||||
|
||||
@@ -67,7 +67,6 @@ tempfile.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
edge = ["reth-provider/edge"]
|
||||
serde = [
|
||||
"reth-exex-types/serde",
|
||||
"reth-revm/serde",
|
||||
|
||||
@@ -251,6 +251,8 @@ impl<DB, ChainSpec: EthChainSpec> NodeBuilder<DB, ChainSpec> {
|
||||
}
|
||||
|
||||
/// Creates a preconfigured node for testing purposes with a specific datadir.
|
||||
///
|
||||
/// The entire `datadir` will be cleaned up when the node is dropped.
|
||||
#[cfg(feature = "test-utils")]
|
||||
pub fn testing_node_with_datadir(
|
||||
mut self,
|
||||
@@ -268,7 +270,7 @@ impl<DB, ChainSpec: EthChainSpec> NodeBuilder<DB, ChainSpec> {
|
||||
let data_dir =
|
||||
path.unwrap_or_chain_default(self.config.chain.chain(), self.config.datadir.clone());
|
||||
|
||||
let db = reth_db::test_utils::create_test_rw_db_with_path(data_dir.db());
|
||||
let db = reth_db::test_utils::create_test_rw_db_with_datadir(data_dir.data_dir());
|
||||
|
||||
WithLaunchContext { builder: self.with_database(db), task_executor }
|
||||
}
|
||||
|
||||
@@ -218,9 +218,9 @@ impl<Node: FullNodeComponents, AddOns: NodeAddOns<Node>> DerefMut for FullNode<N
|
||||
}
|
||||
|
||||
/// Helper type alias to define [`FullNode`] for a given [`Node`].
|
||||
pub type FullNodeFor<N, DB = Arc<DatabaseEnv>> =
|
||||
pub type FullNodeFor<N, DB = DatabaseEnv> =
|
||||
FullNode<NodeAdapter<RethFullAdapter<DB, N>>, <N as Node<RethFullAdapter<DB, N>>>::AddOns>;
|
||||
|
||||
/// Helper type alias to define [`NodeHandle`] for a given [`Node`].
|
||||
pub type NodeHandleFor<N, DB = Arc<DatabaseEnv>> =
|
||||
pub type NodeHandleFor<N, DB = DatabaseEnv> =
|
||||
NodeHandle<NodeAdapter<RethFullAdapter<DB, N>>, <N as Node<RethFullAdapter<DB, N>>>::AddOns>;
|
||||
|
||||
@@ -1192,6 +1192,7 @@ impl<'a, N: FullNodeComponents<Types: NodeTypes<ChainSpec: Hardforks + EthereumH
|
||||
.pending_block_kind(self.config.pending_block_kind)
|
||||
.raw_tx_forwarder(self.config.raw_tx_forwarder)
|
||||
.evm_memory_limit(self.config.rpc_evm_memory_limit)
|
||||
.force_blob_sidecar_upcasting(self.config.force_blob_sidecar_upcasting)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
//! clap [Args](clap::Args) for engine purposes
|
||||
|
||||
use clap::{builder::Resettable, Args};
|
||||
use reth_engine_primitives::{TreeConfig, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE};
|
||||
use reth_engine_primitives::{
|
||||
TreeConfig, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE, DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES,
|
||||
DEFAULT_SPARSE_TRIE_PRUNE_DEPTH,
|
||||
};
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use crate::node_config::{
|
||||
@@ -38,6 +41,8 @@ pub struct DefaultEngineValues {
|
||||
disable_proof_v2: bool,
|
||||
cache_metrics_disabled: bool,
|
||||
enable_sparse_trie_as_cache: bool,
|
||||
sparse_trie_prune_depth: usize,
|
||||
sparse_trie_max_storage_tries: usize,
|
||||
}
|
||||
|
||||
impl DefaultEngineValues {
|
||||
@@ -179,6 +184,18 @@ impl DefaultEngineValues {
|
||||
self.enable_sparse_trie_as_cache = v;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the sparse trie prune depth by default
|
||||
pub const fn with_sparse_trie_prune_depth(mut self, v: usize) -> Self {
|
||||
self.sparse_trie_prune_depth = v;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the maximum number of storage tries to retain after sparse trie pruning by default
|
||||
pub const fn with_sparse_trie_max_storage_tries(mut self, v: usize) -> Self {
|
||||
self.sparse_trie_max_storage_tries = v;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DefaultEngineValues {
|
||||
@@ -205,6 +222,8 @@ impl Default for DefaultEngineValues {
|
||||
disable_proof_v2: false,
|
||||
cache_metrics_disabled: false,
|
||||
enable_sparse_trie_as_cache: false,
|
||||
sparse_trie_prune_depth: DEFAULT_SPARSE_TRIE_PRUNE_DEPTH,
|
||||
sparse_trie_max_storage_tries: DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -336,6 +355,14 @@ pub struct EngineArgs {
|
||||
/// Enable sparse trie as cache.
|
||||
#[arg(long = "engine.enable-sparse-trie-as-cache", default_value_t = DefaultEngineValues::get_global().enable_sparse_trie_as_cache, conflicts_with = "disable_proof_v2")]
|
||||
pub enable_sparse_trie_as_cache: bool,
|
||||
|
||||
/// Sparse trie prune depth.
|
||||
#[arg(long = "engine.sparse-trie-prune-depth", default_value_t = DefaultEngineValues::get_global().sparse_trie_prune_depth, requires = "enable_sparse_trie_as_cache")]
|
||||
pub sparse_trie_prune_depth: usize,
|
||||
|
||||
/// Maximum number of storage tries to retain after sparse trie pruning.
|
||||
#[arg(long = "engine.sparse-trie-max-storage-tries", default_value_t = DefaultEngineValues::get_global().sparse_trie_max_storage_tries, requires = "enable_sparse_trie_as_cache")]
|
||||
pub sparse_trie_max_storage_tries: usize,
|
||||
}
|
||||
|
||||
#[allow(deprecated)]
|
||||
@@ -363,6 +390,8 @@ impl Default for EngineArgs {
|
||||
disable_proof_v2,
|
||||
cache_metrics_disabled,
|
||||
enable_sparse_trie_as_cache,
|
||||
sparse_trie_prune_depth,
|
||||
sparse_trie_max_storage_tries,
|
||||
} = DefaultEngineValues::get_global().clone();
|
||||
Self {
|
||||
persistence_threshold,
|
||||
@@ -390,6 +419,8 @@ impl Default for EngineArgs {
|
||||
disable_proof_v2,
|
||||
cache_metrics_disabled,
|
||||
enable_sparse_trie_as_cache,
|
||||
sparse_trie_prune_depth,
|
||||
sparse_trie_max_storage_tries,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -397,7 +428,7 @@ impl Default for EngineArgs {
|
||||
impl EngineArgs {
|
||||
/// Creates a [`TreeConfig`] from the engine arguments.
|
||||
pub fn tree_config(&self) -> TreeConfig {
|
||||
let mut config = TreeConfig::default()
|
||||
TreeConfig::default()
|
||||
.with_persistence_threshold(self.persistence_threshold)
|
||||
.with_memory_block_buffer_target(self.memory_block_buffer_target)
|
||||
.with_legacy_state_root(self.legacy_state_root_task_enabled)
|
||||
@@ -414,21 +445,14 @@ impl EngineArgs {
|
||||
.with_always_process_payload_attributes_on_canonical_head(
|
||||
self.always_process_payload_attributes_on_canonical_head,
|
||||
)
|
||||
.with_unwind_canonical_header(self.allow_unwind_canonical_header);
|
||||
|
||||
if let Some(count) = self.storage_worker_count {
|
||||
config = config.with_storage_worker_count(count);
|
||||
}
|
||||
|
||||
if let Some(count) = self.account_worker_count {
|
||||
config = config.with_account_worker_count(count);
|
||||
}
|
||||
|
||||
config = config.with_disable_proof_v2(self.disable_proof_v2);
|
||||
config = config.without_cache_metrics(self.cache_metrics_disabled);
|
||||
config = config.with_enable_sparse_trie_as_cache(self.enable_sparse_trie_as_cache);
|
||||
|
||||
config
|
||||
.with_unwind_canonical_header(self.allow_unwind_canonical_header)
|
||||
.with_storage_worker_count_opt(self.storage_worker_count)
|
||||
.with_account_worker_count_opt(self.account_worker_count)
|
||||
.with_disable_proof_v2(self.disable_proof_v2)
|
||||
.without_cache_metrics(self.cache_metrics_disabled)
|
||||
.with_enable_sparse_trie_as_cache(self.enable_sparse_trie_as_cache)
|
||||
.with_sparse_trie_prune_depth(self.sparse_trie_prune_depth)
|
||||
.with_sparse_trie_max_storage_tries(self.sparse_trie_max_storage_tries)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -479,7 +503,9 @@ mod tests {
|
||||
account_worker_count: Some(8),
|
||||
disable_proof_v2: false,
|
||||
cache_metrics_disabled: true,
|
||||
enable_sparse_trie_as_cache: false,
|
||||
enable_sparse_trie_as_cache: true,
|
||||
sparse_trie_prune_depth: 10,
|
||||
sparse_trie_max_storage_tries: 100,
|
||||
};
|
||||
|
||||
let parsed_args = CommandParser::<EngineArgs>::parse_from([
|
||||
@@ -510,6 +536,11 @@ mod tests {
|
||||
"--engine.account-worker-count",
|
||||
"8",
|
||||
"--engine.disable-cache-metrics",
|
||||
"--engine.enable-sparse-trie-as-cache",
|
||||
"--engine.sparse-trie-prune-depth",
|
||||
"10",
|
||||
"--engine.sparse-trie-max-storage-tries",
|
||||
"100",
|
||||
])
|
||||
.args;
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ use clap::{builder::RangedU64ValueParser, Args};
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_config::config::PruneConfig;
|
||||
use reth_prune_types::{
|
||||
PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_UNWIND_SAFE_DISTANCE,
|
||||
PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_DISTANCE, MINIMUM_UNWIND_SAFE_DISTANCE,
|
||||
};
|
||||
use std::{collections::BTreeMap, ops::Not, sync::OnceLock};
|
||||
|
||||
@@ -81,7 +81,7 @@ impl Default for DefaultPruningValues {
|
||||
minimal_prune_modes: PruneModes {
|
||||
sender_recovery: Some(PruneMode::Full),
|
||||
transaction_lookup: Some(PruneMode::Full),
|
||||
receipts: Some(PruneMode::Full),
|
||||
receipts: Some(PruneMode::Distance(MINIMUM_DISTANCE)),
|
||||
account_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
|
||||
storage_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
|
||||
bodies_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
|
||||
|
||||
@@ -647,6 +647,14 @@ pub struct RpcServerArgs {
|
||||
/// transactions from the same sender will also be skipped.
|
||||
#[arg(long = "testing.skip-invalid-transactions", default_value_t = true)]
|
||||
pub testing_skip_invalid_transactions: bool,
|
||||
|
||||
/// Force upcasting EIP-4844 blob sidecars to EIP-7594 format when Osaka is active.
|
||||
///
|
||||
/// When enabled, blob transactions submitted via `eth_sendRawTransaction` with EIP-4844
|
||||
/// sidecars will be automatically converted to EIP-7594 format if the next block is Osaka.
|
||||
/// By default this is disabled, meaning transactions are submitted as-is.
|
||||
#[arg(long = "rpc.force-blob-sidecar-upcasting", default_value_t = false)]
|
||||
pub rpc_force_blob_sidecar_upcasting: bool,
|
||||
}
|
||||
|
||||
impl RpcServerArgs {
|
||||
@@ -768,6 +776,12 @@ impl RpcServerArgs {
|
||||
self.rpc_send_raw_transaction_sync_timeout = timeout;
|
||||
self
|
||||
}
|
||||
|
||||
/// Enables forced blob sidecar upcasting from EIP-4844 to EIP-7594 format.
|
||||
pub const fn with_force_blob_sidecar_upcasting(mut self) -> Self {
|
||||
self.rpc_force_blob_sidecar_upcasting = true;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for RpcServerArgs {
|
||||
@@ -860,6 +874,7 @@ impl Default for RpcServerArgs {
|
||||
gas_price_oracle,
|
||||
rpc_send_raw_transaction_sync_timeout,
|
||||
testing_skip_invalid_transactions: true,
|
||||
rpc_force_blob_sidecar_upcasting: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1036,6 +1051,7 @@ mod tests {
|
||||
},
|
||||
rpc_send_raw_transaction_sync_timeout: std::time::Duration::from_secs(30),
|
||||
testing_skip_invalid_transactions: true,
|
||||
rpc_force_blob_sidecar_upcasting: false,
|
||||
};
|
||||
|
||||
let parsed_args = CommandParser::<RpcServerArgs>::parse_from([
|
||||
|
||||
@@ -21,7 +21,7 @@ alloy-primitives.workspace = true
|
||||
alloy-consensus.workspace = true
|
||||
|
||||
tokio.workspace = true
|
||||
tokio-tungstenite = { workspace = true, features = ["rustls-tls-native-roots"] }
|
||||
tokio-tungstenite.workspace = true
|
||||
futures-util.workspace = true
|
||||
tokio-stream.workspace = true
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ tempfile = { workspace = true, optional = true }
|
||||
tikv-jemalloc-ctl = { workspace = true, optional = true, features = ["stats"] }
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
procfs = "0.17.0"
|
||||
procfs = "0.18.0"
|
||||
|
||||
[dev-dependencies]
|
||||
reqwest.workspace = true
|
||||
|
||||
@@ -37,7 +37,7 @@ pub use commands::{import::ImportOpCommand, import_receipts::ImportReceiptsOpCom
|
||||
use reth_optimism_chainspec::OpChainSpec;
|
||||
use reth_rpc_server_types::{DefaultRpcModuleValidator, RpcModuleValidator};
|
||||
|
||||
use std::{ffi::OsString, fmt, marker::PhantomData, sync::Arc};
|
||||
use std::{ffi::OsString, fmt, marker::PhantomData};
|
||||
|
||||
use chainspec::OpChainSpecParser;
|
||||
use clap::Parser;
|
||||
@@ -121,7 +121,7 @@ where
|
||||
/// [`NodeCommand`](reth_cli_commands::node::NodeCommand).
|
||||
pub fn run<L, Fut>(self, launcher: L) -> eyre::Result<()>
|
||||
where
|
||||
L: FnOnce(WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>, Ext) -> Fut,
|
||||
L: FnOnce(WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>, Ext) -> Fut,
|
||||
Fut: Future<Output = eyre::Result<()>>,
|
||||
{
|
||||
self.with_runner(CliRunner::try_default_runtime()?, launcher)
|
||||
@@ -130,7 +130,7 @@ where
|
||||
/// Execute the configured cli command with the provided [`CliRunner`].
|
||||
pub fn with_runner<L, Fut>(self, runner: CliRunner, launcher: L) -> eyre::Result<()>
|
||||
where
|
||||
L: FnOnce(WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>, Ext) -> Fut,
|
||||
L: FnOnce(WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>, Ext) -> Fut,
|
||||
Fut: Future<Output = eyre::Result<()>>,
|
||||
{
|
||||
let mut this = self.configure();
|
||||
|
||||
@@ -38,7 +38,7 @@ op-alloy-rpc-types-engine = { workspace = true, features = ["k256"] }
|
||||
|
||||
# io
|
||||
tokio.workspace = true
|
||||
tokio-tungstenite = { workspace = true, features = ["rustls-tls-native-roots"] }
|
||||
tokio-tungstenite.workspace = true
|
||||
serde_json.workspace = true
|
||||
url.workspace = true
|
||||
futures-util.workspace = true
|
||||
|
||||
@@ -101,7 +101,7 @@ impl<T: SignedTransaction> SequenceManager<T> {
|
||||
// Bundle completed sequence with its decoded transactions and push to cache
|
||||
// Ring buffer automatically evicts oldest entry when full
|
||||
let txs = std::mem::take(&mut self.pending_transactions);
|
||||
self.completed_cache.push((completed, txs));
|
||||
self.completed_cache.enqueue((completed, txs));
|
||||
|
||||
// ensure cache is wiped on new flashblock
|
||||
let _ = self.pending.take_cached_reads();
|
||||
|
||||
@@ -2,6 +2,7 @@ use futures_util::stream::StreamExt;
|
||||
use reth_optimism_flashblocks::WsFlashBlockStream;
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "depends on external service availability"]
|
||||
async fn test_streaming_flashblocks_from_remote_source_is_successful() {
|
||||
let items = 3;
|
||||
let ws_url = "wss://sepolia.flashblocks.base.org/ws".parse().unwrap();
|
||||
|
||||
@@ -219,10 +219,9 @@ impl OpNode {
|
||||
/// use reth_optimism_chainspec::OpChainSpecBuilder;
|
||||
/// use reth_optimism_node::OpNode;
|
||||
/// use reth_provider::providers::{RocksDBProvider, StaticFileProvider};
|
||||
/// use std::sync::Arc;
|
||||
///
|
||||
/// let factory = OpNode::provider_factory_builder()
|
||||
/// .db(Arc::new(open_db_read_only("db", Default::default()).unwrap()))
|
||||
/// .db(open_db_read_only("db", Default::default()).unwrap())
|
||||
/// .chainspec(OpChainSpecBuilder::base_mainnet().build().into())
|
||||
/// .static_file(StaticFileProvider::read_only("db/static_files", false).unwrap())
|
||||
/// .rocksdb_provider(RocksDBProvider::builder("db/rocksdb").build().unwrap())
|
||||
|
||||
@@ -435,19 +435,11 @@ impl reth_codecs::Compact for OpTransactionSigned {
|
||||
|
||||
let tx_bits = if zstd_bit {
|
||||
let mut tmp = Vec::with_capacity(256);
|
||||
if cfg!(feature = "std") {
|
||||
reth_zstd_compressors::TRANSACTION_COMPRESSOR.with(|compressor| {
|
||||
let mut compressor = compressor.borrow_mut();
|
||||
let tx_bits = self.transaction.to_compact(&mut tmp);
|
||||
buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress"));
|
||||
tx_bits as u8
|
||||
})
|
||||
} else {
|
||||
let mut compressor = reth_zstd_compressors::create_tx_compressor();
|
||||
reth_zstd_compressors::with_tx_compressor(|compressor| {
|
||||
let tx_bits = self.transaction.to_compact(&mut tmp);
|
||||
buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress"));
|
||||
tx_bits as u8
|
||||
}
|
||||
})
|
||||
} else {
|
||||
self.transaction.to_compact(buf) as u8
|
||||
};
|
||||
@@ -469,29 +461,15 @@ impl reth_codecs::Compact for OpTransactionSigned {
|
||||
|
||||
let zstd_bit = bitflags >> 3;
|
||||
let (transaction, buf) = if zstd_bit != 0 {
|
||||
if cfg!(feature = "std") {
|
||||
reth_zstd_compressors::TRANSACTION_DECOMPRESSOR.with(|decompressor| {
|
||||
let mut decompressor = decompressor.borrow_mut();
|
||||
|
||||
// TODO: enforce that zstd is only present at a "top" level type
|
||||
let transaction_type = (bitflags & 0b110) >> 1;
|
||||
let (transaction, _) = OpTypedTransaction::from_compact(
|
||||
decompressor.decompress(buf),
|
||||
transaction_type,
|
||||
);
|
||||
|
||||
(transaction, buf)
|
||||
})
|
||||
} else {
|
||||
let mut decompressor = reth_zstd_compressors::create_tx_decompressor();
|
||||
reth_zstd_compressors::with_tx_decompressor(|decompressor| {
|
||||
// TODO: enforce that zstd is only present at a "top" level type
|
||||
let transaction_type = (bitflags & 0b110) >> 1;
|
||||
let (transaction, _) = OpTypedTransaction::from_compact(
|
||||
decompressor.decompress(buf),
|
||||
transaction_type,
|
||||
);
|
||||
|
||||
(transaction, buf)
|
||||
}
|
||||
})
|
||||
} else {
|
||||
let transaction_type = bitflags >> 1;
|
||||
OpTypedTransaction::from_compact(buf, transaction_type)
|
||||
|
||||
@@ -61,7 +61,7 @@ op-revm.workspace = true
|
||||
tokio.workspace = true
|
||||
futures.workspace = true
|
||||
tokio-stream.workspace = true
|
||||
reqwest = { workspace = true, features = ["rustls-tls-native-roots"] }
|
||||
reqwest.workspace = true
|
||||
async-trait.workspace = true
|
||||
tower.workspace = true
|
||||
|
||||
|
||||
@@ -188,6 +188,12 @@ pub mod serde_bincode_compat;
|
||||
pub mod size;
|
||||
pub use size::InMemorySize;
|
||||
|
||||
/// Rayon utilities
|
||||
#[cfg(feature = "rayon")]
|
||||
pub mod rayon;
|
||||
#[cfg(feature = "rayon")]
|
||||
pub use rayon::ParallelBridgeBuffered;
|
||||
|
||||
/// Node traits
|
||||
pub mod node;
|
||||
pub use node::{BlockTy, BodyTy, HeaderTy, NodePrimitives, ReceiptTy, TxTy};
|
||||
|
||||
32
crates/primitives-traits/src/rayon.rs
Normal file
32
crates/primitives-traits/src/rayon.rs
Normal file
@@ -0,0 +1,32 @@
|
||||
//! Rayon parallel iterator utilities.
|
||||
|
||||
use alloc::vec::Vec;
|
||||
use rayon::iter::IntoParallelIterator;
|
||||
|
||||
/// Extension trait for iterators to convert them to parallel iterators via collection.
|
||||
///
|
||||
/// This is an alternative to [`rayon::iter::ParallelBridge`] that first collects the iterator
|
||||
/// into a `Vec`, then calls [`IntoParallelIterator`] on it. This avoids the mutex contention
|
||||
/// that can occur with `par_bridge` when either the iterator's `next()` method is fast or the
|
||||
/// parallel tasks are fast, as `par_bridge` wraps the iterator in a mutex.
|
||||
///
|
||||
/// # When to use
|
||||
///
|
||||
/// Use `par_bridge_buffered` instead of `par_bridge` when:
|
||||
/// - The iterator produces items quickly
|
||||
/// - The parallel work per item is relatively light
|
||||
/// - The total number of items is known to be reasonable for memory
|
||||
///
|
||||
/// Stick with `par_bridge` when:
|
||||
/// - The iterator is slow (e.g., I/O bound) and you want to overlap iteration with processing
|
||||
/// - Memory is constrained and you cannot afford to collect all items upfront
|
||||
pub trait ParallelBridgeBuffered: Iterator<Item: Send> + Sized {
|
||||
/// Collects this iterator into a `Vec` and returns a parallel iterator over it.
|
||||
///
|
||||
/// See [this trait's documentation](ParallelBridgeBuffered) for more details.
|
||||
fn par_bridge_buffered(self) -> rayon::vec::IntoIter<Self::Item> {
|
||||
self.collect::<Vec<_>>().into_par_iter()
|
||||
}
|
||||
}
|
||||
|
||||
impl<I: Iterator<Item: Send>> ParallelBridgeBuffered for I {}
|
||||
@@ -42,11 +42,15 @@ rayon.workspace = true
|
||||
tokio.workspace = true
|
||||
rustc-hash.workspace = true
|
||||
|
||||
[features]
|
||||
rocksdb = ["reth-provider/rocksdb"]
|
||||
|
||||
[dev-dependencies]
|
||||
# reth
|
||||
reth-db = { workspace = true, features = ["test-utils"] }
|
||||
reth-stages = { workspace = true, features = ["test-utils"] }
|
||||
reth-stages = { workspace = true, features = ["test-utils", "rocksdb"] }
|
||||
reth-primitives-traits = { workspace = true, features = ["arbitrary"] }
|
||||
reth-storage-api.workspace = true
|
||||
reth-testing-utils.workspace = true
|
||||
reth-tracing.workspace = true
|
||||
|
||||
|
||||
@@ -7,10 +7,10 @@ use reth_primitives_traits::NodePrimitives;
|
||||
use reth_provider::{
|
||||
providers::StaticFileProvider, BlockReader, ChainStateBlockReader, DBProvider,
|
||||
DatabaseProviderFactory, NodePrimitivesProvider, PruneCheckpointReader, PruneCheckpointWriter,
|
||||
StageCheckpointReader, StaticFileProviderFactory, StorageSettingsCache,
|
||||
RocksDBProviderFactory, StageCheckpointReader, StaticFileProviderFactory,
|
||||
};
|
||||
use reth_prune_types::PruneModes;
|
||||
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader};
|
||||
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader, StorageSettingsCache};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::watch;
|
||||
|
||||
@@ -85,6 +85,7 @@ impl PrunerBuilder {
|
||||
+ StageCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ RocksDBProviderFactory
|
||||
+ StaticFileProviderFactory<
|
||||
Primitives: NodePrimitives<SignedTx: Value, Receipt: Value, BlockHeader: Value>,
|
||||
>,
|
||||
@@ -121,7 +122,8 @@ impl PrunerBuilder {
|
||||
+ StorageSettingsCache
|
||||
+ StageCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader,
|
||||
+ StorageChangeSetReader
|
||||
+ RocksDBProviderFactory,
|
||||
{
|
||||
let segments = SegmentSet::<Provider>::from_components(static_file_provider, self.segments);
|
||||
|
||||
|
||||
@@ -54,6 +54,38 @@ where
|
||||
})
|
||||
}
|
||||
|
||||
/// Deletes ALL static file jars for a given segment.
|
||||
///
|
||||
/// This is used for `PruneMode::Full` where all data should be removed, including the highest jar.
|
||||
/// Unlike [`prune_static_files`], this does not preserve the most recent jar.
|
||||
pub(crate) fn delete_static_files_segment<Provider>(
|
||||
provider: &Provider,
|
||||
input: PruneInput,
|
||||
segment: StaticFileSegment,
|
||||
) -> Result<SegmentOutput, PrunerError>
|
||||
where
|
||||
Provider: StaticFileProviderFactory,
|
||||
{
|
||||
let deleted_headers = provider.static_file_provider().delete_segment(segment)?;
|
||||
|
||||
if deleted_headers.is_empty() {
|
||||
return Ok(SegmentOutput::done())
|
||||
}
|
||||
|
||||
let tx_ranges = deleted_headers.iter().filter_map(|header| header.tx_range());
|
||||
|
||||
let pruned = tx_ranges.clone().map(|range| range.len()).sum::<u64>() as usize;
|
||||
|
||||
Ok(SegmentOutput {
|
||||
progress: PruneProgress::Finished,
|
||||
pruned,
|
||||
checkpoint: Some(SegmentOutputCheckpoint {
|
||||
block_number: Some(input.to_block),
|
||||
tx_number: tx_ranges.map(|range| range.end()).max(),
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
/// A segment represents a pruning of some portion of the data.
|
||||
///
|
||||
/// Segments are called from [`Pruner`](crate::Pruner) with the following lifecycle:
|
||||
|
||||
@@ -7,10 +7,11 @@ use reth_db_api::{table::Value, transaction::DbTxMut};
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_provider::{
|
||||
providers::StaticFileProvider, BlockReader, ChainStateBlockReader, DBProvider,
|
||||
PruneCheckpointReader, PruneCheckpointWriter, StaticFileProviderFactory, StorageSettingsCache,
|
||||
PruneCheckpointReader, PruneCheckpointWriter, RocksDBProviderFactory,
|
||||
StaticFileProviderFactory,
|
||||
};
|
||||
use reth_prune_types::PruneModes;
|
||||
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader};
|
||||
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader, StorageSettingsCache};
|
||||
|
||||
/// Collection of [`Segment`]. Thread-safe, allocated on the heap.
|
||||
#[derive(Debug)]
|
||||
@@ -55,7 +56,8 @@ where
|
||||
+ ChainStateBlockReader
|
||||
+ StorageSettingsCache
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader,
|
||||
+ StorageChangeSetReader
|
||||
+ RocksDBProviderFactory,
|
||||
{
|
||||
/// Creates a [`SegmentSet`] from an existing components, such as [`StaticFileProvider`] and
|
||||
/// [`PruneModes`].
|
||||
|
||||
@@ -10,20 +10,20 @@ use alloy_primitives::BlockNumber;
|
||||
use reth_db_api::{models::ShardedKey, tables, transaction::DbTxMut};
|
||||
use reth_provider::{
|
||||
changeset_walker::StaticFileAccountChangesetWalker, DBProvider, EitherWriter,
|
||||
StaticFileProviderFactory, StorageSettingsCache,
|
||||
RocksDBProviderFactory, StaticFileProviderFactory,
|
||||
};
|
||||
use reth_prune_types::{
|
||||
PruneMode, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint,
|
||||
};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::ChangeSetReader;
|
||||
use reth_storage_api::{ChangeSetReader, StorageSettingsCache};
|
||||
use rustc_hash::FxHashMap;
|
||||
use tracing::{instrument, trace};
|
||||
|
||||
/// Number of account history tables to prune in one step.
|
||||
///
|
||||
/// Account History consists of two tables: [`tables::AccountChangeSets`] and
|
||||
/// [`tables::AccountsHistory`]. We want to prune them to the same block number.
|
||||
/// Account History consists of two tables: [`tables::AccountChangeSets`] (either in database or
|
||||
/// static files) and [`tables::AccountsHistory`]. We want to prune them to the same block number.
|
||||
const ACCOUNT_HISTORY_TABLES_TO_PRUNE: usize = 2;
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -42,7 +42,8 @@ where
|
||||
Provider: DBProvider<Tx: DbTxMut>
|
||||
+ StaticFileProviderFactory
|
||||
+ StorageSettingsCache
|
||||
+ ChangeSetReader,
|
||||
+ ChangeSetReader
|
||||
+ RocksDBProviderFactory,
|
||||
{
|
||||
fn segment(&self) -> PruneSegment {
|
||||
PruneSegment::AccountHistory
|
||||
@@ -67,7 +68,13 @@ where
|
||||
};
|
||||
let range_end = *range.end();
|
||||
|
||||
// Check where account changesets are stored
|
||||
// Check where account history indices are stored
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().account_history_in_rocksdb {
|
||||
return self.prune_rocksdb(provider, input, range, range_end);
|
||||
}
|
||||
|
||||
// Check where account changesets are stored (MDBX path)
|
||||
if EitherWriter::account_changesets_destination(provider).is_static_file() {
|
||||
self.prune_static_files(provider, input, range, range_end)
|
||||
} else {
|
||||
@@ -94,6 +101,8 @@ impl AccountHistory {
|
||||
input.limiter
|
||||
};
|
||||
|
||||
// The limiter may already be exhausted from a previous segment in the same prune run.
|
||||
// Early exit avoids unnecessary iteration when no budget remains.
|
||||
if limiter.is_limit_reached() {
|
||||
return Ok(SegmentOutput::not_done(
|
||||
limiter.interrupt_reason(),
|
||||
@@ -101,11 +110,14 @@ impl AccountHistory {
|
||||
))
|
||||
}
|
||||
|
||||
// The size of this map it's limited by `prune_delete_limit * blocks_since_last_run /
|
||||
// ACCOUNT_HISTORY_TABLES_TO_PRUNE`, and with the current defaults it's usually `3500 * 5 /
|
||||
// 2`, so 8750 entries. Each entry is `160 bit + 64 bit`, so the total size should be up to
|
||||
// ~0.25MB + some hashmap overhead. `blocks_since_last_run` is additionally limited by the
|
||||
// `max_reorg_depth`, so no OOM is expected here.
|
||||
// Deleted account changeset keys (account addresses) with the highest block number deleted
|
||||
// for that key.
|
||||
//
|
||||
// The size of this map is limited by `prune_delete_limit * blocks_since_last_run /
|
||||
// ACCOUNT_HISTORY_TABLES_TO_PRUNE`, and with current default it's usually `3500 * 5
|
||||
// / 2`, so 8750 entries. Each entry is `160 bit + 64 bit`, so the total
|
||||
// size should be up to ~0.25MB + some hashmap overhead. `blocks_since_last_run` is
|
||||
// additionally limited by the `max_reorg_depth`, so no OOM is expected here.
|
||||
let mut highest_deleted_accounts = FxHashMap::default();
|
||||
let mut last_changeset_pruned_block = None;
|
||||
let mut pruned_changesets = 0;
|
||||
@@ -124,8 +136,8 @@ impl AccountHistory {
|
||||
limiter.increment_deleted_entries_count();
|
||||
}
|
||||
|
||||
// Delete static file jars below the pruned block
|
||||
if let Some(last_block) = last_changeset_pruned_block {
|
||||
// Delete static file jars only when fully processed
|
||||
if done && let Some(last_block) = last_changeset_pruned_block {
|
||||
provider
|
||||
.static_file_provider()
|
||||
.delete_segment_below_block(StaticFileSegment::AccountChangeSets, last_block + 1)?;
|
||||
@@ -210,6 +222,106 @@ impl AccountHistory {
|
||||
)
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Prunes account history when indices are stored in `RocksDB`.
|
||||
///
|
||||
/// Reads account changesets from static files and prunes the corresponding
|
||||
/// `RocksDB` history shards.
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
fn prune_rocksdb<Provider>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
input: PruneInput,
|
||||
range: std::ops::RangeInclusive<BlockNumber>,
|
||||
range_end: BlockNumber,
|
||||
) -> Result<SegmentOutput, PrunerError>
|
||||
where
|
||||
Provider: DBProvider + StaticFileProviderFactory + ChangeSetReader + RocksDBProviderFactory,
|
||||
{
|
||||
use reth_provider::PruneShardOutcome;
|
||||
|
||||
// Unlike MDBX path, we don't divide the limit by 2 because RocksDB path only prunes
|
||||
// history shards (no separate changeset table to delete from). The changesets are in
|
||||
// static files which are deleted separately.
|
||||
let mut limiter = input.limiter;
|
||||
|
||||
if limiter.is_limit_reached() {
|
||||
return Ok(SegmentOutput::not_done(
|
||||
limiter.interrupt_reason(),
|
||||
input.previous_checkpoint.map(SegmentOutputCheckpoint::from_prune_checkpoint),
|
||||
))
|
||||
}
|
||||
|
||||
let mut highest_deleted_accounts = FxHashMap::default();
|
||||
let mut last_changeset_pruned_block = None;
|
||||
let mut changesets_processed = 0usize;
|
||||
let mut done = true;
|
||||
|
||||
// Walk account changesets from static files using a streaming iterator.
|
||||
// For each changeset, track the highest block number seen for each address
|
||||
// to determine which history shard entries need pruning.
|
||||
let walker = StaticFileAccountChangesetWalker::new(provider, range);
|
||||
for result in walker {
|
||||
if limiter.is_limit_reached() {
|
||||
done = false;
|
||||
break;
|
||||
}
|
||||
let (block_number, changeset) = result?;
|
||||
highest_deleted_accounts.insert(changeset.address, block_number);
|
||||
last_changeset_pruned_block = Some(block_number);
|
||||
changesets_processed += 1;
|
||||
limiter.increment_deleted_entries_count();
|
||||
}
|
||||
trace!(target: "pruner", processed = %changesets_processed, %done, "Scanned account changesets from static files");
|
||||
|
||||
let last_changeset_pruned_block = last_changeset_pruned_block
|
||||
.map(|block_number| if done { block_number } else { block_number.saturating_sub(1) })
|
||||
.unwrap_or(range_end);
|
||||
|
||||
// Prune RocksDB history shards for affected accounts
|
||||
let mut deleted_shards = 0usize;
|
||||
let mut updated_shards = 0usize;
|
||||
|
||||
// Sort by address for better RocksDB cache locality
|
||||
let mut sorted_accounts: Vec<_> = highest_deleted_accounts.into_iter().collect();
|
||||
sorted_accounts.sort_unstable_by_key(|(addr, _)| *addr);
|
||||
|
||||
provider.with_rocksdb_batch(|mut batch| {
|
||||
for (address, highest_block) in &sorted_accounts {
|
||||
let prune_to = (*highest_block).min(last_changeset_pruned_block);
|
||||
match batch.prune_account_history_to(*address, prune_to)? {
|
||||
PruneShardOutcome::Deleted => deleted_shards += 1,
|
||||
PruneShardOutcome::Updated => updated_shards += 1,
|
||||
PruneShardOutcome::Unchanged => {}
|
||||
}
|
||||
}
|
||||
Ok(((), Some(batch.into_inner())))
|
||||
})?;
|
||||
trace!(target: "pruner", deleted = deleted_shards, updated = updated_shards, %done, "Pruned account history (RocksDB indices)");
|
||||
|
||||
// Delete static file jars only when fully processed. During provider.commit(), RocksDB
|
||||
// batch is committed before the MDBX checkpoint. If crash occurs after RocksDB commit
|
||||
// but before MDBX commit, on restart the pruner checkpoint indicates data needs
|
||||
// re-pruning, but the RocksDB shards are already pruned - this is safe because pruning
|
||||
// is idempotent (re-pruning already-pruned shards is a no-op).
|
||||
if done {
|
||||
provider.static_file_provider().delete_segment_below_block(
|
||||
StaticFileSegment::AccountChangeSets,
|
||||
last_changeset_pruned_block + 1,
|
||||
)?;
|
||||
}
|
||||
|
||||
let progress = limiter.progress(done);
|
||||
|
||||
Ok(SegmentOutput {
|
||||
progress,
|
||||
pruned: changesets_processed + deleted_shards + updated_shards,
|
||||
checkpoint: Some(SegmentOutputCheckpoint {
|
||||
block_number: Some(last_changeset_pruned_block),
|
||||
tx_number: None,
|
||||
}),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -539,4 +651,272 @@ mod tests {
|
||||
test_prune(998, 2, (PruneProgress::Finished, 1000));
|
||||
test_prune(1400, 3, (PruneProgress::Finished, 804));
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
#[test]
|
||||
fn prune_rocksdb_path() {
|
||||
use reth_db_api::models::ShardedKey;
|
||||
use reth_provider::{RocksDBProviderFactory, StaticFileProviderFactory};
|
||||
|
||||
let db = TestStageDB::default();
|
||||
let mut rng = generators::rng();
|
||||
|
||||
let blocks = random_block_range(
|
||||
&mut rng,
|
||||
0..=100,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() },
|
||||
);
|
||||
db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
|
||||
|
||||
let accounts = random_eoa_accounts(&mut rng, 2).into_iter().collect::<BTreeMap<_, _>>();
|
||||
|
||||
let (changesets, _) = random_changeset_range(
|
||||
&mut rng,
|
||||
blocks.iter(),
|
||||
accounts.into_iter().map(|(addr, acc)| (addr, (acc, Vec::new()))),
|
||||
0..0,
|
||||
0..0,
|
||||
);
|
||||
|
||||
db.insert_changesets_to_static_files(changesets.clone(), None)
|
||||
.expect("insert changesets to static files");
|
||||
|
||||
let mut account_blocks: BTreeMap<_, Vec<u64>> = BTreeMap::new();
|
||||
for (block, changeset) in changesets.iter().enumerate() {
|
||||
for (address, _, _) in changeset {
|
||||
account_blocks.entry(*address).or_default().push(block as u64);
|
||||
}
|
||||
}
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let mut batch = rocksdb.batch();
|
||||
for (address, block_numbers) in &account_blocks {
|
||||
let shard = BlockNumberList::new_pre_sorted(block_numbers.iter().copied());
|
||||
batch
|
||||
.put::<tables::AccountsHistory>(ShardedKey::new(*address, u64::MAX), &shard)
|
||||
.unwrap();
|
||||
}
|
||||
batch.commit().unwrap();
|
||||
|
||||
for (address, expected_blocks) in &account_blocks {
|
||||
let shards = rocksdb.account_history_shards(*address).unwrap();
|
||||
assert_eq!(shards.len(), 1);
|
||||
assert_eq!(shards[0].1.iter().collect::<Vec<_>>(), *expected_blocks);
|
||||
}
|
||||
|
||||
let to_block: BlockNumber = 50;
|
||||
let prune_mode = PruneMode::Before(to_block);
|
||||
let input =
|
||||
PruneInput { previous_checkpoint: None, to_block, limiter: PruneLimiter::default() };
|
||||
let segment = AccountHistory::new(prune_mode);
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::default()
|
||||
.with_account_changesets_in_static_files(true)
|
||||
.with_account_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
provider.commit().expect("commit");
|
||||
|
||||
assert_matches!(
|
||||
result,
|
||||
SegmentOutput { progress: PruneProgress::Finished, pruned, checkpoint: Some(_) }
|
||||
if pruned > 0
|
||||
);
|
||||
|
||||
for (address, original_blocks) in &account_blocks {
|
||||
let shards = rocksdb.account_history_shards(*address).unwrap();
|
||||
|
||||
let expected_blocks: Vec<u64> =
|
||||
original_blocks.iter().copied().filter(|b| *b > to_block).collect();
|
||||
|
||||
if expected_blocks.is_empty() {
|
||||
assert!(
|
||||
shards.is_empty(),
|
||||
"Expected no shards for address {address:?} after pruning"
|
||||
);
|
||||
} else {
|
||||
assert_eq!(shards.len(), 1, "Expected 1 shard for address {address:?}");
|
||||
assert_eq!(
|
||||
shards[0].1.iter().collect::<Vec<_>>(),
|
||||
expected_blocks,
|
||||
"Shard blocks mismatch for address {address:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let static_file_provider = db.factory.static_file_provider();
|
||||
let highest_block = static_file_provider.get_highest_static_file_block(
|
||||
reth_static_file_types::StaticFileSegment::AccountChangeSets,
|
||||
);
|
||||
if let Some(block) = highest_block {
|
||||
assert!(
|
||||
block > to_block,
|
||||
"Static files should only contain blocks above to_block ({to_block}), got {block}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Tests that when a limiter stops mid-block (with multiple changes for the same block),
|
||||
/// the checkpoint is set to `block_number - 1` to avoid dangling index entries.
|
||||
#[test]
|
||||
fn prune_partial_progress_mid_block() {
|
||||
use alloy_primitives::{Address, U256};
|
||||
use reth_primitives_traits::Account;
|
||||
use reth_testing_utils::generators::ChangeSet;
|
||||
|
||||
let db = TestStageDB::default();
|
||||
let mut rng = generators::rng();
|
||||
|
||||
// Create blocks 0..=10
|
||||
let blocks = random_block_range(
|
||||
&mut rng,
|
||||
0..=10,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() },
|
||||
);
|
||||
db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
|
||||
|
||||
// Create specific changesets where block 5 has 4 account changes
|
||||
let addr1 = Address::with_last_byte(1);
|
||||
let addr2 = Address::with_last_byte(2);
|
||||
let addr3 = Address::with_last_byte(3);
|
||||
let addr4 = Address::with_last_byte(4);
|
||||
let addr5 = Address::with_last_byte(5);
|
||||
|
||||
let account = Account { nonce: 1, balance: U256::from(100), bytecode_hash: None };
|
||||
|
||||
// Build changesets: blocks 0-4 have 1 change each, block 5 has 4 changes, block 6 has 1
|
||||
let changesets: Vec<ChangeSet> = vec![
|
||||
vec![(addr1, account, vec![])], // block 0
|
||||
vec![(addr1, account, vec![])], // block 1
|
||||
vec![(addr1, account, vec![])], // block 2
|
||||
vec![(addr1, account, vec![])], // block 3
|
||||
vec![(addr1, account, vec![])], // block 4
|
||||
// block 5: 4 different account changes (sorted by address for consistency)
|
||||
vec![
|
||||
(addr1, account, vec![]),
|
||||
(addr2, account, vec![]),
|
||||
(addr3, account, vec![]),
|
||||
(addr4, account, vec![]),
|
||||
],
|
||||
vec![(addr5, account, vec![])], // block 6
|
||||
];
|
||||
|
||||
db.insert_changesets(changesets.clone(), None).expect("insert changesets");
|
||||
db.insert_history(changesets.clone(), None).expect("insert history");
|
||||
|
||||
// Total changesets: 5 (blocks 0-4) + 4 (block 5) + 1 (block 6) = 10
|
||||
assert_eq!(
|
||||
db.table::<tables::AccountChangeSets>().unwrap().len(),
|
||||
changesets.iter().flatten().count()
|
||||
);
|
||||
|
||||
let prune_mode = PruneMode::Before(10);
|
||||
|
||||
// Set limiter to stop after 7 entries (mid-block 5: 5 from blocks 0-4, then 2 of 4 from
|
||||
// block 5). Due to ACCOUNT_HISTORY_TABLES_TO_PRUNE=2, actual limit is 7/2=3
|
||||
// changesets. So we'll process blocks 0, 1, 2 (3 changesets), stopping before block
|
||||
// 3. Actually, let's use a higher limit to reach block 5. With limit=14, we get 7
|
||||
// changeset slots. Blocks 0-4 use 5 slots, leaving 2 for block 5 (which has 4), so
|
||||
// we stop mid-block 5.
|
||||
let deleted_entries_limit = 14; // 14/2 = 7 changeset entries before limit
|
||||
let limiter = PruneLimiter::default().set_deleted_entries_limit(deleted_entries_limit);
|
||||
|
||||
let input = PruneInput { previous_checkpoint: None, to_block: 10, limiter };
|
||||
let segment = AccountHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default().with_account_changesets_in_static_files(false),
|
||||
);
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
|
||||
// Should report that there's more data
|
||||
assert!(!result.progress.is_finished(), "Expected HasMoreData since we stopped mid-block");
|
||||
|
||||
// Save checkpoint and commit
|
||||
segment
|
||||
.save_checkpoint(&provider, result.checkpoint.unwrap().as_prune_checkpoint(prune_mode))
|
||||
.unwrap();
|
||||
provider.commit().expect("commit");
|
||||
|
||||
// Verify checkpoint is set to block 4 (not 5), since block 5 is incomplete
|
||||
let checkpoint = db
|
||||
.factory
|
||||
.provider()
|
||||
.unwrap()
|
||||
.get_prune_checkpoint(PruneSegment::AccountHistory)
|
||||
.unwrap()
|
||||
.expect("checkpoint should exist");
|
||||
|
||||
assert_eq!(
|
||||
checkpoint.block_number,
|
||||
Some(4),
|
||||
"Checkpoint should be block 4 (block before incomplete block 5)"
|
||||
);
|
||||
|
||||
// Verify remaining changesets (block 5 and 6 should still have entries)
|
||||
let remaining_changesets = db.table::<tables::AccountChangeSets>().unwrap();
|
||||
// After pruning blocks 0-4, remaining should be block 5 (4 entries) + block 6 (1 entry) = 5
|
||||
// But since we stopped mid-block 5, some of block 5 might be pruned
|
||||
// However, checkpoint is 4, so on re-run we should re-process from block 5
|
||||
assert!(
|
||||
!remaining_changesets.is_empty(),
|
||||
"Should have remaining changesets for blocks 5-6"
|
||||
);
|
||||
|
||||
// Verify no dangling history indices for blocks that weren't fully pruned
|
||||
// The indices for block 5 should still reference blocks <= 5 appropriately
|
||||
let history = db.table::<tables::AccountsHistory>().unwrap();
|
||||
for (key, _blocks) in &history {
|
||||
// All blocks in the history should be > checkpoint block number
|
||||
// OR the shard's highest_block_number should be > checkpoint
|
||||
assert!(
|
||||
key.highest_block_number > 4,
|
||||
"Found stale history shard with highest_block_number {} <= checkpoint 4",
|
||||
key.highest_block_number
|
||||
);
|
||||
}
|
||||
|
||||
// Run prune again to complete - should finish processing block 5 and 6
|
||||
let input2 = PruneInput {
|
||||
previous_checkpoint: Some(checkpoint),
|
||||
to_block: 10,
|
||||
limiter: PruneLimiter::default().set_deleted_entries_limit(100), // high limit
|
||||
};
|
||||
|
||||
let provider2 = db.factory.database_provider_rw().unwrap();
|
||||
provider2.set_storage_settings_cache(
|
||||
StorageSettings::default().with_account_changesets_in_static_files(false),
|
||||
);
|
||||
let result2 = segment.prune(&provider2, input2).unwrap();
|
||||
|
||||
assert!(result2.progress.is_finished(), "Second run should complete");
|
||||
|
||||
segment
|
||||
.save_checkpoint(
|
||||
&provider2,
|
||||
result2.checkpoint.unwrap().as_prune_checkpoint(prune_mode),
|
||||
)
|
||||
.unwrap();
|
||||
provider2.commit().expect("commit");
|
||||
|
||||
// Verify final checkpoint
|
||||
let final_checkpoint = db
|
||||
.factory
|
||||
.provider()
|
||||
.unwrap()
|
||||
.get_prune_checkpoint(PruneSegment::AccountHistory)
|
||||
.unwrap()
|
||||
.expect("checkpoint should exist");
|
||||
|
||||
// Should now be at block 6 (the last block with changesets)
|
||||
assert_eq!(final_checkpoint.block_number, Some(6), "Final checkpoint should be at block 6");
|
||||
|
||||
// All changesets should be pruned
|
||||
let final_changesets = db.table::<tables::AccountChangeSets>().unwrap();
|
||||
assert!(final_changesets.is_empty(), "All changesets up to block 10 should be pruned");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,6 +49,16 @@ where
|
||||
fn prune(&self, provider: &Provider, input: PruneInput) -> Result<SegmentOutput, PrunerError> {
|
||||
if EitherWriterDestination::senders(provider).is_static_file() {
|
||||
debug!(target: "pruner", "Pruning transaction senders from static files.");
|
||||
|
||||
if self.mode.is_full() {
|
||||
debug!(target: "pruner", "PruneMode::Full: deleting all transaction senders static files.");
|
||||
return segments::delete_static_files_segment(
|
||||
provider,
|
||||
input,
|
||||
StaticFileSegment::TransactionSenders,
|
||||
)
|
||||
}
|
||||
|
||||
return segments::prune_static_files(
|
||||
provider,
|
||||
input,
|
||||
|
||||
@@ -12,7 +12,7 @@ use reth_db_api::{
|
||||
tables,
|
||||
transaction::DbTxMut,
|
||||
};
|
||||
use reth_provider::{DBProvider, EitherWriter, StaticFileProviderFactory};
|
||||
use reth_provider::{DBProvider, EitherWriter, RocksDBProviderFactory, StaticFileProviderFactory};
|
||||
use reth_prune_types::{
|
||||
PruneMode, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint,
|
||||
};
|
||||
@@ -43,7 +43,8 @@ where
|
||||
Provider: DBProvider<Tx: DbTxMut>
|
||||
+ StaticFileProviderFactory
|
||||
+ StorageChangeSetReader
|
||||
+ StorageSettingsCache,
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory,
|
||||
{
|
||||
fn segment(&self) -> PruneSegment {
|
||||
PruneSegment::StorageHistory
|
||||
@@ -68,6 +69,13 @@ where
|
||||
};
|
||||
let range_end = *range.end();
|
||||
|
||||
// Check where storage history indices are stored
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().storages_history_in_rocksdb {
|
||||
return self.prune_rocksdb(provider, input, range, range_end);
|
||||
}
|
||||
|
||||
// Check where storage changesets are stored (MDBX path)
|
||||
if EitherWriter::storage_changesets_destination(provider).is_static_file() {
|
||||
self.prune_static_files(provider, input, range, range_end)
|
||||
} else {
|
||||
@@ -94,6 +102,8 @@ impl StorageHistory {
|
||||
input.limiter
|
||||
};
|
||||
|
||||
// The limiter may already be exhausted from a previous segment in the same prune run.
|
||||
// Early exit avoids unnecessary iteration when no budget remains.
|
||||
if limiter.is_limit_reached() {
|
||||
return Ok(SegmentOutput::not_done(
|
||||
limiter.interrupt_reason(),
|
||||
@@ -126,8 +136,8 @@ impl StorageHistory {
|
||||
limiter.increment_deleted_entries_count();
|
||||
}
|
||||
|
||||
// Delete static file jars below the pruned block
|
||||
if let Some(last_block) = last_changeset_pruned_block {
|
||||
// Delete static file jars only when fully processed
|
||||
if done && let Some(last_block) = last_changeset_pruned_block {
|
||||
provider
|
||||
.static_file_provider()
|
||||
.delete_segment_below_block(StaticFileSegment::StorageChangeSets, last_block + 1)?;
|
||||
@@ -216,6 +226,107 @@ impl StorageHistory {
|
||||
)
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Prunes storage history when indices are stored in `RocksDB`.
|
||||
///
|
||||
/// Reads storage changesets from static files and prunes the corresponding
|
||||
/// `RocksDB` history shards.
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
fn prune_rocksdb<Provider>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
input: PruneInput,
|
||||
range: std::ops::RangeInclusive<BlockNumber>,
|
||||
range_end: BlockNumber,
|
||||
) -> Result<SegmentOutput, PrunerError>
|
||||
where
|
||||
Provider: DBProvider + StaticFileProviderFactory + RocksDBProviderFactory,
|
||||
{
|
||||
use reth_provider::PruneShardOutcome;
|
||||
|
||||
let mut limiter = input.limiter;
|
||||
|
||||
if limiter.is_limit_reached() {
|
||||
return Ok(SegmentOutput::not_done(
|
||||
limiter.interrupt_reason(),
|
||||
input.previous_checkpoint.map(SegmentOutputCheckpoint::from_prune_checkpoint),
|
||||
))
|
||||
}
|
||||
|
||||
let mut highest_deleted_storages: FxHashMap<_, _> = FxHashMap::default();
|
||||
let mut last_changeset_pruned_block = None;
|
||||
let mut changesets_processed = 0usize;
|
||||
let mut done = true;
|
||||
|
||||
// Walk storage changesets from static files using a streaming iterator.
|
||||
// For each changeset, track the highest block number seen for each (address, storage_key)
|
||||
// pair to determine which history shard entries need pruning.
|
||||
let walker = provider.static_file_provider().walk_storage_changeset_range(range);
|
||||
for result in walker {
|
||||
if limiter.is_limit_reached() {
|
||||
done = false;
|
||||
break;
|
||||
}
|
||||
let (block_address, entry) = result?;
|
||||
let block_number = block_address.block_number();
|
||||
let address = block_address.address();
|
||||
highest_deleted_storages.insert((address, entry.key), block_number);
|
||||
last_changeset_pruned_block = Some(block_number);
|
||||
changesets_processed += 1;
|
||||
limiter.increment_deleted_entries_count();
|
||||
}
|
||||
|
||||
trace!(target: "pruner", processed = %changesets_processed, %done, "Scanned storage changesets from static files");
|
||||
|
||||
let last_changeset_pruned_block = last_changeset_pruned_block
|
||||
.map(|block_number| if done { block_number } else { block_number.saturating_sub(1) })
|
||||
.unwrap_or(range_end);
|
||||
|
||||
// Prune RocksDB history shards for affected storage slots
|
||||
let mut deleted_shards = 0usize;
|
||||
let mut updated_shards = 0usize;
|
||||
|
||||
// Sort by (address, storage_key) for better RocksDB cache locality
|
||||
let mut sorted_storages: Vec<_> = highest_deleted_storages.into_iter().collect();
|
||||
sorted_storages.sort_unstable_by_key(|((addr, key), _)| (*addr, *key));
|
||||
|
||||
provider.with_rocksdb_batch(|mut batch| {
|
||||
for ((address, storage_key), highest_block) in &sorted_storages {
|
||||
let prune_to = (*highest_block).min(last_changeset_pruned_block);
|
||||
match batch.prune_storage_history_to(*address, *storage_key, prune_to)? {
|
||||
PruneShardOutcome::Deleted => deleted_shards += 1,
|
||||
PruneShardOutcome::Updated => updated_shards += 1,
|
||||
PruneShardOutcome::Unchanged => {}
|
||||
}
|
||||
}
|
||||
Ok(((), Some(batch.into_inner())))
|
||||
})?;
|
||||
|
||||
trace!(target: "pruner", deleted = deleted_shards, updated = updated_shards, %done, "Pruned storage history (RocksDB indices)");
|
||||
|
||||
// Delete static file jars only when fully processed. During provider.commit(), RocksDB
|
||||
// batch is committed before the MDBX checkpoint. If crash occurs after RocksDB commit
|
||||
// but before MDBX commit, on restart the pruner checkpoint indicates data needs
|
||||
// re-pruning, but the RocksDB shards are already pruned - this is safe because pruning
|
||||
// is idempotent (re-pruning already-pruned shards is a no-op).
|
||||
if done {
|
||||
provider.static_file_provider().delete_segment_below_block(
|
||||
StaticFileSegment::StorageChangeSets,
|
||||
last_changeset_pruned_block + 1,
|
||||
)?;
|
||||
}
|
||||
|
||||
let progress = limiter.progress(done);
|
||||
|
||||
Ok(SegmentOutput {
|
||||
progress,
|
||||
pruned: changesets_processed + deleted_shards + updated_shards,
|
||||
checkpoint: Some(SegmentOutputCheckpoint {
|
||||
block_number: Some(last_changeset_pruned_block),
|
||||
tx_number: None,
|
||||
}),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -553,4 +664,270 @@ mod tests {
|
||||
test_prune(998, 2, (PruneProgress::Finished, 500));
|
||||
test_prune(1200, 3, (PruneProgress::Finished, 202));
|
||||
}
|
||||
|
||||
/// Tests that when a limiter stops mid-block (with multiple storage changes for the same
|
||||
/// block), the checkpoint is set to `block_number - 1` to avoid dangling index entries.
|
||||
#[test]
|
||||
fn prune_partial_progress_mid_block() {
|
||||
use alloy_primitives::{Address, U256};
|
||||
use reth_primitives_traits::Account;
|
||||
use reth_testing_utils::generators::ChangeSet;
|
||||
|
||||
let db = TestStageDB::default();
|
||||
let mut rng = generators::rng();
|
||||
|
||||
// Create blocks 0..=10
|
||||
let blocks = random_block_range(
|
||||
&mut rng,
|
||||
0..=10,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() },
|
||||
);
|
||||
db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
|
||||
|
||||
// Create specific changesets where block 5 has 4 storage changes
|
||||
let addr1 = Address::with_last_byte(1);
|
||||
let addr2 = Address::with_last_byte(2);
|
||||
|
||||
let account = Account { nonce: 1, balance: U256::from(100), bytecode_hash: None };
|
||||
|
||||
// Create storage entries
|
||||
let storage_entry = |key: u8| reth_primitives_traits::StorageEntry {
|
||||
key: B256::with_last_byte(key),
|
||||
value: U256::from(100),
|
||||
};
|
||||
|
||||
// Build changesets: blocks 0-4 have 1 storage change each, block 5 has 4 changes, block 6
|
||||
// has 1. Entries within each account must be sorted by key.
|
||||
let changesets: Vec<ChangeSet> = vec![
|
||||
vec![(addr1, account, vec![storage_entry(1)])], // block 0
|
||||
vec![(addr1, account, vec![storage_entry(1)])], // block 1
|
||||
vec![(addr1, account, vec![storage_entry(1)])], // block 2
|
||||
vec![(addr1, account, vec![storage_entry(1)])], // block 3
|
||||
vec![(addr1, account, vec![storage_entry(1)])], // block 4
|
||||
// block 5: 4 different storage changes (2 addresses, each with 2 storage slots)
|
||||
// Sorted by address, then by storage key within each address
|
||||
vec![
|
||||
(addr1, account, vec![storage_entry(1), storage_entry(2)]),
|
||||
(addr2, account, vec![storage_entry(1), storage_entry(2)]),
|
||||
],
|
||||
vec![(addr1, account, vec![storage_entry(3)])], // block 6
|
||||
];
|
||||
|
||||
db.insert_changesets(changesets.clone(), None).expect("insert changesets");
|
||||
db.insert_history(changesets.clone(), None).expect("insert history");
|
||||
|
||||
// Total storage changesets
|
||||
let total_storage_entries: usize =
|
||||
changesets.iter().flat_map(|c| c.iter()).map(|(_, _, entries)| entries.len()).sum();
|
||||
assert_eq!(db.table::<tables::StorageChangeSets>().unwrap().len(), total_storage_entries);
|
||||
|
||||
let prune_mode = PruneMode::Before(10);
|
||||
|
||||
// Set limiter to stop mid-block 5
|
||||
// With STORAGE_HISTORY_TABLES_TO_PRUNE=2, limit=14 gives us 7 storage entries before limit
|
||||
// Blocks 0-4 use 5 slots, leaving 2 for block 5 (which has 4), so we stop mid-block 5
|
||||
let deleted_entries_limit = 14; // 14/2 = 7 storage entries before limit
|
||||
let limiter = PruneLimiter::default().set_deleted_entries_limit(deleted_entries_limit);
|
||||
|
||||
let input = PruneInput { previous_checkpoint: None, to_block: 10, limiter };
|
||||
let segment = StorageHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default().with_storage_changesets_in_static_files(false),
|
||||
);
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
|
||||
// Should report that there's more data
|
||||
assert!(!result.progress.is_finished(), "Expected HasMoreData since we stopped mid-block");
|
||||
|
||||
// Save checkpoint and commit
|
||||
segment
|
||||
.save_checkpoint(&provider, result.checkpoint.unwrap().as_prune_checkpoint(prune_mode))
|
||||
.unwrap();
|
||||
provider.commit().expect("commit");
|
||||
|
||||
// Verify checkpoint is set to block 4 (not 5), since block 5 is incomplete
|
||||
let checkpoint = db
|
||||
.factory
|
||||
.provider()
|
||||
.unwrap()
|
||||
.get_prune_checkpoint(PruneSegment::StorageHistory)
|
||||
.unwrap()
|
||||
.expect("checkpoint should exist");
|
||||
|
||||
assert_eq!(
|
||||
checkpoint.block_number,
|
||||
Some(4),
|
||||
"Checkpoint should be block 4 (block before incomplete block 5)"
|
||||
);
|
||||
|
||||
// Verify remaining changesets
|
||||
let remaining_changesets = db.table::<tables::StorageChangeSets>().unwrap();
|
||||
assert!(
|
||||
!remaining_changesets.is_empty(),
|
||||
"Should have remaining changesets for blocks 5-6"
|
||||
);
|
||||
|
||||
// Verify no dangling history indices for blocks that weren't fully pruned
|
||||
let history = db.table::<tables::StoragesHistory>().unwrap();
|
||||
for (key, _blocks) in &history {
|
||||
assert!(
|
||||
key.sharded_key.highest_block_number > 4,
|
||||
"Found stale history shard with highest_block_number {} <= checkpoint 4",
|
||||
key.sharded_key.highest_block_number
|
||||
);
|
||||
}
|
||||
|
||||
// Run prune again to complete - should finish processing block 5 and 6
|
||||
let input2 = PruneInput {
|
||||
previous_checkpoint: Some(checkpoint),
|
||||
to_block: 10,
|
||||
limiter: PruneLimiter::default().set_deleted_entries_limit(100), // high limit
|
||||
};
|
||||
|
||||
let provider2 = db.factory.database_provider_rw().unwrap();
|
||||
provider2.set_storage_settings_cache(
|
||||
StorageSettings::default().with_storage_changesets_in_static_files(false),
|
||||
);
|
||||
let result2 = segment.prune(&provider2, input2).unwrap();
|
||||
|
||||
assert!(result2.progress.is_finished(), "Second run should complete");
|
||||
|
||||
segment
|
||||
.save_checkpoint(
|
||||
&provider2,
|
||||
result2.checkpoint.unwrap().as_prune_checkpoint(prune_mode),
|
||||
)
|
||||
.unwrap();
|
||||
provider2.commit().expect("commit");
|
||||
|
||||
// Verify final checkpoint
|
||||
let final_checkpoint = db
|
||||
.factory
|
||||
.provider()
|
||||
.unwrap()
|
||||
.get_prune_checkpoint(PruneSegment::StorageHistory)
|
||||
.unwrap()
|
||||
.expect("checkpoint should exist");
|
||||
|
||||
// Should now be at block 6 (the last block with changesets)
|
||||
assert_eq!(final_checkpoint.block_number, Some(6), "Final checkpoint should be at block 6");
|
||||
|
||||
// All changesets should be pruned
|
||||
let final_changesets = db.table::<tables::StorageChangeSets>().unwrap();
|
||||
assert!(final_changesets.is_empty(), "All changesets up to block 10 should be pruned");
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
#[test]
|
||||
fn prune_rocksdb() {
|
||||
use reth_db_api::models::storage_sharded_key::StorageShardedKey;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use reth_storage_api::StorageSettings;
|
||||
|
||||
let db = TestStageDB::default();
|
||||
let mut rng = generators::rng();
|
||||
|
||||
let blocks = random_block_range(
|
||||
&mut rng,
|
||||
0..=100,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() },
|
||||
);
|
||||
db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
|
||||
|
||||
let accounts = random_eoa_accounts(&mut rng, 2).into_iter().collect::<BTreeMap<_, _>>();
|
||||
|
||||
let (changesets, _) = random_changeset_range(
|
||||
&mut rng,
|
||||
blocks.iter(),
|
||||
accounts.into_iter().map(|(addr, acc)| (addr, (acc, Vec::new()))),
|
||||
1..2,
|
||||
1..2,
|
||||
);
|
||||
|
||||
db.insert_changesets_to_static_files(changesets.clone(), None)
|
||||
.expect("insert changesets to static files");
|
||||
|
||||
let mut storage_indices: BTreeMap<(alloy_primitives::Address, B256), Vec<u64>> =
|
||||
BTreeMap::new();
|
||||
for (block, changeset) in changesets.iter().enumerate() {
|
||||
for (address, _, storage_entries) in changeset {
|
||||
for entry in storage_entries {
|
||||
storage_indices.entry((*address, entry.key)).or_default().push(block as u64);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let mut batch = rocksdb.batch();
|
||||
for ((address, storage_key), block_numbers) in &storage_indices {
|
||||
let shard = BlockNumberList::new_pre_sorted(block_numbers.clone());
|
||||
batch
|
||||
.put::<tables::StoragesHistory>(
|
||||
StorageShardedKey::last(*address, *storage_key),
|
||||
&shard,
|
||||
)
|
||||
.expect("insert storage history shard");
|
||||
}
|
||||
batch.commit().expect("commit rocksdb batch");
|
||||
}
|
||||
|
||||
{
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
for (address, storage_key) in storage_indices.keys() {
|
||||
let shards = rocksdb.storage_history_shards(*address, *storage_key).unwrap();
|
||||
assert!(!shards.is_empty(), "RocksDB should contain storage history before prune");
|
||||
}
|
||||
}
|
||||
|
||||
let to_block = 50u64;
|
||||
let prune_mode = PruneMode::Before(to_block);
|
||||
let input =
|
||||
PruneInput { previous_checkpoint: None, to_block, limiter: PruneLimiter::default() };
|
||||
let segment = StorageHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default()
|
||||
.with_storage_changesets_in_static_files(true)
|
||||
.with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
provider.commit().expect("commit");
|
||||
|
||||
assert_matches!(
|
||||
result,
|
||||
SegmentOutput { progress: PruneProgress::Finished, checkpoint: Some(_), .. }
|
||||
);
|
||||
|
||||
{
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
for ((address, storage_key), block_numbers) in &storage_indices {
|
||||
let shards = rocksdb.storage_history_shards(*address, *storage_key).unwrap();
|
||||
|
||||
let remaining_blocks: Vec<u64> =
|
||||
block_numbers.iter().copied().filter(|&b| b > to_block).collect();
|
||||
|
||||
if remaining_blocks.is_empty() {
|
||||
assert!(
|
||||
shards.is_empty(),
|
||||
"Shard for {:?}/{:?} should be deleted when all blocks pruned",
|
||||
address,
|
||||
storage_key
|
||||
);
|
||||
} else {
|
||||
assert!(!shards.is_empty(), "Shard should exist with remaining blocks");
|
||||
let actual_blocks: Vec<u64> =
|
||||
shards.iter().flat_map(|(_, list)| list.iter()).collect();
|
||||
assert_eq!(
|
||||
actual_blocks, remaining_blocks,
|
||||
"RocksDB shard should only contain blocks > {}",
|
||||
to_block
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,7 +54,7 @@ pub struct PruneModes {
|
||||
pub transaction_lookup: Option<PruneMode>,
|
||||
/// Receipts pruning configuration. This setting overrides `receipts_log_filter`
|
||||
/// and offers improved performance.
|
||||
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none",))]
|
||||
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none"))]
|
||||
pub receipts: Option<PruneMode>,
|
||||
/// Account History pruning configuration.
|
||||
#[cfg_attr(
|
||||
@@ -75,7 +75,7 @@ pub struct PruneModes {
|
||||
)]
|
||||
pub storage_history: Option<PruneMode>,
|
||||
/// Bodies History pruning configuration.
|
||||
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none",))]
|
||||
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none"))]
|
||||
pub bodies_history: Option<PruneMode>,
|
||||
/// Receipts pruning configuration by retaining only those receipts that contain logs emitted
|
||||
/// by the specified addresses, discarding others. This setting is overridden by `receipts`.
|
||||
@@ -112,7 +112,13 @@ impl PruneModes {
|
||||
///
|
||||
/// Returns `true` if any migration was performed.
|
||||
pub const fn migrate(&mut self) -> bool {
|
||||
false
|
||||
match &self.receipts {
|
||||
Some(PruneMode::Full | PruneMode::Distance(0..MINIMUM_DISTANCE)) => {
|
||||
self.receipts = Some(PruneMode::Distance(MINIMUM_DISTANCE));
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns an error if we can't unwind to the targeted block because the target block is
|
||||
|
||||
@@ -2,7 +2,7 @@ use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::{BlockId, BlockNumberOrTag};
|
||||
use alloy_genesis::ChainConfig;
|
||||
use alloy_json_rpc::RpcObject;
|
||||
use alloy_primitives::{Address, Bytes, B256};
|
||||
use alloy_primitives::{Address, Bytes, B256, U64};
|
||||
use alloy_rpc_types_debug::ExecutionWitness;
|
||||
use alloy_rpc_types_eth::{Bundle, StateContext};
|
||||
use alloy_rpc_types_trace::geth::{
|
||||
@@ -325,7 +325,7 @@ pub trait DebugApi<TxReq: RpcObject> {
|
||||
/// Sets the current head of the local chain by block number. Note, this is a destructive action
|
||||
/// and may severely damage your chain. Use with extreme caution.
|
||||
#[method(name = "setHead")]
|
||||
async fn debug_set_head(&self, number: u64) -> RpcResult<()>;
|
||||
async fn debug_set_head(&self, number: U64) -> RpcResult<()>;
|
||||
|
||||
/// Sets the rate of mutex profiling.
|
||||
#[method(name = "setMutexProfileFraction")]
|
||||
|
||||
@@ -107,6 +107,7 @@ impl RethRpcServerConfig for RpcServerArgs {
|
||||
.pending_block_kind(self.rpc_pending_block)
|
||||
.raw_tx_forwarder(self.rpc_forwarder.clone())
|
||||
.rpc_evm_memory_limit(self.rpc_evm_memory_limit)
|
||||
.force_blob_sidecar_upcasting(self.rpc_force_blob_sidecar_upcasting)
|
||||
}
|
||||
|
||||
fn flashbots_config(&self) -> ValidationApiConfig {
|
||||
|
||||
@@ -49,7 +49,7 @@ jsonrpsee-types.workspace = true
|
||||
futures.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-stream.workspace = true
|
||||
reqwest = { workspace = true, features = ["rustls-tls-native-roots"] }
|
||||
reqwest.workspace = true
|
||||
|
||||
# metrics
|
||||
metrics.workspace = true
|
||||
|
||||
@@ -107,6 +107,11 @@ pub struct EthConfig {
|
||||
pub send_raw_transaction_sync_timeout: Duration,
|
||||
/// Maximum memory the EVM can allocate per RPC request.
|
||||
pub rpc_evm_memory_limit: u64,
|
||||
/// Whether to force upcasting EIP-4844 blob sidecars to EIP-7594 format when Osaka is active.
|
||||
///
|
||||
/// This is disabled by default, allowing blob transactions with EIP-4844 sidecars to be
|
||||
/// submitted without automatic conversion.
|
||||
pub force_blob_sidecar_upcasting: bool,
|
||||
}
|
||||
|
||||
impl EthConfig {
|
||||
@@ -140,6 +145,7 @@ impl Default for EthConfig {
|
||||
raw_tx_forwarder: ForwardConfig::default(),
|
||||
send_raw_transaction_sync_timeout: RPC_DEFAULT_SEND_RAW_TX_SYNC_TIMEOUT_SECS,
|
||||
rpc_evm_memory_limit: (1 << 32) - 1,
|
||||
force_blob_sidecar_upcasting: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -242,6 +248,12 @@ impl EthConfig {
|
||||
self.rpc_evm_memory_limit = memory_limit;
|
||||
self
|
||||
}
|
||||
|
||||
/// Configures whether to force upcasting EIP-4844 blob sidecars to EIP-7594 format.
|
||||
pub const fn force_blob_sidecar_upcasting(mut self, force: bool) -> Self {
|
||||
self.force_blob_sidecar_upcasting = force;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Config for the filter
|
||||
|
||||
@@ -71,10 +71,6 @@ revm-primitives = { workspace = true, features = ["serde"] }
|
||||
|
||||
# rpc
|
||||
jsonrpsee.workspace = true
|
||||
http.workspace = true
|
||||
http-body.workspace = true
|
||||
hyper.workspace = true
|
||||
jsonwebtoken.workspace = true
|
||||
serde_json.workspace = true
|
||||
jsonrpsee-types.workspace = true
|
||||
|
||||
@@ -82,7 +78,6 @@ jsonrpsee-types.workspace = true
|
||||
async-trait.workspace = true
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tokio-stream.workspace = true
|
||||
tower.workspace = true
|
||||
pin-project.workspace = true
|
||||
parking_lot.workspace = true
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::{eip2718::Encodable2718, BlockId, BlockNumberOrTag};
|
||||
use alloy_evm::env::BlockEnvironment;
|
||||
use alloy_genesis::ChainConfig;
|
||||
use alloy_primitives::{hex::decode, uint, Address, Bytes, B256};
|
||||
use alloy_primitives::{hex::decode, uint, Address, Bytes, B256, U64};
|
||||
use alloy_rlp::{Decodable, Encodable};
|
||||
use alloy_rpc_types::BlockTransactionsKind;
|
||||
use alloy_rpc_types_debug::ExecutionWitness;
|
||||
@@ -998,7 +998,7 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn debug_set_head(&self, _number: u64) -> RpcResult<()> {
|
||||
async fn debug_set_head(&self, _number: U64) -> RpcResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -47,6 +47,7 @@ pub struct EthApiBuilder<N: RpcNodeCore, Rpc, NextEnv = ()> {
|
||||
raw_tx_forwarder: ForwardConfig,
|
||||
send_raw_transaction_sync_timeout: Duration,
|
||||
evm_memory_limit: u64,
|
||||
force_blob_sidecar_upcasting: bool,
|
||||
}
|
||||
|
||||
impl<Provider, Pool, Network, EvmConfig, ChainSpec>
|
||||
@@ -99,6 +100,7 @@ impl<N: RpcNodeCore, Rpc, NextEnv> EthApiBuilder<N, Rpc, NextEnv> {
|
||||
raw_tx_forwarder,
|
||||
send_raw_transaction_sync_timeout,
|
||||
evm_memory_limit,
|
||||
force_blob_sidecar_upcasting,
|
||||
} = self;
|
||||
EthApiBuilder {
|
||||
components,
|
||||
@@ -121,6 +123,7 @@ impl<N: RpcNodeCore, Rpc, NextEnv> EthApiBuilder<N, Rpc, NextEnv> {
|
||||
raw_tx_forwarder,
|
||||
send_raw_transaction_sync_timeout,
|
||||
evm_memory_limit,
|
||||
force_blob_sidecar_upcasting,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -154,6 +157,7 @@ where
|
||||
raw_tx_forwarder: ForwardConfig::default(),
|
||||
send_raw_transaction_sync_timeout: Duration::from_secs(30),
|
||||
evm_memory_limit: (1 << 32) - 1,
|
||||
force_blob_sidecar_upcasting: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -194,6 +198,7 @@ where
|
||||
raw_tx_forwarder,
|
||||
send_raw_transaction_sync_timeout,
|
||||
evm_memory_limit,
|
||||
force_blob_sidecar_upcasting,
|
||||
} = self;
|
||||
EthApiBuilder {
|
||||
components,
|
||||
@@ -216,6 +221,7 @@ where
|
||||
raw_tx_forwarder,
|
||||
send_raw_transaction_sync_timeout,
|
||||
evm_memory_limit,
|
||||
force_blob_sidecar_upcasting,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -245,6 +251,7 @@ where
|
||||
raw_tx_forwarder,
|
||||
send_raw_transaction_sync_timeout,
|
||||
evm_memory_limit,
|
||||
force_blob_sidecar_upcasting,
|
||||
} = self;
|
||||
EthApiBuilder {
|
||||
components,
|
||||
@@ -267,6 +274,7 @@ where
|
||||
raw_tx_forwarder,
|
||||
send_raw_transaction_sync_timeout,
|
||||
evm_memory_limit,
|
||||
force_blob_sidecar_upcasting,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -502,6 +510,7 @@ where
|
||||
raw_tx_forwarder,
|
||||
send_raw_transaction_sync_timeout,
|
||||
evm_memory_limit,
|
||||
force_blob_sidecar_upcasting,
|
||||
} = self;
|
||||
|
||||
let provider = components.provider().clone();
|
||||
@@ -544,6 +553,7 @@ where
|
||||
raw_tx_forwarder.forwarder_client(),
|
||||
send_raw_transaction_sync_timeout,
|
||||
evm_memory_limit,
|
||||
force_blob_sidecar_upcasting,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -574,4 +584,10 @@ where
|
||||
self.evm_memory_limit = memory_limit;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets whether to force upcasting EIP-4844 blob sidecars to EIP-7594 format.
|
||||
pub const fn force_blob_sidecar_upcasting(mut self, force: bool) -> Self {
|
||||
self.force_blob_sidecar_upcasting = force;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
@@ -157,6 +157,7 @@ where
|
||||
raw_tx_forwarder: ForwardConfig,
|
||||
send_raw_transaction_sync_timeout: Duration,
|
||||
evm_memory_limit: u64,
|
||||
force_blob_sidecar_upcasting: bool,
|
||||
) -> Self {
|
||||
let inner = EthApiInner::new(
|
||||
components,
|
||||
@@ -177,6 +178,7 @@ where
|
||||
raw_tx_forwarder.forwarder_client(),
|
||||
send_raw_transaction_sync_timeout,
|
||||
evm_memory_limit,
|
||||
force_blob_sidecar_upcasting,
|
||||
);
|
||||
|
||||
Self { inner: Arc::new(inner) }
|
||||
@@ -333,6 +335,9 @@ pub struct EthApiInner<N: RpcNodeCore, Rpc: RpcConvert> {
|
||||
|
||||
/// Maximum memory the EVM can allocate per RPC request.
|
||||
evm_memory_limit: u64,
|
||||
|
||||
/// Whether to force upcasting EIP-4844 blob sidecars to EIP-7594 format when Osaka is active.
|
||||
force_blob_sidecar_upcasting: bool,
|
||||
}
|
||||
|
||||
impl<N, Rpc> EthApiInner<N, Rpc>
|
||||
@@ -361,6 +366,7 @@ where
|
||||
raw_tx_forwarder: Option<RpcClient>,
|
||||
send_raw_transaction_sync_timeout: Duration,
|
||||
evm_memory_limit: u64,
|
||||
force_blob_sidecar_upcasting: bool,
|
||||
) -> Self {
|
||||
let signers = parking_lot::RwLock::new(Default::default());
|
||||
// get the block number of the latest block
|
||||
@@ -405,6 +411,7 @@ where
|
||||
send_raw_transaction_sync_timeout,
|
||||
blob_sidecar_converter: BlobSidecarConverter::new(),
|
||||
evm_memory_limit,
|
||||
force_blob_sidecar_upcasting,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -596,6 +603,12 @@ where
|
||||
pub const fn blocking_io_request_semaphore(&self) -> &Arc<Semaphore> {
|
||||
&self.blocking_io_request_semaphore
|
||||
}
|
||||
|
||||
/// Returns whether to force upcasting EIP-4844 blob sidecars to EIP-7594 format.
|
||||
#[inline]
|
||||
pub const fn force_blob_sidecar_upcasting(&self) -> bool {
|
||||
self.force_blob_sidecar_upcasting
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -44,9 +44,9 @@ where
|
||||
let mut pool_transaction =
|
||||
<Self::Pool as TransactionPool>::Transaction::from_pooled(recovered);
|
||||
|
||||
// TODO: remove this after Osaka transition
|
||||
// Convert legacy blob sidecars to EIP-7594 format
|
||||
if pool_transaction.is_eip4844() {
|
||||
// Optionally convert legacy blob sidecars to EIP-7594 format when Osaka is active
|
||||
// This is opt-in via --rpc.force-blob-sidecar-upcasting
|
||||
if self.inner.force_blob_sidecar_upcasting() && pool_transaction.is_eip4844() {
|
||||
let EthBlobTransactionSidecar::Present(sidecar) = pool_transaction.take_blob() else {
|
||||
return Err(EthApiError::PoolError(RpcPoolError::Eip4844(
|
||||
Eip4844PoolTransactionError::MissingEip4844BlobSidecar,
|
||||
|
||||
@@ -25,13 +25,6 @@
|
||||
#![cfg_attr(docsrs, feature(doc_cfg))]
|
||||
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
|
||||
|
||||
use http as _;
|
||||
use http_body as _;
|
||||
use hyper as _;
|
||||
use jsonwebtoken as _;
|
||||
use pin_project as _;
|
||||
use tower as _;
|
||||
|
||||
mod admin;
|
||||
mod aliases;
|
||||
mod debug;
|
||||
|
||||
@@ -61,7 +61,7 @@ rayon.workspace = true
|
||||
num-traits.workspace = true
|
||||
tempfile = { workspace = true, optional = true }
|
||||
bincode.workspace = true
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls-tls-native-roots", "blocking"] }
|
||||
reqwest.workspace = true
|
||||
eyre.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -2,7 +2,7 @@ use reth_db_api::{table::Value, transaction::DbTxMut};
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_provider::{
|
||||
BlockReader, ChainStateBlockReader, DBProvider, PruneCheckpointReader, PruneCheckpointWriter,
|
||||
StageCheckpointReader, StaticFileProviderFactory, StorageSettingsCache,
|
||||
RocksDBProviderFactory, StageCheckpointReader, StaticFileProviderFactory,
|
||||
};
|
||||
use reth_prune::{
|
||||
PruneMode, PruneModes, PruneSegment, PrunerBuilder, SegmentOutput, SegmentOutputCheckpoint,
|
||||
@@ -10,7 +10,7 @@ use reth_prune::{
|
||||
use reth_stages_api::{
|
||||
ExecInput, ExecOutput, Stage, StageCheckpoint, StageError, StageId, UnwindInput, UnwindOutput,
|
||||
};
|
||||
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader};
|
||||
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader, StorageSettingsCache};
|
||||
use tracing::info;
|
||||
|
||||
/// The prune stage that runs the pruner with the provided prune modes.
|
||||
@@ -49,7 +49,8 @@ where
|
||||
Primitives: NodePrimitives<SignedTx: Value, Receipt: Value, BlockHeader: Value>,
|
||||
> + StorageSettingsCache
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader,
|
||||
+ StorageChangeSetReader
|
||||
+ RocksDBProviderFactory,
|
||||
{
|
||||
fn id(&self) -> StageId {
|
||||
StageId::Prune
|
||||
@@ -156,7 +157,8 @@ where
|
||||
Primitives: NodePrimitives<SignedTx: Value, Receipt: Value, BlockHeader: Value>,
|
||||
> + StorageSettingsCache
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader,
|
||||
+ StorageChangeSetReader
|
||||
+ RocksDBProviderFactory,
|
||||
{
|
||||
fn id(&self) -> StageId {
|
||||
StageId::PruneSenderRecovery
|
||||
|
||||
@@ -232,11 +232,8 @@ where
|
||||
let mut writer = EitherWriter::new_transaction_hash_numbers(provider, rocksdb_batch)?;
|
||||
|
||||
let static_file_provider = provider.static_file_provider();
|
||||
let rev_walker = provider
|
||||
.block_body_indices_range(range.clone())?
|
||||
.into_iter()
|
||||
.zip(range.collect::<Vec<_>>())
|
||||
.rev();
|
||||
let rev_walker =
|
||||
provider.block_body_indices_range(range.clone())?.into_iter().rev().zip(range.rev());
|
||||
|
||||
for (body, number) in rev_walker {
|
||||
if number <= unwind_to {
|
||||
|
||||
@@ -38,4 +38,3 @@ assert_matches.workspace = true
|
||||
tempfile.workspace = true
|
||||
|
||||
[features]
|
||||
edge = ["reth-stages/edge"]
|
||||
|
||||
@@ -13,30 +13,28 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
alloy-primitives.workspace = true
|
||||
tracing = { workspace = true, optional = true }
|
||||
|
||||
clap = { workspace = true, features = ["derive"], optional = true }
|
||||
fixed-map.workspace = true
|
||||
derive_more.workspace = true
|
||||
serde = { workspace = true, features = ["alloc", "derive"] }
|
||||
reth-stages-types.workspace = true
|
||||
strum = { workspace = true, features = ["derive"] }
|
||||
|
||||
[dev-dependencies]
|
||||
reth-nippy-jar.workspace = true
|
||||
serde_json.workspace = true
|
||||
insta.workspace = true
|
||||
tempfile.workspace = true
|
||||
|
||||
[features]
|
||||
default = ["std"]
|
||||
std = [
|
||||
"alloy-primitives/std",
|
||||
"derive_more/std",
|
||||
"reth-stages-types/std",
|
||||
"serde/std",
|
||||
"strum/std",
|
||||
"serde_json/std",
|
||||
"fixed-map/std",
|
||||
"dep:tracing",
|
||||
"tracing?/std",
|
||||
]
|
||||
clap = ["dep:clap"]
|
||||
|
||||
@@ -1,471 +0,0 @@
|
||||
//! Changeset offset sidecar file I/O.
|
||||
//!
|
||||
//! Provides append-only writing and O(1) random-access reading for changeset offsets.
|
||||
//! The file format is fixed-width 16-byte records: `[offset: u64 LE][num_changes: u64 LE]`.
|
||||
|
||||
use crate::ChangesetOffset;
|
||||
use std::{
|
||||
fs::{File, OpenOptions},
|
||||
io::{self, Read, Seek, SeekFrom, Write},
|
||||
path::Path,
|
||||
};
|
||||
|
||||
/// Writer for appending changeset offsets to a sidecar file.
|
||||
#[derive(Debug)]
|
||||
pub struct ChangesetOffsetWriter {
|
||||
file: File,
|
||||
/// Number of records written.
|
||||
records_written: u64,
|
||||
}
|
||||
|
||||
impl ChangesetOffsetWriter {
|
||||
/// Record size in bytes.
|
||||
const RECORD_SIZE: usize = 16;
|
||||
|
||||
/// Opens or creates the changeset offset file for appending.
|
||||
///
|
||||
/// The file is healed to match `committed_len` (from the segment header):
|
||||
/// - Partial records (from crash mid-write) are truncated to record boundary
|
||||
/// - Extra complete records (from crash after sidecar sync but before header commit)
|
||||
/// are truncated to match the committed length
|
||||
/// - If the file has fewer records than committed, returns an error (data corruption)
|
||||
///
|
||||
/// This mirrors NippyJar's healing behavior where config/header is the commit boundary.
|
||||
pub fn new(path: impl AsRef<Path>, committed_len: u64) -> io::Result<Self> {
|
||||
let file = OpenOptions::new()
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(path.as_ref())?;
|
||||
|
||||
let file_len = file.metadata()?.len();
|
||||
let remainder = file_len % Self::RECORD_SIZE as u64;
|
||||
|
||||
// First, truncate any partial record from crash mid-write
|
||||
let aligned_len = if remainder != 0 {
|
||||
let truncated_len = file_len - remainder;
|
||||
#[cfg(feature = "std")]
|
||||
tracing::warn!(
|
||||
target: "reth::static_file",
|
||||
path = %path.as_ref().display(),
|
||||
original_len = file_len,
|
||||
truncated_len,
|
||||
"Truncating partial changeset offset record"
|
||||
);
|
||||
file.set_len(truncated_len)?;
|
||||
file.sync_all()?; // Sync required for crash safety
|
||||
truncated_len
|
||||
} else {
|
||||
file_len
|
||||
};
|
||||
|
||||
let records_in_file = aligned_len / Self::RECORD_SIZE as u64;
|
||||
|
||||
// Heal sidecar to match committed header length
|
||||
match records_in_file.cmp(&committed_len) {
|
||||
std::cmp::Ordering::Greater => {
|
||||
// Sidecar has uncommitted records from a crash - truncate them
|
||||
let target_len = committed_len * Self::RECORD_SIZE as u64;
|
||||
#[cfg(feature = "std")]
|
||||
tracing::warn!(
|
||||
target: "reth::static_file",
|
||||
path = %path.as_ref().display(),
|
||||
sidecar_records = records_in_file,
|
||||
committed_len,
|
||||
"Truncating uncommitted changeset offset records after crash recovery"
|
||||
);
|
||||
file.set_len(target_len)?;
|
||||
file.sync_all()?; // Sync required for crash safety
|
||||
}
|
||||
std::cmp::Ordering::Less => {
|
||||
// Unlikely: sidecar is shorter than header claims - data corruption or
|
||||
// incomplete prune
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!(
|
||||
"Changeset offset sidecar has {} records but header expects {}: {}",
|
||||
records_in_file,
|
||||
committed_len,
|
||||
path.as_ref().display()
|
||||
),
|
||||
));
|
||||
}
|
||||
std::cmp::Ordering::Equal => {}
|
||||
}
|
||||
|
||||
let records_written = committed_len;
|
||||
let file = OpenOptions::new().create(true).append(true).open(path)?;
|
||||
|
||||
Ok(Self { file, records_written })
|
||||
}
|
||||
|
||||
/// Appends a single changeset offset record.
|
||||
pub fn append(&mut self, offset: &ChangesetOffset) -> io::Result<()> {
|
||||
let mut buf = [0u8; Self::RECORD_SIZE];
|
||||
buf[..8].copy_from_slice(&offset.offset().to_le_bytes());
|
||||
buf[8..].copy_from_slice(&offset.num_changes().to_le_bytes());
|
||||
self.file.write_all(&buf)?;
|
||||
self.records_written += 1;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Appends multiple changeset offset records.
|
||||
pub fn append_many(&mut self, offsets: &[ChangesetOffset]) -> io::Result<()> {
|
||||
for offset in offsets {
|
||||
self.append(offset)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Syncs all data to disk. Must be called before committing the header.
|
||||
pub fn sync(&mut self) -> io::Result<()> {
|
||||
self.file.sync_all()
|
||||
}
|
||||
|
||||
/// Truncates the file to contain exactly `len` records and syncs to disk.
|
||||
/// Used after prune operations to reclaim space.
|
||||
///
|
||||
/// The sync is required for crash safety - without it, a crash could
|
||||
/// resurrect the old file length.
|
||||
pub fn truncate(&mut self, len: u64) -> io::Result<()> {
|
||||
self.file.set_len(len * Self::RECORD_SIZE as u64)?;
|
||||
self.file.sync_all()?;
|
||||
self.records_written = len;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns the number of records in the file.
|
||||
pub const fn len(&self) -> u64 {
|
||||
self.records_written
|
||||
}
|
||||
|
||||
/// Returns true if the file is empty.
|
||||
pub const fn is_empty(&self) -> bool {
|
||||
self.records_written == 0
|
||||
}
|
||||
}
|
||||
|
||||
/// Reader for changeset offsets with O(1) random access.
|
||||
#[derive(Debug)]
|
||||
pub struct ChangesetOffsetReader {
|
||||
file: File,
|
||||
/// Cached file length in records.
|
||||
len: u64,
|
||||
}
|
||||
|
||||
impl ChangesetOffsetReader {
|
||||
/// Record size in bytes.
|
||||
const RECORD_SIZE: usize = 16;
|
||||
|
||||
/// Opens the changeset offset file for reading.
|
||||
pub fn new(path: impl AsRef<Path>) -> io::Result<Self> {
|
||||
let file = File::open(path)?;
|
||||
let len = file.metadata()?.len() / Self::RECORD_SIZE as u64;
|
||||
Ok(Self { file, len })
|
||||
}
|
||||
|
||||
/// Opens with an explicit length (from header metadata).
|
||||
/// Any records beyond `len` are ignored.
|
||||
///
|
||||
/// Returns an error if the file has fewer records than `len` (data corruption).
|
||||
pub fn with_len(path: impl AsRef<Path>, len: u64) -> io::Result<Self> {
|
||||
let file = File::open(&path)?;
|
||||
let file_len = file.metadata()?.len();
|
||||
let records_in_file = file_len / Self::RECORD_SIZE as u64;
|
||||
|
||||
if records_in_file < len {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!(
|
||||
"Changeset offset sidecar has {} records but expected at least {}: {}",
|
||||
records_in_file,
|
||||
len,
|
||||
path.as_ref().display()
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(Self { file, len })
|
||||
}
|
||||
|
||||
/// Reads a single changeset offset by block index.
|
||||
/// Returns None if index is out of bounds.
|
||||
pub fn get(&mut self, block_index: u64) -> io::Result<Option<ChangesetOffset>> {
|
||||
if block_index >= self.len {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let byte_pos = block_index * Self::RECORD_SIZE as u64;
|
||||
self.file.seek(SeekFrom::Start(byte_pos))?;
|
||||
|
||||
let mut buf = [0u8; Self::RECORD_SIZE];
|
||||
self.file.read_exact(&mut buf)?;
|
||||
|
||||
let offset = u64::from_le_bytes(buf[..8].try_into().unwrap());
|
||||
let num_changes = u64::from_le_bytes(buf[8..].try_into().unwrap());
|
||||
|
||||
Ok(Some(ChangesetOffset::new(offset, num_changes)))
|
||||
}
|
||||
|
||||
/// Reads a range of changeset offsets.
|
||||
pub fn get_range(&mut self, start: u64, end: u64) -> io::Result<Vec<ChangesetOffset>> {
|
||||
let end = end.min(self.len);
|
||||
if start >= end {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let count = (end - start) as usize;
|
||||
let byte_pos = start * Self::RECORD_SIZE as u64;
|
||||
self.file.seek(SeekFrom::Start(byte_pos))?;
|
||||
|
||||
let mut result = Vec::with_capacity(count);
|
||||
let mut buf = [0u8; Self::RECORD_SIZE];
|
||||
|
||||
for _ in 0..count {
|
||||
self.file.read_exact(&mut buf)?;
|
||||
let offset = u64::from_le_bytes(buf[..8].try_into().unwrap());
|
||||
let num_changes = u64::from_le_bytes(buf[8..].try_into().unwrap());
|
||||
result.push(ChangesetOffset::new(offset, num_changes));
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Returns the number of valid records.
|
||||
pub const fn len(&self) -> u64 {
|
||||
self.len
|
||||
}
|
||||
|
||||
/// Returns true if there are no records.
|
||||
pub const fn is_empty(&self) -> bool {
|
||||
self.len == 0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn test_write_and_read() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.csoff");
|
||||
|
||||
// Write (new file, committed_len=0)
|
||||
{
|
||||
let mut writer = ChangesetOffsetWriter::new(&path, 0).unwrap();
|
||||
writer.append(&ChangesetOffset::new(0, 5)).unwrap();
|
||||
writer.append(&ChangesetOffset::new(5, 3)).unwrap();
|
||||
writer.append(&ChangesetOffset::new(8, 10)).unwrap();
|
||||
writer.sync().unwrap();
|
||||
assert_eq!(writer.len(), 3);
|
||||
}
|
||||
|
||||
// Read
|
||||
{
|
||||
let mut reader = ChangesetOffsetReader::new(&path).unwrap();
|
||||
assert_eq!(reader.len(), 3);
|
||||
|
||||
let entry = reader.get(0).unwrap().unwrap();
|
||||
assert_eq!(entry.offset(), 0);
|
||||
assert_eq!(entry.num_changes(), 5);
|
||||
|
||||
let entry = reader.get(1).unwrap().unwrap();
|
||||
assert_eq!(entry.offset(), 5);
|
||||
assert_eq!(entry.num_changes(), 3);
|
||||
|
||||
let entry = reader.get(2).unwrap().unwrap();
|
||||
assert_eq!(entry.offset(), 8);
|
||||
assert_eq!(entry.num_changes(), 10);
|
||||
|
||||
assert!(reader.get(3).unwrap().is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_truncate() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.csoff");
|
||||
|
||||
let mut writer = ChangesetOffsetWriter::new(&path, 0).unwrap();
|
||||
writer.append(&ChangesetOffset::new(0, 1)).unwrap();
|
||||
writer.append(&ChangesetOffset::new(1, 2)).unwrap();
|
||||
writer.append(&ChangesetOffset::new(3, 3)).unwrap();
|
||||
writer.sync().unwrap();
|
||||
|
||||
writer.truncate(2).unwrap();
|
||||
assert_eq!(writer.len(), 2);
|
||||
|
||||
let mut reader = ChangesetOffsetReader::new(&path).unwrap();
|
||||
assert_eq!(reader.len(), 2);
|
||||
assert!(reader.get(2).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_partial_record_recovery() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.csoff");
|
||||
|
||||
// Write 1 full record (16 bytes) + 8 trailing bytes (partial record)
|
||||
{
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
// Full record: offset=100, num_changes=5
|
||||
file.write_all(&100u64.to_le_bytes()).unwrap();
|
||||
file.write_all(&5u64.to_le_bytes()).unwrap();
|
||||
// Partial record: only 8 bytes (incomplete)
|
||||
file.write_all(&200u64.to_le_bytes()).unwrap();
|
||||
file.sync_all().unwrap();
|
||||
}
|
||||
|
||||
// Verify file has 24 bytes before opening with writer
|
||||
assert_eq!(std::fs::metadata(&path).unwrap().len(), 24);
|
||||
|
||||
// Open with writer, committed_len=1 (header committed 1 record)
|
||||
// Should truncate partial record and match committed length
|
||||
let writer = ChangesetOffsetWriter::new(&path, 1).unwrap();
|
||||
assert_eq!(writer.len(), 1);
|
||||
|
||||
// Verify file was truncated to 16 bytes
|
||||
assert_eq!(std::fs::metadata(&path).unwrap().len(), 16);
|
||||
|
||||
// Verify the complete record is readable
|
||||
let mut reader = ChangesetOffsetReader::new(&path).unwrap();
|
||||
assert_eq!(reader.len(), 1);
|
||||
let entry = reader.get(0).unwrap().unwrap();
|
||||
assert_eq!(entry.offset(), 100);
|
||||
assert_eq!(entry.num_changes(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_len_bounds_reads() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.csoff");
|
||||
|
||||
// Write 3 records
|
||||
{
|
||||
let mut writer = ChangesetOffsetWriter::new(&path, 0).unwrap();
|
||||
writer.append(&ChangesetOffset::new(0, 10)).unwrap();
|
||||
writer.append(&ChangesetOffset::new(10, 20)).unwrap();
|
||||
writer.append(&ChangesetOffset::new(30, 30)).unwrap();
|
||||
writer.sync().unwrap();
|
||||
assert_eq!(writer.len(), 3);
|
||||
}
|
||||
|
||||
// Open with explicit len=2, ignoring the 3rd record
|
||||
let mut reader = ChangesetOffsetReader::with_len(&path, 2).unwrap();
|
||||
assert_eq!(reader.len(), 2);
|
||||
|
||||
// First two records should be readable
|
||||
let entry0 = reader.get(0).unwrap().unwrap();
|
||||
assert_eq!(entry0.offset(), 0);
|
||||
assert_eq!(entry0.num_changes(), 10);
|
||||
|
||||
let entry1 = reader.get(1).unwrap().unwrap();
|
||||
assert_eq!(entry1.offset(), 10);
|
||||
assert_eq!(entry1.num_changes(), 20);
|
||||
|
||||
// Third record should be out of bounds (due to len=2)
|
||||
assert!(reader.get(2).unwrap().is_none());
|
||||
|
||||
// get_range should also respect the len bound
|
||||
let range = reader.get_range(0, 5).unwrap();
|
||||
assert_eq!(range.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_truncate_uncommitted_records_on_open() {
|
||||
// Simulates crash recovery where sidecar has more records than committed header length.
|
||||
// ChangesetOffsetWriter::new() should automatically truncate to committed_len.
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.csoff");
|
||||
|
||||
// Simulate: wrote 3 records, synced sidecar, but header only committed len=2
|
||||
{
|
||||
let mut writer = ChangesetOffsetWriter::new(&path, 0).unwrap();
|
||||
writer.append(&ChangesetOffset::new(0, 5)).unwrap();
|
||||
writer.append(&ChangesetOffset::new(5, 10)).unwrap();
|
||||
writer.append(&ChangesetOffset::new(15, 7)).unwrap(); // uncommitted
|
||||
writer.sync().unwrap();
|
||||
assert_eq!(writer.len(), 3);
|
||||
}
|
||||
|
||||
// On "restart", new() heals by truncating to committed length
|
||||
let committed_len = 2u64;
|
||||
{
|
||||
let writer = ChangesetOffsetWriter::new(&path, committed_len).unwrap();
|
||||
assert_eq!(writer.len(), 2); // Healed to committed length
|
||||
}
|
||||
|
||||
// Verify file is now correct length and new appends go to the right place
|
||||
{
|
||||
let mut writer = ChangesetOffsetWriter::new(&path, 2).unwrap();
|
||||
assert_eq!(writer.len(), 2);
|
||||
|
||||
// Append a new record - should be at index 2, not index 3
|
||||
writer.append(&ChangesetOffset::new(15, 20)).unwrap();
|
||||
writer.sync().unwrap();
|
||||
assert_eq!(writer.len(), 3);
|
||||
}
|
||||
|
||||
// Verify the records are correct
|
||||
{
|
||||
let mut reader = ChangesetOffsetReader::new(&path).unwrap();
|
||||
assert_eq!(reader.len(), 3);
|
||||
|
||||
let entry0 = reader.get(0).unwrap().unwrap();
|
||||
assert_eq!(entry0.offset(), 0);
|
||||
assert_eq!(entry0.num_changes(), 5);
|
||||
|
||||
let entry1 = reader.get(1).unwrap().unwrap();
|
||||
assert_eq!(entry1.offset(), 5);
|
||||
assert_eq!(entry1.num_changes(), 10);
|
||||
|
||||
// This should be the NEW record, not the old uncommitted one
|
||||
let entry2 = reader.get(2).unwrap().unwrap();
|
||||
assert_eq!(entry2.offset(), 15);
|
||||
assert_eq!(entry2.num_changes(), 20); // Not 7 from the old uncommitted record
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sidecar_shorter_than_committed_errors() {
|
||||
// If sidecar has fewer records than committed, it's data corruption - should error.
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.csoff");
|
||||
|
||||
// Write 1 record
|
||||
{
|
||||
let mut writer = ChangesetOffsetWriter::new(&path, 0).unwrap();
|
||||
writer.append(&ChangesetOffset::new(0, 5)).unwrap();
|
||||
writer.sync().unwrap();
|
||||
}
|
||||
|
||||
// Try to open with committed_len=3 (header claims more than file has)
|
||||
let result = ChangesetOffsetWriter::new(&path, 3);
|
||||
assert!(result.is_err());
|
||||
let err = result.unwrap_err();
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reader_with_len_shorter_than_file_errors() {
|
||||
// If header claims more records than file has, with_len should error (data corruption).
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.csoff");
|
||||
|
||||
// Write 1 record
|
||||
{
|
||||
let mut writer = ChangesetOffsetWriter::new(&path, 0).unwrap();
|
||||
writer.append(&ChangesetOffset::new(0, 5)).unwrap();
|
||||
writer.sync().unwrap();
|
||||
}
|
||||
|
||||
// Try to open reader with len=3 (header claims more than file has)
|
||||
let result = ChangesetOffsetReader::with_len(&path, 3);
|
||||
assert!(result.is_err());
|
||||
let err = result.unwrap_err();
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
}
|
||||
}
|
||||
@@ -15,18 +15,11 @@ mod compression;
|
||||
mod event;
|
||||
mod segment;
|
||||
|
||||
#[cfg(feature = "std")]
|
||||
mod changeset_offsets;
|
||||
#[cfg(feature = "std")]
|
||||
pub use changeset_offsets::{ChangesetOffsetReader, ChangesetOffsetWriter};
|
||||
|
||||
use alloy_primitives::BlockNumber;
|
||||
pub use compression::Compression;
|
||||
use core::ops::RangeInclusive;
|
||||
pub use event::StaticFileProducerEvent;
|
||||
pub use segment::{
|
||||
ChangesetOffset, SegmentConfig, SegmentHeader, SegmentRangeInclusive, StaticFileSegment,
|
||||
};
|
||||
pub use segment::{SegmentConfig, SegmentHeader, SegmentRangeInclusive, StaticFileSegment};
|
||||
|
||||
/// Map keyed by [`StaticFileSegment`].
|
||||
pub type StaticFileMap<T> = alloc::boxed::Box<fixed_map::Map<StaticFileSegment, T>>;
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
use crate::{find_fixed_range, BlockNumber, Compression};
|
||||
use alloc::{format, string::String};
|
||||
use alloc::{format, string::String, vec::Vec};
|
||||
use alloy_primitives::TxNumber;
|
||||
use core::{
|
||||
ops::{Range, RangeInclusive},
|
||||
str::FromStr,
|
||||
};
|
||||
use reth_stages_types::StageId;
|
||||
use serde::{de::Visitor, ser::SerializeStruct, Deserialize, Deserializer, Serialize, Serializer};
|
||||
use strum::{EnumIs, EnumString};
|
||||
|
||||
@@ -198,6 +199,18 @@ impl StaticFileSegment {
|
||||
pub const fn is_block_or_change_based(&self) -> bool {
|
||||
self.is_block_based() || self.is_change_based()
|
||||
}
|
||||
|
||||
/// Maps this segment to the [`StageId`] responsible for it.
|
||||
pub const fn to_stage_id(&self) -> StageId {
|
||||
match self {
|
||||
Self::Headers => StageId::Headers,
|
||||
Self::Transactions => StageId::Bodies,
|
||||
Self::Receipts | Self::AccountChangeSets | Self::StorageChangeSets => {
|
||||
StageId::Execution
|
||||
}
|
||||
Self::TransactionSenders => StageId::SenderRecovery,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A changeset offset, also with the number of elements in the offset for convenience
|
||||
@@ -211,11 +224,6 @@ pub struct ChangesetOffset {
|
||||
}
|
||||
|
||||
impl ChangesetOffset {
|
||||
/// Creates a new changeset offset.
|
||||
pub const fn new(offset: u64, num_changes: u64) -> Self {
|
||||
Self { offset, num_changes }
|
||||
}
|
||||
|
||||
/// Returns the start offset for the row for this block
|
||||
pub const fn offset(&self) -> u64 {
|
||||
self.offset
|
||||
@@ -230,11 +238,6 @@ impl ChangesetOffset {
|
||||
pub const fn changeset_range(&self) -> Range<u64> {
|
||||
self.offset..(self.offset + self.num_changes)
|
||||
}
|
||||
|
||||
/// Increments the number of changes by 1.
|
||||
pub const fn increment_num_changes(&mut self) {
|
||||
self.num_changes += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// A segment header that contains information common to all segments. Used for storage.
|
||||
@@ -252,7 +255,7 @@ pub struct SegmentHeader {
|
||||
/// Segment type
|
||||
segment: StaticFileSegment,
|
||||
/// List of offsets, for where each block's changeset starts.
|
||||
changeset_offsets_len: u64,
|
||||
changeset_offsets: Option<Vec<ChangesetOffset>>,
|
||||
}
|
||||
|
||||
struct SegmentHeaderVisitor;
|
||||
@@ -281,18 +284,21 @@ impl<'de> Visitor<'de> for SegmentHeaderVisitor {
|
||||
let segment: StaticFileSegment =
|
||||
seq.next_element()?.ok_or_else(|| serde::de::Error::invalid_length(3, &self))?;
|
||||
|
||||
let changeset_offsets_len = if segment.is_change_based() {
|
||||
// Try to read the 5th field (changeset_offsets_len)
|
||||
match seq.next_element::<u64>()? {
|
||||
Some(len) => len,
|
||||
let changeset_offsets = if segment.is_change_based() {
|
||||
// Try to read the 5th field (changeset_offsets)
|
||||
// If it doesn't exist (old format), this will return None
|
||||
match seq.next_element()? {
|
||||
Some(Some(offsets)) => Some(offsets),
|
||||
// Changesets should have offsets
|
||||
Some(None) => None,
|
||||
None => {
|
||||
return Err(serde::de::Error::custom(
|
||||
"changeset_offsets_len should exist for changeset static files",
|
||||
"changeset_offsets should exist for static files",
|
||||
))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
0
|
||||
None
|
||||
};
|
||||
|
||||
Ok(SegmentHeader {
|
||||
@@ -300,7 +306,7 @@ impl<'de> Visitor<'de> for SegmentHeaderVisitor {
|
||||
block_range,
|
||||
tx_range,
|
||||
segment,
|
||||
changeset_offsets_len,
|
||||
changeset_offsets,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -335,7 +341,7 @@ impl Serialize for SegmentHeader {
|
||||
state.serialize_field("segment", &self.segment)?;
|
||||
|
||||
if self.segment.is_change_based() {
|
||||
state.serialize_field("changeset_offsets", &self.changeset_offsets_len)?;
|
||||
state.serialize_field("changeset_offsets", &self.changeset_offsets)?;
|
||||
}
|
||||
|
||||
state.end()
|
||||
@@ -350,7 +356,7 @@ impl SegmentHeader {
|
||||
tx_range: Option<SegmentRangeInclusive>,
|
||||
segment: StaticFileSegment,
|
||||
) -> Self {
|
||||
Self { expected_block_range, block_range, tx_range, segment, changeset_offsets_len: 0 }
|
||||
Self { expected_block_range, block_range, tx_range, segment, changeset_offsets: None }
|
||||
}
|
||||
|
||||
/// Returns the static file segment kind.
|
||||
@@ -373,20 +379,9 @@ impl SegmentHeader {
|
||||
self.tx_range
|
||||
}
|
||||
|
||||
/// Returns the number of changeset offset entries.
|
||||
/// The actual offsets are stored in the .csoff sidecar file.
|
||||
pub const fn changeset_offsets_len(&self) -> u64 {
|
||||
self.changeset_offsets_len
|
||||
}
|
||||
|
||||
/// Sets the changeset offsets length.
|
||||
pub const fn set_changeset_offsets_len(&mut self, len: u64) {
|
||||
self.changeset_offsets_len = len;
|
||||
}
|
||||
|
||||
/// Increments the changeset offsets length by 1.
|
||||
pub const fn increment_changeset_offsets_len(&mut self) {
|
||||
self.changeset_offsets_len += 1;
|
||||
/// Returns the changeset offsets.
|
||||
pub const fn changeset_offsets(&self) -> Option<&Vec<ChangesetOffset>> {
|
||||
self.changeset_offsets.as_ref()
|
||||
}
|
||||
|
||||
/// The expected block start of the segment.
|
||||
@@ -444,7 +439,7 @@ impl SegmentHeader {
|
||||
}
|
||||
|
||||
/// Increments block end range depending on segment
|
||||
pub const fn increment_block(&mut self) -> BlockNumber {
|
||||
pub fn increment_block(&mut self) -> BlockNumber {
|
||||
let block_num = if let Some(block_range) = &mut self.block_range {
|
||||
block_range.end += 1;
|
||||
block_range.end
|
||||
@@ -456,10 +451,20 @@ impl SegmentHeader {
|
||||
self.expected_block_start()
|
||||
};
|
||||
|
||||
// For changeset segments, increment the offsets length.
|
||||
// The actual offset entry is written to the sidecar file by the caller.
|
||||
// For changeset segments, initialize an offset entry for the new block
|
||||
if self.segment.is_change_based() {
|
||||
self.changeset_offsets_len += 1;
|
||||
let offsets = self.changeset_offsets.get_or_insert_default();
|
||||
// Calculate the offset for the new block
|
||||
let new_offset = if let Some(last_offset) = offsets.last() {
|
||||
// The new block starts after the last block's changes
|
||||
last_offset.offset + last_offset.num_changes
|
||||
} else {
|
||||
// First block starts at offset 0
|
||||
0
|
||||
};
|
||||
|
||||
// Add a new offset entry with 0 changes initially
|
||||
offsets.push(ChangesetOffset { offset: new_offset, num_changes: 0 });
|
||||
}
|
||||
|
||||
block_num
|
||||
@@ -476,11 +481,23 @@ impl SegmentHeader {
|
||||
}
|
||||
}
|
||||
|
||||
/// Increments the latest block's number of changes.
|
||||
pub fn increment_block_changes(&mut self) {
|
||||
debug_assert!(self.segment().is_change_based());
|
||||
if self.segment.is_change_based() {
|
||||
let offsets = self.changeset_offsets.get_or_insert_with(Default::default);
|
||||
if let Some(last_offset) = offsets.last_mut() {
|
||||
last_offset.num_changes += 1;
|
||||
} else {
|
||||
// If offsets is empty, we are adding the first change for a block
|
||||
// The offset for the first block is 0
|
||||
offsets.push(ChangesetOffset { offset: 0, num_changes: 1 });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Removes `num` elements from end of tx or block range.
|
||||
///
|
||||
/// For changeset segments, also decrements the changeset offsets length.
|
||||
/// The caller must truncate the sidecar file accordingly.
|
||||
pub const fn prune(&mut self, num: u64) {
|
||||
pub fn prune(&mut self, num: u64) {
|
||||
// Changesets also contain a block range, but are not strictly block-based
|
||||
if self.segment.is_block_or_change_based() {
|
||||
if let Some(range) = &mut self.block_range {
|
||||
@@ -488,18 +505,26 @@ impl SegmentHeader {
|
||||
self.block_range = None;
|
||||
// Clear all changeset offsets if we're clearing all blocks
|
||||
if self.segment.is_change_based() {
|
||||
self.changeset_offsets_len = 0;
|
||||
self.changeset_offsets = None;
|
||||
}
|
||||
} else {
|
||||
let old_end = range.end;
|
||||
range.end = range.end.saturating_sub(num);
|
||||
|
||||
// Update changeset offsets length for changeset segments
|
||||
if self.segment.is_change_based() {
|
||||
// Update changeset offsets for account changesets
|
||||
if self.segment.is_change_based() &&
|
||||
let Some(offsets) = &mut self.changeset_offsets
|
||||
{
|
||||
// Calculate how many blocks we're removing
|
||||
let blocks_to_remove = old_end - range.end;
|
||||
self.changeset_offsets_len =
|
||||
self.changeset_offsets_len.saturating_sub(blocks_to_remove);
|
||||
// Remove the last `blocks_to_remove` entries from offsets
|
||||
let new_len = offsets.len().saturating_sub(blocks_to_remove as usize);
|
||||
offsets.truncate(new_len);
|
||||
|
||||
// If we removed all offsets, set to None
|
||||
if offsets.is_empty() {
|
||||
self.changeset_offsets = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -522,24 +547,28 @@ impl SegmentHeader {
|
||||
}
|
||||
}
|
||||
|
||||
/// Synchronizes changeset offsets length with the current block range for changeset segments.
|
||||
/// Synchronizes changeset offsets with the current block range for account changeset segments.
|
||||
///
|
||||
/// This should be called after modifying the block range when dealing with changeset segments
|
||||
/// to ensure the offsets length matches the block range size.
|
||||
/// The caller must also truncate the sidecar file accordingly.
|
||||
pub const fn sync_changeset_offsets(&mut self) {
|
||||
/// to ensure the offsets vector matches the block range size.
|
||||
pub fn sync_changeset_offsets(&mut self) {
|
||||
if !self.segment.is_change_based() {
|
||||
return;
|
||||
}
|
||||
|
||||
if let Some(block_range) = &self.block_range {
|
||||
let expected_len = block_range.end - block_range.start + 1;
|
||||
if self.changeset_offsets_len > expected_len {
|
||||
self.changeset_offsets_len = expected_len;
|
||||
if let Some(offsets) = &mut self.changeset_offsets {
|
||||
let expected_len = (block_range.end - block_range.start + 1) as usize;
|
||||
if offsets.len() > expected_len {
|
||||
offsets.truncate(expected_len);
|
||||
if offsets.is_empty() {
|
||||
self.changeset_offsets = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// No block range means no offsets
|
||||
self.changeset_offsets_len = 0;
|
||||
self.changeset_offsets = None;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -565,22 +594,21 @@ impl SegmentHeader {
|
||||
self.tx_start()
|
||||
}
|
||||
|
||||
/// Returns the index into the sidecar file for a given block's changeset offset.
|
||||
/// Returns the `ChangesetOffset` corresponding for the given block, if it's in the block
|
||||
/// range.
|
||||
///
|
||||
/// Returns `None` if the block is not in the block range.
|
||||
/// The caller must read the actual offset from the sidecar file at this index.
|
||||
pub fn changeset_offset_index(&self, block: BlockNumber) -> Option<u64> {
|
||||
/// If it is not in the block range or the changeset list in the header does not contain a
|
||||
/// value for the block, this returns `None`.
|
||||
pub fn changeset_offset(&self, block: BlockNumber) -> Option<&ChangesetOffset> {
|
||||
let block_range = self.block_range()?;
|
||||
if !block_range.contains(block) {
|
||||
return None;
|
||||
return None
|
||||
}
|
||||
|
||||
let index = block - block_range.start();
|
||||
if index >= self.changeset_offsets_len {
|
||||
return None;
|
||||
}
|
||||
let offsets = self.changeset_offsets.as_ref()?;
|
||||
let index = (block - block_range.start()) as usize;
|
||||
|
||||
Some(index)
|
||||
offsets.get(index)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -739,42 +767,42 @@ mod tests {
|
||||
block_range: Some(SegmentRangeInclusive::new(0, 100)),
|
||||
tx_range: None,
|
||||
segment: StaticFileSegment::Headers,
|
||||
changeset_offsets_len: 0,
|
||||
changeset_offsets: None,
|
||||
},
|
||||
SegmentHeader {
|
||||
expected_block_range: SegmentRangeInclusive::new(0, 200),
|
||||
block_range: None,
|
||||
tx_range: Some(SegmentRangeInclusive::new(0, 300)),
|
||||
segment: StaticFileSegment::Transactions,
|
||||
changeset_offsets_len: 0,
|
||||
changeset_offsets: None,
|
||||
},
|
||||
SegmentHeader {
|
||||
expected_block_range: SegmentRangeInclusive::new(0, 200),
|
||||
block_range: Some(SegmentRangeInclusive::new(0, 100)),
|
||||
tx_range: Some(SegmentRangeInclusive::new(0, 300)),
|
||||
segment: StaticFileSegment::Receipts,
|
||||
changeset_offsets_len: 0,
|
||||
changeset_offsets: None,
|
||||
},
|
||||
SegmentHeader {
|
||||
expected_block_range: SegmentRangeInclusive::new(0, 200),
|
||||
block_range: Some(SegmentRangeInclusive::new(0, 100)),
|
||||
tx_range: Some(SegmentRangeInclusive::new(0, 300)),
|
||||
segment: StaticFileSegment::TransactionSenders,
|
||||
changeset_offsets_len: 0,
|
||||
changeset_offsets: None,
|
||||
},
|
||||
SegmentHeader {
|
||||
expected_block_range: SegmentRangeInclusive::new(0, 200),
|
||||
block_range: Some(SegmentRangeInclusive::new(0, 100)),
|
||||
tx_range: Some(SegmentRangeInclusive::new(0, 300)),
|
||||
segment: StaticFileSegment::AccountChangeSets,
|
||||
changeset_offsets_len: 100,
|
||||
changeset_offsets: Some(vec![ChangesetOffset { offset: 1, num_changes: 1 }; 100]),
|
||||
},
|
||||
SegmentHeader {
|
||||
expected_block_range: SegmentRangeInclusive::new(0, 200),
|
||||
block_range: Some(SegmentRangeInclusive::new(0, 100)),
|
||||
tx_range: None,
|
||||
segment: StaticFileSegment::StorageChangeSets,
|
||||
changeset_offsets_len: 100,
|
||||
changeset_offsets: Some(vec![ChangesetOffset { offset: 1, num_changes: 1 }; 100]),
|
||||
},
|
||||
];
|
||||
// Check that we test all segments
|
||||
|
||||
@@ -2,4 +2,4 @@
|
||||
source: crates/static-file/types/src/segment.rs
|
||||
expression: "Bytes::from(serialized)"
|
||||
---
|
||||
0x01000000000000000000000000000000c80000000000000001000000000000000064000000000000000100000000000000002c0100000000000004000000640000000000000001000000000000000000000000000000000000000000000000
|
||||
0x01000000000000000000000000000000c80000000000000001000000000000000064000000000000000100000000000000002c01000000000000040000000164000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000000000000000000000000000000000000
|
||||
|
||||
@@ -2,4 +2,4 @@
|
||||
source: crates/static-file/types/src/segment.rs
|
||||
expression: "Bytes::from(serialized)"
|
||||
---
|
||||
0x01000000000000000000000000000000c80000000000000001000000000000000064000000000000000005000000640000000000000001000000000000000000000000000000000000000000000000
|
||||
0x01000000000000000000000000000000c800000000000000010000000000000000640000000000000000050000000164000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000000000000000000000000000000000000
|
||||
|
||||
@@ -133,8 +133,7 @@ fn generate_from_compact(
|
||||
let decompressor = zstd.decompressor;
|
||||
quote! {
|
||||
if flags.__zstd() != 0 {
|
||||
#decompressor.with(|decompressor| {
|
||||
let decompressor = &mut decompressor.borrow_mut();
|
||||
#decompressor(|decompressor| {
|
||||
let decompressed = decompressor.decompress(buf);
|
||||
let mut original_buf = buf;
|
||||
|
||||
@@ -203,9 +202,7 @@ fn generate_to_compact(
|
||||
let compressor = zstd.compressor;
|
||||
lines.push(quote! {
|
||||
if zstd {
|
||||
#compressor.with(|compressor| {
|
||||
let mut compressor = compressor.borrow_mut();
|
||||
|
||||
#compressor(|compressor| {
|
||||
let compressed = compressor.compress(&buffer).expect("Failed to compress.");
|
||||
buf.put(compressed.as_slice());
|
||||
});
|
||||
|
||||
@@ -10,8 +10,8 @@ use reth_codecs_derive::CompactZstd;
|
||||
#[derive(CompactZstd)]
|
||||
#[reth_codecs(crate = "crate")]
|
||||
#[reth_zstd(
|
||||
compressor = reth_zstd_compressors::RECEIPT_COMPRESSOR,
|
||||
decompressor = reth_zstd_compressors::RECEIPT_DECOMPRESSOR
|
||||
compressor = reth_zstd_compressors::with_receipt_compressor,
|
||||
decompressor = reth_zstd_compressors::with_receipt_decompressor
|
||||
)]
|
||||
struct CompactOpReceipt<'a> {
|
||||
tx_type: OpTxType,
|
||||
|
||||
@@ -149,20 +149,9 @@ impl<T: Envelope + ToTxCompact + Transaction + Send + Sync> CompactEnvelope for
|
||||
self.to_tx_compact(&mut tx_buf);
|
||||
|
||||
buf.put_slice(
|
||||
&{
|
||||
#[cfg(feature = "std")]
|
||||
{
|
||||
reth_zstd_compressors::TRANSACTION_COMPRESSOR.with(|compressor| {
|
||||
let mut compressor = compressor.borrow_mut();
|
||||
compressor.compress(&tx_buf)
|
||||
})
|
||||
}
|
||||
#[cfg(not(feature = "std"))]
|
||||
{
|
||||
let mut compressor = reth_zstd_compressors::create_tx_compressor();
|
||||
compressor.compress(&tx_buf)
|
||||
}
|
||||
}
|
||||
&reth_zstd_compressors::with_tx_compressor(|compressor| {
|
||||
compressor.compress(&tx_buf)
|
||||
})
|
||||
.expect("Failed to compress"),
|
||||
);
|
||||
tx_bits
|
||||
@@ -188,27 +177,12 @@ impl<T: Envelope + ToTxCompact + Transaction + Send + Sync> CompactEnvelope for
|
||||
let (signature, buf) = Signature::from_compact(buf, sig_bit);
|
||||
|
||||
let (transaction, buf) = if zstd_bit != 0 {
|
||||
#[cfg(feature = "std")]
|
||||
{
|
||||
reth_zstd_compressors::TRANSACTION_DECOMPRESSOR.with(|decompressor| {
|
||||
let mut decompressor = decompressor.borrow_mut();
|
||||
let decompressed = decompressor.decompress(buf);
|
||||
|
||||
let (tx_type, tx_buf) = T::TxType::from_compact(decompressed, tx_bits);
|
||||
let (tx, _) = Self::from_tx_compact(tx_buf, tx_type, signature);
|
||||
|
||||
(tx, buf)
|
||||
})
|
||||
}
|
||||
#[cfg(not(feature = "std"))]
|
||||
{
|
||||
let mut decompressor = reth_zstd_compressors::create_tx_decompressor();
|
||||
reth_zstd_compressors::with_tx_decompressor(|decompressor| {
|
||||
let decompressed = decompressor.decompress(buf);
|
||||
let (tx_type, tx_buf) = T::TxType::from_compact(decompressed, tx_bits);
|
||||
let (tx, _) = Self::from_tx_compact(tx_buf, tx_type, signature);
|
||||
|
||||
(tx, buf)
|
||||
}
|
||||
})
|
||||
} else {
|
||||
let (tx_type, buf) = T::TxType::from_compact(buf, tx_bits);
|
||||
Self::from_tx_compact(buf, tx_type, signature)
|
||||
|
||||
@@ -62,7 +62,7 @@ impl IntegerList {
|
||||
|
||||
/// Pushes a new integer to the list.
|
||||
pub fn push(&mut self, value: u64) -> Result<(), IntegerListError> {
|
||||
self.0.push(value).then_some(()).ok_or(IntegerListError::UnsortedInput)
|
||||
self.0.try_push(value).map_err(|_| IntegerListError::UnsortedInput)
|
||||
}
|
||||
|
||||
/// Clears the list.
|
||||
|
||||
@@ -960,8 +960,8 @@ mod tests {
|
||||
let provider = factory.provider().unwrap();
|
||||
let tx = provider.tx_ref();
|
||||
(
|
||||
collect_table_entries::<Arc<DatabaseEnv>, tables::AccountsHistory>(tx).unwrap(),
|
||||
collect_table_entries::<Arc<DatabaseEnv>, tables::StoragesHistory>(tx).unwrap(),
|
||||
collect_table_entries::<DatabaseEnv, tables::AccountsHistory>(tx).unwrap(),
|
||||
collect_table_entries::<DatabaseEnv, tables::StoragesHistory>(tx).unwrap(),
|
||||
)
|
||||
};
|
||||
|
||||
|
||||
@@ -374,10 +374,9 @@ mod tests {
|
||||
transaction::{DbTx, DbTxMut},
|
||||
};
|
||||
use reth_primitives_traits::StorageEntry;
|
||||
use std::sync::Arc;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn create_test_db() -> Arc<DatabaseEnv> {
|
||||
fn create_test_db() -> DatabaseEnv {
|
||||
let path = TempDir::new().unwrap();
|
||||
let mut db = DatabaseEnv::open(
|
||||
path.path(),
|
||||
@@ -386,7 +385,7 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
db.create_tables().unwrap();
|
||||
Arc::new(db)
|
||||
db
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -222,7 +222,7 @@ impl DatabaseArguments {
|
||||
}
|
||||
|
||||
/// Wrapper for the libmdbx environment: [Environment]
|
||||
#[derive(Debug)]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DatabaseEnv {
|
||||
/// Libmdbx-sys environment.
|
||||
inner: Environment,
|
||||
@@ -488,6 +488,10 @@ impl DatabaseEnv {
|
||||
inner_env.set_max_read_transaction_duration(max_read_transaction_duration);
|
||||
}
|
||||
|
||||
// Disable prefault writes: avoids mincore() syscall overhead since pages are likely
|
||||
// already warm from Engine Task reads in WRITEMAP mode.
|
||||
inner_env.set_prefault_write(false);
|
||||
|
||||
let env = Self {
|
||||
inner: inner_env.open(path).map_err(|e| DatabaseError::Open(e.into()))?,
|
||||
dbis: Arc::default(),
|
||||
@@ -644,11 +648,8 @@ mod tests {
|
||||
use tempfile::TempDir;
|
||||
|
||||
/// Create database for testing
|
||||
fn create_test_db(kind: DatabaseEnvKind) -> Arc<DatabaseEnv> {
|
||||
Arc::new(create_test_db_with_path(
|
||||
kind,
|
||||
&tempfile::TempDir::new().expect(ERROR_TEMPDIR).keep(),
|
||||
))
|
||||
fn create_test_db(kind: DatabaseEnvKind) -> DatabaseEnv {
|
||||
create_test_db_with_path(kind, &tempfile::TempDir::new().expect(ERROR_TEMPDIR).keep())
|
||||
}
|
||||
|
||||
/// Create database for testing with specified path
|
||||
@@ -737,7 +738,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_dup_cursor_delete_first() {
|
||||
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
|
||||
let mut dup_cursor = tx.cursor_dup_write::<PlainStorageState>().unwrap();
|
||||
@@ -802,7 +803,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_walk_range() {
|
||||
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT (0, 0), (1, 0), (2, 0), (3, 0)
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -866,7 +867,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_walk_range_on_dup_table() {
|
||||
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
let address0 = Address::ZERO;
|
||||
let address1 = Address::with_last_byte(1);
|
||||
@@ -926,7 +927,7 @@ mod tests {
|
||||
#[expect(clippy::reversed_empty_ranges)]
|
||||
#[test]
|
||||
fn db_cursor_walk_range_invalid() {
|
||||
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT (0, 0), (1, 0), (2, 0), (3, 0)
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -954,7 +955,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_walker() {
|
||||
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT (0, 0), (1, 0), (3, 0)
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -984,7 +985,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_reverse_walker() {
|
||||
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT (0, 0), (1, 0), (3, 0)
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -1014,7 +1015,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_walk_back() {
|
||||
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT (0, 0), (1, 0), (3, 0)
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -1053,7 +1054,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_seek_exact_or_previous_key() {
|
||||
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -1077,7 +1078,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_insert() {
|
||||
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -1117,7 +1118,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_insert_dup() {
|
||||
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
|
||||
let mut dup_cursor = tx.cursor_dup_write::<PlainStorageState>().unwrap();
|
||||
@@ -1135,7 +1136,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_delete_current_non_existent() {
|
||||
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
|
||||
let key1 = Address::with_last_byte(1);
|
||||
@@ -1165,7 +1166,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_insert_wherever_cursor_is() {
|
||||
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
|
||||
// PUT
|
||||
@@ -1198,7 +1199,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_append() {
|
||||
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -1225,7 +1226,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_append_failure() {
|
||||
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -1260,7 +1261,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_upsert() {
|
||||
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
|
||||
let mut cursor = tx.cursor_write::<PlainAccountState>().unwrap();
|
||||
@@ -1295,7 +1296,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_dupsort_append() {
|
||||
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
let transition_id = 2;
|
||||
|
||||
@@ -1557,7 +1558,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_sharded_key() {
|
||||
let db: Arc<DatabaseEnv> = create_test_db(DatabaseEnvKind::RW);
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let real_key = address!("0xa2c122be93b0074270ebee7f6b7292c7deb45047");
|
||||
|
||||
let shards = 5;
|
||||
|
||||
@@ -203,6 +203,26 @@ pub mod test_utils {
|
||||
Arc::new(TempDatabase::new(db, path))
|
||||
}
|
||||
|
||||
/// Create read/write database for testing within a data directory.
|
||||
///
|
||||
/// The database is created at `datadir/db`, and `TempDatabase` will clean up the entire
|
||||
/// `datadir` on drop.
|
||||
#[track_caller]
|
||||
pub fn create_test_rw_db_with_datadir<P: AsRef<Path>>(
|
||||
datadir: P,
|
||||
) -> Arc<TempDatabase<DatabaseEnv>> {
|
||||
let datadir = datadir.as_ref().to_path_buf();
|
||||
let db_path = datadir.join("db");
|
||||
let emsg = format!("{ERROR_DB_CREATION}: {db_path:?}");
|
||||
let db = init_db(
|
||||
&db_path,
|
||||
DatabaseArguments::new(ClientVersion::default())
|
||||
.with_max_read_transaction_duration(Some(MaxReadTransactionDuration::Unbounded)),
|
||||
)
|
||||
.expect(&emsg);
|
||||
Arc::new(TempDatabase::new(db, datadir))
|
||||
}
|
||||
|
||||
/// Create read only database for testing
|
||||
#[track_caller]
|
||||
pub fn create_test_ro_db() -> Arc<TempDatabase<DatabaseEnv>> {
|
||||
@@ -235,6 +255,24 @@ mod tests {
|
||||
use std::time::Duration;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn test_temp_database_cleanup() {
|
||||
// Test that TempDatabase properly cleans up its directory when dropped
|
||||
let temp_path = {
|
||||
let db = crate::test_utils::create_test_rw_db();
|
||||
let path = db.path().to_path_buf();
|
||||
assert!(path.exists(), "Database directory should exist while TempDatabase is alive");
|
||||
path
|
||||
// TempDatabase dropped here
|
||||
};
|
||||
|
||||
// Verify the directory was cleaned up
|
||||
assert!(
|
||||
!temp_path.exists(),
|
||||
"Database directory should be cleaned up after TempDatabase is dropped"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn db_version() {
|
||||
let path = tempdir().unwrap();
|
||||
|
||||
@@ -55,6 +55,7 @@ impl Environment {
|
||||
handle_slow_readers: None,
|
||||
#[cfg(feature = "read-tx-timeouts")]
|
||||
max_read_transaction_duration: None,
|
||||
prefault_write: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -602,6 +603,9 @@ pub struct EnvironmentBuilder {
|
||||
/// The maximum duration of a read transaction. If [None], but the `read-tx-timeout` feature is
|
||||
/// enabled, the default value of [`DEFAULT_MAX_READ_TRANSACTION_DURATION`] is used.
|
||||
max_read_transaction_duration: Option<read_transactions::MaxReadTransactionDuration>,
|
||||
/// Controls whether prefault write is enabled. If [None], MDBX uses its default.
|
||||
/// When disabled, avoids `mincore()` syscall overhead in WRITEMAP mode.
|
||||
prefault_write: Option<bool>,
|
||||
}
|
||||
|
||||
impl EnvironmentBuilder {
|
||||
@@ -723,6 +727,14 @@ impl EnvironmentBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(prefault_write) = self.prefault_write {
|
||||
mdbx_result(ffi::mdbx_env_set_option(
|
||||
env,
|
||||
ffi::MDBX_opt_prefault_write_enable,
|
||||
prefault_write as u64,
|
||||
))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})() {
|
||||
ffi::mdbx_env_close_ex(env, false);
|
||||
@@ -873,6 +885,19 @@ impl EnvironmentBuilder {
|
||||
self.handle_slow_readers = Some(hsr);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether to enable prefault writes.
|
||||
///
|
||||
/// In WRITEMAP mode, MDBX by default uses `mincore()` to check if pages are resident before
|
||||
/// touching them. This avoids page faults but adds syscall overhead. Disabling prefault
|
||||
/// writes skips the `mincore()` check and lets the kernel handle page faults directly.
|
||||
///
|
||||
/// This is beneficial when pages are likely already in memory (e.g., recently read by
|
||||
/// other transactions), as it eliminates unnecessary syscall overhead.
|
||||
pub const fn set_prefault_write(&mut self, prefault_write: bool) -> &mut Self {
|
||||
self.prefault_write = Some(prefault_write);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "read-tx-timeouts")]
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user