mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
44 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1f93ee97bb | ||
|
|
3adac16571 | ||
|
|
bafe9943fd | ||
|
|
ac21a606a0 | ||
|
|
8f940c2d69 | ||
|
|
1af8a03ab6 | ||
|
|
3ab7774c5e | ||
|
|
912ce282dd | ||
|
|
6a4e5eb409 | ||
|
|
fd3b778f04 | ||
|
|
5db0e519ca | ||
|
|
fc8931c30e | ||
|
|
72bfdfe6ca | ||
|
|
b6d0e04a16 | ||
|
|
35d214f616 | ||
|
|
ed093cd5a0 | ||
|
|
04d22f6469 | ||
|
|
26db968153 | ||
|
|
d2420598b0 | ||
|
|
0b4d697ef3 | ||
|
|
f257508554 | ||
|
|
c6628ae3a4 | ||
|
|
3f2f5baaca | ||
|
|
d44a500750 | ||
|
|
b7b3f327fc | ||
|
|
2c70f5157f | ||
|
|
6d364b1379 | ||
|
|
3b67d78e24 | ||
|
|
b5ccdb89ac | ||
|
|
3e4e764b5a | ||
|
|
39c725e66c | ||
|
|
19bdc02e57 | ||
|
|
c68bd5036c | ||
|
|
f49b9542f5 | ||
|
|
04824d304b | ||
|
|
0d38ba9fc5 | ||
|
|
63ac93a431 | ||
|
|
90492aa3cf | ||
|
|
dd7a10ba4a | ||
|
|
14453ccca5 | ||
|
|
13e9c00ae9 | ||
|
|
3b129f5a34 | ||
|
|
70480acd66 | ||
|
|
7856d256e6 |
@@ -12,7 +12,7 @@ workflows:
|
||||
# Check that `A` activates the features of `B`.
|
||||
"propagate-feature",
|
||||
# These are the features to check:
|
||||
"--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,js-tracer,portable,keccak-cache-global",
|
||||
"--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,js-tracer,portable",
|
||||
# Do not try to add a new section to `[features]` of `A` only because `B` exposes that feature. There are edge-cases where this is still needed, but we can add them manually.
|
||||
"--left-side-feature-missing=ignore",
|
||||
# Ignore the case that `A` it outside of the workspace. Otherwise it will report errors in external dependencies that we have no influence on.
|
||||
|
||||
50
Cargo.lock
generated
50
Cargo.lock
generated
@@ -329,9 +329,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-json-abi"
|
||||
version = "1.5.0"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6bfca3dbbcb7498f0f60e67aff2ad6aff57032e22eb2fd03189854be11a22c03"
|
||||
checksum = "5513d5e6bd1cba6bdcf5373470f559f320c05c8c59493b6e98912fbe6733943f"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"alloy-sol-type-parser",
|
||||
@@ -426,9 +426,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-primitives"
|
||||
version = "1.5.0"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c850e6ccbd34b8a463a1e934ffc8fc00e1efc5e5489f2ad82d7797949f3bd4e"
|
||||
checksum = "355bf68a433e0fd7f7d33d5a9fc2583fde70bf5c530f63b80845f8da5505cf28"
|
||||
dependencies = [
|
||||
"alloy-rlp",
|
||||
"arbitrary",
|
||||
@@ -447,7 +447,6 @@ dependencies = [
|
||||
"proptest",
|
||||
"proptest-derive 0.6.0",
|
||||
"rand 0.9.2",
|
||||
"rapidhash",
|
||||
"ruint",
|
||||
"rustc-hash",
|
||||
"serde",
|
||||
@@ -782,9 +781,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-sol-macro"
|
||||
version = "1.5.0"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2218e3aeb3ee665d117fdf188db0d5acfdc3f7b7502c827421cb78f26a2aec0"
|
||||
checksum = "f3ce480400051b5217f19d6e9a82d9010cdde20f1ae9c00d53591e4a1afbb312"
|
||||
dependencies = [
|
||||
"alloy-sol-macro-expander",
|
||||
"alloy-sol-macro-input",
|
||||
@@ -796,9 +795,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-sol-macro-expander"
|
||||
version = "1.5.0"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b231cb8cc48e66dd1c6e11a1402f3ac86c3667cbc13a6969a0ac030ba7bb8c88"
|
||||
checksum = "6d792e205ed3b72f795a8044c52877d2e6b6e9b1d13f431478121d8d4eaa9028"
|
||||
dependencies = [
|
||||
"alloy-sol-macro-input",
|
||||
"const-hex",
|
||||
@@ -814,9 +813,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-sol-macro-input"
|
||||
version = "1.5.0"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49a522d79929c1bf0152b07567a38f7eaed3ab149e53e7528afa78ff11994668"
|
||||
checksum = "0bd1247a8f90b465ef3f1207627547ec16940c35597875cdc09c49d58b19693c"
|
||||
dependencies = [
|
||||
"const-hex",
|
||||
"dunce",
|
||||
@@ -830,9 +829,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-sol-type-parser"
|
||||
version = "1.5.0"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0475c459859c8d9428af6ff3736614655a57efda8cc435a3b8b4796fa5ac1dd0"
|
||||
checksum = "954d1b2533b9b2c7959652df3076954ecb1122a28cc740aa84e7b0a49f6ac0a9"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"winnow",
|
||||
@@ -840,9 +839,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-sol-types"
|
||||
version = "1.5.0"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35287d9d821d5f26011bcd8d9101340898f761c9933cf50fca689bb7ed62fdeb"
|
||||
checksum = "70319350969a3af119da6fb3e9bddb1bce66c9ea933600cb297c8b1850ad2a3c"
|
||||
dependencies = [
|
||||
"alloy-json-abi",
|
||||
"alloy-primitives",
|
||||
@@ -7210,16 +7209,6 @@ dependencies = [
|
||||
"rand_core 0.9.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rapidhash"
|
||||
version = "4.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d8e65c75143ce5d47c55b510297eeb1182f3c739b6043c537670e9fc18612dae"
|
||||
dependencies = [
|
||||
"rand 0.9.2",
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ratatui"
|
||||
version = "0.29.0"
|
||||
@@ -8247,7 +8236,6 @@ dependencies = [
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"mini-moka",
|
||||
"moka",
|
||||
"parking_lot",
|
||||
"proptest",
|
||||
"rand 0.8.5",
|
||||
@@ -8281,7 +8269,6 @@ dependencies = [
|
||||
"reth-stages",
|
||||
"reth-stages-api",
|
||||
"reth-static-file",
|
||||
"reth-storage-errors",
|
||||
"reth-tasks",
|
||||
"reth-testing-utils",
|
||||
"reth-tracing",
|
||||
@@ -9129,7 +9116,6 @@ dependencies = [
|
||||
"fdlimit",
|
||||
"futures",
|
||||
"jsonrpsee",
|
||||
"parking_lot",
|
||||
"rayon",
|
||||
"reth-basic-payload-builder",
|
||||
"reth-chain-state",
|
||||
@@ -9289,7 +9275,6 @@ dependencies = [
|
||||
"reth-rpc-eth-api",
|
||||
"reth-rpc-eth-types",
|
||||
"reth-rpc-server-types",
|
||||
"reth-stages-types",
|
||||
"reth-tasks",
|
||||
"reth-testing-utils",
|
||||
"reth-tracing",
|
||||
@@ -9653,7 +9638,6 @@ dependencies = [
|
||||
"reth-rpc-engine-api",
|
||||
"reth-rpc-eth-types",
|
||||
"reth-rpc-server-types",
|
||||
"reth-stages-types",
|
||||
"reth-tasks",
|
||||
"reth-tracing",
|
||||
"reth-transaction-pool",
|
||||
@@ -10397,7 +10381,6 @@ dependencies = [
|
||||
"reth-ethereum-engine-primitives",
|
||||
"reth-ethereum-primitives",
|
||||
"reth-metrics",
|
||||
"reth-network-api",
|
||||
"reth-node-ethereum",
|
||||
"reth-payload-builder",
|
||||
"reth-payload-builder-primitives",
|
||||
@@ -10584,7 +10567,6 @@ dependencies = [
|
||||
"reth-stages-api",
|
||||
"reth-static-file",
|
||||
"reth-static-file-types",
|
||||
"reth-storage-api",
|
||||
"reth-storage-errors",
|
||||
"reth-testing-utils",
|
||||
"reth-trie",
|
||||
@@ -12358,9 +12340,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "syn-solidity"
|
||||
version = "1.5.0"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60ceeb7c95a4536de0c0e1649bd98d1a72a4bb9590b1f3e45a8a0bfdb7c188c0"
|
||||
checksum = "ff790eb176cc81bb8936aed0f7b9f14fc4670069a2d371b3e3b0ecce908b2cb3"
|
||||
dependencies = [
|
||||
"paste",
|
||||
"proc-macro2",
|
||||
|
||||
@@ -489,10 +489,10 @@ alloy-dyn-abi = "1.4.1"
|
||||
alloy-eip2124 = { version = "0.2.0", default-features = false }
|
||||
alloy-eip7928 = { version = "0.1.0" }
|
||||
alloy-evm = { version = "0.25.1", default-features = false }
|
||||
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
|
||||
alloy-primitives = { version = "1.4.1", default-features = false, features = ["map-foldhash"] }
|
||||
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
|
||||
alloy-sol-macro = "1.5.0"
|
||||
alloy-sol-types = { version = "1.5.0", default-features = false }
|
||||
alloy-sol-macro = "1.4.1"
|
||||
alloy-sol-types = { version = "1.4.1", default-features = false }
|
||||
alloy-trie = { version = "0.9.1", default-features = false }
|
||||
|
||||
alloy-hardforks = "0.4.5"
|
||||
@@ -587,7 +587,6 @@ url = { version = "2.3", default-features = false }
|
||||
zstd = "0.13"
|
||||
byteorder = "1"
|
||||
mini-moka = "0.10"
|
||||
moka = "0.12"
|
||||
tar-no-std = { version = "0.3.2", default-features = false }
|
||||
miniz_oxide = { version = "0.8.4", default-features = false }
|
||||
chrono = "0.4.41"
|
||||
|
||||
@@ -18,7 +18,7 @@ FROM chef AS builder
|
||||
COPY --from=planner /app/recipe.json recipe.json
|
||||
|
||||
# Build profile, release by default
|
||||
ARG BUILD_PROFILE=maxperf
|
||||
ARG BUILD_PROFILE=release
|
||||
ENV BUILD_PROFILE=$BUILD_PROFILE
|
||||
|
||||
# Extra Cargo flags
|
||||
|
||||
@@ -14,7 +14,7 @@ RUN cargo chef prepare --recipe-path recipe.json
|
||||
FROM chef AS builder
|
||||
COPY --from=planner /app/recipe.json recipe.json
|
||||
|
||||
ARG BUILD_PROFILE=maxperf
|
||||
ARG BUILD_PROFILE=release
|
||||
ENV BUILD_PROFILE=$BUILD_PROFILE
|
||||
|
||||
ARG RUSTFLAGS=""
|
||||
|
||||
@@ -81,7 +81,7 @@ backon.workspace = true
|
||||
tempfile.workspace = true
|
||||
|
||||
[features]
|
||||
default = ["jemalloc", "otlp", "reth-revm/portable", "js-tracer", "keccak-cache-global", "asm-keccak"]
|
||||
default = ["jemalloc", "otlp", "reth-revm/portable", "js-tracer"]
|
||||
|
||||
otlp = [
|
||||
"reth-ethereum-cli/otlp",
|
||||
@@ -102,9 +102,7 @@ asm-keccak = [
|
||||
"reth-ethereum-cli/asm-keccak",
|
||||
"reth-node-ethereum/asm-keccak",
|
||||
]
|
||||
keccak-cache-global = [
|
||||
"reth-node-ethereum/keccak-cache-global",
|
||||
]
|
||||
|
||||
jemalloc = [
|
||||
"reth-cli-util/jemalloc",
|
||||
"reth-node-core/jemalloc",
|
||||
|
||||
@@ -80,8 +80,6 @@ pub fn make_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Hea
|
||||
.then_some(EMPTY_REQUESTS_HASH);
|
||||
|
||||
Header {
|
||||
number: genesis.number.unwrap_or_default(),
|
||||
parent_hash: genesis.parent_hash.unwrap_or_default(),
|
||||
gas_limit: genesis.gas_limit,
|
||||
difficulty: genesis.difficulty,
|
||||
nonce: genesis.nonce.into(),
|
||||
|
||||
@@ -23,10 +23,7 @@ use reth_node_core::{
|
||||
dirs::{ChainPath, DataDirPath},
|
||||
};
|
||||
use reth_provider::{
|
||||
providers::{
|
||||
BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider,
|
||||
StaticFileProviderBuilder,
|
||||
},
|
||||
providers::{BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider},
|
||||
ProviderFactory, StaticFileProviderFactory,
|
||||
};
|
||||
use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget};
|
||||
@@ -103,27 +100,18 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
}
|
||||
|
||||
info!(target: "reth::cli", ?db_path, ?sf_path, "Opening storage");
|
||||
let genesis_block_number = self.chain.genesis().number.unwrap_or_default();
|
||||
let (db, sfp) = match access {
|
||||
AccessRights::RW => (
|
||||
Arc::new(init_db(db_path, self.db.database_args())?),
|
||||
StaticFileProviderBuilder::read_write(sf_path)?
|
||||
.with_genesis_block_number(genesis_block_number)
|
||||
.build()?,
|
||||
StaticFileProvider::read_write(sf_path)?,
|
||||
),
|
||||
AccessRights::RO | AccessRights::RoInconsistent => (
|
||||
Arc::new(open_db_read_only(&db_path, self.db.database_args())?),
|
||||
StaticFileProvider::read_only(sf_path, false)?,
|
||||
),
|
||||
AccessRights::RO | AccessRights::RoInconsistent => {
|
||||
(Arc::new(open_db_read_only(&db_path, self.db.database_args())?), {
|
||||
let provider = StaticFileProviderBuilder::read_only(sf_path)?
|
||||
.with_genesis_block_number(genesis_block_number)
|
||||
.build()?;
|
||||
provider.watch_directory();
|
||||
provider
|
||||
})
|
||||
}
|
||||
};
|
||||
// TransactionDB only support read-write mode
|
||||
let rocksdb_provider = RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level)
|
||||
.build()?;
|
||||
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
//! Command that initializes the node from a genesis file.
|
||||
|
||||
use crate::common::{AccessRights, CliNodeTypes, Environment, EnvironmentArgs};
|
||||
use alloy_consensus::BlockHeader;
|
||||
use clap::Parser;
|
||||
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks};
|
||||
use reth_chainspec::{EthChainSpec, EthereumHardforks};
|
||||
use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_provider::BlockHashReader;
|
||||
use std::sync::Arc;
|
||||
@@ -23,9 +22,8 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> InitComman
|
||||
|
||||
let Environment { provider_factory, .. } = self.env.init::<N>(AccessRights::RW)?;
|
||||
|
||||
let genesis_block_number = provider_factory.chain_spec().genesis_header().number();
|
||||
let hash = provider_factory
|
||||
.block_hash(genesis_block_number)?
|
||||
.block_hash(0)?
|
||||
.ok_or_else(|| eyre::eyre!("Genesis hash not found."))?;
|
||||
|
||||
info!(target: "reth::cli", hash = ?hash, "Genesis block written");
|
||||
|
||||
@@ -72,7 +72,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
.split();
|
||||
if result.len() != 1 {
|
||||
eyre::bail!(
|
||||
"Invalid number of bodies received. Expected: 1. Received: {}",
|
||||
"Invalid number of headers received. Expected: 1. Received: {}",
|
||||
result.len()
|
||||
)
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ use reth_node_builder::{
|
||||
PayloadTypes,
|
||||
};
|
||||
use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs};
|
||||
use reth_primitives_traits::AlloyBlockHeader;
|
||||
use reth_provider::providers::BlockchainProvider;
|
||||
use reth_rpc_server_types::RpcModuleSelection;
|
||||
use reth_tasks::TaskManager;
|
||||
@@ -158,8 +157,8 @@ where
|
||||
.await?;
|
||||
|
||||
let node = NodeTestContext::new(node, self.attributes_generator).await?;
|
||||
let genesis_number = self.chain_spec.genesis_header().number();
|
||||
let genesis = node.block_hash(genesis_number);
|
||||
|
||||
let genesis = node.block_hash(0);
|
||||
node.update_forkchoice(genesis, genesis).await?;
|
||||
|
||||
eyre::Ok(node)
|
||||
|
||||
@@ -29,7 +29,6 @@ reth-provider.workspace = true
|
||||
reth-prune.workspace = true
|
||||
reth-revm.workspace = true
|
||||
reth-stages-api.workspace = true
|
||||
reth-storage-errors.workspace = true
|
||||
reth-tasks.workspace = true
|
||||
reth-trie-parallel.workspace = true
|
||||
reth-trie-sparse = { workspace = true, features = ["std", "metrics"] }
|
||||
@@ -53,7 +52,6 @@ futures.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio = { workspace = true, features = ["rt", "rt-multi-thread", "sync", "macros"] }
|
||||
mini-moka = { workspace = true, features = ["sync"] }
|
||||
moka = { workspace = true, features = ["sync"] }
|
||||
smallvec.workspace = true
|
||||
|
||||
# metrics
|
||||
|
||||
@@ -241,7 +241,6 @@ fn bench_state_root(c: &mut Criterion) {
|
||||
StateProviderBuilder::new(provider.clone(), genesis_hash, None),
|
||||
OverlayStateProviderFactory::new(provider),
|
||||
&TreeConfig::default(),
|
||||
None,
|
||||
);
|
||||
|
||||
let mut state_hook = handle.state_hook();
|
||||
|
||||
@@ -128,12 +128,12 @@ we send them along with the state updates to the [Sparse Trie Task](#sparse-trie
|
||||
|
||||
### Finishing the calculation
|
||||
|
||||
Once all transactions are executed, the [Engine](#engine) sends a `StateRootMessage::FinishedStateUpdates` message
|
||||
Once all transactions are executed, the [Engine](#engine) sends a `StateRootMessage::FinishStateUpdates` message
|
||||
to the State Root Task, marking the end of receiving state updates.
|
||||
|
||||
Every time we receive a new proof from the [MultiProof Manager](#multiproof-manager), we also check
|
||||
the following conditions:
|
||||
1. Are all updates received? (`StateRootMessage::FinishedStateUpdates` was sent)
|
||||
1. Are all updates received? (`StateRootMessage::FinishStateUpdates` was sent)
|
||||
2. Is `ProofSequencer` empty? (no proofs are pending for sequencing)
|
||||
3. Are all proofs that were sent to the [`MultiProofManager::spawn_or_queue`](#multiproof-manager) finished
|
||||
calculating and were sent to the [Sparse Trie Task](#sparse-trie-task)?
|
||||
|
||||
@@ -219,19 +219,10 @@ pub enum HandlerEvent<T> {
|
||||
}
|
||||
|
||||
/// Internal events issued by the [`ChainOrchestrator`].
|
||||
#[derive(Debug)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum FromOrchestrator {
|
||||
/// Invoked when backfill sync finished
|
||||
BackfillSyncFinished(ControlFlow),
|
||||
/// Invoked when backfill sync started
|
||||
BackfillSyncStarted,
|
||||
/// Gracefully terminate the engine service.
|
||||
///
|
||||
/// When this variant is received, the engine will persist all remaining in-memory blocks
|
||||
/// to disk before shutting down. Once persistence is complete, a signal is sent through
|
||||
/// the oneshot channel to notify the caller.
|
||||
Terminate {
|
||||
/// Channel to signal termination completion.
|
||||
tx: tokio::sync::oneshot::Sender<()>,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -31,9 +31,6 @@ pub(crate) struct CachedStateProvider<S> {
|
||||
|
||||
/// Metrics for the cached state provider
|
||||
metrics: CachedStateMetrics,
|
||||
|
||||
/// If prewarm enabled we populate every cache miss
|
||||
prewarm: bool,
|
||||
}
|
||||
|
||||
impl<S> CachedStateProvider<S>
|
||||
@@ -42,32 +39,12 @@ where
|
||||
{
|
||||
/// Creates a new [`CachedStateProvider`] from an [`ExecutionCache`], state provider, and
|
||||
/// [`CachedStateMetrics`].
|
||||
pub(crate) const fn new(
|
||||
pub(crate) const fn new_with_caches(
|
||||
state_provider: S,
|
||||
caches: ExecutionCache,
|
||||
metrics: CachedStateMetrics,
|
||||
) -> Self {
|
||||
Self { state_provider, caches, metrics, prewarm: false }
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> CachedStateProvider<S> {
|
||||
/// Enables pre-warm mode so that every cache miss is populated.
|
||||
///
|
||||
/// This is only relevant for pre-warm transaction execution with the intention to pre-populate
|
||||
/// the cache with data for regular block execution. During regular block execution the
|
||||
/// cache doesn't need to be populated because the actual EVM database
|
||||
/// [`State`](revm::database::State) also caches internally during block execution and the cache
|
||||
/// is then updated after the block with the entire [`BundleState`] output of that block which
|
||||
/// contains all accessed accounts,code,storage. See also [`ExecutionCache::insert_state`].
|
||||
pub(crate) const fn prewarm(mut self) -> Self {
|
||||
self.prewarm = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether this provider should pre-warm cache misses.
|
||||
const fn is_prewarm(&self) -> bool {
|
||||
self.prewarm
|
||||
Self { state_provider, caches, metrics }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -146,10 +123,7 @@ impl<S: AccountReader> AccountReader for CachedStateProvider<S> {
|
||||
self.metrics.account_cache_misses.increment(1);
|
||||
|
||||
let res = self.state_provider.basic_account(address)?;
|
||||
|
||||
if self.is_prewarm() {
|
||||
self.caches.account_cache.insert(*address, res);
|
||||
}
|
||||
self.caches.account_cache.insert(*address, res);
|
||||
Ok(res)
|
||||
}
|
||||
}
|
||||
@@ -174,19 +148,15 @@ impl<S: StateProvider> StateProvider for CachedStateProvider<S> {
|
||||
match self.caches.get_storage(&account, &storage_key) {
|
||||
(SlotStatus::NotCached, maybe_cache) => {
|
||||
let final_res = self.state_provider.storage(account, storage_key)?;
|
||||
|
||||
if self.is_prewarm() {
|
||||
let account_cache = maybe_cache.unwrap_or_default();
|
||||
account_cache.insert_storage(storage_key, final_res);
|
||||
// we always need to insert the value to update the weights.
|
||||
// Note: there exists a race when the storage cache did not exist yet and two
|
||||
// consumers looking up the a storage value for this account for the first time,
|
||||
// however we can assume that this will only happen for the very first
|
||||
// (mostlikely the same) value, and don't expect that this
|
||||
// will accidentally replace an account storage cache with
|
||||
// additional values.
|
||||
self.caches.insert_storage_cache(account, account_cache);
|
||||
}
|
||||
let account_cache = maybe_cache.unwrap_or_default();
|
||||
account_cache.insert_storage(storage_key, final_res);
|
||||
// we always need to insert the value to update the weights.
|
||||
// Note: there exists a race when the storage cache did not exist yet and two
|
||||
// consumers looking up the a storage value for this account for the first time,
|
||||
// however we can assume that this will only happen for the very first (mostlikely
|
||||
// the same) value, and don't expect that this will accidentally
|
||||
// replace an account storage cache with additional values.
|
||||
self.caches.insert_storage_cache(account, account_cache);
|
||||
|
||||
self.metrics.storage_cache_misses.increment(1);
|
||||
Ok(final_res)
|
||||
@@ -213,11 +183,7 @@ impl<S: BytecodeReader> BytecodeReader for CachedStateProvider<S> {
|
||||
self.metrics.code_cache_misses.increment(1);
|
||||
|
||||
let final_res = self.state_provider.bytecode_by_hash(code_hash)?;
|
||||
|
||||
if self.is_prewarm() {
|
||||
self.caches.code_cache.insert(*code_hash, final_res.clone());
|
||||
}
|
||||
|
||||
self.caches.code_cache.insert(*code_hash, final_res.clone());
|
||||
Ok(final_res)
|
||||
}
|
||||
}
|
||||
@@ -819,7 +785,7 @@ mod tests {
|
||||
|
||||
let caches = ExecutionCacheBuilder::default().build_caches(1000);
|
||||
let state_provider =
|
||||
CachedStateProvider::new(provider, caches, CachedStateMetrics::zeroed());
|
||||
CachedStateProvider::new_with_caches(provider, caches, CachedStateMetrics::zeroed());
|
||||
|
||||
// check that the storage is empty
|
||||
let res = state_provider.storage(address, storage_key);
|
||||
@@ -842,7 +808,7 @@ mod tests {
|
||||
|
||||
let caches = ExecutionCacheBuilder::default().build_caches(1000);
|
||||
let state_provider =
|
||||
CachedStateProvider::new(provider, caches, CachedStateMetrics::zeroed());
|
||||
CachedStateProvider::new_with_caches(provider, caches, CachedStateMetrics::zeroed());
|
||||
|
||||
// check that the storage returns the expected value
|
||||
let res = state_provider.storage(address, storage_key);
|
||||
|
||||
@@ -83,7 +83,7 @@ where
|
||||
{
|
||||
/// Creates a new [`InstrumentedStateProvider`] from a state provider with the provided label
|
||||
/// for metrics.
|
||||
pub fn new(state_provider: S, source: &'static str) -> Self {
|
||||
pub fn from_state_provider(state_provider: S, source: &'static str) -> Self {
|
||||
Self {
|
||||
state_provider,
|
||||
metrics: StateProviderMetrics::new_with_labels(&[("source", source)]),
|
||||
|
||||
@@ -39,7 +39,6 @@ use revm::state::EvmState;
|
||||
use state::TreeState;
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
ops,
|
||||
sync::{
|
||||
mpsc::{Receiver, RecvError, RecvTimeoutError, Sender},
|
||||
Arc,
|
||||
@@ -427,13 +426,9 @@ where
|
||||
match self.try_recv_engine_message() {
|
||||
Ok(Some(msg)) => {
|
||||
debug!(target: "engine::tree", %msg, "received new engine message");
|
||||
match self.on_engine_message(msg) {
|
||||
Ok(ops::ControlFlow::Break(())) => return,
|
||||
Ok(ops::ControlFlow::Continue(())) => {}
|
||||
Err(fatal) => {
|
||||
error!(target: "engine::tree", %fatal, "insert block fatal error");
|
||||
return
|
||||
}
|
||||
if let Err(fatal) = self.on_engine_message(msg) {
|
||||
error!(target: "engine::tree", %fatal, "insert block fatal error");
|
||||
return
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
@@ -1307,7 +1302,22 @@ where
|
||||
// Check if persistence has complete
|
||||
match rx.try_recv() {
|
||||
Ok(last_persisted_hash_num) => {
|
||||
self.on_persistence_complete(last_persisted_hash_num, start_time)?;
|
||||
self.metrics.engine.persistence_duration.record(start_time.elapsed());
|
||||
let Some(BlockNumHash {
|
||||
hash: last_persisted_block_hash,
|
||||
number: last_persisted_block_number,
|
||||
}) = last_persisted_hash_num
|
||||
else {
|
||||
// if this happened, then we persisted no blocks because we sent an
|
||||
// empty vec of blocks
|
||||
warn!(target: "engine::tree", "Persistence task completed but did not persist any blocks");
|
||||
return Ok(())
|
||||
};
|
||||
|
||||
debug!(target: "engine::tree", ?last_persisted_block_hash, ?last_persisted_block_number, elapsed=?start_time.elapsed(), "Finished persisting, calling finish");
|
||||
self.persistence_state
|
||||
.finish(last_persisted_block_hash, last_persisted_block_number);
|
||||
self.on_new_persisted_block()?;
|
||||
}
|
||||
Err(TryRecvError::Closed) => return Err(TryRecvError::Closed.into()),
|
||||
Err(TryRecvError::Empty) => {
|
||||
@@ -1320,8 +1330,7 @@ where
|
||||
if let Some(new_tip_num) = self.find_disk_reorg()? {
|
||||
self.remove_blocks(new_tip_num)
|
||||
} else if self.should_persist() {
|
||||
let blocks_to_persist =
|
||||
self.get_canonical_blocks_to_persist(PersistTarget::Threshold)?;
|
||||
let blocks_to_persist = self.get_canonical_blocks_to_persist()?;
|
||||
self.persist_blocks(blocks_to_persist);
|
||||
}
|
||||
}
|
||||
@@ -1329,72 +1338,11 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Finishes termination by persisting all remaining blocks and signaling completion.
|
||||
///
|
||||
/// This blocks until all persistence is complete. Always signals completion,
|
||||
/// even if an error occurs.
|
||||
fn finish_termination(
|
||||
&mut self,
|
||||
pending_termination: oneshot::Sender<()>,
|
||||
) -> Result<(), AdvancePersistenceError> {
|
||||
trace!(target: "engine::tree", "finishing termination, persisting remaining blocks");
|
||||
let result = self.persist_until_complete();
|
||||
let _ = pending_termination.send(());
|
||||
result
|
||||
}
|
||||
|
||||
/// Persists all remaining blocks until none are left.
|
||||
fn persist_until_complete(&mut self) -> Result<(), AdvancePersistenceError> {
|
||||
loop {
|
||||
// Wait for any in-progress persistence to complete (blocking)
|
||||
if let Some((rx, start_time, _action)) = self.persistence_state.rx.take() {
|
||||
let result = rx.blocking_recv().map_err(|_| TryRecvError::Closed)?;
|
||||
self.on_persistence_complete(result, start_time)?;
|
||||
}
|
||||
|
||||
let blocks_to_persist = self.get_canonical_blocks_to_persist(PersistTarget::Head)?;
|
||||
|
||||
if blocks_to_persist.is_empty() {
|
||||
debug!(target: "engine::tree", "persistence complete, signaling termination");
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
debug!(target: "engine::tree", count = blocks_to_persist.len(), "persisting remaining blocks before shutdown");
|
||||
self.persist_blocks(blocks_to_persist);
|
||||
}
|
||||
}
|
||||
|
||||
/// Handles a completed persistence task.
|
||||
fn on_persistence_complete(
|
||||
&mut self,
|
||||
last_persisted_hash_num: Option<BlockNumHash>,
|
||||
start_time: Instant,
|
||||
) -> Result<(), AdvancePersistenceError> {
|
||||
self.metrics.engine.persistence_duration.record(start_time.elapsed());
|
||||
|
||||
let Some(BlockNumHash {
|
||||
hash: last_persisted_block_hash,
|
||||
number: last_persisted_block_number,
|
||||
}) = last_persisted_hash_num
|
||||
else {
|
||||
// if this happened, then we persisted no blocks because we sent an empty vec of blocks
|
||||
warn!(target: "engine::tree", "Persistence task completed but did not persist any blocks");
|
||||
return Ok(())
|
||||
};
|
||||
|
||||
debug!(target: "engine::tree", ?last_persisted_block_hash, ?last_persisted_block_number, elapsed=?start_time.elapsed(), "Finished persisting, calling finish");
|
||||
self.persistence_state.finish(last_persisted_block_hash, last_persisted_block_number);
|
||||
self.on_new_persisted_block()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Handles a message from the engine.
|
||||
///
|
||||
/// Returns `ControlFlow::Break(())` if the engine should terminate.
|
||||
fn on_engine_message(
|
||||
&mut self,
|
||||
msg: FromEngine<EngineApiRequest<T, N>, N::Block>,
|
||||
) -> Result<ops::ControlFlow<()>, InsertBlockFatalError> {
|
||||
) -> Result<(), InsertBlockFatalError> {
|
||||
match msg {
|
||||
FromEngine::Event(event) => match event {
|
||||
FromOrchestrator::BackfillSyncStarted => {
|
||||
@@ -1404,13 +1352,6 @@ where
|
||||
FromOrchestrator::BackfillSyncFinished(ctrl) => {
|
||||
self.on_backfill_sync_finished(ctrl)?;
|
||||
}
|
||||
FromOrchestrator::Terminate { tx } => {
|
||||
debug!(target: "engine::tree", "received terminate request");
|
||||
if let Err(err) = self.finish_termination(tx) {
|
||||
error!(target: "engine::tree", %err, "Termination failed");
|
||||
}
|
||||
return Ok(ops::ControlFlow::Break(()))
|
||||
}
|
||||
},
|
||||
FromEngine::Request(request) => {
|
||||
match request {
|
||||
@@ -1418,7 +1359,7 @@ where
|
||||
let block_num_hash = block.recovered_block().num_hash();
|
||||
if block_num_hash.number <= self.state.tree_state.canonical_block_number() {
|
||||
// outdated block that can be skipped
|
||||
return Ok(ops::ControlFlow::Continue(()))
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
debug!(target: "engine::tree", block=?block_num_hash, "inserting already executed block");
|
||||
@@ -1526,7 +1467,7 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(ops::ControlFlow::Continue(()))
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Invoked if the backfill sync has finished to target.
|
||||
@@ -1760,10 +1701,10 @@ where
|
||||
}
|
||||
|
||||
/// Returns a batch of consecutive canonical blocks to persist in the range
|
||||
/// `(last_persisted_number .. target]`. The expected order is oldest -> newest.
|
||||
/// `(last_persisted_number .. canonical_head - threshold]`. The expected
|
||||
/// order is oldest -> newest.
|
||||
fn get_canonical_blocks_to_persist(
|
||||
&self,
|
||||
target: PersistTarget,
|
||||
) -> Result<Vec<ExecutedBlock<N>>, AdvancePersistenceError> {
|
||||
// We will calculate the state root using the database, so we need to be sure there are no
|
||||
// changes
|
||||
@@ -1774,12 +1715,9 @@ where
|
||||
let last_persisted_number = self.persistence_state.last_persisted_block.number;
|
||||
let canonical_head_number = self.state.tree_state.canonical_block_number();
|
||||
|
||||
let target_number = match target {
|
||||
PersistTarget::Head => canonical_head_number,
|
||||
PersistTarget::Threshold => {
|
||||
canonical_head_number.saturating_sub(self.config.memory_block_buffer_target())
|
||||
}
|
||||
};
|
||||
// Persist only up to block buffer target
|
||||
let target_number =
|
||||
canonical_head_number.saturating_sub(self.config.memory_block_buffer_target());
|
||||
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
@@ -2922,12 +2860,3 @@ pub enum InsertPayloadOk {
|
||||
/// The payload was valid and inserted into the tree.
|
||||
Inserted(BlockStatus),
|
||||
}
|
||||
|
||||
/// Target for block persistence.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum PersistTarget {
|
||||
/// Persist up to `canonical_head - memory_block_buffer_target`.
|
||||
Threshold,
|
||||
/// Persist all blocks up to and including the canonical head.
|
||||
Head,
|
||||
}
|
||||
|
||||
@@ -1,318 +0,0 @@
|
||||
//! BAL (Block Access List, EIP-7928) related functionality.
|
||||
|
||||
use alloy_consensus::constants::KECCAK_EMPTY;
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use alloy_primitives::{keccak256, U256};
|
||||
use reth_primitives_traits::Account;
|
||||
use reth_provider::{AccountReader, ProviderError};
|
||||
use reth_trie::{HashedPostState, HashedStorage};
|
||||
|
||||
/// Converts a Block Access List into a [`HashedPostState`] by extracting the final state
|
||||
/// of modified accounts and storage slots.
|
||||
pub fn bal_to_hashed_post_state<P>(
|
||||
bal: &BlockAccessList,
|
||||
provider: &P,
|
||||
) -> Result<HashedPostState, ProviderError>
|
||||
where
|
||||
P: AccountReader,
|
||||
{
|
||||
let mut hashed_state = HashedPostState::with_capacity(bal.len());
|
||||
|
||||
for account_changes in bal {
|
||||
let address = account_changes.address;
|
||||
let hashed_address = keccak256(address);
|
||||
|
||||
// Get the latest balance (last balance change if any)
|
||||
let balance = account_changes.balance_changes.last().map(|change| change.post_balance);
|
||||
|
||||
// Get the latest nonce (last nonce change if any)
|
||||
let nonce = account_changes.nonce_changes.last().map(|change| change.new_nonce);
|
||||
|
||||
// Get the latest code (last code change if any)
|
||||
let code_hash = if let Some(code_change) = account_changes.code_changes.last() {
|
||||
if code_change.new_code.is_empty() {
|
||||
Some(Some(KECCAK_EMPTY))
|
||||
} else {
|
||||
Some(Some(keccak256(&code_change.new_code)))
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Only fetch account from provider if we're missing any field
|
||||
let existing_account = if balance.is_none() || nonce.is_none() || code_hash.is_none() {
|
||||
provider.basic_account(&address)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Build the final account state
|
||||
let account = Account {
|
||||
balance: balance.unwrap_or_else(|| {
|
||||
existing_account.as_ref().map(|acc| acc.balance).unwrap_or(U256::ZERO)
|
||||
}),
|
||||
nonce: nonce
|
||||
.unwrap_or_else(|| existing_account.as_ref().map(|acc| acc.nonce).unwrap_or(0)),
|
||||
bytecode_hash: code_hash.unwrap_or_else(|| {
|
||||
existing_account.as_ref().and_then(|acc| acc.bytecode_hash).or(Some(KECCAK_EMPTY))
|
||||
}),
|
||||
};
|
||||
|
||||
hashed_state.accounts.insert(hashed_address, Some(account));
|
||||
|
||||
// Process storage changes
|
||||
if !account_changes.storage_changes.is_empty() {
|
||||
let mut storage_map = HashedStorage::new(false);
|
||||
|
||||
for slot_changes in &account_changes.storage_changes {
|
||||
let hashed_slot = keccak256(slot_changes.slot);
|
||||
|
||||
// Get the last change for this slot
|
||||
if let Some(last_change) = slot_changes.changes.last() {
|
||||
storage_map
|
||||
.storage
|
||||
.insert(hashed_slot, U256::from_be_bytes(last_change.new_value.0));
|
||||
}
|
||||
}
|
||||
|
||||
if !storage_map.storage.is_empty() {
|
||||
hashed_state.storages.insert(hashed_address, storage_map);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(hashed_state)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_eip7928::{
|
||||
AccountChanges, BalanceChange, CodeChange, NonceChange, SlotChanges, StorageChange,
|
||||
};
|
||||
use alloy_primitives::{Address, Bytes, StorageKey, B256};
|
||||
use reth_revm::test_utils::StateProviderTest;
|
||||
|
||||
#[test]
|
||||
fn test_bal_to_hashed_post_state_basic() {
|
||||
let provider = StateProviderTest::default();
|
||||
|
||||
let address = Address::random();
|
||||
let account_changes = AccountChanges {
|
||||
address,
|
||||
storage_changes: vec![],
|
||||
storage_reads: vec![],
|
||||
balance_changes: vec![BalanceChange::new(0, U256::from(100))],
|
||||
nonce_changes: vec![NonceChange::new(0, 1)],
|
||||
code_changes: vec![],
|
||||
};
|
||||
|
||||
let bal = vec![account_changes];
|
||||
let result = bal_to_hashed_post_state(&bal, &provider).unwrap();
|
||||
|
||||
assert_eq!(result.accounts.len(), 1);
|
||||
|
||||
let hashed_address = keccak256(address);
|
||||
let account_opt = result.accounts.get(&hashed_address).unwrap();
|
||||
assert!(account_opt.is_some());
|
||||
|
||||
let account = account_opt.as_ref().unwrap();
|
||||
assert_eq!(account.balance, U256::from(100));
|
||||
assert_eq!(account.nonce, 1);
|
||||
assert_eq!(account.bytecode_hash, Some(KECCAK_EMPTY));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bal_with_storage_changes() {
|
||||
let provider = StateProviderTest::default();
|
||||
|
||||
let address = Address::random();
|
||||
let slot = StorageKey::random();
|
||||
let value = B256::random();
|
||||
|
||||
let slot_changes = SlotChanges { slot, changes: vec![StorageChange::new(0, value)] };
|
||||
|
||||
let account_changes = AccountChanges {
|
||||
address,
|
||||
storage_changes: vec![slot_changes],
|
||||
storage_reads: vec![],
|
||||
balance_changes: vec![BalanceChange::new(0, U256::from(500))],
|
||||
nonce_changes: vec![NonceChange::new(0, 2)],
|
||||
code_changes: vec![],
|
||||
};
|
||||
|
||||
let bal = vec![account_changes];
|
||||
let result = bal_to_hashed_post_state(&bal, &provider).unwrap();
|
||||
|
||||
let hashed_address = keccak256(address);
|
||||
assert!(result.storages.contains_key(&hashed_address));
|
||||
|
||||
let storage = result.storages.get(&hashed_address).unwrap();
|
||||
let hashed_slot = keccak256(slot);
|
||||
|
||||
let stored_value = storage.storage.get(&hashed_slot).unwrap();
|
||||
assert_eq!(*stored_value, U256::from_be_bytes(value.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bal_with_code_change() {
|
||||
let provider = StateProviderTest::default();
|
||||
|
||||
let address = Address::random();
|
||||
let code = Bytes::from(vec![0x60, 0x80, 0x60, 0x40]); // Some bytecode
|
||||
|
||||
let account_changes = AccountChanges {
|
||||
address,
|
||||
storage_changes: vec![],
|
||||
storage_reads: vec![],
|
||||
balance_changes: vec![BalanceChange::new(0, U256::from(1000))],
|
||||
nonce_changes: vec![NonceChange::new(0, 1)],
|
||||
code_changes: vec![CodeChange::new(0, code.clone())],
|
||||
};
|
||||
|
||||
let bal = vec![account_changes];
|
||||
let result = bal_to_hashed_post_state(&bal, &provider).unwrap();
|
||||
|
||||
let hashed_address = keccak256(address);
|
||||
let account_opt = result.accounts.get(&hashed_address).unwrap();
|
||||
let account = account_opt.as_ref().unwrap();
|
||||
|
||||
let expected_code_hash = keccak256(&code);
|
||||
assert_eq!(account.bytecode_hash, Some(expected_code_hash));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bal_with_empty_code() {
|
||||
let provider = StateProviderTest::default();
|
||||
|
||||
let address = Address::random();
|
||||
let empty_code = Bytes::default();
|
||||
|
||||
let account_changes = AccountChanges {
|
||||
address,
|
||||
storage_changes: vec![],
|
||||
storage_reads: vec![],
|
||||
balance_changes: vec![BalanceChange::new(0, U256::from(1000))],
|
||||
nonce_changes: vec![NonceChange::new(0, 1)],
|
||||
code_changes: vec![CodeChange::new(0, empty_code)],
|
||||
};
|
||||
|
||||
let bal = vec![account_changes];
|
||||
let result = bal_to_hashed_post_state(&bal, &provider).unwrap();
|
||||
|
||||
let hashed_address = keccak256(address);
|
||||
let account_opt = result.accounts.get(&hashed_address).unwrap();
|
||||
let account = account_opt.as_ref().unwrap();
|
||||
|
||||
assert_eq!(account.bytecode_hash, Some(KECCAK_EMPTY));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bal_multiple_changes_takes_last() {
|
||||
let provider = StateProviderTest::default();
|
||||
|
||||
let address = Address::random();
|
||||
|
||||
// Multiple balance changes - should take the last one
|
||||
let account_changes = AccountChanges {
|
||||
address,
|
||||
storage_changes: vec![],
|
||||
storage_reads: vec![],
|
||||
balance_changes: vec![
|
||||
BalanceChange::new(0, U256::from(100)),
|
||||
BalanceChange::new(1, U256::from(200)),
|
||||
BalanceChange::new(2, U256::from(300)),
|
||||
],
|
||||
nonce_changes: vec![
|
||||
NonceChange::new(0, 1),
|
||||
NonceChange::new(1, 2),
|
||||
NonceChange::new(2, 3),
|
||||
],
|
||||
code_changes: vec![],
|
||||
};
|
||||
|
||||
let bal = vec![account_changes];
|
||||
let result = bal_to_hashed_post_state(&bal, &provider).unwrap();
|
||||
|
||||
let hashed_address = keccak256(address);
|
||||
let account_opt = result.accounts.get(&hashed_address).unwrap();
|
||||
let account = account_opt.as_ref().unwrap();
|
||||
|
||||
// Should have the last values
|
||||
assert_eq!(account.balance, U256::from(300));
|
||||
assert_eq!(account.nonce, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bal_uses_provider_for_missing_fields() {
|
||||
let mut provider = StateProviderTest::default();
|
||||
|
||||
let address = Address::random();
|
||||
let code_hash = B256::random();
|
||||
let existing_account =
|
||||
Account { balance: U256::from(999), nonce: 42, bytecode_hash: Some(code_hash) };
|
||||
provider.insert_account(address, existing_account, None, Default::default());
|
||||
|
||||
// Only change balance, nonce and code should come from provider
|
||||
let account_changes = AccountChanges {
|
||||
address,
|
||||
storage_changes: vec![],
|
||||
storage_reads: vec![],
|
||||
balance_changes: vec![BalanceChange::new(0, U256::from(1500))],
|
||||
nonce_changes: vec![],
|
||||
code_changes: vec![],
|
||||
};
|
||||
|
||||
let bal = vec![account_changes];
|
||||
let result = bal_to_hashed_post_state(&bal, &provider).unwrap();
|
||||
|
||||
let hashed_address = keccak256(address);
|
||||
let account_opt = result.accounts.get(&hashed_address).unwrap();
|
||||
let account = account_opt.as_ref().unwrap();
|
||||
|
||||
// Balance should be updated
|
||||
assert_eq!(account.balance, U256::from(1500));
|
||||
// Nonce and bytecode_hash should come from provider
|
||||
assert_eq!(account.nonce, 42);
|
||||
assert_eq!(account.bytecode_hash, Some(code_hash));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bal_multiple_storage_changes_per_slot() {
|
||||
let provider = StateProviderTest::default();
|
||||
|
||||
let address = Address::random();
|
||||
let slot = StorageKey::random();
|
||||
|
||||
// Multiple changes to the same slot - should take the last one
|
||||
let slot_changes = SlotChanges {
|
||||
slot,
|
||||
changes: vec![
|
||||
StorageChange::new(0, B256::from(U256::from(100).to_be_bytes::<32>())),
|
||||
StorageChange::new(1, B256::from(U256::from(200).to_be_bytes::<32>())),
|
||||
StorageChange::new(2, B256::from(U256::from(300).to_be_bytes::<32>())),
|
||||
],
|
||||
};
|
||||
|
||||
let account_changes = AccountChanges {
|
||||
address,
|
||||
storage_changes: vec![slot_changes],
|
||||
storage_reads: vec![],
|
||||
balance_changes: vec![BalanceChange::new(0, U256::from(100))],
|
||||
nonce_changes: vec![NonceChange::new(0, 1)],
|
||||
code_changes: vec![],
|
||||
};
|
||||
|
||||
let bal = vec![account_changes];
|
||||
let result = bal_to_hashed_post_state(&bal, &provider).unwrap();
|
||||
|
||||
let hashed_address = keccak256(address);
|
||||
let storage = result.storages.get(&hashed_address).unwrap();
|
||||
let hashed_slot = keccak256(slot);
|
||||
|
||||
let stored_value = storage.storage.get(&hashed_slot).unwrap();
|
||||
|
||||
// Should have the last value
|
||||
assert_eq!(*stored_value, U256::from(300));
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,6 @@ use crate::tree::{
|
||||
sparse_trie::SparseTrieTask,
|
||||
StateProviderBuilder, TreeConfig,
|
||||
};
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::eip1898::BlockWithParent;
|
||||
use alloy_evm::{block::StateChangeSource, ToTxEnv};
|
||||
use alloy_primitives::B256;
|
||||
@@ -23,12 +22,11 @@ use multiproof::{SparseTrieUpdate, *};
|
||||
use parking_lot::RwLock;
|
||||
use prewarm::PrewarmMetrics;
|
||||
use rayon::prelude::*;
|
||||
use reth_engine_primitives::ExecutableTxIterator;
|
||||
use reth_evm::{
|
||||
execute::{ExecutableTxFor, WithTxEnv},
|
||||
ConfigureEvm, EvmEnvFor, ExecutableTxIterator, ExecutableTxTuple, OnStateHook, SpecFor,
|
||||
TxEnvFor,
|
||||
ConfigureEvm, EvmEnvFor, OnStateHook, SpecFor, TxEnvFor,
|
||||
};
|
||||
use reth_execution_types::ExecutionOutcome;
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_provider::{BlockReader, DatabaseProviderROFactory, StateProviderFactory, StateReader};
|
||||
use reth_revm::{db::BundleState, state::EvmState};
|
||||
@@ -51,9 +49,8 @@ use std::{
|
||||
},
|
||||
time::Instant,
|
||||
};
|
||||
use tracing::{debug, debug_span, error, instrument, warn, Span};
|
||||
use tracing::{debug, debug_span, instrument, warn, Span};
|
||||
|
||||
pub mod bal;
|
||||
mod configured_sparse_trie;
|
||||
pub mod executor;
|
||||
pub mod multiproof;
|
||||
@@ -93,13 +90,6 @@ pub const SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY: usize = 1_000_000;
|
||||
/// 144MB.
|
||||
pub const SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY: usize = 1_000_000;
|
||||
|
||||
/// Type alias for [`PayloadHandle`] returned by payload processor spawn methods.
|
||||
type IteratorPayloadHandle<Evm, I, N> = PayloadHandle<
|
||||
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxTuple>::Tx>,
|
||||
<I as ExecutableTxTuple>::Error,
|
||||
<N as NodePrimitives>::Receipt,
|
||||
>;
|
||||
|
||||
/// Entrypoint for executing the payload.
|
||||
#[derive(Debug)]
|
||||
pub struct PayloadProcessor<Evm>
|
||||
@@ -208,6 +198,7 @@ where
|
||||
///
|
||||
/// This returns a handle to await the final state root and to interact with the tasks (e.g.
|
||||
/// canceling)
|
||||
#[allow(clippy::type_complexity)]
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor",
|
||||
@@ -221,8 +212,7 @@ where
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
multiproof_provider_factory: F,
|
||||
config: &TreeConfig,
|
||||
bal: Option<Arc<BlockAccessList>>,
|
||||
) -> IteratorPayloadHandle<Evm, I, N>
|
||||
) -> PayloadHandle<WithTxEnv<TxEnvFor<Evm>, I::Tx>, I::Error>
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
F: DatabaseProviderROFactory<Provider: TrieCursorFactory + HashedCursorFactory>
|
||||
@@ -262,45 +252,19 @@ where
|
||||
// wire the multiproof task to the prewarm task
|
||||
let to_multi_proof = Some(multi_proof_task.state_root_message_sender());
|
||||
|
||||
// Handle BAL-based optimization if available
|
||||
let prewarm_handle = if let Some(bal) = bal {
|
||||
// When BAL is present, skip spawning prewarm tasks entirely and send BAL to multiproof
|
||||
debug!(target: "engine::tree::payload_processor", "BAL present, skipping prewarm tasks");
|
||||
|
||||
// Send BAL message immediately to MultiProofTask
|
||||
if let Some(ref sender) = to_multi_proof &&
|
||||
let Err(err) = sender.send(MultiProofMessage::BlockAccessList(bal))
|
||||
{
|
||||
// In this case state root validation will simply fail
|
||||
error!(target: "engine::tree::payload_processor", ?err, "Failed to send BAL to MultiProofTask");
|
||||
}
|
||||
|
||||
// Spawn minimal cache-only task without prewarming
|
||||
self.spawn_caching_with(
|
||||
env,
|
||||
prewarm_rx,
|
||||
transaction_count_hint,
|
||||
provider_builder.clone(),
|
||||
None, // Don't send proof targets when BAL is present
|
||||
)
|
||||
} else {
|
||||
// Normal path: spawn with full prewarming
|
||||
self.spawn_caching_with(
|
||||
env,
|
||||
prewarm_rx,
|
||||
transaction_count_hint,
|
||||
provider_builder.clone(),
|
||||
to_multi_proof.clone(),
|
||||
)
|
||||
};
|
||||
let prewarm_handle = self.spawn_caching_with(
|
||||
env,
|
||||
prewarm_rx,
|
||||
transaction_count_hint,
|
||||
provider_builder,
|
||||
to_multi_proof.clone(),
|
||||
);
|
||||
|
||||
// spawn multi-proof task
|
||||
let parent_span = span.clone();
|
||||
self.executor.spawn_blocking(move || {
|
||||
let _enter = parent_span.entered();
|
||||
// Build a state provider for the multiproof task
|
||||
let provider = provider_builder.build().expect("failed to build provider");
|
||||
multi_proof_task.run(provider);
|
||||
multi_proof_task.run();
|
||||
});
|
||||
|
||||
// wire the sparse trie to the state root response receiver
|
||||
@@ -327,7 +291,7 @@ where
|
||||
env: ExecutionEnv<Evm>,
|
||||
transactions: I,
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
) -> IteratorPayloadHandle<Evm, I, N>
|
||||
) -> PayloadHandle<WithTxEnv<TxEnvFor<Evm>, I::Tx>, I::Error>
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
{
|
||||
@@ -407,7 +371,7 @@ where
|
||||
transaction_count_hint: usize,
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
) -> CacheTaskHandle<N::Receipt>
|
||||
) -> CacheTaskHandle
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
{
|
||||
@@ -588,15 +552,12 @@ where
|
||||
}
|
||||
|
||||
/// Handle to all the spawned tasks.
|
||||
///
|
||||
/// Generic over `R` (receipt type) to allow sharing `Arc<ExecutionOutcome<R>>` with the
|
||||
/// caching task without cloning the expensive `BundleState`.
|
||||
#[derive(Debug)]
|
||||
pub struct PayloadHandle<Tx, Err, R> {
|
||||
pub struct PayloadHandle<Tx, Err> {
|
||||
/// Channel for evm state updates
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
// must include the receiver of the state root wired to the sparse trie
|
||||
prewarm_handle: CacheTaskHandle<R>,
|
||||
prewarm_handle: CacheTaskHandle,
|
||||
/// Stream of block transactions
|
||||
transactions: mpsc::Receiver<Result<Tx, Err>>,
|
||||
/// Receiver for the state root
|
||||
@@ -605,7 +566,7 @@ pub struct PayloadHandle<Tx, Err, R> {
|
||||
_span: Span,
|
||||
}
|
||||
|
||||
impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
|
||||
impl<Tx, Err> PayloadHandle<Tx, Err> {
|
||||
/// Awaits the state root
|
||||
///
|
||||
/// # Panics
|
||||
@@ -634,7 +595,7 @@ impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
|
||||
|
||||
move |source: StateChangeSource, state: &EvmState| {
|
||||
if let Some(sender) = &to_multi_proof {
|
||||
let _ = sender.send(MultiProofMessage::StateUpdate(source.into(), state.clone()));
|
||||
let _ = sender.send(MultiProofMessage::StateUpdate(source, state.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -658,14 +619,9 @@ impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
|
||||
|
||||
/// Terminates the entire caching task.
|
||||
///
|
||||
/// If the [`ExecutionOutcome`] is provided it will update the shared cache using its
|
||||
/// bundle state. Using `Arc<ExecutionOutcome>` allows sharing with the main execution
|
||||
/// path without cloning the expensive `BundleState`.
|
||||
pub(super) fn terminate_caching(
|
||||
&mut self,
|
||||
execution_outcome: Option<Arc<ExecutionOutcome<R>>>,
|
||||
) {
|
||||
self.prewarm_handle.terminate_caching(execution_outcome)
|
||||
/// If the [`BundleState`] is provided it will update the shared cache.
|
||||
pub(super) fn terminate_caching(&mut self, block_output: Option<&BundleState>) {
|
||||
self.prewarm_handle.terminate_caching(block_output)
|
||||
}
|
||||
|
||||
/// Returns iterator yielding transactions from the stream.
|
||||
@@ -677,20 +633,17 @@ impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
|
||||
}
|
||||
|
||||
/// Access to the spawned [`PrewarmCacheTask`].
|
||||
///
|
||||
/// Generic over `R` (receipt type) to allow sharing `Arc<ExecutionOutcome<R>>` with the
|
||||
/// prewarm task without cloning the expensive `BundleState`.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct CacheTaskHandle<R> {
|
||||
pub(crate) struct CacheTaskHandle {
|
||||
/// The shared cache the task operates with.
|
||||
cache: Option<StateExecutionCache>,
|
||||
/// Metrics for the caches
|
||||
cache_metrics: Option<CachedStateMetrics>,
|
||||
/// Channel to the spawned prewarm task if any
|
||||
to_prewarm_task: Option<std::sync::mpsc::Sender<PrewarmTaskEvent<R>>>,
|
||||
to_prewarm_task: Option<std::sync::mpsc::Sender<PrewarmTaskEvent>>,
|
||||
}
|
||||
|
||||
impl<R: Send + Sync + 'static> CacheTaskHandle<R> {
|
||||
impl CacheTaskHandle {
|
||||
/// Terminates the pre-warming transaction processing.
|
||||
///
|
||||
/// Note: This does not terminate the task yet.
|
||||
@@ -702,25 +655,20 @@ impl<R: Send + Sync + 'static> CacheTaskHandle<R> {
|
||||
|
||||
/// Terminates the entire pre-warming task.
|
||||
///
|
||||
/// If the [`ExecutionOutcome`] is provided it will update the shared cache using its
|
||||
/// bundle state. Using `Arc<ExecutionOutcome>` avoids cloning the expensive `BundleState`.
|
||||
pub(super) fn terminate_caching(
|
||||
&mut self,
|
||||
execution_outcome: Option<Arc<ExecutionOutcome<R>>>,
|
||||
) {
|
||||
/// If the [`BundleState`] is provided it will update the shared cache.
|
||||
pub(super) fn terminate_caching(&mut self, block_output: Option<&BundleState>) {
|
||||
if let Some(tx) = self.to_prewarm_task.take() {
|
||||
let event = PrewarmTaskEvent::Terminate { execution_outcome };
|
||||
// Only clone when we have an active task and a state to send
|
||||
let event = PrewarmTaskEvent::Terminate { block_output: block_output.cloned() };
|
||||
let _ = tx.send(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<R> Drop for CacheTaskHandle<R> {
|
||||
impl Drop for CacheTaskHandle {
|
||||
fn drop(&mut self) {
|
||||
// Ensure we always terminate on drop - send None without needing Send + Sync bounds
|
||||
if let Some(tx) = self.to_prewarm_task.take() {
|
||||
let _ = tx.send(PrewarmTaskEvent::Terminate { execution_outcome: None });
|
||||
}
|
||||
// Ensure we always terminate on drop
|
||||
self.terminate_caching(None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -773,8 +721,6 @@ impl ExecutionCache {
|
||||
|
||||
cache
|
||||
.as_ref()
|
||||
// Check `is_available()` to ensure no other tasks (e.g., prewarming) currently hold
|
||||
// a reference to this cache. We can only reuse it when we have exclusive access.
|
||||
.filter(|c| c.executed_block_hash() == parent_hash && c.is_available())
|
||||
.cloned()
|
||||
}
|
||||
@@ -1116,7 +1062,6 @@ mod tests {
|
||||
StateProviderBuilder::new(provider_factory.clone(), genesis_hash, None),
|
||||
OverlayStateProviderFactory::new(provider_factory),
|
||||
&TreeConfig::default(),
|
||||
None, // No BAL for test
|
||||
);
|
||||
|
||||
let mut state_hook = handle.state_hook();
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
//! Multiproof task related functionality.
|
||||
|
||||
use crate::tree::payload_processor::bal::bal_to_hashed_post_state;
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use alloy_evm::block::StateChangeSource;
|
||||
use alloy_primitives::{
|
||||
keccak256,
|
||||
@@ -13,7 +11,6 @@ use dashmap::DashMap;
|
||||
use derive_more::derive::Deref;
|
||||
use metrics::{Gauge, Histogram};
|
||||
use reth_metrics::Metrics;
|
||||
use reth_provider::AccountReader;
|
||||
use reth_revm::state::EvmState;
|
||||
use reth_trie::{
|
||||
added_removed_keys::MultiAddedRemovedKeys, DecodedMultiProof, HashedPostState, HashedStorage,
|
||||
@@ -29,30 +26,6 @@ use reth_trie_parallel::{
|
||||
use std::{collections::BTreeMap, mem, ops::DerefMut, sync::Arc, time::Instant};
|
||||
use tracing::{debug, error, instrument, trace};
|
||||
|
||||
/// Source of state changes, either from EVM execution or from a Block Access List.
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum Source {
|
||||
/// State changes from EVM execution.
|
||||
Evm(StateChangeSource),
|
||||
/// State changes from Block Access List (EIP-7928).
|
||||
BlockAccessList,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Source {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Evm(source) => source.fmt(f),
|
||||
Self::BlockAccessList => f.write_str("BlockAccessList"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<StateChangeSource> for Source {
|
||||
fn from(source: StateChangeSource) -> Self {
|
||||
Self::Evm(source)
|
||||
}
|
||||
}
|
||||
|
||||
/// Maximum number of targets to batch together for prefetch batching.
|
||||
/// Prefetches are just proof requests (no state merging), so we allow a higher cap than state
|
||||
/// updates
|
||||
@@ -109,7 +82,7 @@ pub(super) enum MultiProofMessage {
|
||||
/// Prefetch proof targets
|
||||
PrefetchProofs(MultiProofTargets),
|
||||
/// New state update from transaction execution with its source
|
||||
StateUpdate(Source, EvmState),
|
||||
StateUpdate(StateChangeSource, EvmState),
|
||||
/// State update that can be applied to the sparse trie without any new proofs.
|
||||
///
|
||||
/// It can be the case when all accounts and storage slots from the state update were already
|
||||
@@ -120,11 +93,6 @@ pub(super) enum MultiProofMessage {
|
||||
/// The state update that was used to calculate the proof
|
||||
state: HashedPostState,
|
||||
},
|
||||
/// Block Access List (EIP-7928; BAL) containing complete state changes for the block.
|
||||
///
|
||||
/// When received, the task generates a single state update from the BAL and processes it.
|
||||
/// No further messages are expected after receiving this variant.
|
||||
BlockAccessList(Arc<BlockAccessList>),
|
||||
/// Signals state update stream end.
|
||||
///
|
||||
/// This is triggered by block execution, indicating that no additional state updates are
|
||||
@@ -170,6 +138,11 @@ impl ProofSequencer {
|
||||
while let Some(pending) = self.pending_proofs.remove(¤t_sequence) {
|
||||
consecutive_proofs.push(pending);
|
||||
current_sequence += 1;
|
||||
|
||||
// if we don't have the next number, stop collecting
|
||||
if !self.pending_proofs.contains_key(¤t_sequence) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
self.next_to_deliver += consecutive_proofs.len() as u64;
|
||||
@@ -307,7 +280,7 @@ impl StorageMultiproofInput {
|
||||
/// Input parameters for dispatching a multiproof calculation.
|
||||
#[derive(Debug)]
|
||||
struct MultiproofInput {
|
||||
source: Option<Source>,
|
||||
source: Option<StateChangeSource>,
|
||||
hashed_state_update: HashedPostState,
|
||||
proof_targets: MultiProofTargets,
|
||||
proof_sequence_number: u64,
|
||||
@@ -910,19 +883,9 @@ impl MultiProofTask {
|
||||
skip(self, update),
|
||||
fields(accounts = update.len(), chunks = 0)
|
||||
)]
|
||||
fn on_state_update(&mut self, source: Source, update: EvmState) -> u64 {
|
||||
fn on_state_update(&mut self, source: StateChangeSource, update: EvmState) -> u64 {
|
||||
let hashed_state_update = evm_state_to_hashed_post_state(update);
|
||||
self.on_hashed_state_update(source, hashed_state_update)
|
||||
}
|
||||
|
||||
/// Processes a hashed state update and dispatches multiproofs as needed.
|
||||
///
|
||||
/// Returns the number of state updates dispatched (both `EmptyProof` and regular multiproofs).
|
||||
fn on_hashed_state_update(
|
||||
&mut self,
|
||||
source: Source,
|
||||
hashed_state_update: HashedPostState,
|
||||
) -> u64 {
|
||||
// Update removed keys based on the state update.
|
||||
self.multi_added_removed_keys.update_with_state(&hashed_state_update);
|
||||
|
||||
@@ -1019,16 +982,12 @@ impl MultiProofTask {
|
||||
/// This preserves ordering without requeuing onto the channel.
|
||||
///
|
||||
/// Returns `true` if done, `false` to continue.
|
||||
fn process_multiproof_message<P>(
|
||||
fn process_multiproof_message(
|
||||
&mut self,
|
||||
msg: MultiProofMessage,
|
||||
ctx: &mut MultiproofBatchCtx,
|
||||
batch_metrics: &mut MultiproofBatchMetrics,
|
||||
provider: &P,
|
||||
) -> bool
|
||||
where
|
||||
P: AccountReader,
|
||||
{
|
||||
) -> bool {
|
||||
match msg {
|
||||
// Prefetch proofs: batch consecutive prefetch requests up to target/message limits
|
||||
MultiProofMessage::PrefetchProofs(targets) => {
|
||||
@@ -1187,56 +1146,6 @@ impl MultiProofTask {
|
||||
|
||||
false
|
||||
}
|
||||
// Process Block Access List (BAL) - complete state changes provided upfront
|
||||
MultiProofMessage::BlockAccessList(bal) => {
|
||||
trace!(target: "engine::tree::payload_processor::multiproof", "processing MultiProofMessage::BAL");
|
||||
|
||||
if ctx.first_update_time.is_none() {
|
||||
self.metrics
|
||||
.first_update_wait_time_histogram
|
||||
.record(ctx.start.elapsed().as_secs_f64());
|
||||
ctx.first_update_time = Some(Instant::now());
|
||||
debug!(target: "engine::tree::payload_processor::multiproof", "Started state root calculation from BAL");
|
||||
}
|
||||
|
||||
// Convert BAL to HashedPostState and process it
|
||||
match bal_to_hashed_post_state(&bal, &provider) {
|
||||
Ok(hashed_state) => {
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor::multiproof",
|
||||
accounts = hashed_state.accounts.len(),
|
||||
storages = hashed_state.storages.len(),
|
||||
"Processing BAL state update"
|
||||
);
|
||||
|
||||
// Use BlockAccessList as source for BAL-derived state updates
|
||||
batch_metrics.state_update_proofs_requested +=
|
||||
self.on_hashed_state_update(Source::BlockAccessList, hashed_state);
|
||||
}
|
||||
Err(err) => {
|
||||
error!(target: "engine::tree::payload_processor::multiproof", ?err, "Failed to convert BAL to hashed state");
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Mark updates as finished since BAL provides complete state
|
||||
ctx.updates_finished_time = Some(Instant::now());
|
||||
|
||||
// Check if we're done (might need to wait for proofs to complete)
|
||||
if self.is_done(
|
||||
batch_metrics.proofs_processed,
|
||||
batch_metrics.state_update_proofs_requested,
|
||||
batch_metrics.prefetch_proofs_requested,
|
||||
ctx.updates_finished(),
|
||||
) {
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor::multiproof",
|
||||
"BAL processed and all proofs complete, ending calculation"
|
||||
);
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
// Signal that no more state updates will arrive
|
||||
MultiProofMessage::FinishedStateUpdates => {
|
||||
trace!(target: "engine::tree::payload_processor::multiproof", "processing MultiProofMessage::FinishedStateUpdates");
|
||||
@@ -1329,10 +1238,7 @@ impl MultiProofTask {
|
||||
target = "engine::tree::payload_processor::multiproof",
|
||||
skip_all
|
||||
)]
|
||||
pub(crate) fn run<P>(mut self, provider: P)
|
||||
where
|
||||
P: AccountReader,
|
||||
{
|
||||
pub(crate) fn run(mut self) {
|
||||
let mut ctx = MultiproofBatchCtx::new(Instant::now());
|
||||
let mut batch_metrics = MultiproofBatchMetrics::default();
|
||||
|
||||
@@ -1342,7 +1248,7 @@ impl MultiProofTask {
|
||||
trace!(target: "engine::tree::payload_processor::multiproof", "entering main channel receiving loop");
|
||||
|
||||
if let Some(msg) = ctx.pending_msg.take() {
|
||||
if self.process_multiproof_message(msg, &mut ctx, &mut batch_metrics, &provider) {
|
||||
if self.process_multiproof_message(msg, &mut ctx, &mut batch_metrics) {
|
||||
break 'main;
|
||||
}
|
||||
continue;
|
||||
@@ -1417,7 +1323,7 @@ impl MultiProofTask {
|
||||
}
|
||||
};
|
||||
|
||||
if self.process_multiproof_message(msg, &mut ctx, &mut batch_metrics, &provider) {
|
||||
if self.process_multiproof_message(msg, &mut ctx, &mut batch_metrics) {
|
||||
break 'main;
|
||||
}
|
||||
}
|
||||
@@ -1453,9 +1359,6 @@ impl MultiProofTask {
|
||||
/// Context for multiproof message batching loop.
|
||||
///
|
||||
/// Contains processing state that persists across loop iterations.
|
||||
///
|
||||
/// Used by `process_multiproof_message` to batch consecutive same-type messages received via
|
||||
/// `try_recv` for efficient processing.
|
||||
struct MultiproofBatchCtx {
|
||||
/// Buffers a non-matching message type encountered during batching.
|
||||
/// Processed first in next iteration to preserve ordering while allowing same-type
|
||||
@@ -1471,7 +1374,7 @@ struct MultiproofBatchCtx {
|
||||
/// Reusable buffer for accumulating prefetch targets during batching.
|
||||
accumulated_prefetch_targets: Vec<MultiProofTargets>,
|
||||
/// Reusable buffer for accumulating state updates during batching.
|
||||
accumulated_state_updates: Vec<(Source, EvmState)>,
|
||||
accumulated_state_updates: Vec<(StateChangeSource, EvmState)>,
|
||||
}
|
||||
|
||||
impl MultiproofBatchCtx {
|
||||
@@ -1589,44 +1492,34 @@ where
|
||||
/// are safe to merge because they originate from the same logical execution and can be
|
||||
/// coalesced to amortize proof work.
|
||||
fn can_batch_state_update(
|
||||
batch_source: Source,
|
||||
batch_source: StateChangeSource,
|
||||
batch_update: &EvmState,
|
||||
next_source: Source,
|
||||
next_source: StateChangeSource,
|
||||
next_update: &EvmState,
|
||||
) -> bool {
|
||||
if !same_source(batch_source, next_source) {
|
||||
if !same_state_change_source(batch_source, next_source) {
|
||||
return false;
|
||||
}
|
||||
|
||||
match (batch_source, next_source) {
|
||||
(
|
||||
Source::Evm(StateChangeSource::PreBlock(_)),
|
||||
Source::Evm(StateChangeSource::PreBlock(_)),
|
||||
) |
|
||||
(
|
||||
Source::Evm(StateChangeSource::PostBlock(_)),
|
||||
Source::Evm(StateChangeSource::PostBlock(_)),
|
||||
) => batch_update == next_update,
|
||||
(StateChangeSource::PreBlock(_), StateChangeSource::PreBlock(_)) |
|
||||
(StateChangeSource::PostBlock(_), StateChangeSource::PostBlock(_)) => {
|
||||
batch_update == next_update
|
||||
}
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks whether two sources refer to the same origin.
|
||||
fn same_source(lhs: Source, rhs: Source) -> bool {
|
||||
/// Checks whether two state change sources refer to the same origin.
|
||||
fn same_state_change_source(lhs: StateChangeSource, rhs: StateChangeSource) -> bool {
|
||||
match (lhs, rhs) {
|
||||
(
|
||||
Source::Evm(StateChangeSource::Transaction(a)),
|
||||
Source::Evm(StateChangeSource::Transaction(b)),
|
||||
) => a == b,
|
||||
(
|
||||
Source::Evm(StateChangeSource::PreBlock(a)),
|
||||
Source::Evm(StateChangeSource::PreBlock(b)),
|
||||
) => mem::discriminant(&a) == mem::discriminant(&b),
|
||||
(
|
||||
Source::Evm(StateChangeSource::PostBlock(a)),
|
||||
Source::Evm(StateChangeSource::PostBlock(b)),
|
||||
) => mem::discriminant(&a) == mem::discriminant(&b),
|
||||
(Source::BlockAccessList, Source::BlockAccessList) => true,
|
||||
(StateChangeSource::Transaction(a), StateChangeSource::Transaction(b)) => a == b,
|
||||
(StateChangeSource::PreBlock(a), StateChangeSource::PreBlock(b)) => {
|
||||
mem::discriminant(&a) == mem::discriminant(&b)
|
||||
}
|
||||
(StateChangeSource::PostBlock(a), StateChangeSource::PostBlock(b)) => {
|
||||
mem::discriminant(&a) == mem::discriminant(&b)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -1646,8 +1539,7 @@ fn estimate_evm_state_targets(state: &EvmState) -> usize {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_eip7928::{AccountChanges, BalanceChange};
|
||||
use alloy_primitives::{map::B256Set, Address};
|
||||
use alloy_primitives::map::B256Set;
|
||||
use reth_provider::{
|
||||
providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory,
|
||||
BlockReader, DatabaseProviderFactory, PruneCheckpointReader, StageCheckpointReader,
|
||||
@@ -1656,7 +1548,7 @@ mod tests {
|
||||
use reth_trie::MultiProof;
|
||||
use reth_trie_parallel::proof_task::{ProofTaskCtx, ProofWorkerHandle};
|
||||
use revm_primitives::{B256, U256};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::sync::OnceLock;
|
||||
use tokio::runtime::{Handle, Runtime};
|
||||
|
||||
/// Get a handle to the test runtime, creating it if necessary
|
||||
@@ -2217,8 +2109,8 @@ mod tests {
|
||||
let source = StateChangeSource::Transaction(0);
|
||||
|
||||
let tx = task.state_root_message_sender();
|
||||
tx.send(MultiProofMessage::StateUpdate(source.into(), update1.clone())).unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source.into(), update2.clone())).unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source, update1.clone())).unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source, update2.clone())).unwrap();
|
||||
|
||||
let proofs_requested =
|
||||
if let Ok(MultiProofMessage::StateUpdate(_src, update)) = task.rx.recv() {
|
||||
@@ -2237,7 +2129,7 @@ mod tests {
|
||||
assert!(merged_update.contains_key(&addr1));
|
||||
assert!(merged_update.contains_key(&addr2));
|
||||
|
||||
task.on_state_update(source.into(), merged_update)
|
||||
task.on_state_update(source, merged_update)
|
||||
} else {
|
||||
panic!("Expected StateUpdate message");
|
||||
};
|
||||
@@ -2281,20 +2173,20 @@ mod tests {
|
||||
|
||||
// Queue: A1 (immediate dispatch), B1 (batched), A2 (should become pending)
|
||||
let tx = task.state_root_message_sender();
|
||||
tx.send(MultiProofMessage::StateUpdate(source_a.into(), create_state_update(addr_a1, 100)))
|
||||
tx.send(MultiProofMessage::StateUpdate(source_a, create_state_update(addr_a1, 100)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source_b.into(), create_state_update(addr_b1, 200)))
|
||||
tx.send(MultiProofMessage::StateUpdate(source_b, create_state_update(addr_b1, 200)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source_a.into(), create_state_update(addr_a2, 300)))
|
||||
tx.send(MultiProofMessage::StateUpdate(source_a, create_state_update(addr_a2, 300)))
|
||||
.unwrap();
|
||||
|
||||
let mut pending_msg: Option<MultiProofMessage> = None;
|
||||
|
||||
if let Ok(MultiProofMessage::StateUpdate(first_source, _)) = task.rx.recv() {
|
||||
assert!(same_source(first_source, source_a.into()));
|
||||
assert!(same_state_change_source(first_source, source_a));
|
||||
|
||||
// Simulate batching loop for remaining messages
|
||||
let mut accumulated_updates: Vec<(Source, EvmState)> = Vec::new();
|
||||
let mut accumulated_updates: Vec<(StateChangeSource, EvmState)> = Vec::new();
|
||||
let mut accumulated_targets = 0usize;
|
||||
|
||||
loop {
|
||||
@@ -2342,7 +2234,7 @@ mod tests {
|
||||
|
||||
assert_eq!(accumulated_updates.len(), 1, "Should only batch matching sources");
|
||||
let batch_source = accumulated_updates[0].0;
|
||||
assert!(same_source(batch_source, source_b.into()));
|
||||
assert!(same_state_change_source(batch_source, source_b));
|
||||
|
||||
let batch_source = accumulated_updates[0].0;
|
||||
let mut merged_update = accumulated_updates.remove(0).1;
|
||||
@@ -2350,7 +2242,10 @@ mod tests {
|
||||
merged_update.extend(next_update);
|
||||
}
|
||||
|
||||
assert!(same_source(batch_source, source_b.into()), "Batch should use matching source");
|
||||
assert!(
|
||||
same_state_change_source(batch_source, source_b),
|
||||
"Batch should use matching source"
|
||||
);
|
||||
assert!(merged_update.contains_key(&addr_b1));
|
||||
assert!(!merged_update.contains_key(&addr_a1));
|
||||
assert!(!merged_update.contains_key(&addr_a2));
|
||||
@@ -2360,7 +2255,7 @@ mod tests {
|
||||
|
||||
match pending_msg {
|
||||
Some(MultiProofMessage::StateUpdate(pending_source, pending_update)) => {
|
||||
assert!(same_source(pending_source, source_a.into()));
|
||||
assert!(same_state_change_source(pending_source, source_a));
|
||||
assert!(pending_update.contains_key(&addr_a2));
|
||||
}
|
||||
other => panic!("Expected pending StateUpdate with source_a, got {:?}", other),
|
||||
@@ -2403,20 +2298,17 @@ mod tests {
|
||||
|
||||
// Queue: first update dispatched immediately, next two should not merge
|
||||
let tx = task.state_root_message_sender();
|
||||
tx.send(MultiProofMessage::StateUpdate(source.into(), create_state_update(addr1, 100)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source.into(), create_state_update(addr2, 200)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source.into(), create_state_update(addr3, 300)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source, create_state_update(addr1, 100))).unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source, create_state_update(addr2, 200))).unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source, create_state_update(addr3, 300))).unwrap();
|
||||
|
||||
let mut pending_msg: Option<MultiProofMessage> = None;
|
||||
|
||||
if let Ok(MultiProofMessage::StateUpdate(first_source, first_update)) = task.rx.recv() {
|
||||
assert!(same_source(first_source, source.into()));
|
||||
assert!(same_state_change_source(first_source, source));
|
||||
assert!(first_update.contains_key(&addr1));
|
||||
|
||||
let mut accumulated_updates: Vec<(Source, EvmState)> = Vec::new();
|
||||
let mut accumulated_updates: Vec<(StateChangeSource, EvmState)> = Vec::new();
|
||||
let mut accumulated_targets = 0usize;
|
||||
|
||||
loop {
|
||||
@@ -2468,7 +2360,7 @@ mod tests {
|
||||
"Second pre-block update should not merge with a different payload"
|
||||
);
|
||||
let (batched_source, batched_update) = accumulated_updates.remove(0);
|
||||
assert!(same_source(batched_source, source.into()));
|
||||
assert!(same_state_change_source(batched_source, source));
|
||||
assert!(batched_update.contains_key(&addr2));
|
||||
assert!(!batched_update.contains_key(&addr3));
|
||||
|
||||
@@ -2548,8 +2440,8 @@ mod tests {
|
||||
let tx = task.state_root_message_sender();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(targets1)).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(targets2)).unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update1)).unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update2)).unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source, state_update1)).unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source, state_update2)).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(targets3.clone())).unwrap();
|
||||
|
||||
// Step 1: Receive and batch PrefetchProofs (should get targets1 + targets2)
|
||||
@@ -2616,7 +2508,6 @@ mod tests {
|
||||
use revm_state::Account;
|
||||
|
||||
let test_provider_factory = create_test_provider_factory();
|
||||
let test_provider = test_provider_factory.latest().unwrap();
|
||||
let mut task = create_test_state_root_task(test_provider_factory);
|
||||
|
||||
// Queue: Prefetch1, StateUpdate, Prefetch2
|
||||
@@ -2648,7 +2539,7 @@ mod tests {
|
||||
|
||||
let tx = task.state_root_message_sender();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(prefetch1)).unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update)).unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source, state_update)).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(prefetch2.clone())).unwrap();
|
||||
|
||||
let mut ctx = MultiproofBatchCtx::new(Instant::now());
|
||||
@@ -2657,22 +2548,12 @@ mod tests {
|
||||
// First message: Prefetch1 batches; StateUpdate becomes pending.
|
||||
let first = task.rx.recv().unwrap();
|
||||
assert!(matches!(first, MultiProofMessage::PrefetchProofs(_)));
|
||||
assert!(!task.process_multiproof_message(
|
||||
first,
|
||||
&mut ctx,
|
||||
&mut batch_metrics,
|
||||
&test_provider
|
||||
));
|
||||
assert!(!task.process_multiproof_message(first, &mut ctx, &mut batch_metrics));
|
||||
let pending = ctx.pending_msg.take().expect("pending message captured");
|
||||
assert!(matches!(pending, MultiProofMessage::StateUpdate(_, _)));
|
||||
|
||||
// Pending message should be handled before the next select loop.
|
||||
assert!(!task.process_multiproof_message(
|
||||
pending,
|
||||
&mut ctx,
|
||||
&mut batch_metrics,
|
||||
&test_provider
|
||||
));
|
||||
assert!(!task.process_multiproof_message(pending, &mut ctx, &mut batch_metrics));
|
||||
|
||||
// Prefetch2 should now be in pending_msg (captured by StateUpdate's batching loop).
|
||||
match ctx.pending_msg.take() {
|
||||
@@ -2744,21 +2625,12 @@ mod tests {
|
||||
// Queue: [Prefetch1, State1, State2, State3, Prefetch2]
|
||||
let tx = task.state_root_message_sender();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(prefetch1.clone())).unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(
|
||||
source.into(),
|
||||
create_state_update(state_addr1, 100),
|
||||
))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(
|
||||
source.into(),
|
||||
create_state_update(state_addr2, 200),
|
||||
))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(
|
||||
source.into(),
|
||||
create_state_update(state_addr3, 300),
|
||||
))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source, create_state_update(state_addr1, 100)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source, create_state_update(state_addr2, 200)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source, create_state_update(state_addr3, 300)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(prefetch2.clone())).unwrap();
|
||||
|
||||
// Simulate the state-machine loop behavior
|
||||
@@ -2831,44 +2703,4 @@ mod tests {
|
||||
_ => panic!("Prefetch2 was lost!"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Verifies that BAL messages are processed correctly and generate state updates.
|
||||
#[test]
|
||||
fn test_bal_message_processing() {
|
||||
let test_provider_factory = create_test_provider_factory();
|
||||
let test_provider = test_provider_factory.latest().unwrap();
|
||||
let mut task = create_test_state_root_task(test_provider_factory);
|
||||
|
||||
// Create a simple BAL with one account change
|
||||
let account_address = Address::random();
|
||||
let account_changes = AccountChanges {
|
||||
address: account_address,
|
||||
balance_changes: vec![BalanceChange::new(0, U256::from(1000))],
|
||||
nonce_changes: vec![],
|
||||
code_changes: vec![],
|
||||
storage_changes: vec![],
|
||||
storage_reads: vec![],
|
||||
};
|
||||
|
||||
let bal = Arc::new(vec![account_changes]);
|
||||
|
||||
let mut ctx = MultiproofBatchCtx::new(Instant::now());
|
||||
let mut batch_metrics = MultiproofBatchMetrics::default();
|
||||
|
||||
let should_finish = task.process_multiproof_message(
|
||||
MultiProofMessage::BlockAccessList(bal),
|
||||
&mut ctx,
|
||||
&mut batch_metrics,
|
||||
&test_provider,
|
||||
);
|
||||
|
||||
// BAL should mark updates as finished
|
||||
assert!(ctx.updates_finished_time.is_some());
|
||||
|
||||
// Should have dispatched state update proofs
|
||||
assert!(batch_metrics.state_update_proofs_requested > 0);
|
||||
|
||||
// Should need to wait for the results of those proofs to arrive
|
||||
assert!(!should_finish, "Should continue waiting for proofs");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,11 +27,10 @@ use alloy_primitives::{keccak256, map::B256Set, B256};
|
||||
use crossbeam_channel::Sender as CrossbeamSender;
|
||||
use metrics::{Counter, Gauge, Histogram};
|
||||
use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, SpecFor};
|
||||
use reth_execution_types::ExecutionOutcome;
|
||||
use reth_metrics::Metrics;
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_provider::{BlockReader, StateProviderFactory, StateReader};
|
||||
use reth_revm::{database::StateProviderDatabase, state::EvmState};
|
||||
use reth_provider::{BlockReader, StateProviderBox, StateProviderFactory, StateReader};
|
||||
use reth_revm::{database::StateProviderDatabase, db::BundleState, state::EvmState};
|
||||
use reth_trie::MultiProofTargets;
|
||||
use std::{
|
||||
sync::{
|
||||
@@ -87,7 +86,7 @@ where
|
||||
/// Sender to emit evm state outcome messages, if any.
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
/// Receiver for events produced by tx execution
|
||||
actions_rx: Receiver<PrewarmTaskEvent<N::Receipt>>,
|
||||
actions_rx: Receiver<PrewarmTaskEvent>,
|
||||
/// Parent span for tracing
|
||||
parent_span: Span,
|
||||
}
|
||||
@@ -106,7 +105,7 @@ where
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
transaction_count_hint: usize,
|
||||
max_concurrency: usize,
|
||||
) -> (Self, Sender<PrewarmTaskEvent<N::Receipt>>) {
|
||||
) -> (Self, Sender<PrewarmTaskEvent>) {
|
||||
let (actions_tx, actions_rx) = channel();
|
||||
|
||||
trace!(
|
||||
@@ -136,11 +135,8 @@ where
|
||||
/// For Optimism chains, special handling is applied to the first transaction if it's a
|
||||
/// deposit transaction (type 0x7E/126) which sets critical metadata that affects all
|
||||
/// subsequent transactions in the block.
|
||||
fn spawn_all<Tx>(
|
||||
&self,
|
||||
pending: mpsc::Receiver<Tx>,
|
||||
actions_tx: Sender<PrewarmTaskEvent<N::Receipt>>,
|
||||
) where
|
||||
fn spawn_all<Tx>(&self, pending: mpsc::Receiver<Tx>, actions_tx: Sender<PrewarmTaskEvent>)
|
||||
where
|
||||
Tx: ExecutableTxFor<Evm> + Clone + Send + 'static,
|
||||
{
|
||||
let executor = self.executor.clone();
|
||||
@@ -252,7 +248,7 @@ where
|
||||
///
|
||||
/// This method is called from `run()` only after all execution tasks are complete.
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
|
||||
fn save_cache(self, execution_outcome: Arc<ExecutionOutcome<N::Receipt>>) {
|
||||
fn save_cache(self, state: BundleState) {
|
||||
let start = Instant::now();
|
||||
|
||||
let Self { execution_cache, ctx: PrewarmContext { env, metrics, saved_cache, .. }, .. } =
|
||||
@@ -269,8 +265,7 @@ where
|
||||
let new_cache = SavedCache::new(hash, caches, cache_metrics);
|
||||
|
||||
// Insert state into cache while holding the lock
|
||||
// Access the BundleState through the shared ExecutionOutcome
|
||||
if new_cache.cache().insert_state(execution_outcome.state()).is_err() {
|
||||
if new_cache.cache().insert_state(&state).is_err() {
|
||||
// Clear the cache on error to prevent having a polluted cache
|
||||
*cached = None;
|
||||
debug!(target: "engine::caching", "cleared execution cache on update error");
|
||||
@@ -305,12 +300,12 @@ where
|
||||
pub(super) fn run(
|
||||
self,
|
||||
pending: mpsc::Receiver<impl ExecutableTxFor<Evm> + Clone + Send + 'static>,
|
||||
actions_tx: Sender<PrewarmTaskEvent<N::Receipt>>,
|
||||
actions_tx: Sender<PrewarmTaskEvent>,
|
||||
) {
|
||||
// spawn execution tasks.
|
||||
self.spawn_all(pending, actions_tx);
|
||||
|
||||
let mut final_execution_outcome = None;
|
||||
let mut final_block_output = None;
|
||||
let mut finished_execution = false;
|
||||
while let Ok(event) = self.actions_rx.recv() {
|
||||
match event {
|
||||
@@ -323,9 +318,9 @@ where
|
||||
// completed executing a set of transactions
|
||||
self.send_multi_proof_targets(proof_targets);
|
||||
}
|
||||
PrewarmTaskEvent::Terminate { execution_outcome } => {
|
||||
PrewarmTaskEvent::Terminate { block_output } => {
|
||||
trace!(target: "engine::tree::payload_processor::prewarm", "Received termination signal");
|
||||
final_execution_outcome = Some(execution_outcome);
|
||||
final_block_output = Some(block_output);
|
||||
|
||||
if finished_execution {
|
||||
// all tasks are done, we can exit, which will save caches and exit
|
||||
@@ -339,7 +334,7 @@ where
|
||||
|
||||
finished_execution = true;
|
||||
|
||||
if final_execution_outcome.is_some() {
|
||||
if final_block_output.is_some() {
|
||||
// all tasks are done, we can exit, which will save caches and exit
|
||||
break
|
||||
}
|
||||
@@ -349,9 +344,9 @@ where
|
||||
|
||||
debug!(target: "engine::tree::payload_processor::prewarm", "Completed prewarm execution");
|
||||
|
||||
// save caches and finish using the shared ExecutionOutcome
|
||||
if let Some(Some(execution_outcome)) = final_execution_outcome {
|
||||
self.save_cache(execution_outcome);
|
||||
// save caches and finish
|
||||
if let Some(Some(state)) = final_block_output {
|
||||
self.save_cache(state);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -393,10 +388,10 @@ where
|
||||
metrics,
|
||||
terminate_execution,
|
||||
precompile_cache_disabled,
|
||||
precompile_cache_map,
|
||||
mut precompile_cache_map,
|
||||
} = self;
|
||||
|
||||
let mut state_provider = match provider.build() {
|
||||
let state_provider = match provider.build() {
|
||||
Ok(provider) => provider,
|
||||
Err(err) => {
|
||||
trace!(
|
||||
@@ -409,15 +404,13 @@ where
|
||||
};
|
||||
|
||||
// Use the caches to create a new provider with caching
|
||||
if let Some(saved_cache) = saved_cache {
|
||||
let state_provider: StateProviderBox = if let Some(saved_cache) = saved_cache {
|
||||
let caches = saved_cache.cache().clone();
|
||||
let cache_metrics = saved_cache.metrics().clone();
|
||||
state_provider = Box::new(
|
||||
CachedStateProvider::new(state_provider, caches, cache_metrics)
|
||||
// ensure we pre-warm the cache
|
||||
.prewarm(),
|
||||
);
|
||||
}
|
||||
Box::new(CachedStateProvider::new_with_caches(state_provider, caches, cache_metrics))
|
||||
} else {
|
||||
state_provider
|
||||
};
|
||||
|
||||
let state_provider = StateProviderDatabase::new(state_provider);
|
||||
|
||||
@@ -459,7 +452,7 @@ where
|
||||
fn transact_batch<Tx>(
|
||||
self,
|
||||
txs: mpsc::Receiver<IndexedTransaction<Tx>>,
|
||||
sender: Sender<PrewarmTaskEvent<N::Receipt>>,
|
||||
sender: Sender<PrewarmTaskEvent>,
|
||||
done_tx: Sender<()>,
|
||||
) where
|
||||
Tx: ExecutableTxFor<Evm>,
|
||||
@@ -540,7 +533,7 @@ where
|
||||
&self,
|
||||
idx: usize,
|
||||
executor: &WorkloadExecutor,
|
||||
actions_tx: Sender<PrewarmTaskEvent<N::Receipt>>,
|
||||
actions_tx: Sender<PrewarmTaskEvent>,
|
||||
done_tx: Sender<()>,
|
||||
) -> mpsc::Sender<IndexedTransaction<Tx>>
|
||||
where
|
||||
@@ -596,18 +589,14 @@ fn multiproof_targets_from_state(state: EvmState) -> (MultiProofTargets, usize)
|
||||
}
|
||||
|
||||
/// The events the pre-warm task can handle.
|
||||
///
|
||||
/// Generic over `R` (receipt type) to allow sharing `Arc<ExecutionOutcome<R>>` with the main
|
||||
/// execution path without cloning the expensive `BundleState`.
|
||||
pub(super) enum PrewarmTaskEvent<R> {
|
||||
pub(super) enum PrewarmTaskEvent {
|
||||
/// Forcefully terminate all remaining transaction execution.
|
||||
TerminateTransactionExecution,
|
||||
/// Forcefully terminate the task on demand and update the shared cache with the given output
|
||||
/// before exiting.
|
||||
Terminate {
|
||||
/// The final execution outcome. Using `Arc` allows sharing with the main execution
|
||||
/// path without cloning the expensive `BundleState`.
|
||||
execution_outcome: Option<Arc<ExecutionOutcome<R>>>,
|
||||
/// The final block state output.
|
||||
block_output: Option<BundleState>,
|
||||
},
|
||||
/// The outcome of a pre-warm task
|
||||
Outcome {
|
||||
|
||||
@@ -35,13 +35,12 @@ use reth_primitives_traits::{
|
||||
};
|
||||
use reth_provider::{
|
||||
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockReader,
|
||||
DatabaseProviderFactory, DatabaseProviderROFactory, ExecutionOutcome, HashedPostStateProvider,
|
||||
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
|
||||
StateProviderFactory, StateReader, TrieReader,
|
||||
DatabaseProviderFactory, ExecutionOutcome, HashedPostStateProvider, ProviderError,
|
||||
PruneCheckpointReader, StageCheckpointReader, StateProvider, StateProviderFactory, StateReader,
|
||||
StateRootProvider, TrieReader,
|
||||
};
|
||||
use reth_revm::db::State;
|
||||
use reth_storage_errors::db::DatabaseError;
|
||||
use reth_trie::{updates::TrieUpdates, HashedPostState, StateRoot, TrieInputSorted};
|
||||
use reth_trie::{updates::TrieUpdates, HashedPostState, TrieInputSorted};
|
||||
use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError};
|
||||
use revm_primitives::Address;
|
||||
use std::{
|
||||
@@ -375,8 +374,7 @@ where
|
||||
let mut state_provider = ensure_ok!(provider_builder.build());
|
||||
drop(_enter);
|
||||
|
||||
// Fetch parent block. This goes to memory most of the time unless the parent block is
|
||||
// beyond the in-memory buffer.
|
||||
// fetch parent block
|
||||
let Some(parent_block) = ensure_ok!(self.sealed_header_by_hash(parent_hash, ctx.state()))
|
||||
else {
|
||||
return Err(InsertBlockError::new(
|
||||
@@ -401,17 +399,9 @@ where
|
||||
"Decided which state root algorithm to run"
|
||||
);
|
||||
|
||||
// Get an iterator over the transactions in the payload
|
||||
// use prewarming background task
|
||||
let txs = self.tx_iterator_for(&input)?;
|
||||
|
||||
// Extract the BAL, if valid and available
|
||||
let block_access_list = ensure_ok!(input
|
||||
.block_access_list()
|
||||
.transpose()
|
||||
// Eventually gets converted to a `InsertBlockErrorKind::Other`
|
||||
.map_err(Box::<dyn std::error::Error + Send + Sync>::from))
|
||||
.map(Arc::new);
|
||||
|
||||
// Spawn the appropriate processor based on strategy
|
||||
let mut handle = ensure_ok!(self.spawn_payload_processor(
|
||||
env.clone(),
|
||||
@@ -420,23 +410,26 @@ where
|
||||
parent_hash,
|
||||
ctx.state(),
|
||||
strategy,
|
||||
block_access_list,
|
||||
));
|
||||
|
||||
// Use cached state provider before executing, used in execution after prewarming threads
|
||||
// complete
|
||||
if let Some((caches, cache_metrics)) = handle.caches().zip(handle.cache_metrics()) {
|
||||
state_provider =
|
||||
Box::new(CachedStateProvider::new(state_provider, caches, cache_metrics));
|
||||
state_provider = Box::new(CachedStateProvider::new_with_caches(
|
||||
state_provider,
|
||||
caches,
|
||||
cache_metrics,
|
||||
));
|
||||
};
|
||||
|
||||
if self.config.state_provider_metrics() {
|
||||
state_provider = Box::new(InstrumentedStateProvider::new(state_provider, "engine"));
|
||||
}
|
||||
|
||||
// Execute the block and handle any execution errors
|
||||
let (output, senders) = match self.execute_block(&state_provider, env, &input, &mut handle)
|
||||
{
|
||||
let (output, senders) = match if self.config.state_provider_metrics() {
|
||||
let state_provider =
|
||||
InstrumentedStateProvider::from_state_provider(&state_provider, "engine");
|
||||
self.execute_block(&state_provider, env, &input, &mut handle)
|
||||
} else {
|
||||
self.execute_block(&state_provider, env, &input, &mut handle)
|
||||
} {
|
||||
Ok(output) => output,
|
||||
Err(err) => return self.handle_execution_error(input, err, &parent_block),
|
||||
};
|
||||
@@ -520,7 +513,7 @@ where
|
||||
}
|
||||
|
||||
let (root, updates) = ensure_ok_post_block!(
|
||||
self.compute_state_root_serial(block.parent_hash(), &hashed_state, ctx.state()),
|
||||
state_provider.state_root_with_updates(hashed_state.clone()),
|
||||
block
|
||||
);
|
||||
(root, updates, root_time.elapsed())
|
||||
@@ -550,14 +543,17 @@ where
|
||||
.into())
|
||||
}
|
||||
|
||||
// Create ExecutionOutcome and wrap in Arc for sharing with both the caching task
|
||||
// and the deferred trie task. This avoids cloning the expensive BundleState.
|
||||
let execution_outcome = Arc::new(ExecutionOutcome::from((output, block_num_hash.number)));
|
||||
// terminate prewarming task with good state output
|
||||
handle.terminate_caching(Some(&output.state));
|
||||
|
||||
// Terminate prewarming task with the shared execution outcome
|
||||
handle.terminate_caching(Some(Arc::clone(&execution_outcome)));
|
||||
|
||||
Ok(self.spawn_deferred_trie_task(block, execution_outcome, &ctx, hashed_state, trie_output))
|
||||
Ok(self.spawn_deferred_trie_task(
|
||||
block,
|
||||
output,
|
||||
block_num_hash.number,
|
||||
&ctx,
|
||||
hashed_state,
|
||||
trie_output,
|
||||
))
|
||||
}
|
||||
|
||||
/// Return sealed block header from database or in-memory state by hash.
|
||||
@@ -600,7 +596,7 @@ where
|
||||
state_provider: S,
|
||||
env: ExecutionEnv<Evm>,
|
||||
input: &BlockOrPayload<T>,
|
||||
handle: &mut PayloadHandle<impl ExecutableTxFor<Evm>, Err, N::Receipt>,
|
||||
handle: &mut PayloadHandle<impl ExecutableTxFor<Evm>, Err>,
|
||||
) -> Result<(BlockExecutionOutput<N::Receipt>, Vec<Address>), InsertBlockErrorKind>
|
||||
where
|
||||
S: StateProvider,
|
||||
@@ -612,7 +608,7 @@ where
|
||||
debug!(target: "engine::tree::payload_validator", "Executing block");
|
||||
|
||||
let mut db = State::builder()
|
||||
.with_database(StateProviderDatabase::new(state_provider))
|
||||
.with_database(StateProviderDatabase::new(&state_provider))
|
||||
.with_bundle_update()
|
||||
.without_state_clear()
|
||||
.build();
|
||||
@@ -658,6 +654,8 @@ where
|
||||
///
|
||||
/// Returns `Ok(_)` if computed successfully.
|
||||
/// Returns `Err(_)` if error was encountered during computation.
|
||||
/// `Err(ProviderError::ConsistentView(_))` can be safely ignored and fallback computation
|
||||
/// should be used instead.
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)]
|
||||
fn compute_state_root_parallel(
|
||||
&self,
|
||||
@@ -687,36 +685,6 @@ where
|
||||
ParallelStateRoot::new(factory, prefix_sets).incremental_root_with_updates()
|
||||
}
|
||||
|
||||
/// Compute state root for the given hashed post state in serial.
|
||||
fn compute_state_root_serial(
|
||||
&self,
|
||||
parent_hash: B256,
|
||||
hashed_state: &HashedPostState,
|
||||
state: &EngineApiTreeState<N>,
|
||||
) -> ProviderResult<(B256, TrieUpdates)> {
|
||||
let (mut input, block_hash) = self.compute_trie_input(parent_hash, state)?;
|
||||
|
||||
// Extend state overlay with current block's sorted state.
|
||||
input.prefix_sets.extend(hashed_state.construct_prefix_sets());
|
||||
let sorted_hashed_state = hashed_state.clone_into_sorted();
|
||||
Arc::make_mut(&mut input.state).extend_ref(&sorted_hashed_state);
|
||||
|
||||
let TrieInputSorted { nodes, state, .. } = input;
|
||||
let prefix_sets = hashed_state.construct_prefix_sets();
|
||||
|
||||
let factory = OverlayStateProviderFactory::new(self.provider.clone())
|
||||
.with_block_hash(Some(block_hash))
|
||||
.with_trie_overlay(Some(nodes))
|
||||
.with_hashed_state_overlay(Some(state));
|
||||
|
||||
let provider = factory.database_provider_ro()?;
|
||||
|
||||
Ok(StateRoot::new(&provider, &provider)
|
||||
.with_prefix_sets(prefix_sets.freeze())
|
||||
.root_with_updates()
|
||||
.map_err(Into::<DatabaseError>::into)?)
|
||||
}
|
||||
|
||||
/// Validates the block after execution.
|
||||
///
|
||||
/// This performs:
|
||||
@@ -811,12 +779,10 @@ where
|
||||
parent_hash: B256,
|
||||
state: &EngineApiTreeState<N>,
|
||||
strategy: StateRootStrategy,
|
||||
block_access_list: Option<Arc<BlockAccessList>>,
|
||||
) -> Result<
|
||||
PayloadHandle<
|
||||
impl ExecutableTxFor<Evm> + use<N, P, Evm, V, T>,
|
||||
impl core::error::Error + Send + Sync + 'static + use<N, P, Evm, V, T>,
|
||||
N::Receipt,
|
||||
>,
|
||||
InsertBlockErrorKind,
|
||||
> {
|
||||
@@ -842,14 +808,12 @@ where
|
||||
.record(trie_input_start.elapsed().as_secs_f64());
|
||||
|
||||
let spawn_start = Instant::now();
|
||||
|
||||
let handle = self.payload_processor.spawn(
|
||||
env,
|
||||
txs,
|
||||
provider_builder,
|
||||
multiproof_provider_factory,
|
||||
&self.config,
|
||||
block_access_list,
|
||||
);
|
||||
|
||||
// record prewarming initialization duration
|
||||
@@ -1051,7 +1015,8 @@ where
|
||||
fn spawn_deferred_trie_task(
|
||||
&self,
|
||||
block: RecoveredBlock<N::Block>,
|
||||
execution_outcome: Arc<ExecutionOutcome<N::Receipt>>,
|
||||
output: BlockExecutionOutput<N::Receipt>,
|
||||
block_number: u64,
|
||||
ctx: &TreeCtx<'_, N>,
|
||||
hashed_state: HashedPostState,
|
||||
trie_output: TrieUpdates,
|
||||
@@ -1101,7 +1066,7 @@ where
|
||||
|
||||
ExecutedBlock::with_deferred_trie_data(
|
||||
Arc::new(block),
|
||||
execution_outcome,
|
||||
Arc::new(ExecutionOutcome::from((output, block_number))),
|
||||
deferred_trie_data,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,56 +1,50 @@
|
||||
//! Contains a precompile cache backed by `schnellru::LruMap` (LRU by length).
|
||||
|
||||
use alloy_primitives::Bytes;
|
||||
use dashmap::DashMap;
|
||||
use parking_lot::Mutex;
|
||||
use reth_evm::precompiles::{DynPrecompile, Precompile, PrecompileInput};
|
||||
use revm::precompile::{PrecompileId, PrecompileOutput, PrecompileResult};
|
||||
use revm_primitives::Address;
|
||||
use std::{hash::Hash, sync::Arc};
|
||||
use schnellru::LruMap;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
hash::{Hash, Hasher},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
/// Default max cache size for [`PrecompileCache`]
|
||||
const MAX_CACHE_SIZE: u32 = 10_000;
|
||||
|
||||
/// Stores caches for each precompile.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct PrecompileCacheMap<S>(Arc<DashMap<Address, PrecompileCache<S>>>)
|
||||
pub struct PrecompileCacheMap<S>(HashMap<Address, PrecompileCache<S>>)
|
||||
where
|
||||
S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone + 'static;
|
||||
S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone;
|
||||
|
||||
impl<S> PrecompileCacheMap<S>
|
||||
where
|
||||
S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone + 'static,
|
||||
{
|
||||
pub(crate) fn cache_for_address(&self, address: Address) -> PrecompileCache<S> {
|
||||
// Try just using `.get` first to avoid acquiring a write lock.
|
||||
if let Some(cache) = self.0.get(&address) {
|
||||
return cache.clone();
|
||||
}
|
||||
// Otherwise, fallback to `.entry` and initialize the cache.
|
||||
//
|
||||
// This should be very rare as caches for all precompiles will be initialized as soon as
|
||||
// first EVM is created.
|
||||
pub(crate) fn cache_for_address(&mut self, address: Address) -> PrecompileCache<S> {
|
||||
self.0.entry(address).or_default().clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// Cache for precompiles, for each input stores the result.
|
||||
///
|
||||
/// [`LruMap`] requires a mutable reference on `get` since it updates the LRU order,
|
||||
/// so we use a [`Mutex`] instead of an `RwLock`.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PrecompileCache<S>(
|
||||
moka::sync::Cache<Bytes, CacheEntry<S>, alloy_primitives::map::DefaultHashBuilder>,
|
||||
)
|
||||
pub struct PrecompileCache<S>(Arc<Mutex<LruMap<CacheKey<S>, CacheEntry>>>)
|
||||
where
|
||||
S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone + 'static;
|
||||
S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone;
|
||||
|
||||
impl<S> Default for PrecompileCache<S>
|
||||
where
|
||||
S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone + 'static,
|
||||
{
|
||||
fn default() -> Self {
|
||||
Self(
|
||||
moka::sync::CacheBuilder::new(MAX_CACHE_SIZE as u64)
|
||||
.initial_capacity(MAX_CACHE_SIZE as usize)
|
||||
.build_with_hasher(Default::default()),
|
||||
)
|
||||
Self(Arc::new(Mutex::new(LruMap::new(schnellru::ByLength::new(MAX_CACHE_SIZE)))))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,31 +52,63 @@ impl<S> PrecompileCache<S>
|
||||
where
|
||||
S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone + 'static,
|
||||
{
|
||||
fn get(&self, input: &[u8], spec: S) -> Option<CacheEntry<S>> {
|
||||
self.0.get(input).filter(|e| e.spec == spec)
|
||||
fn get(&self, key: &CacheKeyRef<'_, S>) -> Option<CacheEntry> {
|
||||
self.0.lock().get(key).cloned()
|
||||
}
|
||||
|
||||
/// Inserts the given key and value into the cache, returning the new cache size.
|
||||
fn insert(&self, input: Bytes, value: CacheEntry<S>) -> usize {
|
||||
self.0.insert(input, value);
|
||||
self.0.entry_count() as usize
|
||||
fn insert(&self, key: CacheKey<S>, value: CacheEntry) -> usize {
|
||||
let mut cache = self.0.lock();
|
||||
cache.insert(key, value);
|
||||
cache.len()
|
||||
}
|
||||
}
|
||||
|
||||
/// Cache key, spec id and precompile call input. spec id is included in the key to account for
|
||||
/// precompile repricing across fork activations.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct CacheKey<S>((S, Bytes));
|
||||
|
||||
impl<S> CacheKey<S> {
|
||||
const fn new(spec_id: S, input: Bytes) -> Self {
|
||||
Self((spec_id, input))
|
||||
}
|
||||
}
|
||||
|
||||
/// Cache key reference, used to avoid cloning the input bytes when looking up using a [`CacheKey`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CacheKeyRef<'a, S>((S, &'a [u8]));
|
||||
|
||||
impl<'a, S> CacheKeyRef<'a, S> {
|
||||
const fn new(spec_id: S, input: &'a [u8]) -> Self {
|
||||
Self((spec_id, input))
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: PartialEq> PartialEq<CacheKey<S>> for CacheKeyRef<'_, S> {
|
||||
fn eq(&self, other: &CacheKey<S>) -> bool {
|
||||
self.0 .0 == other.0 .0 && self.0 .1 == other.0 .1.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, S: Hash> Hash for CacheKeyRef<'a, S> {
|
||||
fn hash<H: Hasher>(&self, state: &mut H) {
|
||||
self.0 .0.hash(state);
|
||||
self.0 .1.hash(state);
|
||||
}
|
||||
}
|
||||
|
||||
/// Cache entry, precompile successful output.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CacheEntry<S> {
|
||||
output: PrecompileOutput,
|
||||
spec: S,
|
||||
}
|
||||
pub struct CacheEntry(PrecompileOutput);
|
||||
|
||||
impl<S> CacheEntry<S> {
|
||||
impl CacheEntry {
|
||||
const fn gas_used(&self) -> u64 {
|
||||
self.output.gas_used
|
||||
self.0.gas_used
|
||||
}
|
||||
|
||||
fn to_precompile_result(&self) -> PrecompileResult {
|
||||
Ok(self.output.clone())
|
||||
Ok(self.0.clone())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -164,7 +190,9 @@ where
|
||||
}
|
||||
|
||||
fn call(&self, input: PrecompileInput<'_>) -> PrecompileResult {
|
||||
if let Some(entry) = &self.cache.get(input.data, self.spec_id.clone()) {
|
||||
let key = CacheKeyRef::new(self.spec_id.clone(), input.data);
|
||||
|
||||
if let Some(entry) = &self.cache.get(&key) {
|
||||
self.increment_by_one_precompile_cache_hits();
|
||||
if input.gas >= entry.gas_used() {
|
||||
return entry.to_precompile_result()
|
||||
@@ -176,10 +204,8 @@ where
|
||||
|
||||
match &result {
|
||||
Ok(output) => {
|
||||
let size = self.cache.insert(
|
||||
Bytes::copy_from_slice(calldata),
|
||||
CacheEntry { output: output.clone(), spec: self.spec_id.clone() },
|
||||
);
|
||||
let key = CacheKey::new(self.spec_id.clone(), Bytes::copy_from_slice(calldata));
|
||||
let size = self.cache.insert(key, CacheEntry(output.clone()));
|
||||
self.set_precompile_cache_size_metric(size as f64);
|
||||
self.increment_by_one_precompile_cache_misses();
|
||||
}
|
||||
@@ -220,12 +246,31 @@ impl CachedPrecompileMetrics {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::hash::DefaultHasher;
|
||||
|
||||
use super::*;
|
||||
use reth_evm::{EthEvmFactory, Evm, EvmEnv, EvmFactory};
|
||||
use reth_revm::db::EmptyDB;
|
||||
use revm::{context::TxEnv, precompile::PrecompileOutput};
|
||||
use revm_primitives::hardfork::SpecId;
|
||||
|
||||
#[test]
|
||||
fn test_cache_key_ref_hash() {
|
||||
let key1 = CacheKey::new(SpecId::PRAGUE, b"test_input".into());
|
||||
let key2 = CacheKeyRef::new(SpecId::PRAGUE, b"test_input");
|
||||
assert!(PartialEq::eq(&key2, &key1));
|
||||
|
||||
let mut hasher = DefaultHasher::new();
|
||||
key1.hash(&mut hasher);
|
||||
let hash1 = hasher.finish();
|
||||
|
||||
let mut hasher = DefaultHasher::new();
|
||||
key2.hash(&mut hasher);
|
||||
let hash2 = hasher.finish();
|
||||
|
||||
assert_eq!(hash1, hash2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_precompile_cache_basic() {
|
||||
let dyn_precompile: DynPrecompile = (|_input: PrecompileInput<'_>| -> PrecompileResult {
|
||||
@@ -248,11 +293,12 @@ mod tests {
|
||||
reverted: false,
|
||||
};
|
||||
|
||||
let input = b"test_input";
|
||||
let expected = CacheEntry { output, spec: SpecId::PRAGUE };
|
||||
cache.cache.insert(input.into(), expected.clone());
|
||||
let key = CacheKey::new(SpecId::PRAGUE, b"test_input".into());
|
||||
let expected = CacheEntry(output);
|
||||
cache.cache.insert(key, expected.clone());
|
||||
|
||||
let actual = cache.cache.get(input, SpecId::PRAGUE).unwrap();
|
||||
let key = CacheKeyRef::new(SpecId::PRAGUE, b"test_input");
|
||||
let actual = cache.cache.get(&key).unwrap();
|
||||
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
@@ -266,7 +312,7 @@ mod tests {
|
||||
let address1 = Address::repeat_byte(1);
|
||||
let address2 = Address::repeat_byte(2);
|
||||
|
||||
let cache_map = PrecompileCacheMap::default();
|
||||
let mut cache_map = PrecompileCacheMap::default();
|
||||
|
||||
// create the first precompile with a specific output
|
||||
let precompile1: DynPrecompile = (PrecompileId::custom("custom"), {
|
||||
|
||||
@@ -4,7 +4,7 @@ use crate::{
|
||||
tree::{
|
||||
payload_validator::{BasicEngineValidator, TreeCtx, ValidationOutcome},
|
||||
persistence_state::CurrentPersistenceAction,
|
||||
PersistTarget, TreeConfig,
|
||||
TreeConfig,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -285,8 +285,7 @@ impl TestHarness {
|
||||
let fcu_state = self.fcu_state(block_hash);
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let _ = self
|
||||
.tree
|
||||
self.tree
|
||||
.on_engine_message(FromEngine::Request(
|
||||
BeaconEngineMessage::ForkchoiceUpdated {
|
||||
state: fcu_state,
|
||||
@@ -499,7 +498,7 @@ fn test_tree_persist_block_batch() {
|
||||
|
||||
// process the message
|
||||
let msg = test_harness.tree.try_recv_engine_message().unwrap().unwrap();
|
||||
let _ = test_harness.tree.on_engine_message(msg).unwrap();
|
||||
test_harness.tree.on_engine_message(msg).unwrap();
|
||||
|
||||
// we now should receive the other batch
|
||||
let msg = test_harness.tree.try_recv_engine_message().unwrap().unwrap();
|
||||
@@ -578,7 +577,7 @@ async fn test_engine_request_during_backfill() {
|
||||
.with_backfill_state(BackfillSyncState::Active);
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let _ = test_harness
|
||||
test_harness
|
||||
.tree
|
||||
.on_engine_message(FromEngine::Request(
|
||||
BeaconEngineMessage::ForkchoiceUpdated {
|
||||
@@ -659,7 +658,7 @@ async fn test_holesky_payload() {
|
||||
TestHarness::new(HOLESKY.clone()).with_backfill_state(BackfillSyncState::Active);
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let _ = test_harness
|
||||
test_harness
|
||||
.tree
|
||||
.on_engine_message(FromEngine::Request(
|
||||
BeaconEngineMessage::NewPayload {
|
||||
@@ -884,8 +883,7 @@ async fn test_get_canonical_blocks_to_persist() {
|
||||
.with_persistence_threshold(persistence_threshold)
|
||||
.with_memory_block_buffer_target(memory_block_buffer_target);
|
||||
|
||||
let blocks_to_persist =
|
||||
test_harness.tree.get_canonical_blocks_to_persist(PersistTarget::Threshold).unwrap();
|
||||
let blocks_to_persist = test_harness.tree.get_canonical_blocks_to_persist().unwrap();
|
||||
|
||||
let expected_blocks_to_persist_length: usize =
|
||||
(canonical_head_number - memory_block_buffer_target - last_persisted_block_number)
|
||||
@@ -904,8 +902,7 @@ async fn test_get_canonical_blocks_to_persist() {
|
||||
|
||||
assert!(test_harness.tree.state.tree_state.sealed_header_by_hash(&fork_block_hash).is_some());
|
||||
|
||||
let blocks_to_persist =
|
||||
test_harness.tree.get_canonical_blocks_to_persist(PersistTarget::Threshold).unwrap();
|
||||
let blocks_to_persist = test_harness.tree.get_canonical_blocks_to_persist().unwrap();
|
||||
assert_eq!(blocks_to_persist.len(), expected_blocks_to_persist_length);
|
||||
|
||||
// check that the fork block is not included in the blocks to persist
|
||||
@@ -984,7 +981,7 @@ async fn test_engine_tree_live_sync_transition_required_blocks_requested() {
|
||||
let backfill_tip_block = main_chain[(backfill_finished_block_number - 1) as usize].clone();
|
||||
// add block to mock provider to enable persistence clean up.
|
||||
test_harness.provider.add_block(backfill_tip_block.hash(), backfill_tip_block.into_block());
|
||||
let _ = test_harness.tree.on_engine_message(FromEngine::Event(backfill_finished)).unwrap();
|
||||
test_harness.tree.on_engine_message(FromEngine::Event(backfill_finished)).unwrap();
|
||||
|
||||
let event = test_harness.from_tree_rx.recv().await.unwrap();
|
||||
match event {
|
||||
@@ -994,7 +991,7 @@ async fn test_engine_tree_live_sync_transition_required_blocks_requested() {
|
||||
_ => panic!("Unexpected event: {event:#?}"),
|
||||
}
|
||||
|
||||
let _ = test_harness
|
||||
test_harness
|
||||
.tree
|
||||
.on_engine_message(FromEngine::DownloadedBlocks(vec![main_chain
|
||||
.last()
|
||||
@@ -1050,7 +1047,7 @@ async fn test_fcu_with_canonical_ancestor_updates_latest_block() {
|
||||
|
||||
// Send FCU to the canonical ancestor
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let _ = test_harness
|
||||
test_harness
|
||||
.tree
|
||||
.on_engine_message(FromEngine::Request(
|
||||
BeaconEngineMessage::ForkchoiceUpdated {
|
||||
@@ -1946,53 +1943,4 @@ mod forkchoice_updated_tests {
|
||||
.unwrap();
|
||||
assert!(result.is_some(), "OpStack should handle canonical head");
|
||||
}
|
||||
|
||||
/// Test that engine termination persists all blocks and signals completion.
|
||||
#[test]
|
||||
fn test_engine_termination_with_everything_persisted() {
|
||||
let chain_spec = MAINNET.clone();
|
||||
let mut test_block_builder = TestBlockBuilder::eth().with_chain_spec((*chain_spec).clone());
|
||||
|
||||
// Create 10 blocks to persist
|
||||
let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..11).collect();
|
||||
let canonical_tip = blocks.last().unwrap().recovered_block().number;
|
||||
let test_harness = TestHarness::new(chain_spec).with_blocks(blocks);
|
||||
|
||||
// Create termination channel
|
||||
let (terminate_tx, mut terminate_rx) = oneshot::channel();
|
||||
|
||||
let to_tree_tx = test_harness.to_tree_tx.clone();
|
||||
let action_rx = test_harness.action_rx;
|
||||
|
||||
// Spawn tree in background thread
|
||||
std::thread::Builder::new()
|
||||
.name("Engine Task".to_string())
|
||||
.spawn(|| test_harness.tree.run())
|
||||
.unwrap();
|
||||
|
||||
// Send terminate request
|
||||
to_tree_tx
|
||||
.send(FromEngine::Event(FromOrchestrator::Terminate { tx: terminate_tx }))
|
||||
.unwrap();
|
||||
|
||||
// Handle persistence actions until termination completes
|
||||
let mut last_persisted_number = 0;
|
||||
loop {
|
||||
if terminate_rx.try_recv().is_ok() {
|
||||
break;
|
||||
}
|
||||
|
||||
if let Ok(PersistenceAction::SaveBlocks(saved_blocks, sender)) =
|
||||
action_rx.recv_timeout(std::time::Duration::from_millis(100))
|
||||
{
|
||||
if let Some(last) = saved_blocks.last() {
|
||||
last_persisted_number = last.recovered_block().number;
|
||||
}
|
||||
sender.send(saved_blocks.last().map(|b| b.recovered_block().num_hash())).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure we persisted right to the tip
|
||||
assert_eq!(last_persisted_number, canonical_tip);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,12 +150,6 @@ where
|
||||
let era1_id = Era1Id::new(&config.network, start_block, block_count as u32)
|
||||
.with_hash(historical_root);
|
||||
|
||||
let era1_id = if config.max_blocks_per_file == MAX_BLOCKS_PER_ERA1 as u64 {
|
||||
era1_id
|
||||
} else {
|
||||
era1_id.with_era_count()
|
||||
};
|
||||
|
||||
debug!("Final file name {}", era1_id.to_file_name());
|
||||
let file_path = config.dir.join(era1_id.to_file_name());
|
||||
let file = std::fs::File::create(&file_path)?;
|
||||
|
||||
@@ -24,7 +24,7 @@ fn test_export_with_genesis_only() {
|
||||
assert!(file_path.exists(), "Exported file should exist on disk");
|
||||
let file_name = file_path.file_name().unwrap().to_str().unwrap();
|
||||
assert!(
|
||||
file_name.starts_with("mainnet-00000-"),
|
||||
file_name.starts_with("mainnet-00000-00001-"),
|
||||
"File should have correct prefix with era format"
|
||||
);
|
||||
assert!(file_name.ends_with(".era1"), "File should have correct extension");
|
||||
|
||||
@@ -30,11 +30,8 @@ pub trait EraFileFormat: Sized {
|
||||
|
||||
/// Era file identifiers
|
||||
pub trait EraFileId: Clone {
|
||||
/// File type for this identifier
|
||||
const FILE_TYPE: EraFileType;
|
||||
|
||||
/// Number of items, slots for `era`, blocks for `era1`, per era
|
||||
const ITEMS_PER_ERA: u64;
|
||||
/// Convert to standardized file name
|
||||
fn to_file_name(&self) -> String;
|
||||
|
||||
/// Get the network name
|
||||
fn network_name(&self) -> &str;
|
||||
@@ -44,43 +41,6 @@ pub trait EraFileId: Clone {
|
||||
|
||||
/// Get the count of items
|
||||
fn count(&self) -> u32;
|
||||
|
||||
/// Get the optional hash identifier
|
||||
fn hash(&self) -> Option<[u8; 4]>;
|
||||
|
||||
/// Whether to include era count in filename
|
||||
fn include_era_count(&self) -> bool;
|
||||
|
||||
/// Calculate era number
|
||||
fn era_number(&self) -> u64 {
|
||||
self.start_number() / Self::ITEMS_PER_ERA
|
||||
}
|
||||
|
||||
/// Calculate the number of eras spanned per file.
|
||||
///
|
||||
/// If the user can decide how many slots/blocks per era file there are, we need to calculate
|
||||
/// it. Most of the time it should be 1, but it can never be more than 2 eras per file
|
||||
/// as there is a maximum of 8192 slots/blocks per era file.
|
||||
fn era_count(&self) -> u64 {
|
||||
if self.count() == 0 {
|
||||
return 0;
|
||||
}
|
||||
let first_era = self.era_number();
|
||||
let last_number = self.start_number() + self.count() as u64 - 1;
|
||||
let last_era = last_number / Self::ITEMS_PER_ERA;
|
||||
last_era - first_era + 1
|
||||
}
|
||||
|
||||
/// Convert to standardized file name.
|
||||
fn to_file_name(&self) -> String {
|
||||
Self::FILE_TYPE.format_filename(
|
||||
self.network_name(),
|
||||
self.era_number(),
|
||||
self.hash(),
|
||||
self.include_era_count(),
|
||||
self.era_count(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// [`StreamReader`] for reading era-format files
|
||||
@@ -194,37 +154,6 @@ impl EraFileType {
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate era file name.
|
||||
///
|
||||
/// Standard format: `<config-name>-<era-number>-<short-historical-root>.<ext>`
|
||||
/// See also <https://github.com/eth-clients/e2store-format-specs/blob/main/formats/era.md#file-name>
|
||||
///
|
||||
/// With era count (for custom exports):
|
||||
/// `<config-name>-<era-number>-<era-count>-<short-historical-root>.<ext>`
|
||||
pub fn format_filename(
|
||||
&self,
|
||||
network_name: &str,
|
||||
era_number: u64,
|
||||
hash: Option<[u8; 4]>,
|
||||
include_era_count: bool,
|
||||
era_count: u64,
|
||||
) -> String {
|
||||
let hash = format_hash(hash);
|
||||
|
||||
if include_era_count {
|
||||
format!(
|
||||
"{}-{:05}-{:05}-{}{}",
|
||||
network_name,
|
||||
era_number,
|
||||
era_count,
|
||||
hash,
|
||||
self.extension()
|
||||
)
|
||||
} else {
|
||||
format!("{}-{:05}-{}{}", network_name, era_number, hash, self.extension())
|
||||
}
|
||||
}
|
||||
|
||||
/// Detect file type from URL
|
||||
/// By default, it assumes `Era` type
|
||||
pub fn from_url(url: &str) -> Self {
|
||||
@@ -235,11 +164,3 @@ impl EraFileType {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Format hash as hex string, or placeholder if none
|
||||
pub fn format_hash(hash: Option<[u8; 4]>) -> String {
|
||||
match hash {
|
||||
Some(h) => format!("{:02x}{:02x}{:02x}{:02x}", h[0], h[1], h[2], h[3]),
|
||||
None => "00000000".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//! See also <https://github.com/eth-clients/e2store-format-specs/blob/main/formats/era.md>
|
||||
|
||||
use crate::{
|
||||
common::file_ops::{EraFileId, EraFileType},
|
||||
common::file_ops::EraFileId,
|
||||
e2s::types::{Entry, IndexEntry, SLOT_INDEX},
|
||||
era::types::consensus::{CompressedBeaconState, CompressedSignedBeaconBlock},
|
||||
};
|
||||
@@ -163,22 +163,12 @@ pub struct EraId {
|
||||
/// Optional hash identifier for this file
|
||||
/// First 4 bytes of the last historical root in the last state in the era file
|
||||
pub hash: Option<[u8; 4]>,
|
||||
|
||||
/// Whether to include era count in filename
|
||||
/// It is used for custom exports when we don't use the max number of items per file
|
||||
include_era_count: bool,
|
||||
}
|
||||
|
||||
impl EraId {
|
||||
/// Create a new [`EraId`]
|
||||
pub fn new(network_name: impl Into<String>, start_slot: u64, slot_count: u32) -> Self {
|
||||
Self {
|
||||
network_name: network_name.into(),
|
||||
start_slot,
|
||||
slot_count,
|
||||
hash: None,
|
||||
include_era_count: false,
|
||||
}
|
||||
Self { network_name: network_name.into(), start_slot, slot_count, hash: None }
|
||||
}
|
||||
|
||||
/// Add a hash identifier to [`EraId`]
|
||||
@@ -187,18 +177,32 @@ impl EraId {
|
||||
self
|
||||
}
|
||||
|
||||
/// Include era count in filename, for custom slot-per-file exports
|
||||
pub const fn with_era_count(mut self) -> Self {
|
||||
self.include_era_count = true;
|
||||
self
|
||||
/// Calculate which era number the file starts at
|
||||
pub const fn era_number(&self) -> u64 {
|
||||
self.start_slot / SLOTS_PER_HISTORICAL_ROOT
|
||||
}
|
||||
|
||||
// Helper function to calculate the number of eras per era1 file,
|
||||
// If the user can decide how many blocks per era1 file there are, we need to calculate it.
|
||||
// Most of the time it should be 1, but it can never be more than 2 eras per file
|
||||
// as there is a maximum of 8192 blocks per era1 file.
|
||||
const fn calculate_era_count(&self) -> u64 {
|
||||
if self.slot_count == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let first_era = self.era_number();
|
||||
|
||||
// Calculate the actual last slot number in the range
|
||||
let last_slot = self.start_slot + self.slot_count as u64 - 1;
|
||||
// Find which era the last block belongs to
|
||||
let last_era = last_slot / SLOTS_PER_HISTORICAL_ROOT;
|
||||
// Count how many eras we span
|
||||
last_era - first_era + 1
|
||||
}
|
||||
}
|
||||
|
||||
impl EraFileId for EraId {
|
||||
const FILE_TYPE: EraFileType = EraFileType::Era;
|
||||
|
||||
const ITEMS_PER_ERA: u64 = SLOTS_PER_HISTORICAL_ROOT;
|
||||
|
||||
fn network_name(&self) -> &str {
|
||||
&self.network_name
|
||||
}
|
||||
@@ -210,13 +214,24 @@ impl EraFileId for EraId {
|
||||
fn count(&self) -> u32 {
|
||||
self.slot_count
|
||||
}
|
||||
/// Convert to file name following the era file naming:
|
||||
/// `<config-name>-<era-number>-<era-count>-<short-historical-root>.era`
|
||||
/// <https://github.com/eth-clients/e2store-format-specs/blob/main/formats/era.md#file-name>
|
||||
/// See also <https://github.com/eth-clients/e2store-format-specs/blob/main/formats/era.md>
|
||||
fn to_file_name(&self) -> String {
|
||||
let era_number = self.era_number();
|
||||
let era_count = self.calculate_era_count();
|
||||
|
||||
fn hash(&self) -> Option<[u8; 4]> {
|
||||
self.hash
|
||||
}
|
||||
|
||||
fn include_era_count(&self) -> bool {
|
||||
self.include_era_count
|
||||
if let Some(hash) = self.hash {
|
||||
format!(
|
||||
"{}-{:05}-{:05}-{:02x}{:02x}{:02x}{:02x}.era",
|
||||
self.network_name, era_number, era_count, hash[0], hash[1], hash[2], hash[3]
|
||||
)
|
||||
} else {
|
||||
// era spec format with placeholder hash when no hash available
|
||||
// Format: `<config-name>-<era-number>-<era-count>-00000000.era`
|
||||
format!("{}-{:05}-{:05}-00000000.era", self.network_name, era_number, era_count)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -384,40 +399,4 @@ mod tests {
|
||||
let parsed_offset = index.offsets[0];
|
||||
assert_eq!(parsed_offset, -1024);
|
||||
}
|
||||
|
||||
#[test_case::test_case(
|
||||
EraId::new("mainnet", 0, 8192).with_hash([0x4b, 0x36, 0x3d, 0xb9]),
|
||||
"mainnet-00000-4b363db9.era";
|
||||
"Mainnet era 0"
|
||||
)]
|
||||
#[test_case::test_case(
|
||||
EraId::new("mainnet", 8192, 8192).with_hash([0x40, 0xcf, 0x2f, 0x3c]),
|
||||
"mainnet-00001-40cf2f3c.era";
|
||||
"Mainnet era 1"
|
||||
)]
|
||||
#[test_case::test_case(
|
||||
EraId::new("mainnet", 0, 8192),
|
||||
"mainnet-00000-00000000.era";
|
||||
"Without hash"
|
||||
)]
|
||||
fn test_era_id_file_naming(id: EraId, expected_file_name: &str) {
|
||||
let actual_file_name = id.to_file_name();
|
||||
assert_eq!(actual_file_name, expected_file_name);
|
||||
}
|
||||
|
||||
// File naming with era-count, for custom exports
|
||||
#[test_case::test_case(
|
||||
EraId::new("mainnet", 0, 8192).with_hash([0x4b, 0x36, 0x3d, 0xb9]).with_era_count(),
|
||||
"mainnet-00000-00001-4b363db9.era";
|
||||
"Mainnet era 0 with count"
|
||||
)]
|
||||
#[test_case::test_case(
|
||||
EraId::new("mainnet", 8000, 500).with_hash([0xab, 0xcd, 0xef, 0x12]).with_era_count(),
|
||||
"mainnet-00000-00002-abcdef12.era";
|
||||
"Spanning two eras with count"
|
||||
)]
|
||||
fn test_era_id_file_naming_with_era_count(id: EraId, expected_file_name: &str) {
|
||||
let actual_file_name = id.to_file_name();
|
||||
assert_eq!(actual_file_name, expected_file_name);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//! See also <https://github.com/eth-clients/e2store-format-specs/blob/main/formats/era1.md>
|
||||
|
||||
use crate::{
|
||||
common::file_ops::{EraFileId, EraFileType},
|
||||
common::file_ops::EraFileId,
|
||||
e2s::types::{Entry, IndexEntry},
|
||||
era1::types::execution::{Accumulator, BlockTuple, MAX_BLOCKS_PER_ERA1},
|
||||
};
|
||||
@@ -105,10 +105,6 @@ pub struct Era1Id {
|
||||
/// Optional hash identifier for this file
|
||||
/// First 4 bytes of the last historical root in the last state in the era file
|
||||
pub hash: Option<[u8; 4]>,
|
||||
|
||||
/// Whether to include era count in filename
|
||||
/// It is used for custom exports when we don't use the max number of items per file
|
||||
pub include_era_count: bool,
|
||||
}
|
||||
|
||||
impl Era1Id {
|
||||
@@ -118,13 +114,7 @@ impl Era1Id {
|
||||
start_block: BlockNumber,
|
||||
block_count: u32,
|
||||
) -> Self {
|
||||
Self {
|
||||
network_name: network_name.into(),
|
||||
start_block,
|
||||
block_count,
|
||||
hash: None,
|
||||
include_era_count: false,
|
||||
}
|
||||
Self { network_name: network_name.into(), start_block, block_count, hash: None }
|
||||
}
|
||||
|
||||
/// Add a hash identifier to [`Era1Id`]
|
||||
@@ -133,17 +123,21 @@ impl Era1Id {
|
||||
self
|
||||
}
|
||||
|
||||
/// Include era count in filename, for custom block-per-file exports
|
||||
pub const fn with_era_count(mut self) -> Self {
|
||||
self.include_era_count = true;
|
||||
self
|
||||
// Helper function to calculate the number of eras per era1 file,
|
||||
// If the user can decide how many blocks per era1 file there are, we need to calculate it.
|
||||
// Most of the time it should be 1, but it can never be more than 2 eras per file
|
||||
// as there is a maximum of 8192 blocks per era1 file.
|
||||
const fn calculate_era_count(&self, first_era: u64) -> u64 {
|
||||
// Calculate the actual last block number in the range
|
||||
let last_block = self.start_block + self.block_count as u64 - 1;
|
||||
// Find which era the last block belongs to
|
||||
let last_era = last_block / MAX_BLOCKS_PER_ERA1 as u64;
|
||||
// Count how many eras we span
|
||||
last_era - first_era + 1
|
||||
}
|
||||
}
|
||||
|
||||
impl EraFileId for Era1Id {
|
||||
const FILE_TYPE: EraFileType = EraFileType::Era1;
|
||||
|
||||
const ITEMS_PER_ERA: u64 = MAX_BLOCKS_PER_ERA1 as u64;
|
||||
fn network_name(&self) -> &str {
|
||||
&self.network_name
|
||||
}
|
||||
@@ -155,13 +149,24 @@ impl EraFileId for Era1Id {
|
||||
fn count(&self) -> u32 {
|
||||
self.block_count
|
||||
}
|
||||
|
||||
fn hash(&self) -> Option<[u8; 4]> {
|
||||
self.hash
|
||||
}
|
||||
|
||||
fn include_era_count(&self) -> bool {
|
||||
self.include_era_count
|
||||
/// Convert to file name following the era file naming:
|
||||
/// `<config-name>-<era-number>-<era-count>-<short-historical-root>.era(1)`
|
||||
/// <https://github.com/eth-clients/e2store-format-specs/blob/main/formats/era.md#file-name>
|
||||
/// See also <https://github.com/eth-clients/e2store-format-specs/blob/main/formats/era1.md>
|
||||
fn to_file_name(&self) -> String {
|
||||
// Find which era the first block belongs to
|
||||
let era_number = self.start_block / MAX_BLOCKS_PER_ERA1 as u64;
|
||||
let era_count = self.calculate_era_count(era_number);
|
||||
if let Some(hash) = self.hash {
|
||||
format!(
|
||||
"{}-{:05}-{:05}-{:02x}{:02x}{:02x}{:02x}.era1",
|
||||
self.network_name, era_number, era_count, hash[0], hash[1], hash[2], hash[3]
|
||||
)
|
||||
} else {
|
||||
// era spec format with placeholder hash when no hash available
|
||||
// Format: `<config-name>-<era-number>-<era-count>-00000000.era1`
|
||||
format!("{}-{:05}-{:05}-00000000.era1", self.network_name, era_number, era_count)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -309,51 +314,35 @@ mod tests {
|
||||
|
||||
#[test_case::test_case(
|
||||
Era1Id::new("mainnet", 0, 8192).with_hash([0x5e, 0xc1, 0xff, 0xb8]),
|
||||
"mainnet-00000-5ec1ffb8.era1";
|
||||
"mainnet-00000-00001-5ec1ffb8.era1";
|
||||
"Mainnet era 0"
|
||||
)]
|
||||
#[test_case::test_case(
|
||||
Era1Id::new("mainnet", 8192, 8192).with_hash([0x5e, 0xcb, 0x9b, 0xf9]),
|
||||
"mainnet-00001-5ecb9bf9.era1";
|
||||
"mainnet-00001-00001-5ecb9bf9.era1";
|
||||
"Mainnet era 1"
|
||||
)]
|
||||
#[test_case::test_case(
|
||||
Era1Id::new("sepolia", 0, 8192).with_hash([0x90, 0x91, 0x84, 0x72]),
|
||||
"sepolia-00000-90918472.era1";
|
||||
"sepolia-00000-00001-90918472.era1";
|
||||
"Sepolia era 0"
|
||||
)]
|
||||
#[test_case::test_case(
|
||||
Era1Id::new("sepolia", 155648, 8192).with_hash([0xfa, 0x77, 0x00, 0x19]),
|
||||
"sepolia-00019-fa770019.era1";
|
||||
"sepolia-00019-00001-fa770019.era1";
|
||||
"Sepolia era 19"
|
||||
)]
|
||||
#[test_case::test_case(
|
||||
Era1Id::new("mainnet", 1000, 100),
|
||||
"mainnet-00000-00000000.era1";
|
||||
"mainnet-00000-00001-00000000.era1";
|
||||
"ID without hash"
|
||||
)]
|
||||
#[test_case::test_case(
|
||||
Era1Id::new("sepolia", 101130240, 8192).with_hash([0xab, 0xcd, 0xef, 0x12]),
|
||||
"sepolia-12345-abcdef12.era1";
|
||||
"sepolia-12345-00001-abcdef12.era1";
|
||||
"Large block number era 12345"
|
||||
)]
|
||||
fn test_era1_id_file_naming(id: Era1Id, expected_file_name: &str) {
|
||||
let actual_file_name = id.to_file_name();
|
||||
assert_eq!(actual_file_name, expected_file_name);
|
||||
}
|
||||
|
||||
// File naming with era-count, for custom exports
|
||||
#[test_case::test_case(
|
||||
Era1Id::new("mainnet", 0, 8192).with_hash([0x5e, 0xc1, 0xff, 0xb8]).with_era_count(),
|
||||
"mainnet-00000-00001-5ec1ffb8.era1";
|
||||
"Mainnet era 0 with count"
|
||||
)]
|
||||
#[test_case::test_case(
|
||||
Era1Id::new("mainnet", 8000, 500).with_hash([0xab, 0xcd, 0xef, 0x12]).with_era_count(),
|
||||
"mainnet-00000-00002-abcdef12.era1";
|
||||
"Spanning two eras with count"
|
||||
)]
|
||||
fn test_era1_id_file_naming_with_era_count(id: Era1Id, expected_file_name: &str) {
|
||||
fn test_era1id_file_naming(id: Era1Id, expected_file_name: &str) {
|
||||
let actual_file_name = id.to_file_name();
|
||||
assert_eq!(actual_file_name, expected_file_name);
|
||||
}
|
||||
|
||||
@@ -170,7 +170,7 @@ impl DisplayHardforks {
|
||||
let mut post_merge = Vec::new();
|
||||
|
||||
for (fork, condition, metadata) in hardforks {
|
||||
let display_fork = DisplayFork {
|
||||
let mut display_fork = DisplayFork {
|
||||
name: fork.name().to_string(),
|
||||
activated_at: condition,
|
||||
eip: None,
|
||||
@@ -181,7 +181,12 @@ impl DisplayHardforks {
|
||||
ForkCondition::Block(_) => {
|
||||
pre_merge.push(display_fork);
|
||||
}
|
||||
ForkCondition::TTD { .. } => {
|
||||
ForkCondition::TTD { activation_block_number, total_difficulty, fork_block } => {
|
||||
display_fork.activated_at = ForkCondition::TTD {
|
||||
activation_block_number,
|
||||
fork_block,
|
||||
total_difficulty,
|
||||
};
|
||||
with_merge.push(display_fork);
|
||||
}
|
||||
ForkCondition::Timestamp(_) => {
|
||||
|
||||
@@ -61,7 +61,6 @@ reth-node-core.workspace = true
|
||||
reth-e2e-test-utils.workspace = true
|
||||
reth-tasks.workspace = true
|
||||
reth-testing-utils.workspace = true
|
||||
reth-stages-types.workspace = true
|
||||
tempfile.workspace = true
|
||||
jsonrpsee-core.workspace = true
|
||||
|
||||
@@ -89,9 +88,6 @@ asm-keccak = [
|
||||
"reth-node-core/asm-keccak",
|
||||
"revm/asm-keccak",
|
||||
]
|
||||
keccak-cache-global = [
|
||||
"alloy-primitives/keccak-cache-global",
|
||||
]
|
||||
js-tracer = [
|
||||
"reth-node-builder/js-tracer",
|
||||
"reth-rpc/js-tracer",
|
||||
@@ -110,5 +106,4 @@ test-utils = [
|
||||
"reth-evm/test-utils",
|
||||
"reth-primitives-traits/test-utils",
|
||||
"reth-evm-ethereum/test-utils",
|
||||
"reth-stages-types/test-utils",
|
||||
]
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
use crate::utils::eth_payload_attributes;
|
||||
use alloy_genesis::Genesis;
|
||||
use alloy_primitives::B256;
|
||||
use reth_chainspec::{ChainSpecBuilder, MAINNET};
|
||||
use reth_e2e_test_utils::{setup, transaction::TransactionTestContext};
|
||||
use reth_node_ethereum::EthereumNode;
|
||||
use reth_provider::{HeaderProvider, StageCheckpointReader};
|
||||
use reth_stages_types::StageId;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Tests that a node can initialize and advance with a custom genesis block number.
|
||||
#[tokio::test]
|
||||
async fn can_run_eth_node_with_custom_genesis_number() -> eyre::Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
// Create genesis with custom block number (e.g., 1000)
|
||||
let mut genesis: Genesis =
|
||||
serde_json::from_str(include_str!("../assets/genesis.json")).unwrap();
|
||||
genesis.number = Some(1000);
|
||||
genesis.parent_hash = Some(B256::random());
|
||||
|
||||
let chain_spec = Arc::new(
|
||||
ChainSpecBuilder::default()
|
||||
.chain(MAINNET.chain)
|
||||
.genesis(genesis)
|
||||
.cancun_activated()
|
||||
.build(),
|
||||
);
|
||||
|
||||
let (mut nodes, _tasks, wallet) =
|
||||
setup::<EthereumNode>(1, chain_spec, false, eth_payload_attributes).await?;
|
||||
|
||||
let mut node = nodes.pop().unwrap();
|
||||
|
||||
// Verify stage checkpoints are initialized to genesis block number (1000)
|
||||
for stage in StageId::ALL {
|
||||
let checkpoint = node.inner.provider.get_stage_checkpoint(stage)?;
|
||||
assert!(checkpoint.is_some(), "Stage {:?} checkpoint should exist", stage);
|
||||
assert_eq!(
|
||||
checkpoint.unwrap().block_number,
|
||||
1000,
|
||||
"Stage {:?} checkpoint should be at genesis block 1000",
|
||||
stage
|
||||
);
|
||||
}
|
||||
|
||||
// Advance the chain (block 1001)
|
||||
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
|
||||
let tx_hash = node.rpc.inject_tx(raw_tx).await?;
|
||||
let payload = node.advance_block().await?;
|
||||
|
||||
let block_hash = payload.block().hash();
|
||||
let block_number = payload.block().number;
|
||||
|
||||
// Verify we're at block 1001 (genesis + 1)
|
||||
assert_eq!(block_number, 1001, "Block number should be 1001 after advancing from genesis 1000");
|
||||
|
||||
// Assert the block has been committed
|
||||
node.assert_new_block(tx_hash, block_hash, block_number).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Tests that block queries respect custom genesis boundaries.
|
||||
#[tokio::test]
|
||||
async fn custom_genesis_block_query_boundaries() -> eyre::Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let genesis_number = 5000u64;
|
||||
|
||||
let mut genesis: Genesis =
|
||||
serde_json::from_str(include_str!("../assets/genesis.json")).unwrap();
|
||||
genesis.number = Some(genesis_number);
|
||||
genesis.parent_hash = Some(B256::random());
|
||||
|
||||
let chain_spec = Arc::new(
|
||||
ChainSpecBuilder::default()
|
||||
.chain(MAINNET.chain)
|
||||
.genesis(genesis)
|
||||
.cancun_activated()
|
||||
.build(),
|
||||
);
|
||||
|
||||
let (mut nodes, _tasks, _wallet) =
|
||||
setup::<EthereumNode>(1, chain_spec, false, eth_payload_attributes).await?;
|
||||
|
||||
let node = nodes.pop().unwrap();
|
||||
|
||||
// Query genesis block should succeed
|
||||
let genesis_header = node.inner.provider.header_by_number(genesis_number)?;
|
||||
assert!(genesis_header.is_some(), "Genesis block at {} should exist", genesis_number);
|
||||
|
||||
// Query blocks before genesis should return None
|
||||
for block_num in [0, 1, genesis_number - 1] {
|
||||
let header = node.inner.provider.header_by_number(block_num)?;
|
||||
assert!(header.is_none(), "Block {} before genesis should not exist", block_num);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -7,7 +7,6 @@ use reth_e2e_test_utils::{
|
||||
use reth_node_builder::{NodeBuilder, NodeHandle};
|
||||
use reth_node_core::{args::RpcServerArgs, node_config::NodeConfig};
|
||||
use reth_node_ethereum::EthereumNode;
|
||||
use reth_provider::BlockNumReader;
|
||||
use reth_tasks::TaskManager;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -128,55 +127,3 @@ async fn test_failed_run_eth_node_with_no_auth_engine_api_over_ipc_opts() -> eyr
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_engine_graceful_shutdown() -> eyre::Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let (mut nodes, _tasks, wallet) = setup::<EthereumNode>(
|
||||
1,
|
||||
Arc::new(
|
||||
ChainSpecBuilder::default()
|
||||
.chain(MAINNET.chain)
|
||||
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
|
||||
.cancun_activated()
|
||||
.build(),
|
||||
),
|
||||
false,
|
||||
eth_payload_attributes,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut node = nodes.pop().unwrap();
|
||||
|
||||
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
|
||||
let tx_hash = node.rpc.inject_tx(raw_tx).await?;
|
||||
let payload = node.advance_block().await?;
|
||||
node.assert_new_block(tx_hash, payload.block().hash(), payload.block().number).await?;
|
||||
|
||||
// Get block number before shutdown
|
||||
let block_before = node.inner.provider.best_block_number()?;
|
||||
assert_eq!(block_before, 1, "Expected 1 block before shutdown");
|
||||
|
||||
// Verify block is NOT yet persisted to database
|
||||
let db_block_before = node.inner.provider.last_block_number()?;
|
||||
assert_eq!(db_block_before, 0, "Block should not be persisted yet");
|
||||
|
||||
// Trigger graceful shutdown
|
||||
let done_rx = node
|
||||
.inner
|
||||
.add_ons_handle
|
||||
.engine_shutdown
|
||||
.shutdown()
|
||||
.expect("shutdown should return receiver");
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(2), done_rx)
|
||||
.await
|
||||
.expect("shutdown timed out")
|
||||
.expect("shutdown completion channel should not be closed");
|
||||
|
||||
let db_block = node.inner.provider.last_block_number()?;
|
||||
assert_eq!(db_block, 1, "Database should have persisted block 1");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
#![allow(missing_docs)]
|
||||
|
||||
mod blobs;
|
||||
mod custom_genesis;
|
||||
mod dev;
|
||||
mod eth;
|
||||
mod p2p;
|
||||
|
||||
@@ -79,9 +79,7 @@ arbitrary = [
|
||||
"alloy-rpc-types-engine?/arbitrary",
|
||||
"reth-codecs?/arbitrary",
|
||||
]
|
||||
keccak-cache-global = [
|
||||
"reth-node-ethereum?/keccak-cache-global",
|
||||
]
|
||||
|
||||
test-utils = [
|
||||
"reth-chainspec/test-utils",
|
||||
"reth-consensus?/test-utils",
|
||||
|
||||
@@ -373,13 +373,13 @@ impl<N: NetworkPrimitives> EthMessage<N> {
|
||||
///
|
||||
/// This handles up/downcasting where appropriate, for example for different receipt request
|
||||
/// types.
|
||||
pub fn map_versioned(self, version: EthVersion) -> Self {
|
||||
pub fn map_versioned(mut self, version: EthVersion) -> Self {
|
||||
// For eth/70 peers we send `GetReceipts` using the new eth/70
|
||||
// encoding with `firstBlockReceiptIndex = 0`, while keeping the
|
||||
// user-facing `PeerRequest` API unchanged.
|
||||
if version >= EthVersion::Eth70 {
|
||||
return match self {
|
||||
Self::GetReceipts(pair) => {
|
||||
EthMessage::GetReceipts(pair) => {
|
||||
let RequestPair { request_id, message } = pair;
|
||||
let req = RequestPair {
|
||||
request_id,
|
||||
@@ -388,7 +388,7 @@ impl<N: NetworkPrimitives> EthMessage<N> {
|
||||
block_hashes: message.0,
|
||||
},
|
||||
};
|
||||
Self::GetReceipts70(req)
|
||||
EthMessage::GetReceipts70(req)
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
|
||||
@@ -101,9 +101,8 @@ where
|
||||
.or(Err(P2PStreamError::HandshakeError(P2PHandshakeError::Timeout)))?
|
||||
.ok_or(P2PStreamError::HandshakeError(P2PHandshakeError::NoResponse))??;
|
||||
|
||||
// Check that the uncompressed message length does not exceed the max payload size.
|
||||
// Note: The first message (Hello/Disconnect) is not snappy compressed. We will check the
|
||||
// decompressed length again for subsequent messages after the handshake.
|
||||
// let's check the compressed length first, we will need to check again once confirming
|
||||
// that it contains snappy-compressed data (this will be the case for all non-p2p messages).
|
||||
if first_message_bytes.len() > MAX_PAYLOAD_SIZE {
|
||||
return Err(P2PStreamError::MessageTooBig {
|
||||
message_size: first_message_bytes.len(),
|
||||
|
||||
@@ -218,9 +218,6 @@ where
|
||||
let _ = response.send(Ok(Receipts69(receipts)));
|
||||
}
|
||||
|
||||
/// Handles partial responses for [`GetReceipts70`] queries.
|
||||
///
|
||||
/// This will adhere to the soft limit but allow filling the last vec partially.
|
||||
fn on_receipts70_request(
|
||||
&self,
|
||||
_peer_id: PeerId,
|
||||
|
||||
@@ -318,7 +318,7 @@ impl<N: NetworkPrimitives> ActiveSession<N> {
|
||||
fn on_internal_peer_request(&mut self, request: PeerRequest<N>, deadline: Instant) {
|
||||
let request_id = self.next_id();
|
||||
trace!(?request, peer_id=?self.remote_peer_id, ?request_id, "sending request to peer");
|
||||
let msg = request.create_request_message(request_id).map_versioned(self.conn.version());
|
||||
let mut msg = request.create_request_message(request_id).map_versioned(self.conn.version());
|
||||
|
||||
self.queued_outgoing.push_back(msg.into());
|
||||
let req = InflightRequest {
|
||||
@@ -374,6 +374,7 @@ impl<N: NetworkPrimitives> ActiveSession<N> {
|
||||
fn handle_outgoing_response(&mut self, id: u64, resp: PeerResponseResult<N>) {
|
||||
match resp.try_into_message(id) {
|
||||
Ok(msg) => {
|
||||
let msg: EthMessage<N> = msg;
|
||||
self.queued_outgoing.push_back(msg.into());
|
||||
}
|
||||
Err(err) => {
|
||||
|
||||
@@ -76,7 +76,6 @@ secp256k1 = { workspace = true, features = ["global-context", "std", "recovery"]
|
||||
## misc
|
||||
aquamarine.workspace = true
|
||||
eyre.workspace = true
|
||||
parking_lot.workspace = true
|
||||
jsonrpsee.workspace = true
|
||||
fdlimit.workspace = true
|
||||
rayon.workspace = true
|
||||
|
||||
@@ -483,12 +483,10 @@ where
|
||||
StaticFileProviderBuilder::read_write(self.data_dir().static_files())?
|
||||
.with_metrics()
|
||||
.with_blocks_per_file_for_segments(static_files_config.as_blocks_per_file_map())
|
||||
.with_genesis_block_number(self.chain_spec().genesis().number.unwrap_or_default())
|
||||
.build()?;
|
||||
|
||||
// Initialize RocksDB provider with metrics, statistics, and default tables
|
||||
// Initialize RocksDB provider with metrics and statistics enabled
|
||||
let rocksdb_provider = RocksDBProvider::builder(self.data_dir().rocksdb())
|
||||
.with_default_tables()
|
||||
.with_metrics()
|
||||
.with_statistics()
|
||||
.build()?;
|
||||
@@ -938,15 +936,11 @@ where
|
||||
///
|
||||
/// A target block hash if the pipeline is inconsistent, otherwise `None`.
|
||||
pub fn check_pipeline_consistency(&self) -> ProviderResult<Option<B256>> {
|
||||
// Get the expected first stage based on config.
|
||||
let first_stage =
|
||||
if self.era_import_source().is_some() { StageId::Era } else { StageId::Headers };
|
||||
|
||||
// If no target was provided, check if the stages are congruent - check if the
|
||||
// checkpoint of the last stage matches the checkpoint of the first.
|
||||
let first_stage_checkpoint = self
|
||||
.blockchain_db()
|
||||
.get_stage_checkpoint(first_stage)?
|
||||
.get_stage_checkpoint(*StageId::ALL.first().unwrap())?
|
||||
.unwrap_or_default()
|
||||
.block_number;
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use crate::{
|
||||
common::{Attached, LaunchContextWith, WithConfigs},
|
||||
hooks::NodeHooks,
|
||||
rpc::{EngineShutdown, EngineValidatorAddOn, EngineValidatorBuilder, RethRpcAddOns, RpcHandle},
|
||||
rpc::{EngineValidatorAddOn, EngineValidatorBuilder, RethRpcAddOns, RpcHandle},
|
||||
setup::build_networked_pipeline,
|
||||
AddOns, AddOnsContext, FullNode, LaunchContext, LaunchNode, NodeAdapter,
|
||||
NodeBuilderWithComponents, NodeComponents, NodeComponentsBuilder, NodeHandle, NodeTypesAdapter,
|
||||
@@ -13,7 +13,6 @@ use futures::{stream_select, StreamExt};
|
||||
use reth_chainspec::{EthChainSpec, EthereumHardforks};
|
||||
use reth_engine_service::service::{ChainEvent, EngineService};
|
||||
use reth_engine_tree::{
|
||||
chain::FromOrchestrator,
|
||||
engine::{EngineApiRequest, EngineRequestHandler},
|
||||
tree::TreeConfig,
|
||||
};
|
||||
@@ -261,16 +260,8 @@ impl EngineNodeLauncher {
|
||||
)),
|
||||
);
|
||||
|
||||
let RpcHandle {
|
||||
rpc_server_handles,
|
||||
rpc_registry,
|
||||
engine_events,
|
||||
beacon_engine_handle,
|
||||
engine_shutdown: _,
|
||||
} = add_ons.launch_add_ons(add_ons_ctx).await?;
|
||||
|
||||
// Create engine shutdown handle
|
||||
let (engine_shutdown, mut shutdown_rx) = EngineShutdown::new();
|
||||
let RpcHandle { rpc_server_handles, rpc_registry, engine_events, beacon_engine_handle } =
|
||||
add_ons.launch_add_ons(add_ons_ctx).await?;
|
||||
|
||||
// Run consensus engine to completion
|
||||
let initial_target = ctx.initial_backfill_target()?;
|
||||
@@ -304,14 +295,6 @@ impl EngineNodeLauncher {
|
||||
// advance the chain and await payloads built locally to add into the engine api tree handler to prevent re-execution if that block is received as payload from the CL
|
||||
loop {
|
||||
tokio::select! {
|
||||
shutdown_req = &mut shutdown_rx => {
|
||||
if let Ok(req) = shutdown_req {
|
||||
debug!(target: "reth::cli", "received engine shutdown request");
|
||||
engine_service.orchestrator_mut().handler_mut().handler_mut().on_event(
|
||||
FromOrchestrator::Terminate { tx: req.done_tx }.into()
|
||||
);
|
||||
}
|
||||
}
|
||||
payload = built_payloads.select_next_some() => {
|
||||
if let Some(executed_block) = payload.executed_block() {
|
||||
debug!(target: "reth::cli", block=?executed_block.recovered_block.num_hash(), "inserting built payload");
|
||||
@@ -383,7 +366,6 @@ impl EngineNodeLauncher {
|
||||
rpc_registry,
|
||||
engine_events,
|
||||
beacon_engine_handle,
|
||||
engine_shutdown,
|
||||
},
|
||||
};
|
||||
// Notify on node started
|
||||
|
||||
@@ -11,7 +11,6 @@ use crate::{
|
||||
use alloy_rpc_types::engine::ClientVersionV1;
|
||||
use alloy_rpc_types_engine::ExecutionData;
|
||||
use jsonrpsee::{core::middleware::layer::Either, RpcModule};
|
||||
use parking_lot::Mutex;
|
||||
use reth_chain_state::CanonStateSubscriptions;
|
||||
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks, Hardforks};
|
||||
use reth_node_api::{
|
||||
@@ -42,9 +41,7 @@ use std::{
|
||||
fmt::{self, Debug},
|
||||
future::Future,
|
||||
ops::{Deref, DerefMut},
|
||||
sync::Arc,
|
||||
};
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
/// Contains the handles to the spawned RPC servers.
|
||||
///
|
||||
@@ -335,8 +332,6 @@ pub struct RpcHandle<Node: FullNodeComponents, EthApi: EthApiTypes> {
|
||||
pub engine_events: EventSender<ConsensusEngineEvent<<Node::Types as NodeTypes>::Primitives>>,
|
||||
/// Handle to the beacon consensus engine.
|
||||
pub beacon_engine_handle: ConsensusEngineHandle<<Node::Types as NodeTypes>::Payload>,
|
||||
/// Handle to trigger engine shutdown.
|
||||
pub engine_shutdown: EngineShutdown,
|
||||
}
|
||||
|
||||
impl<Node: FullNodeComponents, EthApi: EthApiTypes> Clone for RpcHandle<Node, EthApi> {
|
||||
@@ -346,7 +341,6 @@ impl<Node: FullNodeComponents, EthApi: EthApiTypes> Clone for RpcHandle<Node, Et
|
||||
rpc_registry: self.rpc_registry.clone(),
|
||||
engine_events: self.engine_events.clone(),
|
||||
beacon_engine_handle: self.beacon_engine_handle.clone(),
|
||||
engine_shutdown: self.engine_shutdown.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -367,7 +361,6 @@ where
|
||||
f.debug_struct("RpcHandle")
|
||||
.field("rpc_server_handles", &self.rpc_server_handles)
|
||||
.field("rpc_registry", &self.rpc_registry)
|
||||
.field("engine_shutdown", &self.engine_shutdown)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -963,7 +956,6 @@ where
|
||||
rpc_registry: registry,
|
||||
engine_events,
|
||||
beacon_engine_handle: engine_handle,
|
||||
engine_shutdown: EngineShutdown::default(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1389,7 +1381,6 @@ where
|
||||
version: version_metadata().cargo_pkg_version.to_string(),
|
||||
commit: version_metadata().vergen_git_sha.to_string(),
|
||||
};
|
||||
|
||||
Ok(EngineApi::new(
|
||||
ctx.node.provider().clone(),
|
||||
ctx.config.chain.clone(),
|
||||
@@ -1401,7 +1392,6 @@ where
|
||||
EngineCapabilities::default(),
|
||||
engine_validator,
|
||||
ctx.config.engine.accept_execution_requests_hash,
|
||||
ctx.node.network().clone(),
|
||||
))
|
||||
}
|
||||
}
|
||||
@@ -1436,48 +1426,3 @@ impl IntoEngineApiRpcModule for NoopEngineApi {
|
||||
RpcModule::new(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle to trigger graceful engine shutdown.
|
||||
///
|
||||
/// This handle can be used to request a graceful shutdown of the engine,
|
||||
/// which will persist all remaining in-memory blocks before terminating.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct EngineShutdown {
|
||||
/// Channel to send shutdown signal.
|
||||
tx: Arc<Mutex<Option<oneshot::Sender<EngineShutdownRequest>>>>,
|
||||
}
|
||||
|
||||
impl EngineShutdown {
|
||||
/// Creates a new [`EngineShutdown`] handle and returns the receiver.
|
||||
pub fn new() -> (Self, oneshot::Receiver<EngineShutdownRequest>) {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
(Self { tx: Arc::new(Mutex::new(Some(tx))) }, rx)
|
||||
}
|
||||
|
||||
/// Requests a graceful engine shutdown.
|
||||
///
|
||||
/// All remaining in-memory blocks will be persisted before the engine terminates.
|
||||
///
|
||||
/// Returns a receiver that resolves when shutdown is complete.
|
||||
/// Returns `None` if shutdown was already triggered.
|
||||
pub fn shutdown(&self) -> Option<oneshot::Receiver<()>> {
|
||||
let mut guard = self.tx.lock();
|
||||
let tx = guard.take()?;
|
||||
let (done_tx, done_rx) = oneshot::channel();
|
||||
let _ = tx.send(EngineShutdownRequest { done_tx });
|
||||
Some(done_rx)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for EngineShutdown {
|
||||
fn default() -> Self {
|
||||
Self { tx: Arc::new(Mutex::new(None)) }
|
||||
}
|
||||
}
|
||||
|
||||
/// Request to shutdown the engine.
|
||||
#[derive(Debug)]
|
||||
pub struct EngineShutdownRequest {
|
||||
/// Channel to signal shutdown completion.
|
||||
pub done_tx: oneshot::Sender<()>,
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ use reth_network::{
|
||||
DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ,
|
||||
SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE,
|
||||
},
|
||||
HelloMessageWithProtocols, NetworkConfigBuilder, NetworkPrimitives,
|
||||
HelloMessageWithProtocols, NetworkConfigBuilder, NetworkPrimitives, SessionsConfig,
|
||||
};
|
||||
use reth_network_peers::{mainnet_nodes, TrustedPeer};
|
||||
use secp256k1::SecretKey;
|
||||
@@ -339,7 +339,7 @@ impl NetworkArgs {
|
||||
NetworkConfigBuilder::<N>::new(secret_key)
|
||||
.external_ip_resolver(self.nat.clone())
|
||||
.sessions_config(
|
||||
config.sessions.clone().with_upscaled_event_buffer(peers_config.max_peers()),
|
||||
SessionsConfig::default().with_upscaled_event_buffer(peers_config.max_peers()),
|
||||
)
|
||||
.peer_config(peers_config)
|
||||
.boot_nodes(chain_bootnodes.clone())
|
||||
|
||||
@@ -208,7 +208,7 @@ where
|
||||
active: true,
|
||||
syncing: self.network.is_syncing(),
|
||||
peers: self.network.num_connected_peers() as u64,
|
||||
gas_price: self.pool.block_info().pending_basefee,
|
||||
gas_price: 0, // TODO
|
||||
uptime: 100,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -27,7 +27,7 @@ tracing.workspace = true
|
||||
workspace = true
|
||||
|
||||
[features]
|
||||
default = ["jemalloc", "otlp", "reth-optimism-evm/portable", "js-tracer", "keccak-cache-global", "asm-keccak"]
|
||||
default = ["jemalloc", "otlp", "reth-optimism-evm/portable", "js-tracer"]
|
||||
|
||||
otlp = ["reth-optimism-cli/otlp"]
|
||||
|
||||
@@ -40,9 +40,7 @@ jemalloc-prof = ["reth-cli-util/jemalloc-prof"]
|
||||
tracy-allocator = ["reth-cli-util/tracy-allocator"]
|
||||
|
||||
asm-keccak = ["reth-optimism-cli/asm-keccak", "reth-optimism-node/asm-keccak"]
|
||||
keccak-cache-global = [
|
||||
"reth-optimism-node/keccak-cache-global",
|
||||
]
|
||||
|
||||
dev = [
|
||||
"reth-optimism-cli/dev",
|
||||
"reth-optimism-primitives/arbitrary",
|
||||
|
||||
@@ -80,7 +80,6 @@ reth-payload-util.workspace = true
|
||||
reth-revm = { workspace = true, features = ["std"] }
|
||||
reth-rpc.workspace = true
|
||||
reth-rpc-eth-types.workspace = true
|
||||
reth-stages-types.workspace = true
|
||||
|
||||
alloy-network.workspace = true
|
||||
futures.workspace = true
|
||||
@@ -94,10 +93,6 @@ asm-keccak = [
|
||||
"reth-node-core/asm-keccak",
|
||||
"revm/asm-keccak",
|
||||
]
|
||||
keccak-cache-global = [
|
||||
"alloy-primitives/keccak-cache-global",
|
||||
"reth-optimism-node/keccak-cache-global",
|
||||
]
|
||||
js-tracer = [
|
||||
"reth-node-builder/js-tracer",
|
||||
"reth-optimism-node/js-tracer",
|
||||
@@ -123,7 +118,6 @@ test-utils = [
|
||||
"reth-optimism-primitives/arbitrary",
|
||||
"reth-primitives-traits/test-utils",
|
||||
"reth-trie-common/test-utils",
|
||||
"reth-stages-types/test-utils",
|
||||
]
|
||||
reth-codec = ["reth-optimism-primitives/reth-codec"]
|
||||
|
||||
|
||||
@@ -146,7 +146,6 @@ where
|
||||
EngineCapabilities::new(OP_ENGINE_CAPABILITIES.iter().copied()),
|
||||
engine_validator,
|
||||
ctx.config.engine.accept_execution_requests_hash,
|
||||
ctx.node.network().clone(),
|
||||
);
|
||||
|
||||
Ok(OpEngineApi::new(inner))
|
||||
|
||||
@@ -1,123 +0,0 @@
|
||||
//! Tests for custom genesis block number support.
|
||||
|
||||
use alloy_consensus::BlockHeader;
|
||||
use alloy_genesis::Genesis;
|
||||
use alloy_primitives::B256;
|
||||
use reth_chainspec::EthChainSpec;
|
||||
use reth_db::test_utils::create_test_rw_db_with_path;
|
||||
use reth_e2e_test_utils::{
|
||||
node::NodeTestContext, transaction::TransactionTestContext, wallet::Wallet,
|
||||
};
|
||||
use reth_node_builder::{EngineNodeLauncher, Node, NodeBuilder, NodeConfig};
|
||||
use reth_node_core::args::DatadirArgs;
|
||||
use reth_optimism_chainspec::OpChainSpecBuilder;
|
||||
use reth_optimism_node::{utils::optimism_payload_attributes, OpNode};
|
||||
use reth_provider::{providers::BlockchainProvider, HeaderProvider, StageCheckpointReader};
|
||||
use reth_stages_types::StageId;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
/// Tests that an OP node can initialize with a custom genesis block number.
|
||||
#[tokio::test]
|
||||
async fn test_op_node_custom_genesis_number() {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let genesis_number = 1000;
|
||||
|
||||
// Create genesis with custom block number (1000)
|
||||
let mut genesis: Genesis =
|
||||
serde_json::from_str(include_str!("../assets/genesis.json")).unwrap();
|
||||
genesis.number = Some(genesis_number);
|
||||
genesis.parent_hash = Some(B256::random());
|
||||
|
||||
let chain_spec =
|
||||
Arc::new(OpChainSpecBuilder::base_mainnet().genesis(genesis).ecotone_activated().build());
|
||||
|
||||
let wallet = Arc::new(Mutex::new(Wallet::default().with_chain_id(chain_spec.chain().into())));
|
||||
|
||||
// Configure and launch the node
|
||||
let config = NodeConfig::new(chain_spec.clone()).with_datadir_args(DatadirArgs {
|
||||
datadir: reth_db::test_utils::tempdir_path().into(),
|
||||
..Default::default()
|
||||
});
|
||||
let db = create_test_rw_db_with_path(
|
||||
config
|
||||
.datadir
|
||||
.datadir
|
||||
.unwrap_or_chain_default(config.chain.chain(), config.datadir.clone())
|
||||
.db(),
|
||||
);
|
||||
let tasks = reth_tasks::TaskManager::current();
|
||||
let node_handle = NodeBuilder::new(config.clone())
|
||||
.with_database(db)
|
||||
.with_types_and_provider::<OpNode, BlockchainProvider<_>>()
|
||||
.with_components(OpNode::default().components())
|
||||
.with_add_ons(OpNode::new(Default::default()).add_ons())
|
||||
.launch_with_fn(|builder| {
|
||||
let launcher = EngineNodeLauncher::new(
|
||||
tasks.executor(),
|
||||
builder.config.datadir(),
|
||||
Default::default(),
|
||||
);
|
||||
builder.launch_with(launcher)
|
||||
})
|
||||
.await
|
||||
.expect("Failed to launch node");
|
||||
|
||||
let mut node =
|
||||
NodeTestContext::new(node_handle.node, optimism_payload_attributes).await.unwrap();
|
||||
|
||||
// Verify stage checkpoints are initialized to genesis block number (1000)
|
||||
for stage in StageId::ALL {
|
||||
let checkpoint = node.inner.provider.get_stage_checkpoint(stage).unwrap();
|
||||
assert!(checkpoint.is_some(), "Stage {:?} checkpoint should exist", stage);
|
||||
assert_eq!(
|
||||
checkpoint.unwrap().block_number,
|
||||
1000,
|
||||
"Stage {:?} checkpoint should be at genesis block 1000",
|
||||
stage
|
||||
);
|
||||
}
|
||||
|
||||
// Query genesis block should succeed
|
||||
let genesis_header = node.inner.provider.header_by_number(genesis_number).unwrap();
|
||||
assert!(genesis_header.is_some(), "Genesis block at {} should exist", genesis_number);
|
||||
|
||||
// Query blocks before genesis should return None
|
||||
for block_num in [0, 1, genesis_number - 1] {
|
||||
let header = node.inner.provider.header_by_number(block_num).unwrap();
|
||||
assert!(header.is_none(), "Block {} before genesis should not exist", block_num);
|
||||
}
|
||||
|
||||
// Advance the chain with a single block
|
||||
let _ = wallet; // wallet available for future use
|
||||
let block_payloads = node
|
||||
.advance(1, |_| {
|
||||
Box::pin({
|
||||
let value = wallet.clone();
|
||||
async move {
|
||||
let mut wallet = value.lock().await;
|
||||
let tx_fut = TransactionTestContext::optimism_l1_block_info_tx(
|
||||
wallet.chain_id,
|
||||
wallet.inner.clone(),
|
||||
wallet.inner_nonce,
|
||||
);
|
||||
wallet.inner_nonce += 1;
|
||||
|
||||
tx_fut.await
|
||||
}
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(block_payloads.len(), 1);
|
||||
let block = block_payloads.first().unwrap().block();
|
||||
|
||||
// Verify the new block is at 1001 (genesis 1000 + 1)
|
||||
assert_eq!(
|
||||
block.number(),
|
||||
1001,
|
||||
"Block number should be 1001 after advancing from genesis 100"
|
||||
);
|
||||
}
|
||||
@@ -6,6 +6,4 @@ mod priority;
|
||||
|
||||
mod rpc;
|
||||
|
||||
mod custom_genesis;
|
||||
|
||||
const fn main() {}
|
||||
|
||||
@@ -74,9 +74,7 @@ arbitrary = [
|
||||
"reth-eth-wire?/arbitrary",
|
||||
"reth-codecs?/arbitrary",
|
||||
]
|
||||
keccak-cache-global = [
|
||||
"reth-optimism-node?/keccak-cache-global",
|
||||
]
|
||||
|
||||
test-utils = [
|
||||
"reth-chainspec/test-utils",
|
||||
"reth-consensus?/test-utils",
|
||||
|
||||
@@ -7,7 +7,7 @@ use alloy_rpc_types_eth::{Log, TransactionReceipt};
|
||||
use op_alloy_consensus::{OpReceipt, OpTransaction};
|
||||
use op_alloy_rpc_types::{L1BlockInfo, OpTransactionReceipt, OpTransactionReceiptFields};
|
||||
use op_revm::estimate_tx_compressed_size;
|
||||
use reth_chainspec::{ChainSpecProvider, EthChainSpec};
|
||||
use reth_chainspec::ChainSpecProvider;
|
||||
use reth_node_api::NodePrimitives;
|
||||
use reth_optimism_evm::RethL1BlockInfo;
|
||||
use reth_optimism_forks::OpHardforks;
|
||||
@@ -74,11 +74,9 @@ where
|
||||
let mut l1_block_info = match reth_optimism_evm::extract_l1_info(block.body()) {
|
||||
Ok(l1_block_info) => l1_block_info,
|
||||
Err(err) => {
|
||||
let genesis_number =
|
||||
self.provider.chain_spec().genesis().number.unwrap_or_default();
|
||||
// If it is the genesis block (i.e. block number is 0), there is no L1 info, so
|
||||
// we return an empty l1_block_info.
|
||||
if block.header().number() == genesis_number {
|
||||
if block.header().number() == 0 {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
return Err(err.into());
|
||||
|
||||
@@ -19,11 +19,8 @@ pub trait PayloadTransactions {
|
||||
ctx: (),
|
||||
) -> Option<Self::Transaction>;
|
||||
|
||||
/// Marks the transaction identified by `sender` and `nonce` as invalid for this iterator.
|
||||
///
|
||||
/// Implementations must ensure that subsequent transactions returned from this iterator do not
|
||||
/// depend on this transaction. For example, they may choose to stop yielding any further
|
||||
/// transactions from this sender in the current iteration.
|
||||
/// Exclude descendants of the transaction with given sender and nonce from the iterator,
|
||||
/// because this transaction won't be included in the block.
|
||||
fn mark_invalid(&mut self, sender: Address, nonce: u64);
|
||||
}
|
||||
|
||||
@@ -49,9 +46,6 @@ impl<T> PayloadTransactions for NoopPayloadTransactions<T> {
|
||||
|
||||
/// Wrapper struct that allows to convert `BestTransactions` (used in tx pool) to
|
||||
/// `PayloadTransactions` (used in block composition).
|
||||
///
|
||||
/// Note: `mark_invalid` for this type filters out all further transactions from the given sender
|
||||
/// in the current iteration, mirroring the semantics of `BestTransactions::mark_invalid`.
|
||||
#[derive(Debug)]
|
||||
pub struct BestPayloadTransactions<T, I>
|
||||
where
|
||||
|
||||
@@ -218,7 +218,7 @@ impl<B: Block> RecoveredBlock<B> {
|
||||
|
||||
/// A safer variant of [`Self::new_sealed`] that checks if the number of senders is equal to
|
||||
/// the number of transactions in the block and recovers the senders from the transactions, if
|
||||
/// not using [`SignedTransaction::recover_signer`](crate::transaction::signed::SignedTransaction)
|
||||
/// not using [`SignedTransaction::recover_signer_unchecked`](crate::transaction::signed::SignedTransaction)
|
||||
/// to recover the senders.
|
||||
///
|
||||
/// Returns an error if any of the transactions fail to recover the sender.
|
||||
|
||||
@@ -179,7 +179,7 @@ impl<B: Block> SealedBlock<B> {
|
||||
|
||||
/// Recovers all senders from the transactions in the block.
|
||||
///
|
||||
/// Returns an error if any of the transactions fail to recover the sender.
|
||||
/// Returns `None` if any of the transactions fail to recover the sender.
|
||||
pub fn senders(&self) -> Result<Vec<Address>, RecoveryError> {
|
||||
self.body().recover_signers()
|
||||
}
|
||||
|
||||
@@ -94,7 +94,7 @@ impl<H: Sealable> SealedHeader<H> {
|
||||
*self.hash_ref()
|
||||
}
|
||||
|
||||
/// This is the inverse of [`Self::seal_slow`] which returns the raw header and hash.
|
||||
/// This is the inverse of [`Header::seal_slow`] which returns the raw header and hash.
|
||||
pub fn split(self) -> (H, BlockHash) {
|
||||
let hash = self.hash();
|
||||
(self.header, hash)
|
||||
|
||||
@@ -42,12 +42,13 @@ fn validate_blob_tx(
|
||||
blob_sidecar.blobs.extend(blob_sidecar_ext.blobs);
|
||||
blob_sidecar.proofs.extend(blob_sidecar_ext.proofs);
|
||||
blob_sidecar.commitments.extend(blob_sidecar_ext.commitments);
|
||||
}
|
||||
|
||||
// ensure exactly num_blobs blobs
|
||||
blob_sidecar.blobs.truncate(num_blobs as usize);
|
||||
blob_sidecar.proofs.truncate(num_blobs as usize);
|
||||
blob_sidecar.commitments.truncate(num_blobs as usize);
|
||||
if blob_sidecar.blobs.len() > num_blobs as usize {
|
||||
blob_sidecar.blobs.truncate(num_blobs as usize);
|
||||
blob_sidecar.proofs.truncate(num_blobs as usize);
|
||||
blob_sidecar.commitments.truncate(num_blobs as usize);
|
||||
}
|
||||
}
|
||||
|
||||
tx.blob_versioned_hashes = blob_sidecar.versioned_hashes().collect();
|
||||
|
||||
|
||||
@@ -240,18 +240,6 @@ pub trait EngineApi<Engine: EngineTypes> {
|
||||
&self,
|
||||
versioned_hashes: Vec<B256>,
|
||||
) -> RpcResult<Option<Vec<BlobAndProofV2>>>;
|
||||
|
||||
/// Fetch blobs for the consensus layer from the blob store.
|
||||
///
|
||||
/// Returns a response of the same length as the request. Missing or older-version blobs are
|
||||
/// returned as `null` elements.
|
||||
///
|
||||
/// Returns `null` if syncing.
|
||||
#[method(name = "getBlobsV3")]
|
||||
async fn get_blobs_v3(
|
||||
&self,
|
||||
versioned_hashes: Vec<B256>,
|
||||
) -> RpcResult<Option<Vec<Option<BlobAndProofV2>>>>;
|
||||
}
|
||||
|
||||
/// A subset of the ETH rpc interface: <https://ethereum.github.io/execution-apis/api-documentation>
|
||||
|
||||
@@ -54,7 +54,6 @@ pub async fn launch_auth(secret: JwtSecret) -> AuthServerHandle {
|
||||
EngineCapabilities::default(),
|
||||
EthereumEngineValidator::new(MAINNET.clone()),
|
||||
false,
|
||||
NoopNetwork::default(),
|
||||
);
|
||||
let module = AuthRpcModule::new(engine_api);
|
||||
module.start_server(config).await.unwrap()
|
||||
|
||||
@@ -23,7 +23,6 @@ reth-tasks.workspace = true
|
||||
reth-engine-primitives.workspace = true
|
||||
reth-transaction-pool.workspace = true
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-network-api.workspace = true
|
||||
|
||||
# ethereum
|
||||
alloy-eips.workspace = true
|
||||
|
||||
@@ -19,7 +19,6 @@ pub const CAPABILITIES: &[&str] = &[
|
||||
"engine_getPayloadBodiesByRangeV1",
|
||||
"engine_getBlobsV1",
|
||||
"engine_getBlobsV2",
|
||||
"engine_getBlobsV3",
|
||||
];
|
||||
|
||||
// The list of all supported Engine capabilities available over the engine endpoint.
|
||||
|
||||
@@ -18,7 +18,6 @@ use async_trait::async_trait;
|
||||
use jsonrpsee_core::{server::RpcModule, RpcResult};
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_engine_primitives::{ConsensusEngineHandle, EngineApiValidator, EngineTypes};
|
||||
use reth_network_api::NetworkInfo;
|
||||
use reth_payload_builder::PayloadStore;
|
||||
use reth_payload_primitives::{
|
||||
validate_payload_timestamp, EngineApiMessageVersion, MessageValidationKind,
|
||||
@@ -95,9 +94,7 @@ where
|
||||
capabilities: EngineCapabilities,
|
||||
validator: Validator,
|
||||
accept_execution_requests_hash: bool,
|
||||
network: impl NetworkInfo + 'static,
|
||||
) -> Self {
|
||||
let is_syncing = Arc::new(move || network.is_syncing());
|
||||
let inner = Arc::new(EngineApiInner {
|
||||
provider,
|
||||
chain_spec,
|
||||
@@ -110,7 +107,6 @@ where
|
||||
tx_pool,
|
||||
validator,
|
||||
accept_execution_requests_hash,
|
||||
is_syncing,
|
||||
});
|
||||
Self { inner }
|
||||
}
|
||||
@@ -796,35 +792,6 @@ where
|
||||
.map_err(|err| EngineApiError::Internal(Box::new(err)))
|
||||
}
|
||||
|
||||
fn get_blobs_v3(
|
||||
&self,
|
||||
versioned_hashes: Vec<B256>,
|
||||
) -> EngineApiResult<Option<Vec<Option<BlobAndProofV2>>>> {
|
||||
// Check if Osaka fork is active
|
||||
let current_timestamp =
|
||||
SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
if !self.inner.chain_spec.is_osaka_active_at_timestamp(current_timestamp) {
|
||||
return Err(EngineApiError::EngineObjectValidationError(
|
||||
reth_payload_primitives::EngineObjectValidationError::UnsupportedFork,
|
||||
));
|
||||
}
|
||||
|
||||
if versioned_hashes.len() > MAX_BLOB_LIMIT {
|
||||
return Err(EngineApiError::BlobRequestTooLarge { len: versioned_hashes.len() })
|
||||
}
|
||||
|
||||
// Spec requires returning `null` if syncing.
|
||||
if (*self.inner.is_syncing)() {
|
||||
return Ok(None)
|
||||
}
|
||||
|
||||
self.inner
|
||||
.tx_pool
|
||||
.get_blobs_for_versioned_hashes_v3(&versioned_hashes)
|
||||
.map(Some)
|
||||
.map_err(|err| EngineApiError::Internal(Box::new(err)))
|
||||
}
|
||||
|
||||
/// Metered version of `get_blobs_v2`.
|
||||
pub fn get_blobs_v2_metered(
|
||||
&self,
|
||||
@@ -860,27 +827,6 @@ where
|
||||
|
||||
res
|
||||
}
|
||||
|
||||
/// Metered version of `get_blobs_v3`.
|
||||
pub fn get_blobs_v3_metered(
|
||||
&self,
|
||||
versioned_hashes: Vec<B256>,
|
||||
) -> EngineApiResult<Option<Vec<Option<BlobAndProofV2>>>> {
|
||||
let hashes_len = versioned_hashes.len();
|
||||
let start = Instant::now();
|
||||
let res = Self::get_blobs_v3(self, versioned_hashes);
|
||||
self.inner.metrics.latency.get_blobs_v3.record(start.elapsed());
|
||||
|
||||
if let Ok(Some(blobs)) = &res {
|
||||
let blobs_found = blobs.iter().flatten().count();
|
||||
let blobs_missed = hashes_len - blobs_found;
|
||||
|
||||
self.inner.metrics.blob_metrics.blob_count.increment(blobs_found as u64);
|
||||
self.inner.metrics.blob_metrics.blob_misses.increment(blobs_missed as u64);
|
||||
}
|
||||
|
||||
res
|
||||
}
|
||||
}
|
||||
|
||||
// This is the concrete ethereum engine API implementation.
|
||||
@@ -1153,14 +1099,6 @@ where
|
||||
trace!(target: "rpc::engine", "Serving engine_getBlobsV2");
|
||||
Ok(self.get_blobs_v2_metered(versioned_hashes)?)
|
||||
}
|
||||
|
||||
async fn get_blobs_v3(
|
||||
&self,
|
||||
versioned_hashes: Vec<B256>,
|
||||
) -> RpcResult<Option<Vec<Option<BlobAndProofV2>>>> {
|
||||
trace!(target: "rpc::engine", "Serving engine_getBlobsV3");
|
||||
Ok(self.get_blobs_v3_metered(versioned_hashes)?)
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider, EngineT, Pool, Validator, ChainSpec> IntoEngineApiRpcModule
|
||||
@@ -1217,8 +1155,6 @@ struct EngineApiInner<Provider, PayloadT: PayloadTypes, Pool, Validator, ChainSp
|
||||
/// Engine validator.
|
||||
validator: Validator,
|
||||
accept_execution_requests_hash: bool,
|
||||
/// Returns `true` if the node is currently syncing.
|
||||
is_syncing: Arc<dyn Fn() -> bool + Send + Sync>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1226,13 +1162,10 @@ mod tests {
|
||||
use super::*;
|
||||
use alloy_rpc_types_engine::{ClientCode, ClientVersionV1};
|
||||
use assert_matches::assert_matches;
|
||||
use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET};
|
||||
use reth_chainspec::{ChainSpec, MAINNET};
|
||||
use reth_engine_primitives::BeaconEngineMessage;
|
||||
use reth_ethereum_engine_primitives::EthEngineTypes;
|
||||
use reth_ethereum_primitives::Block;
|
||||
use reth_network_api::{
|
||||
noop::NoopNetwork, EthProtocolInfo, NetworkError, NetworkInfo, NetworkStatus,
|
||||
};
|
||||
use reth_node_ethereum::EthereumEngineValidator;
|
||||
use reth_payload_builder::test_utils::spawn_test_payload_service;
|
||||
use reth_provider::test_utils::MockEthProvider;
|
||||
@@ -1273,7 +1206,6 @@ mod tests {
|
||||
EngineCapabilities::default(),
|
||||
EthereumEngineValidator::new(chain_spec.clone()),
|
||||
false,
|
||||
NoopNetwork::default(),
|
||||
);
|
||||
let handle = EngineApiTestHandle { chain_spec, provider, from_api: engine_rx };
|
||||
(handle, api)
|
||||
@@ -1315,76 +1247,6 @@ mod tests {
|
||||
assert_matches!(handle.from_api.recv().await, Some(BeaconEngineMessage::NewPayload { .. }));
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TestNetworkInfo {
|
||||
syncing: bool,
|
||||
}
|
||||
|
||||
impl NetworkInfo for TestNetworkInfo {
|
||||
fn local_addr(&self) -> std::net::SocketAddr {
|
||||
(std::net::Ipv4Addr::UNSPECIFIED, 0).into()
|
||||
}
|
||||
|
||||
async fn network_status(&self) -> Result<NetworkStatus, NetworkError> {
|
||||
#[allow(deprecated)]
|
||||
Ok(NetworkStatus {
|
||||
client_version: "test".to_string(),
|
||||
protocol_version: 5,
|
||||
eth_protocol_info: EthProtocolInfo {
|
||||
network: 1,
|
||||
difficulty: None,
|
||||
genesis: Default::default(),
|
||||
config: Default::default(),
|
||||
head: Default::default(),
|
||||
},
|
||||
capabilities: vec![],
|
||||
})
|
||||
}
|
||||
|
||||
fn chain_id(&self) -> u64 {
|
||||
1
|
||||
}
|
||||
|
||||
fn is_syncing(&self) -> bool {
|
||||
self.syncing
|
||||
}
|
||||
|
||||
fn is_initially_syncing(&self) -> bool {
|
||||
self.syncing
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_blobs_v3_returns_null_when_syncing() {
|
||||
let chain_spec: Arc<ChainSpec> =
|
||||
Arc::new(ChainSpecBuilder::mainnet().osaka_activated().build());
|
||||
let provider = Arc::new(MockEthProvider::default());
|
||||
let payload_store = spawn_test_payload_service::<EthEngineTypes>();
|
||||
let (to_engine, _engine_rx) = unbounded_channel::<BeaconEngineMessage<EthEngineTypes>>();
|
||||
|
||||
let api = EngineApi::new(
|
||||
provider,
|
||||
chain_spec.clone(),
|
||||
ConsensusEngineHandle::new(to_engine),
|
||||
payload_store.into(),
|
||||
NoopTransactionPool::default(),
|
||||
Box::<TokioTaskExecutor>::default(),
|
||||
ClientVersionV1 {
|
||||
code: ClientCode::RH,
|
||||
name: "Reth".to_string(),
|
||||
version: "v0.0.0-test".to_string(),
|
||||
commit: "test".to_string(),
|
||||
},
|
||||
EngineCapabilities::default(),
|
||||
EthereumEngineValidator::new(chain_spec),
|
||||
false,
|
||||
TestNetworkInfo { syncing: true },
|
||||
);
|
||||
|
||||
let res = api.get_blobs_v3_metered(vec![B256::ZERO]);
|
||||
assert_matches!(res, Ok(None));
|
||||
}
|
||||
|
||||
// tests covering `engine_getPayloadBodiesByRange` and `engine_getPayloadBodiesByHash`
|
||||
mod get_payload_bodies {
|
||||
use super::*;
|
||||
|
||||
@@ -46,8 +46,6 @@ pub(crate) struct EngineApiLatencyMetrics {
|
||||
pub(crate) get_blobs_v1: Histogram,
|
||||
/// Latency for `engine_getBlobsV2`
|
||||
pub(crate) get_blobs_v2: Histogram,
|
||||
/// Latency for `engine_getBlobsV3`
|
||||
pub(crate) get_blobs_v3: Histogram,
|
||||
}
|
||||
|
||||
#[derive(Metrics)]
|
||||
|
||||
@@ -92,7 +92,7 @@ where
|
||||
///
|
||||
/// Can fail if the element is rejected by the limiter or if we fail to grow an empty map.
|
||||
///
|
||||
/// See [`LruMap::insert`] for more info.
|
||||
/// See [`Schnellru::insert`](LruMap::insert) for more info.
|
||||
pub fn insert<'a>(&mut self, key: L::KeyToInsert<'a>, value: V) -> bool
|
||||
where
|
||||
L::KeyToInsert<'a>: Hash + PartialEq<K>,
|
||||
|
||||
@@ -175,6 +175,8 @@ where
|
||||
// need to apply the state changes of this call before executing the
|
||||
// next call
|
||||
if calls.peek().is_some() {
|
||||
// need to apply the state changes of this call before executing
|
||||
// the next call
|
||||
db.commit(res.state)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,7 +75,6 @@ reth-network-p2p = { workspace = true, features = ["test-utils"] }
|
||||
reth-downloaders.workspace = true
|
||||
reth-static-file.workspace = true
|
||||
reth-stages-api = { workspace = true, features = ["test-utils"] }
|
||||
reth-storage-api.workspace = true
|
||||
reth-testing-utils.workspace = true
|
||||
reth-trie = { workspace = true, features = ["test-utils"] }
|
||||
reth-provider = { workspace = true, features = ["test-utils"] }
|
||||
@@ -117,7 +116,6 @@ test-utils = [
|
||||
"reth-ethereum-primitives?/test-utils",
|
||||
"reth-evm-ethereum/test-utils",
|
||||
]
|
||||
rocksdb = ["reth-provider/rocksdb"]
|
||||
|
||||
[[bench]]
|
||||
name = "criterion"
|
||||
|
||||
@@ -23,7 +23,7 @@ use reth_stages_api::{
|
||||
};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use std::{
|
||||
cmp::{max, Ordering},
|
||||
cmp::Ordering,
|
||||
ops::RangeInclusive,
|
||||
sync::Arc,
|
||||
task::{ready, Context, Poll},
|
||||
@@ -620,11 +620,7 @@ where
|
||||
// Otherwise, we recalculate the whole stage checkpoint including the amount of gas
|
||||
// already processed, if there's any.
|
||||
_ => {
|
||||
let genesis_block_number = provider.genesis_block_number();
|
||||
let processed = calculate_gas_used_from_headers(
|
||||
provider,
|
||||
genesis_block_number..=max(start_block - 1, genesis_block_number),
|
||||
)?;
|
||||
let processed = calculate_gas_used_from_headers(provider, 0..=start_block - 1)?;
|
||||
|
||||
ExecutionCheckpoint {
|
||||
block_range: CheckpointBlockRange { from: start_block, to: max_block },
|
||||
|
||||
@@ -3,16 +3,17 @@ use alloy_primitives::{TxHash, TxNumber};
|
||||
use num_traits::Zero;
|
||||
use reth_config::config::{EtlConfig, TransactionLookupConfig};
|
||||
use reth_db_api::{
|
||||
table::{Decode, Decompress, Value},
|
||||
cursor::{DbCursorRO, DbCursorRW},
|
||||
table::Value,
|
||||
tables,
|
||||
transaction::DbTxMut,
|
||||
RawKey, RawValue,
|
||||
};
|
||||
use reth_etl::Collector;
|
||||
use reth_primitives_traits::{NodePrimitives, SignedTransaction};
|
||||
use reth_provider::{
|
||||
BlockReader, DBProvider, EitherWriter, PruneCheckpointReader, PruneCheckpointWriter,
|
||||
RocksDBProviderFactory, StaticFileProviderFactory, StatsReader, StorageSettingsCache,
|
||||
TransactionsProvider, TransactionsProviderExt,
|
||||
BlockReader, DBProvider, PruneCheckpointReader, PruneCheckpointWriter,
|
||||
StaticFileProviderFactory, StatsReader, TransactionsProvider, TransactionsProviderExt,
|
||||
};
|
||||
use reth_prune_types::{PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment};
|
||||
use reth_stages_api::{
|
||||
@@ -64,9 +65,7 @@ where
|
||||
+ PruneCheckpointReader
|
||||
+ StatsReader
|
||||
+ StaticFileProviderFactory<Primitives: NodePrimitives<SignedTx: Value + SignedTransaction>>
|
||||
+ TransactionsProviderExt
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory,
|
||||
+ TransactionsProviderExt,
|
||||
{
|
||||
/// Return the id of the stage
|
||||
fn id(&self) -> StageId {
|
||||
@@ -151,27 +150,16 @@ where
|
||||
);
|
||||
|
||||
if range_output.is_final_range {
|
||||
let total_hashes = hash_collector.len();
|
||||
let interval = (total_hashes / 10).max(1);
|
||||
|
||||
// Use append mode when table is empty (first sync) - significantly faster
|
||||
let append_only =
|
||||
provider.count_entries::<tables::TransactionHashNumbers>()?.is_zero();
|
||||
let mut txhash_cursor = provider
|
||||
.tx_ref()
|
||||
.cursor_write::<tables::RawTable<tables::TransactionHashNumbers>>()?;
|
||||
|
||||
// Create RocksDB batch if feature is enabled
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
let rocksdb = provider.rocksdb_provider();
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
let rocksdb_batch = rocksdb.batch();
|
||||
#[cfg(not(all(unix, feature = "rocksdb")))]
|
||||
let rocksdb_batch = ();
|
||||
|
||||
// Create writer that routes to either MDBX or RocksDB based on settings
|
||||
let mut writer =
|
||||
EitherWriter::new_transaction_hash_numbers(provider, rocksdb_batch)?;
|
||||
|
||||
let total_hashes = hash_collector.len();
|
||||
let interval = (total_hashes / 10).max(1);
|
||||
for (index, hash_to_number) in hash_collector.iter()?.enumerate() {
|
||||
let (hash_bytes, number_bytes) = hash_to_number?;
|
||||
let (hash, number) = hash_to_number?;
|
||||
if index > 0 && index.is_multiple_of(interval) {
|
||||
info!(
|
||||
target: "sync::stages::transaction_lookup",
|
||||
@@ -181,16 +169,12 @@ where
|
||||
);
|
||||
}
|
||||
|
||||
// Decode from raw ETL bytes
|
||||
let hash = TxHash::decode(&hash_bytes)?;
|
||||
let tx_num = TxNumber::decompress(&number_bytes)?;
|
||||
writer.put_transaction_hash_number(hash, tx_num, append_only)?;
|
||||
}
|
||||
|
||||
// Extract and register RocksDB batch for commit at provider level
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if let Some(batch) = writer.into_raw_rocksdb_batch() {
|
||||
provider.set_pending_rocksdb_batch(batch);
|
||||
let key = RawKey::<TxHash>::from_vec(hash);
|
||||
if append_only {
|
||||
txhash_cursor.append(key, &RawValue::<TxNumber>::from_vec(number))?
|
||||
} else {
|
||||
txhash_cursor.insert(key, &RawValue::<TxNumber>::from_vec(number))?
|
||||
}
|
||||
}
|
||||
|
||||
trace!(target: "sync::stages::transaction_lookup",
|
||||
@@ -215,19 +199,11 @@ where
|
||||
provider: &Provider,
|
||||
input: UnwindInput,
|
||||
) -> Result<UnwindOutput, StageError> {
|
||||
let tx = provider.tx_ref();
|
||||
let (range, unwind_to, _) = input.unwind_block_range_with_threshold(self.chunk_size);
|
||||
|
||||
// Create RocksDB batch if feature is enabled
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
let rocksdb = provider.rocksdb_provider();
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
let rocksdb_batch = rocksdb.batch();
|
||||
#[cfg(not(all(unix, feature = "rocksdb")))]
|
||||
let rocksdb_batch = ();
|
||||
|
||||
// Create writer that routes to either MDBX or RocksDB based on settings
|
||||
let mut writer = EitherWriter::new_transaction_hash_numbers(provider, rocksdb_batch)?;
|
||||
|
||||
// Cursor to unwind tx hash to number
|
||||
let mut tx_hash_number_cursor = tx.cursor_write::<tables::TransactionHashNumbers>()?;
|
||||
let static_file_provider = provider.static_file_provider();
|
||||
let rev_walker = provider
|
||||
.block_body_indices_range(range.clone())?
|
||||
@@ -242,18 +218,15 @@ where
|
||||
|
||||
// Delete all transactions that belong to this block
|
||||
for tx_id in body.tx_num_range() {
|
||||
if let Some(transaction) = static_file_provider.transaction_by_id(tx_id)? {
|
||||
writer.delete_transaction_hash_number(transaction.trie_hash())?;
|
||||
// First delete the transaction and hash to id mapping
|
||||
if let Some(transaction) = static_file_provider.transaction_by_id(tx_id)? &&
|
||||
tx_hash_number_cursor.seek_exact(transaction.trie_hash())?.is_some()
|
||||
{
|
||||
tx_hash_number_cursor.delete_current()?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract and register RocksDB batch for commit at provider level
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if let Some(batch) = writer.into_raw_rocksdb_batch() {
|
||||
provider.set_pending_rocksdb_batch(batch);
|
||||
}
|
||||
|
||||
Ok(UnwindOutput {
|
||||
checkpoint: StageCheckpoint::new(unwind_to)
|
||||
.with_entities_stage_checkpoint(stage_checkpoint(provider)?),
|
||||
@@ -293,7 +266,7 @@ mod tests {
|
||||
};
|
||||
use alloy_primitives::{BlockNumber, B256};
|
||||
use assert_matches::assert_matches;
|
||||
use reth_db_api::{cursor::DbCursorRO, transaction::DbTx};
|
||||
use reth_db_api::transaction::DbTx;
|
||||
use reth_ethereum_primitives::Block;
|
||||
use reth_primitives_traits::SealedBlock;
|
||||
use reth_provider::{
|
||||
@@ -608,160 +581,4 @@ mod tests {
|
||||
self.ensure_no_hash_by_block(input.unwind_to)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
mod rocksdb_tests {
|
||||
use super::*;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use reth_storage_api::StorageSettings;
|
||||
|
||||
/// Test that when `transaction_hash_numbers_in_rocksdb` is enabled, the stage
|
||||
/// writes transaction hash mappings to `RocksDB` instead of MDBX.
|
||||
#[tokio::test]
|
||||
async fn execute_writes_to_rocksdb_when_enabled() {
|
||||
let (previous_stage, stage_progress) = (110, 100);
|
||||
let mut rng = generators::rng();
|
||||
|
||||
// Set up the runner
|
||||
let runner = TransactionLookupTestRunner::default();
|
||||
|
||||
// Enable RocksDB for transaction hash numbers
|
||||
runner.db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let input = ExecInput {
|
||||
target: Some(previous_stage),
|
||||
checkpoint: Some(StageCheckpoint::new(stage_progress)),
|
||||
};
|
||||
|
||||
// Insert blocks with transactions
|
||||
let blocks = random_block_range(
|
||||
&mut rng,
|
||||
stage_progress + 1..=previous_stage,
|
||||
BlockRangeParams {
|
||||
parent: Some(B256::ZERO),
|
||||
tx_count: 1..3, // Ensure we have transactions
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
runner
|
||||
.db
|
||||
.insert_blocks(blocks.iter(), StorageKind::Static)
|
||||
.expect("failed to insert blocks");
|
||||
|
||||
// Count expected transactions
|
||||
let expected_tx_count: usize = blocks.iter().map(|b| b.body().transactions.len()).sum();
|
||||
assert!(expected_tx_count > 0, "test requires at least one transaction");
|
||||
|
||||
// Execute the stage
|
||||
let rx = runner.execute(input);
|
||||
let result = rx.await.unwrap();
|
||||
assert!(result.is_ok(), "stage execution failed: {:?}", result);
|
||||
|
||||
// Verify MDBX table is empty (data should be in RocksDB)
|
||||
let mdbx_count = runner.db.count_entries::<tables::TransactionHashNumbers>().unwrap();
|
||||
assert_eq!(
|
||||
mdbx_count, 0,
|
||||
"MDBX TransactionHashNumbers should be empty when RocksDB is enabled"
|
||||
);
|
||||
|
||||
// Verify RocksDB has the data
|
||||
let rocksdb = runner.db.factory.rocksdb_provider();
|
||||
let mut rocksdb_count = 0;
|
||||
for block in &blocks {
|
||||
for tx in &block.body().transactions {
|
||||
let hash = *tx.tx_hash();
|
||||
let result = rocksdb.get::<tables::TransactionHashNumbers>(hash).unwrap();
|
||||
assert!(result.is_some(), "Transaction hash {:?} not found in RocksDB", hash);
|
||||
rocksdb_count += 1;
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
rocksdb_count, expected_tx_count,
|
||||
"RocksDB should contain all transaction hashes"
|
||||
);
|
||||
}
|
||||
|
||||
/// Test that when `transaction_hash_numbers_in_rocksdb` is enabled, the stage
|
||||
/// unwind deletes transaction hash mappings from `RocksDB` instead of MDBX.
|
||||
#[tokio::test]
|
||||
async fn unwind_deletes_from_rocksdb_when_enabled() {
|
||||
let (previous_stage, stage_progress) = (110, 100);
|
||||
let mut rng = generators::rng();
|
||||
|
||||
// Set up the runner
|
||||
let runner = TransactionLookupTestRunner::default();
|
||||
|
||||
// Enable RocksDB for transaction hash numbers
|
||||
runner.db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Insert blocks with transactions
|
||||
let blocks = random_block_range(
|
||||
&mut rng,
|
||||
stage_progress + 1..=previous_stage,
|
||||
BlockRangeParams {
|
||||
parent: Some(B256::ZERO),
|
||||
tx_count: 1..3, // Ensure we have transactions
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
runner
|
||||
.db
|
||||
.insert_blocks(blocks.iter(), StorageKind::Static)
|
||||
.expect("failed to insert blocks");
|
||||
|
||||
// Count expected transactions
|
||||
let expected_tx_count: usize = blocks.iter().map(|b| b.body().transactions.len()).sum();
|
||||
assert!(expected_tx_count > 0, "test requires at least one transaction");
|
||||
|
||||
// Execute the stage first to populate RocksDB
|
||||
let exec_input = ExecInput {
|
||||
target: Some(previous_stage),
|
||||
checkpoint: Some(StageCheckpoint::new(stage_progress)),
|
||||
};
|
||||
let rx = runner.execute(exec_input);
|
||||
let result = rx.await.unwrap();
|
||||
assert!(result.is_ok(), "stage execution failed: {:?}", result);
|
||||
|
||||
// Verify RocksDB has the data before unwind
|
||||
let rocksdb = runner.db.factory.rocksdb_provider();
|
||||
for block in &blocks {
|
||||
for tx in &block.body().transactions {
|
||||
let hash = *tx.tx_hash();
|
||||
let result = rocksdb.get::<tables::TransactionHashNumbers>(hash).unwrap();
|
||||
assert!(
|
||||
result.is_some(),
|
||||
"Transaction hash {:?} should exist before unwind",
|
||||
hash
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Now unwind to stage_progress (removing all the blocks we added)
|
||||
let unwind_input = UnwindInput {
|
||||
checkpoint: StageCheckpoint::new(previous_stage),
|
||||
unwind_to: stage_progress,
|
||||
bad_block: None,
|
||||
};
|
||||
let unwind_result = runner.unwind(unwind_input).await;
|
||||
assert!(unwind_result.is_ok(), "stage unwind failed: {:?}", unwind_result);
|
||||
|
||||
// Verify RocksDB data is deleted after unwind
|
||||
let rocksdb = runner.db.factory.rocksdb_provider();
|
||||
for block in &blocks {
|
||||
for tx in &block.body().transactions {
|
||||
let hash = *tx.tx_hash();
|
||||
let result = rocksdb.get::<tables::TransactionHashNumbers>(hash).unwrap();
|
||||
assert!(
|
||||
result.is_none(),
|
||||
"Transaction hash {:?} should be deleted from RocksDB after unwind",
|
||||
hash
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,7 +50,7 @@ impl Default for TestStageDB {
|
||||
create_test_rw_db(),
|
||||
MAINNET.clone(),
|
||||
StaticFileProvider::read_write(static_dir_path).unwrap(),
|
||||
RocksDBProvider::builder(rocksdb_dir_path).with_default_tables().build().unwrap(),
|
||||
RocksDBProvider::builder(rocksdb_dir_path).build().unwrap(),
|
||||
)
|
||||
.expect("failed to create test provider factory"),
|
||||
}
|
||||
@@ -68,7 +68,7 @@ impl TestStageDB {
|
||||
create_test_rw_db_with_path(path),
|
||||
MAINNET.clone(),
|
||||
StaticFileProvider::read_write(static_dir_path).unwrap(),
|
||||
RocksDBProvider::builder(rocksdb_dir_path).with_default_tables().build().unwrap(),
|
||||
RocksDBProvider::builder(rocksdb_dir_path).build().unwrap(),
|
||||
)
|
||||
.expect("failed to create test provider factory"),
|
||||
}
|
||||
|
||||
@@ -100,7 +100,6 @@ where
|
||||
+ StateWriter
|
||||
+ TrieWriter
|
||||
+ MetadataWriter
|
||||
+ ChainSpecProvider
|
||||
+ AsRef<PF::ProviderRW>,
|
||||
PF::ChainSpec: EthChainSpec<Header = <PF::Primitives as NodePrimitives>::BlockHeader>,
|
||||
{
|
||||
@@ -127,7 +126,6 @@ where
|
||||
+ StateWriter
|
||||
+ TrieWriter
|
||||
+ MetadataWriter
|
||||
+ ChainSpecProvider
|
||||
+ AsRef<PF::ProviderRW>,
|
||||
PF::ChainSpec: EthChainSpec<Header = <PF::Primitives as NodePrimitives>::BlockHeader>,
|
||||
{
|
||||
@@ -136,12 +134,9 @@ where
|
||||
let genesis = chain.genesis();
|
||||
let hash = chain.genesis_hash();
|
||||
|
||||
// Get the genesis block number from the chain spec
|
||||
let genesis_block_number = chain.genesis_header().number();
|
||||
|
||||
// Check if we already have the genesis header or if we have the wrong one.
|
||||
match factory.block_hash(genesis_block_number) {
|
||||
Ok(None) | Err(ProviderError::MissingStaticFileBlock(StaticFileSegment::Headers, _)) => {}
|
||||
match factory.block_hash(0) {
|
||||
Ok(None) | Err(ProviderError::MissingStaticFileBlock(StaticFileSegment::Headers, 0)) => {}
|
||||
Ok(Some(block_hash)) => {
|
||||
if block_hash == hash {
|
||||
// Some users will at times attempt to re-sync from scratch by just deleting the
|
||||
@@ -184,26 +179,15 @@ where
|
||||
// compute state root to populate trie tables
|
||||
compute_state_root(&provider_rw, None)?;
|
||||
|
||||
// set stage checkpoint to genesis block number for all stages
|
||||
let checkpoint = StageCheckpoint::new(genesis_block_number);
|
||||
// insert sync stage
|
||||
for stage in StageId::ALL {
|
||||
provider_rw.save_stage_checkpoint(stage, checkpoint)?;
|
||||
provider_rw.save_stage_checkpoint(stage, Default::default())?;
|
||||
}
|
||||
|
||||
// Static file segments start empty, so we need to initialize the genesis block.
|
||||
let static_file_provider = provider_rw.static_file_provider();
|
||||
|
||||
// Static file segments start empty, so we need to initialize the genesis block.
|
||||
// For genesis blocks with non-zero block numbers, we need to use get_writer() instead of
|
||||
// latest_writer() to ensure the genesis block is stored in the correct static file range.
|
||||
static_file_provider
|
||||
.get_writer(genesis_block_number, StaticFileSegment::Receipts)?
|
||||
.user_header_mut()
|
||||
.set_block_range(genesis_block_number, genesis_block_number);
|
||||
static_file_provider
|
||||
.get_writer(genesis_block_number, StaticFileSegment::Transactions)?
|
||||
.user_header_mut()
|
||||
.set_block_range(genesis_block_number, genesis_block_number);
|
||||
static_file_provider.latest_writer(StaticFileSegment::Receipts)?.increment_block(0)?;
|
||||
static_file_provider.latest_writer(StaticFileSegment::Transactions)?.increment_block(0)?;
|
||||
|
||||
// Behaviour reserved only for new nodes should be set here.
|
||||
provider_rw.write_storage_settings(storage_settings)?;
|
||||
@@ -226,11 +210,9 @@ where
|
||||
+ DBProvider<Tx: DbTxMut>
|
||||
+ HeaderProvider
|
||||
+ StateWriter
|
||||
+ ChainSpecProvider
|
||||
+ AsRef<Provider>,
|
||||
{
|
||||
let genesis_block_number = provider.chain_spec().genesis_header().number();
|
||||
insert_state(provider, alloc, genesis_block_number)
|
||||
insert_state(provider, alloc, 0)
|
||||
}
|
||||
|
||||
/// Inserts state at given block into database.
|
||||
@@ -353,10 +335,9 @@ pub fn insert_genesis_history<'a, 'b, Provider>(
|
||||
alloc: impl Iterator<Item = (&'a Address, &'b GenesisAccount)> + Clone,
|
||||
) -> ProviderResult<()>
|
||||
where
|
||||
Provider: DBProvider<Tx: DbTxMut> + HistoryWriter + ChainSpecProvider,
|
||||
Provider: DBProvider<Tx: DbTxMut> + HistoryWriter,
|
||||
{
|
||||
let genesis_block_number = provider.chain_spec().genesis_header().number();
|
||||
insert_history(provider, alloc, genesis_block_number)
|
||||
insert_history(provider, alloc, 0)
|
||||
}
|
||||
|
||||
/// Inserts history indices for genesis accounts and storage.
|
||||
@@ -396,37 +377,17 @@ where
|
||||
let (header, block_hash) = (chain.genesis_header(), chain.genesis_hash());
|
||||
let static_file_provider = provider.static_file_provider();
|
||||
|
||||
// Get the actual genesis block number from the header
|
||||
let genesis_block_number = header.number();
|
||||
|
||||
match static_file_provider.block_hash(genesis_block_number) {
|
||||
Ok(None) | Err(ProviderError::MissingStaticFileBlock(StaticFileSegment::Headers, _)) => {
|
||||
let difficulty = header.difficulty();
|
||||
|
||||
// For genesis blocks with non-zero block numbers, we need to ensure they are stored
|
||||
// in the correct static file range. We use get_writer() with the genesis block number
|
||||
// to ensure the genesis block is stored in the correct static file range.
|
||||
let mut writer = static_file_provider
|
||||
.get_writer(genesis_block_number, StaticFileSegment::Headers)?;
|
||||
|
||||
// For non-zero genesis blocks, we need to set block range to genesis_block_number and
|
||||
// append header without increment block
|
||||
if genesis_block_number > 0 {
|
||||
writer
|
||||
.user_header_mut()
|
||||
.set_block_range(genesis_block_number, genesis_block_number);
|
||||
writer.append_header_direct(header, difficulty, &block_hash)?;
|
||||
} else {
|
||||
// For zero genesis blocks, use normal append_header
|
||||
writer.append_header(header, &block_hash)?;
|
||||
}
|
||||
match static_file_provider.block_hash(0) {
|
||||
Ok(None) | Err(ProviderError::MissingStaticFileBlock(StaticFileSegment::Headers, 0)) => {
|
||||
let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?;
|
||||
writer.append_header(header, &block_hash)?;
|
||||
}
|
||||
Ok(Some(_)) => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
|
||||
provider.tx_ref().put::<tables::HeaderNumbers>(block_hash, genesis_block_number)?;
|
||||
provider.tx_ref().put::<tables::BlockBodyIndices>(genesis_block_number, Default::default())?;
|
||||
provider.tx_ref().put::<tables::HeaderNumbers>(block_hash, 0)?;
|
||||
provider.tx_ref().put::<tables::BlockBodyIndices>(0, Default::default())?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
119
crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c
vendored
119
crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c
vendored
@@ -12936,125 +12936,6 @@ int mdbx_txn_renew(MDBX_txn *txn) {
|
||||
return LOG_IFERR(rc);
|
||||
}
|
||||
|
||||
int mdbx_txn_clone(const MDBX_txn *src, MDBX_txn **dest) {
|
||||
if (unlikely(!dest))
|
||||
return LOG_IFERR(MDBX_EINVAL);
|
||||
*dest = nullptr;
|
||||
|
||||
int rc = check_txn(src, MDBX_TXN_BLOCKED);
|
||||
if (unlikely(rc != MDBX_SUCCESS))
|
||||
return LOG_IFERR(rc);
|
||||
|
||||
if (unlikely((src->flags & MDBX_TXN_RDONLY) == 0))
|
||||
return LOG_IFERR(MDBX_EINVAL);
|
||||
|
||||
MDBX_env *const env = src->env;
|
||||
rc = check_env(env, true);
|
||||
if (unlikely(rc != MDBX_SUCCESS))
|
||||
return LOG_IFERR(rc);
|
||||
|
||||
if (unlikely((env->flags & MDBX_NOSTICKYTHREADS) == 0))
|
||||
return LOG_IFERR(MDBX_TXN_OVERLAPPING);
|
||||
|
||||
MDBX_txn *txn = nullptr;
|
||||
const intptr_t bitmap_bytes =
|
||||
#if MDBX_ENABLE_DBI_SPARSE
|
||||
ceil_powerof2(env->max_dbi, CHAR_BIT * sizeof(txn->dbi_sparse[0])) / CHAR_BIT;
|
||||
#else
|
||||
0;
|
||||
#endif /* MDBX_ENABLE_DBI_SPARSE */
|
||||
|
||||
STATIC_ASSERT(sizeof(txn->tw) > sizeof(txn->to));
|
||||
const size_t base = sizeof(MDBX_txn) - sizeof(txn->tw) + sizeof(txn->to);
|
||||
const size_t size = base + (size_t)bitmap_bytes + env->max_dbi * sizeof(txn->dbi_seqs[0]) +
|
||||
env->max_dbi * (sizeof(txn->dbs[0]) + sizeof(txn->cursors[0]) + sizeof(txn->dbi_state[0]));
|
||||
|
||||
txn = osal_malloc(size);
|
||||
if (unlikely(txn == nullptr))
|
||||
return LOG_IFERR(MDBX_ENOMEM);
|
||||
#if MDBX_DEBUG
|
||||
memset(txn, 0xCD, size);
|
||||
VALGRIND_MAKE_MEM_UNDEFINED(txn, size);
|
||||
#endif /* MDBX_DEBUG */
|
||||
MDBX_ANALYSIS_ASSUME(size > base);
|
||||
memset(txn, 0, (MDBX_GOOFY_MSVC_STATIC_ANALYZER && base > size) ? size : base);
|
||||
txn->dbs = ptr_disp(txn, base);
|
||||
txn->cursors = ptr_disp(txn->dbs, env->max_dbi * sizeof(txn->dbs[0]));
|
||||
#if MDBX_DEBUG
|
||||
txn->cursors[FREE_DBI] = nullptr; /* avoid SIGSEGV in an assertion later */
|
||||
#endif
|
||||
txn->dbi_state = ptr_disp(txn, size - env->max_dbi * sizeof(txn->dbi_state[0]));
|
||||
txn->dbi_seqs = ptr_disp(txn->cursors, env->max_dbi * sizeof(txn->cursors[0]));
|
||||
#if MDBX_ENABLE_DBI_SPARSE
|
||||
txn->dbi_sparse = ptr_disp(txn->dbi_state, -bitmap_bytes);
|
||||
#endif /* MDBX_ENABLE_DBI_SPARSE */
|
||||
|
||||
txn->env = env;
|
||||
txn->flags = src->flags & ~txn_state_flags;
|
||||
txn->parent = nullptr;
|
||||
txn->nested = nullptr;
|
||||
|
||||
txn->txnid = src->txnid;
|
||||
txn->front_txnid = src->front_txnid;
|
||||
txn->geo = src->geo;
|
||||
txn->canary = src->canary;
|
||||
txn->owner = (env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self();
|
||||
|
||||
if (unlikely(src->n_dbi > env->max_dbi)) {
|
||||
rc = MDBX_CORRUPTED;
|
||||
goto bailout;
|
||||
}
|
||||
|
||||
txn->n_dbi = src->n_dbi;
|
||||
memset(txn->cursors, 0, env->max_dbi * sizeof(txn->cursors[0]));
|
||||
memset(txn->dbi_state, 0, env->max_dbi * sizeof(txn->dbi_state[0]));
|
||||
memset(txn->dbi_seqs, 0, env->max_dbi * sizeof(txn->dbi_seqs[0]));
|
||||
#if MDBX_ENABLE_DBI_SPARSE
|
||||
if (bitmap_bytes)
|
||||
memset(txn->dbi_sparse, 0, bitmap_bytes);
|
||||
#endif /* MDBX_ENABLE_DBI_SPARSE */
|
||||
|
||||
memcpy(txn->dbs, src->dbs, txn->n_dbi * sizeof(txn->dbs[0]));
|
||||
memcpy(txn->dbi_state, src->dbi_state, txn->n_dbi * sizeof(txn->dbi_state[0]));
|
||||
memcpy(txn->dbi_seqs, src->dbi_seqs, txn->n_dbi * sizeof(txn->dbi_seqs[0]));
|
||||
#if MDBX_ENABLE_DBI_SPARSE
|
||||
if (bitmap_bytes)
|
||||
memcpy(txn->dbi_sparse, src->dbi_sparse, bitmap_bytes);
|
||||
#endif /* MDBX_ENABLE_DBI_SPARSE */
|
||||
|
||||
txn->to.reader = nullptr;
|
||||
if (env->lck_mmap.lck) {
|
||||
bsr_t brs = mvcc_bind_slot(env);
|
||||
if (unlikely(brs.err != MDBX_SUCCESS)) {
|
||||
rc = brs.err;
|
||||
goto bailout;
|
||||
}
|
||||
txn->to.reader = brs.rslot;
|
||||
safe64_reset(&txn->to.reader->txnid, true);
|
||||
if (src->to.reader) {
|
||||
atomic_store32(&txn->to.reader->snapshot_pages_used,
|
||||
atomic_load32(&src->to.reader->snapshot_pages_used, mo_Relaxed), mo_Relaxed);
|
||||
atomic_store64(&txn->to.reader->snapshot_pages_retired,
|
||||
atomic_load64(&src->to.reader->snapshot_pages_retired, mo_Relaxed), mo_Relaxed);
|
||||
} else {
|
||||
atomic_store32(&txn->to.reader->snapshot_pages_used, src->geo.first_unallocated, mo_Relaxed);
|
||||
atomic_store64(&txn->to.reader->snapshot_pages_retired, 0, mo_Relaxed);
|
||||
}
|
||||
safe64_write(&txn->to.reader->txnid, src->txnid);
|
||||
atomic_store32(&env->lck->rdt_refresh_flag, true, mo_AcquireRelease);
|
||||
}
|
||||
|
||||
txn->signature = txn_signature;
|
||||
txn->userctx = nullptr;
|
||||
*dest = txn;
|
||||
DEBUG("clone txn %" PRIaTXN "r %p from %p on env %p", txn->txnid, (void *)txn, (void *)src, (void *)env);
|
||||
return MDBX_SUCCESS;
|
||||
|
||||
bailout:
|
||||
osal_free(txn);
|
||||
return LOG_IFERR(rc);
|
||||
}
|
||||
|
||||
int mdbx_txn_set_userctx(MDBX_txn *txn, void *ctx) {
|
||||
int rc = check_txn(txn, MDBX_TXN_FINISHED);
|
||||
if (unlikely(rc != MDBX_SUCCESS))
|
||||
|
||||
@@ -3882,35 +3882,6 @@ MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API void *mdbx_env_get_userctx(const MDBX_env
|
||||
LIBMDBX_API int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, MDBX_txn **txn,
|
||||
void *context);
|
||||
|
||||
/** \brief Clone a read-only transaction snapshot.
|
||||
* \ingroup c_transactions
|
||||
*
|
||||
* Creates a new read-only transaction that uses the same MVCC snapshot as
|
||||
* the \p src transaction. This allows parallel read operations across threads
|
||||
* without re-opening a read transaction and re-fetching state.
|
||||
*
|
||||
* \note This function requires \ref MDBX_NOSTICKYTHREADS (aka MDBX_NOTLS)
|
||||
* to be enabled for the environment. Otherwise it will return
|
||||
* \ref MDBX_TXN_OVERLAPPING.
|
||||
*
|
||||
* \note The \p src transaction must be an active read-only transaction.
|
||||
*
|
||||
* \note The \p src transaction and the cloned transaction must not be used
|
||||
* concurrently from multiple threads. Each transaction and its cursors must
|
||||
* be confined to a single thread at a time.
|
||||
*
|
||||
* \param [in] src A read-only transaction handle returned by
|
||||
* \ref mdbx_txn_begin_ex() or \ref mdbx_txn_begin().
|
||||
* \param [out] dest Address where the cloned \ref MDBX_txn handle will be
|
||||
* stored. Must not be NULL.
|
||||
*
|
||||
* \returns A non-zero error value on failure and 0 on success.
|
||||
* \retval MDBX_EINVAL Invalid arguments or \p src is not read-only.
|
||||
* \retval MDBX_TXN_OVERLAPPING \ref MDBX_NOSTICKYTHREADS is not enabled.
|
||||
* \retval MDBX_READERS_FULL Reader lock table is full.
|
||||
* \retval MDBX_ENOMEM Out of memory. */
|
||||
LIBMDBX_API int mdbx_txn_clone(const MDBX_txn *src, MDBX_txn **dest);
|
||||
|
||||
/** \brief Create a transaction for use with the environment.
|
||||
* \ingroup c_transactions
|
||||
*
|
||||
|
||||
@@ -483,20 +483,6 @@ impl Transaction<RW> {
|
||||
}
|
||||
|
||||
impl Transaction<RO> {
|
||||
/// Clones this read-only transaction, preserving the same MVCC snapshot.
|
||||
///
|
||||
/// This requires the environment to be opened with `MDBX_NOSTICKYTHREADS` (aka `MDBX_NOTLS`).
|
||||
/// The cloned transaction must not be used concurrently with this transaction from multiple
|
||||
/// threads.
|
||||
pub fn clone_snapshot(&self) -> Result<Self> {
|
||||
let cloned = self.txn_execute(|txn| {
|
||||
let mut cloned: *mut ffi::MDBX_txn = ptr::null_mut();
|
||||
mdbx_result(unsafe { ffi::mdbx_txn_clone(txn, &mut cloned) }).map(|_| cloned)
|
||||
})??;
|
||||
|
||||
Ok(Self::new_from_ptr(self.env().clone(), cloned))
|
||||
}
|
||||
|
||||
/// Closes the database handle.
|
||||
///
|
||||
/// # Safety
|
||||
|
||||
@@ -373,35 +373,3 @@ fn test_stat_dupsort() {
|
||||
assert_eq!(stat.entries(), 8);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_txn_clone_snapshot() {
|
||||
let dir = tempdir().unwrap();
|
||||
let env = Environment::builder().open(dir.path()).unwrap();
|
||||
{
|
||||
let txn = env.begin_rw_txn().unwrap();
|
||||
let db = txn.open_db(None).unwrap();
|
||||
txn.put(db.dbi(), b"k", b"v1", WriteFlags::empty()).unwrap();
|
||||
txn.commit().unwrap();
|
||||
}
|
||||
|
||||
let ro = env.begin_ro_txn().unwrap();
|
||||
let clone = ro.clone_snapshot().unwrap();
|
||||
|
||||
{
|
||||
let txn = env.begin_rw_txn().unwrap();
|
||||
let db = txn.open_db(None).unwrap();
|
||||
txn.put(db.dbi(), b"k", b"v2", WriteFlags::empty()).unwrap();
|
||||
txn.commit().unwrap();
|
||||
}
|
||||
|
||||
let db = ro.open_db(None).unwrap();
|
||||
assert_eq!(ro.get::<[u8; 2]>(db.dbi(), b"k").unwrap(), Some(*b"v1"));
|
||||
|
||||
let db = clone.open_db(None).unwrap();
|
||||
assert_eq!(clone.get::<[u8; 2]>(db.dbi(), b"k").unwrap(), Some(*b"v1"));
|
||||
|
||||
let ro2 = env.begin_ro_txn().unwrap();
|
||||
let db = ro2.open_db(None).unwrap();
|
||||
assert_eq!(ro2.get::<[u8; 2]>(db.dbi(), b"k").unwrap(), Some(*b"v2"));
|
||||
}
|
||||
|
||||
@@ -9,19 +9,14 @@ use crate::{
|
||||
providers::{StaticFileProvider, StaticFileProviderRWRefMut},
|
||||
StaticFileProviderFactory,
|
||||
};
|
||||
use alloy_primitives::{map::HashMap, Address, BlockNumber, TxHash, TxNumber};
|
||||
use alloy_primitives::{map::HashMap, Address, BlockNumber, TxNumber};
|
||||
use reth_db::{
|
||||
cursor::DbCursorRO,
|
||||
static_file::TransactionSenderMask,
|
||||
table::Value,
|
||||
transaction::{CursorMutTy, CursorTy, DbTx, DbTxMut},
|
||||
};
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRW,
|
||||
models::{storage_sharded_key::StorageShardedKey, ShardedKey},
|
||||
tables,
|
||||
tables::BlockNumberList,
|
||||
};
|
||||
use reth_db_api::{cursor::DbCursorRW, tables};
|
||||
use reth_errors::ProviderError;
|
||||
use reth_node_types::NodePrimitives;
|
||||
use reth_primitives_traits::ReceiptTy;
|
||||
@@ -187,21 +182,6 @@ impl<'a> EitherWriter<'a, (), ()> {
|
||||
}
|
||||
|
||||
impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N> {
|
||||
/// Extracts the raw `RocksDB` write batch from this writer, if it contains one.
|
||||
///
|
||||
/// Returns `Some(WriteBatchWithTransaction)` for [`Self::RocksDB`] variant,
|
||||
/// `None` for other variants.
|
||||
///
|
||||
/// This is used to defer `RocksDB` commits to the provider level, ensuring all
|
||||
/// storage commits (MDBX, static files, `RocksDB`) happen atomically in a single place.
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
pub fn into_raw_rocksdb_batch(self) -> Option<rocksdb::WriteBatchWithTransaction<true>> {
|
||||
match self {
|
||||
Self::Database(_) | Self::StaticFile(_) => None,
|
||||
Self::RocksDB(batch) => Some(batch.into_inner()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Increment the block number.
|
||||
///
|
||||
/// Relevant only for [`Self::StaticFile`]. It is a no-op for [`Self::Database`].
|
||||
@@ -314,119 +294,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N>
|
||||
where
|
||||
CURSOR: DbCursorRW<tables::TransactionHashNumbers> + DbCursorRO<tables::TransactionHashNumbers>,
|
||||
{
|
||||
/// Puts a transaction hash number mapping.
|
||||
///
|
||||
/// When `append_only` is true, uses `cursor.append()` which is significantly faster
|
||||
/// but requires entries to be inserted in order and the table to be empty.
|
||||
/// When false, uses `cursor.insert()` which handles arbitrary insertion order.
|
||||
pub fn put_transaction_hash_number(
|
||||
&mut self,
|
||||
hash: TxHash,
|
||||
tx_num: TxNumber,
|
||||
append_only: bool,
|
||||
) -> ProviderResult<()> {
|
||||
match self {
|
||||
Self::Database(cursor) => {
|
||||
if append_only {
|
||||
Ok(cursor.append(hash, &tx_num)?)
|
||||
} else {
|
||||
Ok(cursor.insert(hash, &tx_num)?)
|
||||
}
|
||||
}
|
||||
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(batch) => batch.put::<tables::TransactionHashNumbers>(hash, &tx_num),
|
||||
}
|
||||
}
|
||||
|
||||
/// Deletes a transaction hash number mapping.
|
||||
pub fn delete_transaction_hash_number(&mut self, hash: TxHash) -> ProviderResult<()> {
|
||||
match self {
|
||||
Self::Database(cursor) => {
|
||||
if cursor.seek_exact(hash)?.is_some() {
|
||||
cursor.delete_current()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(batch) => batch.delete::<tables::TransactionHashNumbers>(hash),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N>
|
||||
where
|
||||
CURSOR: DbCursorRW<tables::StoragesHistory> + DbCursorRO<tables::StoragesHistory>,
|
||||
{
|
||||
/// Puts a storage history entry.
|
||||
pub fn put_storage_history(
|
||||
&mut self,
|
||||
key: StorageShardedKey,
|
||||
value: &BlockNumberList,
|
||||
) -> ProviderResult<()> {
|
||||
match self {
|
||||
Self::Database(cursor) => Ok(cursor.upsert(key, value)?),
|
||||
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(batch) => batch.put::<tables::StoragesHistory>(key, value),
|
||||
}
|
||||
}
|
||||
|
||||
/// Deletes a storage history entry.
|
||||
pub fn delete_storage_history(&mut self, key: StorageShardedKey) -> ProviderResult<()> {
|
||||
match self {
|
||||
Self::Database(cursor) => {
|
||||
if cursor.seek_exact(key)?.is_some() {
|
||||
cursor.delete_current()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(batch) => batch.delete::<tables::StoragesHistory>(key),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N>
|
||||
where
|
||||
CURSOR: DbCursorRW<tables::AccountsHistory> + DbCursorRO<tables::AccountsHistory>,
|
||||
{
|
||||
/// Puts an account history entry.
|
||||
pub fn put_account_history(
|
||||
&mut self,
|
||||
key: ShardedKey<Address>,
|
||||
value: &BlockNumberList,
|
||||
) -> ProviderResult<()> {
|
||||
match self {
|
||||
Self::Database(cursor) => Ok(cursor.upsert(key, value)?),
|
||||
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(batch) => batch.put::<tables::AccountsHistory>(key, value),
|
||||
}
|
||||
}
|
||||
|
||||
/// Deletes an account history entry.
|
||||
pub fn delete_account_history(&mut self, key: ShardedKey<Address>) -> ProviderResult<()> {
|
||||
match self {
|
||||
Self::Database(cursor) => {
|
||||
if cursor.seek_exact(key)?.is_some() {
|
||||
cursor.delete_current()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Self::StaticFile(_) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(batch) => batch.delete::<tables::AccountsHistory>(key),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Represents a source for reading data, either from database, static files, or `RocksDB`.
|
||||
#[derive(Debug, Display)]
|
||||
pub enum EitherReader<'a, CURSOR, N> {
|
||||
@@ -551,60 +418,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<CURSOR, N: NodePrimitives> EitherReader<'_, CURSOR, N>
|
||||
where
|
||||
CURSOR: DbCursorRO<tables::TransactionHashNumbers>,
|
||||
{
|
||||
/// Gets a transaction number by its hash.
|
||||
pub fn get_transaction_hash_number(
|
||||
&mut self,
|
||||
hash: TxHash,
|
||||
) -> ProviderResult<Option<TxNumber>> {
|
||||
match self {
|
||||
Self::Database(cursor, _) => Ok(cursor.seek_exact(hash)?.map(|(_, v)| v)),
|
||||
Self::StaticFile(_, _) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(tx) => tx.get::<tables::TransactionHashNumbers>(hash),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<CURSOR, N: NodePrimitives> EitherReader<'_, CURSOR, N>
|
||||
where
|
||||
CURSOR: DbCursorRO<tables::StoragesHistory>,
|
||||
{
|
||||
/// Gets a storage history entry.
|
||||
pub fn get_storage_history(
|
||||
&mut self,
|
||||
key: StorageShardedKey,
|
||||
) -> ProviderResult<Option<BlockNumberList>> {
|
||||
match self {
|
||||
Self::Database(cursor, _) => Ok(cursor.seek_exact(key)?.map(|(_, v)| v)),
|
||||
Self::StaticFile(_, _) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(tx) => tx.get::<tables::StoragesHistory>(key),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<CURSOR, N: NodePrimitives> EitherReader<'_, CURSOR, N>
|
||||
where
|
||||
CURSOR: DbCursorRO<tables::AccountsHistory>,
|
||||
{
|
||||
/// Gets an account history entry.
|
||||
pub fn get_account_history(
|
||||
&mut self,
|
||||
key: ShardedKey<Address>,
|
||||
) -> ProviderResult<Option<BlockNumberList>> {
|
||||
match self {
|
||||
Self::Database(cursor, _) => Ok(cursor.seek_exact(key)?.map(|(_, v)| v)),
|
||||
Self::StaticFile(_, _) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(tx) => tx.get::<tables::AccountsHistory>(key),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Destination for writing data.
|
||||
#[derive(Debug, EnumIs)]
|
||||
pub enum EitherWriterDestination {
|
||||
@@ -686,308 +499,3 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix, feature = "rocksdb"))]
|
||||
mod rocksdb_tests {
|
||||
use super::*;
|
||||
use crate::{
|
||||
providers::rocksdb::{RocksDBBuilder, RocksDBProvider},
|
||||
test_utils::create_test_provider_factory,
|
||||
RocksDBProviderFactory,
|
||||
};
|
||||
use alloy_primitives::{Address, B256};
|
||||
use reth_db_api::{
|
||||
models::{storage_sharded_key::StorageShardedKey, IntegerList, ShardedKey},
|
||||
tables,
|
||||
};
|
||||
use reth_storage_api::{DatabaseProviderFactory, StorageSettings};
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn create_rocksdb_provider() -> (TempDir, RocksDBProvider) {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let provider = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::TransactionHashNumbers>()
|
||||
.with_table::<tables::StoragesHistory>()
|
||||
.with_table::<tables::AccountsHistory>()
|
||||
.build()
|
||||
.unwrap();
|
||||
(temp_dir, provider)
|
||||
}
|
||||
|
||||
/// Test that `EitherWriter::new_transaction_hash_numbers` creates a `RocksDB` writer
|
||||
/// when the storage setting is enabled, and that put operations followed by commit
|
||||
/// persist the data to `RocksDB`.
|
||||
#[test]
|
||||
fn test_either_writer_transaction_hash_numbers_with_rocksdb() {
|
||||
let factory = create_test_provider_factory();
|
||||
|
||||
// Enable RocksDB for transaction hash numbers
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let hash1 = B256::from([1u8; 32]);
|
||||
let hash2 = B256::from([2u8; 32]);
|
||||
let tx_num1 = 100u64;
|
||||
let tx_num2 = 200u64;
|
||||
|
||||
// Get the RocksDB batch from the provider
|
||||
let rocksdb = factory.rocksdb_provider();
|
||||
let batch = rocksdb.batch();
|
||||
|
||||
// Create EitherWriter with RocksDB
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
let mut writer = EitherWriter::new_transaction_hash_numbers(&provider, batch).unwrap();
|
||||
|
||||
// Verify we got a RocksDB writer
|
||||
assert!(matches!(writer, EitherWriter::RocksDB(_)));
|
||||
|
||||
// Write transaction hash numbers (append_only=false since we're using RocksDB)
|
||||
writer.put_transaction_hash_number(hash1, tx_num1, false).unwrap();
|
||||
writer.put_transaction_hash_number(hash2, tx_num2, false).unwrap();
|
||||
|
||||
// Extract the batch and register with provider for commit
|
||||
if let Some(batch) = writer.into_raw_rocksdb_batch() {
|
||||
provider.set_pending_rocksdb_batch(batch);
|
||||
}
|
||||
|
||||
// Commit via provider - this commits RocksDB batch too
|
||||
provider.commit().unwrap();
|
||||
|
||||
// Verify data was written to RocksDB
|
||||
let rocksdb = factory.rocksdb_provider();
|
||||
assert_eq!(rocksdb.get::<tables::TransactionHashNumbers>(hash1).unwrap(), Some(tx_num1));
|
||||
assert_eq!(rocksdb.get::<tables::TransactionHashNumbers>(hash2).unwrap(), Some(tx_num2));
|
||||
}
|
||||
|
||||
/// Test that `EitherWriter::delete_transaction_hash_number` works with `RocksDB`.
|
||||
#[test]
|
||||
fn test_either_writer_delete_transaction_hash_number_with_rocksdb() {
|
||||
let factory = create_test_provider_factory();
|
||||
|
||||
// Enable RocksDB for transaction hash numbers
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let hash = B256::from([1u8; 32]);
|
||||
let tx_num = 100u64;
|
||||
|
||||
// First, write a value directly to RocksDB
|
||||
let rocksdb = factory.rocksdb_provider();
|
||||
rocksdb.put::<tables::TransactionHashNumbers>(hash, &tx_num).unwrap();
|
||||
assert_eq!(rocksdb.get::<tables::TransactionHashNumbers>(hash).unwrap(), Some(tx_num));
|
||||
|
||||
// Now delete using EitherWriter
|
||||
let batch = rocksdb.batch();
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
let mut writer = EitherWriter::new_transaction_hash_numbers(&provider, batch).unwrap();
|
||||
writer.delete_transaction_hash_number(hash).unwrap();
|
||||
|
||||
// Extract the batch and commit via provider
|
||||
if let Some(batch) = writer.into_raw_rocksdb_batch() {
|
||||
provider.set_pending_rocksdb_batch(batch);
|
||||
}
|
||||
provider.commit().unwrap();
|
||||
|
||||
// Verify deletion
|
||||
let rocksdb = factory.rocksdb_provider();
|
||||
assert_eq!(rocksdb.get::<tables::TransactionHashNumbers>(hash).unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rocksdb_batch_transaction_hash_numbers() {
|
||||
let (_temp_dir, provider) = create_rocksdb_provider();
|
||||
|
||||
let hash1 = B256::from([1u8; 32]);
|
||||
let hash2 = B256::from([2u8; 32]);
|
||||
let tx_num1 = 100u64;
|
||||
let tx_num2 = 200u64;
|
||||
|
||||
// Write via RocksDBBatch (same as EitherWriter::RocksDB would use internally)
|
||||
let mut batch = provider.batch();
|
||||
batch.put::<tables::TransactionHashNumbers>(hash1, &tx_num1).unwrap();
|
||||
batch.put::<tables::TransactionHashNumbers>(hash2, &tx_num2).unwrap();
|
||||
batch.commit().unwrap();
|
||||
|
||||
// Read via RocksTx (same as EitherReader::RocksDB would use internally)
|
||||
let tx = provider.tx();
|
||||
assert_eq!(tx.get::<tables::TransactionHashNumbers>(hash1).unwrap(), Some(tx_num1));
|
||||
assert_eq!(tx.get::<tables::TransactionHashNumbers>(hash2).unwrap(), Some(tx_num2));
|
||||
|
||||
// Test missing key
|
||||
let missing_hash = B256::from([99u8; 32]);
|
||||
assert_eq!(tx.get::<tables::TransactionHashNumbers>(missing_hash).unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rocksdb_batch_storage_history() {
|
||||
let (_temp_dir, provider) = create_rocksdb_provider();
|
||||
|
||||
let address = Address::random();
|
||||
let storage_key = B256::from([1u8; 32]);
|
||||
let key = StorageShardedKey::new(address, storage_key, 1000);
|
||||
let value = IntegerList::new([1, 5, 10, 50]).unwrap();
|
||||
|
||||
// Write via RocksDBBatch
|
||||
let mut batch = provider.batch();
|
||||
batch.put::<tables::StoragesHistory>(key.clone(), &value).unwrap();
|
||||
batch.commit().unwrap();
|
||||
|
||||
// Read via RocksTx
|
||||
let tx = provider.tx();
|
||||
let result = tx.get::<tables::StoragesHistory>(key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
|
||||
// Test missing key
|
||||
let missing_key = StorageShardedKey::new(Address::random(), B256::ZERO, 0);
|
||||
assert_eq!(tx.get::<tables::StoragesHistory>(missing_key).unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rocksdb_batch_account_history() {
|
||||
let (_temp_dir, provider) = create_rocksdb_provider();
|
||||
|
||||
let address = Address::random();
|
||||
let key = ShardedKey::new(address, 1000);
|
||||
let value = IntegerList::new([1, 10, 100, 500]).unwrap();
|
||||
|
||||
// Write via RocksDBBatch
|
||||
let mut batch = provider.batch();
|
||||
batch.put::<tables::AccountsHistory>(key.clone(), &value).unwrap();
|
||||
batch.commit().unwrap();
|
||||
|
||||
// Read via RocksTx
|
||||
let tx = provider.tx();
|
||||
let result = tx.get::<tables::AccountsHistory>(key).unwrap();
|
||||
assert_eq!(result, Some(value));
|
||||
|
||||
// Test missing key
|
||||
let missing_key = ShardedKey::new(Address::random(), 0);
|
||||
assert_eq!(tx.get::<tables::AccountsHistory>(missing_key).unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rocksdb_batch_delete_transaction_hash_number() {
|
||||
let (_temp_dir, provider) = create_rocksdb_provider();
|
||||
|
||||
let hash = B256::from([1u8; 32]);
|
||||
let tx_num = 100u64;
|
||||
|
||||
// First write
|
||||
provider.put::<tables::TransactionHashNumbers>(hash, &tx_num).unwrap();
|
||||
assert_eq!(provider.get::<tables::TransactionHashNumbers>(hash).unwrap(), Some(tx_num));
|
||||
|
||||
// Delete via RocksDBBatch
|
||||
let mut batch = provider.batch();
|
||||
batch.delete::<tables::TransactionHashNumbers>(hash).unwrap();
|
||||
batch.commit().unwrap();
|
||||
|
||||
// Verify deletion
|
||||
assert_eq!(provider.get::<tables::TransactionHashNumbers>(hash).unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rocksdb_batch_delete_storage_history() {
|
||||
let (_temp_dir, provider) = create_rocksdb_provider();
|
||||
|
||||
let address = Address::random();
|
||||
let storage_key = B256::from([1u8; 32]);
|
||||
let key = StorageShardedKey::new(address, storage_key, 1000);
|
||||
let value = IntegerList::new([1, 5, 10]).unwrap();
|
||||
|
||||
// First write
|
||||
provider.put::<tables::StoragesHistory>(key.clone(), &value).unwrap();
|
||||
assert!(provider.get::<tables::StoragesHistory>(key.clone()).unwrap().is_some());
|
||||
|
||||
// Delete via RocksDBBatch
|
||||
let mut batch = provider.batch();
|
||||
batch.delete::<tables::StoragesHistory>(key.clone()).unwrap();
|
||||
batch.commit().unwrap();
|
||||
|
||||
// Verify deletion
|
||||
assert_eq!(provider.get::<tables::StoragesHistory>(key).unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rocksdb_batch_delete_account_history() {
|
||||
let (_temp_dir, provider) = create_rocksdb_provider();
|
||||
|
||||
let address = Address::random();
|
||||
let key = ShardedKey::new(address, 1000);
|
||||
let value = IntegerList::new([1, 10, 100]).unwrap();
|
||||
|
||||
// First write
|
||||
provider.put::<tables::AccountsHistory>(key.clone(), &value).unwrap();
|
||||
assert!(provider.get::<tables::AccountsHistory>(key.clone()).unwrap().is_some());
|
||||
|
||||
// Delete via RocksDBBatch
|
||||
let mut batch = provider.batch();
|
||||
batch.delete::<tables::AccountsHistory>(key.clone()).unwrap();
|
||||
batch.commit().unwrap();
|
||||
|
||||
// Verify deletion
|
||||
assert_eq!(provider.get::<tables::AccountsHistory>(key).unwrap(), None);
|
||||
}
|
||||
|
||||
/// Test that `RocksDB` commits happen at `provider.commit()` level, not at writer level.
|
||||
///
|
||||
/// This ensures all storage commits (MDBX, static files, `RocksDB`) happen atomically
|
||||
/// in a single place, making it easier to reason about commit ordering and consistency.
|
||||
#[test]
|
||||
fn test_rocksdb_commits_at_provider_level() {
|
||||
let factory = create_test_provider_factory();
|
||||
|
||||
// Enable RocksDB for transaction hash numbers
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let hash1 = B256::from([1u8; 32]);
|
||||
let hash2 = B256::from([2u8; 32]);
|
||||
let tx_num1 = 100u64;
|
||||
let tx_num2 = 200u64;
|
||||
|
||||
// Get the RocksDB batch from the provider
|
||||
let rocksdb = factory.rocksdb_provider();
|
||||
let batch = rocksdb.batch();
|
||||
|
||||
// Create provider and EitherWriter
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
let mut writer = EitherWriter::new_transaction_hash_numbers(&provider, batch).unwrap();
|
||||
|
||||
// Write transaction hash numbers (append_only=false since we're using RocksDB)
|
||||
writer.put_transaction_hash_number(hash1, tx_num1, false).unwrap();
|
||||
writer.put_transaction_hash_number(hash2, tx_num2, false).unwrap();
|
||||
|
||||
// Extract the raw batch from the writer and register it with the provider
|
||||
let raw_batch = writer.into_raw_rocksdb_batch();
|
||||
if let Some(batch) = raw_batch {
|
||||
provider.set_pending_rocksdb_batch(batch);
|
||||
}
|
||||
|
||||
// Data should NOT be visible yet (batch not committed)
|
||||
let rocksdb = factory.rocksdb_provider();
|
||||
assert_eq!(
|
||||
rocksdb.get::<tables::TransactionHashNumbers>(hash1).unwrap(),
|
||||
None,
|
||||
"Data should not be visible before provider.commit()"
|
||||
);
|
||||
|
||||
// Commit the provider - this should commit both MDBX and RocksDB
|
||||
provider.commit().unwrap();
|
||||
|
||||
// Now data should be visible in RocksDB
|
||||
let rocksdb = factory.rocksdb_provider();
|
||||
assert_eq!(
|
||||
rocksdb.get::<tables::TransactionHashNumbers>(hash1).unwrap(),
|
||||
Some(tx_num1),
|
||||
"Data should be visible after provider.commit()"
|
||||
);
|
||||
assert_eq!(
|
||||
rocksdb.get::<tables::TransactionHashNumbers>(hash2).unwrap(),
|
||||
Some(tx_num2),
|
||||
"Data should be visible after provider.commit()"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,11 +181,6 @@ impl<N: ProviderNodeTypes> RocksDBProviderFactory for BlockchainProvider<N> {
|
||||
fn rocksdb_provider(&self) -> RocksDBProvider {
|
||||
self.database.rocksdb_provider()
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
fn set_pending_rocksdb_batch(&self, _batch: rocksdb::WriteBatchWithTransaction<true>) {
|
||||
unimplemented!("BlockchainProvider wraps ProviderFactory - use DatabaseProvider::set_pending_rocksdb_batch instead")
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: ProviderNodeTypes> HeaderProvider for BlockchainProvider<N> {
|
||||
|
||||
@@ -109,7 +109,7 @@ impl<N> ProviderFactoryBuilder<N> {
|
||||
self.db(Arc::new(open_db_read_only(db_dir, db_args)?))
|
||||
.chainspec(chainspec)
|
||||
.static_file(StaticFileProvider::read_only(static_files_dir, watch_static_files)?)
|
||||
.rocksdb_provider(RocksDBProvider::builder(&rocksdb_dir).with_default_tables().build()?)
|
||||
.rocksdb_provider(RocksDBProvider::builder(&rocksdb_dir).build()?)
|
||||
.build_provider_factory()
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
@@ -153,11 +153,6 @@ impl<N: NodeTypesWithDB> RocksDBProviderFactory for ProviderFactory<N> {
|
||||
fn rocksdb_provider(&self) -> RocksDBProvider {
|
||||
self.rocksdb_provider.clone()
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
fn set_pending_rocksdb_batch(&self, _batch: rocksdb::WriteBatchWithTransaction<true>) {
|
||||
unimplemented!("ProviderFactory is a factory, not a provider - use DatabaseProvider::set_pending_rocksdb_batch instead")
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>> ProviderFactory<N> {
|
||||
|
||||
@@ -151,6 +151,7 @@ impl<DB: Database, N: NodeTypes> From<DatabaseProviderRW<DB, N>>
|
||||
|
||||
/// A provider struct that fetches data from the database.
|
||||
/// Wrapper around [`DbTx`] and [`DbTxMut`]. Example: [`HeaderProvider`] [`BlockHashReader`]
|
||||
#[derive(Debug)]
|
||||
pub struct DatabaseProvider<TX, N: NodeTypes> {
|
||||
/// Database transaction.
|
||||
tx: TX,
|
||||
@@ -166,29 +167,10 @@ pub struct DatabaseProvider<TX, N: NodeTypes> {
|
||||
storage_settings: Arc<RwLock<StorageSettings>>,
|
||||
/// `RocksDB` provider
|
||||
rocksdb_provider: RocksDBProvider,
|
||||
/// Pending `RocksDB` batches to be committed at provider commit time.
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
pending_rocksdb_batches: parking_lot::Mutex<Vec<rocksdb::WriteBatchWithTransaction<true>>>,
|
||||
/// Minimum distance from tip required for pruning
|
||||
minimum_pruning_distance: u64,
|
||||
}
|
||||
|
||||
impl<TX: Debug, N: NodeTypes> Debug for DatabaseProvider<TX, N> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let mut s = f.debug_struct("DatabaseProvider");
|
||||
s.field("tx", &self.tx)
|
||||
.field("chain_spec", &self.chain_spec)
|
||||
.field("static_file_provider", &self.static_file_provider)
|
||||
.field("prune_modes", &self.prune_modes)
|
||||
.field("storage", &self.storage)
|
||||
.field("storage_settings", &self.storage_settings)
|
||||
.field("rocksdb_provider", &self.rocksdb_provider);
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
s.field("pending_rocksdb_batches", &"<pending batches>");
|
||||
s.field("minimum_pruning_distance", &self.minimum_pruning_distance).finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl<TX, N: NodeTypes> DatabaseProvider<TX, N> {
|
||||
/// Returns reference to prune modes.
|
||||
pub const fn prune_modes_ref(&self) -> &PruneModes {
|
||||
@@ -277,11 +259,6 @@ impl<TX, N: NodeTypes> RocksDBProviderFactory for DatabaseProvider<TX, N> {
|
||||
fn rocksdb_provider(&self) -> RocksDBProvider {
|
||||
self.rocksdb_provider.clone()
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
fn set_pending_rocksdb_batch(&self, batch: rocksdb::WriteBatchWithTransaction<true>) {
|
||||
self.pending_rocksdb_batches.lock().push(batch);
|
||||
}
|
||||
}
|
||||
|
||||
impl<TX: Debug + Send + Sync, N: NodeTypes<ChainSpec: EthChainSpec + 'static>> ChainSpecProvider
|
||||
@@ -313,8 +290,6 @@ impl<TX: DbTxMut, N: NodeTypes> DatabaseProvider<TX, N> {
|
||||
storage,
|
||||
storage_settings,
|
||||
rocksdb_provider,
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
pending_rocksdb_batches: parking_lot::Mutex::new(Vec::new()),
|
||||
minimum_pruning_distance: MINIMUM_PRUNING_DISTANCE,
|
||||
}
|
||||
}
|
||||
@@ -570,8 +545,6 @@ impl<TX: DbTx + 'static, N: NodeTypesForProvider> DatabaseProvider<TX, N> {
|
||||
storage,
|
||||
storage_settings,
|
||||
rocksdb_provider,
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
pending_rocksdb_batches: parking_lot::Mutex::new(Vec::new()),
|
||||
minimum_pruning_distance: MINIMUM_PRUNING_DISTANCE,
|
||||
}
|
||||
}
|
||||
@@ -3205,7 +3178,7 @@ impl<TX: DbTx + 'static, N: NodeTypes + 'static> DBProvider for DatabaseProvider
|
||||
self.prune_modes_ref()
|
||||
}
|
||||
|
||||
/// Commit database transaction, static files, and pending `RocksDB` batches.
|
||||
/// Commit database transaction and static files.
|
||||
fn commit(self) -> ProviderResult<bool> {
|
||||
// For unwinding it makes more sense to commit the database first, since if
|
||||
// it is interrupted before the static files commit, we can just
|
||||
@@ -3213,27 +3186,9 @@ impl<TX: DbTx + 'static, N: NodeTypes + 'static> DBProvider for DatabaseProvider
|
||||
// checkpoints on the next start-up.
|
||||
if self.static_file_provider.has_unwind_queued() {
|
||||
self.tx.commit()?;
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
{
|
||||
let batches = std::mem::take(&mut *self.pending_rocksdb_batches.lock());
|
||||
for batch in batches {
|
||||
self.rocksdb_provider.commit_batch(batch)?;
|
||||
}
|
||||
}
|
||||
|
||||
self.static_file_provider.commit()?;
|
||||
} else {
|
||||
self.static_file_provider.commit()?;
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
{
|
||||
let batches = std::mem::take(&mut *self.pending_rocksdb_batches.lock());
|
||||
for batch in batches {
|
||||
self.rocksdb_provider.commit_batch(batch)?;
|
||||
}
|
||||
}
|
||||
|
||||
self.tx.commit()?;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,913 +0,0 @@
|
||||
//! Invariant checking for `RocksDB` tables.
|
||||
//!
|
||||
//! This module provides consistency checks for tables stored in `RocksDB`, similar to the
|
||||
//! consistency checks for static files. The goal is to detect and potentially heal
|
||||
//! inconsistencies between `RocksDB` data and MDBX checkpoints.
|
||||
|
||||
use super::RocksDBProvider;
|
||||
use crate::StaticFileProviderFactory;
|
||||
use alloy_eips::eip2718::Encodable2718;
|
||||
use alloy_primitives::BlockNumber;
|
||||
use rayon::prelude::*;
|
||||
use reth_db::cursor::DbCursorRO;
|
||||
use reth_db_api::{tables, transaction::DbTx};
|
||||
use reth_stages_types::StageId;
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::{
|
||||
DBProvider, StageCheckpointReader, StorageSettingsCache, TransactionsProvider,
|
||||
};
|
||||
use reth_storage_errors::provider::ProviderResult;
|
||||
|
||||
impl RocksDBProvider {
|
||||
/// Checks consistency of `RocksDB` tables against MDBX stage checkpoints.
|
||||
///
|
||||
/// Returns an unwind target block number if the pipeline needs to unwind to rebuild
|
||||
/// `RocksDB` data. Returns `None` if all invariants pass or if inconsistencies were healed.
|
||||
///
|
||||
/// # Invariants checked
|
||||
///
|
||||
/// For `TransactionHashNumbers`:
|
||||
/// - The maximum `TxNumber` value should not exceed what the `TransactionLookup` stage
|
||||
/// checkpoint indicates has been processed.
|
||||
/// - If `RocksDB` is ahead, excess entries are pruned (healed).
|
||||
/// - If `RocksDB` is behind, an unwind is required.
|
||||
///
|
||||
/// For `StoragesHistory`:
|
||||
/// - The maximum block number in shards should not exceed the `IndexStorageHistory` stage
|
||||
/// checkpoint.
|
||||
/// - Similar healing/unwind logic applies.
|
||||
///
|
||||
/// # Requirements
|
||||
///
|
||||
/// For pruning `TransactionHashNumbers`, the provider must be able to supply transaction
|
||||
/// data (typically from static files) so that transaction hashes can be computed. This
|
||||
/// implies that static files should be ahead of or in sync with `RocksDB`.
|
||||
pub fn check_consistency<Provider>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
) -> ProviderResult<Option<BlockNumber>>
|
||||
where
|
||||
Provider: DBProvider
|
||||
+ StageCheckpointReader
|
||||
+ StorageSettingsCache
|
||||
+ StaticFileProviderFactory
|
||||
+ TransactionsProvider<Transaction: Encodable2718>,
|
||||
{
|
||||
let mut unwind_target: Option<BlockNumber> = None;
|
||||
|
||||
// Check TransactionHashNumbers if stored in RocksDB
|
||||
if provider.cached_storage_settings().transaction_hash_numbers_in_rocksdb &&
|
||||
let Some(target) = self.check_transaction_hash_numbers(provider)?
|
||||
{
|
||||
unwind_target = Some(unwind_target.map_or(target, |t| t.min(target)));
|
||||
}
|
||||
|
||||
// Check StoragesHistory if stored in RocksDB
|
||||
if provider.cached_storage_settings().storages_history_in_rocksdb &&
|
||||
let Some(target) = self.check_storages_history(provider)?
|
||||
{
|
||||
unwind_target = Some(unwind_target.map_or(target, |t| t.min(target)));
|
||||
}
|
||||
|
||||
Ok(unwind_target)
|
||||
}
|
||||
|
||||
/// Checks invariants for the `TransactionHashNumbers` table.
|
||||
///
|
||||
/// Returns a block number to unwind to if MDBX is behind the checkpoint.
|
||||
/// If static files are ahead of MDBX, excess `RocksDB` entries are pruned (healed).
|
||||
///
|
||||
/// # Approach
|
||||
///
|
||||
/// Instead of iterating `RocksDB` entries (which is expensive and doesn't give us the
|
||||
/// tx range we need), we use static files and MDBX to determine what needs pruning:
|
||||
/// - Static files are committed before `RocksDB`, so they're at least at the same height
|
||||
/// - MDBX `TransactionBlocks` tells us what's been fully committed
|
||||
/// - If static files have more transactions than MDBX, prune the excess range
|
||||
fn check_transaction_hash_numbers<Provider>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
) -> ProviderResult<Option<BlockNumber>>
|
||||
where
|
||||
Provider: DBProvider
|
||||
+ StageCheckpointReader
|
||||
+ StaticFileProviderFactory
|
||||
+ TransactionsProvider<Transaction: Encodable2718>,
|
||||
{
|
||||
// Get the TransactionLookup stage checkpoint
|
||||
let checkpoint = provider
|
||||
.get_stage_checkpoint(StageId::TransactionLookup)?
|
||||
.map(|cp| cp.block_number)
|
||||
.unwrap_or(0);
|
||||
|
||||
// Get last tx_num from MDBX - this tells us what MDBX has fully committed
|
||||
let mut cursor = provider.tx_ref().cursor_read::<tables::TransactionBlocks>()?;
|
||||
let mdbx_last = cursor.last()?;
|
||||
|
||||
// Get highest tx_num from static files - this tells us what tx data is available
|
||||
let highest_static_tx = provider
|
||||
.static_file_provider()
|
||||
.get_highest_static_file_tx(StaticFileSegment::Transactions);
|
||||
|
||||
match (mdbx_last, highest_static_tx) {
|
||||
(Some((mdbx_tx, mdbx_block)), Some(highest_tx)) if highest_tx > mdbx_tx => {
|
||||
// Static files are ahead of MDBX - prune RocksDB entries for the excess range.
|
||||
// This is the common case during recovery from a crash during unwinding.
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
mdbx_last_tx = mdbx_tx,
|
||||
mdbx_block,
|
||||
highest_static_tx = highest_tx,
|
||||
"Static files ahead of MDBX, pruning TransactionHashNumbers excess data"
|
||||
);
|
||||
self.prune_transaction_hash_numbers_in_range(provider, (mdbx_tx + 1)..=highest_tx)?;
|
||||
|
||||
// After pruning, check if MDBX is behind checkpoint
|
||||
if checkpoint > mdbx_block {
|
||||
tracing::warn!(
|
||||
target: "reth::providers::rocksdb",
|
||||
mdbx_block,
|
||||
checkpoint,
|
||||
"MDBX behind checkpoint after pruning, unwind needed"
|
||||
);
|
||||
return Ok(Some(mdbx_block));
|
||||
}
|
||||
}
|
||||
(Some((_mdbx_tx, mdbx_block)), _) => {
|
||||
// MDBX and static files are in sync (or static files don't have more data).
|
||||
// Check if MDBX is behind checkpoint.
|
||||
if checkpoint > mdbx_block {
|
||||
tracing::warn!(
|
||||
target: "reth::providers::rocksdb",
|
||||
mdbx_block,
|
||||
checkpoint,
|
||||
"MDBX behind checkpoint, unwind needed"
|
||||
);
|
||||
return Ok(Some(mdbx_block));
|
||||
}
|
||||
}
|
||||
(None, Some(highest_tx)) => {
|
||||
// MDBX has no transactions but static files have data.
|
||||
// This means RocksDB might have stale entries - prune them all.
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
highest_static_tx = highest_tx,
|
||||
"MDBX empty but static files have data, pruning all TransactionHashNumbers"
|
||||
);
|
||||
self.prune_transaction_hash_numbers_in_range(provider, 0..=highest_tx)?;
|
||||
}
|
||||
(None, None) => {
|
||||
// Both MDBX and static files are empty.
|
||||
// If checkpoint says we should have data, that's an inconsistency.
|
||||
if checkpoint > 0 {
|
||||
tracing::warn!(
|
||||
target: "reth::providers::rocksdb",
|
||||
checkpoint,
|
||||
"Checkpoint set but no transaction data exists, unwind needed"
|
||||
);
|
||||
return Ok(Some(0));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Prunes `TransactionHashNumbers` entries for transactions in the given range.
|
||||
///
|
||||
/// This fetches transactions from the provider, computes their hashes in parallel,
|
||||
/// and deletes the corresponding entries from `RocksDB` by key. This approach is more
|
||||
/// scalable than iterating all rows because it only processes the transactions that
|
||||
/// need to be pruned.
|
||||
///
|
||||
/// # Requirements
|
||||
///
|
||||
/// The provider must be able to supply transaction data (typically from static files)
|
||||
/// so that transaction hashes can be computed. This implies that static files should
|
||||
/// be ahead of or in sync with `RocksDB`.
|
||||
fn prune_transaction_hash_numbers_in_range<Provider>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
tx_range: std::ops::RangeInclusive<u64>,
|
||||
) -> ProviderResult<()>
|
||||
where
|
||||
Provider: TransactionsProvider<Transaction: Encodable2718>,
|
||||
{
|
||||
if tx_range.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Fetch transactions in the range and compute their hashes in parallel
|
||||
let hashes: Vec<_> = provider
|
||||
.transactions_by_tx_range(tx_range.clone())?
|
||||
.into_par_iter()
|
||||
.map(|tx| tx.trie_hash())
|
||||
.collect();
|
||||
|
||||
if !hashes.is_empty() {
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
deleted_count = hashes.len(),
|
||||
tx_range_start = *tx_range.start(),
|
||||
tx_range_end = *tx_range.end(),
|
||||
"Pruning TransactionHashNumbers entries by tx range"
|
||||
);
|
||||
|
||||
let mut batch = self.batch();
|
||||
for hash in hashes {
|
||||
batch.delete::<tables::TransactionHashNumbers>(hash)?;
|
||||
}
|
||||
batch.commit()?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Checks invariants for the `StoragesHistory` table.
|
||||
///
|
||||
/// Returns a block number to unwind to if `RocksDB` is behind the checkpoint.
|
||||
/// If `RocksDB` is ahead of the checkpoint, excess entries are pruned (healed).
|
||||
fn check_storages_history<Provider>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
) -> ProviderResult<Option<BlockNumber>>
|
||||
where
|
||||
Provider: DBProvider + StageCheckpointReader,
|
||||
{
|
||||
// Get the IndexStorageHistory stage checkpoint
|
||||
let checkpoint = provider
|
||||
.get_stage_checkpoint(StageId::IndexStorageHistory)?
|
||||
.map(|cp| cp.block_number)
|
||||
.unwrap_or(0);
|
||||
|
||||
// Check if RocksDB has any data
|
||||
let rocks_first = self.first::<tables::StoragesHistory>()?;
|
||||
|
||||
match rocks_first {
|
||||
Some(_) => {
|
||||
// If checkpoint is 0 but we have data, clear everything
|
||||
if checkpoint == 0 {
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
"StoragesHistory has data but checkpoint is 0, clearing all"
|
||||
);
|
||||
self.prune_storages_history_above(0)?;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// Find the max highest_block_number (excluding u64::MAX sentinel) across all
|
||||
// entries
|
||||
let mut max_highest_block = 0u64;
|
||||
for result in self.iter::<tables::StoragesHistory>()? {
|
||||
let (key, _) = result?;
|
||||
let highest = key.sharded_key.highest_block_number;
|
||||
if highest != u64::MAX && highest > max_highest_block {
|
||||
max_highest_block = highest;
|
||||
}
|
||||
}
|
||||
|
||||
// If any entry has highest_block > checkpoint, prune excess
|
||||
if max_highest_block > checkpoint {
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
rocks_highest = max_highest_block,
|
||||
checkpoint,
|
||||
"StoragesHistory ahead of checkpoint, pruning excess data"
|
||||
);
|
||||
self.prune_storages_history_above(checkpoint)?;
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
None => {
|
||||
// Empty RocksDB table
|
||||
if checkpoint > 0 {
|
||||
// Stage says we should have data but we don't
|
||||
return Ok(Some(0));
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Prunes `StoragesHistory` entries where `highest_block_number` > `max_block`.
|
||||
///
|
||||
/// For `StoragesHistory`, the key contains `highest_block_number`, so we can iterate
|
||||
/// and delete entries where `key.sharded_key.highest_block_number > max_block`.
|
||||
///
|
||||
/// TODO(<https://github.com/paradigmxyz/reth/issues/20417>): this iterates the whole table,
|
||||
/// which is inefficient. Use changeset-based pruning instead.
|
||||
fn prune_storages_history_above(&self, max_block: BlockNumber) -> ProviderResult<()> {
|
||||
use reth_db_api::models::storage_sharded_key::StorageShardedKey;
|
||||
|
||||
let mut to_delete: Vec<StorageShardedKey> = Vec::new();
|
||||
for result in self.iter::<tables::StoragesHistory>()? {
|
||||
let (key, _) = result?;
|
||||
let highest_block = key.sharded_key.highest_block_number;
|
||||
if max_block == 0 || (highest_block != u64::MAX && highest_block > max_block) {
|
||||
to_delete.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
let deleted = to_delete.len();
|
||||
if deleted > 0 {
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
deleted_count = deleted,
|
||||
max_block,
|
||||
"Pruning StoragesHistory entries"
|
||||
);
|
||||
|
||||
let mut batch = self.batch();
|
||||
for key in to_delete {
|
||||
batch.delete::<tables::StoragesHistory>(key)?;
|
||||
}
|
||||
batch.commit()?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{
|
||||
providers::rocksdb::RocksDBBuilder, test_utils::create_test_provider_factory, BlockWriter,
|
||||
DatabaseProviderFactory, StageCheckpointWriter, TransactionsProvider,
|
||||
};
|
||||
use alloy_primitives::{Address, B256};
|
||||
use reth_db::cursor::DbCursorRW;
|
||||
use reth_db_api::{
|
||||
models::{storage_sharded_key::StorageShardedKey, StorageSettings},
|
||||
tables::{self, BlockNumberList},
|
||||
transaction::DbTxMut,
|
||||
};
|
||||
use reth_stages_types::StageCheckpoint;
|
||||
use reth_testing_utils::generators::{self, BlockRangeParams};
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_first_last_empty_rocksdb() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let provider = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::TransactionHashNumbers>()
|
||||
.with_table::<tables::StoragesHistory>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Empty RocksDB, no checkpoints - should be consistent
|
||||
let first = provider.first::<tables::TransactionHashNumbers>().unwrap();
|
||||
let last = provider.last::<tables::TransactionHashNumbers>().unwrap();
|
||||
|
||||
assert!(first.is_none());
|
||||
assert!(last.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_first_last_with_data() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let provider = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::TransactionHashNumbers>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Insert some data
|
||||
let tx_hash = B256::from([1u8; 32]);
|
||||
provider.put::<tables::TransactionHashNumbers>(tx_hash, &100).unwrap();
|
||||
|
||||
// RocksDB has data
|
||||
let last = provider.last::<tables::TransactionHashNumbers>().unwrap();
|
||||
assert!(last.is_some());
|
||||
assert_eq!(last.unwrap().1, 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_empty_rocksdb_no_checkpoint_is_ok() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::TransactionHashNumbers>()
|
||||
.with_table::<tables::StoragesHistory>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Create a test provider factory for MDBX
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy()
|
||||
.with_transaction_hash_numbers_in_rocksdb(true)
|
||||
.with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// Empty RocksDB and no checkpoints - should be consistent (None = no unwind needed)
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_empty_rocksdb_with_checkpoint_needs_unwind() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::TransactionHashNumbers>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Create a test provider factory for MDBX
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Set a checkpoint indicating we should have processed up to block 100
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
provider
|
||||
.save_stage_checkpoint(StageId::TransactionLookup, StageCheckpoint::new(100))
|
||||
.unwrap();
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// RocksDB is empty but checkpoint says block 100 was processed
|
||||
// This means RocksDB is missing data and we need to unwind to rebuild
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(result, Some(0), "Should require unwind to block 0 to rebuild RocksDB");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_mdbx_empty_static_files_have_data_prunes_rocksdb() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::TransactionHashNumbers>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Generate blocks with real transactions and insert them
|
||||
let mut rng = generators::rng();
|
||||
let blocks = generators::random_block_range(
|
||||
&mut rng,
|
||||
0..=2,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() },
|
||||
);
|
||||
|
||||
let mut tx_hashes = Vec::new();
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
let mut tx_count = 0u64;
|
||||
for block in &blocks {
|
||||
provider.insert_block(block.clone().try_recover().expect("recover block")).unwrap();
|
||||
for tx in &block.body().transactions {
|
||||
let hash = tx.trie_hash();
|
||||
tx_hashes.push(hash);
|
||||
rocksdb.put::<tables::TransactionHashNumbers>(hash, &tx_count).unwrap();
|
||||
tx_count += 1;
|
||||
}
|
||||
}
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
// Simulate crash recovery: MDBX was reset but static files and RocksDB still have data.
|
||||
// Clear TransactionBlocks to simulate empty MDBX state.
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
let mut cursor = provider.tx_ref().cursor_write::<tables::TransactionBlocks>().unwrap();
|
||||
let mut to_delete = Vec::new();
|
||||
let mut walker = cursor.walk(Some(0)).unwrap();
|
||||
while let Some((tx_num, _)) = walker.next().transpose().unwrap() {
|
||||
to_delete.push(tx_num);
|
||||
}
|
||||
drop(walker);
|
||||
for tx_num in to_delete {
|
||||
cursor.seek_exact(tx_num).unwrap();
|
||||
cursor.delete_current().unwrap();
|
||||
}
|
||||
// No checkpoint set (checkpoint = 0)
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
// Verify RocksDB data exists
|
||||
assert!(rocksdb.last::<tables::TransactionHashNumbers>().unwrap().is_some());
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// MDBX TransactionBlocks is empty, but static files have transaction data.
|
||||
// This means RocksDB has stale data that should be pruned (healed).
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(result, None, "Should heal by pruning, no unwind needed");
|
||||
|
||||
// Verify data was pruned
|
||||
for hash in &tx_hashes {
|
||||
assert!(
|
||||
rocksdb.get::<tables::TransactionHashNumbers>(*hash).unwrap().is_none(),
|
||||
"RocksDB should be empty after pruning"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_storages_history_empty_with_checkpoint_needs_unwind() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::StoragesHistory>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Create a test provider factory for MDBX
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Set a checkpoint indicating we should have processed up to block 100
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
provider
|
||||
.save_stage_checkpoint(StageId::IndexStorageHistory, StageCheckpoint::new(100))
|
||||
.unwrap();
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// RocksDB is empty but checkpoint says block 100 was processed
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(result, Some(0), "Should require unwind to block 0 to rebuild StoragesHistory");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_storages_history_has_data_no_checkpoint_prunes_data() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::StoragesHistory>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Insert data into RocksDB
|
||||
let key = StorageShardedKey::new(Address::ZERO, B256::ZERO, 50);
|
||||
let block_list = BlockNumberList::new_pre_sorted([10, 20, 30, 50]);
|
||||
rocksdb.put::<tables::StoragesHistory>(key, &block_list).unwrap();
|
||||
|
||||
// Verify data exists
|
||||
assert!(rocksdb.last::<tables::StoragesHistory>().unwrap().is_some());
|
||||
|
||||
// Create a test provider factory for MDBX with NO checkpoint
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// RocksDB has data but checkpoint is 0
|
||||
// This means RocksDB has stale data that should be pruned (healed)
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(result, None, "Should heal by pruning, no unwind needed");
|
||||
|
||||
// Verify data was pruned
|
||||
assert!(
|
||||
rocksdb.last::<tables::StoragesHistory>().unwrap().is_none(),
|
||||
"RocksDB should be empty after pruning"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_mdbx_behind_checkpoint_needs_unwind() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::TransactionHashNumbers>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Generate blocks with real transactions (blocks 0-2, 6 transactions total)
|
||||
let mut rng = generators::rng();
|
||||
let blocks = generators::random_block_range(
|
||||
&mut rng,
|
||||
0..=2,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() },
|
||||
);
|
||||
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
let mut tx_count = 0u64;
|
||||
for block in &blocks {
|
||||
provider.insert_block(block.clone().try_recover().expect("recover block")).unwrap();
|
||||
for tx in &block.body().transactions {
|
||||
let hash = tx.trie_hash();
|
||||
rocksdb.put::<tables::TransactionHashNumbers>(hash, &tx_count).unwrap();
|
||||
tx_count += 1;
|
||||
}
|
||||
}
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
// Now simulate a scenario where checkpoint is ahead of MDBX.
|
||||
// This happens when the checkpoint was saved but MDBX data was lost/corrupted.
|
||||
// Set checkpoint to block 10 (beyond our actual data at block 2)
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
provider
|
||||
.save_stage_checkpoint(StageId::TransactionLookup, StageCheckpoint::new(10))
|
||||
.unwrap();
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// MDBX has data up to block 2, but checkpoint says block 10 was processed.
|
||||
// The static files highest tx matches MDBX last tx (both at block 2).
|
||||
// Checkpoint > mdbx_block means we need to unwind to rebuild.
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(
|
||||
result,
|
||||
Some(2),
|
||||
"Should require unwind to block 2 (MDBX's last block) to rebuild from checkpoint"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_rocksdb_ahead_of_checkpoint_prunes_excess() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::TransactionHashNumbers>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Create a test provider factory for MDBX
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Generate blocks with real transactions:
|
||||
// Blocks 0-5, each with 2 transactions = 12 total transactions (0-11)
|
||||
let mut rng = generators::rng();
|
||||
let blocks = generators::random_block_range(
|
||||
&mut rng,
|
||||
0..=5,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() },
|
||||
);
|
||||
|
||||
// Track which hashes belong to which blocks
|
||||
let mut tx_hashes = Vec::new();
|
||||
let mut tx_count = 0u64;
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
// Insert ALL blocks (0-5) to write transactions to static files
|
||||
for block in &blocks {
|
||||
provider.insert_block(block.clone().try_recover().expect("recover block")).unwrap();
|
||||
for tx in &block.body().transactions {
|
||||
let hash = tx.trie_hash();
|
||||
tx_hashes.push(hash);
|
||||
rocksdb.put::<tables::TransactionHashNumbers>(hash, &tx_count).unwrap();
|
||||
tx_count += 1;
|
||||
}
|
||||
}
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
// Simulate crash recovery scenario:
|
||||
// MDBX was unwound to block 2, but RocksDB and static files still have more data.
|
||||
// Remove TransactionBlocks entries for blocks 3-5 to simulate MDBX unwind.
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
// Delete TransactionBlocks entries for tx > 5 (i.e., for blocks 3-5)
|
||||
// TransactionBlocks maps last_tx_in_block -> block_number
|
||||
// After unwind, only entries for blocks 0-2 should remain (tx 5 -> block 2)
|
||||
let mut cursor = provider.tx_ref().cursor_write::<tables::TransactionBlocks>().unwrap();
|
||||
// Walk and delete entries where block > 2
|
||||
let mut to_delete = Vec::new();
|
||||
let mut walker = cursor.walk(Some(0)).unwrap();
|
||||
while let Some((tx_num, block_num)) = walker.next().transpose().unwrap() {
|
||||
if block_num > 2 {
|
||||
to_delete.push(tx_num);
|
||||
}
|
||||
}
|
||||
drop(walker);
|
||||
for tx_num in to_delete {
|
||||
cursor.seek_exact(tx_num).unwrap();
|
||||
cursor.delete_current().unwrap();
|
||||
}
|
||||
|
||||
// Set checkpoint to block 2
|
||||
provider
|
||||
.save_stage_checkpoint(StageId::TransactionLookup, StageCheckpoint::new(2))
|
||||
.unwrap();
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// RocksDB has tx hashes for all blocks (0-5)
|
||||
// MDBX TransactionBlocks only goes up to tx 5 (block 2)
|
||||
// Static files have data for all txs (0-11)
|
||||
// This means RocksDB is ahead and should prune entries for tx 6-11
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(result, None, "Should heal by pruning, no unwind needed");
|
||||
|
||||
// Verify: hashes for blocks 0-2 (tx 0-5) should remain, blocks 3-5 (tx 6-11) should be
|
||||
// pruned First 6 hashes should remain
|
||||
for (i, hash) in tx_hashes.iter().take(6).enumerate() {
|
||||
assert!(
|
||||
rocksdb.get::<tables::TransactionHashNumbers>(*hash).unwrap().is_some(),
|
||||
"tx {} should remain",
|
||||
i
|
||||
);
|
||||
}
|
||||
// Last 6 hashes should be pruned
|
||||
for (i, hash) in tx_hashes.iter().skip(6).enumerate() {
|
||||
assert!(
|
||||
rocksdb.get::<tables::TransactionHashNumbers>(*hash).unwrap().is_none(),
|
||||
"tx {} should be pruned",
|
||||
i + 6
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_storages_history_ahead_of_checkpoint_prunes_excess() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::StoragesHistory>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Insert data into RocksDB with different highest_block_numbers
|
||||
let key_block_50 = StorageShardedKey::new(Address::ZERO, B256::ZERO, 50);
|
||||
let key_block_100 = StorageShardedKey::new(Address::ZERO, B256::from([1u8; 32]), 100);
|
||||
let key_block_150 = StorageShardedKey::new(Address::ZERO, B256::from([2u8; 32]), 150);
|
||||
let key_block_max = StorageShardedKey::new(Address::ZERO, B256::from([3u8; 32]), u64::MAX);
|
||||
|
||||
let block_list = BlockNumberList::new_pre_sorted([10, 20, 30]);
|
||||
rocksdb.put::<tables::StoragesHistory>(key_block_50.clone(), &block_list).unwrap();
|
||||
rocksdb.put::<tables::StoragesHistory>(key_block_100.clone(), &block_list).unwrap();
|
||||
rocksdb.put::<tables::StoragesHistory>(key_block_150.clone(), &block_list).unwrap();
|
||||
rocksdb.put::<tables::StoragesHistory>(key_block_max.clone(), &block_list).unwrap();
|
||||
|
||||
// Create a test provider factory for MDBX
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Set checkpoint to block 100
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
provider
|
||||
.save_stage_checkpoint(StageId::IndexStorageHistory, StageCheckpoint::new(100))
|
||||
.unwrap();
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// RocksDB has entries with highest_block = 150 which exceeds checkpoint (100)
|
||||
// Should prune entries where highest_block > 100 (but not u64::MAX sentinel)
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(result, None, "Should heal by pruning, no unwind needed");
|
||||
|
||||
// Verify key_block_150 was pruned, but others remain
|
||||
assert!(
|
||||
rocksdb.get::<tables::StoragesHistory>(key_block_50).unwrap().is_some(),
|
||||
"Entry with highest_block=50 should remain"
|
||||
);
|
||||
assert!(
|
||||
rocksdb.get::<tables::StoragesHistory>(key_block_100).unwrap().is_some(),
|
||||
"Entry with highest_block=100 should remain"
|
||||
);
|
||||
assert!(
|
||||
rocksdb.get::<tables::StoragesHistory>(key_block_150).unwrap().is_none(),
|
||||
"Entry with highest_block=150 should be pruned"
|
||||
);
|
||||
assert!(
|
||||
rocksdb.get::<tables::StoragesHistory>(key_block_max).unwrap().is_some(),
|
||||
"Entry with highest_block=u64::MAX (sentinel) should remain"
|
||||
);
|
||||
}
|
||||
|
||||
/// Test that pruning works by fetching transactions and computing their hashes,
|
||||
/// rather than iterating all rows. This test uses random blocks with unique
|
||||
/// transactions so we can verify the correct entries are pruned.
|
||||
#[test]
|
||||
fn test_prune_transaction_hash_numbers_by_range() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::TransactionHashNumbers>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Create a test provider factory for MDBX
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Generate random blocks with unique transactions
|
||||
// Block 0 (genesis) has no transactions
|
||||
// Blocks 1-5 each have 2 transactions = 10 transactions total
|
||||
let mut rng = generators::rng();
|
||||
let blocks = generators::random_block_range(
|
||||
&mut rng,
|
||||
0..=5,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() },
|
||||
);
|
||||
|
||||
// Insert blocks into the database
|
||||
let mut tx_count = 0u64;
|
||||
let mut tx_hashes = Vec::new();
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
|
||||
for block in &blocks {
|
||||
provider.insert_block(block.clone().try_recover().expect("recover block")).unwrap();
|
||||
|
||||
// Store transaction hash -> tx_number mappings in RocksDB
|
||||
for tx in &block.body().transactions {
|
||||
let hash = tx.trie_hash();
|
||||
tx_hashes.push(hash);
|
||||
rocksdb.put::<tables::TransactionHashNumbers>(hash, &tx_count).unwrap();
|
||||
tx_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Set checkpoint to block 2 (meaning we should only have tx hashes for blocks 0-2)
|
||||
// Blocks 0, 1, 2 have 6 transactions (2 each), so tx 0-5 should remain
|
||||
provider
|
||||
.save_stage_checkpoint(StageId::TransactionLookup, StageCheckpoint::new(2))
|
||||
.unwrap();
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
// At this point:
|
||||
// - RocksDB has tx hashes for blocks 0-5 (10 total: 2 per block)
|
||||
// - Checkpoint says we only processed up to block 2
|
||||
// - We need to prune tx hashes for blocks 3, 4, 5 (tx 6-9)
|
||||
|
||||
// Verify RocksDB has the expected number of entries before pruning
|
||||
let rocksdb_count_before: usize =
|
||||
rocksdb.iter::<tables::TransactionHashNumbers>().unwrap().count();
|
||||
assert_eq!(
|
||||
rocksdb_count_before, tx_count as usize,
|
||||
"RocksDB should have all {} transaction hashes before pruning",
|
||||
tx_count
|
||||
);
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// Verify we can fetch transactions by tx range
|
||||
let all_txs = provider.transactions_by_tx_range(0..tx_count).unwrap();
|
||||
assert_eq!(all_txs.len(), tx_count as usize, "Should be able to fetch all transactions");
|
||||
|
||||
// Verify the hashes match between what we stored and what we compute from fetched txs
|
||||
for (i, tx) in all_txs.iter().enumerate() {
|
||||
let computed_hash = tx.trie_hash();
|
||||
assert_eq!(
|
||||
computed_hash, tx_hashes[i],
|
||||
"Hash mismatch for tx {}: stored {:?} vs computed {:?}",
|
||||
i, tx_hashes[i], computed_hash
|
||||
);
|
||||
}
|
||||
|
||||
// Blocks 0, 1, 2 have 2 tx each = 6 tx total (indices 0-5)
|
||||
// We want to keep tx 0-5, prune tx 6-9
|
||||
let max_tx_to_keep = 5u64;
|
||||
let tx_to_prune_start = max_tx_to_keep + 1;
|
||||
|
||||
// Prune transactions 6-9 (blocks 3-5)
|
||||
rocksdb
|
||||
.prune_transaction_hash_numbers_in_range(&provider, tx_to_prune_start..=(tx_count - 1))
|
||||
.expect("prune should succeed");
|
||||
|
||||
// Verify: transactions 0-5 should remain, 6-9 should be pruned
|
||||
let mut remaining_count = 0;
|
||||
for result in rocksdb.iter::<tables::TransactionHashNumbers>().unwrap() {
|
||||
let (_hash, tx_num) = result.unwrap();
|
||||
assert!(
|
||||
tx_num <= max_tx_to_keep,
|
||||
"Transaction {} should have been pruned (> {})",
|
||||
tx_num,
|
||||
max_tx_to_keep
|
||||
);
|
||||
remaining_count += 1;
|
||||
}
|
||||
assert_eq!(
|
||||
remaining_count,
|
||||
(max_tx_to_keep + 1) as usize,
|
||||
"Should have {} transactions (0-{})",
|
||||
max_tx_to_keep + 1,
|
||||
max_tx_to_keep
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,5 @@
|
||||
//! [`RocksDBProvider`] implementation
|
||||
|
||||
mod invariants;
|
||||
mod metrics;
|
||||
mod provider;
|
||||
|
||||
pub use provider::{RocksDBBatch, RocksDBBuilder, RocksDBProvider, RocksTx};
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use super::metrics::{RocksDBMetrics, RocksDBOperation};
|
||||
use reth_db_api::{
|
||||
table::{Compress, Decompress, Encode, Table},
|
||||
tables, DatabaseError,
|
||||
DatabaseError,
|
||||
};
|
||||
use reth_storage_errors::{
|
||||
db::{DatabaseErrorInfo, DatabaseWriteError, DatabaseWriteOperation, LogLevel},
|
||||
@@ -143,18 +143,6 @@ impl RocksDBBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Registers the default tables used by reth for `RocksDB` storage.
|
||||
///
|
||||
/// This registers:
|
||||
/// - [`tables::TransactionHashNumbers`] - Transaction hash to number mapping
|
||||
/// - [`tables::AccountsHistory`] - Account history index
|
||||
/// - [`tables::StoragesHistory`] - Storage history index
|
||||
pub fn with_default_tables(self) -> Self {
|
||||
self.with_table::<tables::TransactionHashNumbers>()
|
||||
.with_table::<tables::AccountsHistory>()
|
||||
.with_table::<tables::StoragesHistory>()
|
||||
}
|
||||
|
||||
/// Enables metrics.
|
||||
pub const fn with_metrics(mut self) -> Self {
|
||||
self.enable_metrics = true;
|
||||
@@ -380,65 +368,6 @@ impl RocksDBProvider {
|
||||
})
|
||||
}
|
||||
|
||||
/// Gets the first (smallest key) entry from the specified table.
|
||||
pub fn first<T: Table>(&self) -> ProviderResult<Option<(T::Key, T::Value)>> {
|
||||
self.execute_with_operation_metric(RocksDBOperation::Get, T::NAME, |this| {
|
||||
let cf = this.get_cf_handle::<T>()?;
|
||||
let mut iter = this.0.db.iterator_cf(cf, IteratorMode::Start);
|
||||
|
||||
match iter.next() {
|
||||
Some(Ok((key_bytes, value_bytes))) => {
|
||||
let key = <T::Key as reth_db_api::table::Decode>::decode(&key_bytes)
|
||||
.map_err(|_| ProviderError::Database(DatabaseError::Decode))?;
|
||||
let value = T::Value::decompress(&value_bytes)
|
||||
.map_err(|_| ProviderError::Database(DatabaseError::Decode))?;
|
||||
Ok(Some((key, value)))
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
Err(ProviderError::Database(DatabaseError::Read(DatabaseErrorInfo {
|
||||
message: e.to_string().into(),
|
||||
code: -1,
|
||||
})))
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Gets the last (largest key) entry from the specified table.
|
||||
pub fn last<T: Table>(&self) -> ProviderResult<Option<(T::Key, T::Value)>> {
|
||||
self.execute_with_operation_metric(RocksDBOperation::Get, T::NAME, |this| {
|
||||
let cf = this.get_cf_handle::<T>()?;
|
||||
let mut iter = this.0.db.iterator_cf(cf, IteratorMode::End);
|
||||
|
||||
match iter.next() {
|
||||
Some(Ok((key_bytes, value_bytes))) => {
|
||||
let key = <T::Key as reth_db_api::table::Decode>::decode(&key_bytes)
|
||||
.map_err(|_| ProviderError::Database(DatabaseError::Decode))?;
|
||||
let value = T::Value::decompress(&value_bytes)
|
||||
.map_err(|_| ProviderError::Database(DatabaseError::Decode))?;
|
||||
Ok(Some((key, value)))
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
Err(ProviderError::Database(DatabaseError::Read(DatabaseErrorInfo {
|
||||
message: e.to_string().into(),
|
||||
code: -1,
|
||||
})))
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Creates an iterator over all entries in the specified table.
|
||||
///
|
||||
/// Returns decoded `(Key, Value)` pairs in key order.
|
||||
pub fn iter<T: Table>(&self) -> ProviderResult<RocksDBIter<'_, T>> {
|
||||
let cf = self.get_cf_handle::<T>()?;
|
||||
let iter = self.0.db.iterator_cf(cf, IteratorMode::Start);
|
||||
Ok(RocksDBIter { inner: iter, _marker: std::marker::PhantomData })
|
||||
}
|
||||
|
||||
/// Writes a batch of operations atomically.
|
||||
pub fn write_batch<F>(&self, f: F) -> ProviderResult<()>
|
||||
where
|
||||
@@ -450,19 +379,6 @@ impl RocksDBProvider {
|
||||
batch_handle.commit()
|
||||
})
|
||||
}
|
||||
|
||||
/// Commits a raw `WriteBatchWithTransaction` to `RocksDB`.
|
||||
///
|
||||
/// This is used when the batch was extracted via [`RocksDBBatch::into_inner`]
|
||||
/// and needs to be committed at a later point (e.g., at provider commit time).
|
||||
pub fn commit_batch(&self, batch: WriteBatchWithTransaction<true>) -> ProviderResult<()> {
|
||||
self.0.db.write_opt(batch, &WriteOptions::default()).map_err(|e| {
|
||||
ProviderError::Database(DatabaseError::Commit(DatabaseErrorInfo {
|
||||
message: e.to_string().into(),
|
||||
code: -1,
|
||||
}))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle for building a batch of operations atomically.
|
||||
@@ -537,18 +453,6 @@ impl<'a> RocksDBBatch<'a> {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.inner.is_empty()
|
||||
}
|
||||
|
||||
/// Returns a reference to the underlying `RocksDB` provider.
|
||||
pub const fn provider(&self) -> &RocksDBProvider {
|
||||
self.provider
|
||||
}
|
||||
|
||||
/// Consumes the batch and returns the underlying `WriteBatchWithTransaction`.
|
||||
///
|
||||
/// This is used to defer commits to the provider level.
|
||||
pub fn into_inner(self) -> WriteBatchWithTransaction<true> {
|
||||
self.inner
|
||||
}
|
||||
}
|
||||
|
||||
/// `RocksDB` transaction wrapper providing MDBX-like semantics.
|
||||
@@ -668,50 +572,6 @@ impl<'db> RocksTx<'db> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterator over a `RocksDB` table (non-transactional).
|
||||
///
|
||||
/// Yields decoded `(Key, Value)` pairs in key order.
|
||||
pub struct RocksDBIter<'db, T: Table> {
|
||||
inner: rocksdb::DBIteratorWithThreadMode<'db, TransactionDB>,
|
||||
_marker: std::marker::PhantomData<T>,
|
||||
}
|
||||
|
||||
impl<T: Table> fmt::Debug for RocksDBIter<'_, T> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("RocksDBIter").field("table", &T::NAME).finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Table> Iterator for RocksDBIter<'_, T> {
|
||||
type Item = ProviderResult<(T::Key, T::Value)>;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
let (key_bytes, value_bytes) = match self.inner.next()? {
|
||||
Ok(kv) => kv,
|
||||
Err(e) => {
|
||||
return Some(Err(ProviderError::Database(DatabaseError::Read(DatabaseErrorInfo {
|
||||
message: e.to_string().into(),
|
||||
code: -1,
|
||||
}))))
|
||||
}
|
||||
};
|
||||
|
||||
// Decode key
|
||||
let key = match <T::Key as reth_db_api::table::Decode>::decode(&key_bytes) {
|
||||
Ok(k) => k,
|
||||
Err(_) => return Some(Err(ProviderError::Database(DatabaseError::Decode))),
|
||||
};
|
||||
|
||||
// Decompress value
|
||||
let value = match T::Value::decompress(&value_bytes) {
|
||||
Ok(v) => v,
|
||||
Err(_) => return Some(Err(ProviderError::Database(DatabaseError::Decode))),
|
||||
};
|
||||
|
||||
Some(Ok((key, value)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterator over a `RocksDB` table within a transaction.
|
||||
///
|
||||
/// Yields decoded `(Key, Value)` pairs. Sees uncommitted writes.
|
||||
@@ -770,38 +630,10 @@ const fn convert_log_level(level: LogLevel) -> rocksdb::LogLevel {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_primitives::{Address, TxHash, B256};
|
||||
use reth_db_api::{
|
||||
models::{sharded_key::ShardedKey, storage_sharded_key::StorageShardedKey, IntegerList},
|
||||
table::Table,
|
||||
tables,
|
||||
};
|
||||
use alloy_primitives::{TxHash, B256};
|
||||
use reth_db_api::{table::Table, tables};
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_with_default_tables_registers_required_column_families() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
// Build with default tables
|
||||
let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap();
|
||||
|
||||
// Should be able to write/read TransactionHashNumbers
|
||||
let tx_hash = TxHash::from(B256::from([1u8; 32]));
|
||||
provider.put::<tables::TransactionHashNumbers>(tx_hash, &100).unwrap();
|
||||
assert_eq!(provider.get::<tables::TransactionHashNumbers>(tx_hash).unwrap(), Some(100));
|
||||
|
||||
// Should be able to write/read AccountsHistory
|
||||
let key = ShardedKey::new(Address::ZERO, 100);
|
||||
let value = IntegerList::default();
|
||||
provider.put::<tables::AccountsHistory>(key.clone(), &value).unwrap();
|
||||
assert!(provider.get::<tables::AccountsHistory>(key).unwrap().is_some());
|
||||
|
||||
// Should be able to write/read StoragesHistory
|
||||
let key = StorageShardedKey::new(Address::ZERO, B256::ZERO, 100);
|
||||
provider.put::<tables::StoragesHistory>(key.clone(), &value).unwrap();
|
||||
assert!(provider.get::<tables::StoragesHistory>(key).unwrap().is_some());
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TestTable;
|
||||
|
||||
@@ -1070,28 +902,4 @@ mod tests {
|
||||
assert_eq!(provider.get::<TestTable>(i).unwrap(), Some(value));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_first_and_last_entry() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let provider =
|
||||
RocksDBBuilder::new(temp_dir.path()).with_table::<TestTable>().build().unwrap();
|
||||
|
||||
// Empty table should return None for both
|
||||
assert_eq!(provider.first::<TestTable>().unwrap(), None);
|
||||
assert_eq!(provider.last::<TestTable>().unwrap(), None);
|
||||
|
||||
// Insert some entries
|
||||
provider.put::<TestTable>(10, &b"value_10".to_vec()).unwrap();
|
||||
provider.put::<TestTable>(20, &b"value_20".to_vec()).unwrap();
|
||||
provider.put::<TestTable>(5, &b"value_5".to_vec()).unwrap();
|
||||
|
||||
// First should return the smallest key
|
||||
let first = provider.first::<TestTable>().unwrap();
|
||||
assert_eq!(first, Some((5, b"value_5".to_vec())));
|
||||
|
||||
// Last should return the largest key
|
||||
let last = provider.last::<TestTable>().unwrap();
|
||||
assert_eq!(last, Some((20, b"value_20".to_vec())));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,11 +119,6 @@ impl RocksDBBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Registers the default tables used by reth for `RocksDB` storage (stub implementation).
|
||||
pub const fn with_default_tables(self) -> Self {
|
||||
self
|
||||
}
|
||||
|
||||
/// Enables metrics (stub implementation).
|
||||
pub const fn with_metrics(self) -> Self {
|
||||
self
|
||||
|
||||
@@ -151,23 +151,6 @@ impl<N: NodePrimitives> StaticFileProviderBuilder<N> {
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the genesis block number for the [`StaticFileProvider`].
|
||||
///
|
||||
/// This configures the genesis block number, which is used to determine the starting point
|
||||
/// for block indexing and querying operations.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `genesis_block_number` - The block number of the genesis block.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Returns `Self` to allow method chaining.
|
||||
pub const fn with_genesis_block_number(mut self, genesis_block_number: u64) -> Self {
|
||||
self.inner.genesis_block_number = genesis_block_number;
|
||||
self
|
||||
}
|
||||
|
||||
/// Builds the final [`StaticFileProvider`] and initializes the index.
|
||||
pub fn build(self) -> ProviderResult<StaticFileProvider<N>> {
|
||||
let provider = StaticFileProvider(Arc::new(self.inner));
|
||||
@@ -325,8 +308,6 @@ pub struct StaticFileProviderInner<N> {
|
||||
blocks_per_file: HashMap<StaticFileSegment, u64>,
|
||||
/// Write lock for when access is [`StaticFileAccess::RW`].
|
||||
_lock_file: Option<StorageLock>,
|
||||
/// Genesis block number, default is 0;
|
||||
genesis_block_number: u64,
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives> StaticFileProviderInner<N> {
|
||||
@@ -353,7 +334,6 @@ impl<N: NodePrimitives> StaticFileProviderInner<N> {
|
||||
access,
|
||||
blocks_per_file,
|
||||
_lock_file,
|
||||
genesis_block_number: 0,
|
||||
};
|
||||
|
||||
Ok(provider)
|
||||
@@ -429,11 +409,6 @@ impl<N: NodePrimitives> StaticFileProviderInner<N> {
|
||||
block,
|
||||
)
|
||||
}
|
||||
|
||||
/// Get genesis block number
|
||||
pub const fn genesis_block_number(&self) -> u64 {
|
||||
self.genesis_block_number
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
@@ -1751,11 +1726,7 @@ impl<N: NodePrimitives> StaticFileWriter for StaticFileProvider<N> {
|
||||
&self,
|
||||
segment: StaticFileSegment,
|
||||
) -> ProviderResult<StaticFileProviderRWRefMut<'_, Self::Primitives>> {
|
||||
let genesis_number = self.0.as_ref().genesis_block_number();
|
||||
self.get_writer(
|
||||
self.get_highest_static_file_block(segment).unwrap_or(genesis_number),
|
||||
segment,
|
||||
)
|
||||
self.get_writer(self.get_highest_static_file_block(segment).unwrap_or_default(), segment)
|
||||
}
|
||||
|
||||
fn commit(&self) -> ProviderResult<()> {
|
||||
|
||||
@@ -363,9 +363,8 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
|
||||
.as_ref()
|
||||
.map(|block_range| block_range.end())
|
||||
.or_else(|| {
|
||||
(self.writer.user_header().expected_block_start() >
|
||||
self.reader().genesis_block_number())
|
||||
.then(|| self.writer.user_header().expected_block_start() - 1)
|
||||
(self.writer.user_header().expected_block_start() > 0)
|
||||
.then(|| self.writer.user_header().expected_block_start() - 1)
|
||||
});
|
||||
|
||||
self.reader().update_index(self.writer.user_header().segment(), segment_max_block)
|
||||
@@ -646,37 +645,6 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Appends header to static file without calling `increment_block`.
|
||||
/// This is useful for genesis blocks with non-zero block numbers.
|
||||
pub fn append_header_direct(
|
||||
&mut self,
|
||||
header: &N::BlockHeader,
|
||||
total_difficulty: U256,
|
||||
hash: &BlockHash,
|
||||
) -> ProviderResult<()>
|
||||
where
|
||||
N::BlockHeader: Compact,
|
||||
{
|
||||
let start = Instant::now();
|
||||
self.ensure_no_queued_prune()?;
|
||||
|
||||
debug_assert!(self.writer.user_header().segment() == StaticFileSegment::Headers);
|
||||
|
||||
self.append_column(header)?;
|
||||
self.append_column(CompactU256::from(total_difficulty))?;
|
||||
self.append_column(hash)?;
|
||||
|
||||
if let Some(metrics) = &self.metrics {
|
||||
metrics.record_segment_operation(
|
||||
StaticFileSegment::Headers,
|
||||
StaticFileProviderOperation::Append,
|
||||
Some(start.elapsed()),
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Appends transaction to static file.
|
||||
///
|
||||
/// It **DOES NOT CALL** `increment_block()`, it should be handled elsewhere. There might be
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use crate::{
|
||||
providers::{NodeTypesForProvider, ProviderNodeTypes, RocksDBBuilder, StaticFileProvider},
|
||||
providers::{NodeTypesForProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider},
|
||||
HashingWriter, ProviderFactory, TrieWriter,
|
||||
};
|
||||
use alloy_primitives::B256;
|
||||
@@ -62,10 +62,7 @@ pub fn create_test_provider_factory_with_node_types<N: NodeTypesForProvider>(
|
||||
db,
|
||||
chain_spec,
|
||||
StaticFileProvider::read_write(static_dir.keep()).expect("static file provider"),
|
||||
RocksDBBuilder::new(&rocksdb_dir)
|
||||
.with_default_tables()
|
||||
.build()
|
||||
.expect("failed to create test RocksDB provider"),
|
||||
RocksDBProvider::new(&rocksdb_dir).expect("failed to create test RocksDB provider"),
|
||||
)
|
||||
.expect("failed to create test provider factory")
|
||||
}
|
||||
|
||||
@@ -29,9 +29,4 @@ impl<C: Send + Sync, N: NodePrimitives> RocksDBProviderFactory for NoopProvider<
|
||||
fn rocksdb_provider(&self) -> RocksDBProvider {
|
||||
RocksDBProvider::builder(PathBuf::default()).build().unwrap()
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
fn set_pending_rocksdb_batch(&self, _batch: rocksdb::WriteBatchWithTransaction<true>) {
|
||||
// No-op for NoopProvider
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,11 +6,4 @@ use crate::providers::RocksDBProvider;
|
||||
pub trait RocksDBProviderFactory {
|
||||
/// Returns the `RocksDB` provider.
|
||||
fn rocksdb_provider(&self) -> RocksDBProvider;
|
||||
|
||||
/// Adds a pending `RocksDB` batch to be committed when this provider is committed.
|
||||
///
|
||||
/// This allows deferring `RocksDB` commits to happen at the same time as MDBX and static file
|
||||
/// commits, ensuring atomicity across all storage backends.
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
fn set_pending_rocksdb_batch(&self, batch: rocksdb::WriteBatchWithTransaction<true>);
|
||||
}
|
||||
|
||||
@@ -59,73 +59,6 @@ impl DiskFileBlobStore {
|
||||
fn clear_cache(&self) {
|
||||
self.inner.blob_cache.lock().clear()
|
||||
}
|
||||
|
||||
/// Look up EIP-7594 blobs by their versioned hashes.
|
||||
///
|
||||
/// This returns a result vector with the **same length and order** as the input
|
||||
/// `versioned_hashes`. Each element is `Some(BlobAndProofV2)` if the blob is available, or
|
||||
/// `None` if it is missing or an older sidecar version.
|
||||
///
|
||||
/// The lookup first scans the in-memory cache and, if not all blobs are found, falls back to
|
||||
/// reading candidate sidecars from disk using the `versioned_hash -> tx_hash` index.
|
||||
fn get_by_versioned_hashes_eip7594(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Vec<Option<BlobAndProofV2>>, BlobStoreError> {
|
||||
// we must return the blobs in order but we don't necessarily find them in the requested
|
||||
// order
|
||||
let mut result = vec![None; versioned_hashes.len()];
|
||||
|
||||
// first scan all cached full sidecars
|
||||
for (_tx_hash, blob_sidecar) in self.inner.blob_cache.lock().iter() {
|
||||
if let Some(blob_sidecar) = blob_sidecar.as_eip7594() {
|
||||
for (hash_idx, match_result) in
|
||||
blob_sidecar.match_versioned_hashes(versioned_hashes)
|
||||
{
|
||||
result[hash_idx] = Some(match_result);
|
||||
}
|
||||
}
|
||||
|
||||
// return early if all blobs are found.
|
||||
if result.iter().all(|blob| blob.is_some()) {
|
||||
return Ok(result);
|
||||
}
|
||||
}
|
||||
|
||||
// not all versioned hashes were found, try to look up a matching tx
|
||||
let mut missing_tx_hashes = Vec::new();
|
||||
|
||||
{
|
||||
let mut versioned_to_txhashes = self.inner.versioned_hashes_to_txhash.lock();
|
||||
for (idx, _) in
|
||||
result.iter().enumerate().filter(|(_, blob_and_proof)| blob_and_proof.is_none())
|
||||
{
|
||||
// this is safe because the result vec has the same len
|
||||
let versioned_hash = versioned_hashes[idx];
|
||||
if let Some(tx_hash) = versioned_to_txhashes.get(&versioned_hash).copied() {
|
||||
missing_tx_hashes.push(tx_hash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if we have missing blobs, try to read them from disk and try again
|
||||
if !missing_tx_hashes.is_empty() {
|
||||
let blobs_from_disk = self.inner.read_many_decoded(missing_tx_hashes);
|
||||
for (_, blob_sidecar) in blobs_from_disk {
|
||||
if let Some(blob_sidecar) = blob_sidecar.as_eip7594() {
|
||||
for (hash_idx, match_result) in
|
||||
blob_sidecar.match_versioned_hashes(versioned_hashes)
|
||||
{
|
||||
if result[hash_idx].is_none() {
|
||||
result[hash_idx] = Some(match_result);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
impl BlobStore for DiskFileBlobStore {
|
||||
@@ -272,7 +205,58 @@ impl BlobStore for DiskFileBlobStore {
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Option<Vec<BlobAndProofV2>>, BlobStoreError> {
|
||||
let result = self.get_by_versioned_hashes_eip7594(versioned_hashes)?;
|
||||
// we must return the blobs in order but we don't necessarily find them in the requested
|
||||
// order
|
||||
let mut result = vec![None; versioned_hashes.len()];
|
||||
|
||||
// first scan all cached full sidecars
|
||||
for (_tx_hash, blob_sidecar) in self.inner.blob_cache.lock().iter() {
|
||||
if let Some(blob_sidecar) = blob_sidecar.as_eip7594() {
|
||||
for (hash_idx, match_result) in
|
||||
blob_sidecar.match_versioned_hashes(versioned_hashes)
|
||||
{
|
||||
result[hash_idx] = Some(match_result);
|
||||
}
|
||||
}
|
||||
|
||||
// return early if all blobs are found.
|
||||
if result.iter().all(|blob| blob.is_some()) {
|
||||
// got all blobs, can return early
|
||||
return Ok(Some(result.into_iter().map(Option::unwrap).collect()))
|
||||
}
|
||||
}
|
||||
|
||||
// not all versioned hashes were found, try to look up a matching tx
|
||||
let mut missing_tx_hashes = Vec::new();
|
||||
|
||||
{
|
||||
let mut versioned_to_txhashes = self.inner.versioned_hashes_to_txhash.lock();
|
||||
for (idx, _) in
|
||||
result.iter().enumerate().filter(|(_, blob_and_proof)| blob_and_proof.is_none())
|
||||
{
|
||||
// this is safe because the result vec has the same len
|
||||
let versioned_hash = versioned_hashes[idx];
|
||||
if let Some(tx_hash) = versioned_to_txhashes.get(&versioned_hash).copied() {
|
||||
missing_tx_hashes.push(tx_hash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if we have missing blobs, try to read them from disk and try again
|
||||
if !missing_tx_hashes.is_empty() {
|
||||
let blobs_from_disk = self.inner.read_many_decoded(missing_tx_hashes);
|
||||
for (_, blob_sidecar) in blobs_from_disk {
|
||||
if let Some(blob_sidecar) = blob_sidecar.as_eip7594() {
|
||||
for (hash_idx, match_result) in
|
||||
blob_sidecar.match_versioned_hashes(versioned_hashes)
|
||||
{
|
||||
if result[hash_idx].is_none() {
|
||||
result[hash_idx] = Some(match_result);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// only return the blobs if we found all requested versioned hashes
|
||||
if result.iter().all(|blob| blob.is_some()) {
|
||||
@@ -282,13 +266,6 @@ impl BlobStore for DiskFileBlobStore {
|
||||
}
|
||||
}
|
||||
|
||||
fn get_by_versioned_hashes_v3(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Vec<Option<BlobAndProofV2>>, BlobStoreError> {
|
||||
self.get_by_versioned_hashes_eip7594(versioned_hashes)
|
||||
}
|
||||
|
||||
fn data_size_hint(&self) -> Option<usize> {
|
||||
Some(self.inner.size_tracker.data_size())
|
||||
}
|
||||
@@ -679,12 +656,7 @@ pub enum OpenDiskFileBlobStore {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use alloy_consensus::BlobTransactionSidecar;
|
||||
use alloy_eips::{
|
||||
eip4844::{kzg_to_versioned_hash, Blob, BlobAndProofV2, Bytes48},
|
||||
eip7594::{
|
||||
BlobTransactionSidecarEip7594, BlobTransactionSidecarVariant, CELLS_PER_EXT_BLOB,
|
||||
},
|
||||
};
|
||||
use alloy_eips::eip7594::BlobTransactionSidecarVariant;
|
||||
|
||||
use super::*;
|
||||
use std::sync::atomic::Ordering;
|
||||
@@ -710,20 +682,6 @@ mod tests {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn eip7594_single_blob_sidecar() -> (BlobTransactionSidecarVariant, B256, BlobAndProofV2) {
|
||||
let blob = Blob::default();
|
||||
let commitment = Bytes48::default();
|
||||
let cell_proofs = vec![Bytes48::default(); CELLS_PER_EXT_BLOB];
|
||||
|
||||
let versioned_hash = kzg_to_versioned_hash(commitment.as_slice());
|
||||
|
||||
let expected =
|
||||
BlobAndProofV2 { blob: Box::new(Blob::default()), proofs: cell_proofs.clone() };
|
||||
let sidecar = BlobTransactionSidecarEip7594::new(vec![blob], vec![commitment], cell_proofs);
|
||||
|
||||
(BlobTransactionSidecarVariant::Eip7594(sidecar), versioned_hash, expected)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_insert_all_get_all() {
|
||||
let (store, _dir) = tmp_store();
|
||||
@@ -893,33 +851,4 @@ mod tests {
|
||||
assert_eq!(stat.delete_succeed, 3);
|
||||
assert_eq!(stat.delete_failed, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_get_blobs_v3_returns_partial_results() {
|
||||
let (store, _dir) = tmp_store();
|
||||
|
||||
let (sidecar, versioned_hash, expected) = eip7594_single_blob_sidecar();
|
||||
store.insert(TxHash::random(), sidecar).unwrap();
|
||||
|
||||
assert_ne!(versioned_hash, B256::ZERO);
|
||||
|
||||
let request = vec![versioned_hash, B256::ZERO];
|
||||
let v2 = store.get_by_versioned_hashes_v2(&request).unwrap();
|
||||
assert!(v2.is_none(), "v2 must return null if any requested blob is missing");
|
||||
|
||||
let v3 = store.get_by_versioned_hashes_v3(&request).unwrap();
|
||||
assert_eq!(v3, vec![Some(expected), None]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_get_blobs_v3_can_fallback_to_disk() {
|
||||
let (store, _dir) = tmp_store();
|
||||
|
||||
let (sidecar, versioned_hash, expected) = eip7594_single_blob_sidecar();
|
||||
store.insert(TxHash::random(), sidecar).unwrap();
|
||||
store.clear_cache();
|
||||
|
||||
let v3 = store.get_by_versioned_hashes_v3(&[versioned_hash]).unwrap();
|
||||
assert_eq!(v3, vec![Some(expected)]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,35 +13,6 @@ pub struct InMemoryBlobStore {
|
||||
inner: Arc<InMemoryBlobStoreInner>,
|
||||
}
|
||||
|
||||
impl InMemoryBlobStore {
|
||||
/// Look up EIP-7594 blobs by their versioned hashes.
|
||||
///
|
||||
/// This returns a result vector with the **same length and order** as the input
|
||||
/// `versioned_hashes`. Each element is `Some(BlobAndProofV2)` if the blob is available, or
|
||||
/// `None` if it is missing or an older sidecar version.
|
||||
fn get_by_versioned_hashes_eip7594(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Vec<Option<BlobAndProofV2>> {
|
||||
let mut result = vec![None; versioned_hashes.len()];
|
||||
for (_tx_hash, blob_sidecar) in self.inner.store.read().iter() {
|
||||
if let Some(blob_sidecar) = blob_sidecar.as_eip7594() {
|
||||
for (hash_idx, match_result) in
|
||||
blob_sidecar.match_versioned_hashes(versioned_hashes)
|
||||
{
|
||||
result[hash_idx] = Some(match_result);
|
||||
}
|
||||
}
|
||||
|
||||
// Return early if all blobs are found.
|
||||
if result.iter().all(|blob| blob.is_some()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct InMemoryBlobStoreInner {
|
||||
/// Storage for all blob data.
|
||||
@@ -163,7 +134,20 @@ impl BlobStore for InMemoryBlobStore {
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Option<Vec<BlobAndProofV2>>, BlobStoreError> {
|
||||
let result = self.get_by_versioned_hashes_eip7594(versioned_hashes);
|
||||
let mut result = vec![None; versioned_hashes.len()];
|
||||
for (_tx_hash, blob_sidecar) in self.inner.store.read().iter() {
|
||||
if let Some(blob_sidecar) = blob_sidecar.as_eip7594() {
|
||||
for (hash_idx, match_result) in
|
||||
blob_sidecar.match_versioned_hashes(versioned_hashes)
|
||||
{
|
||||
result[hash_idx] = Some(match_result);
|
||||
}
|
||||
}
|
||||
|
||||
if result.iter().all(|blob| blob.is_some()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if result.iter().all(|blob| blob.is_some()) {
|
||||
Ok(Some(result.into_iter().map(Option::unwrap).collect()))
|
||||
} else {
|
||||
@@ -171,13 +155,6 @@ impl BlobStore for InMemoryBlobStore {
|
||||
}
|
||||
}
|
||||
|
||||
fn get_by_versioned_hashes_v3(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Vec<Option<BlobAndProofV2>>, BlobStoreError> {
|
||||
Ok(self.get_by_versioned_hashes_eip7594(versioned_hashes))
|
||||
}
|
||||
|
||||
fn data_size_hint(&self) -> Option<usize> {
|
||||
Some(self.inner.size_tracker.data_size())
|
||||
}
|
||||
@@ -206,45 +183,3 @@ fn insert_size(
|
||||
store.insert(tx, Arc::new(blob));
|
||||
add
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_eips::{
|
||||
eip4844::{kzg_to_versioned_hash, Blob, BlobAndProofV2, Bytes48},
|
||||
eip7594::{
|
||||
BlobTransactionSidecarEip7594, BlobTransactionSidecarVariant, CELLS_PER_EXT_BLOB,
|
||||
},
|
||||
};
|
||||
|
||||
fn eip7594_single_blob_sidecar() -> (BlobTransactionSidecarVariant, B256, BlobAndProofV2) {
|
||||
let blob = Blob::default();
|
||||
let commitment = Bytes48::default();
|
||||
let cell_proofs = vec![Bytes48::default(); CELLS_PER_EXT_BLOB];
|
||||
|
||||
let versioned_hash = kzg_to_versioned_hash(commitment.as_slice());
|
||||
|
||||
let expected =
|
||||
BlobAndProofV2 { blob: Box::new(Blob::default()), proofs: cell_proofs.clone() };
|
||||
let sidecar = BlobTransactionSidecarEip7594::new(vec![blob], vec![commitment], cell_proofs);
|
||||
|
||||
(BlobTransactionSidecarVariant::Eip7594(sidecar), versioned_hash, expected)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mem_get_blobs_v3_returns_partial_results() {
|
||||
let store = InMemoryBlobStore::default();
|
||||
|
||||
let (sidecar, versioned_hash, expected) = eip7594_single_blob_sidecar();
|
||||
store.insert(B256::random(), sidecar).unwrap();
|
||||
|
||||
assert_ne!(versioned_hash, B256::ZERO);
|
||||
|
||||
let request = vec![versioned_hash, B256::ZERO];
|
||||
let v2 = store.get_by_versioned_hashes_v2(&request).unwrap();
|
||||
assert!(v2.is_none(), "v2 must return null if any requested blob is missing");
|
||||
|
||||
let v3 = store.get_by_versioned_hashes_v3(&request).unwrap();
|
||||
assert_eq!(v3, vec![Some(expected), None]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,15 +100,6 @@ pub trait BlobStore: fmt::Debug + Send + Sync + 'static {
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Option<Vec<BlobAndProofV2>>, BlobStoreError>;
|
||||
|
||||
/// Return the [`BlobAndProofV2`]s for a list of blob versioned hashes.
|
||||
///
|
||||
/// The response is always the same length as the request. Missing or older-version blobs are
|
||||
/// returned as `None` elements.
|
||||
fn get_by_versioned_hashes_v3(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Vec<Option<BlobAndProofV2>>, BlobStoreError>;
|
||||
|
||||
/// Data size of all transactions in the blob store.
|
||||
fn data_size_hint(&self) -> Option<usize>;
|
||||
|
||||
|
||||
@@ -78,13 +78,6 @@ impl BlobStore for NoopBlobStore {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn get_by_versioned_hashes_v3(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Vec<Option<BlobAndProofV2>>, BlobStoreError> {
|
||||
Ok(vec![None; versioned_hashes.len()])
|
||||
}
|
||||
|
||||
fn data_size_hint(&self) -> Option<usize> {
|
||||
Some(0)
|
||||
}
|
||||
|
||||
@@ -751,13 +751,6 @@ where
|
||||
) -> Result<Option<Vec<BlobAndProofV2>>, BlobStoreError> {
|
||||
self.pool.blob_store().get_by_versioned_hashes_v2(versioned_hashes)
|
||||
}
|
||||
|
||||
fn get_blobs_for_versioned_hashes_v3(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Vec<Option<BlobAndProofV2>>, BlobStoreError> {
|
||||
self.pool.blob_store().get_by_versioned_hashes_v3(versioned_hashes)
|
||||
}
|
||||
}
|
||||
|
||||
impl<V, T, S> TransactionPoolExt for Pool<V, T, S>
|
||||
|
||||
@@ -345,13 +345,6 @@ impl<T: EthPoolTransaction> TransactionPool for NoopTransactionPool<T> {
|
||||
) -> Result<Option<Vec<BlobAndProofV2>>, BlobStoreError> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn get_blobs_for_versioned_hashes_v3(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Vec<Option<BlobAndProofV2>>, BlobStoreError> {
|
||||
Ok(vec![None; versioned_hashes.len()])
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`TransactionValidator`] that does nothing.
|
||||
|
||||
@@ -638,15 +638,6 @@ pub trait TransactionPool: Clone + Debug + Send + Sync {
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Option<Vec<BlobAndProofV2>>, BlobStoreError>;
|
||||
|
||||
/// Return the [`BlobAndProofV2`]s for a list of blob versioned hashes.
|
||||
///
|
||||
/// The response is always the same length as the request. Missing or older-version blobs are
|
||||
/// returned as `None` elements.
|
||||
fn get_blobs_for_versioned_hashes_v3(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Vec<Option<BlobAndProofV2>>, BlobStoreError>;
|
||||
}
|
||||
|
||||
/// Extension for [`TransactionPool`] trait that allows to set the current block info.
|
||||
|
||||
@@ -78,7 +78,7 @@ Logging:
|
||||
|
||||
Possible values:
|
||||
- always: Colors on
|
||||
- auto: Auto-detect
|
||||
- auto: Colors on
|
||||
- never: Colors off
|
||||
|
||||
Display:
|
||||
@@ -93,4 +93,4 @@ Display:
|
||||
|
||||
-q, --quiet
|
||||
Silence all log output
|
||||
```
|
||||
```
|
||||
96
docs/vocs/docs/pages/cli/reth/debug.mdx
Normal file
96
docs/vocs/docs/pages/cli/reth/debug.mdx
Normal file
@@ -0,0 +1,96 @@
|
||||
# reth debug
|
||||
|
||||
Various debug routines
|
||||
|
||||
```bash
|
||||
$ reth debug --help
|
||||
```
|
||||
```txt
|
||||
Usage: reth debug [OPTIONS] <COMMAND>
|
||||
|
||||
Commands:
|
||||
merkle Debug the clean & incremental state root calculations
|
||||
in-memory-merkle Debug in-memory state root calculation
|
||||
help Print this message or the help of the given subcommand(s)
|
||||
|
||||
Options:
|
||||
-h, --help
|
||||
Print help (see a summary with '-h')
|
||||
|
||||
Logging:
|
||||
--log.stdout.format <FORMAT>
|
||||
The format to use for logs written to stdout
|
||||
|
||||
[default: terminal]
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
--log.stdout.filter <FILTER>
|
||||
The filter to use for logs written to stdout
|
||||
|
||||
[default: ]
|
||||
|
||||
--log.file.format <FORMAT>
|
||||
The format to use for logs written to the log file
|
||||
|
||||
[default: terminal]
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
--log.file.filter <FILTER>
|
||||
The filter to use for logs written to the log file
|
||||
|
||||
[default: debug]
|
||||
|
||||
--log.file.directory <PATH>
|
||||
The path to put log files in
|
||||
|
||||
[default: <CACHE_DIR>/logs]
|
||||
|
||||
--log.file.max-size <SIZE>
|
||||
The maximum size (in MB) of one log file
|
||||
|
||||
[default: 200]
|
||||
|
||||
--log.file.max-files <COUNT>
|
||||
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
|
||||
|
||||
[default: 5]
|
||||
|
||||
--log.journald
|
||||
Write logs to journald
|
||||
|
||||
--log.journald.filter <FILTER>
|
||||
The filter to use for logs written to journald
|
||||
|
||||
[default: error]
|
||||
|
||||
--color <COLOR>
|
||||
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
|
||||
|
||||
[default: always]
|
||||
|
||||
Possible values:
|
||||
- always: Colors on
|
||||
- auto: Colors on
|
||||
- never: Colors off
|
||||
|
||||
Display:
|
||||
-v, --verbosity...
|
||||
Set the minimum log level.
|
||||
|
||||
-v Errors
|
||||
-vv Warnings
|
||||
-vvv Info
|
||||
-vvvv Debug
|
||||
-vvvvv Traces (warning: very verbose!)
|
||||
|
||||
-q, --quiet
|
||||
Silence all log output
|
||||
```
|
||||
100
docs/vocs/docs/pages/cli/reth/recover.mdx
Normal file
100
docs/vocs/docs/pages/cli/reth/recover.mdx
Normal file
@@ -0,0 +1,100 @@
|
||||
# reth recover
|
||||
|
||||
Scripts for node recovery
|
||||
|
||||
```bash
|
||||
$ reth recover --help
|
||||
```
|
||||
```txt
|
||||
Usage: reth recover [OPTIONS] <COMMAND>
|
||||
|
||||
Commands:
|
||||
storage-tries Recover the node by deleting dangling storage tries
|
||||
help Print this message or the help of the given subcommand(s)
|
||||
|
||||
Options:
|
||||
-h, --help
|
||||
Print help (see a summary with '-h')
|
||||
|
||||
Logging:
|
||||
--log.stdout.format <FORMAT>
|
||||
The format to use for logs written to stdout
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.stdout.filter <FILTER>
|
||||
The filter to use for logs written to stdout
|
||||
|
||||
[default: ]
|
||||
|
||||
--log.file.format <FORMAT>
|
||||
The format to use for logs written to the log file
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.file.filter <FILTER>
|
||||
The filter to use for logs written to the log file
|
||||
|
||||
[default: debug]
|
||||
|
||||
--log.file.directory <PATH>
|
||||
The path to put log files in
|
||||
|
||||
[default: <CACHE_DIR>/logs]
|
||||
|
||||
--log.file.name <NAME>
|
||||
The prefix name of the log files
|
||||
|
||||
[default: reth.log]
|
||||
|
||||
--log.file.max-size <SIZE>
|
||||
The maximum size (in MB) of one log file
|
||||
|
||||
[default: 200]
|
||||
|
||||
--log.file.max-files <COUNT>
|
||||
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
|
||||
|
||||
[default: 5]
|
||||
|
||||
--log.journald
|
||||
Write logs to journald
|
||||
|
||||
--log.journald.filter <FILTER>
|
||||
The filter to use for logs written to journald
|
||||
|
||||
[default: error]
|
||||
|
||||
--color <COLOR>
|
||||
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
|
||||
|
||||
Possible values:
|
||||
- always: Colors on
|
||||
- auto: Colors on
|
||||
- never: Colors off
|
||||
|
||||
[default: always]
|
||||
|
||||
Display:
|
||||
-v, --verbosity...
|
||||
Set the minimum log level.
|
||||
|
||||
-v Errors
|
||||
-vv Warnings
|
||||
-vvv Info
|
||||
-vvvv Debug
|
||||
-vvvvv Traces (warning: very verbose!)
|
||||
|
||||
-q, --quiet
|
||||
Silence all log output
|
||||
```
|
||||
154
docs/vocs/docs/pages/cli/reth/recover/storage-tries.mdx
Normal file
154
docs/vocs/docs/pages/cli/reth/recover/storage-tries.mdx
Normal file
@@ -0,0 +1,154 @@
|
||||
# reth recover storage-tries
|
||||
|
||||
Recover the node by deleting dangling storage tries
|
||||
|
||||
```bash
|
||||
$ reth recover storage-tries --help
|
||||
```
|
||||
```txt
|
||||
Usage: reth recover storage-tries [OPTIONS]
|
||||
|
||||
Options:
|
||||
-h, --help
|
||||
Print help (see a summary with '-h')
|
||||
|
||||
Datadir:
|
||||
--datadir <DATA_DIR>
|
||||
The path to the data dir for all reth files and subdirectories.
|
||||
|
||||
Defaults to the OS-specific data directory:
|
||||
|
||||
- Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/`
|
||||
- Windows: `{FOLDERID_RoamingAppData}/reth/`
|
||||
- macOS: `$HOME/Library/Application Support/reth/`
|
||||
|
||||
[default: default]
|
||||
|
||||
--datadir.static-files <PATH>
|
||||
The absolute path to store static files in.
|
||||
|
||||
--config <FILE>
|
||||
The path to the configuration file to use
|
||||
|
||||
--chain <CHAIN_OR_PATH>
|
||||
The chain this node is running.
|
||||
Possible values are either a built-in chain or the path to a chain specification file.
|
||||
|
||||
Built-in chains:
|
||||
mainnet, sepolia, holesky, hoodi, dev
|
||||
|
||||
[default: mainnet]
|
||||
|
||||
Database:
|
||||
--db.log-level <LOG_LEVEL>
|
||||
Database logging level. Levels higher than "notice" require a debug build
|
||||
|
||||
Possible values:
|
||||
- fatal: Enables logging for critical conditions, i.e. assertion failures
|
||||
- error: Enables logging for error conditions
|
||||
- warn: Enables logging for warning conditions
|
||||
- notice: Enables logging for normal but significant condition
|
||||
- verbose: Enables logging for verbose informational
|
||||
- debug: Enables logging for debug-level messages
|
||||
- trace: Enables logging for trace debug-level messages
|
||||
- extra: Enables logging for extra debug-level messages
|
||||
|
||||
--db.exclusive <EXCLUSIVE>
|
||||
Open environment in exclusive/monopolistic mode. Makes it possible to open a database on an NFS volume
|
||||
|
||||
[possible values: true, false]
|
||||
|
||||
--db.max-size <MAX_SIZE>
|
||||
Maximum database size (e.g., 4TB, 8MB)
|
||||
|
||||
--db.growth-step <GROWTH_STEP>
|
||||
Database growth step (e.g., 4GB, 4KB)
|
||||
|
||||
--db.read-transaction-timeout <READ_TRANSACTION_TIMEOUT>
|
||||
Read transaction timeout in seconds, 0 means no timeout
|
||||
|
||||
--db.max-readers <MAX_READERS>
|
||||
Maximum number of readers allowed to access the database concurrently
|
||||
|
||||
Logging:
|
||||
--log.stdout.format <FORMAT>
|
||||
The format to use for logs written to stdout
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.stdout.filter <FILTER>
|
||||
The filter to use for logs written to stdout
|
||||
|
||||
[default: ]
|
||||
|
||||
--log.file.format <FORMAT>
|
||||
The format to use for logs written to the log file
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.file.filter <FILTER>
|
||||
The filter to use for logs written to the log file
|
||||
|
||||
[default: debug]
|
||||
|
||||
--log.file.directory <PATH>
|
||||
The path to put log files in
|
||||
|
||||
[default: <CACHE_DIR>/logs]
|
||||
|
||||
--log.file.name <NAME>
|
||||
The prefix name of the log files
|
||||
|
||||
[default: reth.log]
|
||||
|
||||
--log.file.max-size <SIZE>
|
||||
The maximum size (in MB) of one log file
|
||||
|
||||
[default: 200]
|
||||
|
||||
--log.file.max-files <COUNT>
|
||||
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
|
||||
|
||||
[default: 5]
|
||||
|
||||
--log.journald
|
||||
Write logs to journald
|
||||
|
||||
--log.journald.filter <FILTER>
|
||||
The filter to use for logs written to journald
|
||||
|
||||
[default: error]
|
||||
|
||||
--color <COLOR>
|
||||
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
|
||||
|
||||
Possible values:
|
||||
- always: Colors on
|
||||
- auto: Colors on
|
||||
- never: Colors off
|
||||
|
||||
[default: always]
|
||||
|
||||
Display:
|
||||
-v, --verbosity...
|
||||
Set the minimum log level.
|
||||
|
||||
-v Errors
|
||||
-vv Warnings
|
||||
-vvv Info
|
||||
-vvvv Debug
|
||||
-vvvvv Traces (warning: very verbose!)
|
||||
|
||||
-q, --quiet
|
||||
Silence all log output
|
||||
```
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user