mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
19 Commits
performanc
...
yk/thread_
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
54d07c55e5 | ||
|
|
a8d0a89a62 | ||
|
|
8e00e81af4 | ||
|
|
453514c48f | ||
|
|
432ac7afa1 | ||
|
|
c7fca9f2b4 | ||
|
|
715ca5b980 | ||
|
|
9ae62aad26 | ||
|
|
c65df40526 | ||
|
|
d8acc1e4cf | ||
|
|
852aad8126 | ||
|
|
61c072ad20 | ||
|
|
6a5b985113 | ||
|
|
1adc6aec00 | ||
|
|
5edc16ad85 | ||
|
|
f54a8a1ef5 | ||
|
|
c681851ec8 | ||
|
|
d964fcbcde | ||
|
|
e79691aae7 |
5
Cargo.lock
generated
5
Cargo.lock
generated
@@ -8244,6 +8244,7 @@ dependencies = [
|
||||
"derive_more",
|
||||
"eyre",
|
||||
"futures",
|
||||
"libc",
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"mini-moka",
|
||||
@@ -9286,6 +9287,7 @@ dependencies = [
|
||||
"reth-rpc-eth-api",
|
||||
"reth-rpc-eth-types",
|
||||
"reth-rpc-server-types",
|
||||
"reth-stages-types",
|
||||
"reth-tasks",
|
||||
"reth-testing-utils",
|
||||
"reth-tracing",
|
||||
@@ -9649,6 +9651,7 @@ dependencies = [
|
||||
"reth-rpc-engine-api",
|
||||
"reth-rpc-eth-types",
|
||||
"reth-rpc-server-types",
|
||||
"reth-stages-types",
|
||||
"reth-tasks",
|
||||
"reth-tracing",
|
||||
"reth-transaction-pool",
|
||||
@@ -10392,6 +10395,7 @@ dependencies = [
|
||||
"reth-ethereum-engine-primitives",
|
||||
"reth-ethereum-primitives",
|
||||
"reth-metrics",
|
||||
"reth-network-api",
|
||||
"reth-node-ethereum",
|
||||
"reth-payload-builder",
|
||||
"reth-payload-builder-primitives",
|
||||
@@ -10992,6 +10996,7 @@ dependencies = [
|
||||
"dashmap 6.1.0",
|
||||
"derive_more",
|
||||
"itertools 0.14.0",
|
||||
"libc",
|
||||
"metrics",
|
||||
"proptest",
|
||||
"proptest-arbitrary-interop",
|
||||
|
||||
@@ -18,7 +18,7 @@ FROM chef AS builder
|
||||
COPY --from=planner /app/recipe.json recipe.json
|
||||
|
||||
# Build profile, release by default
|
||||
ARG BUILD_PROFILE=release
|
||||
ARG BUILD_PROFILE=maxperf
|
||||
ENV BUILD_PROFILE=$BUILD_PROFILE
|
||||
|
||||
# Extra Cargo flags
|
||||
|
||||
@@ -14,7 +14,7 @@ RUN cargo chef prepare --recipe-path recipe.json
|
||||
FROM chef AS builder
|
||||
COPY --from=planner /app/recipe.json recipe.json
|
||||
|
||||
ARG BUILD_PROFILE=release
|
||||
ARG BUILD_PROFILE=maxperf
|
||||
ENV BUILD_PROFILE=$BUILD_PROFILE
|
||||
|
||||
ARG RUSTFLAGS=""
|
||||
|
||||
@@ -81,7 +81,7 @@ backon.workspace = true
|
||||
tempfile.workspace = true
|
||||
|
||||
[features]
|
||||
default = ["jemalloc", "otlp", "reth-revm/portable", "js-tracer", "keccak-cache-global"]
|
||||
default = ["jemalloc", "otlp", "reth-revm/portable", "js-tracer", "keccak-cache-global", "asm-keccak"]
|
||||
|
||||
otlp = [
|
||||
"reth-ethereum-cli/otlp",
|
||||
|
||||
@@ -80,6 +80,8 @@ pub fn make_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Hea
|
||||
.then_some(EMPTY_REQUESTS_HASH);
|
||||
|
||||
Header {
|
||||
number: genesis.number.unwrap_or_default(),
|
||||
parent_hash: genesis.parent_hash.unwrap_or_default(),
|
||||
gas_limit: genesis.gas_limit,
|
||||
difficulty: genesis.difficulty,
|
||||
nonce: genesis.nonce.into(),
|
||||
|
||||
@@ -23,7 +23,10 @@ use reth_node_core::{
|
||||
dirs::{ChainPath, DataDirPath},
|
||||
};
|
||||
use reth_provider::{
|
||||
providers::{BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider},
|
||||
providers::{
|
||||
BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider,
|
||||
StaticFileProviderBuilder,
|
||||
},
|
||||
ProviderFactory, StaticFileProviderFactory,
|
||||
};
|
||||
use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget};
|
||||
@@ -100,15 +103,23 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
}
|
||||
|
||||
info!(target: "reth::cli", ?db_path, ?sf_path, "Opening storage");
|
||||
let genesis_block_number = self.chain.genesis().number.unwrap();
|
||||
let (db, sfp) = match access {
|
||||
AccessRights::RW => (
|
||||
Arc::new(init_db(db_path, self.db.database_args())?),
|
||||
StaticFileProvider::read_write(sf_path)?,
|
||||
),
|
||||
AccessRights::RO | AccessRights::RoInconsistent => (
|
||||
Arc::new(open_db_read_only(&db_path, self.db.database_args())?),
|
||||
StaticFileProvider::read_only(sf_path, false)?,
|
||||
StaticFileProviderBuilder::read_write(sf_path)?
|
||||
.with_genesis_block_number(genesis_block_number)
|
||||
.build()?,
|
||||
),
|
||||
AccessRights::RO | AccessRights::RoInconsistent => {
|
||||
(Arc::new(open_db_read_only(&db_path, self.db.database_args())?), {
|
||||
let provider = StaticFileProviderBuilder::read_only(sf_path)?
|
||||
.with_genesis_block_number(genesis_block_number)
|
||||
.build()?;
|
||||
provider.watch_directory();
|
||||
provider
|
||||
})
|
||||
}
|
||||
};
|
||||
// TransactionDB only support read-write mode
|
||||
let rocksdb_provider = RocksDBProvider::builder(data_dir.rocksdb())
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
//! Command that initializes the node from a genesis file.
|
||||
|
||||
use crate::common::{AccessRights, CliNodeTypes, Environment, EnvironmentArgs};
|
||||
use alloy_consensus::BlockHeader;
|
||||
use clap::Parser;
|
||||
use reth_chainspec::{EthChainSpec, EthereumHardforks};
|
||||
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks};
|
||||
use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_provider::BlockHashReader;
|
||||
use std::sync::Arc;
|
||||
@@ -22,8 +23,9 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> InitComman
|
||||
|
||||
let Environment { provider_factory, .. } = self.env.init::<N>(AccessRights::RW)?;
|
||||
|
||||
let genesis_block_number = provider_factory.chain_spec().genesis_header().number();
|
||||
let hash = provider_factory
|
||||
.block_hash(0)?
|
||||
.block_hash(genesis_block_number)?
|
||||
.ok_or_else(|| eyre::eyre!("Genesis hash not found."))?;
|
||||
|
||||
info!(target: "reth::cli", hash = ?hash, "Genesis block written");
|
||||
|
||||
@@ -11,6 +11,7 @@ use reth_node_builder::{
|
||||
PayloadTypes,
|
||||
};
|
||||
use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs};
|
||||
use reth_primitives_traits::AlloyBlockHeader;
|
||||
use reth_provider::providers::BlockchainProvider;
|
||||
use reth_rpc_server_types::RpcModuleSelection;
|
||||
use reth_tasks::TaskManager;
|
||||
@@ -157,8 +158,8 @@ where
|
||||
.await?;
|
||||
|
||||
let node = NodeTestContext::new(node, self.attributes_generator).await?;
|
||||
|
||||
let genesis = node.block_hash(0);
|
||||
let genesis_number = self.chain_spec.genesis_header().number();
|
||||
let genesis = node.block_hash(genesis_number);
|
||||
node.update_forkchoice(genesis, genesis).await?;
|
||||
|
||||
eyre::Ok(node)
|
||||
|
||||
@@ -67,6 +67,9 @@ derive_more.workspace = true
|
||||
parking_lot.workspace = true
|
||||
crossbeam-channel.workspace = true
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
libc = "0.2"
|
||||
|
||||
# optional deps for test-utils
|
||||
reth-prune-types = { workspace = true, optional = true }
|
||||
reth-stages = { workspace = true, optional = true }
|
||||
|
||||
@@ -31,6 +31,9 @@ pub(crate) struct CachedStateProvider<S> {
|
||||
|
||||
/// Metrics for the cached state provider
|
||||
metrics: CachedStateMetrics,
|
||||
|
||||
/// If prewarm enabled we populate every cache miss
|
||||
prewarm: bool,
|
||||
}
|
||||
|
||||
impl<S> CachedStateProvider<S>
|
||||
@@ -39,12 +42,32 @@ where
|
||||
{
|
||||
/// Creates a new [`CachedStateProvider`] from an [`ExecutionCache`], state provider, and
|
||||
/// [`CachedStateMetrics`].
|
||||
pub(crate) const fn new_with_caches(
|
||||
pub(crate) const fn new(
|
||||
state_provider: S,
|
||||
caches: ExecutionCache,
|
||||
metrics: CachedStateMetrics,
|
||||
) -> Self {
|
||||
Self { state_provider, caches, metrics }
|
||||
Self { state_provider, caches, metrics, prewarm: false }
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> CachedStateProvider<S> {
|
||||
/// Enables pre-warm mode so that every cache miss is populated.
|
||||
///
|
||||
/// This is only relevant for pre-warm transaction execution with the intention to pre-populate
|
||||
/// the cache with data for regular block execution. During regular block execution the
|
||||
/// cache doesn't need to be populated because the actual EVM database
|
||||
/// [`State`](revm::database::State) also caches internally during block execution and the cache
|
||||
/// is then updated after the block with the entire [`BundleState`] output of that block which
|
||||
/// contains all accessed accounts,code,storage. See also [`ExecutionCache::insert_state`].
|
||||
pub(crate) const fn prewarm(mut self) -> Self {
|
||||
self.prewarm = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether this provider should pre-warm cache misses.
|
||||
const fn is_prewarm(&self) -> bool {
|
||||
self.prewarm
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,7 +146,10 @@ impl<S: AccountReader> AccountReader for CachedStateProvider<S> {
|
||||
self.metrics.account_cache_misses.increment(1);
|
||||
|
||||
let res = self.state_provider.basic_account(address)?;
|
||||
self.caches.account_cache.insert(*address, res);
|
||||
|
||||
if self.is_prewarm() {
|
||||
self.caches.account_cache.insert(*address, res);
|
||||
}
|
||||
Ok(res)
|
||||
}
|
||||
}
|
||||
@@ -148,15 +174,19 @@ impl<S: StateProvider> StateProvider for CachedStateProvider<S> {
|
||||
match self.caches.get_storage(&account, &storage_key) {
|
||||
(SlotStatus::NotCached, maybe_cache) => {
|
||||
let final_res = self.state_provider.storage(account, storage_key)?;
|
||||
let account_cache = maybe_cache.unwrap_or_default();
|
||||
account_cache.insert_storage(storage_key, final_res);
|
||||
// we always need to insert the value to update the weights.
|
||||
// Note: there exists a race when the storage cache did not exist yet and two
|
||||
// consumers looking up the a storage value for this account for the first time,
|
||||
// however we can assume that this will only happen for the very first (mostlikely
|
||||
// the same) value, and don't expect that this will accidentally
|
||||
// replace an account storage cache with additional values.
|
||||
self.caches.insert_storage_cache(account, account_cache);
|
||||
|
||||
if self.is_prewarm() {
|
||||
let account_cache = maybe_cache.unwrap_or_default();
|
||||
account_cache.insert_storage(storage_key, final_res);
|
||||
// we always need to insert the value to update the weights.
|
||||
// Note: there exists a race when the storage cache did not exist yet and two
|
||||
// consumers looking up the a storage value for this account for the first time,
|
||||
// however we can assume that this will only happen for the very first
|
||||
// (mostlikely the same) value, and don't expect that this
|
||||
// will accidentally replace an account storage cache with
|
||||
// additional values.
|
||||
self.caches.insert_storage_cache(account, account_cache);
|
||||
}
|
||||
|
||||
self.metrics.storage_cache_misses.increment(1);
|
||||
Ok(final_res)
|
||||
@@ -183,7 +213,11 @@ impl<S: BytecodeReader> BytecodeReader for CachedStateProvider<S> {
|
||||
self.metrics.code_cache_misses.increment(1);
|
||||
|
||||
let final_res = self.state_provider.bytecode_by_hash(code_hash)?;
|
||||
self.caches.code_cache.insert(*code_hash, final_res.clone());
|
||||
|
||||
if self.is_prewarm() {
|
||||
self.caches.code_cache.insert(*code_hash, final_res.clone());
|
||||
}
|
||||
|
||||
Ok(final_res)
|
||||
}
|
||||
}
|
||||
@@ -785,7 +819,7 @@ mod tests {
|
||||
|
||||
let caches = ExecutionCacheBuilder::default().build_caches(1000);
|
||||
let state_provider =
|
||||
CachedStateProvider::new_with_caches(provider, caches, CachedStateMetrics::zeroed());
|
||||
CachedStateProvider::new(provider, caches, CachedStateMetrics::zeroed());
|
||||
|
||||
// check that the storage is empty
|
||||
let res = state_provider.storage(address, storage_key);
|
||||
@@ -808,7 +842,7 @@ mod tests {
|
||||
|
||||
let caches = ExecutionCacheBuilder::default().build_caches(1000);
|
||||
let state_provider =
|
||||
CachedStateProvider::new_with_caches(provider, caches, CachedStateMetrics::zeroed());
|
||||
CachedStateProvider::new(provider, caches, CachedStateMetrics::zeroed());
|
||||
|
||||
// check that the storage returns the expected value
|
||||
let res = state_provider.storage(address, storage_key);
|
||||
|
||||
@@ -83,7 +83,7 @@ where
|
||||
{
|
||||
/// Creates a new [`InstrumentedStateProvider`] from a state provider with the provided label
|
||||
/// for metrics.
|
||||
pub fn from_state_provider(state_provider: S, source: &'static str) -> Self {
|
||||
pub fn new(state_provider: S, source: &'static str) -> Self {
|
||||
Self {
|
||||
state_provider,
|
||||
metrics: StateProviderMetrics::new_with_labels(&[("source", source)]),
|
||||
|
||||
@@ -1302,22 +1302,7 @@ where
|
||||
// Check if persistence has complete
|
||||
match rx.try_recv() {
|
||||
Ok(last_persisted_hash_num) => {
|
||||
self.metrics.engine.persistence_duration.record(start_time.elapsed());
|
||||
let Some(BlockNumHash {
|
||||
hash: last_persisted_block_hash,
|
||||
number: last_persisted_block_number,
|
||||
}) = last_persisted_hash_num
|
||||
else {
|
||||
// if this happened, then we persisted no blocks because we sent an
|
||||
// empty vec of blocks
|
||||
warn!(target: "engine::tree", "Persistence task completed but did not persist any blocks");
|
||||
return Ok(())
|
||||
};
|
||||
|
||||
debug!(target: "engine::tree", ?last_persisted_block_hash, ?last_persisted_block_number, elapsed=?start_time.elapsed(), "Finished persisting, calling finish");
|
||||
self.persistence_state
|
||||
.finish(last_persisted_block_hash, last_persisted_block_number);
|
||||
self.on_new_persisted_block()?;
|
||||
self.on_persistence_complete(last_persisted_hash_num, start_time)?;
|
||||
}
|
||||
Err(TryRecvError::Closed) => return Err(TryRecvError::Closed.into()),
|
||||
Err(TryRecvError::Empty) => {
|
||||
@@ -1338,6 +1323,30 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Handles a completed persistence task.
|
||||
fn on_persistence_complete(
|
||||
&mut self,
|
||||
last_persisted_hash_num: Option<BlockNumHash>,
|
||||
start_time: Instant,
|
||||
) -> Result<(), AdvancePersistenceError> {
|
||||
self.metrics.engine.persistence_duration.record(start_time.elapsed());
|
||||
|
||||
let Some(BlockNumHash {
|
||||
hash: last_persisted_block_hash,
|
||||
number: last_persisted_block_number,
|
||||
}) = last_persisted_hash_num
|
||||
else {
|
||||
// if this happened, then we persisted no blocks because we sent an empty vec of blocks
|
||||
warn!(target: "engine::tree", "Persistence task completed but did not persist any blocks");
|
||||
return Ok(())
|
||||
};
|
||||
|
||||
debug!(target: "engine::tree", ?last_persisted_block_hash, ?last_persisted_block_number, elapsed=?start_time.elapsed(), "Finished persisting, calling finish");
|
||||
self.persistence_state.finish(last_persisted_block_hash, last_persisted_block_number);
|
||||
self.on_new_persisted_block()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Handles a message from the engine.
|
||||
fn on_engine_message(
|
||||
&mut self,
|
||||
|
||||
@@ -6,6 +6,34 @@ use tokio::{
|
||||
task::JoinHandle,
|
||||
};
|
||||
|
||||
/// Sets the current thread's name for profiling visibility.
|
||||
#[inline]
|
||||
fn set_thread_name(name: &str) {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
// SAFETY: name is a valid string, prctl with PR_SET_NAME is safe
|
||||
unsafe {
|
||||
// PR_SET_NAME expects a null-terminated string, truncated to 16 bytes (including null)
|
||||
let mut buf = [0u8; 16];
|
||||
let len = name.len().min(15);
|
||||
buf[..len].copy_from_slice(&name.as_bytes()[..len]);
|
||||
libc::prctl(libc::PR_SET_NAME, buf.as_ptr());
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
// SAFETY: name is a valid string
|
||||
unsafe {
|
||||
let c_name = std::ffi::CString::new(name).unwrap_or_default();
|
||||
libc::pthread_setname_np(c_name.as_ptr());
|
||||
}
|
||||
}
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
{
|
||||
let _ = name;
|
||||
}
|
||||
}
|
||||
|
||||
/// An executor for mixed I/O and CPU workloads.
|
||||
///
|
||||
/// This type uses tokio to spawn blocking tasks and will reuse an existing tokio
|
||||
@@ -36,6 +64,22 @@ impl WorkloadExecutor {
|
||||
{
|
||||
self.inner.handle.spawn_blocking(func)
|
||||
}
|
||||
|
||||
/// Spawns a blocking task with a descriptive thread name for profiling.
|
||||
///
|
||||
/// Sets the thread name at runtime, making it identifiable in profiling tools like Samply.
|
||||
/// Uses Tokio's blocking thread pool for efficiency.
|
||||
#[track_caller]
|
||||
pub fn spawn_blocking_named<F, R>(&self, name: &'static str, func: F) -> JoinHandle<R>
|
||||
where
|
||||
F: FnOnce() -> R + Send + 'static,
|
||||
R: Send + 'static,
|
||||
{
|
||||
self.inner.handle.spawn_blocking(move || {
|
||||
set_thread_name(name);
|
||||
func()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
|
||||
@@ -23,11 +23,12 @@ use multiproof::{SparseTrieUpdate, *};
|
||||
use parking_lot::RwLock;
|
||||
use prewarm::PrewarmMetrics;
|
||||
use rayon::prelude::*;
|
||||
use reth_engine_primitives::ExecutableTxIterator;
|
||||
use reth_evm::{
|
||||
execute::{ExecutableTxFor, WithTxEnv},
|
||||
ConfigureEvm, EvmEnvFor, OnStateHook, SpecFor, TxEnvFor,
|
||||
ConfigureEvm, EvmEnvFor, ExecutableTxIterator, ExecutableTxTuple, OnStateHook, SpecFor,
|
||||
TxEnvFor,
|
||||
};
|
||||
use reth_execution_types::ExecutionOutcome;
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_provider::{BlockReader, DatabaseProviderROFactory, StateProviderFactory, StateReader};
|
||||
use reth_revm::{db::BundleState, state::EvmState};
|
||||
@@ -92,6 +93,13 @@ pub const SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY: usize = 1_000_000;
|
||||
/// 144MB.
|
||||
pub const SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY: usize = 1_000_000;
|
||||
|
||||
/// Type alias for [`PayloadHandle`] returned by payload processor spawn methods.
|
||||
type IteratorPayloadHandle<Evm, I, N> = PayloadHandle<
|
||||
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxTuple>::Tx>,
|
||||
<I as ExecutableTxTuple>::Error,
|
||||
<N as NodePrimitives>::Receipt,
|
||||
>;
|
||||
|
||||
/// Entrypoint for executing the payload.
|
||||
#[derive(Debug)]
|
||||
pub struct PayloadProcessor<Evm>
|
||||
@@ -200,7 +208,6 @@ where
|
||||
///
|
||||
/// This returns a handle to await the final state root and to interact with the tasks (e.g.
|
||||
/// canceling)
|
||||
#[allow(clippy::type_complexity)]
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor",
|
||||
@@ -215,7 +222,7 @@ where
|
||||
multiproof_provider_factory: F,
|
||||
config: &TreeConfig,
|
||||
bal: Option<Arc<BlockAccessList>>,
|
||||
) -> PayloadHandle<WithTxEnv<TxEnvFor<Evm>, I::Tx>, I::Error>
|
||||
) -> IteratorPayloadHandle<Evm, I, N>
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
F: DatabaseProviderROFactory<Provider: TrieCursorFactory + HashedCursorFactory>
|
||||
@@ -289,7 +296,7 @@ where
|
||||
|
||||
// spawn multi-proof task
|
||||
let parent_span = span.clone();
|
||||
self.executor.spawn_blocking(move || {
|
||||
self.executor.spawn_blocking_named("reth-multiproof", move || {
|
||||
let _enter = parent_span.entered();
|
||||
// Build a state provider for the multiproof task
|
||||
let provider = provider_builder.build().expect("failed to build provider");
|
||||
@@ -320,7 +327,7 @@ where
|
||||
env: ExecutionEnv<Evm>,
|
||||
transactions: I,
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
) -> PayloadHandle<WithTxEnv<TxEnvFor<Evm>, I::Tx>, I::Error>
|
||||
) -> IteratorPayloadHandle<Evm, I, N>
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
{
|
||||
@@ -355,7 +362,7 @@ where
|
||||
let (execute_tx, execute_rx) = mpsc::channel();
|
||||
|
||||
// Spawn a task that `convert`s all transactions in parallel and sends them out-of-order.
|
||||
self.executor.spawn_blocking(move || {
|
||||
self.executor.spawn_blocking_named("reth-tx-conv", move || {
|
||||
transactions.enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
|
||||
let tx = convert(tx);
|
||||
let tx = tx.map(|tx| WithTxEnv { tx_env: tx.to_tx_env(), tx: Arc::new(tx) });
|
||||
@@ -369,7 +376,7 @@ where
|
||||
|
||||
// Spawn a task that processes out-of-order transactions from the task above and sends them
|
||||
// to the execution task in order.
|
||||
self.executor.spawn_blocking(move || {
|
||||
self.executor.spawn_blocking_named("reth-tx-order", move || {
|
||||
let mut next_for_execution = 0;
|
||||
let mut queue = BTreeMap::new();
|
||||
while let Ok((idx, tx)) = ooo_rx.recv() {
|
||||
@@ -400,7 +407,7 @@ where
|
||||
transaction_count_hint: usize,
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
) -> CacheTaskHandle
|
||||
) -> CacheTaskHandle<N::Receipt>
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
{
|
||||
@@ -443,7 +450,7 @@ where
|
||||
// spawn pre-warm task
|
||||
{
|
||||
let to_prewarm_task = to_prewarm_task.clone();
|
||||
self.executor.spawn_blocking(move || {
|
||||
self.executor.spawn_blocking_named("reth-prewarm", move || {
|
||||
prewarm_task.run(transactions, to_prewarm_task);
|
||||
});
|
||||
}
|
||||
@@ -508,7 +515,7 @@ where
|
||||
);
|
||||
|
||||
let span = Span::current();
|
||||
self.executor.spawn_blocking(move || {
|
||||
self.executor.spawn_blocking_named("reth-sparse", move || {
|
||||
let _enter = span.entered();
|
||||
|
||||
let (result, trie) = task.run();
|
||||
@@ -581,12 +588,15 @@ where
|
||||
}
|
||||
|
||||
/// Handle to all the spawned tasks.
|
||||
///
|
||||
/// Generic over `R` (receipt type) to allow sharing `Arc<ExecutionOutcome<R>>` with the
|
||||
/// caching task without cloning the expensive `BundleState`.
|
||||
#[derive(Debug)]
|
||||
pub struct PayloadHandle<Tx, Err> {
|
||||
pub struct PayloadHandle<Tx, Err, R> {
|
||||
/// Channel for evm state updates
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
// must include the receiver of the state root wired to the sparse trie
|
||||
prewarm_handle: CacheTaskHandle,
|
||||
prewarm_handle: CacheTaskHandle<R>,
|
||||
/// Stream of block transactions
|
||||
transactions: mpsc::Receiver<Result<Tx, Err>>,
|
||||
/// Receiver for the state root
|
||||
@@ -595,7 +605,7 @@ pub struct PayloadHandle<Tx, Err> {
|
||||
_span: Span,
|
||||
}
|
||||
|
||||
impl<Tx, Err> PayloadHandle<Tx, Err> {
|
||||
impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
|
||||
/// Awaits the state root
|
||||
///
|
||||
/// # Panics
|
||||
@@ -648,9 +658,14 @@ impl<Tx, Err> PayloadHandle<Tx, Err> {
|
||||
|
||||
/// Terminates the entire caching task.
|
||||
///
|
||||
/// If the [`BundleState`] is provided it will update the shared cache.
|
||||
pub(super) fn terminate_caching(&mut self, block_output: Option<&BundleState>) {
|
||||
self.prewarm_handle.terminate_caching(block_output)
|
||||
/// If the [`ExecutionOutcome`] is provided it will update the shared cache using its
|
||||
/// bundle state. Using `Arc<ExecutionOutcome>` allows sharing with the main execution
|
||||
/// path without cloning the expensive `BundleState`.
|
||||
pub(super) fn terminate_caching(
|
||||
&mut self,
|
||||
execution_outcome: Option<Arc<ExecutionOutcome<R>>>,
|
||||
) {
|
||||
self.prewarm_handle.terminate_caching(execution_outcome)
|
||||
}
|
||||
|
||||
/// Returns iterator yielding transactions from the stream.
|
||||
@@ -662,17 +677,20 @@ impl<Tx, Err> PayloadHandle<Tx, Err> {
|
||||
}
|
||||
|
||||
/// Access to the spawned [`PrewarmCacheTask`].
|
||||
///
|
||||
/// Generic over `R` (receipt type) to allow sharing `Arc<ExecutionOutcome<R>>` with the
|
||||
/// prewarm task without cloning the expensive `BundleState`.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct CacheTaskHandle {
|
||||
pub(crate) struct CacheTaskHandle<R> {
|
||||
/// The shared cache the task operates with.
|
||||
cache: Option<StateExecutionCache>,
|
||||
/// Metrics for the caches
|
||||
cache_metrics: Option<CachedStateMetrics>,
|
||||
/// Channel to the spawned prewarm task if any
|
||||
to_prewarm_task: Option<std::sync::mpsc::Sender<PrewarmTaskEvent>>,
|
||||
to_prewarm_task: Option<std::sync::mpsc::Sender<PrewarmTaskEvent<R>>>,
|
||||
}
|
||||
|
||||
impl CacheTaskHandle {
|
||||
impl<R: Send + Sync + 'static> CacheTaskHandle<R> {
|
||||
/// Terminates the pre-warming transaction processing.
|
||||
///
|
||||
/// Note: This does not terminate the task yet.
|
||||
@@ -684,20 +702,25 @@ impl CacheTaskHandle {
|
||||
|
||||
/// Terminates the entire pre-warming task.
|
||||
///
|
||||
/// If the [`BundleState`] is provided it will update the shared cache.
|
||||
pub(super) fn terminate_caching(&mut self, block_output: Option<&BundleState>) {
|
||||
/// If the [`ExecutionOutcome`] is provided it will update the shared cache using its
|
||||
/// bundle state. Using `Arc<ExecutionOutcome>` avoids cloning the expensive `BundleState`.
|
||||
pub(super) fn terminate_caching(
|
||||
&mut self,
|
||||
execution_outcome: Option<Arc<ExecutionOutcome<R>>>,
|
||||
) {
|
||||
if let Some(tx) = self.to_prewarm_task.take() {
|
||||
// Only clone when we have an active task and a state to send
|
||||
let event = PrewarmTaskEvent::Terminate { block_output: block_output.cloned() };
|
||||
let event = PrewarmTaskEvent::Terminate { execution_outcome };
|
||||
let _ = tx.send(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CacheTaskHandle {
|
||||
impl<R> Drop for CacheTaskHandle<R> {
|
||||
fn drop(&mut self) {
|
||||
// Ensure we always terminate on drop
|
||||
self.terminate_caching(None);
|
||||
// Ensure we always terminate on drop - send None without needing Send + Sync bounds
|
||||
if let Some(tx) = self.to_prewarm_task.take() {
|
||||
let _ = tx.send(PrewarmTaskEvent::Terminate { execution_outcome: None });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -750,6 +773,8 @@ impl ExecutionCache {
|
||||
|
||||
cache
|
||||
.as_ref()
|
||||
// Check `is_available()` to ensure no other tasks (e.g., prewarming) currently hold
|
||||
// a reference to this cache. We can only reuse it when we have exclusive access.
|
||||
.filter(|c| c.executed_block_hash() == parent_hash && c.is_available())
|
||||
.cloned()
|
||||
}
|
||||
|
||||
@@ -170,11 +170,6 @@ impl ProofSequencer {
|
||||
while let Some(pending) = self.pending_proofs.remove(¤t_sequence) {
|
||||
consecutive_proofs.push(pending);
|
||||
current_sequence += 1;
|
||||
|
||||
// if we don't have the next number, stop collecting
|
||||
if !self.pending_proofs.contains_key(¤t_sequence) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
self.next_to_deliver += consecutive_proofs.len() as u64;
|
||||
@@ -1458,6 +1453,9 @@ impl MultiProofTask {
|
||||
/// Context for multiproof message batching loop.
|
||||
///
|
||||
/// Contains processing state that persists across loop iterations.
|
||||
///
|
||||
/// Used by `process_multiproof_message` to batch consecutive same-type messages received via
|
||||
/// `try_recv` for efficient processing.
|
||||
struct MultiproofBatchCtx {
|
||||
/// Buffers a non-matching message type encountered during batching.
|
||||
/// Processed first in next iteration to preserve ordering while allowing same-type
|
||||
|
||||
@@ -27,10 +27,11 @@ use alloy_primitives::{keccak256, map::B256Set, B256};
|
||||
use crossbeam_channel::Sender as CrossbeamSender;
|
||||
use metrics::{Counter, Gauge, Histogram};
|
||||
use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, SpecFor};
|
||||
use reth_execution_types::ExecutionOutcome;
|
||||
use reth_metrics::Metrics;
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_provider::{BlockReader, StateProviderBox, StateProviderFactory, StateReader};
|
||||
use reth_revm::{database::StateProviderDatabase, db::BundleState, state::EvmState};
|
||||
use reth_provider::{BlockReader, StateProviderFactory, StateReader};
|
||||
use reth_revm::{database::StateProviderDatabase, state::EvmState};
|
||||
use reth_trie::MultiProofTargets;
|
||||
use std::{
|
||||
sync::{
|
||||
@@ -86,7 +87,7 @@ where
|
||||
/// Sender to emit evm state outcome messages, if any.
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
/// Receiver for events produced by tx execution
|
||||
actions_rx: Receiver<PrewarmTaskEvent>,
|
||||
actions_rx: Receiver<PrewarmTaskEvent<N::Receipt>>,
|
||||
/// Parent span for tracing
|
||||
parent_span: Span,
|
||||
}
|
||||
@@ -105,7 +106,7 @@ where
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
transaction_count_hint: usize,
|
||||
max_concurrency: usize,
|
||||
) -> (Self, Sender<PrewarmTaskEvent>) {
|
||||
) -> (Self, Sender<PrewarmTaskEvent<N::Receipt>>) {
|
||||
let (actions_tx, actions_rx) = channel();
|
||||
|
||||
trace!(
|
||||
@@ -135,8 +136,11 @@ where
|
||||
/// For Optimism chains, special handling is applied to the first transaction if it's a
|
||||
/// deposit transaction (type 0x7E/126) which sets critical metadata that affects all
|
||||
/// subsequent transactions in the block.
|
||||
fn spawn_all<Tx>(&self, pending: mpsc::Receiver<Tx>, actions_tx: Sender<PrewarmTaskEvent>)
|
||||
where
|
||||
fn spawn_all<Tx>(
|
||||
&self,
|
||||
pending: mpsc::Receiver<Tx>,
|
||||
actions_tx: Sender<PrewarmTaskEvent<N::Receipt>>,
|
||||
) where
|
||||
Tx: ExecutableTxFor<Evm> + Clone + Send + 'static,
|
||||
{
|
||||
let executor = self.executor.clone();
|
||||
@@ -145,7 +149,7 @@ where
|
||||
let transaction_count_hint = self.transaction_count_hint;
|
||||
let span = Span::current();
|
||||
|
||||
self.executor.spawn_blocking(move || {
|
||||
self.executor.spawn_blocking_named("reth-prewarm", move || {
|
||||
let _enter = debug_span!(target: "engine::tree::payload_processor::prewarm", parent: span, "spawn_all").entered();
|
||||
|
||||
let (done_tx, done_rx) = mpsc::channel();
|
||||
@@ -248,7 +252,7 @@ where
|
||||
///
|
||||
/// This method is called from `run()` only after all execution tasks are complete.
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
|
||||
fn save_cache(self, state: BundleState) {
|
||||
fn save_cache(self, execution_outcome: Arc<ExecutionOutcome<N::Receipt>>) {
|
||||
let start = Instant::now();
|
||||
|
||||
let Self { execution_cache, ctx: PrewarmContext { env, metrics, saved_cache, .. }, .. } =
|
||||
@@ -265,7 +269,8 @@ where
|
||||
let new_cache = SavedCache::new(hash, caches, cache_metrics);
|
||||
|
||||
// Insert state into cache while holding the lock
|
||||
if new_cache.cache().insert_state(&state).is_err() {
|
||||
// Access the BundleState through the shared ExecutionOutcome
|
||||
if new_cache.cache().insert_state(execution_outcome.state()).is_err() {
|
||||
// Clear the cache on error to prevent having a polluted cache
|
||||
*cached = None;
|
||||
debug!(target: "engine::caching", "cleared execution cache on update error");
|
||||
@@ -300,12 +305,12 @@ where
|
||||
pub(super) fn run(
|
||||
self,
|
||||
pending: mpsc::Receiver<impl ExecutableTxFor<Evm> + Clone + Send + 'static>,
|
||||
actions_tx: Sender<PrewarmTaskEvent>,
|
||||
actions_tx: Sender<PrewarmTaskEvent<N::Receipt>>,
|
||||
) {
|
||||
// spawn execution tasks.
|
||||
self.spawn_all(pending, actions_tx);
|
||||
|
||||
let mut final_block_output = None;
|
||||
let mut final_execution_outcome = None;
|
||||
let mut finished_execution = false;
|
||||
while let Ok(event) = self.actions_rx.recv() {
|
||||
match event {
|
||||
@@ -318,9 +323,9 @@ where
|
||||
// completed executing a set of transactions
|
||||
self.send_multi_proof_targets(proof_targets);
|
||||
}
|
||||
PrewarmTaskEvent::Terminate { block_output } => {
|
||||
PrewarmTaskEvent::Terminate { execution_outcome } => {
|
||||
trace!(target: "engine::tree::payload_processor::prewarm", "Received termination signal");
|
||||
final_block_output = Some(block_output);
|
||||
final_execution_outcome = Some(execution_outcome);
|
||||
|
||||
if finished_execution {
|
||||
// all tasks are done, we can exit, which will save caches and exit
|
||||
@@ -334,7 +339,7 @@ where
|
||||
|
||||
finished_execution = true;
|
||||
|
||||
if final_block_output.is_some() {
|
||||
if final_execution_outcome.is_some() {
|
||||
// all tasks are done, we can exit, which will save caches and exit
|
||||
break
|
||||
}
|
||||
@@ -344,9 +349,9 @@ where
|
||||
|
||||
debug!(target: "engine::tree::payload_processor::prewarm", "Completed prewarm execution");
|
||||
|
||||
// save caches and finish
|
||||
if let Some(Some(state)) = final_block_output {
|
||||
self.save_cache(state);
|
||||
// save caches and finish using the shared ExecutionOutcome
|
||||
if let Some(Some(execution_outcome)) = final_execution_outcome {
|
||||
self.save_cache(execution_outcome);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -391,7 +396,7 @@ where
|
||||
mut precompile_cache_map,
|
||||
} = self;
|
||||
|
||||
let state_provider = match provider.build() {
|
||||
let mut state_provider = match provider.build() {
|
||||
Ok(provider) => provider,
|
||||
Err(err) => {
|
||||
trace!(
|
||||
@@ -404,13 +409,15 @@ where
|
||||
};
|
||||
|
||||
// Use the caches to create a new provider with caching
|
||||
let state_provider: StateProviderBox = if let Some(saved_cache) = saved_cache {
|
||||
if let Some(saved_cache) = saved_cache {
|
||||
let caches = saved_cache.cache().clone();
|
||||
let cache_metrics = saved_cache.metrics().clone();
|
||||
Box::new(CachedStateProvider::new_with_caches(state_provider, caches, cache_metrics))
|
||||
} else {
|
||||
state_provider
|
||||
};
|
||||
state_provider = Box::new(
|
||||
CachedStateProvider::new(state_provider, caches, cache_metrics)
|
||||
// ensure we pre-warm the cache
|
||||
.prewarm(),
|
||||
);
|
||||
}
|
||||
|
||||
let state_provider = StateProviderDatabase::new(state_provider);
|
||||
|
||||
@@ -452,7 +459,7 @@ where
|
||||
fn transact_batch<Tx>(
|
||||
self,
|
||||
txs: mpsc::Receiver<IndexedTransaction<Tx>>,
|
||||
sender: Sender<PrewarmTaskEvent>,
|
||||
sender: Sender<PrewarmTaskEvent<N::Receipt>>,
|
||||
done_tx: Sender<()>,
|
||||
) where
|
||||
Tx: ExecutableTxFor<Evm>,
|
||||
@@ -533,7 +540,7 @@ where
|
||||
&self,
|
||||
idx: usize,
|
||||
executor: &WorkloadExecutor,
|
||||
actions_tx: Sender<PrewarmTaskEvent>,
|
||||
actions_tx: Sender<PrewarmTaskEvent<N::Receipt>>,
|
||||
done_tx: Sender<()>,
|
||||
) -> mpsc::Sender<IndexedTransaction<Tx>>
|
||||
where
|
||||
@@ -544,7 +551,7 @@ where
|
||||
let span =
|
||||
debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm worker", idx);
|
||||
|
||||
executor.spawn_blocking(move || {
|
||||
executor.spawn_blocking_named("reth-prewarm-w", move || {
|
||||
let _enter = span.entered();
|
||||
ctx.transact_batch(rx, actions_tx, done_tx);
|
||||
});
|
||||
@@ -589,14 +596,18 @@ fn multiproof_targets_from_state(state: EvmState) -> (MultiProofTargets, usize)
|
||||
}
|
||||
|
||||
/// The events the pre-warm task can handle.
|
||||
pub(super) enum PrewarmTaskEvent {
|
||||
///
|
||||
/// Generic over `R` (receipt type) to allow sharing `Arc<ExecutionOutcome<R>>` with the main
|
||||
/// execution path without cloning the expensive `BundleState`.
|
||||
pub(super) enum PrewarmTaskEvent<R> {
|
||||
/// Forcefully terminate all remaining transaction execution.
|
||||
TerminateTransactionExecution,
|
||||
/// Forcefully terminate the task on demand and update the shared cache with the given output
|
||||
/// before exiting.
|
||||
Terminate {
|
||||
/// The final block state output.
|
||||
block_output: Option<BundleState>,
|
||||
/// The final execution outcome. Using `Arc` allows sharing with the main execution
|
||||
/// path without cloning the expensive `BundleState`.
|
||||
execution_outcome: Option<Arc<ExecutionOutcome<R>>>,
|
||||
},
|
||||
/// The outcome of a pre-warm task
|
||||
Outcome {
|
||||
|
||||
@@ -374,7 +374,8 @@ where
|
||||
let mut state_provider = ensure_ok!(provider_builder.build());
|
||||
drop(_enter);
|
||||
|
||||
// fetch parent block
|
||||
// Fetch parent block. This goes to memory most of the time unless the parent block is
|
||||
// beyond the in-memory buffer.
|
||||
let Some(parent_block) = ensure_ok!(self.sealed_header_by_hash(parent_hash, ctx.state()))
|
||||
else {
|
||||
return Err(InsertBlockError::new(
|
||||
@@ -399,7 +400,7 @@ where
|
||||
"Decided which state root algorithm to run"
|
||||
);
|
||||
|
||||
// use prewarming background task
|
||||
// Get an iterator over the transactions in the payload
|
||||
let txs = self.tx_iterator_for(&input)?;
|
||||
|
||||
// Extract the BAL, if valid and available
|
||||
@@ -424,21 +425,17 @@ where
|
||||
// Use cached state provider before executing, used in execution after prewarming threads
|
||||
// complete
|
||||
if let Some((caches, cache_metrics)) = handle.caches().zip(handle.cache_metrics()) {
|
||||
state_provider = Box::new(CachedStateProvider::new_with_caches(
|
||||
state_provider,
|
||||
caches,
|
||||
cache_metrics,
|
||||
));
|
||||
state_provider =
|
||||
Box::new(CachedStateProvider::new(state_provider, caches, cache_metrics));
|
||||
};
|
||||
|
||||
if self.config.state_provider_metrics() {
|
||||
state_provider = Box::new(InstrumentedStateProvider::new(state_provider, "engine"));
|
||||
}
|
||||
|
||||
// Execute the block and handle any execution errors
|
||||
let (output, senders) = match if self.config.state_provider_metrics() {
|
||||
let state_provider =
|
||||
InstrumentedStateProvider::from_state_provider(&state_provider, "engine");
|
||||
self.execute_block(&state_provider, env, &input, &mut handle)
|
||||
} else {
|
||||
self.execute_block(&state_provider, env, &input, &mut handle)
|
||||
} {
|
||||
let (output, senders) = match self.execute_block(&state_provider, env, &input, &mut handle)
|
||||
{
|
||||
Ok(output) => output,
|
||||
Err(err) => return self.handle_execution_error(input, err, &parent_block),
|
||||
};
|
||||
@@ -552,17 +549,14 @@ where
|
||||
.into())
|
||||
}
|
||||
|
||||
// terminate prewarming task with good state output
|
||||
handle.terminate_caching(Some(&output.state));
|
||||
// Create ExecutionOutcome and wrap in Arc for sharing with both the caching task
|
||||
// and the deferred trie task. This avoids cloning the expensive BundleState.
|
||||
let execution_outcome = Arc::new(ExecutionOutcome::from((output, block_num_hash.number)));
|
||||
|
||||
Ok(self.spawn_deferred_trie_task(
|
||||
block,
|
||||
output,
|
||||
block_num_hash.number,
|
||||
&ctx,
|
||||
hashed_state,
|
||||
trie_output,
|
||||
))
|
||||
// Terminate prewarming task with the shared execution outcome
|
||||
handle.terminate_caching(Some(Arc::clone(&execution_outcome)));
|
||||
|
||||
Ok(self.spawn_deferred_trie_task(block, execution_outcome, &ctx, hashed_state, trie_output))
|
||||
}
|
||||
|
||||
/// Return sealed block header from database or in-memory state by hash.
|
||||
@@ -605,7 +599,7 @@ where
|
||||
state_provider: S,
|
||||
env: ExecutionEnv<Evm>,
|
||||
input: &BlockOrPayload<T>,
|
||||
handle: &mut PayloadHandle<impl ExecutableTxFor<Evm>, Err>,
|
||||
handle: &mut PayloadHandle<impl ExecutableTxFor<Evm>, Err, N::Receipt>,
|
||||
) -> Result<(BlockExecutionOutput<N::Receipt>, Vec<Address>), InsertBlockErrorKind>
|
||||
where
|
||||
S: StateProvider,
|
||||
@@ -617,7 +611,7 @@ where
|
||||
debug!(target: "engine::tree::payload_validator", "Executing block");
|
||||
|
||||
let mut db = State::builder()
|
||||
.with_database(StateProviderDatabase::new(&state_provider))
|
||||
.with_database(StateProviderDatabase::new(state_provider))
|
||||
.with_bundle_update()
|
||||
.without_state_clear()
|
||||
.build();
|
||||
@@ -793,6 +787,7 @@ where
|
||||
PayloadHandle<
|
||||
impl ExecutableTxFor<Evm> + use<N, P, Evm, V, T>,
|
||||
impl core::error::Error + Send + Sync + 'static + use<N, P, Evm, V, T>,
|
||||
N::Receipt,
|
||||
>,
|
||||
InsertBlockErrorKind,
|
||||
> {
|
||||
@@ -1027,8 +1022,7 @@ where
|
||||
fn spawn_deferred_trie_task(
|
||||
&self,
|
||||
block: RecoveredBlock<N::Block>,
|
||||
output: BlockExecutionOutput<N::Receipt>,
|
||||
block_number: u64,
|
||||
execution_outcome: Arc<ExecutionOutcome<N::Receipt>>,
|
||||
ctx: &TreeCtx<'_, N>,
|
||||
hashed_state: HashedPostState,
|
||||
trie_output: TrieUpdates,
|
||||
@@ -1078,7 +1072,7 @@ where
|
||||
|
||||
ExecutedBlock::with_deferred_trie_data(
|
||||
Arc::new(block),
|
||||
Arc::new(ExecutionOutcome::from((output, block_number))),
|
||||
execution_outcome,
|
||||
deferred_trie_data,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -61,6 +61,7 @@ reth-node-core.workspace = true
|
||||
reth-e2e-test-utils.workspace = true
|
||||
reth-tasks.workspace = true
|
||||
reth-testing-utils.workspace = true
|
||||
reth-stages-types.workspace = true
|
||||
tempfile.workspace = true
|
||||
jsonrpsee-core.workspace = true
|
||||
|
||||
@@ -109,4 +110,5 @@ test-utils = [
|
||||
"reth-evm/test-utils",
|
||||
"reth-primitives-traits/test-utils",
|
||||
"reth-evm-ethereum/test-utils",
|
||||
"reth-stages-types/test-utils",
|
||||
]
|
||||
|
||||
100
crates/ethereum/node/tests/e2e/custom_genesis.rs
Normal file
100
crates/ethereum/node/tests/e2e/custom_genesis.rs
Normal file
@@ -0,0 +1,100 @@
|
||||
use crate::utils::eth_payload_attributes;
|
||||
use alloy_genesis::Genesis;
|
||||
use alloy_primitives::B256;
|
||||
use reth_chainspec::{ChainSpecBuilder, MAINNET};
|
||||
use reth_e2e_test_utils::{setup, transaction::TransactionTestContext};
|
||||
use reth_node_ethereum::EthereumNode;
|
||||
use reth_provider::{HeaderProvider, StageCheckpointReader};
|
||||
use reth_stages_types::StageId;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Tests that a node can initialize and advance with a custom genesis block number.
|
||||
#[tokio::test]
|
||||
async fn can_run_eth_node_with_custom_genesis_number() -> eyre::Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
// Create genesis with custom block number (e.g., 1000)
|
||||
let mut genesis: Genesis =
|
||||
serde_json::from_str(include_str!("../assets/genesis.json")).unwrap();
|
||||
genesis.number = Some(1000);
|
||||
genesis.parent_hash = Some(B256::random());
|
||||
|
||||
let chain_spec = Arc::new(
|
||||
ChainSpecBuilder::default()
|
||||
.chain(MAINNET.chain)
|
||||
.genesis(genesis)
|
||||
.cancun_activated()
|
||||
.build(),
|
||||
);
|
||||
|
||||
let (mut nodes, _tasks, wallet) =
|
||||
setup::<EthereumNode>(1, chain_spec, false, eth_payload_attributes).await?;
|
||||
|
||||
let mut node = nodes.pop().unwrap();
|
||||
|
||||
// Verify stage checkpoints are initialized to genesis block number (1000)
|
||||
for stage in StageId::ALL {
|
||||
let checkpoint = node.inner.provider.get_stage_checkpoint(stage)?;
|
||||
assert!(checkpoint.is_some(), "Stage {:?} checkpoint should exist", stage);
|
||||
assert_eq!(
|
||||
checkpoint.unwrap().block_number,
|
||||
1000,
|
||||
"Stage {:?} checkpoint should be at genesis block 1000",
|
||||
stage
|
||||
);
|
||||
}
|
||||
|
||||
// Advance the chain (block 1001)
|
||||
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
|
||||
let tx_hash = node.rpc.inject_tx(raw_tx).await?;
|
||||
let payload = node.advance_block().await?;
|
||||
|
||||
let block_hash = payload.block().hash();
|
||||
let block_number = payload.block().number;
|
||||
|
||||
// Verify we're at block 1001 (genesis + 1)
|
||||
assert_eq!(block_number, 1001, "Block number should be 1001 after advancing from genesis 1000");
|
||||
|
||||
// Assert the block has been committed
|
||||
node.assert_new_block(tx_hash, block_hash, block_number).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Tests that block queries respect custom genesis boundaries.
|
||||
#[tokio::test]
|
||||
async fn custom_genesis_block_query_boundaries() -> eyre::Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let genesis_number = 5000u64;
|
||||
|
||||
let mut genesis: Genesis =
|
||||
serde_json::from_str(include_str!("../assets/genesis.json")).unwrap();
|
||||
genesis.number = Some(genesis_number);
|
||||
genesis.parent_hash = Some(B256::random());
|
||||
|
||||
let chain_spec = Arc::new(
|
||||
ChainSpecBuilder::default()
|
||||
.chain(MAINNET.chain)
|
||||
.genesis(genesis)
|
||||
.cancun_activated()
|
||||
.build(),
|
||||
);
|
||||
|
||||
let (mut nodes, _tasks, _wallet) =
|
||||
setup::<EthereumNode>(1, chain_spec, false, eth_payload_attributes).await?;
|
||||
|
||||
let node = nodes.pop().unwrap();
|
||||
|
||||
// Query genesis block should succeed
|
||||
let genesis_header = node.inner.provider.header_by_number(genesis_number)?;
|
||||
assert!(genesis_header.is_some(), "Genesis block at {} should exist", genesis_number);
|
||||
|
||||
// Query blocks before genesis should return None
|
||||
for block_num in [0, 1, genesis_number - 1] {
|
||||
let header = node.inner.provider.header_by_number(block_num)?;
|
||||
assert!(header.is_none(), "Block {} before genesis should not exist", block_num);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
#![allow(missing_docs)]
|
||||
|
||||
mod blobs;
|
||||
mod custom_genesis;
|
||||
mod dev;
|
||||
mod eth;
|
||||
mod p2p;
|
||||
|
||||
@@ -483,6 +483,7 @@ where
|
||||
StaticFileProviderBuilder::read_write(self.data_dir().static_files())?
|
||||
.with_metrics()
|
||||
.with_blocks_per_file_for_segments(static_files_config.as_blocks_per_file_map())
|
||||
.with_genesis_block_number(self.chain_spec().genesis().number.unwrap_or_default())
|
||||
.build()?;
|
||||
|
||||
// Initialize RocksDB provider with metrics, statistics, and default tables
|
||||
|
||||
@@ -1381,6 +1381,7 @@ where
|
||||
version: version_metadata().cargo_pkg_version.to_string(),
|
||||
commit: version_metadata().vergen_git_sha.to_string(),
|
||||
};
|
||||
|
||||
Ok(EngineApi::new(
|
||||
ctx.node.provider().clone(),
|
||||
ctx.config.chain.clone(),
|
||||
@@ -1392,6 +1393,7 @@ where
|
||||
EngineCapabilities::default(),
|
||||
engine_validator,
|
||||
ctx.config.engine.accept_execution_requests_hash,
|
||||
ctx.node.network().clone(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,7 +208,7 @@ where
|
||||
active: true,
|
||||
syncing: self.network.is_syncing(),
|
||||
peers: self.network.num_connected_peers() as u64,
|
||||
gas_price: 0, // TODO
|
||||
gas_price: self.pool.block_info().pending_basefee,
|
||||
uptime: 100,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -27,7 +27,7 @@ tracing.workspace = true
|
||||
workspace = true
|
||||
|
||||
[features]
|
||||
default = ["jemalloc", "otlp", "reth-optimism-evm/portable", "js-tracer", "keccak-cache-global"]
|
||||
default = ["jemalloc", "otlp", "reth-optimism-evm/portable", "js-tracer", "keccak-cache-global", "asm-keccak"]
|
||||
|
||||
otlp = ["reth-optimism-cli/otlp"]
|
||||
|
||||
|
||||
@@ -80,6 +80,7 @@ reth-payload-util.workspace = true
|
||||
reth-revm = { workspace = true, features = ["std"] }
|
||||
reth-rpc.workspace = true
|
||||
reth-rpc-eth-types.workspace = true
|
||||
reth-stages-types.workspace = true
|
||||
|
||||
alloy-network.workspace = true
|
||||
futures.workspace = true
|
||||
@@ -122,6 +123,7 @@ test-utils = [
|
||||
"reth-optimism-primitives/arbitrary",
|
||||
"reth-primitives-traits/test-utils",
|
||||
"reth-trie-common/test-utils",
|
||||
"reth-stages-types/test-utils",
|
||||
]
|
||||
reth-codec = ["reth-optimism-primitives/reth-codec"]
|
||||
|
||||
|
||||
@@ -146,6 +146,7 @@ where
|
||||
EngineCapabilities::new(OP_ENGINE_CAPABILITIES.iter().copied()),
|
||||
engine_validator,
|
||||
ctx.config.engine.accept_execution_requests_hash,
|
||||
ctx.node.network().clone(),
|
||||
);
|
||||
|
||||
Ok(OpEngineApi::new(inner))
|
||||
|
||||
123
crates/optimism/node/tests/it/custom_genesis.rs
Normal file
123
crates/optimism/node/tests/it/custom_genesis.rs
Normal file
@@ -0,0 +1,123 @@
|
||||
//! Tests for custom genesis block number support.
|
||||
|
||||
use alloy_consensus::BlockHeader;
|
||||
use alloy_genesis::Genesis;
|
||||
use alloy_primitives::B256;
|
||||
use reth_chainspec::EthChainSpec;
|
||||
use reth_db::test_utils::create_test_rw_db_with_path;
|
||||
use reth_e2e_test_utils::{
|
||||
node::NodeTestContext, transaction::TransactionTestContext, wallet::Wallet,
|
||||
};
|
||||
use reth_node_builder::{EngineNodeLauncher, Node, NodeBuilder, NodeConfig};
|
||||
use reth_node_core::args::DatadirArgs;
|
||||
use reth_optimism_chainspec::OpChainSpecBuilder;
|
||||
use reth_optimism_node::{utils::optimism_payload_attributes, OpNode};
|
||||
use reth_provider::{providers::BlockchainProvider, HeaderProvider, StageCheckpointReader};
|
||||
use reth_stages_types::StageId;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
/// Tests that an OP node can initialize with a custom genesis block number.
|
||||
#[tokio::test]
|
||||
async fn test_op_node_custom_genesis_number() {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let genesis_number = 1000;
|
||||
|
||||
// Create genesis with custom block number (1000)
|
||||
let mut genesis: Genesis =
|
||||
serde_json::from_str(include_str!("../assets/genesis.json")).unwrap();
|
||||
genesis.number = Some(genesis_number);
|
||||
genesis.parent_hash = Some(B256::random());
|
||||
|
||||
let chain_spec =
|
||||
Arc::new(OpChainSpecBuilder::base_mainnet().genesis(genesis).ecotone_activated().build());
|
||||
|
||||
let wallet = Arc::new(Mutex::new(Wallet::default().with_chain_id(chain_spec.chain().into())));
|
||||
|
||||
// Configure and launch the node
|
||||
let config = NodeConfig::new(chain_spec.clone()).with_datadir_args(DatadirArgs {
|
||||
datadir: reth_db::test_utils::tempdir_path().into(),
|
||||
..Default::default()
|
||||
});
|
||||
let db = create_test_rw_db_with_path(
|
||||
config
|
||||
.datadir
|
||||
.datadir
|
||||
.unwrap_or_chain_default(config.chain.chain(), config.datadir.clone())
|
||||
.db(),
|
||||
);
|
||||
let tasks = reth_tasks::TaskManager::current();
|
||||
let node_handle = NodeBuilder::new(config.clone())
|
||||
.with_database(db)
|
||||
.with_types_and_provider::<OpNode, BlockchainProvider<_>>()
|
||||
.with_components(OpNode::default().components())
|
||||
.with_add_ons(OpNode::new(Default::default()).add_ons())
|
||||
.launch_with_fn(|builder| {
|
||||
let launcher = EngineNodeLauncher::new(
|
||||
tasks.executor(),
|
||||
builder.config.datadir(),
|
||||
Default::default(),
|
||||
);
|
||||
builder.launch_with(launcher)
|
||||
})
|
||||
.await
|
||||
.expect("Failed to launch node");
|
||||
|
||||
let mut node =
|
||||
NodeTestContext::new(node_handle.node, optimism_payload_attributes).await.unwrap();
|
||||
|
||||
// Verify stage checkpoints are initialized to genesis block number (1000)
|
||||
for stage in StageId::ALL {
|
||||
let checkpoint = node.inner.provider.get_stage_checkpoint(stage).unwrap();
|
||||
assert!(checkpoint.is_some(), "Stage {:?} checkpoint should exist", stage);
|
||||
assert_eq!(
|
||||
checkpoint.unwrap().block_number,
|
||||
1000,
|
||||
"Stage {:?} checkpoint should be at genesis block 1000",
|
||||
stage
|
||||
);
|
||||
}
|
||||
|
||||
// Query genesis block should succeed
|
||||
let genesis_header = node.inner.provider.header_by_number(genesis_number).unwrap();
|
||||
assert!(genesis_header.is_some(), "Genesis block at {} should exist", genesis_number);
|
||||
|
||||
// Query blocks before genesis should return None
|
||||
for block_num in [0, 1, genesis_number - 1] {
|
||||
let header = node.inner.provider.header_by_number(block_num).unwrap();
|
||||
assert!(header.is_none(), "Block {} before genesis should not exist", block_num);
|
||||
}
|
||||
|
||||
// Advance the chain with a single block
|
||||
let _ = wallet; // wallet available for future use
|
||||
let block_payloads = node
|
||||
.advance(1, |_| {
|
||||
Box::pin({
|
||||
let value = wallet.clone();
|
||||
async move {
|
||||
let mut wallet = value.lock().await;
|
||||
let tx_fut = TransactionTestContext::optimism_l1_block_info_tx(
|
||||
wallet.chain_id,
|
||||
wallet.inner.clone(),
|
||||
wallet.inner_nonce,
|
||||
);
|
||||
wallet.inner_nonce += 1;
|
||||
|
||||
tx_fut.await
|
||||
}
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(block_payloads.len(), 1);
|
||||
let block = block_payloads.first().unwrap().block();
|
||||
|
||||
// Verify the new block is at 1001 (genesis 1000 + 1)
|
||||
assert_eq!(
|
||||
block.number(),
|
||||
1001,
|
||||
"Block number should be 1001 after advancing from genesis 100"
|
||||
);
|
||||
}
|
||||
@@ -6,4 +6,6 @@ mod priority;
|
||||
|
||||
mod rpc;
|
||||
|
||||
mod custom_genesis;
|
||||
|
||||
const fn main() {}
|
||||
|
||||
@@ -7,7 +7,7 @@ use alloy_rpc_types_eth::{Log, TransactionReceipt};
|
||||
use op_alloy_consensus::{OpReceipt, OpTransaction};
|
||||
use op_alloy_rpc_types::{L1BlockInfo, OpTransactionReceipt, OpTransactionReceiptFields};
|
||||
use op_revm::estimate_tx_compressed_size;
|
||||
use reth_chainspec::ChainSpecProvider;
|
||||
use reth_chainspec::{ChainSpecProvider, EthChainSpec};
|
||||
use reth_node_api::NodePrimitives;
|
||||
use reth_optimism_evm::RethL1BlockInfo;
|
||||
use reth_optimism_forks::OpHardforks;
|
||||
@@ -74,9 +74,11 @@ where
|
||||
let mut l1_block_info = match reth_optimism_evm::extract_l1_info(block.body()) {
|
||||
Ok(l1_block_info) => l1_block_info,
|
||||
Err(err) => {
|
||||
let genesis_number =
|
||||
self.provider.chain_spec().genesis().number.unwrap_or_default();
|
||||
// If it is the genesis block (i.e. block number is 0), there is no L1 info, so
|
||||
// we return an empty l1_block_info.
|
||||
if block.header().number() == 0 {
|
||||
if block.header().number() == genesis_number {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
return Err(err.into());
|
||||
|
||||
@@ -19,8 +19,11 @@ pub trait PayloadTransactions {
|
||||
ctx: (),
|
||||
) -> Option<Self::Transaction>;
|
||||
|
||||
/// Exclude descendants of the transaction with given sender and nonce from the iterator,
|
||||
/// because this transaction won't be included in the block.
|
||||
/// Marks the transaction identified by `sender` and `nonce` as invalid for this iterator.
|
||||
///
|
||||
/// Implementations must ensure that subsequent transactions returned from this iterator do not
|
||||
/// depend on this transaction. For example, they may choose to stop yielding any further
|
||||
/// transactions from this sender in the current iteration.
|
||||
fn mark_invalid(&mut self, sender: Address, nonce: u64);
|
||||
}
|
||||
|
||||
@@ -46,6 +49,9 @@ impl<T> PayloadTransactions for NoopPayloadTransactions<T> {
|
||||
|
||||
/// Wrapper struct that allows to convert `BestTransactions` (used in tx pool) to
|
||||
/// `PayloadTransactions` (used in block composition).
|
||||
///
|
||||
/// Note: `mark_invalid` for this type filters out all further transactions from the given sender
|
||||
/// in the current iteration, mirroring the semantics of `BestTransactions::mark_invalid`.
|
||||
#[derive(Debug)]
|
||||
pub struct BestPayloadTransactions<T, I>
|
||||
where
|
||||
|
||||
@@ -42,14 +42,13 @@ fn validate_blob_tx(
|
||||
blob_sidecar.blobs.extend(blob_sidecar_ext.blobs);
|
||||
blob_sidecar.proofs.extend(blob_sidecar_ext.proofs);
|
||||
blob_sidecar.commitments.extend(blob_sidecar_ext.commitments);
|
||||
|
||||
if blob_sidecar.blobs.len() > num_blobs as usize {
|
||||
blob_sidecar.blobs.truncate(num_blobs as usize);
|
||||
blob_sidecar.proofs.truncate(num_blobs as usize);
|
||||
blob_sidecar.commitments.truncate(num_blobs as usize);
|
||||
}
|
||||
}
|
||||
|
||||
// ensure exactly num_blobs blobs
|
||||
blob_sidecar.blobs.truncate(num_blobs as usize);
|
||||
blob_sidecar.proofs.truncate(num_blobs as usize);
|
||||
blob_sidecar.commitments.truncate(num_blobs as usize);
|
||||
|
||||
tx.blob_versioned_hashes = blob_sidecar.versioned_hashes().collect();
|
||||
|
||||
(tx, blob_sidecar)
|
||||
|
||||
@@ -240,6 +240,18 @@ pub trait EngineApi<Engine: EngineTypes> {
|
||||
&self,
|
||||
versioned_hashes: Vec<B256>,
|
||||
) -> RpcResult<Option<Vec<BlobAndProofV2>>>;
|
||||
|
||||
/// Fetch blobs for the consensus layer from the blob store.
|
||||
///
|
||||
/// Returns a response of the same length as the request. Missing or older-version blobs are
|
||||
/// returned as `null` elements.
|
||||
///
|
||||
/// Returns `null` if syncing.
|
||||
#[method(name = "getBlobsV3")]
|
||||
async fn get_blobs_v3(
|
||||
&self,
|
||||
versioned_hashes: Vec<B256>,
|
||||
) -> RpcResult<Option<Vec<Option<BlobAndProofV2>>>>;
|
||||
}
|
||||
|
||||
/// A subset of the ETH rpc interface: <https://ethereum.github.io/execution-apis/api-documentation>
|
||||
|
||||
@@ -54,6 +54,7 @@ pub async fn launch_auth(secret: JwtSecret) -> AuthServerHandle {
|
||||
EngineCapabilities::default(),
|
||||
EthereumEngineValidator::new(MAINNET.clone()),
|
||||
false,
|
||||
NoopNetwork::default(),
|
||||
);
|
||||
let module = AuthRpcModule::new(engine_api);
|
||||
module.start_server(config).await.unwrap()
|
||||
|
||||
@@ -23,6 +23,7 @@ reth-tasks.workspace = true
|
||||
reth-engine-primitives.workspace = true
|
||||
reth-transaction-pool.workspace = true
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-network-api.workspace = true
|
||||
|
||||
# ethereum
|
||||
alloy-eips.workspace = true
|
||||
|
||||
@@ -19,6 +19,7 @@ pub const CAPABILITIES: &[&str] = &[
|
||||
"engine_getPayloadBodiesByRangeV1",
|
||||
"engine_getBlobsV1",
|
||||
"engine_getBlobsV2",
|
||||
"engine_getBlobsV3",
|
||||
];
|
||||
|
||||
// The list of all supported Engine capabilities available over the engine endpoint.
|
||||
|
||||
@@ -18,6 +18,7 @@ use async_trait::async_trait;
|
||||
use jsonrpsee_core::{server::RpcModule, RpcResult};
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_engine_primitives::{ConsensusEngineHandle, EngineApiValidator, EngineTypes};
|
||||
use reth_network_api::NetworkInfo;
|
||||
use reth_payload_builder::PayloadStore;
|
||||
use reth_payload_primitives::{
|
||||
validate_payload_timestamp, EngineApiMessageVersion, MessageValidationKind,
|
||||
@@ -94,7 +95,9 @@ where
|
||||
capabilities: EngineCapabilities,
|
||||
validator: Validator,
|
||||
accept_execution_requests_hash: bool,
|
||||
network: impl NetworkInfo + 'static,
|
||||
) -> Self {
|
||||
let is_syncing = Arc::new(move || network.is_syncing());
|
||||
let inner = Arc::new(EngineApiInner {
|
||||
provider,
|
||||
chain_spec,
|
||||
@@ -107,6 +110,7 @@ where
|
||||
tx_pool,
|
||||
validator,
|
||||
accept_execution_requests_hash,
|
||||
is_syncing,
|
||||
});
|
||||
Self { inner }
|
||||
}
|
||||
@@ -792,6 +796,35 @@ where
|
||||
.map_err(|err| EngineApiError::Internal(Box::new(err)))
|
||||
}
|
||||
|
||||
fn get_blobs_v3(
|
||||
&self,
|
||||
versioned_hashes: Vec<B256>,
|
||||
) -> EngineApiResult<Option<Vec<Option<BlobAndProofV2>>>> {
|
||||
// Check if Osaka fork is active
|
||||
let current_timestamp =
|
||||
SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
if !self.inner.chain_spec.is_osaka_active_at_timestamp(current_timestamp) {
|
||||
return Err(EngineApiError::EngineObjectValidationError(
|
||||
reth_payload_primitives::EngineObjectValidationError::UnsupportedFork,
|
||||
));
|
||||
}
|
||||
|
||||
if versioned_hashes.len() > MAX_BLOB_LIMIT {
|
||||
return Err(EngineApiError::BlobRequestTooLarge { len: versioned_hashes.len() })
|
||||
}
|
||||
|
||||
// Spec requires returning `null` if syncing.
|
||||
if (*self.inner.is_syncing)() {
|
||||
return Ok(None)
|
||||
}
|
||||
|
||||
self.inner
|
||||
.tx_pool
|
||||
.get_blobs_for_versioned_hashes_v3(&versioned_hashes)
|
||||
.map(Some)
|
||||
.map_err(|err| EngineApiError::Internal(Box::new(err)))
|
||||
}
|
||||
|
||||
/// Metered version of `get_blobs_v2`.
|
||||
pub fn get_blobs_v2_metered(
|
||||
&self,
|
||||
@@ -827,6 +860,27 @@ where
|
||||
|
||||
res
|
||||
}
|
||||
|
||||
/// Metered version of `get_blobs_v3`.
|
||||
pub fn get_blobs_v3_metered(
|
||||
&self,
|
||||
versioned_hashes: Vec<B256>,
|
||||
) -> EngineApiResult<Option<Vec<Option<BlobAndProofV2>>>> {
|
||||
let hashes_len = versioned_hashes.len();
|
||||
let start = Instant::now();
|
||||
let res = Self::get_blobs_v3(self, versioned_hashes);
|
||||
self.inner.metrics.latency.get_blobs_v3.record(start.elapsed());
|
||||
|
||||
if let Ok(Some(blobs)) = &res {
|
||||
let blobs_found = blobs.iter().flatten().count();
|
||||
let blobs_missed = hashes_len - blobs_found;
|
||||
|
||||
self.inner.metrics.blob_metrics.blob_count.increment(blobs_found as u64);
|
||||
self.inner.metrics.blob_metrics.blob_misses.increment(blobs_missed as u64);
|
||||
}
|
||||
|
||||
res
|
||||
}
|
||||
}
|
||||
|
||||
// This is the concrete ethereum engine API implementation.
|
||||
@@ -1099,6 +1153,14 @@ where
|
||||
trace!(target: "rpc::engine", "Serving engine_getBlobsV2");
|
||||
Ok(self.get_blobs_v2_metered(versioned_hashes)?)
|
||||
}
|
||||
|
||||
async fn get_blobs_v3(
|
||||
&self,
|
||||
versioned_hashes: Vec<B256>,
|
||||
) -> RpcResult<Option<Vec<Option<BlobAndProofV2>>>> {
|
||||
trace!(target: "rpc::engine", "Serving engine_getBlobsV3");
|
||||
Ok(self.get_blobs_v3_metered(versioned_hashes)?)
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider, EngineT, Pool, Validator, ChainSpec> IntoEngineApiRpcModule
|
||||
@@ -1155,6 +1217,8 @@ struct EngineApiInner<Provider, PayloadT: PayloadTypes, Pool, Validator, ChainSp
|
||||
/// Engine validator.
|
||||
validator: Validator,
|
||||
accept_execution_requests_hash: bool,
|
||||
/// Returns `true` if the node is currently syncing.
|
||||
is_syncing: Arc<dyn Fn() -> bool + Send + Sync>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1162,10 +1226,13 @@ mod tests {
|
||||
use super::*;
|
||||
use alloy_rpc_types_engine::{ClientCode, ClientVersionV1};
|
||||
use assert_matches::assert_matches;
|
||||
use reth_chainspec::{ChainSpec, MAINNET};
|
||||
use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET};
|
||||
use reth_engine_primitives::BeaconEngineMessage;
|
||||
use reth_ethereum_engine_primitives::EthEngineTypes;
|
||||
use reth_ethereum_primitives::Block;
|
||||
use reth_network_api::{
|
||||
noop::NoopNetwork, EthProtocolInfo, NetworkError, NetworkInfo, NetworkStatus,
|
||||
};
|
||||
use reth_node_ethereum::EthereumEngineValidator;
|
||||
use reth_payload_builder::test_utils::spawn_test_payload_service;
|
||||
use reth_provider::test_utils::MockEthProvider;
|
||||
@@ -1206,6 +1273,7 @@ mod tests {
|
||||
EngineCapabilities::default(),
|
||||
EthereumEngineValidator::new(chain_spec.clone()),
|
||||
false,
|
||||
NoopNetwork::default(),
|
||||
);
|
||||
let handle = EngineApiTestHandle { chain_spec, provider, from_api: engine_rx };
|
||||
(handle, api)
|
||||
@@ -1247,6 +1315,76 @@ mod tests {
|
||||
assert_matches!(handle.from_api.recv().await, Some(BeaconEngineMessage::NewPayload { .. }));
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TestNetworkInfo {
|
||||
syncing: bool,
|
||||
}
|
||||
|
||||
impl NetworkInfo for TestNetworkInfo {
|
||||
fn local_addr(&self) -> std::net::SocketAddr {
|
||||
(std::net::Ipv4Addr::UNSPECIFIED, 0).into()
|
||||
}
|
||||
|
||||
async fn network_status(&self) -> Result<NetworkStatus, NetworkError> {
|
||||
#[allow(deprecated)]
|
||||
Ok(NetworkStatus {
|
||||
client_version: "test".to_string(),
|
||||
protocol_version: 5,
|
||||
eth_protocol_info: EthProtocolInfo {
|
||||
network: 1,
|
||||
difficulty: None,
|
||||
genesis: Default::default(),
|
||||
config: Default::default(),
|
||||
head: Default::default(),
|
||||
},
|
||||
capabilities: vec![],
|
||||
})
|
||||
}
|
||||
|
||||
fn chain_id(&self) -> u64 {
|
||||
1
|
||||
}
|
||||
|
||||
fn is_syncing(&self) -> bool {
|
||||
self.syncing
|
||||
}
|
||||
|
||||
fn is_initially_syncing(&self) -> bool {
|
||||
self.syncing
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_blobs_v3_returns_null_when_syncing() {
|
||||
let chain_spec: Arc<ChainSpec> =
|
||||
Arc::new(ChainSpecBuilder::mainnet().osaka_activated().build());
|
||||
let provider = Arc::new(MockEthProvider::default());
|
||||
let payload_store = spawn_test_payload_service::<EthEngineTypes>();
|
||||
let (to_engine, _engine_rx) = unbounded_channel::<BeaconEngineMessage<EthEngineTypes>>();
|
||||
|
||||
let api = EngineApi::new(
|
||||
provider,
|
||||
chain_spec.clone(),
|
||||
ConsensusEngineHandle::new(to_engine),
|
||||
payload_store.into(),
|
||||
NoopTransactionPool::default(),
|
||||
Box::<TokioTaskExecutor>::default(),
|
||||
ClientVersionV1 {
|
||||
code: ClientCode::RH,
|
||||
name: "Reth".to_string(),
|
||||
version: "v0.0.0-test".to_string(),
|
||||
commit: "test".to_string(),
|
||||
},
|
||||
EngineCapabilities::default(),
|
||||
EthereumEngineValidator::new(chain_spec),
|
||||
false,
|
||||
TestNetworkInfo { syncing: true },
|
||||
);
|
||||
|
||||
let res = api.get_blobs_v3_metered(vec![B256::ZERO]);
|
||||
assert_matches!(res, Ok(None));
|
||||
}
|
||||
|
||||
// tests covering `engine_getPayloadBodiesByRange` and `engine_getPayloadBodiesByHash`
|
||||
mod get_payload_bodies {
|
||||
use super::*;
|
||||
|
||||
@@ -46,6 +46,8 @@ pub(crate) struct EngineApiLatencyMetrics {
|
||||
pub(crate) get_blobs_v1: Histogram,
|
||||
/// Latency for `engine_getBlobsV2`
|
||||
pub(crate) get_blobs_v2: Histogram,
|
||||
/// Latency for `engine_getBlobsV3`
|
||||
pub(crate) get_blobs_v3: Histogram,
|
||||
}
|
||||
|
||||
#[derive(Metrics)]
|
||||
|
||||
@@ -23,7 +23,7 @@ use reth_stages_api::{
|
||||
};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
cmp::{max, Ordering},
|
||||
ops::RangeInclusive,
|
||||
sync::Arc,
|
||||
task::{ready, Context, Poll},
|
||||
@@ -620,7 +620,11 @@ where
|
||||
// Otherwise, we recalculate the whole stage checkpoint including the amount of gas
|
||||
// already processed, if there's any.
|
||||
_ => {
|
||||
let processed = calculate_gas_used_from_headers(provider, 0..=start_block - 1)?;
|
||||
let genesis_block_number = provider.genesis_block_number();
|
||||
let processed = calculate_gas_used_from_headers(
|
||||
provider,
|
||||
genesis_block_number..=max(start_block - 1, genesis_block_number),
|
||||
)?;
|
||||
|
||||
ExecutionCheckpoint {
|
||||
block_range: CheckpointBlockRange { from: start_block, to: max_block },
|
||||
|
||||
@@ -100,6 +100,7 @@ where
|
||||
+ StateWriter
|
||||
+ TrieWriter
|
||||
+ MetadataWriter
|
||||
+ ChainSpecProvider
|
||||
+ AsRef<PF::ProviderRW>,
|
||||
PF::ChainSpec: EthChainSpec<Header = <PF::Primitives as NodePrimitives>::BlockHeader>,
|
||||
{
|
||||
@@ -126,6 +127,7 @@ where
|
||||
+ StateWriter
|
||||
+ TrieWriter
|
||||
+ MetadataWriter
|
||||
+ ChainSpecProvider
|
||||
+ AsRef<PF::ProviderRW>,
|
||||
PF::ChainSpec: EthChainSpec<Header = <PF::Primitives as NodePrimitives>::BlockHeader>,
|
||||
{
|
||||
@@ -134,9 +136,12 @@ where
|
||||
let genesis = chain.genesis();
|
||||
let hash = chain.genesis_hash();
|
||||
|
||||
// Get the genesis block number from the chain spec
|
||||
let genesis_block_number = chain.genesis_header().number();
|
||||
|
||||
// Check if we already have the genesis header or if we have the wrong one.
|
||||
match factory.block_hash(0) {
|
||||
Ok(None) | Err(ProviderError::MissingStaticFileBlock(StaticFileSegment::Headers, 0)) => {}
|
||||
match factory.block_hash(genesis_block_number) {
|
||||
Ok(None) | Err(ProviderError::MissingStaticFileBlock(StaticFileSegment::Headers, _)) => {}
|
||||
Ok(Some(block_hash)) => {
|
||||
if block_hash == hash {
|
||||
// Some users will at times attempt to re-sync from scratch by just deleting the
|
||||
@@ -179,15 +184,26 @@ where
|
||||
// compute state root to populate trie tables
|
||||
compute_state_root(&provider_rw, None)?;
|
||||
|
||||
// insert sync stage
|
||||
// set stage checkpoint to genesis block number for all stages
|
||||
let checkpoint = StageCheckpoint::new(genesis_block_number);
|
||||
for stage in StageId::ALL {
|
||||
provider_rw.save_stage_checkpoint(stage, Default::default())?;
|
||||
provider_rw.save_stage_checkpoint(stage, checkpoint)?;
|
||||
}
|
||||
|
||||
// Static file segments start empty, so we need to initialize the genesis block.
|
||||
let static_file_provider = provider_rw.static_file_provider();
|
||||
static_file_provider.latest_writer(StaticFileSegment::Receipts)?.increment_block(0)?;
|
||||
static_file_provider.latest_writer(StaticFileSegment::Transactions)?.increment_block(0)?;
|
||||
|
||||
// Static file segments start empty, so we need to initialize the genesis block.
|
||||
// For genesis blocks with non-zero block numbers, we need to use get_writer() instead of
|
||||
// latest_writer() to ensure the genesis block is stored in the correct static file range.
|
||||
static_file_provider
|
||||
.get_writer(genesis_block_number, StaticFileSegment::Receipts)?
|
||||
.user_header_mut()
|
||||
.set_block_range(genesis_block_number, genesis_block_number);
|
||||
static_file_provider
|
||||
.get_writer(genesis_block_number, StaticFileSegment::Transactions)?
|
||||
.user_header_mut()
|
||||
.set_block_range(genesis_block_number, genesis_block_number);
|
||||
|
||||
// Behaviour reserved only for new nodes should be set here.
|
||||
provider_rw.write_storage_settings(storage_settings)?;
|
||||
@@ -210,9 +226,11 @@ where
|
||||
+ DBProvider<Tx: DbTxMut>
|
||||
+ HeaderProvider
|
||||
+ StateWriter
|
||||
+ ChainSpecProvider
|
||||
+ AsRef<Provider>,
|
||||
{
|
||||
insert_state(provider, alloc, 0)
|
||||
let genesis_block_number = provider.chain_spec().genesis_header().number();
|
||||
insert_state(provider, alloc, genesis_block_number)
|
||||
}
|
||||
|
||||
/// Inserts state at given block into database.
|
||||
@@ -335,9 +353,10 @@ pub fn insert_genesis_history<'a, 'b, Provider>(
|
||||
alloc: impl Iterator<Item = (&'a Address, &'b GenesisAccount)> + Clone,
|
||||
) -> ProviderResult<()>
|
||||
where
|
||||
Provider: DBProvider<Tx: DbTxMut> + HistoryWriter,
|
||||
Provider: DBProvider<Tx: DbTxMut> + HistoryWriter + ChainSpecProvider,
|
||||
{
|
||||
insert_history(provider, alloc, 0)
|
||||
let genesis_block_number = provider.chain_spec().genesis_header().number();
|
||||
insert_history(provider, alloc, genesis_block_number)
|
||||
}
|
||||
|
||||
/// Inserts history indices for genesis accounts and storage.
|
||||
@@ -377,17 +396,37 @@ where
|
||||
let (header, block_hash) = (chain.genesis_header(), chain.genesis_hash());
|
||||
let static_file_provider = provider.static_file_provider();
|
||||
|
||||
match static_file_provider.block_hash(0) {
|
||||
Ok(None) | Err(ProviderError::MissingStaticFileBlock(StaticFileSegment::Headers, 0)) => {
|
||||
let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?;
|
||||
writer.append_header(header, &block_hash)?;
|
||||
// Get the actual genesis block number from the header
|
||||
let genesis_block_number = header.number();
|
||||
|
||||
match static_file_provider.block_hash(genesis_block_number) {
|
||||
Ok(None) | Err(ProviderError::MissingStaticFileBlock(StaticFileSegment::Headers, _)) => {
|
||||
let difficulty = header.difficulty();
|
||||
|
||||
// For genesis blocks with non-zero block numbers, we need to ensure they are stored
|
||||
// in the correct static file range. We use get_writer() with the genesis block number
|
||||
// to ensure the genesis block is stored in the correct static file range.
|
||||
let mut writer = static_file_provider
|
||||
.get_writer(genesis_block_number, StaticFileSegment::Headers)?;
|
||||
|
||||
// For non-zero genesis blocks, we need to set block range to genesis_block_number and
|
||||
// append header without increment block
|
||||
if genesis_block_number > 0 {
|
||||
writer
|
||||
.user_header_mut()
|
||||
.set_block_range(genesis_block_number, genesis_block_number);
|
||||
writer.append_header_direct(header, difficulty, &block_hash)?;
|
||||
} else {
|
||||
// For zero genesis blocks, use normal append_header
|
||||
writer.append_header(header, &block_hash)?;
|
||||
}
|
||||
}
|
||||
Ok(Some(_)) => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
|
||||
provider.tx_ref().put::<tables::HeaderNumbers>(block_hash, 0)?;
|
||||
provider.tx_ref().put::<tables::BlockBodyIndices>(0, Default::default())?;
|
||||
provider.tx_ref().put::<tables::HeaderNumbers>(block_hash, genesis_block_number)?;
|
||||
provider.tx_ref().put::<tables::BlockBodyIndices>(genesis_block_number, Default::default())?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
913
crates/storage/provider/src/providers/rocksdb/invariants.rs
Normal file
913
crates/storage/provider/src/providers/rocksdb/invariants.rs
Normal file
@@ -0,0 +1,913 @@
|
||||
//! Invariant checking for `RocksDB` tables.
|
||||
//!
|
||||
//! This module provides consistency checks for tables stored in `RocksDB`, similar to the
|
||||
//! consistency checks for static files. The goal is to detect and potentially heal
|
||||
//! inconsistencies between `RocksDB` data and MDBX checkpoints.
|
||||
|
||||
use super::RocksDBProvider;
|
||||
use crate::StaticFileProviderFactory;
|
||||
use alloy_eips::eip2718::Encodable2718;
|
||||
use alloy_primitives::BlockNumber;
|
||||
use rayon::prelude::*;
|
||||
use reth_db::cursor::DbCursorRO;
|
||||
use reth_db_api::{tables, transaction::DbTx};
|
||||
use reth_stages_types::StageId;
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::{
|
||||
DBProvider, StageCheckpointReader, StorageSettingsCache, TransactionsProvider,
|
||||
};
|
||||
use reth_storage_errors::provider::ProviderResult;
|
||||
|
||||
impl RocksDBProvider {
|
||||
/// Checks consistency of `RocksDB` tables against MDBX stage checkpoints.
|
||||
///
|
||||
/// Returns an unwind target block number if the pipeline needs to unwind to rebuild
|
||||
/// `RocksDB` data. Returns `None` if all invariants pass or if inconsistencies were healed.
|
||||
///
|
||||
/// # Invariants checked
|
||||
///
|
||||
/// For `TransactionHashNumbers`:
|
||||
/// - The maximum `TxNumber` value should not exceed what the `TransactionLookup` stage
|
||||
/// checkpoint indicates has been processed.
|
||||
/// - If `RocksDB` is ahead, excess entries are pruned (healed).
|
||||
/// - If `RocksDB` is behind, an unwind is required.
|
||||
///
|
||||
/// For `StoragesHistory`:
|
||||
/// - The maximum block number in shards should not exceed the `IndexStorageHistory` stage
|
||||
/// checkpoint.
|
||||
/// - Similar healing/unwind logic applies.
|
||||
///
|
||||
/// # Requirements
|
||||
///
|
||||
/// For pruning `TransactionHashNumbers`, the provider must be able to supply transaction
|
||||
/// data (typically from static files) so that transaction hashes can be computed. This
|
||||
/// implies that static files should be ahead of or in sync with `RocksDB`.
|
||||
pub fn check_consistency<Provider>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
) -> ProviderResult<Option<BlockNumber>>
|
||||
where
|
||||
Provider: DBProvider
|
||||
+ StageCheckpointReader
|
||||
+ StorageSettingsCache
|
||||
+ StaticFileProviderFactory
|
||||
+ TransactionsProvider<Transaction: Encodable2718>,
|
||||
{
|
||||
let mut unwind_target: Option<BlockNumber> = None;
|
||||
|
||||
// Check TransactionHashNumbers if stored in RocksDB
|
||||
if provider.cached_storage_settings().transaction_hash_numbers_in_rocksdb &&
|
||||
let Some(target) = self.check_transaction_hash_numbers(provider)?
|
||||
{
|
||||
unwind_target = Some(unwind_target.map_or(target, |t| t.min(target)));
|
||||
}
|
||||
|
||||
// Check StoragesHistory if stored in RocksDB
|
||||
if provider.cached_storage_settings().storages_history_in_rocksdb &&
|
||||
let Some(target) = self.check_storages_history(provider)?
|
||||
{
|
||||
unwind_target = Some(unwind_target.map_or(target, |t| t.min(target)));
|
||||
}
|
||||
|
||||
Ok(unwind_target)
|
||||
}
|
||||
|
||||
/// Checks invariants for the `TransactionHashNumbers` table.
|
||||
///
|
||||
/// Returns a block number to unwind to if MDBX is behind the checkpoint.
|
||||
/// If static files are ahead of MDBX, excess `RocksDB` entries are pruned (healed).
|
||||
///
|
||||
/// # Approach
|
||||
///
|
||||
/// Instead of iterating `RocksDB` entries (which is expensive and doesn't give us the
|
||||
/// tx range we need), we use static files and MDBX to determine what needs pruning:
|
||||
/// - Static files are committed before `RocksDB`, so they're at least at the same height
|
||||
/// - MDBX `TransactionBlocks` tells us what's been fully committed
|
||||
/// - If static files have more transactions than MDBX, prune the excess range
|
||||
fn check_transaction_hash_numbers<Provider>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
) -> ProviderResult<Option<BlockNumber>>
|
||||
where
|
||||
Provider: DBProvider
|
||||
+ StageCheckpointReader
|
||||
+ StaticFileProviderFactory
|
||||
+ TransactionsProvider<Transaction: Encodable2718>,
|
||||
{
|
||||
// Get the TransactionLookup stage checkpoint
|
||||
let checkpoint = provider
|
||||
.get_stage_checkpoint(StageId::TransactionLookup)?
|
||||
.map(|cp| cp.block_number)
|
||||
.unwrap_or(0);
|
||||
|
||||
// Get last tx_num from MDBX - this tells us what MDBX has fully committed
|
||||
let mut cursor = provider.tx_ref().cursor_read::<tables::TransactionBlocks>()?;
|
||||
let mdbx_last = cursor.last()?;
|
||||
|
||||
// Get highest tx_num from static files - this tells us what tx data is available
|
||||
let highest_static_tx = provider
|
||||
.static_file_provider()
|
||||
.get_highest_static_file_tx(StaticFileSegment::Transactions);
|
||||
|
||||
match (mdbx_last, highest_static_tx) {
|
||||
(Some((mdbx_tx, mdbx_block)), Some(highest_tx)) if highest_tx > mdbx_tx => {
|
||||
// Static files are ahead of MDBX - prune RocksDB entries for the excess range.
|
||||
// This is the common case during recovery from a crash during unwinding.
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
mdbx_last_tx = mdbx_tx,
|
||||
mdbx_block,
|
||||
highest_static_tx = highest_tx,
|
||||
"Static files ahead of MDBX, pruning TransactionHashNumbers excess data"
|
||||
);
|
||||
self.prune_transaction_hash_numbers_in_range(provider, (mdbx_tx + 1)..=highest_tx)?;
|
||||
|
||||
// After pruning, check if MDBX is behind checkpoint
|
||||
if checkpoint > mdbx_block {
|
||||
tracing::warn!(
|
||||
target: "reth::providers::rocksdb",
|
||||
mdbx_block,
|
||||
checkpoint,
|
||||
"MDBX behind checkpoint after pruning, unwind needed"
|
||||
);
|
||||
return Ok(Some(mdbx_block));
|
||||
}
|
||||
}
|
||||
(Some((_mdbx_tx, mdbx_block)), _) => {
|
||||
// MDBX and static files are in sync (or static files don't have more data).
|
||||
// Check if MDBX is behind checkpoint.
|
||||
if checkpoint > mdbx_block {
|
||||
tracing::warn!(
|
||||
target: "reth::providers::rocksdb",
|
||||
mdbx_block,
|
||||
checkpoint,
|
||||
"MDBX behind checkpoint, unwind needed"
|
||||
);
|
||||
return Ok(Some(mdbx_block));
|
||||
}
|
||||
}
|
||||
(None, Some(highest_tx)) => {
|
||||
// MDBX has no transactions but static files have data.
|
||||
// This means RocksDB might have stale entries - prune them all.
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
highest_static_tx = highest_tx,
|
||||
"MDBX empty but static files have data, pruning all TransactionHashNumbers"
|
||||
);
|
||||
self.prune_transaction_hash_numbers_in_range(provider, 0..=highest_tx)?;
|
||||
}
|
||||
(None, None) => {
|
||||
// Both MDBX and static files are empty.
|
||||
// If checkpoint says we should have data, that's an inconsistency.
|
||||
if checkpoint > 0 {
|
||||
tracing::warn!(
|
||||
target: "reth::providers::rocksdb",
|
||||
checkpoint,
|
||||
"Checkpoint set but no transaction data exists, unwind needed"
|
||||
);
|
||||
return Ok(Some(0));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Prunes `TransactionHashNumbers` entries for transactions in the given range.
|
||||
///
|
||||
/// This fetches transactions from the provider, computes their hashes in parallel,
|
||||
/// and deletes the corresponding entries from `RocksDB` by key. This approach is more
|
||||
/// scalable than iterating all rows because it only processes the transactions that
|
||||
/// need to be pruned.
|
||||
///
|
||||
/// # Requirements
|
||||
///
|
||||
/// The provider must be able to supply transaction data (typically from static files)
|
||||
/// so that transaction hashes can be computed. This implies that static files should
|
||||
/// be ahead of or in sync with `RocksDB`.
|
||||
fn prune_transaction_hash_numbers_in_range<Provider>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
tx_range: std::ops::RangeInclusive<u64>,
|
||||
) -> ProviderResult<()>
|
||||
where
|
||||
Provider: TransactionsProvider<Transaction: Encodable2718>,
|
||||
{
|
||||
if tx_range.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Fetch transactions in the range and compute their hashes in parallel
|
||||
let hashes: Vec<_> = provider
|
||||
.transactions_by_tx_range(tx_range.clone())?
|
||||
.into_par_iter()
|
||||
.map(|tx| tx.trie_hash())
|
||||
.collect();
|
||||
|
||||
if !hashes.is_empty() {
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
deleted_count = hashes.len(),
|
||||
tx_range_start = *tx_range.start(),
|
||||
tx_range_end = *tx_range.end(),
|
||||
"Pruning TransactionHashNumbers entries by tx range"
|
||||
);
|
||||
|
||||
let mut batch = self.batch();
|
||||
for hash in hashes {
|
||||
batch.delete::<tables::TransactionHashNumbers>(hash)?;
|
||||
}
|
||||
batch.commit()?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Checks invariants for the `StoragesHistory` table.
|
||||
///
|
||||
/// Returns a block number to unwind to if `RocksDB` is behind the checkpoint.
|
||||
/// If `RocksDB` is ahead of the checkpoint, excess entries are pruned (healed).
|
||||
fn check_storages_history<Provider>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
) -> ProviderResult<Option<BlockNumber>>
|
||||
where
|
||||
Provider: DBProvider + StageCheckpointReader,
|
||||
{
|
||||
// Get the IndexStorageHistory stage checkpoint
|
||||
let checkpoint = provider
|
||||
.get_stage_checkpoint(StageId::IndexStorageHistory)?
|
||||
.map(|cp| cp.block_number)
|
||||
.unwrap_or(0);
|
||||
|
||||
// Check if RocksDB has any data
|
||||
let rocks_first = self.first::<tables::StoragesHistory>()?;
|
||||
|
||||
match rocks_first {
|
||||
Some(_) => {
|
||||
// If checkpoint is 0 but we have data, clear everything
|
||||
if checkpoint == 0 {
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
"StoragesHistory has data but checkpoint is 0, clearing all"
|
||||
);
|
||||
self.prune_storages_history_above(0)?;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// Find the max highest_block_number (excluding u64::MAX sentinel) across all
|
||||
// entries
|
||||
let mut max_highest_block = 0u64;
|
||||
for result in self.iter::<tables::StoragesHistory>()? {
|
||||
let (key, _) = result?;
|
||||
let highest = key.sharded_key.highest_block_number;
|
||||
if highest != u64::MAX && highest > max_highest_block {
|
||||
max_highest_block = highest;
|
||||
}
|
||||
}
|
||||
|
||||
// If any entry has highest_block > checkpoint, prune excess
|
||||
if max_highest_block > checkpoint {
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
rocks_highest = max_highest_block,
|
||||
checkpoint,
|
||||
"StoragesHistory ahead of checkpoint, pruning excess data"
|
||||
);
|
||||
self.prune_storages_history_above(checkpoint)?;
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
None => {
|
||||
// Empty RocksDB table
|
||||
if checkpoint > 0 {
|
||||
// Stage says we should have data but we don't
|
||||
return Ok(Some(0));
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Prunes `StoragesHistory` entries where `highest_block_number` > `max_block`.
|
||||
///
|
||||
/// For `StoragesHistory`, the key contains `highest_block_number`, so we can iterate
|
||||
/// and delete entries where `key.sharded_key.highest_block_number > max_block`.
|
||||
///
|
||||
/// TODO(<https://github.com/paradigmxyz/reth/issues/20417>): this iterates the whole table,
|
||||
/// which is inefficient. Use changeset-based pruning instead.
|
||||
fn prune_storages_history_above(&self, max_block: BlockNumber) -> ProviderResult<()> {
|
||||
use reth_db_api::models::storage_sharded_key::StorageShardedKey;
|
||||
|
||||
let mut to_delete: Vec<StorageShardedKey> = Vec::new();
|
||||
for result in self.iter::<tables::StoragesHistory>()? {
|
||||
let (key, _) = result?;
|
||||
let highest_block = key.sharded_key.highest_block_number;
|
||||
if max_block == 0 || (highest_block != u64::MAX && highest_block > max_block) {
|
||||
to_delete.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
let deleted = to_delete.len();
|
||||
if deleted > 0 {
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
deleted_count = deleted,
|
||||
max_block,
|
||||
"Pruning StoragesHistory entries"
|
||||
);
|
||||
|
||||
let mut batch = self.batch();
|
||||
for key in to_delete {
|
||||
batch.delete::<tables::StoragesHistory>(key)?;
|
||||
}
|
||||
batch.commit()?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{
|
||||
providers::rocksdb::RocksDBBuilder, test_utils::create_test_provider_factory, BlockWriter,
|
||||
DatabaseProviderFactory, StageCheckpointWriter, TransactionsProvider,
|
||||
};
|
||||
use alloy_primitives::{Address, B256};
|
||||
use reth_db::cursor::DbCursorRW;
|
||||
use reth_db_api::{
|
||||
models::{storage_sharded_key::StorageShardedKey, StorageSettings},
|
||||
tables::{self, BlockNumberList},
|
||||
transaction::DbTxMut,
|
||||
};
|
||||
use reth_stages_types::StageCheckpoint;
|
||||
use reth_testing_utils::generators::{self, BlockRangeParams};
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn test_first_last_empty_rocksdb() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let provider = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::TransactionHashNumbers>()
|
||||
.with_table::<tables::StoragesHistory>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Empty RocksDB, no checkpoints - should be consistent
|
||||
let first = provider.first::<tables::TransactionHashNumbers>().unwrap();
|
||||
let last = provider.last::<tables::TransactionHashNumbers>().unwrap();
|
||||
|
||||
assert!(first.is_none());
|
||||
assert!(last.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_first_last_with_data() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let provider = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::TransactionHashNumbers>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Insert some data
|
||||
let tx_hash = B256::from([1u8; 32]);
|
||||
provider.put::<tables::TransactionHashNumbers>(tx_hash, &100).unwrap();
|
||||
|
||||
// RocksDB has data
|
||||
let last = provider.last::<tables::TransactionHashNumbers>().unwrap();
|
||||
assert!(last.is_some());
|
||||
assert_eq!(last.unwrap().1, 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_empty_rocksdb_no_checkpoint_is_ok() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::TransactionHashNumbers>()
|
||||
.with_table::<tables::StoragesHistory>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Create a test provider factory for MDBX
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy()
|
||||
.with_transaction_hash_numbers_in_rocksdb(true)
|
||||
.with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// Empty RocksDB and no checkpoints - should be consistent (None = no unwind needed)
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_empty_rocksdb_with_checkpoint_needs_unwind() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::TransactionHashNumbers>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Create a test provider factory for MDBX
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Set a checkpoint indicating we should have processed up to block 100
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
provider
|
||||
.save_stage_checkpoint(StageId::TransactionLookup, StageCheckpoint::new(100))
|
||||
.unwrap();
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// RocksDB is empty but checkpoint says block 100 was processed
|
||||
// This means RocksDB is missing data and we need to unwind to rebuild
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(result, Some(0), "Should require unwind to block 0 to rebuild RocksDB");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_mdbx_empty_static_files_have_data_prunes_rocksdb() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::TransactionHashNumbers>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Generate blocks with real transactions and insert them
|
||||
let mut rng = generators::rng();
|
||||
let blocks = generators::random_block_range(
|
||||
&mut rng,
|
||||
0..=2,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() },
|
||||
);
|
||||
|
||||
let mut tx_hashes = Vec::new();
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
let mut tx_count = 0u64;
|
||||
for block in &blocks {
|
||||
provider.insert_block(block.clone().try_recover().expect("recover block")).unwrap();
|
||||
for tx in &block.body().transactions {
|
||||
let hash = tx.trie_hash();
|
||||
tx_hashes.push(hash);
|
||||
rocksdb.put::<tables::TransactionHashNumbers>(hash, &tx_count).unwrap();
|
||||
tx_count += 1;
|
||||
}
|
||||
}
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
// Simulate crash recovery: MDBX was reset but static files and RocksDB still have data.
|
||||
// Clear TransactionBlocks to simulate empty MDBX state.
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
let mut cursor = provider.tx_ref().cursor_write::<tables::TransactionBlocks>().unwrap();
|
||||
let mut to_delete = Vec::new();
|
||||
let mut walker = cursor.walk(Some(0)).unwrap();
|
||||
while let Some((tx_num, _)) = walker.next().transpose().unwrap() {
|
||||
to_delete.push(tx_num);
|
||||
}
|
||||
drop(walker);
|
||||
for tx_num in to_delete {
|
||||
cursor.seek_exact(tx_num).unwrap();
|
||||
cursor.delete_current().unwrap();
|
||||
}
|
||||
// No checkpoint set (checkpoint = 0)
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
// Verify RocksDB data exists
|
||||
assert!(rocksdb.last::<tables::TransactionHashNumbers>().unwrap().is_some());
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// MDBX TransactionBlocks is empty, but static files have transaction data.
|
||||
// This means RocksDB has stale data that should be pruned (healed).
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(result, None, "Should heal by pruning, no unwind needed");
|
||||
|
||||
// Verify data was pruned
|
||||
for hash in &tx_hashes {
|
||||
assert!(
|
||||
rocksdb.get::<tables::TransactionHashNumbers>(*hash).unwrap().is_none(),
|
||||
"RocksDB should be empty after pruning"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_storages_history_empty_with_checkpoint_needs_unwind() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::StoragesHistory>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Create a test provider factory for MDBX
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Set a checkpoint indicating we should have processed up to block 100
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
provider
|
||||
.save_stage_checkpoint(StageId::IndexStorageHistory, StageCheckpoint::new(100))
|
||||
.unwrap();
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// RocksDB is empty but checkpoint says block 100 was processed
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(result, Some(0), "Should require unwind to block 0 to rebuild StoragesHistory");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_storages_history_has_data_no_checkpoint_prunes_data() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::StoragesHistory>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Insert data into RocksDB
|
||||
let key = StorageShardedKey::new(Address::ZERO, B256::ZERO, 50);
|
||||
let block_list = BlockNumberList::new_pre_sorted([10, 20, 30, 50]);
|
||||
rocksdb.put::<tables::StoragesHistory>(key, &block_list).unwrap();
|
||||
|
||||
// Verify data exists
|
||||
assert!(rocksdb.last::<tables::StoragesHistory>().unwrap().is_some());
|
||||
|
||||
// Create a test provider factory for MDBX with NO checkpoint
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// RocksDB has data but checkpoint is 0
|
||||
// This means RocksDB has stale data that should be pruned (healed)
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(result, None, "Should heal by pruning, no unwind needed");
|
||||
|
||||
// Verify data was pruned
|
||||
assert!(
|
||||
rocksdb.last::<tables::StoragesHistory>().unwrap().is_none(),
|
||||
"RocksDB should be empty after pruning"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_mdbx_behind_checkpoint_needs_unwind() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::TransactionHashNumbers>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Generate blocks with real transactions (blocks 0-2, 6 transactions total)
|
||||
let mut rng = generators::rng();
|
||||
let blocks = generators::random_block_range(
|
||||
&mut rng,
|
||||
0..=2,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() },
|
||||
);
|
||||
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
let mut tx_count = 0u64;
|
||||
for block in &blocks {
|
||||
provider.insert_block(block.clone().try_recover().expect("recover block")).unwrap();
|
||||
for tx in &block.body().transactions {
|
||||
let hash = tx.trie_hash();
|
||||
rocksdb.put::<tables::TransactionHashNumbers>(hash, &tx_count).unwrap();
|
||||
tx_count += 1;
|
||||
}
|
||||
}
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
// Now simulate a scenario where checkpoint is ahead of MDBX.
|
||||
// This happens when the checkpoint was saved but MDBX data was lost/corrupted.
|
||||
// Set checkpoint to block 10 (beyond our actual data at block 2)
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
provider
|
||||
.save_stage_checkpoint(StageId::TransactionLookup, StageCheckpoint::new(10))
|
||||
.unwrap();
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// MDBX has data up to block 2, but checkpoint says block 10 was processed.
|
||||
// The static files highest tx matches MDBX last tx (both at block 2).
|
||||
// Checkpoint > mdbx_block means we need to unwind to rebuild.
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(
|
||||
result,
|
||||
Some(2),
|
||||
"Should require unwind to block 2 (MDBX's last block) to rebuild from checkpoint"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_rocksdb_ahead_of_checkpoint_prunes_excess() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::TransactionHashNumbers>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Create a test provider factory for MDBX
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Generate blocks with real transactions:
|
||||
// Blocks 0-5, each with 2 transactions = 12 total transactions (0-11)
|
||||
let mut rng = generators::rng();
|
||||
let blocks = generators::random_block_range(
|
||||
&mut rng,
|
||||
0..=5,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() },
|
||||
);
|
||||
|
||||
// Track which hashes belong to which blocks
|
||||
let mut tx_hashes = Vec::new();
|
||||
let mut tx_count = 0u64;
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
// Insert ALL blocks (0-5) to write transactions to static files
|
||||
for block in &blocks {
|
||||
provider.insert_block(block.clone().try_recover().expect("recover block")).unwrap();
|
||||
for tx in &block.body().transactions {
|
||||
let hash = tx.trie_hash();
|
||||
tx_hashes.push(hash);
|
||||
rocksdb.put::<tables::TransactionHashNumbers>(hash, &tx_count).unwrap();
|
||||
tx_count += 1;
|
||||
}
|
||||
}
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
// Simulate crash recovery scenario:
|
||||
// MDBX was unwound to block 2, but RocksDB and static files still have more data.
|
||||
// Remove TransactionBlocks entries for blocks 3-5 to simulate MDBX unwind.
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
// Delete TransactionBlocks entries for tx > 5 (i.e., for blocks 3-5)
|
||||
// TransactionBlocks maps last_tx_in_block -> block_number
|
||||
// After unwind, only entries for blocks 0-2 should remain (tx 5 -> block 2)
|
||||
let mut cursor = provider.tx_ref().cursor_write::<tables::TransactionBlocks>().unwrap();
|
||||
// Walk and delete entries where block > 2
|
||||
let mut to_delete = Vec::new();
|
||||
let mut walker = cursor.walk(Some(0)).unwrap();
|
||||
while let Some((tx_num, block_num)) = walker.next().transpose().unwrap() {
|
||||
if block_num > 2 {
|
||||
to_delete.push(tx_num);
|
||||
}
|
||||
}
|
||||
drop(walker);
|
||||
for tx_num in to_delete {
|
||||
cursor.seek_exact(tx_num).unwrap();
|
||||
cursor.delete_current().unwrap();
|
||||
}
|
||||
|
||||
// Set checkpoint to block 2
|
||||
provider
|
||||
.save_stage_checkpoint(StageId::TransactionLookup, StageCheckpoint::new(2))
|
||||
.unwrap();
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// RocksDB has tx hashes for all blocks (0-5)
|
||||
// MDBX TransactionBlocks only goes up to tx 5 (block 2)
|
||||
// Static files have data for all txs (0-11)
|
||||
// This means RocksDB is ahead and should prune entries for tx 6-11
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(result, None, "Should heal by pruning, no unwind needed");
|
||||
|
||||
// Verify: hashes for blocks 0-2 (tx 0-5) should remain, blocks 3-5 (tx 6-11) should be
|
||||
// pruned First 6 hashes should remain
|
||||
for (i, hash) in tx_hashes.iter().take(6).enumerate() {
|
||||
assert!(
|
||||
rocksdb.get::<tables::TransactionHashNumbers>(*hash).unwrap().is_some(),
|
||||
"tx {} should remain",
|
||||
i
|
||||
);
|
||||
}
|
||||
// Last 6 hashes should be pruned
|
||||
for (i, hash) in tx_hashes.iter().skip(6).enumerate() {
|
||||
assert!(
|
||||
rocksdb.get::<tables::TransactionHashNumbers>(*hash).unwrap().is_none(),
|
||||
"tx {} should be pruned",
|
||||
i + 6
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_storages_history_ahead_of_checkpoint_prunes_excess() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::StoragesHistory>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Insert data into RocksDB with different highest_block_numbers
|
||||
let key_block_50 = StorageShardedKey::new(Address::ZERO, B256::ZERO, 50);
|
||||
let key_block_100 = StorageShardedKey::new(Address::ZERO, B256::from([1u8; 32]), 100);
|
||||
let key_block_150 = StorageShardedKey::new(Address::ZERO, B256::from([2u8; 32]), 150);
|
||||
let key_block_max = StorageShardedKey::new(Address::ZERO, B256::from([3u8; 32]), u64::MAX);
|
||||
|
||||
let block_list = BlockNumberList::new_pre_sorted([10, 20, 30]);
|
||||
rocksdb.put::<tables::StoragesHistory>(key_block_50.clone(), &block_list).unwrap();
|
||||
rocksdb.put::<tables::StoragesHistory>(key_block_100.clone(), &block_list).unwrap();
|
||||
rocksdb.put::<tables::StoragesHistory>(key_block_150.clone(), &block_list).unwrap();
|
||||
rocksdb.put::<tables::StoragesHistory>(key_block_max.clone(), &block_list).unwrap();
|
||||
|
||||
// Create a test provider factory for MDBX
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Set checkpoint to block 100
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
provider
|
||||
.save_stage_checkpoint(StageId::IndexStorageHistory, StageCheckpoint::new(100))
|
||||
.unwrap();
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// RocksDB has entries with highest_block = 150 which exceeds checkpoint (100)
|
||||
// Should prune entries where highest_block > 100 (but not u64::MAX sentinel)
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(result, None, "Should heal by pruning, no unwind needed");
|
||||
|
||||
// Verify key_block_150 was pruned, but others remain
|
||||
assert!(
|
||||
rocksdb.get::<tables::StoragesHistory>(key_block_50).unwrap().is_some(),
|
||||
"Entry with highest_block=50 should remain"
|
||||
);
|
||||
assert!(
|
||||
rocksdb.get::<tables::StoragesHistory>(key_block_100).unwrap().is_some(),
|
||||
"Entry with highest_block=100 should remain"
|
||||
);
|
||||
assert!(
|
||||
rocksdb.get::<tables::StoragesHistory>(key_block_150).unwrap().is_none(),
|
||||
"Entry with highest_block=150 should be pruned"
|
||||
);
|
||||
assert!(
|
||||
rocksdb.get::<tables::StoragesHistory>(key_block_max).unwrap().is_some(),
|
||||
"Entry with highest_block=u64::MAX (sentinel) should remain"
|
||||
);
|
||||
}
|
||||
|
||||
/// Test that pruning works by fetching transactions and computing their hashes,
|
||||
/// rather than iterating all rows. This test uses random blocks with unique
|
||||
/// transactions so we can verify the correct entries are pruned.
|
||||
#[test]
|
||||
fn test_prune_transaction_hash_numbers_by_range() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path())
|
||||
.with_table::<tables::TransactionHashNumbers>()
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Create a test provider factory for MDBX
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
// Generate random blocks with unique transactions
|
||||
// Block 0 (genesis) has no transactions
|
||||
// Blocks 1-5 each have 2 transactions = 10 transactions total
|
||||
let mut rng = generators::rng();
|
||||
let blocks = generators::random_block_range(
|
||||
&mut rng,
|
||||
0..=5,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() },
|
||||
);
|
||||
|
||||
// Insert blocks into the database
|
||||
let mut tx_count = 0u64;
|
||||
let mut tx_hashes = Vec::new();
|
||||
{
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
|
||||
for block in &blocks {
|
||||
provider.insert_block(block.clone().try_recover().expect("recover block")).unwrap();
|
||||
|
||||
// Store transaction hash -> tx_number mappings in RocksDB
|
||||
for tx in &block.body().transactions {
|
||||
let hash = tx.trie_hash();
|
||||
tx_hashes.push(hash);
|
||||
rocksdb.put::<tables::TransactionHashNumbers>(hash, &tx_count).unwrap();
|
||||
tx_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Set checkpoint to block 2 (meaning we should only have tx hashes for blocks 0-2)
|
||||
// Blocks 0, 1, 2 have 6 transactions (2 each), so tx 0-5 should remain
|
||||
provider
|
||||
.save_stage_checkpoint(StageId::TransactionLookup, StageCheckpoint::new(2))
|
||||
.unwrap();
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
// At this point:
|
||||
// - RocksDB has tx hashes for blocks 0-5 (10 total: 2 per block)
|
||||
// - Checkpoint says we only processed up to block 2
|
||||
// - We need to prune tx hashes for blocks 3, 4, 5 (tx 6-9)
|
||||
|
||||
// Verify RocksDB has the expected number of entries before pruning
|
||||
let rocksdb_count_before: usize =
|
||||
rocksdb.iter::<tables::TransactionHashNumbers>().unwrap().count();
|
||||
assert_eq!(
|
||||
rocksdb_count_before, tx_count as usize,
|
||||
"RocksDB should have all {} transaction hashes before pruning",
|
||||
tx_count
|
||||
);
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// Verify we can fetch transactions by tx range
|
||||
let all_txs = provider.transactions_by_tx_range(0..tx_count).unwrap();
|
||||
assert_eq!(all_txs.len(), tx_count as usize, "Should be able to fetch all transactions");
|
||||
|
||||
// Verify the hashes match between what we stored and what we compute from fetched txs
|
||||
for (i, tx) in all_txs.iter().enumerate() {
|
||||
let computed_hash = tx.trie_hash();
|
||||
assert_eq!(
|
||||
computed_hash, tx_hashes[i],
|
||||
"Hash mismatch for tx {}: stored {:?} vs computed {:?}",
|
||||
i, tx_hashes[i], computed_hash
|
||||
);
|
||||
}
|
||||
|
||||
// Blocks 0, 1, 2 have 2 tx each = 6 tx total (indices 0-5)
|
||||
// We want to keep tx 0-5, prune tx 6-9
|
||||
let max_tx_to_keep = 5u64;
|
||||
let tx_to_prune_start = max_tx_to_keep + 1;
|
||||
|
||||
// Prune transactions 6-9 (blocks 3-5)
|
||||
rocksdb
|
||||
.prune_transaction_hash_numbers_in_range(&provider, tx_to_prune_start..=(tx_count - 1))
|
||||
.expect("prune should succeed");
|
||||
|
||||
// Verify: transactions 0-5 should remain, 6-9 should be pruned
|
||||
let mut remaining_count = 0;
|
||||
for result in rocksdb.iter::<tables::TransactionHashNumbers>().unwrap() {
|
||||
let (_hash, tx_num) = result.unwrap();
|
||||
assert!(
|
||||
tx_num <= max_tx_to_keep,
|
||||
"Transaction {} should have been pruned (> {})",
|
||||
tx_num,
|
||||
max_tx_to_keep
|
||||
);
|
||||
remaining_count += 1;
|
||||
}
|
||||
assert_eq!(
|
||||
remaining_count,
|
||||
(max_tx_to_keep + 1) as usize,
|
||||
"Should have {} transactions (0-{})",
|
||||
max_tx_to_keep + 1,
|
||||
max_tx_to_keep
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
//! [`RocksDBProvider`] implementation
|
||||
|
||||
mod invariants;
|
||||
mod metrics;
|
||||
mod provider;
|
||||
|
||||
pub use provider::{RocksDBBatch, RocksDBBuilder, RocksDBProvider, RocksTx};
|
||||
|
||||
@@ -380,6 +380,65 @@ impl RocksDBProvider {
|
||||
})
|
||||
}
|
||||
|
||||
/// Gets the first (smallest key) entry from the specified table.
|
||||
pub fn first<T: Table>(&self) -> ProviderResult<Option<(T::Key, T::Value)>> {
|
||||
self.execute_with_operation_metric(RocksDBOperation::Get, T::NAME, |this| {
|
||||
let cf = this.get_cf_handle::<T>()?;
|
||||
let mut iter = this.0.db.iterator_cf(cf, IteratorMode::Start);
|
||||
|
||||
match iter.next() {
|
||||
Some(Ok((key_bytes, value_bytes))) => {
|
||||
let key = <T::Key as reth_db_api::table::Decode>::decode(&key_bytes)
|
||||
.map_err(|_| ProviderError::Database(DatabaseError::Decode))?;
|
||||
let value = T::Value::decompress(&value_bytes)
|
||||
.map_err(|_| ProviderError::Database(DatabaseError::Decode))?;
|
||||
Ok(Some((key, value)))
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
Err(ProviderError::Database(DatabaseError::Read(DatabaseErrorInfo {
|
||||
message: e.to_string().into(),
|
||||
code: -1,
|
||||
})))
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Gets the last (largest key) entry from the specified table.
|
||||
pub fn last<T: Table>(&self) -> ProviderResult<Option<(T::Key, T::Value)>> {
|
||||
self.execute_with_operation_metric(RocksDBOperation::Get, T::NAME, |this| {
|
||||
let cf = this.get_cf_handle::<T>()?;
|
||||
let mut iter = this.0.db.iterator_cf(cf, IteratorMode::End);
|
||||
|
||||
match iter.next() {
|
||||
Some(Ok((key_bytes, value_bytes))) => {
|
||||
let key = <T::Key as reth_db_api::table::Decode>::decode(&key_bytes)
|
||||
.map_err(|_| ProviderError::Database(DatabaseError::Decode))?;
|
||||
let value = T::Value::decompress(&value_bytes)
|
||||
.map_err(|_| ProviderError::Database(DatabaseError::Decode))?;
|
||||
Ok(Some((key, value)))
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
Err(ProviderError::Database(DatabaseError::Read(DatabaseErrorInfo {
|
||||
message: e.to_string().into(),
|
||||
code: -1,
|
||||
})))
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Creates an iterator over all entries in the specified table.
|
||||
///
|
||||
/// Returns decoded `(Key, Value)` pairs in key order.
|
||||
pub fn iter<T: Table>(&self) -> ProviderResult<RocksDBIter<'_, T>> {
|
||||
let cf = self.get_cf_handle::<T>()?;
|
||||
let iter = self.0.db.iterator_cf(cf, IteratorMode::Start);
|
||||
Ok(RocksDBIter { inner: iter, _marker: std::marker::PhantomData })
|
||||
}
|
||||
|
||||
/// Writes a batch of operations atomically.
|
||||
pub fn write_batch<F>(&self, f: F) -> ProviderResult<()>
|
||||
where
|
||||
@@ -465,6 +524,11 @@ impl<'a> RocksDBBatch<'a> {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.inner.is_empty()
|
||||
}
|
||||
|
||||
/// Returns a reference to the underlying `RocksDB` provider.
|
||||
pub const fn provider(&self) -> &RocksDBProvider {
|
||||
self.provider
|
||||
}
|
||||
}
|
||||
|
||||
/// `RocksDB` transaction wrapper providing MDBX-like semantics.
|
||||
@@ -584,6 +648,50 @@ impl<'db> RocksTx<'db> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterator over a `RocksDB` table (non-transactional).
|
||||
///
|
||||
/// Yields decoded `(Key, Value)` pairs in key order.
|
||||
pub struct RocksDBIter<'db, T: Table> {
|
||||
inner: rocksdb::DBIteratorWithThreadMode<'db, TransactionDB>,
|
||||
_marker: std::marker::PhantomData<T>,
|
||||
}
|
||||
|
||||
impl<T: Table> fmt::Debug for RocksDBIter<'_, T> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("RocksDBIter").field("table", &T::NAME).finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Table> Iterator for RocksDBIter<'_, T> {
|
||||
type Item = ProviderResult<(T::Key, T::Value)>;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
let (key_bytes, value_bytes) = match self.inner.next()? {
|
||||
Ok(kv) => kv,
|
||||
Err(e) => {
|
||||
return Some(Err(ProviderError::Database(DatabaseError::Read(DatabaseErrorInfo {
|
||||
message: e.to_string().into(),
|
||||
code: -1,
|
||||
}))))
|
||||
}
|
||||
};
|
||||
|
||||
// Decode key
|
||||
let key = match <T::Key as reth_db_api::table::Decode>::decode(&key_bytes) {
|
||||
Ok(k) => k,
|
||||
Err(_) => return Some(Err(ProviderError::Database(DatabaseError::Decode))),
|
||||
};
|
||||
|
||||
// Decompress value
|
||||
let value = match T::Value::decompress(&value_bytes) {
|
||||
Ok(v) => v,
|
||||
Err(_) => return Some(Err(ProviderError::Database(DatabaseError::Decode))),
|
||||
};
|
||||
|
||||
Some(Ok((key, value)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterator over a `RocksDB` table within a transaction.
|
||||
///
|
||||
/// Yields decoded `(Key, Value)` pairs. Sees uncommitted writes.
|
||||
@@ -942,4 +1050,28 @@ mod tests {
|
||||
assert_eq!(provider.get::<TestTable>(i).unwrap(), Some(value));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_first_and_last_entry() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let provider =
|
||||
RocksDBBuilder::new(temp_dir.path()).with_table::<TestTable>().build().unwrap();
|
||||
|
||||
// Empty table should return None for both
|
||||
assert_eq!(provider.first::<TestTable>().unwrap(), None);
|
||||
assert_eq!(provider.last::<TestTable>().unwrap(), None);
|
||||
|
||||
// Insert some entries
|
||||
provider.put::<TestTable>(10, &b"value_10".to_vec()).unwrap();
|
||||
provider.put::<TestTable>(20, &b"value_20".to_vec()).unwrap();
|
||||
provider.put::<TestTable>(5, &b"value_5".to_vec()).unwrap();
|
||||
|
||||
// First should return the smallest key
|
||||
let first = provider.first::<TestTable>().unwrap();
|
||||
assert_eq!(first, Some((5, b"value_5".to_vec())));
|
||||
|
||||
// Last should return the largest key
|
||||
let last = provider.last::<TestTable>().unwrap();
|
||||
assert_eq!(last, Some((20, b"value_20".to_vec())));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,6 +151,23 @@ impl<N: NodePrimitives> StaticFileProviderBuilder<N> {
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the genesis block number for the [`StaticFileProvider`].
|
||||
///
|
||||
/// This configures the genesis block number, which is used to determine the starting point
|
||||
/// for block indexing and querying operations.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `genesis_block_number` - The block number of the genesis block.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Returns `Self` to allow method chaining.
|
||||
pub const fn with_genesis_block_number(mut self, genesis_block_number: u64) -> Self {
|
||||
self.inner.genesis_block_number = genesis_block_number;
|
||||
self
|
||||
}
|
||||
|
||||
/// Builds the final [`StaticFileProvider`] and initializes the index.
|
||||
pub fn build(self) -> ProviderResult<StaticFileProvider<N>> {
|
||||
let provider = StaticFileProvider(Arc::new(self.inner));
|
||||
@@ -308,6 +325,8 @@ pub struct StaticFileProviderInner<N> {
|
||||
blocks_per_file: HashMap<StaticFileSegment, u64>,
|
||||
/// Write lock for when access is [`StaticFileAccess::RW`].
|
||||
_lock_file: Option<StorageLock>,
|
||||
/// Genesis block number, default is 0;
|
||||
genesis_block_number: u64,
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives> StaticFileProviderInner<N> {
|
||||
@@ -334,6 +353,7 @@ impl<N: NodePrimitives> StaticFileProviderInner<N> {
|
||||
access,
|
||||
blocks_per_file,
|
||||
_lock_file,
|
||||
genesis_block_number: 0,
|
||||
};
|
||||
|
||||
Ok(provider)
|
||||
@@ -409,6 +429,11 @@ impl<N: NodePrimitives> StaticFileProviderInner<N> {
|
||||
block,
|
||||
)
|
||||
}
|
||||
|
||||
/// Get genesis block number
|
||||
pub const fn genesis_block_number(&self) -> u64 {
|
||||
self.genesis_block_number
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
@@ -1726,7 +1751,11 @@ impl<N: NodePrimitives> StaticFileWriter for StaticFileProvider<N> {
|
||||
&self,
|
||||
segment: StaticFileSegment,
|
||||
) -> ProviderResult<StaticFileProviderRWRefMut<'_, Self::Primitives>> {
|
||||
self.get_writer(self.get_highest_static_file_block(segment).unwrap_or_default(), segment)
|
||||
let genesis_number = self.0.as_ref().genesis_block_number();
|
||||
self.get_writer(
|
||||
self.get_highest_static_file_block(segment).unwrap_or(genesis_number),
|
||||
segment,
|
||||
)
|
||||
}
|
||||
|
||||
fn commit(&self) -> ProviderResult<()> {
|
||||
|
||||
@@ -363,8 +363,9 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
|
||||
.as_ref()
|
||||
.map(|block_range| block_range.end())
|
||||
.or_else(|| {
|
||||
(self.writer.user_header().expected_block_start() > 0)
|
||||
.then(|| self.writer.user_header().expected_block_start() - 1)
|
||||
(self.writer.user_header().expected_block_start() >
|
||||
self.reader().genesis_block_number())
|
||||
.then(|| self.writer.user_header().expected_block_start() - 1)
|
||||
});
|
||||
|
||||
self.reader().update_index(self.writer.user_header().segment(), segment_max_block)
|
||||
@@ -645,6 +646,37 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Appends header to static file without calling `increment_block`.
|
||||
/// This is useful for genesis blocks with non-zero block numbers.
|
||||
pub fn append_header_direct(
|
||||
&mut self,
|
||||
header: &N::BlockHeader,
|
||||
total_difficulty: U256,
|
||||
hash: &BlockHash,
|
||||
) -> ProviderResult<()>
|
||||
where
|
||||
N::BlockHeader: Compact,
|
||||
{
|
||||
let start = Instant::now();
|
||||
self.ensure_no_queued_prune()?;
|
||||
|
||||
debug_assert!(self.writer.user_header().segment() == StaticFileSegment::Headers);
|
||||
|
||||
self.append_column(header)?;
|
||||
self.append_column(CompactU256::from(total_difficulty))?;
|
||||
self.append_column(hash)?;
|
||||
|
||||
if let Some(metrics) = &self.metrics {
|
||||
metrics.record_segment_operation(
|
||||
StaticFileSegment::Headers,
|
||||
StaticFileProviderOperation::Append,
|
||||
Some(start.elapsed()),
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Appends transaction to static file.
|
||||
///
|
||||
/// It **DOES NOT CALL** `increment_block()`, it should be handled elsewhere. There might be
|
||||
|
||||
@@ -59,6 +59,73 @@ impl DiskFileBlobStore {
|
||||
fn clear_cache(&self) {
|
||||
self.inner.blob_cache.lock().clear()
|
||||
}
|
||||
|
||||
/// Look up EIP-7594 blobs by their versioned hashes.
|
||||
///
|
||||
/// This returns a result vector with the **same length and order** as the input
|
||||
/// `versioned_hashes`. Each element is `Some(BlobAndProofV2)` if the blob is available, or
|
||||
/// `None` if it is missing or an older sidecar version.
|
||||
///
|
||||
/// The lookup first scans the in-memory cache and, if not all blobs are found, falls back to
|
||||
/// reading candidate sidecars from disk using the `versioned_hash -> tx_hash` index.
|
||||
fn get_by_versioned_hashes_eip7594(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Vec<Option<BlobAndProofV2>>, BlobStoreError> {
|
||||
// we must return the blobs in order but we don't necessarily find them in the requested
|
||||
// order
|
||||
let mut result = vec![None; versioned_hashes.len()];
|
||||
|
||||
// first scan all cached full sidecars
|
||||
for (_tx_hash, blob_sidecar) in self.inner.blob_cache.lock().iter() {
|
||||
if let Some(blob_sidecar) = blob_sidecar.as_eip7594() {
|
||||
for (hash_idx, match_result) in
|
||||
blob_sidecar.match_versioned_hashes(versioned_hashes)
|
||||
{
|
||||
result[hash_idx] = Some(match_result);
|
||||
}
|
||||
}
|
||||
|
||||
// return early if all blobs are found.
|
||||
if result.iter().all(|blob| blob.is_some()) {
|
||||
return Ok(result);
|
||||
}
|
||||
}
|
||||
|
||||
// not all versioned hashes were found, try to look up a matching tx
|
||||
let mut missing_tx_hashes = Vec::new();
|
||||
|
||||
{
|
||||
let mut versioned_to_txhashes = self.inner.versioned_hashes_to_txhash.lock();
|
||||
for (idx, _) in
|
||||
result.iter().enumerate().filter(|(_, blob_and_proof)| blob_and_proof.is_none())
|
||||
{
|
||||
// this is safe because the result vec has the same len
|
||||
let versioned_hash = versioned_hashes[idx];
|
||||
if let Some(tx_hash) = versioned_to_txhashes.get(&versioned_hash).copied() {
|
||||
missing_tx_hashes.push(tx_hash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if we have missing blobs, try to read them from disk and try again
|
||||
if !missing_tx_hashes.is_empty() {
|
||||
let blobs_from_disk = self.inner.read_many_decoded(missing_tx_hashes);
|
||||
for (_, blob_sidecar) in blobs_from_disk {
|
||||
if let Some(blob_sidecar) = blob_sidecar.as_eip7594() {
|
||||
for (hash_idx, match_result) in
|
||||
blob_sidecar.match_versioned_hashes(versioned_hashes)
|
||||
{
|
||||
if result[hash_idx].is_none() {
|
||||
result[hash_idx] = Some(match_result);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
impl BlobStore for DiskFileBlobStore {
|
||||
@@ -205,58 +272,7 @@ impl BlobStore for DiskFileBlobStore {
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Option<Vec<BlobAndProofV2>>, BlobStoreError> {
|
||||
// we must return the blobs in order but we don't necessarily find them in the requested
|
||||
// order
|
||||
let mut result = vec![None; versioned_hashes.len()];
|
||||
|
||||
// first scan all cached full sidecars
|
||||
for (_tx_hash, blob_sidecar) in self.inner.blob_cache.lock().iter() {
|
||||
if let Some(blob_sidecar) = blob_sidecar.as_eip7594() {
|
||||
for (hash_idx, match_result) in
|
||||
blob_sidecar.match_versioned_hashes(versioned_hashes)
|
||||
{
|
||||
result[hash_idx] = Some(match_result);
|
||||
}
|
||||
}
|
||||
|
||||
// return early if all blobs are found.
|
||||
if result.iter().all(|blob| blob.is_some()) {
|
||||
// got all blobs, can return early
|
||||
return Ok(Some(result.into_iter().map(Option::unwrap).collect()))
|
||||
}
|
||||
}
|
||||
|
||||
// not all versioned hashes were found, try to look up a matching tx
|
||||
let mut missing_tx_hashes = Vec::new();
|
||||
|
||||
{
|
||||
let mut versioned_to_txhashes = self.inner.versioned_hashes_to_txhash.lock();
|
||||
for (idx, _) in
|
||||
result.iter().enumerate().filter(|(_, blob_and_proof)| blob_and_proof.is_none())
|
||||
{
|
||||
// this is safe because the result vec has the same len
|
||||
let versioned_hash = versioned_hashes[idx];
|
||||
if let Some(tx_hash) = versioned_to_txhashes.get(&versioned_hash).copied() {
|
||||
missing_tx_hashes.push(tx_hash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if we have missing blobs, try to read them from disk and try again
|
||||
if !missing_tx_hashes.is_empty() {
|
||||
let blobs_from_disk = self.inner.read_many_decoded(missing_tx_hashes);
|
||||
for (_, blob_sidecar) in blobs_from_disk {
|
||||
if let Some(blob_sidecar) = blob_sidecar.as_eip7594() {
|
||||
for (hash_idx, match_result) in
|
||||
blob_sidecar.match_versioned_hashes(versioned_hashes)
|
||||
{
|
||||
if result[hash_idx].is_none() {
|
||||
result[hash_idx] = Some(match_result);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let result = self.get_by_versioned_hashes_eip7594(versioned_hashes)?;
|
||||
|
||||
// only return the blobs if we found all requested versioned hashes
|
||||
if result.iter().all(|blob| blob.is_some()) {
|
||||
@@ -266,6 +282,13 @@ impl BlobStore for DiskFileBlobStore {
|
||||
}
|
||||
}
|
||||
|
||||
fn get_by_versioned_hashes_v3(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Vec<Option<BlobAndProofV2>>, BlobStoreError> {
|
||||
self.get_by_versioned_hashes_eip7594(versioned_hashes)
|
||||
}
|
||||
|
||||
fn data_size_hint(&self) -> Option<usize> {
|
||||
Some(self.inner.size_tracker.data_size())
|
||||
}
|
||||
@@ -656,7 +679,12 @@ pub enum OpenDiskFileBlobStore {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use alloy_consensus::BlobTransactionSidecar;
|
||||
use alloy_eips::eip7594::BlobTransactionSidecarVariant;
|
||||
use alloy_eips::{
|
||||
eip4844::{kzg_to_versioned_hash, Blob, BlobAndProofV2, Bytes48},
|
||||
eip7594::{
|
||||
BlobTransactionSidecarEip7594, BlobTransactionSidecarVariant, CELLS_PER_EXT_BLOB,
|
||||
},
|
||||
};
|
||||
|
||||
use super::*;
|
||||
use std::sync::atomic::Ordering;
|
||||
@@ -682,6 +710,20 @@ mod tests {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn eip7594_single_blob_sidecar() -> (BlobTransactionSidecarVariant, B256, BlobAndProofV2) {
|
||||
let blob = Blob::default();
|
||||
let commitment = Bytes48::default();
|
||||
let cell_proofs = vec![Bytes48::default(); CELLS_PER_EXT_BLOB];
|
||||
|
||||
let versioned_hash = kzg_to_versioned_hash(commitment.as_slice());
|
||||
|
||||
let expected =
|
||||
BlobAndProofV2 { blob: Box::new(Blob::default()), proofs: cell_proofs.clone() };
|
||||
let sidecar = BlobTransactionSidecarEip7594::new(vec![blob], vec![commitment], cell_proofs);
|
||||
|
||||
(BlobTransactionSidecarVariant::Eip7594(sidecar), versioned_hash, expected)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_insert_all_get_all() {
|
||||
let (store, _dir) = tmp_store();
|
||||
@@ -851,4 +893,33 @@ mod tests {
|
||||
assert_eq!(stat.delete_succeed, 3);
|
||||
assert_eq!(stat.delete_failed, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_get_blobs_v3_returns_partial_results() {
|
||||
let (store, _dir) = tmp_store();
|
||||
|
||||
let (sidecar, versioned_hash, expected) = eip7594_single_blob_sidecar();
|
||||
store.insert(TxHash::random(), sidecar).unwrap();
|
||||
|
||||
assert_ne!(versioned_hash, B256::ZERO);
|
||||
|
||||
let request = vec![versioned_hash, B256::ZERO];
|
||||
let v2 = store.get_by_versioned_hashes_v2(&request).unwrap();
|
||||
assert!(v2.is_none(), "v2 must return null if any requested blob is missing");
|
||||
|
||||
let v3 = store.get_by_versioned_hashes_v3(&request).unwrap();
|
||||
assert_eq!(v3, vec![Some(expected), None]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_get_blobs_v3_can_fallback_to_disk() {
|
||||
let (store, _dir) = tmp_store();
|
||||
|
||||
let (sidecar, versioned_hash, expected) = eip7594_single_blob_sidecar();
|
||||
store.insert(TxHash::random(), sidecar).unwrap();
|
||||
store.clear_cache();
|
||||
|
||||
let v3 = store.get_by_versioned_hashes_v3(&[versioned_hash]).unwrap();
|
||||
assert_eq!(v3, vec![Some(expected)]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,35 @@ pub struct InMemoryBlobStore {
|
||||
inner: Arc<InMemoryBlobStoreInner>,
|
||||
}
|
||||
|
||||
impl InMemoryBlobStore {
|
||||
/// Look up EIP-7594 blobs by their versioned hashes.
|
||||
///
|
||||
/// This returns a result vector with the **same length and order** as the input
|
||||
/// `versioned_hashes`. Each element is `Some(BlobAndProofV2)` if the blob is available, or
|
||||
/// `None` if it is missing or an older sidecar version.
|
||||
fn get_by_versioned_hashes_eip7594(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Vec<Option<BlobAndProofV2>> {
|
||||
let mut result = vec![None; versioned_hashes.len()];
|
||||
for (_tx_hash, blob_sidecar) in self.inner.store.read().iter() {
|
||||
if let Some(blob_sidecar) = blob_sidecar.as_eip7594() {
|
||||
for (hash_idx, match_result) in
|
||||
blob_sidecar.match_versioned_hashes(versioned_hashes)
|
||||
{
|
||||
result[hash_idx] = Some(match_result);
|
||||
}
|
||||
}
|
||||
|
||||
// Return early if all blobs are found.
|
||||
if result.iter().all(|blob| blob.is_some()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct InMemoryBlobStoreInner {
|
||||
/// Storage for all blob data.
|
||||
@@ -134,20 +163,7 @@ impl BlobStore for InMemoryBlobStore {
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Option<Vec<BlobAndProofV2>>, BlobStoreError> {
|
||||
let mut result = vec![None; versioned_hashes.len()];
|
||||
for (_tx_hash, blob_sidecar) in self.inner.store.read().iter() {
|
||||
if let Some(blob_sidecar) = blob_sidecar.as_eip7594() {
|
||||
for (hash_idx, match_result) in
|
||||
blob_sidecar.match_versioned_hashes(versioned_hashes)
|
||||
{
|
||||
result[hash_idx] = Some(match_result);
|
||||
}
|
||||
}
|
||||
|
||||
if result.iter().all(|blob| blob.is_some()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let result = self.get_by_versioned_hashes_eip7594(versioned_hashes);
|
||||
if result.iter().all(|blob| blob.is_some()) {
|
||||
Ok(Some(result.into_iter().map(Option::unwrap).collect()))
|
||||
} else {
|
||||
@@ -155,6 +171,13 @@ impl BlobStore for InMemoryBlobStore {
|
||||
}
|
||||
}
|
||||
|
||||
fn get_by_versioned_hashes_v3(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Vec<Option<BlobAndProofV2>>, BlobStoreError> {
|
||||
Ok(self.get_by_versioned_hashes_eip7594(versioned_hashes))
|
||||
}
|
||||
|
||||
fn data_size_hint(&self) -> Option<usize> {
|
||||
Some(self.inner.size_tracker.data_size())
|
||||
}
|
||||
@@ -183,3 +206,45 @@ fn insert_size(
|
||||
store.insert(tx, Arc::new(blob));
|
||||
add
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_eips::{
|
||||
eip4844::{kzg_to_versioned_hash, Blob, BlobAndProofV2, Bytes48},
|
||||
eip7594::{
|
||||
BlobTransactionSidecarEip7594, BlobTransactionSidecarVariant, CELLS_PER_EXT_BLOB,
|
||||
},
|
||||
};
|
||||
|
||||
fn eip7594_single_blob_sidecar() -> (BlobTransactionSidecarVariant, B256, BlobAndProofV2) {
|
||||
let blob = Blob::default();
|
||||
let commitment = Bytes48::default();
|
||||
let cell_proofs = vec![Bytes48::default(); CELLS_PER_EXT_BLOB];
|
||||
|
||||
let versioned_hash = kzg_to_versioned_hash(commitment.as_slice());
|
||||
|
||||
let expected =
|
||||
BlobAndProofV2 { blob: Box::new(Blob::default()), proofs: cell_proofs.clone() };
|
||||
let sidecar = BlobTransactionSidecarEip7594::new(vec![blob], vec![commitment], cell_proofs);
|
||||
|
||||
(BlobTransactionSidecarVariant::Eip7594(sidecar), versioned_hash, expected)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mem_get_blobs_v3_returns_partial_results() {
|
||||
let store = InMemoryBlobStore::default();
|
||||
|
||||
let (sidecar, versioned_hash, expected) = eip7594_single_blob_sidecar();
|
||||
store.insert(B256::random(), sidecar).unwrap();
|
||||
|
||||
assert_ne!(versioned_hash, B256::ZERO);
|
||||
|
||||
let request = vec![versioned_hash, B256::ZERO];
|
||||
let v2 = store.get_by_versioned_hashes_v2(&request).unwrap();
|
||||
assert!(v2.is_none(), "v2 must return null if any requested blob is missing");
|
||||
|
||||
let v3 = store.get_by_versioned_hashes_v3(&request).unwrap();
|
||||
assert_eq!(v3, vec![Some(expected), None]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,6 +100,15 @@ pub trait BlobStore: fmt::Debug + Send + Sync + 'static {
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Option<Vec<BlobAndProofV2>>, BlobStoreError>;
|
||||
|
||||
/// Return the [`BlobAndProofV2`]s for a list of blob versioned hashes.
|
||||
///
|
||||
/// The response is always the same length as the request. Missing or older-version blobs are
|
||||
/// returned as `None` elements.
|
||||
fn get_by_versioned_hashes_v3(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Vec<Option<BlobAndProofV2>>, BlobStoreError>;
|
||||
|
||||
/// Data size of all transactions in the blob store.
|
||||
fn data_size_hint(&self) -> Option<usize>;
|
||||
|
||||
|
||||
@@ -78,6 +78,13 @@ impl BlobStore for NoopBlobStore {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn get_by_versioned_hashes_v3(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Vec<Option<BlobAndProofV2>>, BlobStoreError> {
|
||||
Ok(vec![None; versioned_hashes.len()])
|
||||
}
|
||||
|
||||
fn data_size_hint(&self) -> Option<usize> {
|
||||
Some(0)
|
||||
}
|
||||
|
||||
@@ -751,6 +751,13 @@ where
|
||||
) -> Result<Option<Vec<BlobAndProofV2>>, BlobStoreError> {
|
||||
self.pool.blob_store().get_by_versioned_hashes_v2(versioned_hashes)
|
||||
}
|
||||
|
||||
fn get_blobs_for_versioned_hashes_v3(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Vec<Option<BlobAndProofV2>>, BlobStoreError> {
|
||||
self.pool.blob_store().get_by_versioned_hashes_v3(versioned_hashes)
|
||||
}
|
||||
}
|
||||
|
||||
impl<V, T, S> TransactionPoolExt for Pool<V, T, S>
|
||||
|
||||
@@ -345,6 +345,13 @@ impl<T: EthPoolTransaction> TransactionPool for NoopTransactionPool<T> {
|
||||
) -> Result<Option<Vec<BlobAndProofV2>>, BlobStoreError> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn get_blobs_for_versioned_hashes_v3(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Vec<Option<BlobAndProofV2>>, BlobStoreError> {
|
||||
Ok(vec![None; versioned_hashes.len()])
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`TransactionValidator`] that does nothing.
|
||||
|
||||
@@ -638,6 +638,15 @@ pub trait TransactionPool: Clone + Debug + Send + Sync {
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Option<Vec<BlobAndProofV2>>, BlobStoreError>;
|
||||
|
||||
/// Return the [`BlobAndProofV2`]s for a list of blob versioned hashes.
|
||||
///
|
||||
/// The response is always the same length as the request. Missing or older-version blobs are
|
||||
/// returned as `None` elements.
|
||||
fn get_blobs_for_versioned_hashes_v3(
|
||||
&self,
|
||||
versioned_hashes: &[B256],
|
||||
) -> Result<Vec<Option<BlobAndProofV2>>, BlobStoreError>;
|
||||
}
|
||||
|
||||
/// Extension for [`TransactionPool`] trait that allows to set the current block info.
|
||||
|
||||
@@ -36,6 +36,9 @@ itertools.workspace = true
|
||||
tokio = { workspace = true, features = ["rt-multi-thread"] }
|
||||
crossbeam-channel.workspace = true
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
libc = "0.2"
|
||||
|
||||
# `metrics` feature
|
||||
reth-metrics = { workspace = true, optional = true }
|
||||
metrics = { workspace = true, optional = true }
|
||||
|
||||
@@ -262,7 +262,6 @@ mod tests {
|
||||
use reth_provider::{test_utils::create_test_provider_factory, HashingWriter};
|
||||
use reth_trie::proof::Proof;
|
||||
use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory};
|
||||
use tokio::runtime::Runtime;
|
||||
|
||||
#[test]
|
||||
fn random_parallel_proof() {
|
||||
@@ -326,7 +325,7 @@ mod tests {
|
||||
let trie_cursor_factory = DatabaseTrieCursorFactory::new(provider_rw.tx_ref());
|
||||
let hashed_cursor_factory = DatabaseHashedCursorFactory::new(provider_rw.tx_ref());
|
||||
|
||||
let rt = Runtime::new().unwrap();
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
|
||||
let factory = reth_provider::providers::OverlayStateProviderFactory::new(factory);
|
||||
let task_ctx = ProofTaskCtx::new(factory);
|
||||
|
||||
@@ -71,6 +71,33 @@ use std::{
|
||||
use tokio::runtime::Handle;
|
||||
use tracing::{debug, debug_span, error, trace};
|
||||
|
||||
/// Sets the current thread's name for profiling visibility.
|
||||
#[inline]
|
||||
fn set_thread_name(name: &str) {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
// SAFETY: prctl with PR_SET_NAME is safe with a valid string pointer
|
||||
unsafe {
|
||||
let mut buf = [0u8; 16];
|
||||
let len = name.len().min(15);
|
||||
buf[..len].copy_from_slice(&name.as_bytes()[..len]);
|
||||
libc::prctl(libc::PR_SET_NAME, buf.as_ptr());
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
// SAFETY: pthread_setname_np is safe with a valid CString
|
||||
unsafe {
|
||||
let c_name = std::ffi::CString::new(name).unwrap_or_default();
|
||||
libc::pthread_setname_np(c_name.as_ptr());
|
||||
}
|
||||
}
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
{
|
||||
let _ = name;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
use crate::proof_task_metrics::{
|
||||
ProofTaskCursorMetrics, ProofTaskCursorMetricsCache, ProofTaskTrieMetrics,
|
||||
@@ -151,6 +178,7 @@ impl ProofWorkerHandle {
|
||||
let storage_available_workers_clone = storage_available_workers.clone();
|
||||
|
||||
executor.spawn_blocking(move || {
|
||||
set_thread_name("reth-stor-proof");
|
||||
#[cfg(feature = "metrics")]
|
||||
let metrics = ProofTaskTrieMetrics::default();
|
||||
#[cfg(feature = "metrics")]
|
||||
@@ -191,6 +219,7 @@ impl ProofWorkerHandle {
|
||||
let account_available_workers_clone = account_available_workers.clone();
|
||||
|
||||
executor.spawn_blocking(move || {
|
||||
set_thread_name("reth-acct-proof");
|
||||
#[cfg(feature = "metrics")]
|
||||
let metrics = ProofTaskTrieMetrics::default();
|
||||
#[cfg(feature = "metrics")]
|
||||
@@ -1588,7 +1617,7 @@ enum AccountWorkerJob {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use reth_provider::test_utils::create_test_provider_factory;
|
||||
use tokio::{runtime::Builder, task};
|
||||
use tokio::runtime::Builder;
|
||||
|
||||
fn test_ctx<Factory>(factory: Factory) -> ProofTaskCtx<Factory> {
|
||||
ProofTaskCtx::new(factory)
|
||||
@@ -1598,21 +1627,16 @@ mod tests {
|
||||
#[test]
|
||||
fn spawn_proof_workers_creates_handle() {
|
||||
let runtime = Builder::new_multi_thread().worker_threads(1).enable_all().build().unwrap();
|
||||
runtime.block_on(async {
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
let provider_factory = create_test_provider_factory();
|
||||
let factory =
|
||||
reth_provider::providers::OverlayStateProviderFactory::new(provider_factory);
|
||||
let ctx = test_ctx(factory);
|
||||
let provider_factory = create_test_provider_factory();
|
||||
let factory = reth_provider::providers::OverlayStateProviderFactory::new(provider_factory);
|
||||
let ctx = test_ctx(factory);
|
||||
|
||||
let proof_handle = ProofWorkerHandle::new(handle.clone(), ctx, 5, 3);
|
||||
let proof_handle = ProofWorkerHandle::new(runtime.handle().clone(), ctx, 5, 3);
|
||||
|
||||
// Verify handle can be cloned
|
||||
let _cloned_handle = proof_handle.clone();
|
||||
// Verify handle can be cloned
|
||||
let _cloned_handle = proof_handle.clone();
|
||||
|
||||
// Workers shut down automatically when handle is dropped
|
||||
drop(proof_handle);
|
||||
task::yield_now().await;
|
||||
});
|
||||
// Workers shut down automatically when handle is dropped
|
||||
drop(proof_handle);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
# reth debug
|
||||
|
||||
Various debug routines
|
||||
|
||||
```bash
|
||||
$ reth debug --help
|
||||
```
|
||||
```txt
|
||||
Usage: reth debug [OPTIONS] <COMMAND>
|
||||
|
||||
Commands:
|
||||
merkle Debug the clean & incremental state root calculations
|
||||
in-memory-merkle Debug in-memory state root calculation
|
||||
help Print this message or the help of the given subcommand(s)
|
||||
|
||||
Options:
|
||||
-h, --help
|
||||
Print help (see a summary with '-h')
|
||||
|
||||
Logging:
|
||||
--log.stdout.format <FORMAT>
|
||||
The format to use for logs written to stdout
|
||||
|
||||
[default: terminal]
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
--log.stdout.filter <FILTER>
|
||||
The filter to use for logs written to stdout
|
||||
|
||||
[default: ]
|
||||
|
||||
--log.file.format <FORMAT>
|
||||
The format to use for logs written to the log file
|
||||
|
||||
[default: terminal]
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
--log.file.filter <FILTER>
|
||||
The filter to use for logs written to the log file
|
||||
|
||||
[default: debug]
|
||||
|
||||
--log.file.directory <PATH>
|
||||
The path to put log files in
|
||||
|
||||
[default: <CACHE_DIR>/logs]
|
||||
|
||||
--log.file.max-size <SIZE>
|
||||
The maximum size (in MB) of one log file
|
||||
|
||||
[default: 200]
|
||||
|
||||
--log.file.max-files <COUNT>
|
||||
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
|
||||
|
||||
[default: 5]
|
||||
|
||||
--log.journald
|
||||
Write logs to journald
|
||||
|
||||
--log.journald.filter <FILTER>
|
||||
The filter to use for logs written to journald
|
||||
|
||||
[default: error]
|
||||
|
||||
--color <COLOR>
|
||||
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
|
||||
|
||||
[default: always]
|
||||
|
||||
Possible values:
|
||||
- always: Colors on
|
||||
- auto: Auto-detect
|
||||
- never: Colors off
|
||||
|
||||
Display:
|
||||
-v, --verbosity...
|
||||
Set the minimum log level.
|
||||
|
||||
-v Errors
|
||||
-vv Warnings
|
||||
-vvv Info
|
||||
-vvvv Debug
|
||||
-vvvvv Traces (warning: very verbose!)
|
||||
|
||||
-q, --quiet
|
||||
Silence all log output
|
||||
```
|
||||
@@ -1,100 +0,0 @@
|
||||
# reth recover
|
||||
|
||||
Scripts for node recovery
|
||||
|
||||
```bash
|
||||
$ reth recover --help
|
||||
```
|
||||
```txt
|
||||
Usage: reth recover [OPTIONS] <COMMAND>
|
||||
|
||||
Commands:
|
||||
storage-tries Recover the node by deleting dangling storage tries
|
||||
help Print this message or the help of the given subcommand(s)
|
||||
|
||||
Options:
|
||||
-h, --help
|
||||
Print help (see a summary with '-h')
|
||||
|
||||
Logging:
|
||||
--log.stdout.format <FORMAT>
|
||||
The format to use for logs written to stdout
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.stdout.filter <FILTER>
|
||||
The filter to use for logs written to stdout
|
||||
|
||||
[default: ]
|
||||
|
||||
--log.file.format <FORMAT>
|
||||
The format to use for logs written to the log file
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.file.filter <FILTER>
|
||||
The filter to use for logs written to the log file
|
||||
|
||||
[default: debug]
|
||||
|
||||
--log.file.directory <PATH>
|
||||
The path to put log files in
|
||||
|
||||
[default: <CACHE_DIR>/logs]
|
||||
|
||||
--log.file.name <NAME>
|
||||
The prefix name of the log files
|
||||
|
||||
[default: reth.log]
|
||||
|
||||
--log.file.max-size <SIZE>
|
||||
The maximum size (in MB) of one log file
|
||||
|
||||
[default: 200]
|
||||
|
||||
--log.file.max-files <COUNT>
|
||||
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
|
||||
|
||||
[default: 5]
|
||||
|
||||
--log.journald
|
||||
Write logs to journald
|
||||
|
||||
--log.journald.filter <FILTER>
|
||||
The filter to use for logs written to journald
|
||||
|
||||
[default: error]
|
||||
|
||||
--color <COLOR>
|
||||
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
|
||||
|
||||
Possible values:
|
||||
- always: Colors on
|
||||
- auto: Auto-detect
|
||||
- never: Colors off
|
||||
|
||||
[default: always]
|
||||
|
||||
Display:
|
||||
-v, --verbosity...
|
||||
Set the minimum log level.
|
||||
|
||||
-v Errors
|
||||
-vv Warnings
|
||||
-vvv Info
|
||||
-vvvv Debug
|
||||
-vvvvv Traces (warning: very verbose!)
|
||||
|
||||
-q, --quiet
|
||||
Silence all log output
|
||||
```
|
||||
@@ -1,154 +0,0 @@
|
||||
# reth recover storage-tries
|
||||
|
||||
Recover the node by deleting dangling storage tries
|
||||
|
||||
```bash
|
||||
$ reth recover storage-tries --help
|
||||
```
|
||||
```txt
|
||||
Usage: reth recover storage-tries [OPTIONS]
|
||||
|
||||
Options:
|
||||
-h, --help
|
||||
Print help (see a summary with '-h')
|
||||
|
||||
Datadir:
|
||||
--datadir <DATA_DIR>
|
||||
The path to the data dir for all reth files and subdirectories.
|
||||
|
||||
Defaults to the OS-specific data directory:
|
||||
|
||||
- Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/`
|
||||
- Windows: `{FOLDERID_RoamingAppData}/reth/`
|
||||
- macOS: `$HOME/Library/Application Support/reth/`
|
||||
|
||||
[default: default]
|
||||
|
||||
--datadir.static-files <PATH>
|
||||
The absolute path to store static files in.
|
||||
|
||||
--config <FILE>
|
||||
The path to the configuration file to use
|
||||
|
||||
--chain <CHAIN_OR_PATH>
|
||||
The chain this node is running.
|
||||
Possible values are either a built-in chain or the path to a chain specification file.
|
||||
|
||||
Built-in chains:
|
||||
mainnet, sepolia, holesky, hoodi, dev
|
||||
|
||||
[default: mainnet]
|
||||
|
||||
Database:
|
||||
--db.log-level <LOG_LEVEL>
|
||||
Database logging level. Levels higher than "notice" require a debug build
|
||||
|
||||
Possible values:
|
||||
- fatal: Enables logging for critical conditions, i.e. assertion failures
|
||||
- error: Enables logging for error conditions
|
||||
- warn: Enables logging for warning conditions
|
||||
- notice: Enables logging for normal but significant condition
|
||||
- verbose: Enables logging for verbose informational
|
||||
- debug: Enables logging for debug-level messages
|
||||
- trace: Enables logging for trace debug-level messages
|
||||
- extra: Enables logging for extra debug-level messages
|
||||
|
||||
--db.exclusive <EXCLUSIVE>
|
||||
Open environment in exclusive/monopolistic mode. Makes it possible to open a database on an NFS volume
|
||||
|
||||
[possible values: true, false]
|
||||
|
||||
--db.max-size <MAX_SIZE>
|
||||
Maximum database size (e.g., 4TB, 8MB)
|
||||
|
||||
--db.growth-step <GROWTH_STEP>
|
||||
Database growth step (e.g., 4GB, 4KB)
|
||||
|
||||
--db.read-transaction-timeout <READ_TRANSACTION_TIMEOUT>
|
||||
Read transaction timeout in seconds, 0 means no timeout
|
||||
|
||||
--db.max-readers <MAX_READERS>
|
||||
Maximum number of readers allowed to access the database concurrently
|
||||
|
||||
Logging:
|
||||
--log.stdout.format <FORMAT>
|
||||
The format to use for logs written to stdout
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.stdout.filter <FILTER>
|
||||
The filter to use for logs written to stdout
|
||||
|
||||
[default: ]
|
||||
|
||||
--log.file.format <FORMAT>
|
||||
The format to use for logs written to the log file
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.file.filter <FILTER>
|
||||
The filter to use for logs written to the log file
|
||||
|
||||
[default: debug]
|
||||
|
||||
--log.file.directory <PATH>
|
||||
The path to put log files in
|
||||
|
||||
[default: <CACHE_DIR>/logs]
|
||||
|
||||
--log.file.name <NAME>
|
||||
The prefix name of the log files
|
||||
|
||||
[default: reth.log]
|
||||
|
||||
--log.file.max-size <SIZE>
|
||||
The maximum size (in MB) of one log file
|
||||
|
||||
[default: 200]
|
||||
|
||||
--log.file.max-files <COUNT>
|
||||
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled
|
||||
|
||||
[default: 5]
|
||||
|
||||
--log.journald
|
||||
Write logs to journald
|
||||
|
||||
--log.journald.filter <FILTER>
|
||||
The filter to use for logs written to journald
|
||||
|
||||
[default: error]
|
||||
|
||||
--color <COLOR>
|
||||
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
|
||||
|
||||
Possible values:
|
||||
- always: Colors on
|
||||
- auto: Auto-detect
|
||||
- never: Colors off
|
||||
|
||||
[default: always]
|
||||
|
||||
Display:
|
||||
-v, --verbosity...
|
||||
Set the minimum log level.
|
||||
|
||||
-v Errors
|
||||
-vv Warnings
|
||||
-vvv Info
|
||||
-vvvv Debug
|
||||
-vvvvv Traces (warning: very verbose!)
|
||||
|
||||
-q, --quiet
|
||||
Silence all log output
|
||||
```
|
||||
@@ -24,6 +24,13 @@ sequenceDiagram
|
||||
ExEx->>ExEx: Rollback & Re-process
|
||||
ExEx->>Reth: New FinishedHeight Event
|
||||
deactivate ExEx
|
||||
|
||||
Note over Reth,ExEx: Revert Flow
|
||||
Reth->>ExEx: ChainRevert Notification
|
||||
activate ExEx
|
||||
ExEx->>ExEx: Rollback & Re-process
|
||||
ExEx->>Reth: New FinishedHeight Event
|
||||
deactivate ExEx
|
||||
```
|
||||
|
||||
ExExes are just [Futures](https://doc.rust-lang.org/std/future/trait.Future.html) that run indefinitely alongside Reth
|
||||
|
||||
Reference in New Issue
Block a user