diff --git a/Cargo.lock b/Cargo.lock index 31bf89d379..8cc2402d31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5760,6 +5760,7 @@ dependencies = [ "alloy-chains", "alloy-rlp", "aquamarine", + "assert_matches", "backon", "clap", "comfy-table", @@ -5774,6 +5775,7 @@ dependencies = [ "itertools 0.12.0", "jemalloc-ctl", "jemallocator", + "jsonrpsee", "metrics", "metrics-exporter-prometheus", "metrics-process", diff --git a/crates/blockchain-tree/src/blockchain_tree.rs b/crates/blockchain-tree/src/blockchain_tree.rs index 0aa5333432..3d1f69fbca 100644 --- a/crates/blockchain-tree/src/blockchain_tree.rs +++ b/crates/blockchain-tree/src/blockchain_tree.rs @@ -1174,7 +1174,13 @@ impl BlockchainTree { } None => { debug!(target: "blockchain_tree", blocks = ?block_hash_numbers, "Recomputing state root for insert"); - let provider = self.externals.provider_factory.provider()?; + let provider = self + .externals + .provider_factory + .provider()? + // State root calculation can take a while, and we're sure no write transaction + // will be open in parallel. See https://github.com/paradigmxyz/reth/issues/6168. + .disable_backtrace_on_long_read_transaction(); let (state_root, trie_updates) = hashed_state .state_root_with_updates(provider.tx_ref()) .map_err(Into::::into)?; diff --git a/crates/storage/db/src/abstraction/mock.rs b/crates/storage/db/src/abstraction/mock.rs index cb20105add..25e8cd0249 100644 --- a/crates/storage/db/src/abstraction/mock.rs +++ b/crates/storage/db/src/abstraction/mock.rs @@ -66,6 +66,8 @@ impl DbTx for TxMock { fn entries(&self) -> Result { Ok(self._table.len()) } + + fn disable_backtrace_on_long_read_transaction(&mut self) {} } impl DbTxMut for TxMock { diff --git a/crates/storage/db/src/abstraction/transaction.rs b/crates/storage/db/src/abstraction/transaction.rs index 472563f352..c2b129afa2 100644 --- a/crates/storage/db/src/abstraction/transaction.rs +++ b/crates/storage/db/src/abstraction/transaction.rs @@ -24,6 +24,8 @@ pub trait DbTx: Send + Sync { fn cursor_dup_read(&self) -> Result, DatabaseError>; /// Returns number of entries in the table. fn entries(&self) -> Result; + /// Disables backtrace recording for this read transaction when it's open for too long. + fn disable_backtrace_on_long_read_transaction(&mut self); } /// Read write transaction that allows writing to database diff --git a/crates/storage/db/src/implementation/mdbx/tx.rs b/crates/storage/db/src/implementation/mdbx/tx.rs index 55ebf85e8b..be69902882 100644 --- a/crates/storage/db/src/implementation/mdbx/tx.rs +++ b/crates/storage/db/src/implementation/mdbx/tx.rs @@ -156,6 +156,9 @@ struct MetricsHandler { /// If `true`, the metric about transaction closing has already been recorded and we don't need /// to do anything on [Drop::drop]. close_recorded: bool, + /// If `true`, the backtrace of transaction will be recorded and logged. + /// See [MetricsHandler::log_backtrace_on_long_read_transaction]. + record_backtrace: bool, /// If `true`, the backtrace of transaction has already been recorded and logged. /// See [MetricsHandler::log_backtrace_on_long_read_transaction]. backtrace_recorded: AtomicBool, @@ -168,6 +171,7 @@ impl MetricsHandler { txn_id, start: Instant::now(), close_recorded: false, + record_backtrace: true, backtrace_recorded: AtomicBool::new(false), _marker: PhantomData, } @@ -194,13 +198,14 @@ impl MetricsHandler { } /// Logs the backtrace of current call if the duration that the read transaction has been open - /// is more than [LONG_TRANSACTION_DURATION]. + /// is more than [LONG_TRANSACTION_DURATION] and `record_backtrace == true`. /// The backtrace is recorded and logged just once, guaranteed by `backtrace_recorded` atomic. /// /// NOTE: Backtrace is recorded using [Backtrace::force_capture], so `RUST_BACKTRACE` env var is /// not needed. fn log_backtrace_on_long_read_transaction(&self) { - if !self.backtrace_recorded.load(Ordering::Relaxed) && + if self.record_backtrace && + !self.backtrace_recorded.load(Ordering::Relaxed) && self.transaction_mode().is_read_only() { let open_duration = self.start.elapsed(); @@ -283,6 +288,13 @@ impl DbTx for Tx { .map_err(|e| DatabaseError::Stats(e.into()))? .entries()) } + + /// Disables backtrace recording for this read transaction when it's open for too long. + fn disable_backtrace_on_long_read_transaction(&mut self) { + if let Some(metrics_handler) = self.metrics_handler.as_mut() { + metrics_handler.record_backtrace = false; + } + } } impl DbTxMut for Tx { diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 1180a755cc..36bb93179a 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -246,6 +246,17 @@ impl DatabaseProvider { .collect::, DatabaseError>>() } + /// Disables backtrace recording for the underlying read database transaction when it's open for + /// too long. + /// + /// CAUTION: In most of the cases, you want to keep backtraces on long read transactions + /// enabled. Use this only if you're sure that no write transaction is open in parallel, meaning + /// that Reth as a node is offline and not progressing. + pub fn disable_backtrace_on_long_read_transaction(mut self) -> Self { + self.tx.disable_backtrace_on_long_read_transaction(); + self + } + /// Gets data within a specified range, potentially spanning different snapshots and database. /// /// # Arguments