From 1ca4348db7d361cf35b342abd14afb7a8b67843c Mon Sep 17 00:00:00 2001 From: stevencartavia <112043913+stevencartavia@users.noreply.github.com> Date: Thu, 20 Nov 2025 09:59:21 -0600 Subject: [PATCH 01/23] chore: replace op-reth OpReceipt with op-alloy's (#19846) Co-authored-by: Arsenii Kulikov --- Cargo.lock | 38 +- Cargo.toml | 10 +- crates/optimism/primitives/Cargo.toml | 27 +- crates/optimism/primitives/src/lib.rs | 7 +- crates/optimism/primitives/src/receipt.rs | 516 +----------------- .../src/serde_bincode_compat.rs | 13 + crates/primitives-traits/src/size.rs | 12 + crates/storage/codecs/src/alloy/mod.rs | 3 + crates/storage/codecs/src/alloy/optimism.rs | 97 ++++ 9 files changed, 168 insertions(+), 555 deletions(-) create mode 100644 crates/storage/codecs/src/alloy/optimism.rs diff --git a/Cargo.lock b/Cargo.lock index c67633c58e..711def69ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3049,7 +3049,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3379,7 +3379,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4728,7 +4728,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.57.0", + "windows-core 0.62.2", ] [[package]] @@ -5073,7 +5073,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5982,7 +5982,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6174,9 +6174,9 @@ dependencies = [ [[package]] name = "op-alloy-consensus" -version = "0.22.1" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0d7ec388eb83a3e6c71774131dbbb2ba9c199b6acac7dce172ed8de2f819e91" +checksum = "3e5506c424b9c761a3231128850b7057b2215abe141c0b56e459a1cff31865f6" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6200,9 +6200,9 @@ checksum = "a79f352fc3893dcd670172e615afef993a41798a1d3fc0db88a3e60ef2e70ecc" [[package]] name = "op-alloy-network" -version = "0.22.1" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "979fe768bbb571d1d0bd7f84bc35124243b4db17f944b94698872a4701e743a0" +checksum = "9ea64f4c3424133fb8987ff70139ca7d212014f9f4d4ce94dd081f786089fff8" dependencies = [ "alloy-consensus", "alloy-network", @@ -6231,9 +6231,9 @@ dependencies = [ [[package]] name = "op-alloy-rpc-jsonrpsee" -version = "0.22.1" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bdbb3c0453fe2605fb008851ea0b45f3f2ba607722c9f2e4ffd7198958ce501" +checksum = "11ff7c4267a5706f2a9e99c0d4d800f2f38f8d7035c170b2b682be6c93b8e146" dependencies = [ "alloy-primitives", "jsonrpsee", @@ -6241,9 +6241,9 @@ dependencies = [ [[package]] name = "op-alloy-rpc-types" -version = "0.22.1" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc252b5fa74dbd33aa2f9a40e5ff9cfe34ed2af9b9b235781bc7cc8ec7d6aca8" +checksum = "35792915c6809bc544258b3e05735b823451b08f7ef91a4ae56424477e348171" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6261,9 +6261,9 @@ dependencies = [ [[package]] name = "op-alloy-rpc-types-engine" -version = "0.22.1" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1abe694cd6718b8932da3f824f46778be0f43289e4103c88abc505c63533a04" +checksum = "d1866acd63d831e732a15b2f15cb421f72505aa45d27941d054f7d068596e92f" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7013,7 +7013,7 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -11463,7 +11463,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -12354,7 +12354,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -13569,7 +13569,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 00db602de9..2de3d2bf7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -520,11 +520,11 @@ alloy-transport-ws = { version = "1.0.41", default-features = false } # op alloy-op-evm = { version = "0.24.1", default-features = false } alloy-op-hardforks = "0.4.4" -op-alloy-rpc-types = { version = "0.22.1", default-features = false } -op-alloy-rpc-types-engine = { version = "0.22.1", default-features = false } -op-alloy-network = { version = "0.22.1", default-features = false } -op-alloy-consensus = { version = "0.22.1", default-features = false } -op-alloy-rpc-jsonrpsee = { version = "0.22.1", default-features = false } +op-alloy-rpc-types = { version = "0.22.2", default-features = false } +op-alloy-rpc-types-engine = { version = "0.22.2", default-features = false } +op-alloy-network = { version = "0.22.2", default-features = false } +op-alloy-consensus = { version = "0.22.2", default-features = false } +op-alloy-rpc-jsonrpsee = { version = "0.22.2", default-features = false } op-alloy-flz = { version = "0.13.1", default-features = false } # misc diff --git a/crates/optimism/primitives/Cargo.toml b/crates/optimism/primitives/Cargo.toml index 30257049c2..ef83fe3ddb 100644 --- a/crates/optimism/primitives/Cargo.toml +++ b/crates/optimism/primitives/Cargo.toml @@ -14,8 +14,6 @@ workspace = true [dependencies] # reth reth-primitives-traits = { workspace = true, features = ["op"] } -reth-codecs = { workspace = true, optional = true, features = ["op"] } -reth-zstd-compressors = { workspace = true, optional = true } # ethereum alloy-primitives.workspace = true @@ -27,17 +25,15 @@ alloy-rlp.workspace = true op-alloy-consensus.workspace = true # codec -bytes = { workspace = true, optional = true } -modular-bitfield = { workspace = true, optional = true } serde = { workspace = true, optional = true } serde_with = { workspace = true, optional = true } -# test -arbitrary = { workspace = true, features = ["derive"], optional = true } - [dev-dependencies] reth-codecs = { workspace = true, features = ["test-utils", "op"] } +bytes.workspace = true +modular-bitfield.workspace = true +reth-zstd-compressors.workspace = true rand.workspace = true arbitrary.workspace = true rstest.workspace = true @@ -53,41 +49,35 @@ secp256k1 = { workspace = true, features = ["rand"] } default = ["std"] std = [ "reth-primitives-traits/std", - "reth-codecs?/std", "alloy-consensus/std", "alloy-primitives/std", "serde?/std", - "bytes?/std", "alloy-rlp/std", - "reth-zstd-compressors?/std", "op-alloy-consensus/std", "serde_json/std", "serde_with?/std", "alloy-eips/std", "secp256k1/std", + "bytes/std", + "reth-zstd-compressors/std", ] alloy-compat = ["op-alloy-consensus/alloy-compat"] reth-codec = [ - "dep:reth-codecs", "std", "reth-primitives-traits/reth-codec", - "reth-codecs?/op", - "dep:bytes", - "dep:modular-bitfield", - "dep:reth-zstd-compressors", ] serde = [ "dep:serde", "reth-primitives-traits/serde", "alloy-primitives/serde", "alloy-consensus/serde", - "bytes?/serde", - "reth-codecs?/serde", "op-alloy-consensus/serde", "alloy-eips/serde", "rand/serde", "rand_08/serde", "secp256k1/serde", + "bytes/serde", + "reth-codecs/serde", ] serde-bincode-compat = [ "serde", @@ -99,11 +89,10 @@ serde-bincode-compat = [ ] arbitrary = [ "std", - "dep:arbitrary", "reth-primitives-traits/arbitrary", - "reth-codecs?/arbitrary", "op-alloy-consensus/arbitrary", "alloy-consensus/arbitrary", "alloy-primitives/arbitrary", "alloy-eips/arbitrary", + "reth-codecs/arbitrary", ] diff --git a/crates/optimism/primitives/src/lib.rs b/crates/optimism/primitives/src/lib.rs index 8100d70c91..f58f73a6e8 100644 --- a/crates/optimism/primitives/src/lib.rs +++ b/crates/optimism/primitives/src/lib.rs @@ -20,7 +20,8 @@ pub mod transaction; pub use transaction::*; mod receipt; -pub use receipt::{DepositReceipt, OpReceipt}; +pub use op_alloy_consensus::OpReceipt; +pub use receipt::DepositReceipt; /// Optimism-specific block type. pub type OpBlock = alloy_consensus::Block; @@ -44,6 +45,6 @@ impl reth_primitives_traits::NodePrimitives for OpPrimitives { /// Bincode-compatible serde implementations. #[cfg(feature = "serde-bincode-compat")] pub mod serde_bincode_compat { - pub use super::receipt::serde_bincode_compat::*; - pub use op_alloy_consensus::serde_bincode_compat::*; + pub use super::receipt::serde_bincode_compat::OpReceipt as LocalOpReceipt; + pub use op_alloy_consensus::serde_bincode_compat::OpReceipt; } diff --git a/crates/optimism/primitives/src/receipt.rs b/crates/optimism/primitives/src/receipt.rs index 74f21eab11..5880b37b24 100644 --- a/crates/optimism/primitives/src/receipt.rs +++ b/crates/optimism/primitives/src/receipt.rs @@ -9,409 +9,9 @@ use alloy_eips::{ }; use alloy_primitives::{Bloom, Log}; use alloy_rlp::{BufMut, Decodable, Encodable, Header}; -use op_alloy_consensus::{OpDepositReceipt, OpTxType}; +use op_alloy_consensus::{OpDepositReceipt, OpReceipt, OpTxType}; use reth_primitives_traits::InMemorySize; -/// Typed ethereum transaction receipt. -/// Receipt containing result of transaction execution. -#[derive(Clone, Debug, PartialEq, Eq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[cfg_attr(feature = "reth-codec", reth_codecs::add_arbitrary_tests(rlp))] -pub enum OpReceipt { - /// Legacy receipt - Legacy(Receipt), - /// EIP-2930 receipt - Eip2930(Receipt), - /// EIP-1559 receipt - Eip1559(Receipt), - /// EIP-7702 receipt - Eip7702(Receipt), - /// Deposit receipt - Deposit(OpDepositReceipt), -} - -impl OpReceipt { - /// Returns [`OpTxType`] of the receipt. - pub const fn tx_type(&self) -> OpTxType { - match self { - Self::Legacy(_) => OpTxType::Legacy, - Self::Eip2930(_) => OpTxType::Eip2930, - Self::Eip1559(_) => OpTxType::Eip1559, - Self::Eip7702(_) => OpTxType::Eip7702, - Self::Deposit(_) => OpTxType::Deposit, - } - } - - /// Returns inner [`Receipt`], - pub const fn as_receipt(&self) -> &Receipt { - match self { - Self::Legacy(receipt) | - Self::Eip2930(receipt) | - Self::Eip1559(receipt) | - Self::Eip7702(receipt) => receipt, - Self::Deposit(receipt) => &receipt.inner, - } - } - - /// Returns a mutable reference to the inner [`Receipt`], - pub const fn as_receipt_mut(&mut self) -> &mut Receipt { - match self { - Self::Legacy(receipt) | - Self::Eip2930(receipt) | - Self::Eip1559(receipt) | - Self::Eip7702(receipt) => receipt, - Self::Deposit(receipt) => &mut receipt.inner, - } - } - - /// Consumes this and returns the inner [`Receipt`]. - pub fn into_receipt(self) -> Receipt { - match self { - Self::Legacy(receipt) | - Self::Eip2930(receipt) | - Self::Eip1559(receipt) | - Self::Eip7702(receipt) => receipt, - Self::Deposit(receipt) => receipt.inner, - } - } - - /// Returns length of RLP-encoded receipt fields with the given [`Bloom`] without an RLP header. - pub fn rlp_encoded_fields_length(&self, bloom: &Bloom) -> usize { - match self { - Self::Legacy(receipt) | - Self::Eip2930(receipt) | - Self::Eip1559(receipt) | - Self::Eip7702(receipt) => receipt.rlp_encoded_fields_length_with_bloom(bloom), - Self::Deposit(receipt) => receipt.rlp_encoded_fields_length_with_bloom(bloom), - } - } - - /// RLP-encodes receipt fields with the given [`Bloom`] without an RLP header. - pub fn rlp_encode_fields(&self, bloom: &Bloom, out: &mut dyn BufMut) { - match self { - Self::Legacy(receipt) | - Self::Eip2930(receipt) | - Self::Eip1559(receipt) | - Self::Eip7702(receipt) => receipt.rlp_encode_fields_with_bloom(bloom, out), - Self::Deposit(receipt) => receipt.rlp_encode_fields_with_bloom(bloom, out), - } - } - - /// Returns RLP header for inner encoding. - pub fn rlp_header_inner(&self, bloom: &Bloom) -> Header { - Header { list: true, payload_length: self.rlp_encoded_fields_length(bloom) } - } - - /// Returns RLP header for inner encoding without bloom. - pub fn rlp_header_inner_without_bloom(&self) -> Header { - Header { list: true, payload_length: self.rlp_encoded_fields_length_without_bloom() } - } - - /// RLP-decodes the receipt from the provided buffer. This does not expect a type byte or - /// network header. - pub fn rlp_decode_inner( - buf: &mut &[u8], - tx_type: OpTxType, - ) -> alloy_rlp::Result> { - match tx_type { - OpTxType::Legacy => { - let ReceiptWithBloom { receipt, logs_bloom } = - RlpDecodableReceipt::rlp_decode_with_bloom(buf)?; - Ok(ReceiptWithBloom { receipt: Self::Legacy(receipt), logs_bloom }) - } - OpTxType::Eip2930 => { - let ReceiptWithBloom { receipt, logs_bloom } = - RlpDecodableReceipt::rlp_decode_with_bloom(buf)?; - Ok(ReceiptWithBloom { receipt: Self::Eip2930(receipt), logs_bloom }) - } - OpTxType::Eip1559 => { - let ReceiptWithBloom { receipt, logs_bloom } = - RlpDecodableReceipt::rlp_decode_with_bloom(buf)?; - Ok(ReceiptWithBloom { receipt: Self::Eip1559(receipt), logs_bloom }) - } - OpTxType::Eip7702 => { - let ReceiptWithBloom { receipt, logs_bloom } = - RlpDecodableReceipt::rlp_decode_with_bloom(buf)?; - Ok(ReceiptWithBloom { receipt: Self::Eip7702(receipt), logs_bloom }) - } - OpTxType::Deposit => { - let ReceiptWithBloom { receipt, logs_bloom } = - RlpDecodableReceipt::rlp_decode_with_bloom(buf)?; - Ok(ReceiptWithBloom { receipt: Self::Deposit(receipt), logs_bloom }) - } - } - } - - /// RLP-encodes receipt fields without an RLP header. - pub fn rlp_encode_fields_without_bloom(&self, out: &mut dyn BufMut) { - match self { - Self::Legacy(receipt) | - Self::Eip2930(receipt) | - Self::Eip1559(receipt) | - Self::Eip7702(receipt) => { - receipt.status.encode(out); - receipt.cumulative_gas_used.encode(out); - receipt.logs.encode(out); - } - Self::Deposit(receipt) => { - receipt.inner.status.encode(out); - receipt.inner.cumulative_gas_used.encode(out); - receipt.inner.logs.encode(out); - if let Some(nonce) = receipt.deposit_nonce { - nonce.encode(out); - } - if let Some(version) = receipt.deposit_receipt_version { - version.encode(out); - } - } - } - } - - /// Returns length of RLP-encoded receipt fields without an RLP header. - pub fn rlp_encoded_fields_length_without_bloom(&self) -> usize { - match self { - Self::Legacy(receipt) | - Self::Eip2930(receipt) | - Self::Eip1559(receipt) | - Self::Eip7702(receipt) => { - receipt.status.length() + - receipt.cumulative_gas_used.length() + - receipt.logs.length() - } - Self::Deposit(receipt) => { - receipt.inner.status.length() + - receipt.inner.cumulative_gas_used.length() + - receipt.inner.logs.length() + - receipt.deposit_nonce.map_or(0, |nonce| nonce.length()) + - receipt.deposit_receipt_version.map_or(0, |version| version.length()) - } - } - } - - /// RLP-decodes the receipt from the provided buffer without bloom. - pub fn rlp_decode_inner_without_bloom( - buf: &mut &[u8], - tx_type: OpTxType, - ) -> alloy_rlp::Result { - let header = Header::decode(buf)?; - if !header.list { - return Err(alloy_rlp::Error::UnexpectedString); - } - - let remaining = buf.len(); - let status = Decodable::decode(buf)?; - let cumulative_gas_used = Decodable::decode(buf)?; - let logs = Decodable::decode(buf)?; - - let mut deposit_nonce = None; - let mut deposit_receipt_version = None; - - // For deposit receipts, try to decode nonce and version if they exist - if tx_type == OpTxType::Deposit && buf.len() + header.payload_length > remaining { - deposit_nonce = Some(Decodable::decode(buf)?); - if buf.len() + header.payload_length > remaining { - deposit_receipt_version = Some(Decodable::decode(buf)?); - } - } - - if buf.len() + header.payload_length != remaining { - return Err(alloy_rlp::Error::UnexpectedLength); - } - - match tx_type { - OpTxType::Legacy => Ok(Self::Legacy(Receipt { status, cumulative_gas_used, logs })), - OpTxType::Eip2930 => Ok(Self::Eip2930(Receipt { status, cumulative_gas_used, logs })), - OpTxType::Eip1559 => Ok(Self::Eip1559(Receipt { status, cumulative_gas_used, logs })), - OpTxType::Eip7702 => Ok(Self::Eip7702(Receipt { status, cumulative_gas_used, logs })), - OpTxType::Deposit => Ok(Self::Deposit(OpDepositReceipt { - inner: Receipt { status, cumulative_gas_used, logs }, - deposit_nonce, - deposit_receipt_version, - })), - } - } -} - -impl Eip2718EncodableReceipt for OpReceipt { - fn eip2718_encoded_length_with_bloom(&self, bloom: &Bloom) -> usize { - !self.tx_type().is_legacy() as usize + self.rlp_header_inner(bloom).length_with_payload() - } - - fn eip2718_encode_with_bloom(&self, bloom: &Bloom, out: &mut dyn BufMut) { - if !self.tx_type().is_legacy() { - out.put_u8(self.tx_type() as u8); - } - self.rlp_header_inner(bloom).encode(out); - self.rlp_encode_fields(bloom, out); - } -} - -impl RlpEncodableReceipt for OpReceipt { - fn rlp_encoded_length_with_bloom(&self, bloom: &Bloom) -> usize { - let mut len = self.eip2718_encoded_length_with_bloom(bloom); - if !self.tx_type().is_legacy() { - len += Header { - list: false, - payload_length: self.eip2718_encoded_length_with_bloom(bloom), - } - .length(); - } - - len - } - - fn rlp_encode_with_bloom(&self, bloom: &Bloom, out: &mut dyn BufMut) { - if !self.tx_type().is_legacy() { - Header { list: false, payload_length: self.eip2718_encoded_length_with_bloom(bloom) } - .encode(out); - } - self.eip2718_encode_with_bloom(bloom, out); - } -} - -impl RlpDecodableReceipt for OpReceipt { - fn rlp_decode_with_bloom(buf: &mut &[u8]) -> alloy_rlp::Result> { - let header_buf = &mut &**buf; - let header = Header::decode(header_buf)?; - - // Legacy receipt, reuse initial buffer without advancing - if header.list { - return Self::rlp_decode_inner(buf, OpTxType::Legacy) - } - - // Otherwise, advance the buffer and try decoding type flag followed by receipt - *buf = *header_buf; - - let remaining = buf.len(); - let tx_type = OpTxType::decode(buf)?; - let this = Self::rlp_decode_inner(buf, tx_type)?; - - if buf.len() + header.payload_length != remaining { - return Err(alloy_rlp::Error::UnexpectedLength); - } - - Ok(this) - } -} - -impl Encodable2718 for OpReceipt { - fn encode_2718_len(&self) -> usize { - !self.tx_type().is_legacy() as usize + - self.rlp_header_inner_without_bloom().length_with_payload() - } - - fn encode_2718(&self, out: &mut dyn BufMut) { - if !self.tx_type().is_legacy() { - out.put_u8(self.tx_type() as u8); - } - self.rlp_header_inner_without_bloom().encode(out); - self.rlp_encode_fields_without_bloom(out); - } -} - -impl Decodable2718 for OpReceipt { - fn typed_decode(ty: u8, buf: &mut &[u8]) -> Eip2718Result { - Ok(Self::rlp_decode_inner_without_bloom(buf, OpTxType::try_from(ty)?)?) - } - - fn fallback_decode(buf: &mut &[u8]) -> Eip2718Result { - Ok(Self::rlp_decode_inner_without_bloom(buf, OpTxType::Legacy)?) - } -} - -impl Encodable for OpReceipt { - fn encode(&self, out: &mut dyn BufMut) { - self.network_encode(out); - } - - fn length(&self) -> usize { - self.network_len() - } -} - -impl Decodable for OpReceipt { - fn decode(buf: &mut &[u8]) -> alloy_rlp::Result { - Ok(Self::network_decode(buf)?) - } -} - -impl TxReceipt for OpReceipt { - type Log = Log; - - fn status_or_post_state(&self) -> Eip658Value { - self.as_receipt().status_or_post_state() - } - - fn status(&self) -> bool { - self.as_receipt().status() - } - - fn bloom(&self) -> Bloom { - self.as_receipt().bloom() - } - - fn cumulative_gas_used(&self) -> u64 { - self.as_receipt().cumulative_gas_used() - } - - fn logs(&self) -> &[Log] { - self.as_receipt().logs() - } - - fn into_logs(self) -> Vec { - match self { - Self::Legacy(receipt) | - Self::Eip2930(receipt) | - Self::Eip1559(receipt) | - Self::Eip7702(receipt) => receipt.logs, - Self::Deposit(receipt) => receipt.inner.logs, - } - } -} - -impl Typed2718 for OpReceipt { - fn ty(&self) -> u8 { - self.tx_type().into() - } -} - -impl IsTyped2718 for OpReceipt { - fn is_type(type_id: u8) -> bool { - ::is_type(type_id) - } -} - -impl InMemorySize for OpReceipt { - fn size(&self) -> usize { - self.as_receipt().size() - } -} - -impl From for OpReceipt { - fn from(envelope: op_alloy_consensus::OpReceiptEnvelope) -> Self { - match envelope { - op_alloy_consensus::OpReceiptEnvelope::Legacy(receipt) => Self::Legacy(receipt.receipt), - op_alloy_consensus::OpReceiptEnvelope::Eip2930(receipt) => { - Self::Eip2930(receipt.receipt) - } - op_alloy_consensus::OpReceiptEnvelope::Eip1559(receipt) => { - Self::Eip1559(receipt.receipt) - } - op_alloy_consensus::OpReceiptEnvelope::Eip7702(receipt) => { - Self::Eip7702(receipt.receipt) - } - op_alloy_consensus::OpReceiptEnvelope::Deposit(receipt) => { - Self::Deposit(OpDepositReceipt { - deposit_nonce: receipt.receipt.deposit_nonce, - deposit_receipt_version: receipt.receipt.deposit_receipt_version, - inner: receipt.receipt.inner, - }) - } - } - } -} - /// Trait for deposit receipt. pub trait DepositReceipt: reth_primitives_traits::Receipt { /// Converts a `Receipt` into a mutable Optimism deposit receipt. @@ -437,100 +37,6 @@ impl DepositReceipt for OpReceipt { } } -#[cfg(feature = "reth-codec")] -mod compact { - use super::*; - use alloc::borrow::Cow; - use reth_codecs::Compact; - - #[derive(reth_codecs::CompactZstd)] - #[reth_zstd( - compressor = reth_zstd_compressors::RECEIPT_COMPRESSOR, - decompressor = reth_zstd_compressors::RECEIPT_DECOMPRESSOR - )] - struct CompactOpReceipt<'a> { - tx_type: OpTxType, - success: bool, - cumulative_gas_used: u64, - #[expect(clippy::owned_cow)] - logs: Cow<'a, Vec>, - deposit_nonce: Option, - deposit_receipt_version: Option, - } - - impl<'a> From<&'a OpReceipt> for CompactOpReceipt<'a> { - fn from(receipt: &'a OpReceipt) -> Self { - Self { - tx_type: receipt.tx_type(), - success: receipt.status(), - cumulative_gas_used: receipt.cumulative_gas_used(), - logs: Cow::Borrowed(&receipt.as_receipt().logs), - deposit_nonce: if let OpReceipt::Deposit(receipt) = receipt { - receipt.deposit_nonce - } else { - None - }, - deposit_receipt_version: if let OpReceipt::Deposit(receipt) = receipt { - receipt.deposit_receipt_version - } else { - None - }, - } - } - } - - impl From> for OpReceipt { - fn from(receipt: CompactOpReceipt<'_>) -> Self { - let CompactOpReceipt { - tx_type, - success, - cumulative_gas_used, - logs, - deposit_nonce, - deposit_receipt_version, - } = receipt; - - let inner = - Receipt { status: success.into(), cumulative_gas_used, logs: logs.into_owned() }; - - match tx_type { - OpTxType::Legacy => Self::Legacy(inner), - OpTxType::Eip2930 => Self::Eip2930(inner), - OpTxType::Eip1559 => Self::Eip1559(inner), - OpTxType::Eip7702 => Self::Eip7702(inner), - OpTxType::Deposit => Self::Deposit(OpDepositReceipt { - inner, - deposit_nonce, - deposit_receipt_version, - }), - } - } - } - - impl Compact for OpReceipt { - fn to_compact(&self, buf: &mut B) -> usize - where - B: bytes::BufMut + AsMut<[u8]>, - { - CompactOpReceipt::from(self).to_compact(buf) - } - - fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { - let (receipt, buf) = CompactOpReceipt::from_compact(buf, len); - (receipt.into(), buf) - } - } - - #[cfg(test)] - #[test] - fn test_ensure_backwards_compatibility() { - use reth_codecs::{test_utils::UnusedBits, validate_bitflag_backwards_compat}; - - assert_eq!(CompactOpReceipt::bitflag_encoded_bytes(), 2); - validate_bitflag_backwards_compat!(CompactOpReceipt<'_>, UnusedBits::NotZero); - } -} - #[cfg(all(feature = "serde", feature = "serde-bincode-compat"))] pub(super) mod serde_bincode_compat { use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -540,17 +46,21 @@ pub(super) mod serde_bincode_compat { /// /// Intended to use with the [`serde_with::serde_as`] macro in the following way: /// ```rust - /// use reth_optimism_primitives::{serde_bincode_compat, OpReceipt}; + /// use reth_optimism_primitives::OpReceipt; + /// use reth_primitives_traits::serde_bincode_compat::SerdeBincodeCompat; /// use serde::{de::DeserializeOwned, Deserialize, Serialize}; /// use serde_with::serde_as; /// /// #[serde_as] /// #[derive(Serialize, Deserialize)] /// struct Data { - /// #[serde_as(as = "serde_bincode_compat::OpReceipt<'_>")] + /// #[serde_as( + /// as = "reth_primitives_traits::serde_bincode_compat::BincodeReprFor<'_, OpReceipt>" + /// )] /// receipt: OpReceipt, /// } /// ``` + #[allow(rustdoc::private_doc_tests)] #[derive(Debug, Serialize, Deserialize)] pub enum OpReceipt<'a> { /// Legacy receipt @@ -609,18 +119,6 @@ pub(super) mod serde_bincode_compat { } } - impl reth_primitives_traits::serde_bincode_compat::SerdeBincodeCompat for super::OpReceipt { - type BincodeRepr<'a> = OpReceipt<'a>; - - fn as_repr(&self) -> Self::BincodeRepr<'_> { - self.into() - } - - fn from_repr(repr: Self::BincodeRepr<'_>) -> Self { - repr.into() - } - } - #[cfg(test)] mod tests { use crate::{receipt::serde_bincode_compat, OpReceipt}; diff --git a/crates/primitives-traits/src/serde_bincode_compat.rs b/crates/primitives-traits/src/serde_bincode_compat.rs index 217ad5ff33..5169f945b8 100644 --- a/crates/primitives-traits/src/serde_bincode_compat.rs +++ b/crates/primitives-traits/src/serde_bincode_compat.rs @@ -345,4 +345,17 @@ mod block_bincode { repr.into() } } + + #[cfg(feature = "op")] + impl super::SerdeBincodeCompat for op_alloy_consensus::OpReceipt { + type BincodeRepr<'a> = op_alloy_consensus::serde_bincode_compat::OpReceipt<'a>; + + fn as_repr(&self) -> Self::BincodeRepr<'_> { + self.into() + } + + fn from_repr(repr: Self::BincodeRepr<'_>) -> Self { + repr.into() + } + } } diff --git a/crates/primitives-traits/src/size.rs b/crates/primitives-traits/src/size.rs index 82c8b5d9c4..e2343cfb95 100644 --- a/crates/primitives-traits/src/size.rs +++ b/crates/primitives-traits/src/size.rs @@ -148,6 +148,18 @@ mod op { } } + impl InMemorySize for op_alloy_consensus::OpReceipt { + fn size(&self) -> usize { + match self { + Self::Legacy(receipt) | + Self::Eip2930(receipt) | + Self::Eip1559(receipt) | + Self::Eip7702(receipt) => receipt.size(), + Self::Deposit(receipt) => receipt.size(), + } + } + } + impl InMemorySize for op_alloy_consensus::OpTypedTransaction { fn size(&self) -> usize { match self { diff --git a/crates/storage/codecs/src/alloy/mod.rs b/crates/storage/codecs/src/alloy/mod.rs index 34fcd6fdc2..9081fad098 100644 --- a/crates/storage/codecs/src/alloy/mod.rs +++ b/crates/storage/codecs/src/alloy/mod.rs @@ -24,6 +24,9 @@ cond_mod!( withdrawal ); +#[cfg(all(feature = "op", feature = "std"))] +pub mod optimism; + pub mod transaction; #[cfg(test)] diff --git a/crates/storage/codecs/src/alloy/optimism.rs b/crates/storage/codecs/src/alloy/optimism.rs new file mode 100644 index 0000000000..7a851a5041 --- /dev/null +++ b/crates/storage/codecs/src/alloy/optimism.rs @@ -0,0 +1,97 @@ +//! Compact implementations for Optimism types. + +use crate::Compact; +use alloc::{borrow::Cow, vec::Vec}; +use alloy_consensus::{Receipt, TxReceipt}; +use alloy_primitives::Log; +use op_alloy_consensus::{OpDepositReceipt, OpReceipt, OpTxType}; +use reth_codecs_derive::CompactZstd; + +#[derive(CompactZstd)] +#[reth_codecs(crate = "crate")] +#[reth_zstd( + compressor = reth_zstd_compressors::RECEIPT_COMPRESSOR, + decompressor = reth_zstd_compressors::RECEIPT_DECOMPRESSOR +)] +struct CompactOpReceipt<'a> { + tx_type: OpTxType, + success: bool, + cumulative_gas_used: u64, + #[expect(clippy::owned_cow)] + logs: Cow<'a, Vec>, + deposit_nonce: Option, + deposit_receipt_version: Option, +} + +impl<'a> From<&'a OpReceipt> for CompactOpReceipt<'a> { + fn from(receipt: &'a OpReceipt) -> Self { + Self { + tx_type: receipt.tx_type(), + success: receipt.status(), + cumulative_gas_used: receipt.cumulative_gas_used(), + logs: Cow::Borrowed(&receipt.as_receipt().logs), + deposit_nonce: if let OpReceipt::Deposit(receipt) = receipt { + receipt.deposit_nonce + } else { + None + }, + deposit_receipt_version: if let OpReceipt::Deposit(receipt) = receipt { + receipt.deposit_receipt_version + } else { + None + }, + } + } +} + +impl From> for OpReceipt { + fn from(receipt: CompactOpReceipt<'_>) -> Self { + let CompactOpReceipt { + tx_type, + success, + cumulative_gas_used, + logs, + deposit_nonce, + deposit_receipt_version, + } = receipt; + + let inner = + Receipt { status: success.into(), cumulative_gas_used, logs: logs.into_owned() }; + + match tx_type { + OpTxType::Legacy => Self::Legacy(inner), + OpTxType::Eip2930 => Self::Eip2930(inner), + OpTxType::Eip1559 => Self::Eip1559(inner), + OpTxType::Eip7702 => Self::Eip7702(inner), + OpTxType::Deposit => { + Self::Deposit(OpDepositReceipt { inner, deposit_nonce, deposit_receipt_version }) + } + } + } +} + +impl Compact for OpReceipt { + fn to_compact(&self, buf: &mut B) -> usize + where + B: bytes::BufMut + AsMut<[u8]>, + { + CompactOpReceipt::from(self).to_compact(buf) + } + + fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { + let (receipt, buf) = CompactOpReceipt::from_compact(buf, len); + (receipt.into(), buf) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{test_utils::UnusedBits, validate_bitflag_backwards_compat}; + + #[test] + fn test_ensure_backwards_compatibility() { + assert_eq!(CompactOpReceipt::bitflag_encoded_bytes(), 2); + validate_bitflag_backwards_compat!(CompactOpReceipt<'_>, UnusedBits::NotZero); + } +} From 55dacfc7392345060d584e0f7e684e5d4225bd02 Mon Sep 17 00:00:00 2001 From: Merkel Tranjes <140164174+rnkrtt@users.noreply.github.com> Date: Thu, 20 Nov 2025 19:18:53 +0100 Subject: [PATCH 02/23] chore: bump op-alloy deps to 0.22.3 (#19885) --- Cargo.lock | 20 ++++++++++---------- Cargo.toml | 10 +++++----- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 711def69ad..f0be65292e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6174,9 +6174,9 @@ dependencies = [ [[package]] name = "op-alloy-consensus" -version = "0.22.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5506c424b9c761a3231128850b7057b2215abe141c0b56e459a1cff31865f6" +checksum = "e82f4f768ba39e52a4efe1b8f3425c04ab0d0e6f90c003fe97e5444cd963405e" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6200,9 +6200,9 @@ checksum = "a79f352fc3893dcd670172e615afef993a41798a1d3fc0db88a3e60ef2e70ecc" [[package]] name = "op-alloy-network" -version = "0.22.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea64f4c3424133fb8987ff70139ca7d212014f9f4d4ce94dd081f786089fff8" +checksum = "f2607d0d985f848f98fa79068d11c612f8476dba7deb7498881794bf51b3cfb5" dependencies = [ "alloy-consensus", "alloy-network", @@ -6231,9 +6231,9 @@ dependencies = [ [[package]] name = "op-alloy-rpc-jsonrpsee" -version = "0.22.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ff7c4267a5706f2a9e99c0d4d800f2f38f8d7035c170b2b682be6c93b8e146" +checksum = "6911db73a4bf59bf8a963dec153ada1057fa426fdc35e0b35fe82657af3501a3" dependencies = [ "alloy-primitives", "jsonrpsee", @@ -6241,9 +6241,9 @@ dependencies = [ [[package]] name = "op-alloy-rpc-types" -version = "0.22.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35792915c6809bc544258b3e05735b823451b08f7ef91a4ae56424477e348171" +checksum = "890b51c3a619c263d52ee5a945dce173a4052d017f93bf5698613b21cbe0d237" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6261,9 +6261,9 @@ dependencies = [ [[package]] name = "op-alloy-rpc-types-engine" -version = "0.22.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1866acd63d831e732a15b2f15cb421f72505aa45d27941d054f7d068596e92f" +checksum = "c92f9dd709b3a769b7604d4d2257846b6de3d3f60e5163982cc4e90c0d0b6f95" dependencies = [ "alloy-consensus", "alloy-eips", diff --git a/Cargo.toml b/Cargo.toml index 2de3d2bf7e..b08a54c0e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -520,11 +520,11 @@ alloy-transport-ws = { version = "1.0.41", default-features = false } # op alloy-op-evm = { version = "0.24.1", default-features = false } alloy-op-hardforks = "0.4.4" -op-alloy-rpc-types = { version = "0.22.2", default-features = false } -op-alloy-rpc-types-engine = { version = "0.22.2", default-features = false } -op-alloy-network = { version = "0.22.2", default-features = false } -op-alloy-consensus = { version = "0.22.2", default-features = false } -op-alloy-rpc-jsonrpsee = { version = "0.22.2", default-features = false } +op-alloy-rpc-types = { version = "0.22.3", default-features = false } +op-alloy-rpc-types-engine = { version = "0.22.3", default-features = false } +op-alloy-network = { version = "0.22.3", default-features = false } +op-alloy-consensus = { version = "0.22.3", default-features = false } +op-alloy-rpc-jsonrpsee = { version = "0.22.3", default-features = false } op-alloy-flz = { version = "0.13.1", default-features = false } # misc From 9cdcc8e087d380a14bcd578bb729a1cd3c145c8a Mon Sep 17 00:00:00 2001 From: gejeduck <47668701+gejeduck@users.noreply.github.com> Date: Thu, 20 Nov 2025 13:19:11 -0500 Subject: [PATCH 03/23] feat: respect BlockRangeInfo when selecting peer for request (#16704) Co-authored-by: Matthias Seitz --- crates/net/eth-wire-types/src/capability.rs | 16 +- crates/net/network/src/fetch/mod.rs | 492 +++++++++++++++++++- crates/net/network/src/session/types.rs | 11 +- 3 files changed, 496 insertions(+), 23 deletions(-) diff --git a/crates/net/eth-wire-types/src/capability.rs b/crates/net/eth-wire-types/src/capability.rs index 3f39bed606..f7cd00671f 100644 --- a/crates/net/eth-wire-types/src/capability.rs +++ b/crates/net/eth-wire-types/src/capability.rs @@ -158,6 +158,15 @@ pub struct Capabilities { } impl Capabilities { + /// Create a new instance from the given vec. + pub fn new(value: Vec) -> Self { + Self { + eth_66: value.iter().any(Capability::is_eth_v66), + eth_67: value.iter().any(Capability::is_eth_v67), + eth_68: value.iter().any(Capability::is_eth_v68), + inner: value, + } + } /// Returns all capabilities. #[inline] pub fn capabilities(&self) -> &[Capability] { @@ -197,12 +206,7 @@ impl Capabilities { impl From> for Capabilities { fn from(value: Vec) -> Self { - Self { - eth_66: value.iter().any(Capability::is_eth_v66), - eth_67: value.iter().any(Capability::is_eth_v67), - eth_68: value.iter().any(Capability::is_eth_v68), - inner: value, - } + Self::new(value) } } diff --git a/crates/net/network/src/fetch/mod.rs b/crates/net/network/src/fetch/mod.rs index 9d603863a9..399f91d12a 100644 --- a/crates/net/network/src/fetch/mod.rs +++ b/crates/net/network/src/fetch/mod.rs @@ -139,8 +139,9 @@ impl StateFetcher { /// Returns the _next_ idle peer that's ready to accept a request, /// prioritizing those with the lowest timeout/latency and those that recently responded with - /// adequate data. - fn next_best_peer(&self) -> Option { + /// adequate data. Additionally, if full blocks are required this prioritizes peers that have + /// full history available + fn next_best_peer(&self, requirement: BestPeerRequirements) -> Option { let mut idle = self.peers.iter().filter(|(_, peer)| peer.state.is_idle()); let mut best_peer = idle.next()?; @@ -152,7 +153,13 @@ impl StateFetcher { continue } - // replace best peer if this peer has better rtt + // replace best peer if this peer meets the requirements better + if maybe_better.1.is_better(best_peer.1, &requirement) { + best_peer = maybe_better; + continue + } + + // replace best peer if this peer has better rtt and both have same range quality if maybe_better.1.timeout() < best_peer.1.timeout() && !maybe_better.1.last_response_likely_bad { @@ -170,9 +177,13 @@ impl StateFetcher { return PollAction::NoRequests } - let Some(peer_id) = self.next_best_peer() else { return PollAction::NoPeersAvailable }; - let request = self.queued_requests.pop_front().expect("not empty"); + let Some(peer_id) = self.next_best_peer(request.best_peer_requirements()) else { + // need to put back the the request + self.queued_requests.push_front(request); + return PollAction::NoPeersAvailable + }; + let request = self.prepare_block_request(peer_id, request); PollAction::Ready(FetchAction::BlockRequest { peer_id, request }) @@ -358,7 +369,6 @@ struct Peer { /// lowest timeout. last_response_likely_bad: bool, /// Tracks the range info for the peer. - #[allow(dead_code)] range_info: Option, } @@ -366,6 +376,74 @@ impl Peer { fn timeout(&self) -> u64 { self.timeout.load(Ordering::Relaxed) } + + /// Returns the earliest block number available from the peer. + fn earliest(&self) -> u64 { + self.range_info.as_ref().map_or(0, |info| info.earliest()) + } + + /// Returns true if the peer has the full history available. + fn has_full_history(&self) -> bool { + self.earliest() == 0 + } + + fn range(&self) -> Option> { + self.range_info.as_ref().map(|info| info.range()) + } + + /// Returns true if this peer has a better range than the other peer for serving the requested + /// range. + /// + /// A peer has a "better range" if: + /// 1. It can fully cover the requested range while the other cannot + /// 2. None can fully cover the range, but this peer has lower start value + /// 3. If a peer doesnt announce a range we assume it has full history, but check the other's + /// range and treat that as better if it can cover the range + fn has_better_range(&self, other: &Self, range: RangeInclusive) -> bool { + let self_range = self.range(); + let other_range = other.range(); + + match (self_range, other_range) { + (Some(self_r), Some(other_r)) => { + // Check if each peer can fully cover the requested range + let self_covers = self_r.contains(range.start()) && self_r.contains(range.end()); + let other_covers = other_r.contains(range.start()) && other_r.contains(range.end()); + + #[allow(clippy::match_same_arms)] + match (self_covers, other_covers) { + (true, false) => true, // Only self covers the range + (false, true) => false, // Only other covers the range + (true, true) => false, // Both cover + (false, false) => { + // neither covers - prefer if peer has lower (better) start range + self_r.start() < other_r.start() + } + } + } + (Some(self_r), None) => { + // Self has range info, other doesn't (treated as full history with unknown latest) + // Self is better only if it covers the range + self_r.contains(range.start()) && self_r.contains(range.end()) + } + (None, Some(other_r)) => { + // Self has no range info (full history), other has range info + // Self is better only if other doesn't cover the range + !(other_r.contains(range.start()) && other_r.contains(range.end())) + } + (None, None) => false, // Neither has range info - no one is better + } + } + + /// Returns true if this peer is better than the other peer based on the given requirements. + fn is_better(&self, other: &Self, requirement: &BestPeerRequirements) -> bool { + match requirement { + BestPeerRequirements::None => false, + BestPeerRequirements::FullBlockRange(range) => { + self.has_better_range(other, range.clone()) + } + BestPeerRequirements::FullBlock => self.has_full_history() && !other.has_full_history(), + } + } } /// Tracks the state of an individual peer @@ -427,7 +505,6 @@ pub(crate) enum DownloadRequest { request: Vec, response: oneshot::Sender>>, priority: Priority, - #[allow(dead_code)] range_hint: Option>, }, } @@ -456,6 +533,20 @@ impl DownloadRequest { const fn is_normal_priority(&self) -> bool { self.get_priority().is_normal() } + + /// Returns the best peer requirements for this request. + fn best_peer_requirements(&self) -> BestPeerRequirements { + match self { + Self::GetBlockHeaders { .. } => BestPeerRequirements::None, + Self::GetBlockBodies { range_hint, .. } => { + if let Some(range) = range_hint { + BestPeerRequirements::FullBlockRange(range.clone()) + } else { + BestPeerRequirements::FullBlock + } + } + } + } } /// An action the syncer can emit. @@ -480,6 +571,16 @@ pub(crate) enum BlockResponseOutcome { BadResponse(PeerId, ReputationChangeKind), } +/// Additional requirements for how to rank peers during selection. +enum BestPeerRequirements { + /// No additional requirements + None, + /// Peer must have this block range available. + FullBlockRange(RangeInclusive), + /// Peer must have full range. + FullBlock, +} + #[cfg(test)] mod tests { use super::*; @@ -536,17 +637,17 @@ mod tests { None, ); - let first_peer = fetcher.next_best_peer().unwrap(); + let first_peer = fetcher.next_best_peer(BestPeerRequirements::None).unwrap(); assert!(first_peer == peer1 || first_peer == peer2); // Pending disconnect for first_peer fetcher.on_pending_disconnect(&first_peer); // first_peer now isn't idle, so we should get other peer - let second_peer = fetcher.next_best_peer().unwrap(); + let second_peer = fetcher.next_best_peer(BestPeerRequirements::None).unwrap(); assert!(first_peer == peer1 || first_peer == peer2); assert_ne!(first_peer, second_peer); // without idle peers, returns None fetcher.on_pending_disconnect(&second_peer); - assert_eq!(fetcher.next_best_peer(), None); + assert_eq!(fetcher.next_best_peer(BestPeerRequirements::None), None); } #[tokio::test] @@ -588,13 +689,13 @@ mod tests { ); // Must always get peer1 (lowest timeout) - assert_eq!(fetcher.next_best_peer(), Some(peer1)); - assert_eq!(fetcher.next_best_peer(), Some(peer1)); + assert_eq!(fetcher.next_best_peer(BestPeerRequirements::None), Some(peer1)); + assert_eq!(fetcher.next_best_peer(BestPeerRequirements::None), Some(peer1)); // peer2's timeout changes below peer1's peer2_timeout.store(10, Ordering::Relaxed); // Then we get peer 2 always (now lowest) - assert_eq!(fetcher.next_best_peer(), Some(peer2)); - assert_eq!(fetcher.next_best_peer(), Some(peer2)); + assert_eq!(fetcher.next_best_peer(BestPeerRequirements::None), Some(peer2)); + assert_eq!(fetcher.next_best_peer(BestPeerRequirements::None), Some(peer2)); } #[tokio::test] @@ -684,4 +785,367 @@ mod tests { assert!(fetcher.peers[&peer_id].state.is_idle()); } + + #[test] + fn test_peer_is_better_none_requirement() { + let peer1 = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: Some(BlockRangeInfo::new(0, 100, B256::random())), + }; + + let peer2 = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 50, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(20)), + last_response_likely_bad: false, + range_info: None, + }; + + // With None requirement, is_better should always return false + assert!(!peer1.is_better(&peer2, &BestPeerRequirements::None)); + assert!(!peer2.is_better(&peer1, &BestPeerRequirements::None)); + } + + #[test] + fn test_peer_is_better_full_block_requirement() { + // Peer with full history (earliest = 0) + let peer_full = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: Some(BlockRangeInfo::new(0, 100, B256::random())), + }; + + // Peer without full history (earliest = 50) + let peer_partial = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: Some(BlockRangeInfo::new(50, 100, B256::random())), + }; + + // Peer without range info (treated as full history) + let peer_no_range = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: None, + }; + + // Peer with full history is better than peer without + assert!(peer_full.is_better(&peer_partial, &BestPeerRequirements::FullBlock)); + assert!(!peer_partial.is_better(&peer_full, &BestPeerRequirements::FullBlock)); + + // Peer without range info (full history) is better than partial + assert!(peer_no_range.is_better(&peer_partial, &BestPeerRequirements::FullBlock)); + assert!(!peer_partial.is_better(&peer_no_range, &BestPeerRequirements::FullBlock)); + + // Both have full history - no improvement + assert!(!peer_full.is_better(&peer_no_range, &BestPeerRequirements::FullBlock)); + assert!(!peer_no_range.is_better(&peer_full, &BestPeerRequirements::FullBlock)); + } + + #[test] + fn test_peer_is_better_full_block_range_requirement() { + let range = RangeInclusive::new(40, 60); + + // Peer that covers the requested range + let peer_covers = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: Some(BlockRangeInfo::new(0, 100, B256::random())), + }; + + // Peer that doesn't cover the range (earliest too high) + let peer_no_cover = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: Some(BlockRangeInfo::new(70, 100, B256::random())), + }; + + // Peer that covers the requested range is better than one that doesn't + assert!(peer_covers + .is_better(&peer_no_cover, &BestPeerRequirements::FullBlockRange(range.clone()))); + assert!( + !peer_no_cover.is_better(&peer_covers, &BestPeerRequirements::FullBlockRange(range)) + ); + } + + #[test] + fn test_peer_is_better_both_cover_range() { + let range = RangeInclusive::new(30, 50); + + // Peer with full history that covers the range + let peer_full = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: Some(BlockRangeInfo::new(0, 50, B256::random())), + }; + + // Peer without full history that also covers the range + let peer_partial = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: Some(BlockRangeInfo::new(30, 50, B256::random())), + }; + + // When both cover the range, prefer none + assert!(!peer_full + .is_better(&peer_partial, &BestPeerRequirements::FullBlockRange(range.clone()))); + assert!(!peer_partial.is_better(&peer_full, &BestPeerRequirements::FullBlockRange(range))); + } + + #[test] + fn test_peer_is_better_lower_start() { + let range = RangeInclusive::new(30, 60); + + // Peer with full history that covers the range + let peer_full = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: Some(BlockRangeInfo::new(0, 50, B256::random())), + }; + + // Peer without full history that also covers the range + let peer_partial = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: Some(BlockRangeInfo::new(30, 50, B256::random())), + }; + + // When both cover the range, prefer lower start value + assert!(peer_full + .is_better(&peer_partial, &BestPeerRequirements::FullBlockRange(range.clone()))); + assert!(!peer_partial.is_better(&peer_full, &BestPeerRequirements::FullBlockRange(range))); + } + + #[test] + fn test_peer_is_better_neither_covers_range() { + let range = RangeInclusive::new(40, 60); + + // Peer with full history that doesn't cover the range (latest too low) + let peer_full = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 30, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: Some(BlockRangeInfo::new(0, 30, B256::random())), + }; + + // Peer without full history that also doesn't cover the range + let peer_partial = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 30, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: Some(BlockRangeInfo::new(10, 30, B256::random())), + }; + + // When neither covers the range, prefer full history + assert!(peer_full + .is_better(&peer_partial, &BestPeerRequirements::FullBlockRange(range.clone()))); + assert!(!peer_partial.is_better(&peer_full, &BestPeerRequirements::FullBlockRange(range))); + } + + #[test] + fn test_peer_is_better_no_range_info() { + let range = RangeInclusive::new(40, 60); + + // Peer with range info + let peer_with_range = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: Some(BlockRangeInfo::new(30, 100, B256::random())), + }; + + // Peer without range info + let peer_no_range = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: None, + }; + + // Peer without range info is not better (we prefer peers with known ranges) + assert!(!peer_no_range + .is_better(&peer_with_range, &BestPeerRequirements::FullBlockRange(range.clone()))); + + // Peer with range info is better than peer without + assert!( + peer_with_range.is_better(&peer_no_range, &BestPeerRequirements::FullBlockRange(range)) + ); + } + + #[test] + fn test_peer_is_better_one_peer_no_range_covers() { + let range = RangeInclusive::new(40, 60); + + // Peer with range info that covers the requested range + let peer_with_range_covers = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: Some(BlockRangeInfo::new(30, 100, B256::random())), + }; + + // Peer without range info (treated as full history with unknown latest) + let peer_no_range = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: None, + }; + + // Peer with range that covers is better than peer without range info + assert!(peer_with_range_covers + .is_better(&peer_no_range, &BestPeerRequirements::FullBlockRange(range.clone()))); + + // Peer without range info is not better when other covers + assert!(!peer_no_range + .is_better(&peer_with_range_covers, &BestPeerRequirements::FullBlockRange(range))); + } + + #[test] + fn test_peer_is_better_one_peer_no_range_doesnt_cover() { + let range = RangeInclusive::new(40, 60); + + // Peer with range info that does NOT cover the requested range (too high) + let peer_with_range_no_cover = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: Some(BlockRangeInfo::new(70, 100, B256::random())), + }; + + // Peer without range info (treated as full history) + let peer_no_range = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: None, + }; + + // Peer with range that doesn't cover is not better + assert!(!peer_with_range_no_cover + .is_better(&peer_no_range, &BestPeerRequirements::FullBlockRange(range.clone()))); + + // Peer without range info (full history) is better when other doesn't cover + assert!(peer_no_range + .is_better(&peer_with_range_no_cover, &BestPeerRequirements::FullBlockRange(range))); + } + + #[test] + fn test_peer_is_better_edge_cases() { + // Test exact range boundaries + let range = RangeInclusive::new(50, 100); + + // Peer that exactly covers the range + let peer_exact = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: Some(BlockRangeInfo::new(50, 100, B256::random())), + }; + + // Peer that's one block short at the start + let peer_short_start = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: Some(BlockRangeInfo::new(51, 100, B256::random())), + }; + + // Peer that's one block short at the end + let peer_short_end = Peer { + state: PeerState::Idle, + best_hash: B256::random(), + best_number: 100, + capabilities: Arc::new(Capabilities::new(vec![])), + timeout: Arc::new(AtomicU64::new(10)), + last_response_likely_bad: false, + range_info: Some(BlockRangeInfo::new(50, 99, B256::random())), + }; + + // Exact coverage is better than short coverage + assert!(peer_exact + .is_better(&peer_short_start, &BestPeerRequirements::FullBlockRange(range.clone()))); + assert!(peer_exact + .is_better(&peer_short_end, &BestPeerRequirements::FullBlockRange(range.clone()))); + + // Short coverage is not better than exact coverage + assert!(!peer_short_start + .is_better(&peer_exact, &BestPeerRequirements::FullBlockRange(range.clone()))); + assert!( + !peer_short_end.is_better(&peer_exact, &BestPeerRequirements::FullBlockRange(range)) + ); + } } diff --git a/crates/net/network/src/session/types.rs b/crates/net/network/src/session/types.rs index b73bfe3b99..02297c2146 100644 --- a/crates/net/network/src/session/types.rs +++ b/crates/net/network/src/session/types.rs @@ -11,7 +11,7 @@ use std::{ }, }; -/// Information about the range of blocks available from a peer. +/// Information about the range of full blocks available from a peer. /// /// This represents the announced `eth69` /// [`BlockRangeUpdate`] of a peer. @@ -45,12 +45,12 @@ impl BlockRangeInfo { RangeInclusive::new(earliest, latest) } - /// Returns the earliest block number available from the peer. + /// Returns the earliest full block number available from the peer. pub fn earliest(&self) -> u64 { self.inner.earliest.load(Ordering::Relaxed) } - /// Returns the latest block number available from the peer. + /// Returns the latest full block number available from the peer. pub fn latest(&self) -> u64 { self.inner.latest.load(Ordering::Relaxed) } @@ -60,6 +60,11 @@ impl BlockRangeInfo { *self.inner.latest_hash.read() } + /// Returns true if the peer has the full history available. + pub fn has_full_history(&self) -> bool { + self.earliest() == 0 + } + /// Updates the range information. pub fn update(&self, earliest: u64, latest: u64, latest_hash: B256) { self.inner.earliest.store(earliest, Ordering::Relaxed); From cc7edeb354c1bb4f5220a1da681cf0d2a192aac3 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 20 Nov 2025 20:51:49 +0100 Subject: [PATCH 04/23] chore: dont treat invalid fork as fatal (#19888) Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com> --- crates/net/network/src/error.rs | 39 +++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/crates/net/network/src/error.rs b/crates/net/network/src/error.rs index 96ba2ff85e..af3fbd1860 100644 --- a/crates/net/network/src/error.rs +++ b/crates/net/network/src/error.rs @@ -113,7 +113,22 @@ impl SessionError for EthStreamError { P2PHandshakeError::HelloNotInHandshake | P2PHandshakeError::NonHelloMessageInHandshake, )) => true, - Self::EthHandshakeError(err) => !matches!(err, EthHandshakeError::NoResponse), + Self::EthHandshakeError(err) => { + #[allow(clippy::match_same_arms)] + match err { + EthHandshakeError::NoResponse => { + // this happens when the conn simply stalled + false + } + EthHandshakeError::InvalidFork(_) => { + // this can occur when the remote or our node is running an outdated client, + // we shouldn't treat this as fatal, because the node can come back online + // with an updated version any time + false + } + _ => true, + } + } _ => false, } } @@ -144,7 +159,22 @@ impl SessionError for EthStreamError { P2PStreamError::MismatchedProtocolVersion { .. } ) } - Self::EthHandshakeError(err) => !matches!(err, EthHandshakeError::NoResponse), + Self::EthHandshakeError(err) => { + #[allow(clippy::match_same_arms)] + match err { + EthHandshakeError::NoResponse => { + // this happens when the conn simply stalled + false + } + EthHandshakeError::InvalidFork(_) => { + // this can occur when the remote or our node is running an outdated client, + // we shouldn't treat this as fatal, because the node can come back online + // with an updated version any time + false + } + _ => true, + } + } _ => false, } } @@ -196,6 +226,11 @@ impl SessionError for EthStreamError { P2PStreamError::PingerError(_) | P2PStreamError::Snap(_), ) => Some(BackoffKind::Medium), + Self::EthHandshakeError(EthHandshakeError::InvalidFork(_)) => { + // the remote can come back online after updating client version, so we can back off + // for a bit + Some(BackoffKind::Medium) + } _ => None, } } From 2a953a821a1e078db15f83a64a09e60f87ba3d68 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 21 Nov 2025 06:48:32 -0500 Subject: [PATCH 05/23] fix: remove noisy storage proof spans (#19892) --- crates/trie/parallel/src/proof_task.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/crates/trie/parallel/src/proof_task.rs b/crates/trie/parallel/src/proof_task.rs index aa3d013d0d..58dc99fc37 100644 --- a/crates/trie/parallel/src/proof_task.rs +++ b/crates/trie/parallel/src/proof_task.rs @@ -1365,11 +1365,6 @@ where match ctx.missed_leaves_storage_roots.entry(hashed_address) { dashmap::Entry::Occupied(occ) => *occ.get(), dashmap::Entry::Vacant(vac) => { - let _guard = debug_span!( - target: "trie::proof_task", - "Waiting on missed leaf storage proof computation", - ?hashed_address, - ); let root = StorageProof::new_hashed(provider, provider, hashed_address) .with_prefix_set_mut(Default::default()) @@ -1413,11 +1408,6 @@ where // Consume remaining storage proof receivers for accounts not encountered during trie walk. for (hashed_address, receiver) in storage_proof_receivers { - let _guard = debug_span!( - target: "trie::proof_task", - "Blocking on final storage proof", - ?hashed_address, - ); if let Ok(proof_msg) = receiver.recv() { // Extract storage proof from the result if let Ok(ProofResult::StorageProof { proof, .. }) = proof_msg.result { From 0ba122923aa826c0ff0b74b7e602b54e97010fe4 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Date: Fri, 21 Nov 2025 12:48:31 +0000 Subject: [PATCH 06/23] ci: partition cargo-checks job (#19897) --- .github/workflows/lint.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 309a25218b..a3f4358ccc 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -92,7 +92,12 @@ jobs: run: .github/assets/check_rv32imac.sh crate-checks: + name: crate-checks (${{ matrix.partition }}/${{ matrix.total_partitions }}) runs-on: ubuntu-latest + strategy: + matrix: + partition: [1, 2] + total_partitions: [2] timeout-minutes: 30 steps: - uses: actions/checkout@v5 @@ -102,7 +107,7 @@ jobs: - uses: Swatinem/rust-cache@v2 with: cache-on-failure: true - - run: cargo hack check --workspace + - run: cargo hack check --workspace --partition ${{ matrix.partition }}/${{ matrix.total_partitions }} msrv: name: MSRV From 86825ac3b7a35f7c7b1fbaa7b19f2854c0076164 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Date: Fri, 21 Nov 2025 14:23:57 +0000 Subject: [PATCH 07/23] refactor(cli): deduplicate ethereum init_tracing implementations (#19898) --- crates/ethereum/cli/src/app.rs | 22 +++------------------ crates/ethereum/cli/src/interface.rs | 29 +++++++++++++++++++++------- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/crates/ethereum/cli/src/app.rs b/crates/ethereum/cli/src/app.rs index 7a9bad459a..edf648d955 100644 --- a/crates/ethereum/cli/src/app.rs +++ b/crates/ethereum/cli/src/app.rs @@ -10,13 +10,11 @@ use reth_cli_runner::CliRunner; use reth_db::DatabaseEnv; use reth_node_api::NodePrimitives; use reth_node_builder::{NodeBuilder, WithLaunchContext}; -use reth_node_core::args::OtlpInitStatus; use reth_node_ethereum::{consensus::EthBeaconConsensus, EthEvmConfig, EthereumNode}; use reth_node_metrics::recorder::install_prometheus_recorder; use reth_rpc_server_types::RpcModuleValidator; use reth_tracing::{FileWorkerGuard, Layers}; use std::{fmt, sync::Arc}; -use tracing::{info, warn}; /// A wrapper around a parsed CLI that handles command execution. #[derive(Debug)] @@ -107,26 +105,12 @@ where /// Initializes tracing with the configured options. /// - /// If file logging is enabled, this function stores guard to the struct. - /// For gRPC OTLP, it requires tokio runtime context. + /// See [`Cli::init_tracing`] for more information. pub fn init_tracing(&mut self, runner: &CliRunner) -> Result<()> { if self.guard.is_none() { - let mut layers = self.layers.take().unwrap_or_default(); - - let otlp_status = runner.block_on(self.cli.traces.init_otlp_tracing(&mut layers))?; - - self.guard = self.cli.logs.init_tracing_with_layers(layers)?; - info!(target: "reth::cli", "Initialized tracing, debug log directory: {}", self.cli.logs.log_file_directory); - match otlp_status { - OtlpInitStatus::Started(endpoint) => { - info!(target: "reth::cli", "Started OTLP {:?} tracing export to {endpoint}", self.cli.traces.protocol); - } - OtlpInitStatus::NoFeature => { - warn!(target: "reth::cli", "Provided OTLP tracing arguments do not have effect, compile with the `otlp` feature") - } - OtlpInitStatus::Disabled => {} - } + self.guard = self.cli.init_tracing(runner, self.layers.take().unwrap_or_default())?; } + Ok(()) } } diff --git a/crates/ethereum/cli/src/interface.rs b/crates/ethereum/cli/src/interface.rs index f41143bb4f..c1a0f3f84b 100644 --- a/crates/ethereum/cli/src/interface.rs +++ b/crates/ethereum/cli/src/interface.rs @@ -19,14 +19,14 @@ use reth_db::DatabaseEnv; use reth_node_api::NodePrimitives; use reth_node_builder::{NodeBuilder, WithLaunchContext}; use reth_node_core::{ - args::{LogArgs, TraceArgs}, + args::{LogArgs, OtlpInitStatus, TraceArgs}, version::version_metadata, }; use reth_node_metrics::recorder::install_prometheus_recorder; use reth_rpc_server_types::{DefaultRpcModuleValidator, RpcModuleValidator}; -use reth_tracing::FileWorkerGuard; +use reth_tracing::{FileWorkerGuard, Layers}; use std::{ffi::OsString, fmt, future::Future, marker::PhantomData, sync::Arc}; -use tracing::info; +use tracing::{info, warn}; /// The main reth cli interface. /// @@ -205,8 +205,7 @@ impl self.logs.log_file_directory = self.logs.log_file_directory.join(chain_spec.chain().to_string()); } - let _guard = self.init_tracing()?; - info!(target: "reth::cli", "Initialized tracing, debug log directory: {}", self.logs.log_file_directory); + let _guard = self.init_tracing(&runner, Layers::new())?; // Install the prometheus recorder to be sure to record all metrics let _ = install_prometheus_recorder(); @@ -219,11 +218,27 @@ impl /// /// If file logging is enabled, this function returns a guard that must be kept alive to ensure /// that all logs are flushed to disk. + /// /// If an OTLP endpoint is specified, it will export metrics to the configured collector. - pub fn init_tracing(&self) -> eyre::Result> { - let layers = reth_tracing::Layers::new(); + pub fn init_tracing( + &mut self, + runner: &CliRunner, + mut layers: Layers, + ) -> eyre::Result> { + let otlp_status = runner.block_on(self.traces.init_otlp_tracing(&mut layers))?; let guard = self.logs.init_tracing_with_layers(layers)?; + info!(target: "reth::cli", "Initialized tracing, debug log directory: {}", self.logs.log_file_directory); + match otlp_status { + OtlpInitStatus::Started(endpoint) => { + info!(target: "reth::cli", "Started OTLP {:?} tracing export to {endpoint}", self.traces.protocol); + } + OtlpInitStatus::NoFeature => { + warn!(target: "reth::cli", "Provided OTLP tracing arguments do not have effect, compile with the `otlp` feature") + } + OtlpInitStatus::Disabled => {} + } + Ok(guard) } } From a43128277f38a87b15f1bc4795e4db3c4dc8f35f Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Fri, 21 Nov 2025 18:41:28 +0400 Subject: [PATCH 08/23] fix: correctly poll tx fetcher (#19900) --- crates/e2e-test-utils/src/setup_builder.rs | 104 ++++++++++---------- crates/ethereum/node/tests/e2e/p2p.rs | 105 ++++++++++++++++++++- crates/net/network/src/transactions/mod.rs | 39 ++++---- 3 files changed, 177 insertions(+), 71 deletions(-) diff --git a/crates/e2e-test-utils/src/setup_builder.rs b/crates/e2e-test-utils/src/setup_builder.rs index 8de2280fe4..c1eabece9f 100644 --- a/crates/e2e-test-utils/src/setup_builder.rs +++ b/crates/e2e-test-utils/src/setup_builder.rs @@ -4,6 +4,7 @@ //! configurations through closures that modify `NodeConfig` and `TreeConfig`. use crate::{node::NodeTestContext, wallet::Wallet, NodeBuilderHelper, NodeHelperType, TmpDB}; +use futures_util::future::TryJoinAll; use reth_chainspec::EthChainSpec; use reth_engine_local::LocalPayloadAttributesBuilder; use reth_node_builder::{ @@ -15,7 +16,7 @@ use reth_provider::providers::BlockchainProvider; use reth_rpc_server_types::RpcModuleSelection; use reth_tasks::TaskManager; use std::sync::Arc; -use tracing::{span, Level}; +use tracing::{span, Instrument, Level}; /// Type alias for tree config modifier closure type TreeConfigModifier = @@ -122,66 +123,71 @@ where reth_node_api::TreeConfig::default() }; - let mut nodes: Vec> = Vec::with_capacity(self.num_nodes); + let mut nodes = (0..self.num_nodes) + .map(async |idx| { + // Create base node config + let base_config = NodeConfig::new(self.chain_spec.clone()) + .with_network(network_config.clone()) + .with_unused_ports() + .with_rpc( + RpcServerArgs::default() + .with_unused_ports() + .with_http() + .with_http_api(RpcModuleSelection::All), + ); + + // Apply node config modifier if present + let node_config = if let Some(modifier) = &self.node_config_modifier { + modifier(base_config) + } else { + base_config + }; + + let span = span!(Level::INFO, "node", idx); + let node = N::default(); + let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config) + .testing_node(exec.clone()) + .with_types_and_provider::>() + .with_components(node.components_builder()) + .with_add_ons(node.add_ons()) + .launch_with_fn(|builder| { + let launcher = EngineNodeLauncher::new( + builder.task_executor().clone(), + builder.config().datadir(), + tree_config.clone(), + ); + builder.launch_with(launcher) + }) + .instrument(span) + .await?; + + let node = NodeTestContext::new(node, self.attributes_generator).await?; + + let genesis = node.block_hash(0); + node.update_forkchoice(genesis, genesis).await?; + + eyre::Ok(node) + }) + .collect::>() + .await?; for idx in 0..self.num_nodes { - // Create base node config - let base_config = NodeConfig::new(self.chain_spec.clone()) - .with_network(network_config.clone()) - .with_unused_ports() - .with_rpc( - RpcServerArgs::default() - .with_unused_ports() - .with_http() - .with_http_api(RpcModuleSelection::All), - ); - - // Apply node config modifier if present - let node_config = if let Some(modifier) = &self.node_config_modifier { - modifier(base_config) - } else { - base_config - }; - - let span = span!(Level::INFO, "node", idx); - let _enter = span.enter(); - let node = N::default(); - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config) - .testing_node(exec.clone()) - .with_types_and_provider::>() - .with_components(node.components_builder()) - .with_add_ons(node.add_ons()) - .launch_with_fn(|builder| { - let launcher = EngineNodeLauncher::new( - builder.task_executor().clone(), - builder.config().datadir(), - tree_config.clone(), - ); - builder.launch_with(launcher) - }) - .await?; - - let mut node = NodeTestContext::new(node, self.attributes_generator).await?; - - let genesis = node.block_hash(0); - node.update_forkchoice(genesis, genesis).await?; - + let (prev, current) = nodes.split_at_mut(idx); + let current = current.first_mut().unwrap(); // Connect nodes if requested if self.connect_nodes { - if let Some(previous_node) = nodes.last_mut() { - previous_node.connect(&mut node).await; + if let Some(prev_idx) = idx.checked_sub(1) { + prev[prev_idx].connect(current).await; } // Connect last node with the first if there are more than two if idx + 1 == self.num_nodes && self.num_nodes > 2 && - let Some(first_node) = nodes.first_mut() + let Some(first) = prev.first_mut() { - node.connect(first_node).await; + current.connect(first).await; } } - - nodes.push(node); } Ok((nodes, tasks, Wallet::default().with_chain_id(self.chain_spec.chain().into()))) diff --git a/crates/ethereum/node/tests/e2e/p2p.rs b/crates/ethereum/node/tests/e2e/p2p.rs index 34a4210538..74266b1675 100644 --- a/crates/ethereum/node/tests/e2e/p2p.rs +++ b/crates/ethereum/node/tests/e2e/p2p.rs @@ -1,10 +1,18 @@ use crate::utils::{advance_with_random_transactions, eth_payload_attributes}; +use alloy_consensus::{SignableTransaction, TxEip1559, TxEnvelope}; +use alloy_eips::Encodable2718; +use alloy_network::TxSignerSync; use alloy_provider::{Provider, ProviderBuilder}; -use rand::{rngs::StdRng, Rng, SeedableRng}; +use futures::future::JoinAll; +use rand::{rngs::StdRng, seq::IndexedRandom, Rng, SeedableRng}; use reth_chainspec::{ChainSpecBuilder, MAINNET}; -use reth_e2e_test_utils::{setup, setup_engine, transaction::TransactionTestContext}; +use reth_e2e_test_utils::{ + setup, setup_engine, setup_engine_with_connection, transaction::TransactionTestContext, + wallet::Wallet, +}; use reth_node_ethereum::EthereumNode; -use std::sync::Arc; +use reth_rpc_api::EthApiServer; +use std::{sync::Arc, time::Duration}; #[tokio::test] async fn can_sync() -> eyre::Result<()> { @@ -195,3 +203,94 @@ async fn test_reorg_through_backfill() -> eyre::Result<()> { Ok(()) } + +#[tokio::test(flavor = "multi_thread")] +async fn test_tx_propagation() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .cancun_activated() + .prague_activated() + .build(), + ); + + // Setup wallet + let chain_id = chain_spec.chain().into(); + let wallet = Wallet::new(1).inner; + let mut nonce = 0; + let mut build_tx = || { + let mut tx = TxEip1559 { + chain_id, + max_priority_fee_per_gas: 1_000_000_000, + max_fee_per_gas: 1_000_000_000, + gas_limit: 100_000, + nonce, + ..Default::default() + }; + nonce += 1; + let signature = wallet.sign_transaction_sync(&mut tx).unwrap(); + TxEnvelope::Eip1559(tx.into_signed(signature)) + }; + + // Setup 10 nodes + let (mut nodes, _tasks, _) = setup_engine_with_connection::( + 10, + chain_spec.clone(), + false, + Default::default(), + eth_payload_attributes, + false, + ) + .await?; + + // Connect all nodes to the first one + let (first, rest) = nodes.split_at_mut(1); + for node in rest { + node.connect(&mut first[0]).await; + } + + // Advance all nodes for 1 block so that they don't consider themselves unsynced + let tx = build_tx(); + nodes[0].rpc.inject_tx(tx.encoded_2718().into()).await?; + let payload = nodes[0].advance_block().await?; + nodes[1..] + .iter_mut() + .map(|node| async { + node.submit_payload(payload.clone()).await.unwrap(); + node.sync_to(payload.block().hash()).await.unwrap(); + }) + .collect::>() + .await; + + // Build and send transaction to first node + let tx = build_tx(); + let tx_hash = *tx.tx_hash(); + let _ = nodes[0].rpc.inject_tx(tx.encoded_2718().into()).await?; + + tokio::time::sleep(Duration::from_millis(100)).await; + + // Assert that all nodes have the transaction + for (i, node) in nodes.iter().enumerate() { + assert!( + node.rpc.inner.eth_api().transaction_by_hash(tx_hash).await?.is_some(), + "Node {i} should have the transaction" + ); + } + + // Build and send one more transaction to a random node + let tx = build_tx(); + let tx_hash = *tx.tx_hash(); + let _ = nodes.choose(&mut rand::rng()).unwrap().rpc.inject_tx(tx.encoded_2718().into()).await?; + + tokio::time::sleep(Duration::from_millis(100)).await; + + // Assert that all nodes have the transaction + for node in nodes { + assert!(node.rpc.inner.eth_api().transaction_by_hash(tx_hash).await?.is_some()); + } + + Ok(()) +} diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index 22ca0a2376..72c704d3ea 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -1554,25 +1554,6 @@ where this.on_new_pending_transactions(new_txs); } - // Advance inflight fetch requests (flush transaction fetcher and queue for - // import to pool). - // - // The smallest decodable transaction is an empty legacy transaction, 10 bytes - // (2 MiB / 10 bytes > 200k transactions). - // - // Since transactions aren't validated until they are inserted into the pool, - // this can potentially queue >200k transactions for insertion to pool. More - // if the message size is bigger than the soft limit on a `PooledTransactions` - // response which is 2 MiB. - let maybe_more_tx_fetch_events = metered_poll_nested_stream_with_budget!( - poll_durations.acc_fetch_events, - "net::tx", - "Transaction fetch events stream", - DEFAULT_BUDGET_TRY_DRAIN_STREAM, - this.transaction_fetcher.poll_next_unpin(cx), - |event| this.on_fetch_event(event), - ); - // Advance incoming transaction events (stream new txns/announcements from // network manager and queue for import to pool/fetch txns). // @@ -1596,6 +1577,25 @@ where |event| this.on_network_tx_event(event), ); + // Advance inflight fetch requests (flush transaction fetcher and queue for + // import to pool). + // + // The smallest decodable transaction is an empty legacy transaction, 10 bytes + // (2 MiB / 10 bytes > 200k transactions). + // + // Since transactions aren't validated until they are inserted into the pool, + // this can potentially queue >200k transactions for insertion to pool. More + // if the message size is bigger than the soft limit on a `PooledTransactions` + // response which is 2 MiB. + let mut maybe_more_tx_fetch_events = metered_poll_nested_stream_with_budget!( + poll_durations.acc_fetch_events, + "net::tx", + "Transaction fetch events stream", + DEFAULT_BUDGET_TRY_DRAIN_STREAM, + this.transaction_fetcher.poll_next_unpin(cx), + |event| this.on_fetch_event(event), + ); + // Advance pool imports (flush txns to pool). // // Note, this is done in batches. A batch is filled from one `Transactions` @@ -1627,6 +1627,7 @@ where { if this.has_capacity_for_fetching_pending_hashes() { this.on_fetch_hashes_pending_fetch(); + maybe_more_tx_fetch_events = true; } }, poll_durations.acc_pending_fetch From 002e755dd4a7f9854acb05e676dfee9eb41fb579 Mon Sep 17 00:00:00 2001 From: YK Date: Fri, 21 Nov 2025 22:52:55 +0800 Subject: [PATCH 09/23] chore(bench-compare): Add latency distribution stats to reth-bench-compare (#19873) Co-authored-by: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> --- bin/reth-bench-compare/src/comparison.rs | 177 ++++++++++++++++++++--- 1 file changed, 156 insertions(+), 21 deletions(-) diff --git a/bin/reth-bench-compare/src/comparison.rs b/bin/reth-bench-compare/src/comparison.rs index e7f03c6c6e..ad6d801e7c 100644 --- a/bin/reth-bench-compare/src/comparison.rs +++ b/bin/reth-bench-compare/src/comparison.rs @@ -6,6 +6,7 @@ use csv::Reader; use eyre::{eyre, Result, WrapErr}; use serde::{Deserialize, Serialize}; use std::{ + cmp::Ordering, collections::HashMap, fs, path::{Path, PathBuf}, @@ -50,13 +51,21 @@ pub(crate) struct TotalGasRow { pub time: u128, } -/// Summary statistics for a benchmark run +/// Summary statistics for a benchmark run. +/// +/// Latencies are derived from per-block `engine_newPayload` timings (converted from µs to ms): +/// - `mean_new_payload_latency_ms`: arithmetic mean latency across blocks. +/// - `median_new_payload_latency_ms`: p50 latency across blocks. +/// - `p90_new_payload_latency_ms` / `p99_new_payload_latency_ms`: tail latencies across blocks. #[derive(Debug, Clone, Serialize)] pub(crate) struct BenchmarkSummary { pub total_blocks: u64, pub total_gas_used: u64, pub total_duration_ms: u128, - pub avg_new_payload_latency_ms: f64, + pub mean_new_payload_latency_ms: f64, + pub median_new_payload_latency_ms: f64, + pub p90_new_payload_latency_ms: f64, + pub p99_new_payload_latency_ms: f64, pub gas_per_second: f64, pub blocks_per_second: f64, pub min_block_number: u64, @@ -82,10 +91,26 @@ pub(crate) struct RefInfo { pub end_timestamp: Option>, } -/// Summary of the comparison between references +/// Summary of the comparison between references. +/// +/// Percent deltas are `(feature - baseline) / baseline * 100`: +/// - `new_payload_latency_p50_change_percent` / p90 / p99: percent changes of the respective +/// per-block percentiles. +/// - `per_block_latency_change_mean_percent` / `per_block_latency_change_median_percent` are the +/// mean and median of per-block percent deltas (feature vs baseline), capturing block-level +/// drift. +/// - `new_payload_total_latency_change_percent` is the percent change of the total newPayload time +/// across the run. +/// +/// Positive means slower/higher; negative means faster/lower. #[derive(Debug, Serialize)] pub(crate) struct ComparisonSummary { - pub new_payload_latency_change_percent: f64, + pub per_block_latency_change_mean_percent: f64, + pub per_block_latency_change_median_percent: f64, + pub new_payload_total_latency_change_percent: f64, + pub new_payload_latency_p50_change_percent: f64, + pub new_payload_latency_p90_change_percent: f64, + pub new_payload_latency_p99_change_percent: f64, pub gas_per_second_change_percent: f64, pub blocks_per_second_change_percent: f64, } @@ -188,10 +213,12 @@ impl ComparisonGenerator { let feature = self.feature_results.as_ref().ok_or_else(|| eyre!("Feature results not loaded"))?; - // Generate comparison - let comparison_summary = - self.calculate_comparison_summary(&baseline.summary, &feature.summary)?; let per_block_comparisons = self.calculate_per_block_comparisons(baseline, feature)?; + let comparison_summary = self.calculate_comparison_summary( + &baseline.summary, + &feature.summary, + &per_block_comparisons, + )?; let report = ComparisonReport { timestamp: self.timestamp.clone(), @@ -281,7 +308,11 @@ impl ComparisonGenerator { Ok(rows) } - /// Calculate summary statistics for a benchmark run + /// Calculate summary statistics for a benchmark run. + /// + /// Computes latency statistics from per-block `new_payload_latency` values in `combined_data` + /// (converting from µs to ms), and throughput metrics using the total run duration from + /// `total_gas_data`. Percentiles (p50/p90/p99) use linear interpolation on sorted latencies. fn calculate_summary( &self, combined_data: &[CombinedLatencyRow], @@ -296,9 +327,16 @@ impl ComparisonGenerator { let total_duration_ms = total_gas_data.last().unwrap().time / 1000; // Convert microseconds to milliseconds - let avg_new_payload_latency_ms: f64 = - combined_data.iter().map(|r| r.new_payload_latency as f64 / 1000.0).sum::() / - total_blocks as f64; + let latencies_ms: Vec = + combined_data.iter().map(|r| r.new_payload_latency as f64 / 1000.0).collect(); + let mean_new_payload_latency_ms: f64 = + latencies_ms.iter().sum::() / total_blocks as f64; + + let mut sorted_latencies_ms = latencies_ms; + sorted_latencies_ms.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal)); + let median_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.5); + let p90_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.9); + let p99_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.99); let total_duration_seconds = total_duration_ms as f64 / 1000.0; let gas_per_second = if total_duration_seconds > f64::EPSILON { @@ -320,7 +358,10 @@ impl ComparisonGenerator { total_blocks, total_gas_used, total_duration_ms, - avg_new_payload_latency_ms, + mean_new_payload_latency_ms, + median_new_payload_latency_ms, + p90_new_payload_latency_ms, + p99_new_payload_latency_ms, gas_per_second, blocks_per_second, min_block_number, @@ -333,6 +374,7 @@ impl ComparisonGenerator { &self, baseline: &BenchmarkSummary, feature: &BenchmarkSummary, + per_block_comparisons: &[BlockComparison], ) -> Result { let calc_percent_change = |baseline: f64, feature: f64| -> f64 { if baseline.abs() > f64::EPSILON { @@ -342,10 +384,43 @@ impl ComparisonGenerator { } }; + let per_block_percent_changes: Vec = + per_block_comparisons.iter().map(|c| c.new_payload_latency_change_percent).collect(); + let per_block_latency_change_mean_percent = if per_block_percent_changes.is_empty() { + 0.0 + } else { + per_block_percent_changes.iter().sum::() / per_block_percent_changes.len() as f64 + }; + let per_block_latency_change_median_percent = if per_block_percent_changes.is_empty() { + 0.0 + } else { + let mut sorted = per_block_percent_changes; + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal)); + percentile(&sorted, 0.5) + }; + + let baseline_total_latency_ms = + baseline.mean_new_payload_latency_ms * baseline.total_blocks as f64; + let feature_total_latency_ms = + feature.mean_new_payload_latency_ms * feature.total_blocks as f64; + let new_payload_total_latency_change_percent = + calc_percent_change(baseline_total_latency_ms, feature_total_latency_ms); + Ok(ComparisonSummary { - new_payload_latency_change_percent: calc_percent_change( - baseline.avg_new_payload_latency_ms, - feature.avg_new_payload_latency_ms, + per_block_latency_change_mean_percent, + per_block_latency_change_median_percent, + new_payload_total_latency_change_percent, + new_payload_latency_p50_change_percent: calc_percent_change( + baseline.median_new_payload_latency_ms, + feature.median_new_payload_latency_ms, + ), + new_payload_latency_p90_change_percent: calc_percent_change( + baseline.p90_new_payload_latency_ms, + feature.p90_new_payload_latency_ms, + ), + new_payload_latency_p99_change_percent: calc_percent_change( + baseline.p99_new_payload_latency_ms, + feature.p99_new_payload_latency_ms, ), gas_per_second_change_percent: calc_percent_change( baseline.gas_per_second, @@ -450,15 +525,35 @@ impl ComparisonGenerator { println!("Performance Changes:"); println!( - " NewPayload Latency: {:+.2}% (total avg change)", - summary.new_payload_latency_change_percent + " NewPayload Latency per-block mean change: {:+.2}%", + summary.per_block_latency_change_mean_percent ); println!( - " Gas/Second: {:+.2}% (total avg change)", + " NewPayload Latency per-block median change: {:+.2}%", + summary.per_block_latency_change_median_percent + ); + println!( + " Total newPayload time change: {:+.2}%", + summary.new_payload_total_latency_change_percent + ); + println!( + " NewPayload Latency p50: {:+.2}%", + summary.new_payload_latency_p50_change_percent + ); + println!( + " NewPayload Latency p90: {:+.2}%", + summary.new_payload_latency_p90_change_percent + ); + println!( + " NewPayload Latency p99: {:+.2}%", + summary.new_payload_latency_p99_change_percent + ); + println!( + " Gas/Second: {:+.2}%", summary.gas_per_second_change_percent ); println!( - " Blocks/Second: {:+.2}% (total avg change)", + " Blocks/Second: {:+.2}%", summary.blocks_per_second_change_percent ); println!(); @@ -473,7 +568,14 @@ impl ComparisonGenerator { baseline.total_gas_used, baseline.total_duration_ms as f64 / 1000.0 ); - println!(" Avg NewPayload: {:.2}ms", baseline.avg_new_payload_latency_ms); + println!(" NewPayload latency (ms):"); + println!( + " mean: {:.2}, p50: {:.2}, p90: {:.2}, p99: {:.2}", + baseline.mean_new_payload_latency_ms, + baseline.median_new_payload_latency_ms, + baseline.p90_new_payload_latency_ms, + baseline.p99_new_payload_latency_ms + ); if let (Some(start), Some(end)) = (&report.baseline.start_timestamp, &report.baseline.end_timestamp) { @@ -495,7 +597,14 @@ impl ComparisonGenerator { feature.total_gas_used, feature.total_duration_ms as f64 / 1000.0 ); - println!(" Avg NewPayload: {:.2}ms", feature.avg_new_payload_latency_ms); + println!(" NewPayload latency (ms):"); + println!( + " mean: {:.2}, p50: {:.2}, p90: {:.2}, p99: {:.2}", + feature.mean_new_payload_latency_ms, + feature.median_new_payload_latency_ms, + feature.p90_new_payload_latency_ms, + feature.p99_new_payload_latency_ms + ); if let (Some(start), Some(end)) = (&report.feature.start_timestamp, &report.feature.end_timestamp) { @@ -508,3 +617,29 @@ impl ComparisonGenerator { println!(); } } + +/// Calculate percentile using linear interpolation on a sorted slice. +/// +/// Computes `rank = percentile × (n - 1)` where n is the array length. If the rank falls +/// between two indices, linearly interpolates between those values. For example, with 100 values, +/// p90 computes rank = 0.9 × 99 = 89.1, then returns `values[89] × 0.9 + values[90] × 0.1`. +/// +/// Returns 0.0 for empty input. +fn percentile(sorted_values: &[f64], percentile: f64) -> f64 { + if sorted_values.is_empty() { + return 0.0; + } + + let clamped = percentile.clamp(0.0, 1.0); + let max_index = sorted_values.len() - 1; + let rank = clamped * max_index as f64; + let lower = rank.floor() as usize; + let upper = rank.ceil() as usize; + + if lower == upper { + sorted_values[lower] + } else { + let weight = rank - lower as f64; + sorted_values[lower].mul_add(1.0 - weight, sorted_values[upper] * weight) + } +} From b0494a158abe3163ec0be61b6a046483f55aa175 Mon Sep 17 00:00:00 2001 From: Mablr <59505383+mablr@users.noreply.github.com> Date: Fri, 21 Nov 2025 16:52:28 +0100 Subject: [PATCH 10/23] chore(rpc-eth-types): use `FillTransaction` from alloy (#19890) --- Cargo.lock | 120 +++++++++--------- Cargo.toml | 54 ++++---- crates/rpc/rpc-eth-api/src/core.rs | 6 +- .../rpc-eth-api/src/helpers/transaction.rs | 6 +- crates/rpc/rpc-eth-types/src/lib.rs | 3 +- crates/rpc/rpc-eth-types/src/transaction.rs | 12 +- 6 files changed, 96 insertions(+), 105 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f0be65292e..8ac94007dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,9 +112,9 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad704069c12f68d0c742d0cad7e0a03882b42767350584627fbf8a47b1bf1846" +checksum = "8b6440213a22df93a87ed512d2f668e7dc1d62a05642d107f82d61edc9e12370" dependencies = [ "alloy-eips", "alloy-primitives", @@ -140,9 +140,9 @@ dependencies = [ [[package]] name = "alloy-consensus-any" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc374f640a5062224d7708402728e3d6879a514ba10f377da62e7dfb14c673e6" +checksum = "15d0bea09287942405c4f9d2a4f22d1e07611c2dbd9d5bf94b75366340f9e6e0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -155,9 +155,9 @@ dependencies = [ [[package]] name = "alloy-contract" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15c493b2812943f7b58191063a8d13ea97c76099900869c08231e8eba3bf2f92" +checksum = "d69af404f1d00ddb42f2419788fa87746a4cd13bab271916d7726fda6c792d94" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -240,9 +240,9 @@ dependencies = [ [[package]] name = "alloy-eips" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e867b5fd52ed0372a95016f3a37cbff95a9d5409230fbaef2d8ea00e8618098" +checksum = "4bd2c7ae05abcab4483ce821f12f285e01c0b33804e6883dd9ca1569a87ee2be" dependencies = [ "alloy-eip2124", "alloy-eip2930", @@ -288,9 +288,9 @@ dependencies = [ [[package]] name = "alloy-genesis" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90be17e9760a6ba6d13cebdb049cea405ebc8bf57d90664ed708cc5bc348342" +checksum = "fc47eaae86488b07ea8e20236184944072a78784a1f4993f8ec17b3aa5d08c21" dependencies = [ "alloy-eips", "alloy-primitives", @@ -329,9 +329,9 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcab4c51fb1273e3b0f59078e0cdf8aa99f697925b09f0d2055c18be46b4d48c" +checksum = "003f46c54f22854a32b9cc7972660a476968008ad505427eabab49225309ec40" dependencies = [ "alloy-primitives", "alloy-sol-types", @@ -344,9 +344,9 @@ dependencies = [ [[package]] name = "alloy-network" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196d7fd3f5d414f7bbd5886a628b7c42bd98d1b126f9a7cff69dbfd72007b39c" +checksum = "4f4029954d9406a40979f3a3b46950928a0fdcfe3ea8a9b0c17490d57e8aa0e3" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -370,9 +370,9 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d3ae2777e900a7a47ad9e3b8ab58eff3d93628265e73bbdee09acf90bf68f75" +checksum = "7805124ad69e57bbae7731c9c344571700b2a18d351bda9e0eba521c991d1bcb" dependencies = [ "alloy-consensus", "alloy-eips", @@ -444,9 +444,9 @@ dependencies = [ [[package]] name = "alloy-provider" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f9bf40c9b2a90c7677f9c39bccd9f06af457f35362439c0497a706f16557703" +checksum = "d369e12c92870d069e0c9dc5350377067af8a056e29e3badf8446099d7e00889" dependencies = [ "alloy-chains", "alloy-consensus", @@ -489,9 +489,9 @@ dependencies = [ [[package]] name = "alloy-pubsub" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acfdbe41e2ef1a7e79b5ea115baa750f9381ac9088fb600f4cedc731cf04a151" +checksum = "f77d20cdbb68a614c7a86b3ffef607b37d087bb47a03c58f4c3f8f99bc3ace3b" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -533,9 +533,9 @@ dependencies = [ [[package]] name = "alloy-rpc-client" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c2630fde9ff6033a780635e1af6ef40e92d74a9cacb8af3defc1b15cfebca5" +checksum = "31c89883fe6b7381744cbe80fef638ac488ead4f1956a4278956a1362c71cd2e" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -559,9 +559,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad098153a12382c22a597e865530033f5e644473742d6c733562d448125e02a2" +checksum = "64e279e6d40ee40fe8f76753b678d8d5d260cb276dc6c8a8026099b16d2b43f4" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -572,9 +572,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-admin" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7604c415f725bd776d46dae44912c276cc3d8af37f37811e5675389791aa0c6" +checksum = "2bcf50ccb65d29b8599f8f5e23dcac685f1d79459654c830cba381345760e901" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -584,9 +584,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-anvil" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214d9d1033c173ab8fa32edd8a4655cd784447c820b0b66cd0d5167e049567d6" +checksum = "5e176c26fdd87893b6afeb5d92099d8f7e7a1fe11d6f4fe0883d6e33ac5f31ba" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -596,9 +596,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-any" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b8429b5b62d21bf3691eb1ae12aaae9bb496894d5a114e3cc73e27e6800ec8" +checksum = "b43c1622aac2508d528743fd4cfdac1dea92d5a8fa894038488ff7edd0af0b32" dependencies = [ "alloy-consensus-any", "alloy-rpc-types-eth", @@ -607,9 +607,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-beacon" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f67f8269e8b5193a5328dd3ef4d60f93524071e53a993776e290581a59aa15fa" +checksum = "1786681640d4c60f22b6b8376b0f3fa200360bf1c3c2cb913e6c97f51928eb1b" dependencies = [ "alloy-eips", "alloy-primitives", @@ -627,9 +627,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-debug" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01731601ea631bd825c652a225701ab466c09457f446b8d8129368a095389c5d" +checksum = "1b2ca3a434a6d49910a7e8e51797eb25db42ef8a5578c52d877fcb26d0afe7bc" dependencies = [ "alloy-primitives", "derive_more", @@ -639,9 +639,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-engine" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9981491bb98e76099983f516ec7de550db0597031f5828c994961eb4bb993cce" +checksum = "d9c4c53a8b0905d931e7921774a1830609713bd3e8222347963172b03a3ecc68" dependencies = [ "alloy-consensus", "alloy-eips", @@ -660,9 +660,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29031a6bf46177d65efce661f7ab37829ca09dd341bc40afb5194e97600655cc" +checksum = "ed5fafb741c19b3cca4cdd04fa215c89413491f9695a3e928dee2ae5657f607e" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -682,9 +682,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-mev" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5c5c78bdd2c72c47e66ab977af420fb4a10279707d4edbd2575693c47aa54a2" +checksum = "49a97bfc6d9b411c85bb08e1174ddd3e5d61b10d3bd13f529d6609f733cb2f6f" dependencies = [ "alloy-consensus", "alloy-eips", @@ -697,9 +697,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-trace" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b842f5aac6676ff4b2e328262d03bdf49807eaec3fe3a4735c45c97388518b" +checksum = "c55324323aa634b01bdecb2d47462a8dce05f5505b14a6e5db361eef16eda476" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -711,9 +711,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-txpool" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fa12c608873beeb7afa392944dce8829fa8a50c487f266863bb2dd6b743c4a2" +checksum = "96b1aa28effb6854be356ce92ed64cea3b323acd04c3f8bfb5126e2839698043" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -723,9 +723,9 @@ dependencies = [ [[package]] name = "alloy-serde" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01e856112bfa0d9adc85bd7c13db03fad0e71d1d6fb4c2010e475b6718108236" +checksum = "a6f180c399ca7c1e2fe17ea58343910cad0090878a696ff5a50241aee12fc529" dependencies = [ "alloy-primitives", "arbitrary", @@ -735,9 +735,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a4f629da632d5279bbc5731634f0f5c9484ad9c4cad0cd974d9669dc1f46d6" +checksum = "ecc39ad2c0a3d2da8891f4081565780703a593f090f768f884049aa3aa929cbc" dependencies = [ "alloy-primitives", "async-trait", @@ -750,9 +750,9 @@ dependencies = [ [[package]] name = "alloy-signer-local" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76c8950810dc43660c0f22883659c4218e090a5c75dce33fa4ca787715997b7b" +checksum = "930e17cb1e46446a193a593a3bfff8d0ecee4e510b802575ebe300ae2e43ef75" dependencies = [ "alloy-consensus", "alloy-network", @@ -839,9 +839,9 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe215a2f9b51d5f1aa5c8cf22c8be8cdb354934de09c9a4e37aefb79b77552fd" +checksum = "cae82426d98f8bc18f53c5223862907cac30ab8fc5e4cd2bb50808e6d3ab43d8" dependencies = [ "alloy-json-rpc", "auto_impl", @@ -862,9 +862,9 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b37b1a30d23deb3a8746e882c70b384c574d355bc2bbea9ea918b0c31366e" +checksum = "90aa6825760905898c106aba9c804b131816a15041523e80b6d4fe7af6380ada" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -877,9 +877,9 @@ dependencies = [ [[package]] name = "alloy-transport-ipc" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c81a4deeaa0d4b022095db17b286188d731e29ea141d4ec765e166732972e4" +checksum = "6ace83a4a6bb896e5894c3479042e6ba78aa5271dde599aa8c36a021d49cc8cc" dependencies = [ "alloy-json-rpc", "alloy-pubsub", @@ -897,9 +897,9 @@ dependencies = [ [[package]] name = "alloy-transport-ws" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e9d6f5f304e8943afede2680e5fc7008780d4fc49387eafd53192ad95e20091" +checksum = "86c9ab4c199e3a8f3520b60ba81aa67bb21fed9ed0d8304e0569094d0758a56f" dependencies = [ "alloy-pubsub", "alloy-transport", @@ -935,9 +935,9 @@ dependencies = [ [[package]] name = "alloy-tx-macros" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ccf423f6de62e8ce1d6c7a11fb7508ae3536d02e0d68aaeb05c8669337d0937" +checksum = "ae109e33814b49fc0a62f2528993aa8a2dd346c26959b151f05441dc0b9da292" dependencies = [ "darling 0.21.3", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index b08a54c0e2..eb63e8977b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -489,33 +489,33 @@ alloy-trie = { version = "0.9.1", default-features = false } alloy-hardforks = "0.4.4" -alloy-consensus = { version = "1.0.41", default-features = false } -alloy-contract = { version = "1.0.41", default-features = false } -alloy-eips = { version = "1.0.41", default-features = false } -alloy-genesis = { version = "1.0.41", default-features = false } -alloy-json-rpc = { version = "1.0.41", default-features = false } -alloy-network = { version = "1.0.41", default-features = false } -alloy-network-primitives = { version = "1.0.41", default-features = false } -alloy-provider = { version = "1.0.41", features = ["reqwest", "debug-api"], default-features = false } -alloy-pubsub = { version = "1.0.41", default-features = false } -alloy-rpc-client = { version = "1.0.41", default-features = false } -alloy-rpc-types = { version = "1.0.41", features = ["eth"], default-features = false } -alloy-rpc-types-admin = { version = "1.0.41", default-features = false } -alloy-rpc-types-anvil = { version = "1.0.41", default-features = false } -alloy-rpc-types-beacon = { version = "1.0.41", default-features = false } -alloy-rpc-types-debug = { version = "1.0.41", default-features = false } -alloy-rpc-types-engine = { version = "1.0.41", default-features = false } -alloy-rpc-types-eth = { version = "1.0.41", default-features = false } -alloy-rpc-types-mev = { version = "1.0.41", default-features = false } -alloy-rpc-types-trace = { version = "1.0.41", default-features = false } -alloy-rpc-types-txpool = { version = "1.0.41", default-features = false } -alloy-serde = { version = "1.0.41", default-features = false } -alloy-signer = { version = "1.0.41", default-features = false } -alloy-signer-local = { version = "1.0.41", default-features = false } -alloy-transport = { version = "1.0.41" } -alloy-transport-http = { version = "1.0.41", features = ["reqwest-rustls-tls"], default-features = false } -alloy-transport-ipc = { version = "1.0.41", default-features = false } -alloy-transport-ws = { version = "1.0.41", default-features = false } +alloy-consensus = { version = "1.1.2", default-features = false } +alloy-contract = { version = "1.1.2", default-features = false } +alloy-eips = { version = "1.1.2", default-features = false } +alloy-genesis = { version = "1.1.2", default-features = false } +alloy-json-rpc = { version = "1.1.2", default-features = false } +alloy-network = { version = "1.1.2", default-features = false } +alloy-network-primitives = { version = "1.1.2", default-features = false } +alloy-provider = { version = "1.1.2", features = ["reqwest", "debug-api"], default-features = false } +alloy-pubsub = { version = "1.1.2", default-features = false } +alloy-rpc-client = { version = "1.1.2", default-features = false } +alloy-rpc-types = { version = "1.1.2", features = ["eth"], default-features = false } +alloy-rpc-types-admin = { version = "1.1.2", default-features = false } +alloy-rpc-types-anvil = { version = "1.1.2", default-features = false } +alloy-rpc-types-beacon = { version = "1.1.2", default-features = false } +alloy-rpc-types-debug = { version = "1.1.2", default-features = false } +alloy-rpc-types-engine = { version = "1.1.2", default-features = false } +alloy-rpc-types-eth = { version = "1.1.2", default-features = false } +alloy-rpc-types-mev = { version = "1.1.2", default-features = false } +alloy-rpc-types-trace = { version = "1.1.2", default-features = false } +alloy-rpc-types-txpool = { version = "1.1.2", default-features = false } +alloy-serde = { version = "1.1.2", default-features = false } +alloy-signer = { version = "1.1.2", default-features = false } +alloy-signer-local = { version = "1.1.2", default-features = false } +alloy-transport = { version = "1.1.2" } +alloy-transport-http = { version = "1.1.2", features = ["reqwest-rustls-tls"], default-features = false } +alloy-transport-ipc = { version = "1.1.2", default-features = false } +alloy-transport-ws = { version = "1.1.2", default-features = false } # op alloy-op-evm = { version = "0.24.1", default-features = false } diff --git a/crates/rpc/rpc-eth-api/src/core.rs b/crates/rpc/rpc-eth-api/src/core.rs index 4e0afbf6ab..9d8af4b803 100644 --- a/crates/rpc/rpc-eth-api/src/core.rs +++ b/crates/rpc/rpc-eth-api/src/core.rs @@ -18,7 +18,7 @@ use alloy_serde::JsonStorageKey; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; use reth_primitives_traits::TxTy; use reth_rpc_convert::RpcTxReq; -use reth_rpc_eth_types::FillTransactionResult; +use reth_rpc_eth_types::FillTransaction; use reth_rpc_server_types::{result::internal_rpc_err, ToRpcResult}; use tracing::trace; @@ -242,7 +242,7 @@ pub trait EthApi< /// Fills the defaults on a given unsigned transaction. #[method(name = "fillTransaction")] - async fn fill_transaction(&self, request: TxReq) -> RpcResult>; + async fn fill_transaction(&self, request: TxReq) -> RpcResult>; /// Simulate arbitrary number of transactions at an arbitrary blockchain index, with the /// optionality of state overrides @@ -703,7 +703,7 @@ where async fn fill_transaction( &self, request: RpcTxReq, - ) -> RpcResult>> { + ) -> RpcResult>> { trace!(target: "rpc::eth", ?request, "Serving eth_fillTransaction"); Ok(EthTransactions::fill_transaction(self, request).await?) } diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index 2b1f3d0533..2f6c3674ed 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -24,7 +24,7 @@ use reth_rpc_convert::{transaction::RpcConvert, RpcTxReq}; use reth_rpc_eth_types::{ utils::{binary_search, recover_raw_transaction}, EthApiError::{self, TransactionConfirmationTimeout}, - FillTransactionResult, SignError, TransactionSource, + FillTransaction, SignError, TransactionSource, }; use reth_storage_api::{ BlockNumReader, BlockReaderIdExt, ProviderBlock, ProviderReceipt, ProviderTx, ReceiptProvider, @@ -450,7 +450,7 @@ pub trait EthTransactions: LoadTransaction { fn fill_transaction( &self, mut request: RpcTxReq, - ) -> impl Future>, Self::Error>> + Send + ) -> impl Future>, Self::Error>> + Send where Self: EthApiSpec + LoadBlock + EstimateCall + LoadFee, { @@ -511,7 +511,7 @@ pub trait EthTransactions: LoadTransaction { let raw = tx.encoded_2718().into(); - Ok(FillTransactionResult { raw, tx }) + Ok(FillTransaction { raw, tx }) } } diff --git a/crates/rpc/rpc-eth-types/src/lib.rs b/crates/rpc/rpc-eth-types/src/lib.rs index 7378ad9962..8d829aebf4 100644 --- a/crates/rpc/rpc-eth-types/src/lib.rs +++ b/crates/rpc/rpc-eth-types/src/lib.rs @@ -23,6 +23,7 @@ pub mod transaction; pub mod tx_forward; pub mod utils; +pub use alloy_rpc_types_eth::FillTransaction; pub use builder::config::{EthConfig, EthFilterConfig}; pub use cache::{ config::EthStateCacheConfig, db::StateCacheDb, multi_consumer::MultiConsumerLruCache, @@ -35,5 +36,5 @@ pub use gas_oracle::{ }; pub use id_provider::EthSubscriptionIdProvider; pub use pending_block::{PendingBlock, PendingBlockEnv, PendingBlockEnvOrigin}; -pub use transaction::{FillTransactionResult, TransactionSource}; +pub use transaction::TransactionSource; pub use tx_forward::ForwardConfig; diff --git a/crates/rpc/rpc-eth-types/src/transaction.rs b/crates/rpc/rpc-eth-types/src/transaction.rs index 3d099f0118..de3323d61e 100644 --- a/crates/rpc/rpc-eth-types/src/transaction.rs +++ b/crates/rpc/rpc-eth-types/src/transaction.rs @@ -2,21 +2,11 @@ //! //! Transaction wrapper that labels transaction with its origin. -use alloy_primitives::{Bytes, B256}; +use alloy_primitives::B256; use alloy_rpc_types_eth::TransactionInfo; use reth_ethereum_primitives::TransactionSigned; use reth_primitives_traits::{NodePrimitives, Recovered, SignedTransaction}; use reth_rpc_convert::{RpcConvert, RpcTransaction}; -use serde::{Deserialize, Serialize}; - -/// Response type for `eth_fillTransaction` RPC method. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct FillTransactionResult { - /// RLP-encoded transaction bytes - pub raw: Bytes, - /// Filled transaction object - pub tx: T, -} /// Represents from where a transaction was fetched. #[derive(Debug, Clone, Eq, PartialEq)] From 9f3949cd35626f2685073103ccde4a759df72832 Mon Sep 17 00:00:00 2001 From: gustavo <1gusredo@gmail.com> Date: Fri, 21 Nov 2025 15:58:49 +0000 Subject: [PATCH 11/23] chore(examples): complete state_provider_example (#19903) --- examples/db-access/src/main.rs | 48 ++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/examples/db-access/src/main.rs b/examples/db-access/src/main.rs index 83d4f9c9b0..1042ac55be 100644 --- a/examples/db-access/src/main.rs +++ b/examples/db-access/src/main.rs @@ -6,8 +6,8 @@ use reth_ethereum::{ node::EthereumNode, primitives::{AlloyBlockHeader, SealedBlock, SealedHeader}, provider::{ - providers::ReadOnlyConfig, AccountReader, BlockReader, BlockSource, HeaderProvider, - ReceiptProvider, StateProvider, TransactionVariant, TransactionsProvider, + providers::ReadOnlyConfig, AccountReader, BlockNumReader, BlockReader, BlockSource, + HeaderProvider, ReceiptProvider, StateProvider, TransactionVariant, TransactionsProvider, }, rpc::eth::primitives::Filter, TransactionSigned, @@ -39,16 +39,13 @@ fn main() -> eyre::Result<()> { txs_provider_example(&provider)?; receipts_provider_example(&provider)?; + state_provider_example(factory.latest()?, &provider, provider.best_block_number()?)?; + state_provider_example(factory.history_by_block_number(block_num)?, &provider, block_num)?; + // Closes the RO transaction opened in the `factory.provider()` call. This is optional and // would happen anyway at the end of the function scope. drop(provider); - // Run the example against latest state - state_provider_example(factory.latest()?)?; - - // Run it with historical state - state_provider_example(factory.history_by_block_number(block_num)?)?; - Ok(()) } @@ -210,16 +207,39 @@ fn receipts_provider_example< Ok(()) } -fn state_provider_example(provider: T) -> eyre::Result<()> { +/// The `StateProvider` allows querying the state tables. +fn state_provider_example( + provider: T, + headers: &H, + number: u64, +) -> eyre::Result<()> { let address = Address::random(); let storage_key = B256::random(); + let slots = [storage_key]; + + let header = headers.header_by_number(number)?.ok_or(eyre::eyre!("header not found"))?; + let state_root = header.state_root(); // Can get account / storage state with simple point queries - let _account = provider.basic_account(&address)?; - let _code = provider.account_code(&address)?; - let _storage = provider.storage(address, storage_key)?; - // TODO: unimplemented. - // let _proof = provider.proof(address, &[])?; + let account = provider.basic_account(&address)?; + let code = provider.account_code(&address)?; + let storage_value = provider.storage(address, storage_key)?; + + println!( + "state at block #{number}: addr={address:?}, nonce={}, balance={}, storage[{:?}]={:?}, has_code={}", + account.as_ref().map(|acc| acc.nonce).unwrap_or_default(), + account.as_ref().map(|acc| acc.balance).unwrap_or_default(), + storage_key, + storage_value, + code.is_some() + ); + + // Returns a bundled proof with the account's info + let proof = provider.proof(Default::default(), address, &slots)?; + + // Can verify the returned proof against the state root + proof.verify(state_root)?; + println!("account proof verified against state root {state_root:?}"); Ok(()) } From 5e0732404c551efdb0d2d40be1d2c391ce61442f Mon Sep 17 00:00:00 2001 From: Francis Li Date: Fri, 21 Nov 2025 10:25:05 -0800 Subject: [PATCH 12/23] chore(op-alloy): update op-alloy to v0.22.4 (#19905) --- Cargo.lock | 20 ++++++++++---------- Cargo.toml | 10 +++++----- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8ac94007dd..bf9b755927 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6174,9 +6174,9 @@ dependencies = [ [[package]] name = "op-alloy-consensus" -version = "0.22.3" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e82f4f768ba39e52a4efe1b8f3425c04ab0d0e6f90c003fe97e5444cd963405e" +checksum = "726da827358a547be9f1e37c2a756b9e3729cb0350f43408164794b370cad8ae" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6200,9 +6200,9 @@ checksum = "a79f352fc3893dcd670172e615afef993a41798a1d3fc0db88a3e60ef2e70ecc" [[package]] name = "op-alloy-network" -version = "0.22.3" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2607d0d985f848f98fa79068d11c612f8476dba7deb7498881794bf51b3cfb5" +checksum = "f63f27e65be273ec8fcb0b6af0fd850b550979465ab93423705ceb3dfddbd2ab" dependencies = [ "alloy-consensus", "alloy-network", @@ -6231,9 +6231,9 @@ dependencies = [ [[package]] name = "op-alloy-rpc-jsonrpsee" -version = "0.22.3" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6911db73a4bf59bf8a963dec153ada1057fa426fdc35e0b35fe82657af3501a3" +checksum = "8ef9114426b16172254555aad34a8ea96c01895e40da92f5d12ea680a1baeaa7" dependencies = [ "alloy-primitives", "jsonrpsee", @@ -6241,9 +6241,9 @@ dependencies = [ [[package]] name = "op-alloy-rpc-types" -version = "0.22.3" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "890b51c3a619c263d52ee5a945dce173a4052d017f93bf5698613b21cbe0d237" +checksum = "562dd4462562c41f9fdc4d860858c40e14a25df7f983ae82047f15f08fce4d19" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6261,9 +6261,9 @@ dependencies = [ [[package]] name = "op-alloy-rpc-types-engine" -version = "0.22.3" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c92f9dd709b3a769b7604d4d2257846b6de3d3f60e5163982cc4e90c0d0b6f95" +checksum = "d8f24b8cb66e4b33e6c9e508bf46b8ecafc92eadd0b93fedd306c0accb477657" dependencies = [ "alloy-consensus", "alloy-eips", diff --git a/Cargo.toml b/Cargo.toml index eb63e8977b..b1a802b587 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -520,11 +520,11 @@ alloy-transport-ws = { version = "1.1.2", default-features = false } # op alloy-op-evm = { version = "0.24.1", default-features = false } alloy-op-hardforks = "0.4.4" -op-alloy-rpc-types = { version = "0.22.3", default-features = false } -op-alloy-rpc-types-engine = { version = "0.22.3", default-features = false } -op-alloy-network = { version = "0.22.3", default-features = false } -op-alloy-consensus = { version = "0.22.3", default-features = false } -op-alloy-rpc-jsonrpsee = { version = "0.22.3", default-features = false } +op-alloy-rpc-types = { version = "0.22.4", default-features = false } +op-alloy-rpc-types-engine = { version = "0.22.4", default-features = false } +op-alloy-network = { version = "0.22.4", default-features = false } +op-alloy-consensus = { version = "0.22.4", default-features = false } +op-alloy-rpc-jsonrpsee = { version = "0.22.4", default-features = false } op-alloy-flz = { version = "0.13.1", default-features = false } # misc From 39ef6216fb5b3bb7467553c58da27b98a5832f0b Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com> Date: Fri, 21 Nov 2025 19:34:16 +0000 Subject: [PATCH 13/23] feat(provider, static-file): transaction senders segment (#19508) Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com> Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- crates/cli/commands/src/db/get.rs | 12 + crates/cli/commands/src/db/settings.rs | 19 +- crates/cli/commands/src/stage/drop.rs | 12 +- crates/config/src/config.rs | 13 +- crates/node/core/src/args/static_files.rs | 25 +- crates/stages/stages/src/stages/bodies.rs | 5 +- crates/stages/stages/src/stages/execution.rs | 9 +- .../stages/src/stages/sender_recovery.rs | 68 +++- crates/static-file/types/src/segment.rs | 24 +- ...s__segment__tests__TransactionSenders.snap | 5 + crates/storage/db-api/src/models/metadata.rs | 30 +- crates/storage/db-api/src/transaction.rs | 3 + crates/storage/db-models/src/blocks.rs | 5 + crates/storage/db/src/static_file/masks.rs | 12 +- crates/storage/provider/src/either_writer.rs | 314 +++++++++++++++--- .../src/providers/database/metrics.rs | 10 + .../provider/src/providers/database/mod.rs | 19 +- .../src/providers/database/provider.rs | 74 +++-- .../provider/src/providers/static_file/jar.rs | 48 +-- .../src/providers/static_file/manager.rs | 119 +++++-- .../provider/src/providers/static_file/mod.rs | 17 +- .../src/providers/static_file/writer.rs | 176 ++++++++-- .../storage-api/src/database_provider.rs | 3 +- docs/vocs/docs/pages/cli/SUMMARY.mdx | 1 + docs/vocs/docs/pages/cli/reth/db.mdx | 10 + .../pages/cli/reth/db/clear/static-file.mdx | 7 +- .../pages/cli/reth/db/get/static-file.mdx | 7 +- .../docs/pages/cli/reth/db/settings/set.mdx | 5 +- .../transaction_senders_in_static_files.mdx | 143 ++++++++ .../cli/reth/db/static-file-header/block.mdx | 7 +- docs/vocs/docs/pages/cli/reth/download.mdx | 10 + docs/vocs/docs/pages/cli/reth/export-era.mdx | 10 + docs/vocs/docs/pages/cli/reth/import-era.mdx | 10 + docs/vocs/docs/pages/cli/reth/import.mdx | 10 + docs/vocs/docs/pages/cli/reth/init-state.mdx | 10 + docs/vocs/docs/pages/cli/reth/init.mdx | 10 + docs/vocs/docs/pages/cli/reth/node.mdx | 10 + docs/vocs/docs/pages/cli/reth/prune.mdx | 10 + docs/vocs/docs/pages/cli/reth/re-execute.mdx | 10 + docs/vocs/docs/pages/cli/reth/stage/drop.mdx | 10 + docs/vocs/docs/pages/cli/reth/stage/dump.mdx | 10 + docs/vocs/docs/pages/cli/reth/stage/run.mdx | 10 + .../vocs/docs/pages/cli/reth/stage/unwind.mdx | 10 + 43 files changed, 1115 insertions(+), 217 deletions(-) create mode 100644 crates/static-file/types/src/snapshots/reth_static_file_types__segment__tests__TransactionSenders.snap create mode 100644 docs/vocs/docs/pages/cli/reth/db/settings/set/transaction_senders_in_static_files.mdx diff --git a/crates/cli/commands/src/db/get.rs b/crates/cli/commands/src/db/get.rs index 2f0fc05311..f3727b6dc6 100644 --- a/crates/cli/commands/src/db/get.rs +++ b/crates/cli/commands/src/db/get.rs @@ -3,6 +3,7 @@ use clap::Parser; use reth_db::{ static_file::{ ColumnSelectorOne, ColumnSelectorTwo, HeaderWithHashMask, ReceiptMask, TransactionMask, + TransactionSenderMask, }, RawDupSort, }; @@ -75,6 +76,10 @@ impl Command { StaticFileSegment::Receipts => { (table_key::(&key)?, >>::MASK) } + StaticFileSegment::TransactionSenders => ( + table_key::(&key)?, + ::MASK, + ), }; let content = tool @@ -114,6 +119,13 @@ impl Command { )?; println!("{}", serde_json::to_string_pretty(&receipt)?); } + StaticFileSegment::TransactionSenders => { + let sender = + <::Value>::decompress( + content[0].as_slice(), + )?; + println!("{}", serde_json::to_string_pretty(&sender)?); + } } } } diff --git a/crates/cli/commands/src/db/settings.rs b/crates/cli/commands/src/db/settings.rs index 6b5169aeda..b4d718d803 100644 --- a/crates/cli/commands/src/db/settings.rs +++ b/crates/cli/commands/src/db/settings.rs @@ -44,6 +44,11 @@ pub enum SetCommand { #[clap(action(ArgAction::Set))] value: bool, }, + /// Store transaction senders in static files instead of the database + TransactionSendersInStaticFiles { + #[clap(action(ArgAction::Set))] + value: bool, + }, } impl Command { @@ -83,8 +88,10 @@ impl Command { println!("No storage settings found, creating new settings."); } - let mut settings @ StorageSettings { receipts_in_static_files: _ } = - settings.unwrap_or_default(); + let mut settings @ StorageSettings { + receipts_in_static_files: _, + transaction_senders_in_static_files: _, + } = settings.unwrap_or_else(StorageSettings::legacy); // Update the setting based on the key match cmd { @@ -96,6 +103,14 @@ impl Command { settings.receipts_in_static_files = value; println!("Set receipts_in_static_files = {}", value); } + SetCommand::TransactionSendersInStaticFiles { value } => { + if settings.transaction_senders_in_static_files == value { + println!("transaction_senders_in_static_files is already set to {}", value); + return Ok(()); + } + settings.transaction_senders_in_static_files = value; + println!("Set transaction_senders_in_static_files = {}", value); + } } // Write updated settings diff --git a/crates/cli/commands/src/stage/drop.rs b/crates/cli/commands/src/stage/drop.rs index 2dfdd1cafa..66505b9046 100644 --- a/crates/cli/commands/src/stage/drop.rs +++ b/crates/cli/commands/src/stage/drop.rs @@ -45,6 +45,7 @@ impl Command { StageEnum::Headers => Some(StaticFileSegment::Headers), StageEnum::Bodies => Some(StaticFileSegment::Transactions), StageEnum::Execution => Some(StaticFileSegment::Receipts), + StageEnum::Senders => Some(StaticFileSegment::TransactionSenders), _ => None, }; @@ -68,17 +69,24 @@ impl Command { StaticFileSegment::Transactions => { let to_delete = static_file_provider .get_highest_static_file_tx(static_file_segment) - .map(|tx| tx + 1) + .map(|tx_num| tx_num + 1) .unwrap_or_default(); writer.prune_transactions(to_delete, 0)?; } StaticFileSegment::Receipts => { let to_delete = static_file_provider .get_highest_static_file_tx(static_file_segment) - .map(|receipt| receipt + 1) + .map(|tx_num| tx_num + 1) .unwrap_or_default(); writer.prune_receipts(to_delete, 0)?; } + StaticFileSegment::TransactionSenders => { + let to_delete = static_file_provider + .get_highest_static_file_tx(static_file_segment) + .map(|tx_num| tx_num + 1) + .unwrap_or_default(); + writer.prune_transaction_senders(to_delete, 0)?; + } } } } diff --git a/crates/config/src/config.rs b/crates/config/src/config.rs index 66c0fdd53e..013df5f4aa 100644 --- a/crates/config/src/config.rs +++ b/crates/config/src/config.rs @@ -436,6 +436,8 @@ pub struct BlocksPerFileConfig { pub transactions: Option, /// Number of blocks per file for the receipts segment. pub receipts: Option, + /// Number of blocks per file for the transaction senders segment. + pub transaction_senders: Option, } impl StaticFilesConfig { @@ -443,7 +445,8 @@ impl StaticFilesConfig { /// /// Returns an error if any blocks per file value is zero. pub fn validate(&self) -> eyre::Result<()> { - let BlocksPerFileConfig { headers, transactions, receipts } = self.blocks_per_file; + let BlocksPerFileConfig { headers, transactions, receipts, transaction_senders } = + self.blocks_per_file; eyre::ensure!(headers != Some(0), "Headers segment blocks per file must be greater than 0"); eyre::ensure!( transactions != Some(0), @@ -453,12 +456,17 @@ impl StaticFilesConfig { receipts != Some(0), "Receipts segment blocks per file must be greater than 0" ); + eyre::ensure!( + transaction_senders != Some(0), + "Transaction senders segment blocks per file must be greater than 0" + ); Ok(()) } /// Converts the blocks per file configuration into a [`HashMap`] per segment. pub fn as_blocks_per_file_map(&self) -> HashMap { - let BlocksPerFileConfig { headers, transactions, receipts } = self.blocks_per_file; + let BlocksPerFileConfig { headers, transactions, receipts, transaction_senders } = + self.blocks_per_file; let mut map = HashMap::new(); // Iterating over all possible segments allows us to do an exhaustive match here, @@ -468,6 +476,7 @@ impl StaticFilesConfig { StaticFileSegment::Headers => headers, StaticFileSegment::Transactions => transactions, StaticFileSegment::Receipts => receipts, + StaticFileSegment::TransactionSenders => transaction_senders, }; if let Some(blocks_per_file) = blocks_per_file { diff --git a/crates/node/core/src/args/static_files.rs b/crates/node/core/src/args/static_files.rs index 9de5b14643..5f7a1510ee 100644 --- a/crates/node/core/src/args/static_files.rs +++ b/crates/node/core/src/args/static_files.rs @@ -20,6 +20,10 @@ pub struct StaticFilesArgs { #[arg(long = "static-files.blocks-per-file.receipts")] pub blocks_per_file_receipts: Option, + /// Number of blocks per file for the transaction senders segment. + #[arg(long = "static-files.blocks-per-file.transaction-senders")] + pub blocks_per_file_transaction_senders: Option, + /// Store receipts in static files instead of the database. /// /// When enabled, receipts will be written to static files on disk instead of the database. @@ -28,6 +32,16 @@ pub struct StaticFilesArgs { /// the node has been initialized, changing this flag requires re-syncing from scratch. #[arg(long = "static-files.receipts")] pub receipts: bool, + + /// Store transaction senders in static files instead of the database. + /// + /// When enabled, transaction senders will be written to static files on disk instead of the + /// database. + /// + /// Note: This setting can only be configured at genesis initialization. Once + /// the node has been initialized, changing this flag requires re-syncing from scratch. + #[arg(long = "static-files.transaction-senders")] + pub transaction_senders: bool, } impl StaticFilesArgs { @@ -41,16 +55,17 @@ impl StaticFilesArgs { .blocks_per_file_transactions .or(config.blocks_per_file.transactions), receipts: self.blocks_per_file_receipts.or(config.blocks_per_file.receipts), + transaction_senders: self + .blocks_per_file_transaction_senders + .or(config.blocks_per_file.transaction_senders), }, } } /// Converts the static files arguments into [`StorageSettings`]. pub const fn to_settings(&self) -> StorageSettings { - if self.receipts { - StorageSettings::new().with_receipts_in_static_files() - } else { - StorageSettings::legacy() - } + StorageSettings::legacy() + .with_receipts_in_static_files(self.receipts) + .with_transaction_senders_in_static_files(self.transaction_senders) } } diff --git a/crates/stages/stages/src/stages/bodies.rs b/crates/stages/stages/src/stages/bodies.rs index 7b6090ca86..bf1fc9bef9 100644 --- a/crates/stages/stages/src/stages/bodies.rs +++ b/crates/stages/stages/src/stages/bodies.rs @@ -772,8 +772,9 @@ mod tests { *range.start()..*range.end() + 1, |cursor, number| cursor.get_two::>(number.into()), )? { - let (header, hash) = header?; - self.headers.push_back(SealedHeader::new(header, hash)); + if let Some((header, hash)) = header? { + self.headers.push_back(SealedHeader::new(header, hash)); + } } Ok(()) diff --git a/crates/stages/stages/src/stages/execution.rs b/crates/stages/stages/src/stages/execution.rs index 8dd76e56b9..c8428f93f3 100644 --- a/crates/stages/stages/src/stages/execution.rs +++ b/crates/stages/stages/src/stages/execution.rs @@ -653,8 +653,9 @@ where *range.start()..*range.end() + 1, |cursor, number| cursor.get_one::>(number.into()), )? { - let entry = entry?; - gas_total += entry.gas_used(); + if let Some(entry) = entry? { + gas_total += entry.gas_used(); + } } let duration = start.elapsed(); @@ -1252,7 +1253,9 @@ mod tests { // but no receipt data is written. let factory = create_test_provider_factory(); - factory.set_storage_settings_cache(StorageSettings::new().with_receipts_in_static_files()); + factory.set_storage_settings_cache( + StorageSettings::legacy().with_receipts_in_static_files(true), + ); // Setup with block 1 let provider_rw = factory.database_provider_rw().unwrap(); diff --git a/crates/stages/stages/src/stages/sender_recovery.rs b/crates/stages/stages/src/stages/sender_recovery.rs index a7cd8128f3..818ba5af08 100644 --- a/crates/stages/stages/src/stages/sender_recovery.rs +++ b/crates/stages/stages/src/stages/sender_recovery.rs @@ -1,4 +1,4 @@ -use alloy_primitives::{Address, TxNumber}; +use alloy_primitives::{Address, BlockNumber, TxNumber}; use reth_config::config::SenderRecoveryConfig; use reth_consensus::ConsensusError; use reth_db::static_file::TransactionMask; @@ -11,8 +11,8 @@ use reth_db_api::{ }; use reth_primitives_traits::{GotExpected, NodePrimitives, SignedTransaction}; use reth_provider::{ - BlockReader, DBProvider, HeaderProvider, ProviderError, PruneCheckpointReader, - StaticFileProviderFactory, StatsReader, + BlockReader, DBProvider, EitherWriter, HeaderProvider, ProviderError, PruneCheckpointReader, + StaticFileProviderFactory, StatsReader, StorageSettingsCache, TransactionsProvider, }; use reth_prune_types::PruneSegment; use reth_stages_api::{ @@ -20,7 +20,7 @@ use reth_stages_api::{ StageId, UnwindInput, UnwindOutput, }; use reth_static_file_types::StaticFileSegment; -use std::{fmt::Debug, ops::Range, sync::mpsc}; +use std::{fmt::Debug, ops::Range, sync::mpsc, time::Instant}; use thiserror::Error; use tracing::*; @@ -64,7 +64,8 @@ where + BlockReader + StaticFileProviderFactory> + StatsReader - + PruneCheckpointReader, + + PruneCheckpointReader + + StorageSettingsCache, { /// Return the id of the stage fn id(&self) -> StageId { @@ -74,7 +75,8 @@ where /// Retrieve the range of transactions to iterate over by querying /// [`BlockBodyIndices`][reth_db_api::tables::BlockBodyIndices], /// collect transactions within that range, recover signer for each transaction and store - /// entries in the [`TransactionSenders`][reth_db_api::tables::TransactionSenders] table. + /// entries in the [`TransactionSenders`][reth_db_api::tables::TransactionSenders] table or + /// static files depending on configuration. fn execute(&mut self, provider: &Provider, input: ExecInput) -> Result { if input.target_reached() { return Ok(ExecOutput::done(input.checkpoint())) @@ -84,6 +86,14 @@ where input.next_block_range_with_transaction_threshold(provider, self.commit_threshold)? else { info!(target: "sync::stages::sender_recovery", "No transaction senders to recover"); + EitherWriter::new_senders( + provider, + provider + .static_file_provider() + .get_highest_static_file_block(StaticFileSegment::TransactionSenders) + .unwrap_or_default(), + )? + .ensure_at_block(input.target())?; return Ok(ExecOutput { checkpoint: StageCheckpoint::new(input.target()) .with_entities_stage_checkpoint(stage_checkpoint(provider)?), @@ -92,8 +102,7 @@ where }; let end_block = *range_output.block_range.end(); - // Acquire the cursor for inserting elements - let mut senders_cursor = provider.tx_ref().cursor_write::()?; + let mut writer = EitherWriter::new_senders(provider, *range_output.block_range.start())?; info!(target: "sync::stages::sender_recovery", tx_range = ?range_output.tx_range, "Recovering senders"); @@ -107,8 +116,28 @@ where let tx_batch_sender = setup_range_recovery(provider); + let start = Instant::now(); + let block_body_indices = + provider.block_body_indices_range(range_output.block_range.clone())?; + let block_body_indices_elapsed = start.elapsed(); + let mut blocks_with_indices = range_output.block_range.zip(block_body_indices).peekable(); + for range in batch { - recover_range(range, provider, tx_batch_sender.clone(), &mut senders_cursor)?; + // Pair each transaction number with its block number + let start = Instant::now(); + let block_numbers = range.clone().fold(Vec::new(), |mut block_numbers, tx| { + while let Some((block, index)) = blocks_with_indices.peek() { + if index.contains_tx(tx) { + block_numbers.push(*block); + return block_numbers + } + blocks_with_indices.next(); + } + block_numbers + }); + let fold_elapsed = start.elapsed(); + debug!(target: "sync::stages::sender_recovery", ?block_body_indices_elapsed, ?fold_elapsed, len = block_numbers.len(), "Calculated block numbers"); + recover_range(range, block_numbers, provider, tx_batch_sender.clone(), &mut writer)?; } Ok(ExecOutput { @@ -141,15 +170,22 @@ where } fn recover_range( - tx_range: Range, + tx_range: Range, + block_numbers: Vec, provider: &Provider, tx_batch_sender: mpsc::Sender, RecoveryResultSender)>>, - senders_cursor: &mut CURSOR, + writer: &mut EitherWriter<'_, CURSOR, Provider::Primitives>, ) -> Result<(), StageError> where - Provider: DBProvider + HeaderProvider + StaticFileProviderFactory, + Provider: DBProvider + HeaderProvider + TransactionsProvider + StaticFileProviderFactory, CURSOR: DbCursorRW, { + debug_assert_eq!( + tx_range.clone().count(), + block_numbers.len(), + "Transaction range and block numbers count mismatch" + ); + debug!(target: "sync::stages::sender_recovery", ?tx_range, "Sending batch for processing"); // Preallocate channels for each chunks in the batch @@ -171,6 +207,7 @@ where debug!(target: "sync::stages::sender_recovery", ?tx_range, "Appending recovered senders to the database"); let mut processed_transactions = 0; + let mut block_numbers = block_numbers.into_iter(); for channel in receivers { while let Ok(recovered) = channel.recv() { let (tx_id, sender) = match recovered { @@ -208,7 +245,12 @@ where } } }; - senders_cursor.append(tx_id, &sender)?; + + let new_block_number = block_numbers + .next() + .expect("block numbers iterator has the same length as the number of transactions"); + writer.ensure_at_block(new_block_number)?; + writer.append_sender(tx_id, &sender)?; processed_transactions += 1; } } diff --git a/crates/static-file/types/src/segment.rs b/crates/static-file/types/src/segment.rs index a7e8e09ac8..cd57f019ba 100644 --- a/crates/static-file/types/src/segment.rs +++ b/crates/static-file/types/src/segment.rs @@ -2,7 +2,6 @@ use crate::{BlockNumber, Compression}; use alloc::{format, string::String}; use alloy_primitives::TxNumber; use core::{ops::RangeInclusive, str::FromStr}; -use derive_more::Display; use serde::{Deserialize, Serialize}; use strum::{EnumIs, EnumString}; @@ -18,7 +17,7 @@ use strum::{EnumIs, EnumString}; Deserialize, Serialize, EnumString, - Display, + derive_more::Display, EnumIs, )] #[strum(serialize_all = "kebab-case")] @@ -32,10 +31,12 @@ pub enum StaticFileSegment { Transactions, /// Static File segment responsible for the `Receipts` table. Receipts, + /// Static File segment responsible for the `TransactionSenders` table. + TransactionSenders, } impl StaticFileSegment { - /// Returns the segment as a string. + /// Returns a string representation of the segment. pub const fn as_str(&self) -> &'static str { // `strum` doesn't generate a doc comment for `into_str` when using `IntoStaticStr` derive // macro, so we need to manually implement it. @@ -46,13 +47,14 @@ impl StaticFileSegment { Self::Headers => "headers", Self::Transactions => "transactions", Self::Receipts => "receipts", + Self::TransactionSenders => "transaction-senders", } } /// Returns an iterator over all segments. pub fn iter() -> impl Iterator { // The order of segments is significant and must be maintained to ensure correctness. - [Self::Headers, Self::Transactions, Self::Receipts].into_iter() + [Self::Headers, Self::Transactions, Self::Receipts, Self::TransactionSenders].into_iter() } /// Returns the default configuration of the segment. @@ -64,7 +66,7 @@ impl StaticFileSegment { pub const fn columns(&self) -> usize { match self { Self::Headers => 3, - Self::Transactions | Self::Receipts => 1, + Self::Transactions | Self::Receipts | Self::TransactionSenders => 1, } } @@ -125,7 +127,7 @@ impl StaticFileSegment { /// Returns `true` if a segment row is linked to a transaction. pub const fn is_tx_based(&self) -> bool { match self { - Self::Receipts | Self::Transactions => true, + Self::Receipts | Self::Transactions | Self::TransactionSenders => true, Self::Headers => false, } } @@ -134,7 +136,7 @@ impl StaticFileSegment { pub const fn is_block_based(&self) -> bool { match self { Self::Headers => true, - Self::Receipts | Self::Transactions => false, + Self::Receipts | Self::Transactions | Self::TransactionSenders => false, } } } @@ -450,6 +452,12 @@ mod tests { tx_range: Some(SegmentRangeInclusive::new(0, 300)), segment: StaticFileSegment::Receipts, }, + SegmentHeader { + expected_block_range: SegmentRangeInclusive::new(0, 200), + block_range: Some(SegmentRangeInclusive::new(0, 100)), + tx_range: Some(SegmentRangeInclusive::new(0, 300)), + segment: StaticFileSegment::TransactionSenders, + }, ]; // Check that we test all segments assert_eq!( @@ -481,6 +489,7 @@ mod tests { StaticFileSegment::Headers => "headers", StaticFileSegment::Transactions => "transactions", StaticFileSegment::Receipts => "receipts", + StaticFileSegment::TransactionSenders => "transaction-senders", }; assert_eq!(static_str, expected_str); } @@ -497,6 +506,7 @@ mod tests { StaticFileSegment::Headers => "Headers", StaticFileSegment::Transactions => "Transactions", StaticFileSegment::Receipts => "Receipts", + StaticFileSegment::TransactionSenders => "TransactionSenders", }; assert_eq!(ser, format!("\"{expected_str}\"")); } diff --git a/crates/static-file/types/src/snapshots/reth_static_file_types__segment__tests__TransactionSenders.snap b/crates/static-file/types/src/snapshots/reth_static_file_types__segment__tests__TransactionSenders.snap new file mode 100644 index 0000000000..c4c85485cc --- /dev/null +++ b/crates/static-file/types/src/snapshots/reth_static_file_types__segment__tests__TransactionSenders.snap @@ -0,0 +1,5 @@ +--- +source: crates/static-file/types/src/segment.rs +expression: "Bytes::from(serialized)" +--- +0x01000000000000000000000000000000c80000000000000001000000000000000064000000000000000100000000000000002c010000000000000300000001000000000000000000000000000000000000000000000000 diff --git a/crates/storage/db-api/src/models/metadata.rs b/crates/storage/db-api/src/models/metadata.rs index 9c2fe405eb..e562ede03b 100644 --- a/crates/storage/db-api/src/models/metadata.rs +++ b/crates/storage/db-api/src/models/metadata.rs @@ -7,33 +7,39 @@ use serde::{Deserialize, Serialize}; /// /// These should be set during `init_genesis` or `init_db` depending on whether we want dictate /// behaviour of new or old nodes respectively. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, Compact)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Compact)] #[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))] #[add_arbitrary_tests(compact)] pub struct StorageSettings { /// Whether this node always writes receipts to static files. /// /// If this is set to FALSE AND receipt pruning IS ENABLED, all receipts should be written to DB. Otherwise, they should be written to static files. This ensures that older nodes do not need to migrate their current DB tables to static files. For more, read: + #[serde(default)] pub receipts_in_static_files: bool, + /// Whether this node always writes transaction senders to static files. + #[serde(default)] + pub transaction_senders_in_static_files: bool, } impl StorageSettings { - /// Creates a new `StorageSettings` with default values. - pub const fn new() -> Self { - Self { receipts_in_static_files: false } - } - /// Creates `StorageSettings` for legacy nodes. /// - /// This explicitly sets `receipts_in_static_files` to `false`, ensuring older nodes - /// continue writing receipts to the database when receipt pruning is enabled. + /// This explicitly sets `receipts_in_static_files` and `transaction_senders_in_static_files` to + /// `false`, ensuring older nodes continue writing receipts and transaction senders to the + /// database when receipt pruning is enabled. pub const fn legacy() -> Self { - Self { receipts_in_static_files: false } + Self { receipts_in_static_files: false, transaction_senders_in_static_files: false } } - /// Sets the `receipts_static_files` flag to true. - pub const fn with_receipts_in_static_files(mut self) -> Self { - self.receipts_in_static_files = true; + /// Sets the `receipts_in_static_files` flag to the provided value. + pub const fn with_receipts_in_static_files(mut self, value: bool) -> Self { + self.receipts_in_static_files = value; + self + } + + /// Sets the `transaction_senders_in_static_files` flag to the provided value. + pub const fn with_transaction_senders_in_static_files(mut self, value: bool) -> Self { + self.transaction_senders_in_static_files = value; self } } diff --git a/crates/storage/db-api/src/transaction.rs b/crates/storage/db-api/src/transaction.rs index 604a4d6bb5..16882ff7c6 100644 --- a/crates/storage/db-api/src/transaction.rs +++ b/crates/storage/db-api/src/transaction.rs @@ -5,6 +5,9 @@ use crate::{ }; use std::fmt::Debug; +/// Helper adapter type for accessing [`DbTx`] cursor. +pub type CursorTy = ::Cursor; + /// Helper adapter type for accessing [`DbTxMut`] mutable cursor. pub type CursorMutTy = ::CursorMut; diff --git a/crates/storage/db-models/src/blocks.rs b/crates/storage/db-models/src/blocks.rs index 2512db1cc9..71f4cfe9aa 100644 --- a/crates/storage/db-models/src/blocks.rs +++ b/crates/storage/db-models/src/blocks.rs @@ -64,6 +64,11 @@ impl StoredBlockBodyIndices { pub const fn tx_count(&self) -> NumTransactions { self.tx_count } + + /// Returns true if the block contains a transaction with the given number. + pub const fn contains_tx(&self, tx_num: TxNumber) -> bool { + tx_num >= self.first_tx_num && tx_num < self.next_tx_num() + } } /// The storage representation of block withdrawals. diff --git a/crates/storage/db/src/static_file/masks.rs b/crates/storage/db/src/static_file/masks.rs index 17833e7ee2..c1d9860278 100644 --- a/crates/storage/db/src/static_file/masks.rs +++ b/crates/storage/db/src/static_file/masks.rs @@ -3,7 +3,7 @@ use crate::{ static_file::mask::{ColumnSelectorOne, ColumnSelectorTwo}, HeaderTerminalDifficulties, }; -use alloy_primitives::BlockHash; +use alloy_primitives::{Address, BlockHash}; use reth_db_api::table::Table; // HEADER MASKS @@ -33,12 +33,18 @@ add_static_file_mask! { // RECEIPT MASKS add_static_file_mask! { - #[doc = "Mask for selecting a single receipt from Receipts static file segment"] + #[doc = "Mask for selecting a single receipt from `Receipts` static file segment"] ReceiptMask, R, 0b1 } // TRANSACTION MASKS add_static_file_mask! { - #[doc = "Mask for selecting a single transaction from Transactions static file segment"] + #[doc = "Mask for selecting a single transaction from `Transactions` static file segment"] TransactionMask, T, 0b1 } + +// TRANSACTION SENDER MASKS +add_static_file_mask! { + #[doc = "Mask for selecting a single transaction sender from `TransactionSenders` static file segment"] + TransactionSenderMask, Address, 0b1 +} diff --git a/crates/storage/provider/src/either_writer.rs b/crates/storage/provider/src/either_writer.rs index a981371d2a..36747f1b2c 100644 --- a/crates/storage/provider/src/either_writer.rs +++ b/crates/storage/provider/src/either_writer.rs @@ -1,18 +1,31 @@ -//! Generic writer abstraction for writing to either database tables or static files. +//! Generic reader and writer abstractions for interacting with either database tables or static +//! files. -use crate::{providers::StaticFileProviderRWRefMut, StaticFileProviderFactory}; -use alloy_primitives::{BlockNumber, TxNumber}; +use std::ops::Range; + +use crate::{ + providers::{StaticFileProvider, StaticFileProviderRWRefMut}, + StaticFileProviderFactory, +}; +use alloy_primitives::{map::HashMap, Address, BlockNumber, TxNumber}; use reth_db::{ + cursor::DbCursorRO, + static_file::TransactionSenderMask, table::Value, - transaction::{CursorMutTy, DbTxMut}, + transaction::{CursorMutTy, CursorTy, DbTx, DbTxMut}, }; use reth_db_api::{cursor::DbCursorRW, tables}; +use reth_errors::ProviderError; use reth_node_types::NodePrimitives; use reth_primitives_traits::ReceiptTy; use reth_static_file_types::StaticFileSegment; use reth_storage_api::{DBProvider, NodePrimitivesProvider, StorageSettingsCache}; use reth_storage_errors::provider::ProviderResult; -use strum::EnumIs; +use strum::{Display, EnumIs}; + +/// Type alias for [`EitherReader`] constructors. +type EitherReaderTy = + EitherReader::Tx, T>,

::Primitives>; /// Type alias for [`EitherWriter`] constructors. type EitherWriterTy<'a, P, T> = EitherWriter< @@ -22,7 +35,7 @@ type EitherWriterTy<'a, P, T> = EitherWriter< >; /// Represents a destination for writing data, either to database or static files. -#[derive(Debug)] +#[derive(Debug, Display)] pub enum EitherWriter<'a, CURSOR, N> { /// Write to database table via cursor Database(CURSOR), @@ -41,11 +54,7 @@ impl<'a> EitherWriter<'a, (), ()> { P::Tx: DbTxMut, ReceiptTy: Value, { - // Write receipts to static files only if they're explicitly enabled or we don't have - // receipts pruning - if provider.cached_storage_settings().receipts_in_static_files || - !provider.prune_modes_ref().has_receipts_pruning() - { + if Self::receipts_destination(provider).is_static_file() { Ok(EitherWriter::StaticFile( provider.get_static_file_writer(block_number, StaticFileSegment::Receipts)?, )) @@ -55,35 +64,7 @@ impl<'a> EitherWriter<'a, (), ()> { )) } } -} -impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N> { - /// Increment the block number. - /// - /// Relevant only for [`Self::StaticFile`]. It is a no-op for [`Self::Database`]. - pub fn increment_block(&mut self, expected_block_number: BlockNumber) -> ProviderResult<()> { - match self { - Self::Database(_) => Ok(()), - Self::StaticFile(writer) => writer.increment_block(expected_block_number), - } - } -} - -impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N> -where - N::Receipt: Value, - CURSOR: DbCursorRW>, -{ - /// Append a transaction receipt. - pub fn append_receipt(&mut self, tx_num: TxNumber, receipt: &N::Receipt) -> ProviderResult<()> { - match self { - Self::Database(cursor) => Ok(cursor.append(tx_num, receipt)?), - Self::StaticFile(writer) => writer.append_receipt(tx_num, receipt), - } - } -} - -impl EitherWriter<'_, (), ()> { /// Returns the destination for writing receipts. /// /// The rules are as follows: @@ -107,11 +88,264 @@ impl EitherWriter<'_, (), ()> { EitherWriterDestination::StaticFile } } + + /// Creates a new [`EitherWriter`] for senders based on storage settings. + pub fn new_senders

( + provider: &'a P, + block_number: BlockNumber, + ) -> ProviderResult> + where + P: DBProvider + NodePrimitivesProvider + StorageSettingsCache + StaticFileProviderFactory, + P::Tx: DbTxMut, + { + if EitherWriterDestination::senders(provider).is_static_file() { + Ok(EitherWriter::StaticFile( + provider + .get_static_file_writer(block_number, StaticFileSegment::TransactionSenders)?, + )) + } else { + Ok(EitherWriter::Database( + provider.tx_ref().cursor_write::()?, + )) + } + } } +impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N> { + /// Increment the block number. + /// + /// Relevant only for [`Self::StaticFile`]. It is a no-op for [`Self::Database`]. + pub fn increment_block(&mut self, expected_block_number: BlockNumber) -> ProviderResult<()> { + match self { + Self::Database(_) => Ok(()), + Self::StaticFile(writer) => writer.increment_block(expected_block_number), + } + } + + /// Ensures that the writer is positioned at the specified block number. + /// + /// If the writer is positioned at a greater block number than the specified one, the writer + /// will NOT be unwound and the error will be returned. + /// + /// Relevant only for [`Self::StaticFile`]. It is a no-op for [`Self::Database`]. + pub fn ensure_at_block(&mut self, block_number: BlockNumber) -> ProviderResult<()> { + match self { + Self::Database(_) => Ok(()), + Self::StaticFile(writer) => writer.ensure_at_block(block_number), + } + } +} + +impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N> +where + N::Receipt: Value, + CURSOR: DbCursorRW>, +{ + /// Append a transaction receipt. + pub fn append_receipt(&mut self, tx_num: TxNumber, receipt: &N::Receipt) -> ProviderResult<()> { + match self { + Self::Database(cursor) => Ok(cursor.append(tx_num, receipt)?), + Self::StaticFile(writer) => writer.append_receipt(tx_num, receipt), + } + } +} + +impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N> +where + CURSOR: DbCursorRW, +{ + /// Append a transaction sender to the destination + pub fn append_sender(&mut self, tx_num: TxNumber, sender: &Address) -> ProviderResult<()> { + match self { + Self::Database(cursor) => Ok(cursor.append(tx_num, sender)?), + Self::StaticFile(writer) => writer.append_transaction_sender(tx_num, sender), + } + } + + /// Append transaction senders to the destination + pub fn append_senders(&mut self, senders: I) -> ProviderResult<()> + where + I: Iterator, + { + match self { + Self::Database(cursor) => { + for (tx_num, sender) in senders { + cursor.append(tx_num, &sender)?; + } + Ok(()) + } + Self::StaticFile(writer) => writer.append_transaction_senders(senders), + } + } + + /// Removes all transaction senders above the given transaction number, and stops at the given + /// block number. + pub fn prune_senders( + &mut self, + unwind_tx_from: TxNumber, + block: BlockNumber, + ) -> ProviderResult<()> + where + CURSOR: DbCursorRO, + { + match self { + Self::Database(cursor) => { + let mut walker = cursor.walk_range(unwind_tx_from..)?; + while walker.next().transpose()?.is_some() { + walker.delete_current()?; + } + } + Self::StaticFile(writer) => { + let static_file_transaction_sender_num = writer + .reader() + .get_highest_static_file_tx(StaticFileSegment::TransactionSenders); + + let to_delete = static_file_transaction_sender_num + .map(|static_num| (static_num + 1).saturating_sub(unwind_tx_from)) + .unwrap_or_default(); + + writer.prune_transaction_senders(to_delete, block)?; + } + } + + Ok(()) + } +} + +/// Represents a source for reading data, either from database or static files. +#[derive(Debug, Display)] +pub enum EitherReader { + /// Read from database table via cursor + Database(CURSOR), + /// Read from static file + StaticFile(StaticFileProvider), +} + +impl EitherReader<(), ()> { + /// Creates a new [`EitherReader`] for senders based on storage settings. + pub fn new_senders

( + provider: &P, + ) -> ProviderResult> + where + P: DBProvider + NodePrimitivesProvider + StorageSettingsCache + StaticFileProviderFactory, + P::Tx: DbTx, + { + if EitherWriterDestination::senders(provider).is_static_file() { + Ok(EitherReader::StaticFile(provider.static_file_provider())) + } else { + Ok(EitherReader::Database( + provider.tx_ref().cursor_read::()?, + )) + } + } +} + +impl EitherReader +where + CURSOR: DbCursorRO, +{ + /// Fetches the senders for a range of transactions. + pub fn senders_by_tx_range( + &mut self, + range: Range, + ) -> ProviderResult> { + match self { + Self::Database(cursor) => cursor + .walk_range(range)? + .map(|result| result.map_err(ProviderError::from)) + .collect::>>(), + Self::StaticFile(provider) => range + .clone() + .zip(provider.fetch_range_iter( + StaticFileSegment::TransactionSenders, + range, + |cursor, number| cursor.get_one::(number.into()), + )?) + .filter_map(|(tx_num, sender)| { + let result = sender.transpose()?; + Some(result.map(|sender| (tx_num, sender))) + }) + .collect::>>(), + } + } +} + +/// Destination for writing data. #[derive(Debug, EnumIs)] -#[allow(missing_docs)] pub enum EitherWriterDestination { + /// Write to database table Database, + /// Write to static file StaticFile, } + +impl EitherWriterDestination { + /// Returns the destination for writing senders based on storage settings. + pub fn senders

(provider: &P) -> Self + where + P: StorageSettingsCache, + { + // Write senders to static files only if they're explicitly enabled + if provider.cached_storage_settings().transaction_senders_in_static_files { + Self::StaticFile + } else { + Self::Database + } + } +} + +#[cfg(test)] +mod tests { + use crate::test_utils::create_test_provider_factory; + + use super::*; + use alloy_primitives::Address; + use reth_storage_api::{DatabaseProviderFactory, StorageSettings}; + + #[test] + fn test_reader_senders_by_tx_range() { + let factory = create_test_provider_factory(); + + // Insert senders only from 1 to 4, but we will query from 0 to 5. + let senders = [ + (1, Address::random()), + (2, Address::random()), + (3, Address::random()), + (4, Address::random()), + ]; + + for transaction_senders_in_static_files in [false, true] { + factory.set_storage_settings_cache( + StorageSettings::legacy() + .with_transaction_senders_in_static_files(transaction_senders_in_static_files), + ); + + let provider = factory.database_provider_rw().unwrap(); + let mut writer = EitherWriter::new_senders(&provider, 0).unwrap(); + if transaction_senders_in_static_files { + assert!(matches!(writer, EitherWriter::StaticFile(_))); + } else { + assert!(matches!(writer, EitherWriter::Database(_))); + } + + writer.increment_block(0).unwrap(); + writer.append_senders(senders.iter().copied()).unwrap(); + drop(writer); + provider.commit().unwrap(); + + let provider = factory.database_provider_ro().unwrap(); + let mut reader = EitherReader::new_senders(&provider).unwrap(); + if transaction_senders_in_static_files { + assert!(matches!(reader, EitherReader::StaticFile(_))); + } else { + assert!(matches!(reader, EitherReader::Database(_))); + } + + assert_eq!( + reader.senders_by_tx_range(0..6).unwrap(), + senders.iter().copied().collect::>(), + "{reader}" + ); + } + } +} diff --git a/crates/storage/provider/src/providers/database/metrics.rs b/crates/storage/provider/src/providers/database/metrics.rs index 4daac3dfdd..22d861c674 100644 --- a/crates/storage/provider/src/providers/database/metrics.rs +++ b/crates/storage/provider/src/providers/database/metrics.rs @@ -45,6 +45,8 @@ pub(crate) enum Action { InsertBlockBodyIndices, InsertTransactionBlocks, GetNextTxNum, + InsertTransactionSenders, + InsertTransactionHashNumbers, } /// Database provider metrics @@ -70,6 +72,10 @@ struct DatabaseProviderMetrics { insert_tx_blocks: Histogram, /// Duration of get next tx num get_next_tx_num: Histogram, + /// Duration of insert transaction senders + insert_transaction_senders: Histogram, + /// Duration of insert transaction hash numbers + insert_transaction_hash_numbers: Histogram, } impl DatabaseProviderMetrics { @@ -85,6 +91,10 @@ impl DatabaseProviderMetrics { Action::InsertBlockBodyIndices => self.insert_block_body_indices.record(duration), Action::InsertTransactionBlocks => self.insert_tx_blocks.record(duration), Action::GetNextTxNum => self.get_next_tx_num.record(duration), + Action::InsertTransactionSenders => self.insert_transaction_senders.record(duration), + Action::InsertTransactionHashNumbers => { + self.insert_transaction_hash_numbers.record(duration) + } } } } diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 6e8f1f1f80..69648a7dfb 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -6,9 +6,10 @@ use crate::{ to_range, traits::{BlockSource, ReceiptProvider}, BlockHashReader, BlockNumReader, BlockReader, ChainSpecProvider, DatabaseProviderFactory, - HashedPostStateProvider, HeaderProvider, HeaderSyncGapProvider, MetadataProvider, - ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProviderBox, - StaticFileProviderFactory, StaticFileWriter, TransactionVariant, TransactionsProvider, + EitherWriterDestination, HashedPostStateProvider, HeaderProvider, HeaderSyncGapProvider, + MetadataProvider, ProviderError, PruneCheckpointReader, StageCheckpointReader, + StateProviderBox, StaticFileProviderFactory, StaticFileWriter, TransactionVariant, + TransactionsProvider, }; use alloy_consensus::transaction::TransactionMeta; use alloy_eips::BlockHashOrNumber; @@ -467,11 +468,19 @@ impl TransactionsProvider for ProviderFactory { &self, range: impl RangeBounds, ) -> ProviderResult> { - self.provider()?.senders_by_tx_range(range) + if EitherWriterDestination::senders(self).is_static_file() { + self.static_file_provider.senders_by_tx_range(range) + } else { + self.provider()?.senders_by_tx_range(range) + } } fn transaction_sender(&self, id: TxNumber) -> ProviderResult> { - self.provider()?.transaction_sender(id) + if EitherWriterDestination::senders(self).is_static_file() { + self.static_file_provider.transaction_sender(id) + } else { + self.provider()?.transaction_sender(id) + } } } diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 793b15db8e..8bf813d1f2 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -13,12 +13,13 @@ use crate::{ }, AccountReader, BlockBodyWriter, BlockExecutionWriter, BlockHashReader, BlockNumReader, BlockReader, BlockWriter, BundleStateInit, ChainStateBlockReader, ChainStateBlockWriter, - DBProvider, EitherWriter, HashingWriter, HeaderProvider, HeaderSyncGapProvider, - HistoricalStateProvider, HistoricalStateProviderRef, HistoryWriter, LatestStateProvider, - LatestStateProviderRef, OriginalValuesKnown, ProviderError, PruneCheckpointReader, - PruneCheckpointWriter, RevertsInit, StageCheckpointReader, StateProviderBox, StateWriter, - StaticFileProviderFactory, StatsReader, StorageReader, StorageTrieWriter, TransactionVariant, - TransactionsProvider, TransactionsProviderExt, TrieReader, TrieWriter, + DBProvider, EitherReader, EitherWriter, EitherWriterDestination, HashingWriter, HeaderProvider, + HeaderSyncGapProvider, HistoricalStateProvider, HistoricalStateProviderRef, HistoryWriter, + LatestStateProvider, LatestStateProviderRef, OriginalValuesKnown, ProviderError, + PruneCheckpointReader, PruneCheckpointWriter, RevertsInit, StageCheckpointReader, + StateProviderBox, StateWriter, StaticFileProviderFactory, StatsReader, StorageReader, + StorageTrieWriter, TransactionVariant, TransactionsProvider, TransactionsProviderExt, + TrieReader, TrieWriter, }; use alloy_consensus::{ transaction::{SignerRecoverable, TransactionMeta, TxHashRef}, @@ -679,17 +680,12 @@ impl DatabaseProvider { HF: Fn(RangeInclusive) -> ProviderResult>, BF: Fn(H, BodyTy, Vec

) -> ProviderResult, { - let mut senders_cursor = self.tx.cursor_read::()?; - self.block_range(range, headers_range, |header, body, tx_range| { let senders = if tx_range.is_empty() { Vec::new() } else { - // fetch senders from the senders table - let known_senders = - senders_cursor - .walk_range(tx_range.clone())? - .collect::, _>>()?; + let known_senders: HashMap = + EitherReader::new_senders(self)?.senders_by_tx_range(tx_range.clone())?; let mut senders = Vec::with_capacity(body.transactions().len()); for (tx_num, tx) in tx_range.zip(body.transactions()) { @@ -1342,11 +1338,19 @@ impl TransactionsProvider for Datab &self, range: impl RangeBounds, ) -> ProviderResult> { - self.cursor_read_collect::(range) + if EitherWriterDestination::senders(self).is_static_file() { + self.static_file_provider.senders_by_tx_range(range) + } else { + self.cursor_read_collect::(range) + } } fn transaction_sender(&self, id: TxNumber) -> ProviderResult> { - Ok(self.tx.get::(id)?) + if EitherWriterDestination::senders(self).is_static_file() { + self.static_file_provider.transaction_sender(id) + } else { + Ok(self.tx.get::(id)?) + } } } @@ -2822,8 +2826,9 @@ impl BlockWrite /// If withdrawals are not empty, this will modify /// [`BlockWithdrawals`](tables::BlockWithdrawals). /// - /// If the provider has __not__ configured full sender pruning, this will modify - /// [`TransactionSenders`](tables::TransactionSenders). + /// If the provider has __not__ configured full sender pruning, this will modify either: + /// * [`StaticFileSegment::TransactionSenders`] if senders are written to static files + /// * [`tables::TransactionSenders`] if senders are written to the database /// /// If the provider has __not__ configured full transaction lookup pruning, this will modify /// [`TransactionHashNumbers`](tables::TransactionHashNumbers). @@ -2832,6 +2837,7 @@ impl BlockWrite block: RecoveredBlock, ) -> ProviderResult { let block_number = block.number(); + let tx_count = block.body().transaction_count() as u64; let mut durations_recorder = metrics::DurationsRecorder::default(); @@ -2842,29 +2848,30 @@ impl BlockWrite self.tx.put::(block.hash(), block_number)?; durations_recorder.record_relative(metrics::Action::InsertHeaderNumbers); - let mut next_tx_num = self + let first_tx_num = self .tx .cursor_read::()? .last()? .map(|(n, _)| n + 1) .unwrap_or_default(); durations_recorder.record_relative(metrics::Action::GetNextTxNum); - let first_tx_num = next_tx_num; - let tx_count = block.body().transaction_count() as u64; + let tx_nums_iter = std::iter::successors(Some(first_tx_num), |n| Some(n + 1)); - // Ensures we have all the senders for the block's transactions. - for (transaction, sender) in block.body().transactions_iter().zip(block.senders_iter()) { - let hash = transaction.tx_hash(); + if self.prune_modes.sender_recovery.as_ref().is_none_or(|m| !m.is_full()) { + let mut senders_writer = EitherWriter::new_senders(self, block.number())?; + senders_writer.increment_block(block.number())?; + senders_writer + .append_senders(tx_nums_iter.clone().zip(block.senders_iter().copied()))?; + durations_recorder.record_relative(metrics::Action::InsertTransactionSenders); + } - if self.prune_modes.sender_recovery.as_ref().is_none_or(|m| !m.is_full()) { - self.tx.put::(next_tx_num, *sender)?; + if self.prune_modes.transaction_lookup.is_none_or(|m| !m.is_full()) { + for (tx_num, transaction) in tx_nums_iter.zip(block.body().transactions_iter()) { + let hash = transaction.tx_hash(); + self.tx.put::(*hash, tx_num)?; } - - if self.prune_modes.transaction_lookup.is_none_or(|m| !m.is_full()) { - self.tx.put::(*hash, next_tx_num)?; - } - next_tx_num += 1; + durations_recorder.record_relative(metrics::Action::InsertTransactionHashNumbers); } self.append_block_bodies(vec![(block_number, Some(block.into_body()))])?; @@ -2932,8 +2939,9 @@ impl BlockWrite } fn remove_blocks_above(&self, block: BlockNumber) -> ProviderResult<()> { + let last_block_number = self.last_block_number()?; // Clean up HeaderNumbers for blocks being removed, we must clear all indexes from MDBX. - for hash in self.canonical_hashes_range(block + 1, self.last_block_number()? + 1)? { + for hash in self.canonical_hashes_range(block + 1, last_block_number + 1)? { self.tx.delete::(hash, None)?; } @@ -2975,7 +2983,7 @@ impl BlockWrite } } - self.remove::(unwind_tx_from..)?; + EitherWriter::new_senders(self, last_block_number)?.prune_senders(unwind_tx_from, block)?; self.remove_bodies_above(block)?; @@ -4752,7 +4760,7 @@ mod tests { // Static files mode { let factory = create_test_provider_factory(); - let storage_settings = StorageSettings::new().with_receipts_in_static_files(); + let storage_settings = StorageSettings::legacy().with_receipts_in_static_files(true); factory.set_storage_settings_cache(storage_settings); let factory = factory.with_prune_modes(PruneModes { receipts: Some(PruneMode::Before(2)), diff --git a/crates/storage/provider/src/providers/static_file/jar.rs b/crates/storage/provider/src/providers/static_file/jar.rs index 4313f81994..5548cb5c1f 100644 --- a/crates/storage/provider/src/providers/static_file/jar.rs +++ b/crates/storage/provider/src/providers/static_file/jar.rs @@ -6,16 +6,18 @@ use crate::{ to_range, BlockHashReader, BlockNumReader, HeaderProvider, ReceiptProvider, TransactionsProvider, }; -use alloy_consensus::transaction::{SignerRecoverable, TransactionMeta}; +use alloy_consensus::transaction::TransactionMeta; use alloy_eips::{eip2718::Encodable2718, BlockHashOrNumber}; use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256}; use reth_chainspec::ChainInfo; use reth_db::static_file::{ BlockHashMask, HeaderMask, HeaderWithHashMask, ReceiptMask, StaticFileCursor, TransactionMask, + TransactionSenderMask, }; use reth_db_api::table::{Decompress, Value}; use reth_node_types::NodePrimitives; use reth_primitives_traits::{SealedHeader, SignedTransaction}; +use reth_storage_api::range_size_hint; use reth_storage_errors::provider::{ProviderError, ProviderResult}; use std::{ fmt::Debug, @@ -104,12 +106,10 @@ impl> HeaderProvider for StaticFileJarProv &self, range: impl RangeBounds, ) -> ProviderResult> { - let range = to_range(range); - let mut cursor = self.cursor()?; - let mut headers = Vec::with_capacity((range.end - range.start) as usize); + let mut headers = Vec::with_capacity(range_size_hint(&range).unwrap_or(1024)); - for num in range { + for num in to_range(range) { if let Some(header) = cursor.get_one::>(num.into())? { headers.push(header); } @@ -133,12 +133,10 @@ impl> HeaderProvider for StaticFileJarProv range: impl RangeBounds, mut predicate: impl FnMut(&SealedHeader) -> bool, ) -> ProviderResult>> { - let range = to_range(range); - let mut cursor = self.cursor()?; - let mut headers = Vec::with_capacity((range.end - range.start) as usize); + let mut headers = Vec::with_capacity(range_size_hint(&range).unwrap_or(1024)); - for number in range { + for number in to_range(range) { if let Some((header, hash)) = cursor.get_two::>(number.into())? { @@ -258,31 +256,34 @@ impl> TransactionsPr &self, range: impl RangeBounds, ) -> ProviderResult> { - let range = to_range(range); let mut cursor = self.cursor()?; - let mut txes = Vec::with_capacity((range.end - range.start) as usize); + let mut txs = Vec::with_capacity(range_size_hint(&range).unwrap_or(1024)); - for num in range { + for num in to_range(range) { if let Some(tx) = cursor.get_one::>(num.into())? { - txes.push(tx) + txs.push(tx) } } - Ok(txes) + Ok(txs) } fn senders_by_tx_range( &self, range: impl RangeBounds, ) -> ProviderResult> { - let txs = self.transactions_by_tx_range(range)?; - Ok(reth_primitives_traits::transaction::recover::recover_signers(&txs)?) + let mut cursor = self.cursor()?; + let mut senders = Vec::with_capacity(range_size_hint(&range).unwrap_or(1024)); + + for num in to_range(range) { + if let Some(tx) = cursor.get_one::(num.into())? { + senders.push(tx) + } + } + Ok(senders) } - fn transaction_sender(&self, num: TxNumber) -> ProviderResult> { - Ok(self - .cursor()? - .get_one::>(num.into())? - .and_then(|tx| tx.recover_signer().ok())) + fn transaction_sender(&self, id: TxNumber) -> ProviderResult> { + self.cursor()?.get_one::(id.into()) } } @@ -317,11 +318,10 @@ impl, ) -> ProviderResult> { - let range = to_range(range); let mut cursor = self.cursor()?; - let mut receipts = Vec::with_capacity((range.end - range.start) as usize); + let mut receipts = Vec::with_capacity(range_size_hint(&range).unwrap_or(1024)); - for num in range { + for num in to_range(range) { if let Some(tx) = cursor.get_one::>(num.into())? { receipts.push(tx) } diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index 772e6dd13e..1c4f86ef49 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -4,13 +4,10 @@ use super::{ }; use crate::{ to_range, BlockHashReader, BlockNumReader, BlockReader, BlockSource, EitherWriter, - HeaderProvider, ReceiptProvider, StageCheckpointReader, StatsReader, TransactionVariant, - TransactionsProvider, TransactionsProviderExt, -}; -use alloy_consensus::{ - transaction::{SignerRecoverable, TransactionMeta}, - Header, + EitherWriterDestination, HeaderProvider, ReceiptProvider, StageCheckpointReader, StatsReader, + TransactionVariant, TransactionsProvider, TransactionsProviderExt, }; +use alloy_consensus::{transaction::TransactionMeta, Header}; use alloy_eips::{eip2718::Encodable2718, BlockHashOrNumber}; use alloy_primitives::{b256, keccak256, Address, BlockHash, BlockNumber, TxHash, TxNumber, B256}; use dashmap::DashMap; @@ -21,7 +18,7 @@ use reth_db::{ lockfile::StorageLock, static_file::{ iter_static_files, BlockHashMask, HeaderMask, HeaderWithHashMask, ReceiptMask, - StaticFileCursor, TransactionMask, + StaticFileCursor, TransactionMask, TransactionSenderMask, }, }; use reth_db_api::{ @@ -461,12 +458,37 @@ impl StaticFileProvider { pub fn get_segment_provider( &self, segment: StaticFileSegment, - start: u64, + number: u64, ) -> ProviderResult> { if segment.is_block_based() { - self.get_segment_provider_for_block(segment, start, None) + self.get_segment_provider_for_block(segment, number, None) } else { - self.get_segment_provider_for_transaction(segment, start, None) + self.get_segment_provider_for_transaction(segment, number, None) + } + } + + /// Gets the [`StaticFileJarProvider`] of the requested segment and start index that can be + /// either block or transaction. + /// + /// If the segment is not found, returns [`None`]. + pub fn get_maybe_segment_provider( + &self, + segment: StaticFileSegment, + number: u64, + ) -> ProviderResult>> { + let provider = if segment.is_block_based() { + self.get_segment_provider_for_block(segment, number, None) + } else { + self.get_segment_provider_for_transaction(segment, number, None) + }; + + match provider { + Ok(provider) => Ok(Some(provider)), + Err( + ProviderError::MissingStaticFileBlock(_, _) | + ProviderError::MissingStaticFileTx(_, _), + ) => Ok(None), + Err(err) => Err(err), } } @@ -1022,6 +1044,11 @@ impl StaticFileProvider { continue; } } + StaticFileSegment::TransactionSenders => { + if EitherWriterDestination::senders(provider).is_database() { + continue + } + } } let initial_highest_block = self.get_highest_static_file_block(segment); @@ -1124,6 +1151,13 @@ impl StaticFileProvider { highest_tx, highest_block, )?, + StaticFileSegment::TransactionSenders => self + .ensure_invariants::<_, tables::TransactionSenders>( + provider, + segment, + highest_tx, + highest_block, + )?, } { debug!(target: "reth::providers::static_file", ?segment, unwind_target=unwind, "Invariants check returned unwind target"); update_unwind_target(unwind); @@ -1224,6 +1258,7 @@ impl StaticFileProvider { StaticFileSegment::Headers => StageId::Headers, StaticFileSegment::Transactions => StageId::Bodies, StaticFileSegment::Receipts => StageId::Execution, + StaticFileSegment::TransactionSenders => StageId::SenderRecovery, }; let checkpoint_block_number = provider.get_stage_checkpoint(stage_id)?.unwrap_or_default().block_number; @@ -1260,7 +1295,9 @@ impl StaticFileProvider { // TODO(joshie): is_block_meta writer.prune_headers(prune_count)?; } - StaticFileSegment::Transactions | StaticFileSegment::Receipts => { + StaticFileSegment::Transactions | + StaticFileSegment::Receipts | + StaticFileSegment::TransactionSenders => { if let Some(block) = provider.block_body_indices(checkpoint_block_number)? { let number = highest_static_file_entry - block.last_tx_num(); debug!(target: "reth::providers::static_file", ?segment, prune_count = number, checkpoint_block_number, "Pruning transaction based segment"); @@ -1272,6 +1309,9 @@ impl StaticFileProvider { StaticFileSegment::Receipts => { writer.prune_receipts(number, checkpoint_block_number)? } + StaticFileSegment::TransactionSenders => { + writer.prune_transaction_senders(number, checkpoint_block_number)? + } StaticFileSegment::Headers => unreachable!(), } } else { @@ -1447,29 +1487,37 @@ impl StaticFileProvider { /// Fetches data within a specified range across multiple static files. /// - /// Returns an iterator over the data + /// Returns an iterator over the data. Yields [`None`] if the data for the specified number is + /// not found. pub fn fetch_range_iter<'a, T, F>( &'a self, segment: StaticFileSegment, range: Range, get_fn: F, - ) -> ProviderResult> + 'a> + ) -> ProviderResult>> + 'a> where F: Fn(&mut StaticFileCursor<'_>, u64) -> ProviderResult> + 'a, T: std::fmt::Debug, { - let mut provider = Some(self.get_segment_provider(segment, range.start)?); - Ok(range.filter_map(move |number| { - match get_fn(&mut provider.as_ref().expect("qed").cursor().ok()?, number).transpose() { - Some(result) => Some(result), + let mut provider = self.get_maybe_segment_provider(segment, range.start)?; + Ok(range.map(move |number| { + match provider + .as_ref() + .map(|provider| get_fn(&mut provider.cursor()?, number)) + .and_then(|result| result.transpose()) + { + Some(result) => result.map(Some), None => { - // There is a very small chance of hitting a deadlock if two consecutive static - // files share the same bucket in the internal dashmap and - // we don't drop the current provider before requesting the - // next one. + // There is a very small chance of hitting a deadlock if two consecutive + // static files share the same bucket in the internal dashmap and we don't drop + // the current provider before requesting the next one. provider.take(); - provider = Some(self.get_segment_provider(segment, number).ok()?); - get_fn(&mut provider.as_ref().expect("qed").cursor().ok()?, number).transpose() + provider = self.get_maybe_segment_provider(segment, number)?; + provider + .as_ref() + .map(|provider| get_fn(&mut provider.cursor()?, number)) + .and_then(|result| result.transpose()) + .transpose() } } })) @@ -1993,15 +2041,24 @@ impl> TransactionsPr &self, range: impl RangeBounds, ) -> ProviderResult> { - let txes = self.transactions_by_tx_range(range)?; - Ok(reth_primitives_traits::transaction::recover::recover_signers(&txes)?) + self.fetch_range_with_predicate( + StaticFileSegment::TransactionSenders, + to_range(range), + |cursor, number| cursor.get_one::(number.into()), + |_| true, + ) } fn transaction_sender(&self, id: TxNumber) -> ProviderResult> { - match self.transaction_by_id_unhashed(id)? { - Some(tx) => Ok(tx.recover_signer().ok()), - None => Ok(None), - } + self.get_segment_provider_for_transaction(StaticFileSegment::TransactionSenders, id, None) + .and_then(|provider| provider.transaction_sender(id)) + .or_else(|err| { + if let ProviderError::MissingStaticFileTx(_, _) = err { + Ok(None) + } else { + Err(err) + } + }) } } @@ -2133,6 +2190,10 @@ impl StatsReader for StaticFileProvider { .map(|txs| txs + 1) .unwrap_or_default() as usize), + tables::TransactionSenders::NAME => Ok(self + .get_highest_static_file_tx(StaticFileSegment::TransactionSenders) + .map(|txs| txs + 1) + .unwrap_or_default() as usize), _ => Err(ProviderError::UnsupportedProvider), } } diff --git a/crates/storage/provider/src/providers/static_file/mod.rs b/crates/storage/provider/src/providers/static_file/mod.rs index 53b878095d..9935c8168f 100644 --- a/crates/storage/provider/src/providers/static_file/mod.rs +++ b/crates/storage/provider/src/providers/static_file/mod.rs @@ -61,7 +61,7 @@ mod tests { test_utils::create_test_provider_factory, HeaderProvider, StaticFileProviderFactory, }; use alloy_consensus::{Header, SignableTransaction, Transaction, TxLegacy}; - use alloy_primitives::{BlockHash, Signature, TxNumber, B256}; + use alloy_primitives::{Address, BlockHash, Signature, TxNumber, B256, U160}; use rand::seq::SliceRandom; use reth_db::test_utils::create_test_static_files_dir; use reth_db_api::{transaction::DbTxMut, CanonicalHeaders, HeaderNumbers, Headers}; @@ -328,6 +328,11 @@ mod tests { receipt.cumulative_gas_used = *next_tx_num; writer.append_receipt(*next_tx_num, &receipt).unwrap(); } + StaticFileSegment::TransactionSenders => { + // Used as ID for validation + let sender = Address::from(U160::from(*next_tx_num)); + writer.append_transaction_sender(*next_tx_num, &sender).unwrap(); + } } *next_tx_num += 1; tx_count -= 1; @@ -430,6 +435,9 @@ mod tests { writer.prune_transactions(prune_count, last_block)? } StaticFileSegment::Receipts => writer.prune_receipts(prune_count, last_block)?, + StaticFileSegment::TransactionSenders => { + writer.prune_transaction_senders(prune_count, last_block)? + } } writer.commit()?; @@ -456,6 +464,13 @@ mod tests { sf_rw.receipt(id)?.map(|r| r.cumulative_gas_used), "receipt mismatch", )?, + StaticFileSegment::TransactionSenders => assert_eyre( + expected_tx_tip, + sf_rw + .transaction_sender(id)? + .map(|s| u64::try_from(U160::from_be_bytes(s.0.into())).unwrap()), + "sender mismatch", + )?, } } diff --git a/crates/storage/provider/src/providers/static_file/writer.rs b/crates/storage/provider/src/providers/static_file/writer.rs index 6fa6aa294d..43545fddf5 100644 --- a/crates/storage/provider/src/providers/static_file/writer.rs +++ b/crates/storage/provider/src/providers/static_file/writer.rs @@ -13,6 +13,7 @@ use reth_static_file_types::{SegmentHeader, SegmentRangeInclusive, StaticFileSeg use reth_storage_errors::provider::{ProviderError, ProviderResult, StaticFileWriterError}; use std::{ borrow::Borrow, + cmp::Ordering, fmt::Debug, path::{Path, PathBuf}, sync::{Arc, Weak}, @@ -29,6 +30,7 @@ pub(crate) struct StaticFileWriters { headers: RwLock>>, transactions: RwLock>>, receipts: RwLock>>, + transaction_senders: RwLock>>, } impl Default for StaticFileWriters { @@ -37,6 +39,7 @@ impl Default for StaticFileWriters { headers: Default::default(), transactions: Default::default(), receipts: Default::default(), + transaction_senders: Default::default(), } } } @@ -51,6 +54,7 @@ impl StaticFileWriters { StaticFileSegment::Headers => self.headers.write(), StaticFileSegment::Transactions => self.transactions.write(), StaticFileSegment::Receipts => self.receipts.write(), + StaticFileSegment::TransactionSenders => self.transaction_senders.write(), }; if write_guard.is_none() { @@ -63,7 +67,9 @@ impl StaticFileWriters { pub(crate) fn commit(&self) -> ProviderResult<()> { debug!(target: "provider::static_file", "Committing all static file segments"); - for writer_lock in [&self.headers, &self.transactions, &self.receipts] { + for writer_lock in + [&self.headers, &self.transactions, &self.receipts, &self.transaction_senders] + { let mut writer = writer_lock.write(); if let Some(writer) = writer.as_mut() { writer.commit()?; @@ -75,7 +81,9 @@ impl StaticFileWriters { } pub(crate) fn has_unwind_queued(&self) -> bool { - for writer_lock in [&self.headers, &self.transactions, &self.receipts] { + for writer_lock in + [&self.headers, &self.transactions, &self.receipts, &self.transaction_senders] + { let writer = writer_lock.read(); if let Some(writer) = writer.as_ref() && writer.will_prune_on_commit() @@ -262,6 +270,10 @@ impl StaticFileProviderRW { StaticFileSegment::Receipts => { self.prune_receipt_data(to_delete, last_block_number.expect("should exist"))? } + StaticFileSegment::TransactionSenders => self.prune_transaction_sender_data( + to_delete, + last_block_number.expect("should exist"), + )?, } } @@ -358,10 +370,39 @@ impl StaticFileProviderRW { self.reader().update_index(self.writer.user_header().segment(), segment_max_block) } + /// Ensures that the writer is positioned at the specified block number. + /// + /// If the writer is positioned at a greater block number than the specified one, the writer + /// will NOT be unwound and the error will be returned. + pub fn ensure_at_block(&mut self, advance_to: BlockNumber) -> ProviderResult<()> { + let current_block = if let Some(current_block_number) = self.current_block_number() { + current_block_number + } else { + self.increment_block(0)?; + 0 + }; + + match current_block.cmp(&advance_to) { + Ordering::Less => { + for block in current_block + 1..=advance_to { + self.increment_block(block)?; + } + } + Ordering::Equal => {} + Ordering::Greater => { + return Err(ProviderError::UnexpectedStaticFileBlockNumber( + self.writer.user_header().segment(), + current_block, + advance_to, + )); + } + } + + Ok(()) + } + /// Allows to increment the [`SegmentHeader`] end block. It will commit the current static file, /// and create the next one if we are past the end range. - /// - /// Returns the current [`BlockNumber`] as seen in the static file. pub fn increment_block(&mut self, expected_block_number: BlockNumber) -> ProviderResult<()> { let segment = self.writer.user_header().segment(); @@ -401,6 +442,11 @@ impl StaticFileProviderRW { Ok(()) } + /// Returns the current block number of the static file writer. + pub fn current_block_number(&self) -> Option { + self.writer.user_header().block_end() + } + /// Returns a block number that is one next to the current tip of static files. pub fn next_block_number(&self) -> u64 { // The next static file block number can be found by checking the one after block_end. @@ -529,8 +575,6 @@ impl StaticFileProviderRW { } /// Appends to tx number-based static file. - /// - /// Returns the current [`TxNumber`] as seen in the static file. fn append_with_tx_number( &mut self, tx_num: TxNumber, @@ -559,8 +603,6 @@ impl StaticFileProviderRW { /// /// It **CALLS** `increment_block()` since the number of headers is equal to the number of /// blocks. - /// - /// Returns the current [`BlockNumber`] as seen in the static file. pub fn append_header(&mut self, header: &N::BlockHeader, hash: &BlockHash) -> ProviderResult<()> where N::BlockHeader: Compact, @@ -572,8 +614,6 @@ impl StaticFileProviderRW { /// /// It **CALLS** `increment_block()` since the number of headers is equal to the number of /// blocks. - /// - /// Returns the current [`BlockNumber`] as seen in the static file. pub fn append_header_with_td( &mut self, header: &N::BlockHeader, @@ -609,8 +649,6 @@ impl StaticFileProviderRW { /// /// It **DOES NOT CALL** `increment_block()`, it should be handled elsewhere. There might be /// empty blocks and this function wouldn't be called. - /// - /// Returns the current [`TxNumber`] as seen in the static file. pub fn append_transaction(&mut self, tx_num: TxNumber, tx: &N::SignedTx) -> ProviderResult<()> where N::SignedTx: Compact, @@ -636,8 +674,6 @@ impl StaticFileProviderRW { /// /// It **DOES NOT** call `increment_block()`, it should be handled elsewhere. There might be /// empty blocks and this function wouldn't be called. - /// - /// Returns the current [`TxNumber`] as seen in the static file. pub fn append_receipt(&mut self, tx_num: TxNumber, receipt: &N::Receipt) -> ProviderResult<()> where N::Receipt: Compact, @@ -660,9 +696,7 @@ impl StaticFileProviderRW { } /// Appends multiple receipts to the static file. - /// - /// Returns the current [`TxNumber`] as seen in the static file, if any. - pub fn append_receipts(&mut self, receipts: I) -> ProviderResult> + pub fn append_receipts(&mut self, receipts: I) -> ProviderResult<()> where I: Iterator>, R: Borrow, @@ -673,20 +707,18 @@ impl StaticFileProviderRW { let mut receipts_iter = receipts.into_iter().peekable(); // If receipts are empty, we can simply return None if receipts_iter.peek().is_none() { - return Ok(None); + return Ok(()); } let start = Instant::now(); self.ensure_no_queued_prune()?; // At this point receipts contains at least one receipt, so this would be overwritten. - let mut tx_number = 0; let mut count: u64 = 0; for receipt_result in receipts_iter { let (tx_num, receipt) = receipt_result?; self.append_with_tx_number(tx_num, receipt.borrow())?; - tx_number = tx_num; count += 1; } @@ -699,7 +731,68 @@ impl StaticFileProviderRW { ); } - Ok(Some(tx_number)) + Ok(()) + } + + /// Appends transaction sender to static file. + /// + /// It **DOES NOT** call `increment_block()`, it should be handled elsewhere. There might be + /// empty blocks and this function wouldn't be called. + pub fn append_transaction_sender( + &mut self, + tx_num: TxNumber, + sender: &alloy_primitives::Address, + ) -> ProviderResult<()> { + let start = Instant::now(); + self.ensure_no_queued_prune()?; + + debug_assert!(self.writer.user_header().segment() == StaticFileSegment::TransactionSenders); + self.append_with_tx_number(tx_num, sender)?; + + if let Some(metrics) = &self.metrics { + metrics.record_segment_operation( + StaticFileSegment::TransactionSenders, + StaticFileProviderOperation::Append, + Some(start.elapsed()), + ); + } + + Ok(()) + } + + /// Appends multiple transaction senders to the static file. + pub fn append_transaction_senders(&mut self, senders: I) -> ProviderResult<()> + where + I: Iterator, + { + debug_assert!(self.writer.user_header().segment() == StaticFileSegment::TransactionSenders); + + let mut senders_iter = senders.into_iter().peekable(); + // If senders are empty, we can simply return + if senders_iter.peek().is_none() { + return Ok(()); + } + + let start = Instant::now(); + self.ensure_no_queued_prune()?; + + // At this point senders contains at least one sender, so this would be overwritten. + let mut count: u64 = 0; + for (tx_num, sender) in senders_iter { + self.append_with_tx_number(tx_num, sender)?; + count += 1; + } + + if let Some(metrics) = &self.metrics { + metrics.record_segment_operations( + StaticFileSegment::TransactionSenders, + StaticFileProviderOperation::Append, + count, + Some(start.elapsed()), + ); + } + + Ok(()) } /// Adds an instruction to prune `to_delete` transactions during commit. @@ -726,6 +819,21 @@ impl StaticFileProviderRW { self.queue_prune(to_delete, Some(last_block)) } + /// Adds an instruction to prune `to_delete` transaction senders during commit. + /// + /// Note: `last_block` refers to the block the unwinds ends at. + pub fn prune_transaction_senders( + &mut self, + to_delete: u64, + last_block: BlockNumber, + ) -> ProviderResult<()> { + debug_assert_eq!( + self.writer.user_header().segment(), + StaticFileSegment::TransactionSenders + ); + self.queue_prune(to_delete, Some(last_block)) + } + /// Adds an instruction to prune `to_delete` headers during commit. pub fn prune_headers(&mut self, to_delete: u64) -> ProviderResult<()> { debug_assert_eq!(self.writer.user_header().segment(), StaticFileSegment::Headers); @@ -802,6 +910,29 @@ impl StaticFileProviderRW { Ok(()) } + /// Prunes the last `to_delete` transaction senders from the data file. + fn prune_transaction_sender_data( + &mut self, + to_delete: u64, + last_block: BlockNumber, + ) -> ProviderResult<()> { + let start = Instant::now(); + + debug_assert!(self.writer.user_header().segment() == StaticFileSegment::TransactionSenders); + + self.truncate(to_delete, Some(last_block))?; + + if let Some(metrics) = &self.metrics { + metrics.record_segment_operation( + StaticFileSegment::TransactionSenders, + StaticFileProviderOperation::Prune, + Some(start.elapsed()), + ); + } + + Ok(()) + } + /// Prunes the last `to_delete` headers from the data file. fn prune_header_data(&mut self, to_delete: u64) -> ProviderResult<()> { let start = Instant::now(); @@ -821,7 +952,8 @@ impl StaticFileProviderRW { Ok(()) } - fn reader(&self) -> StaticFileProvider { + /// Returns a [`StaticFileProvider`] associated with this writer. + pub fn reader(&self) -> StaticFileProvider { Self::upgrade_provider_to_strong_reference(&self.reader) } diff --git a/crates/storage/storage-api/src/database_provider.rs b/crates/storage/storage-api/src/database_provider.rs index 8b5d8281f4..b206ca0922 100644 --- a/crates/storage/storage-api/src/database_provider.rs +++ b/crates/storage/storage-api/src/database_provider.rs @@ -183,7 +183,8 @@ where } } -fn range_size_hint(range: &impl RangeBounds) -> Option { +/// Returns the length of the range if the range has a bounded end. +pub fn range_size_hint(range: &impl RangeBounds) -> Option { let start = match range.start_bound().cloned() { Bound::Included(start) => start, Bound::Excluded(start) => start.checked_add(1)?, diff --git a/docs/vocs/docs/pages/cli/SUMMARY.mdx b/docs/vocs/docs/pages/cli/SUMMARY.mdx index 6a7eb22884..2f6f6e20b9 100644 --- a/docs/vocs/docs/pages/cli/SUMMARY.mdx +++ b/docs/vocs/docs/pages/cli/SUMMARY.mdx @@ -28,6 +28,7 @@ - [`reth db settings get`](/cli/reth/db/settings/get) - [`reth db settings set`](/cli/reth/db/settings/set) - [`reth db settings set receipts_in_static_files`](/cli/reth/db/settings/set/receipts_in_static_files) + - [`reth db settings set transaction_senders_in_static_files`](/cli/reth/db/settings/set/transaction_senders_in_static_files) - [`reth download`](/cli/reth/download) - [`reth stage`](/cli/reth/stage) - [`reth stage run`](/cli/reth/stage/run) diff --git a/docs/vocs/docs/pages/cli/reth/db.mdx b/docs/vocs/docs/pages/cli/reth/db.mdx index fc4b44a66b..95ea29b7bc 100644 --- a/docs/vocs/docs/pages/cli/reth/db.mdx +++ b/docs/vocs/docs/pages/cli/reth/db.mdx @@ -111,6 +111,9 @@ Static Files: --static-files.blocks-per-file.receipts Number of blocks per file for the receipts segment + --static-files.blocks-per-file.transaction-senders + Number of blocks per file for the transaction senders segment + --static-files.receipts Store receipts in static files instead of the database. @@ -118,6 +121,13 @@ Static Files: Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --static-files.transaction-senders + Store transaction senders in static files instead of the database. + + When enabled, transaction senders will be written to static files on disk instead of the database. + + Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + Logging: --log.stdout.format The format to use for logs written to stdout diff --git a/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx b/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx index 9f22178ec4..b9ad2a0cc7 100644 --- a/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx @@ -11,9 +11,10 @@ Usage: reth db clear static-file [OPTIONS] Arguments: Possible values: - - headers: Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTerminalDifficulties` tables - - transactions: Static File segment responsible for the `Transactions` table - - receipts: Static File segment responsible for the `Receipts` table + - headers: Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTerminalDifficulties` tables + - transactions: Static File segment responsible for the `Transactions` table + - receipts: Static File segment responsible for the `Receipts` table + - transaction-senders: Static File segment responsible for the `TransactionSenders` table Options: -h, --help diff --git a/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx b/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx index 8e045a4cdf..36fd873bcc 100644 --- a/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx @@ -11,9 +11,10 @@ Usage: reth db get static-file [OPTIONS] Arguments: Possible values: - - headers: Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTerminalDifficulties` tables - - transactions: Static File segment responsible for the `Transactions` table - - receipts: Static File segment responsible for the `Receipts` table + - headers: Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTerminalDifficulties` tables + - transactions: Static File segment responsible for the `Transactions` table + - receipts: Static File segment responsible for the `Receipts` table + - transaction-senders: Static File segment responsible for the `TransactionSenders` table The key to get content for diff --git a/docs/vocs/docs/pages/cli/reth/db/settings/set.mdx b/docs/vocs/docs/pages/cli/reth/db/settings/set.mdx index 2f4b8df7a9..d1d4e2f216 100644 --- a/docs/vocs/docs/pages/cli/reth/db/settings/set.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/settings/set.mdx @@ -9,8 +9,9 @@ $ reth db settings set --help Usage: reth db settings set [OPTIONS] Commands: - receipts_in_static_files Store receipts in static files instead of the database - help Print this message or the help of the given subcommand(s) + receipts_in_static_files Store receipts in static files instead of the database + transaction_senders_in_static_files Store transaction senders in static files instead of the database + help Print this message or the help of the given subcommand(s) Options: -h, --help diff --git a/docs/vocs/docs/pages/cli/reth/db/settings/set/transaction_senders_in_static_files.mdx b/docs/vocs/docs/pages/cli/reth/db/settings/set/transaction_senders_in_static_files.mdx new file mode 100644 index 0000000000..18115a5094 --- /dev/null +++ b/docs/vocs/docs/pages/cli/reth/db/settings/set/transaction_senders_in_static_files.mdx @@ -0,0 +1,143 @@ +# reth db settings set transaction_senders_in_static_files + +Store transaction senders in static files instead of the database + +```bash +$ reth db settings set transaction_senders_in_static_files --help +``` +```txt +Usage: reth db settings set transaction_senders_in_static_files [OPTIONS] + +Arguments: + + [possible values: true, false] + +Options: + -h, --help + Print help (see a summary with '-h') + +Datadir: + --chain + The chain this node is running. + Possible values are either a built-in chain or the path to a chain specification file. + + Built-in chains: + mainnet, sepolia, holesky, hoodi, dev + + [default: mainnet] + +Logging: + --log.stdout.format + The format to use for logs written to stdout + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + [default: terminal] + + --log.stdout.filter + The filter to use for logs written to stdout + + [default: ] + + --log.file.format + The format to use for logs written to the log file + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + [default: terminal] + + --log.file.filter + The filter to use for logs written to the log file + + [default: debug] + + --log.file.directory + The path to put log files in + + [default: /logs] + + --log.file.name + The prefix name of the log files + + [default: reth.log] + + --log.file.max-size + The maximum size (in MB) of one log file + + [default: 200] + + --log.file.max-files + The maximum amount of log files that will be stored. If set to 0, background file logging is disabled + + [default: 5] + + --log.journald + Write logs to journald + + --log.journald.filter + The filter to use for logs written to journald + + [default: error] + + --color + Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting + + Possible values: + - always: Colors on + - auto: Auto-detect + - never: Colors off + + [default: always] + +Display: + -v, --verbosity... + Set the minimum log level. + + -v Errors + -vv Warnings + -vvv Info + -vvvv Debug + -vvvvv Traces (warning: very verbose!) + + -q, --quiet + Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] +``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/static-file-header/block.mdx b/docs/vocs/docs/pages/cli/reth/db/static-file-header/block.mdx index 4d7fb55544..8f18e10c7f 100644 --- a/docs/vocs/docs/pages/cli/reth/db/static-file-header/block.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/static-file-header/block.mdx @@ -13,9 +13,10 @@ Arguments: Static file segment Possible values: - - headers: Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTerminalDifficulties` tables - - transactions: Static File segment responsible for the `Transactions` table - - receipts: Static File segment responsible for the `Receipts` table + - headers: Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTerminalDifficulties` tables + - transactions: Static File segment responsible for the `Transactions` table + - receipts: Static File segment responsible for the `Receipts` table + - transaction-senders: Static File segment responsible for the `TransactionSenders` table Block number to query diff --git a/docs/vocs/docs/pages/cli/reth/download.mdx b/docs/vocs/docs/pages/cli/reth/download.mdx index 63c8e507c1..ffdbccd669 100644 --- a/docs/vocs/docs/pages/cli/reth/download.mdx +++ b/docs/vocs/docs/pages/cli/reth/download.mdx @@ -96,6 +96,9 @@ Static Files: --static-files.blocks-per-file.receipts Number of blocks per file for the receipts segment + --static-files.blocks-per-file.transaction-senders + Number of blocks per file for the transaction senders segment + --static-files.receipts Store receipts in static files instead of the database. @@ -103,6 +106,13 @@ Static Files: Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --static-files.transaction-senders + Store transaction senders in static files instead of the database. + + When enabled, transaction senders will be written to static files on disk instead of the database. + + Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + -u, --url Specify a snapshot URL or let the command propose a default one. diff --git a/docs/vocs/docs/pages/cli/reth/export-era.mdx b/docs/vocs/docs/pages/cli/reth/export-era.mdx index 7efbe06426..bab9b07966 100644 --- a/docs/vocs/docs/pages/cli/reth/export-era.mdx +++ b/docs/vocs/docs/pages/cli/reth/export-era.mdx @@ -96,6 +96,9 @@ Static Files: --static-files.blocks-per-file.receipts Number of blocks per file for the receipts segment + --static-files.blocks-per-file.transaction-senders + Number of blocks per file for the transaction senders segment + --static-files.receipts Store receipts in static files instead of the database. @@ -103,6 +106,13 @@ Static Files: Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --static-files.transaction-senders + Store transaction senders in static files instead of the database. + + When enabled, transaction senders will be written to static files on disk instead of the database. + + Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --first-block-number Optional first block number to export from the db. It is by default 0. diff --git a/docs/vocs/docs/pages/cli/reth/import-era.mdx b/docs/vocs/docs/pages/cli/reth/import-era.mdx index a04bf89dcf..a535ce750a 100644 --- a/docs/vocs/docs/pages/cli/reth/import-era.mdx +++ b/docs/vocs/docs/pages/cli/reth/import-era.mdx @@ -96,6 +96,9 @@ Static Files: --static-files.blocks-per-file.receipts Number of blocks per file for the receipts segment + --static-files.blocks-per-file.transaction-senders + Number of blocks per file for the transaction senders segment + --static-files.receipts Store receipts in static files instead of the database. @@ -103,6 +106,13 @@ Static Files: Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --static-files.transaction-senders + Store transaction senders in static files instead of the database. + + When enabled, transaction senders will be written to static files on disk instead of the database. + + Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --path The path to a directory for import. diff --git a/docs/vocs/docs/pages/cli/reth/import.mdx b/docs/vocs/docs/pages/cli/reth/import.mdx index 5699512754..8b57f761fe 100644 --- a/docs/vocs/docs/pages/cli/reth/import.mdx +++ b/docs/vocs/docs/pages/cli/reth/import.mdx @@ -96,6 +96,9 @@ Static Files: --static-files.blocks-per-file.receipts Number of blocks per file for the receipts segment + --static-files.blocks-per-file.transaction-senders + Number of blocks per file for the transaction senders segment + --static-files.receipts Store receipts in static files instead of the database. @@ -103,6 +106,13 @@ Static Files: Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --static-files.transaction-senders + Store transaction senders in static files instead of the database. + + When enabled, transaction senders will be written to static files on disk instead of the database. + + Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --no-state Disables stages that require state. diff --git a/docs/vocs/docs/pages/cli/reth/init-state.mdx b/docs/vocs/docs/pages/cli/reth/init-state.mdx index e37d06c63a..80c3701988 100644 --- a/docs/vocs/docs/pages/cli/reth/init-state.mdx +++ b/docs/vocs/docs/pages/cli/reth/init-state.mdx @@ -96,6 +96,9 @@ Static Files: --static-files.blocks-per-file.receipts Number of blocks per file for the receipts segment + --static-files.blocks-per-file.transaction-senders + Number of blocks per file for the transaction senders segment + --static-files.receipts Store receipts in static files instead of the database. @@ -103,6 +106,13 @@ Static Files: Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --static-files.transaction-senders + Store transaction senders in static files instead of the database. + + When enabled, transaction senders will be written to static files on disk instead of the database. + + Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --without-evm Specifies whether to initialize the state without relying on EVM historical data. diff --git a/docs/vocs/docs/pages/cli/reth/init.mdx b/docs/vocs/docs/pages/cli/reth/init.mdx index f0eff64c6a..61f3756af0 100644 --- a/docs/vocs/docs/pages/cli/reth/init.mdx +++ b/docs/vocs/docs/pages/cli/reth/init.mdx @@ -96,6 +96,9 @@ Static Files: --static-files.blocks-per-file.receipts Number of blocks per file for the receipts segment + --static-files.blocks-per-file.transaction-senders + Number of blocks per file for the transaction senders segment + --static-files.receipts Store receipts in static files instead of the database. @@ -103,6 +106,13 @@ Static Files: Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --static-files.transaction-senders + Store transaction senders in static files instead of the database. + + When enabled, transaction senders will be written to static files on disk instead of the database. + + Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + Logging: --log.stdout.format The format to use for logs written to stdout diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index bed2a926da..c39d1b1f43 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -961,6 +961,9 @@ Static Files: --static-files.blocks-per-file.receipts Number of blocks per file for the receipts segment + --static-files.blocks-per-file.transaction-senders + Number of blocks per file for the transaction senders segment + --static-files.receipts Store receipts in static files instead of the database. @@ -968,6 +971,13 @@ Static Files: Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --static-files.transaction-senders + Store transaction senders in static files instead of the database. + + When enabled, transaction senders will be written to static files on disk instead of the database. + + Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + Ress: --ress.enable Enable support for `ress` subprotocol diff --git a/docs/vocs/docs/pages/cli/reth/prune.mdx b/docs/vocs/docs/pages/cli/reth/prune.mdx index d93c0d2302..1d18691922 100644 --- a/docs/vocs/docs/pages/cli/reth/prune.mdx +++ b/docs/vocs/docs/pages/cli/reth/prune.mdx @@ -96,6 +96,9 @@ Static Files: --static-files.blocks-per-file.receipts Number of blocks per file for the receipts segment + --static-files.blocks-per-file.transaction-senders + Number of blocks per file for the transaction senders segment + --static-files.receipts Store receipts in static files instead of the database. @@ -103,6 +106,13 @@ Static Files: Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --static-files.transaction-senders + Store transaction senders in static files instead of the database. + + When enabled, transaction senders will be written to static files on disk instead of the database. + + Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + Logging: --log.stdout.format The format to use for logs written to stdout diff --git a/docs/vocs/docs/pages/cli/reth/re-execute.mdx b/docs/vocs/docs/pages/cli/reth/re-execute.mdx index b51a66c7ac..c3ef601ccd 100644 --- a/docs/vocs/docs/pages/cli/reth/re-execute.mdx +++ b/docs/vocs/docs/pages/cli/reth/re-execute.mdx @@ -96,6 +96,9 @@ Static Files: --static-files.blocks-per-file.receipts Number of blocks per file for the receipts segment + --static-files.blocks-per-file.transaction-senders + Number of blocks per file for the transaction senders segment + --static-files.receipts Store receipts in static files instead of the database. @@ -103,6 +106,13 @@ Static Files: Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --static-files.transaction-senders + Store transaction senders in static files instead of the database. + + When enabled, transaction senders will be written to static files on disk instead of the database. + + Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --from The height to start at diff --git a/docs/vocs/docs/pages/cli/reth/stage/drop.mdx b/docs/vocs/docs/pages/cli/reth/stage/drop.mdx index 0a055a6104..76e6383d10 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/drop.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/drop.mdx @@ -96,6 +96,9 @@ Static Files: --static-files.blocks-per-file.receipts Number of blocks per file for the receipts segment + --static-files.blocks-per-file.transaction-senders + Number of blocks per file for the transaction senders segment + --static-files.receipts Store receipts in static files instead of the database. @@ -103,6 +106,13 @@ Static Files: Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --static-files.transaction-senders + Store transaction senders in static files instead of the database. + + When enabled, transaction senders will be written to static files on disk instead of the database. + + Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + Possible values: - headers: The headers stage within the pipeline diff --git a/docs/vocs/docs/pages/cli/reth/stage/dump.mdx b/docs/vocs/docs/pages/cli/reth/stage/dump.mdx index 4ac4cd5d9b..4089b1510d 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/dump.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/dump.mdx @@ -103,6 +103,9 @@ Static Files: --static-files.blocks-per-file.receipts Number of blocks per file for the receipts segment + --static-files.blocks-per-file.transaction-senders + Number of blocks per file for the transaction senders segment + --static-files.receipts Store receipts in static files instead of the database. @@ -110,6 +113,13 @@ Static Files: Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --static-files.transaction-senders + Store transaction senders in static files instead of the database. + + When enabled, transaction senders will be written to static files on disk instead of the database. + + Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + Logging: --log.stdout.format The format to use for logs written to stdout diff --git a/docs/vocs/docs/pages/cli/reth/stage/run.mdx b/docs/vocs/docs/pages/cli/reth/stage/run.mdx index b77e278676..2c391268ce 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/run.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/run.mdx @@ -96,6 +96,9 @@ Static Files: --static-files.blocks-per-file.receipts Number of blocks per file for the receipts segment + --static-files.blocks-per-file.transaction-senders + Number of blocks per file for the transaction senders segment + --static-files.receipts Store receipts in static files instead of the database. @@ -103,6 +106,13 @@ Static Files: Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --static-files.transaction-senders + Store transaction senders in static files instead of the database. + + When enabled, transaction senders will be written to static files on disk instead of the database. + + Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --metrics Enable Prometheus metrics. diff --git a/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx b/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx index fcba15254d..c1447e4cb0 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx @@ -101,6 +101,9 @@ Static Files: --static-files.blocks-per-file.receipts Number of blocks per file for the receipts segment + --static-files.blocks-per-file.transaction-senders + Number of blocks per file for the transaction senders segment + --static-files.receipts Store receipts in static files instead of the database. @@ -108,6 +111,13 @@ Static Files: Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --static-files.transaction-senders + Store transaction senders in static files instead of the database. + + When enabled, transaction senders will be written to static files on disk instead of the database. + + Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch. + --offline If this is enabled, then all stages except headers, bodies, and sender recovery will be unwound From e15b404a30bb758a19ed907efbc80bf7313e9dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Fri, 21 Nov 2025 22:03:28 +0200 Subject: [PATCH 14/23] feat(era-file): back to era file support (#19482) --- crates/era-utils/src/history.rs | 2 +- crates/era/src/common/decode.rs | 2 +- crates/era/src/era/file.rs | 342 ++++++++++++++++++++ crates/era/src/era/mod.rs | 1 + crates/era/src/era/types/group.rs | 113 +++++++ crates/era/src/era1/types/execution.rs | 16 +- crates/era/src/era1/types/group.rs | 2 +- crates/era/tests/it/era/dd.rs | 190 +++++++++++ crates/era/tests/it/era/genesis.rs | 37 +++ crates/era/tests/it/era/mod.rs | 2 + crates/era/tests/it/{ => era1}/dd.rs | 8 +- crates/era/tests/it/{ => era1}/genesis.rs | 6 +- crates/era/tests/it/era1/mod.rs | 3 + crates/era/tests/it/{ => era1}/roundtrip.rs | 8 +- crates/era/tests/it/main.rs | 118 +++++-- 15 files changed, 799 insertions(+), 51 deletions(-) create mode 100644 crates/era/src/era/file.rs create mode 100644 crates/era/tests/it/era/dd.rs create mode 100644 crates/era/tests/it/era/genesis.rs create mode 100644 crates/era/tests/it/era/mod.rs rename crates/era/tests/it/{ => era1}/dd.rs (97%) rename crates/era/tests/it/{ => era1}/genesis.rs (95%) create mode 100644 crates/era/tests/it/era1/mod.rs rename crates/era/tests/it/{ => era1}/roundtrip.rs (98%) diff --git a/crates/era-utils/src/history.rs b/crates/era-utils/src/history.rs index a1d3e8c859..bb11f6db6a 100644 --- a/crates/era-utils/src/history.rs +++ b/crates/era-utils/src/history.rs @@ -9,7 +9,7 @@ use reth_db_api::{ RawKey, RawTable, RawValue, }; use reth_era::{ - common::{decode::DecodeCompressed, file_ops::StreamReader}, + common::{decode::DecodeCompressedRlp, file_ops::StreamReader}, e2s::error::E2sError, era1::{ file::{BlockTupleIterator, Era1Reader}, diff --git a/crates/era/src/common/decode.rs b/crates/era/src/common/decode.rs index cef3368d74..85f8d534d2 100644 --- a/crates/era/src/common/decode.rs +++ b/crates/era/src/common/decode.rs @@ -5,7 +5,7 @@ use alloy_rlp::Decodable; use ssz::Decode; /// Extension trait for generic decoding from compressed data -pub trait DecodeCompressed { +pub trait DecodeCompressedRlp { /// Decompress and decode the data into the given type fn decode(&self) -> Result; } diff --git a/crates/era/src/era/file.rs b/crates/era/src/era/file.rs new file mode 100644 index 0000000000..1c1a20543f --- /dev/null +++ b/crates/era/src/era/file.rs @@ -0,0 +1,342 @@ +//! Represents a complete Era file +//! +//! The structure of an Era file follows the specification: +//! `Version | block* | era-state | other-entries* | slot-index(block)? | slot-index(state)` +//! +//! See also . + +use crate::{ + common::file_ops::{EraFileFormat, FileReader, StreamReader, StreamWriter}, + e2s::{ + error::E2sError, + file::{E2StoreReader, E2StoreWriter}, + types::{Entry, IndexEntry, Version, SLOT_INDEX}, + }, + era::types::{ + consensus::{ + CompressedBeaconState, CompressedSignedBeaconBlock, COMPRESSED_BEACON_STATE, + COMPRESSED_SIGNED_BEACON_BLOCK, + }, + group::{EraGroup, EraId, SlotIndex}, + }, +}; +use std::{ + fs::File, + io::{Read, Seek, Write}, +}; + +/// Era file interface +#[derive(Debug)] +pub struct EraFile { + /// Version record, must be the first record in the file + pub version: Version, + + /// Main content group of the Era file + pub group: EraGroup, + + /// File identifier + pub id: EraId, +} + +impl EraFileFormat for EraFile { + type EraGroup = EraGroup; + type Id = EraId; + + /// Create a new [`EraFile`] + fn new(group: EraGroup, id: EraId) -> Self { + Self { version: Version, group, id } + } + + fn version(&self) -> &Version { + &self.version + } + + fn group(&self) -> &Self::EraGroup { + &self.group + } + + fn id(&self) -> &Self::Id { + &self.id + } +} + +/// Reader for era files that builds on top of [`E2StoreReader`] +#[derive(Debug)] +pub struct EraReader { + reader: E2StoreReader, +} + +/// An iterator of [`BeaconBlockIterator`] streaming from [`E2StoreReader`]. +#[derive(Debug)] +pub struct BeaconBlockIterator { + reader: E2StoreReader, + state: Option, + other_entries: Vec, + block_slot_index: Option, + state_slot_index: Option, +} + +impl BeaconBlockIterator { + fn new(reader: E2StoreReader) -> Self { + Self { + reader, + state: None, + other_entries: Default::default(), + block_slot_index: None, + state_slot_index: None, + } + } +} + +impl Iterator for BeaconBlockIterator { + type Item = Result; + + fn next(&mut self) -> Option { + self.next_result().transpose() + } +} + +impl BeaconBlockIterator { + fn next_result(&mut self) -> Result, E2sError> { + loop { + let Some(entry) = self.reader.read_next_entry()? else { + return Ok(None); + }; + + match entry.entry_type { + COMPRESSED_SIGNED_BEACON_BLOCK => { + let block = CompressedSignedBeaconBlock::from_entry(&entry)?; + return Ok(Some(block)); + } + COMPRESSED_BEACON_STATE => { + if self.state.is_some() { + return Err(E2sError::Ssz("Multiple state entries found".to_string())); + } + self.state = Some(CompressedBeaconState::from_entry(&entry)?); + } + SLOT_INDEX => { + let slot_index = SlotIndex::from_entry(&entry)?; + // if we haven't seen the state yet, the slot index is for blocks, + // if we have seen the state, the slot index is for the state + if self.state.is_none() { + self.block_slot_index = Some(slot_index); + } else { + self.state_slot_index = Some(slot_index); + } + } + _ => { + self.other_entries.push(entry); + } + } + } + } +} + +impl StreamReader for EraReader { + type File = EraFile; + type Iterator = BeaconBlockIterator; + + /// Create a new [`EraReader`] + fn new(reader: R) -> Self { + Self { reader: E2StoreReader::new(reader) } + } + + /// Returns an iterator of [`BeaconBlockIterator`] streaming from `reader`. + fn iter(self) -> BeaconBlockIterator { + BeaconBlockIterator::new(self.reader) + } + + fn read(self, network_name: String) -> Result { + self.read_and_assemble(network_name) + } +} + +impl EraReader { + /// Reads and parses an era file from the underlying reader, assembling all components + /// into a complete [`EraFile`] with an [`EraId`] that includes the provided network name. + pub fn read_and_assemble(mut self, network_name: String) -> Result { + // Validate version entry + let _version_entry = match self.reader.read_version()? { + Some(entry) if entry.is_version() => entry, + Some(_) => return Err(E2sError::Ssz("First entry is not a Version entry".to_string())), + None => return Err(E2sError::Ssz("Empty Era file".to_string())), + }; + + let mut iter = self.iter(); + let blocks = (&mut iter).collect::, _>>()?; + + let BeaconBlockIterator { + state, other_entries, block_slot_index, state_slot_index, .. + } = iter; + + let state = + state.ok_or_else(|| E2sError::Ssz("Era file missing state entry".to_string()))?; + + let state_slot_index = state_slot_index + .ok_or_else(|| E2sError::Ssz("Era file missing state slot index".to_string()))?; + + // Create appropriate `EraGroup`, genesis vs non-genesis + let mut group = if let Some(block_index) = block_slot_index { + EraGroup::with_block_index(blocks, state, block_index, state_slot_index) + } else { + EraGroup::new(blocks, state, state_slot_index) + }; + + // Add other entries + for entry in other_entries { + group.add_entry(entry); + } + + let (start_slot, slot_count) = group.slot_range(); + + let id = EraId::new(network_name, start_slot, slot_count); + + Ok(EraFile::new(group, id)) + } +} + +impl FileReader for EraReader {} + +/// Writer for Era files that builds on top of [`E2StoreWriter`] +#[derive(Debug)] +pub struct EraWriter { + writer: E2StoreWriter, + has_written_version: bool, + has_written_state: bool, + has_written_block_slot_index: bool, + has_written_state_slot_index: bool, +} + +impl StreamWriter for EraWriter { + type File = EraFile; + + /// Create a new [`EraWriter`] + fn new(writer: W) -> Self { + Self { + writer: E2StoreWriter::new(writer), + has_written_version: false, + has_written_state: false, + has_written_block_slot_index: false, + has_written_state_slot_index: false, + } + } + + /// Write the version entry + fn write_version(&mut self) -> Result<(), E2sError> { + if self.has_written_version { + return Ok(()); + } + + self.writer.write_version()?; + self.has_written_version = true; + Ok(()) + } + + fn write_file(&mut self, file: &Self::File) -> Result<(), E2sError> { + // Write version + self.write_version()?; + + // Write all blocks + for block in &file.group.blocks { + self.write_beacon_block(block)?; + } + + // Write state + self.write_beacon_state(&file.group.era_state)?; + + // Write other entries + for entry in &file.group.other_entries { + self.writer.write_entry(entry)?; + } + + // Write slot index + if let Some(ref block_index) = file.group.slot_index { + self.write_block_slot_index(block_index)?; + } + + // Write state index + self.write_state_slot_index(&file.group.state_slot_index)?; + + self.writer.flush()?; + Ok(()) + } + + /// Flush any buffered data to the underlying writer + fn flush(&mut self) -> Result<(), E2sError> { + self.writer.flush() + } +} + +impl EraWriter { + /// Write beacon block + pub fn write_beacon_block( + &mut self, + block: &CompressedSignedBeaconBlock, + ) -> Result<(), E2sError> { + self.ensure_version_written()?; + + // Ensure blocks are written before state/indices + if self.has_written_state || + self.has_written_block_slot_index || + self.has_written_state_slot_index + { + return Err(E2sError::Ssz("Cannot write blocks after state or indices".to_string())); + } + + let entry = block.to_entry(); + self.writer.write_entry(&entry)?; + Ok(()) + } + + // Write beacon state + fn write_beacon_state(&mut self, state: &CompressedBeaconState) -> Result<(), E2sError> { + self.ensure_version_written()?; + + if self.has_written_state { + return Err(E2sError::Ssz("State already written".to_string())); + } + + let entry = state.to_entry(); + self.writer.write_entry(&entry)?; + self.has_written_state = true; + Ok(()) + } + + /// Write the block slot index + pub fn write_block_slot_index(&mut self, slot_index: &SlotIndex) -> Result<(), E2sError> { + self.ensure_version_written()?; + + if self.has_written_block_slot_index { + return Err(E2sError::Ssz("Block slot index already written".to_string())); + } + + let entry = slot_index.to_entry(); + self.writer.write_entry(&entry)?; + self.has_written_block_slot_index = true; + + Ok(()) + } + + /// Write the state slot index + pub fn write_state_slot_index(&mut self, slot_index: &SlotIndex) -> Result<(), E2sError> { + self.ensure_version_written()?; + + if self.has_written_state_slot_index { + return Err(E2sError::Ssz("State slot index already written".to_string())); + } + + let entry = slot_index.to_entry(); + self.writer.write_entry(&entry)?; + self.has_written_state_slot_index = true; + + Ok(()) + } + + /// Helper to ensure version is written before any data + fn ensure_version_written(&mut self) -> Result<(), E2sError> { + if !self.has_written_version { + self.write_version()?; + } + Ok(()) + } +} diff --git a/crates/era/src/era/mod.rs b/crates/era/src/era/mod.rs index 108eeb3088..864f2ded31 100644 --- a/crates/era/src/era/mod.rs +++ b/crates/era/src/era/mod.rs @@ -1,3 +1,4 @@ //! Core era primitives. +pub mod file; pub mod types; diff --git a/crates/era/src/era/types/group.rs b/crates/era/src/era/types/group.rs index bb250872ed..5051ddae47 100644 --- a/crates/era/src/era/types/group.rs +++ b/crates/era/src/era/types/group.rs @@ -3,10 +3,14 @@ //! See also use crate::{ + common::file_ops::EraFileId, e2s::types::{Entry, IndexEntry, SLOT_INDEX}, era::types::consensus::{CompressedBeaconState, CompressedSignedBeaconBlock}, }; +/// Number of slots per historical root in ERA files +pub const SLOTS_PER_HISTORICAL_ROOT: u64 = 8192; + /// Era file content group /// /// Format: `Version | block* | era-state | other-entries* | slot-index(block)? | slot-index(state)` @@ -64,6 +68,28 @@ impl EraGroup { pub fn add_entry(&mut self, entry: Entry) { self.other_entries.push(entry); } + + /// Get the starting slot and slot count. + pub const fn slot_range(&self) -> (u64, u32) { + if let Some(ref block_index) = self.slot_index { + // Non-genesis era: use block slot index + (block_index.starting_slot, block_index.slot_count() as u32) + } else { + // Genesis era: use state slot index, it should be slot 0 + // Genesis has only the genesis state, no blocks + (self.state_slot_index.starting_slot, 0) + } + } + + /// Get the starting slot number + pub const fn starting_slot(&self) -> u64 { + self.slot_range().0 + } + + /// Get the number of slots + pub const fn slot_count(&self) -> u32 { + self.slot_range().1 + } } /// [`SlotIndex`] records store offsets to data at specific slots @@ -122,6 +148,93 @@ impl IndexEntry for SlotIndex { } } +/// Era file identifier +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EraId { + /// Network configuration name + pub network_name: String, + + /// First slot number in file + pub start_slot: u64, + + /// Number of slots in the file + pub slot_count: u32, + + /// Optional hash identifier for this file + /// First 4 bytes of the last historical root in the last state in the era file + pub hash: Option<[u8; 4]>, +} + +impl EraId { + /// Create a new [`EraId`] + pub fn new(network_name: impl Into, start_slot: u64, slot_count: u32) -> Self { + Self { network_name: network_name.into(), start_slot, slot_count, hash: None } + } + + /// Add a hash identifier to [`EraId`] + pub const fn with_hash(mut self, hash: [u8; 4]) -> Self { + self.hash = Some(hash); + self + } + + /// Calculate which era number the file starts at + pub const fn era_number(&self) -> u64 { + self.start_slot / SLOTS_PER_HISTORICAL_ROOT + } + + // Helper function to calculate the number of eras per era1 file, + // If the user can decide how many blocks per era1 file there are, we need to calculate it. + // Most of the time it should be 1, but it can never be more than 2 eras per file + // as there is a maximum of 8192 blocks per era1 file. + const fn calculate_era_count(&self) -> u64 { + if self.slot_count == 0 { + return 0; + } + + let first_era = self.era_number(); + + // Calculate the actual last slot number in the range + let last_slot = self.start_slot + self.slot_count as u64 - 1; + // Find which era the last block belongs to + let last_era = last_slot / SLOTS_PER_HISTORICAL_ROOT; + // Count how many eras we span + last_era - first_era + 1 + } +} + +impl EraFileId for EraId { + fn network_name(&self) -> &str { + &self.network_name + } + + fn start_number(&self) -> u64 { + self.start_slot + } + + fn count(&self) -> u32 { + self.slot_count + } + /// Convert to file name following the era file naming: + /// `---.era` + /// + /// See also + fn to_file_name(&self) -> String { + let era_number = self.era_number(); + let era_count = self.calculate_era_count(); + + if let Some(hash) = self.hash { + format!( + "{}-{:05}-{:05}-{:02x}{:02x}{:02x}{:02x}.era", + self.network_name, era_number, era_count, hash[0], hash[1], hash[2], hash[3] + ) + } else { + // era spec format with placeholder hash when no hash available + // Format: `---00000000.era` + format!("{}-{:05}-{:05}-00000000.era", self.network_name, era_number, era_count) + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/era/src/era1/types/execution.rs b/crates/era/src/era1/types/execution.rs index 50dee5b4a1..86849419e1 100644 --- a/crates/era/src/era1/types/execution.rs +++ b/crates/era/src/era1/types/execution.rs @@ -16,7 +16,7 @@ //! //! ```rust //! use alloy_consensus::Header; -//! use reth_era::{common::decode::DecodeCompressed, era1::types::execution::CompressedHeader}; +//! use reth_era::{common::decode::DecodeCompressedRlp, era1::types::execution::CompressedHeader}; //! //! let header = Header { number: 100, ..Default::default() }; //! // Compress the header: rlp encoding and Snappy compression @@ -32,7 +32,7 @@ //! ```rust //! use alloy_consensus::{BlockBody, Header}; //! use alloy_primitives::Bytes; -//! use reth_era::{common::decode::DecodeCompressed, era1::types::execution::CompressedBody}; +//! use reth_era::{common::decode::DecodeCompressedRlp, era1::types::execution::CompressedBody}; //! use reth_ethereum_primitives::TransactionSigned; //! //! let body: BlockBody = BlockBody { @@ -53,7 +53,9 @@ //! //! ```rust //! use alloy_consensus::{Eip658Value, Receipt, ReceiptEnvelope, ReceiptWithBloom}; -//! use reth_era::{common::decode::DecodeCompressed, era1::types::execution::CompressedReceipts}; +//! use reth_era::{ +//! common::decode::DecodeCompressedRlp, era1::types::execution::CompressedReceipts, +//! }; //! //! let receipt = //! Receipt { status: Eip658Value::Eip658(true), cumulative_gas_used: 21000, logs: vec![] }; @@ -68,7 +70,7 @@ //! `````` use crate::{ - common::decode::DecodeCompressed, + common::decode::DecodeCompressedRlp, e2s::{error::E2sError, types::Entry}, }; use alloy_consensus::{Block, BlockBody, Header}; @@ -223,7 +225,7 @@ impl CompressedHeader { } } -impl DecodeCompressed for CompressedHeader { +impl DecodeCompressedRlp for CompressedHeader { fn decode(&self) -> Result { let decoder = SnappyRlpCodec::::new(); decoder.decode(&self.data) @@ -310,7 +312,7 @@ impl CompressedBody { } } -impl DecodeCompressed for CompressedBody { +impl DecodeCompressedRlp for CompressedBody { fn decode(&self) -> Result { let decoder = SnappyRlpCodec::::new(); decoder.decode(&self.data) @@ -401,7 +403,7 @@ impl CompressedReceipts { } } -impl DecodeCompressed for CompressedReceipts { +impl DecodeCompressedRlp for CompressedReceipts { fn decode(&self) -> Result { let decoder = SnappyRlpCodec::::new(); decoder.decode(&self.data) diff --git a/crates/era/src/era1/types/group.rs b/crates/era/src/era1/types/group.rs index 5a7e65a404..0f1b8cabd5 100644 --- a/crates/era/src/era1/types/group.rs +++ b/crates/era/src/era1/types/group.rs @@ -174,7 +174,7 @@ impl EraFileId for Era1Id { mod tests { use super::*; use crate::{ - common::decode::DecodeCompressed, + common::decode::DecodeCompressedRlp, test_utils::{create_sample_block, create_test_block_with_compressed_data}, }; use alloy_consensus::ReceiptWithBloom; diff --git a/crates/era/tests/it/era/dd.rs b/crates/era/tests/it/era/dd.rs new file mode 100644 index 0000000000..1537b91225 --- /dev/null +++ b/crates/era/tests/it/era/dd.rs @@ -0,0 +1,190 @@ +//! Simple decoding and decompressing tests +//! for mainnet era files + +use reth_era::{ + common::file_ops::{StreamReader, StreamWriter}, + era::file::{EraReader, EraWriter}, +}; +use std::io::Cursor; + +use crate::{EraTestDownloader, HOODI}; + +// Helper function to test decompression and decoding for a specific era file +async fn test_era_file_decompression_and_decoding( + downloader: &EraTestDownloader, + filename: &str, + network: &str, +) -> eyre::Result<()> { + println!("\nTesting file: {filename}"); + let file = downloader.open_era_file(filename, network).await?; + + // Handle genesis era separately + if file.group.is_genesis() { + // Genesis has no blocks + assert_eq!(file.group.blocks.len(), 0, "Genesis should have no blocks"); + assert!(file.group.slot_index.is_none(), "Genesis should not have block slot index"); + + // Test genesis state decompression + let state_data = file.group.era_state.decompress()?; + assert!(!state_data.is_empty(), "Genesis state should decompress to non-empty data"); + + // Verify state slot index + assert_eq!( + file.group.state_slot_index.slot_count(), + 1, + "Genesis state index should have count of 1" + ); + + let mut buffer = Vec::new(); + { + let mut writer = EraWriter::new(&mut buffer); + writer.write_file(&file)?; + } + + let reader = EraReader::new(Cursor::new(&buffer)); + let read_back_file = reader.read(file.id.network_name.clone())?; + + assert_eq!( + file.group.era_state.decompress()?, + read_back_file.group.era_state.decompress()?, + "Genesis state data should be identical" + ); + + println!("Genesis era verified successfully"); + return Ok(()); + } + + // Non-genesis era - test beacon blocks + println!( + " Non-genesis era with {} beacon blocks, starting at slot {}", + file.group.blocks.len(), + file.group.starting_slot() + ); + + // Test beacon block decompression across different positions + let test_block_indices = [ + 0, // First block + file.group.blocks.len() / 2, // Middle block + file.group.blocks.len() - 1, // Last block + ]; + + for &block_idx in &test_block_indices { + let block = &file.group.blocks[block_idx]; + let slot = file.group.starting_slot() + block_idx as u64; + + println!( + "\n Testing beacon block at slot {}, compressed size: {} bytes", + slot, + block.data.len() + ); + + // Test beacon block decompression + let block_data = block.decompress()?; + assert!( + !block_data.is_empty(), + "Beacon block at slot {slot} decompression should produce non-empty data" + ); + } + + // Test era state decompression + let state_data = file.group.era_state.decompress()?; + assert!(!state_data.is_empty(), "Era state decompression should produce non-empty data"); + println!(" Era state decompressed: {} bytes", state_data.len()); + + // Verify slot indices + if let Some(ref block_slot_index) = file.group.slot_index { + println!( + " Block slot index: starting_slot={}, count={}", + block_slot_index.starting_slot, + block_slot_index.slot_count() + ); + + // Check for empty slots + let empty_slots: Vec = (0..block_slot_index.slot_count()) + .filter(|&i| !block_slot_index.has_data_at_slot(i)) + .collect(); + + if !empty_slots.is_empty() { + println!( + " Found {} empty slots (first few): {:?}", + empty_slots.len(), + &empty_slots[..empty_slots.len().min(5)] + ); + } + } + + // Test round-trip serialization + let mut buffer = Vec::new(); + { + let mut writer = EraWriter::new(&mut buffer); + writer.write_file(&file)?; + } + + // Read back from buffer + let reader = EraReader::new(Cursor::new(&buffer)); + let read_back_file = reader.read(file.id.network_name.clone())?; + + // Verify basic properties are preserved + assert_eq!(file.id.network_name, read_back_file.id.network_name); + assert_eq!(file.id.start_slot, read_back_file.id.start_slot); + assert_eq!(file.id.slot_count, read_back_file.id.slot_count); + assert_eq!(file.group.blocks.len(), read_back_file.group.blocks.len()); + + // Test data preservation for beacon blocks + for &idx in &test_block_indices { + let original_block = &file.group.blocks[idx]; + let read_back_block = &read_back_file.group.blocks[idx]; + let slot = file.group.starting_slot() + idx as u64; + + // Test that decompressed data is identical + assert_eq!( + original_block.decompress()?, + read_back_block.decompress()?, + "Beacon block data should be identical for slot {slot}" + ); + } + + // Test state data preservation + assert_eq!( + file.group.era_state.decompress()?, + read_back_file.group.era_state.decompress()?, + "Era state data should be identical" + ); + + // Test slot indices preservation + if let (Some(original_index), Some(read_index)) = + (&file.group.slot_index, &read_back_file.group.slot_index) + { + assert_eq!( + original_index.starting_slot, read_index.starting_slot, + "Block slot index starting slot should match" + ); + assert_eq!( + original_index.offsets, read_index.offsets, + "Block slot index offsets should match" + ); + } + + assert_eq!( + file.group.state_slot_index.starting_slot, + read_back_file.group.state_slot_index.starting_slot, + "State slot index starting slot should match" + ); + assert_eq!( + file.group.state_slot_index.offsets, read_back_file.group.state_slot_index.offsets, + "State slot index offsets should match" + ); + + Ok(()) +} + +#[test_case::test_case("hoodi-00000-212f13fc.era"; "era_dd_hoodi_0")] +#[test_case::test_case("hoodi-00021-857e418b.era"; "era_dd_hoodi_21")] +#[test_case::test_case("hoodi-00175-202aaa6d.era"; "era_dd_hoodi_175")] +#[test_case::test_case("hoodi-00201-0d521fc8.era"; "era_dd_hoodi_201")] +#[tokio::test(flavor = "multi_thread")] +#[ignore = "download intensive"] +async fn test_hoodi_era1_file_decompression_and_decoding(filename: &str) -> eyre::Result<()> { + let downloader = EraTestDownloader::new().await?; + test_era_file_decompression_and_decoding(&downloader, filename, HOODI).await +} diff --git a/crates/era/tests/it/era/genesis.rs b/crates/era/tests/it/era/genesis.rs new file mode 100644 index 0000000000..66f64421b7 --- /dev/null +++ b/crates/era/tests/it/era/genesis.rs @@ -0,0 +1,37 @@ +//! Genesis block tests for `era1` files. +//! +//! These tests verify proper decompression and decoding of genesis blocks +//! from different networks. + +use crate::{EraTestDownloader, ERA_HOODI_FILES_NAMES, HOODI}; + +#[tokio::test(flavor = "multi_thread")] +#[ignore = "download intensive"] +async fn test_hoodi_genesis_era_decompression() -> eyre::Result<()> { + let downloader = EraTestDownloader::new().await?; + + let file = downloader.open_era_file(ERA_HOODI_FILES_NAMES[0], HOODI).await?; + + // Verify this is genesis era + assert!(file.group.is_genesis(), "First file should be genesis era"); + assert_eq!(file.group.starting_slot(), 0, "Genesis should start at slot 0"); + + // Genesis era has no blocks + assert_eq!(file.group.blocks.len(), 0, "Genesis era should have no blocks"); + + // Genesis should not have block slot index + assert!(file.group.slot_index.is_none(), "Genesis should not have block slot index"); + + // Test state decompression + let state_data = file.group.era_state.decompress()?; + assert!(!state_data.is_empty(), "Decompressed state should not be empty"); + + // Verify state slot index + assert_eq!( + file.group.state_slot_index.slot_count(), + 1, + "Genesis state index should have count of 1" + ); + + Ok(()) +} diff --git a/crates/era/tests/it/era/mod.rs b/crates/era/tests/it/era/mod.rs new file mode 100644 index 0000000000..66ae11afc4 --- /dev/null +++ b/crates/era/tests/it/era/mod.rs @@ -0,0 +1,2 @@ +mod dd; +mod genesis; diff --git a/crates/era/tests/it/dd.rs b/crates/era/tests/it/era1/dd.rs similarity index 97% rename from crates/era/tests/it/dd.rs rename to crates/era/tests/it/era1/dd.rs index 85cc0b1fb9..6b2d2bd42d 100644 --- a/crates/era/tests/it/dd.rs +++ b/crates/era/tests/it/era1/dd.rs @@ -14,11 +14,11 @@ use reth_era::{ use reth_ethereum_primitives::TransactionSigned; use std::io::Cursor; -use crate::{Era1TestDownloader, MAINNET}; +use crate::{EraTestDownloader, MAINNET}; -// Helper function to test decompression and decoding for a specific file +// Helper function to test decompression and decoding for a specific era1 file async fn test_file_decompression( - downloader: &Era1TestDownloader, + downloader: &EraTestDownloader, filename: &str, ) -> eyre::Result<()> { println!("\nTesting file: {filename}"); @@ -154,6 +154,6 @@ async fn test_file_decompression( #[tokio::test(flavor = "multi_thread")] #[ignore = "download intensive"] async fn test_mainnet_era1_file_decompression_and_decoding(filename: &str) -> eyre::Result<()> { - let downloader = Era1TestDownloader::new().await?; + let downloader = EraTestDownloader::new().await?; test_file_decompression(&downloader, filename).await } diff --git a/crates/era/tests/it/genesis.rs b/crates/era/tests/it/era1/genesis.rs similarity index 95% rename from crates/era/tests/it/genesis.rs rename to crates/era/tests/it/era1/genesis.rs index 14f563edf2..0046c8bb89 100644 --- a/crates/era/tests/it/genesis.rs +++ b/crates/era/tests/it/era1/genesis.rs @@ -4,7 +4,7 @@ //! from different networks. use crate::{ - Era1TestDownloader, ERA1_MAINNET_FILES_NAMES, ERA1_SEPOLIA_FILES_NAMES, MAINNET, SEPOLIA, + EraTestDownloader, ERA1_MAINNET_FILES_NAMES, ERA1_SEPOLIA_FILES_NAMES, MAINNET, SEPOLIA, }; use alloy_consensus::{BlockBody, Header}; use reth_era::{e2s::types::IndexEntry, era1::types::execution::CompressedBody}; @@ -13,7 +13,7 @@ use reth_ethereum_primitives::TransactionSigned; #[tokio::test(flavor = "multi_thread")] #[ignore = "download intensive"] async fn test_mainnet_genesis_block_decompression() -> eyre::Result<()> { - let downloader = Era1TestDownloader::new().await?; + let downloader = EraTestDownloader::new().await?; let file = downloader.open_era1_file(ERA1_MAINNET_FILES_NAMES[0], MAINNET).await?; @@ -65,7 +65,7 @@ async fn test_mainnet_genesis_block_decompression() -> eyre::Result<()> { #[tokio::test(flavor = "multi_thread")] #[ignore = "download intensive"] async fn test_sepolia_genesis_block_decompression() -> eyre::Result<()> { - let downloader = Era1TestDownloader::new().await?; + let downloader = EraTestDownloader::new().await?; let file = downloader.open_era1_file(ERA1_SEPOLIA_FILES_NAMES[0], SEPOLIA).await?; diff --git a/crates/era/tests/it/era1/mod.rs b/crates/era/tests/it/era1/mod.rs new file mode 100644 index 0000000000..fb1e16f4e0 --- /dev/null +++ b/crates/era/tests/it/era1/mod.rs @@ -0,0 +1,3 @@ +mod dd; +mod genesis; +mod roundtrip; diff --git a/crates/era/tests/it/roundtrip.rs b/crates/era/tests/it/era1/roundtrip.rs similarity index 98% rename from crates/era/tests/it/roundtrip.rs rename to crates/era/tests/it/era1/roundtrip.rs index cbc51e111c..84250c440e 100644 --- a/crates/era/tests/it/roundtrip.rs +++ b/crates/era/tests/it/era1/roundtrip.rs @@ -24,11 +24,11 @@ use reth_era::{ use reth_ethereum_primitives::TransactionSigned; use std::io::Cursor; -use crate::{Era1TestDownloader, MAINNET, SEPOLIA}; +use crate::{EraTestDownloader, MAINNET, SEPOLIA}; // Helper function to test roundtrip compression/encoding for a specific file async fn test_file_roundtrip( - downloader: &Era1TestDownloader, + downloader: &EraTestDownloader, filename: &str, network: &str, ) -> eyre::Result<()> { @@ -259,7 +259,7 @@ async fn test_file_roundtrip( #[tokio::test(flavor = "multi_thread")] #[ignore = "download intensive"] async fn test_roundtrip_compression_encoding_mainnet(filename: &str) -> eyre::Result<()> { - let downloader = Era1TestDownloader::new().await?; + let downloader = EraTestDownloader::new().await?; test_file_roundtrip(&downloader, filename, MAINNET).await } @@ -270,7 +270,7 @@ async fn test_roundtrip_compression_encoding_mainnet(filename: &str) -> eyre::Re #[tokio::test(flavor = "multi_thread")] #[ignore = "download intensive"] async fn test_roundtrip_compression_encoding_sepolia(filename: &str) -> eyre::Result<()> { - let downloader = Era1TestDownloader::new().await?; + let downloader = EraTestDownloader::new().await?; test_file_roundtrip(&downloader, filename, SEPOLIA).await?; diff --git a/crates/era/tests/it/main.rs b/crates/era/tests/it/main.rs index 625f24a5ec..d9587e8fce 100644 --- a/crates/era/tests/it/main.rs +++ b/crates/era/tests/it/main.rs @@ -8,8 +8,9 @@ use reqwest::{Client, Url}; use reth_era::{ - common::file_ops::FileReader, + common::file_ops::{EraFileType, FileReader}, e2s::error::E2sError, + era::file::{EraFile, EraReader}, era1::file::{Era1File, Era1Reader}, }; use reth_era_downloader::EraClient; @@ -23,9 +24,8 @@ use std::{ use eyre::{eyre, Result}; use tempfile::TempDir; -mod dd; -mod genesis; -mod roundtrip; +mod era; +mod era1; const fn main() {} @@ -33,7 +33,7 @@ const fn main() {} const MAINNET: &str = "mainnet"; /// Default mainnet url /// for downloading mainnet `.era1` files -const MAINNET_URL: &str = "https://era.ithaca.xyz/era1/"; +const ERA1_MAINNET_URL: &str = "https://era.ithaca.xyz/era1/"; /// Succinct list of mainnet files we want to download /// from @@ -54,7 +54,7 @@ const SEPOLIA: &str = "sepolia"; /// Default sepolia url /// for downloading sepolia `.era1` files -const SEPOLIA_URL: &str = "https://era.ithaca.xyz/sepolia-era1/"; +const ERA1_SEPOLIA_URL: &str = "https://era.ithaca.xyz/sepolia-era1/"; /// Succinct list of sepolia files we want to download /// from @@ -66,18 +66,50 @@ const ERA1_SEPOLIA_FILES_NAMES: [&str; 4] = [ "sepolia-00182-a4f0a8a1.era1", ]; +const HOODI: &str = "hoodi"; + +/// Default hoodi url +/// for downloading hoodi `.era` files +/// TODO: to replace with internal era files hosting url +const ERA_HOODI_URL: &str = "https://hoodi.era.nimbus.team/"; + +/// Succinct list of hoodi files we want to download +/// from //TODO: to replace with internal era files hosting url +/// for testing purposes +const ERA_HOODI_FILES_NAMES: [&str; 4] = [ + "hoodi-00000-212f13fc.era", + "hoodi-00021-857e418b.era", + "hoodi-00175-202aaa6d.era", + "hoodi-00201-0d521fc8.era", +]; + +/// Default mainnet url +/// for downloading mainnet `.era` files +//TODO: to replace with internal era files hosting url +const ERA_MAINNET_URL: &str = "https://mainnet.era.nimbus.team/"; + +/// Succinct list of mainnet files we want to download +/// from //TODO: to replace with internal era files hosting url +/// for testing purposes +const ERA_MAINNET_FILES_NAMES: [&str; 4] = [ + "mainnet-00000-4b363db9.era", + "mainnet-00518-4e267a3a.era", + "mainnet-01140-f70d4869.era", + "mainnet-01581-82073d28.era", +]; + /// Utility for downloading `.era1` files for tests /// in a temporary directory /// and caching them in memory #[derive(Debug)] -struct Era1TestDownloader { +struct EraTestDownloader { /// Temporary directory for storing downloaded files temp_dir: TempDir, /// Cache mapping file names to their paths file_cache: Arc>>, } -impl Era1TestDownloader { +impl EraTestDownloader { /// Create a new downloader instance with a temporary directory async fn new() -> Result { let temp_dir = @@ -97,29 +129,9 @@ impl Era1TestDownloader { } // check if the filename is supported - if !ERA1_MAINNET_FILES_NAMES.contains(&filename) && - !ERA1_SEPOLIA_FILES_NAMES.contains(&filename) - { - return Err(eyre!( - "Unknown file: {}. Only the following files are supported: {:?} or {:?}", - filename, - ERA1_MAINNET_FILES_NAMES, - ERA1_SEPOLIA_FILES_NAMES - )); - } - - // initialize the client and build url config - let url = match network { - MAINNET => MAINNET_URL, - SEPOLIA => SEPOLIA_URL, - _ => { - return Err(eyre!( - "Unknown network: {}. Only mainnet and sepolia are supported.", - network - )); - } - }; + self.validate_filename(filename, network)?; + let (url, _): (&str, &[&str]) = self.get_network_config(filename, network)?; let final_url = Url::from_str(url).map_err(|e| eyre!("Failed to parse URL: {}", e))?; let folder = self.temp_dir.path(); @@ -142,6 +154,7 @@ impl Era1TestDownloader { .download_to_file(file_url) .await .map_err(|e| eyre!("Failed to download file: {}", e))?; + // update the cache { let mut cache = self.file_cache.lock().unwrap(); @@ -151,9 +164,54 @@ impl Era1TestDownloader { Ok(downloaded_path.to_path_buf()) } + /// Validate that filename is in the supported list for the network + fn validate_filename(&self, filename: &str, network: &str) -> Result<()> { + let (_, supported_files) = self.get_network_config(filename, network)?; + + if !supported_files.contains(&filename) { + return Err(eyre!( + "Unknown file: '{}' for network '{}'. Supported files: {:?}", + filename, + network, + supported_files + )); + } + + Ok(()) + } + + /// Get network configuration, URL and supported files, based on network and file type + fn get_network_config( + &self, + filename: &str, + network: &str, + ) -> Result<(&'static str, &'static [&'static str])> { + let file_type = EraFileType::from_filename(filename) + .ok_or_else(|| eyre!("Unknown file extension for: {}", filename))?; + + match (network, file_type) { + (MAINNET, EraFileType::Era1) => Ok((ERA1_MAINNET_URL, &ERA1_MAINNET_FILES_NAMES[..])), + (MAINNET, EraFileType::Era) => Ok((ERA_MAINNET_URL, &ERA_MAINNET_FILES_NAMES[..])), + (SEPOLIA, EraFileType::Era1) => Ok((ERA1_SEPOLIA_URL, &ERA1_SEPOLIA_FILES_NAMES[..])), + (HOODI, EraFileType::Era) => Ok((ERA_HOODI_URL, &ERA_HOODI_FILES_NAMES[..])), + _ => Err(eyre!( + "Unsupported combination: network '{}' with file type '{:?}'", + network, + file_type + )), + } + } + /// open .era1 file, downloading it if necessary async fn open_era1_file(&self, filename: &str, network: &str) -> Result { let path = self.download_file(filename, network).await?; Era1Reader::open(&path, network).map_err(|e| eyre!("Failed to open Era1 file: {e}")) } + + /// open .era file, downloading it if necessary + #[allow(dead_code)] + async fn open_era_file(&self, filename: &str, network: &str) -> Result { + let path = self.download_file(filename, network).await?; + EraReader::open(&path, network).map_err(|e| eyre!("Failed to open Era1 file: {e}")) + } } From 6e365949c4b0ed7e0141d6cb39c4cdac86f3783d Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Sat, 22 Nov 2025 00:38:20 +0400 Subject: [PATCH 15/23] feat: allow customizing dev block timestamp (#19904) --- Cargo.lock | 2 ++ crates/engine/local/Cargo.toml | 2 ++ crates/engine/local/src/miner.rs | 38 ++++++++++-------------- crates/engine/local/src/payload.rs | 39 +++++++++++++++++++------ crates/node/builder/src/launch/debug.rs | 12 ++++---- crates/payload/primitives/Cargo.toml | 2 ++ crates/payload/primitives/src/traits.rs | 36 +++++++++++++---------- 7 files changed, 77 insertions(+), 54 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bf9b755927..a729aa53e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8101,6 +8101,7 @@ dependencies = [ "reth-optimism-chainspec", "reth-payload-builder", "reth-payload-primitives", + "reth-primitives-traits", "reth-storage-api", "reth-transaction-pool", "tokio", @@ -9801,6 +9802,7 @@ dependencies = [ name = "reth-payload-primitives" version = "1.9.3" dependencies = [ + "alloy-consensus", "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", diff --git a/crates/engine/local/Cargo.toml b/crates/engine/local/Cargo.toml index dd708dee90..8bf9e28bcb 100644 --- a/crates/engine/local/Cargo.toml +++ b/crates/engine/local/Cargo.toml @@ -15,6 +15,7 @@ reth-engine-primitives = { workspace = true, features = ["std"] } reth-ethereum-engine-primitives.workspace = true reth-payload-builder.workspace = true reth-payload-primitives.workspace = true +reth-primitives-traits.workspace = true reth-storage-api.workspace = true reth-transaction-pool.workspace = true @@ -43,4 +44,5 @@ op = [ "dep:op-alloy-rpc-types-engine", "dep:reth-optimism-chainspec", "reth-payload-primitives/op", + "reth-primitives-traits/op", ] diff --git a/crates/engine/local/src/miner.rs b/crates/engine/local/src/miner.rs index d6298502fb..67001ee73e 100644 --- a/crates/engine/local/src/miner.rs +++ b/crates/engine/local/src/miner.rs @@ -1,6 +1,5 @@ //! Contains the implementation of the mining mode for the local engine. -use alloy_consensus::BlockHeader; use alloy_primitives::{TxHash, B256}; use alloy_rpc_types_engine::ForkchoiceState; use eyre::OptionExt; @@ -10,6 +9,7 @@ use reth_payload_builder::PayloadBuilderHandle; use reth_payload_primitives::{ BuiltPayload, EngineApiMessageVersion, PayloadAttributesBuilder, PayloadKind, PayloadTypes, }; +use reth_primitives_traits::{HeaderTy, SealedHeaderFor}; use reth_storage_api::BlockReader; use reth_transaction_pool::TransactionPool; use std::{ @@ -17,7 +17,7 @@ use std::{ future::Future, pin::Pin, task::{Context, Poll}, - time::{Duration, UNIX_EPOCH}, + time::Duration, }; use tokio::time::Interval; use tokio_stream::wrappers::ReceiverStream; @@ -106,8 +106,8 @@ pub struct LocalMiner { mode: MiningMode, /// The payload builder for the engine payload_builder: PayloadBuilderHandle, - /// Timestamp for the next block. - last_timestamp: u64, + /// Latest block in the chain so far. + last_header: SealedHeaderFor<::Primitives>, /// Stores latest mined blocks. last_block_hashes: VecDeque, } @@ -115,18 +115,21 @@ pub struct LocalMiner { impl LocalMiner where T: PayloadTypes, - B: PayloadAttributesBuilder<::PayloadAttributes>, + B: PayloadAttributesBuilder< + T::PayloadAttributes, + HeaderTy<::Primitives>, + >, Pool: TransactionPool + Unpin, { /// Spawns a new [`LocalMiner`] with the given parameters. pub fn new( - provider: impl BlockReader, + provider: impl BlockReader
::Primitives>>, payload_attributes_builder: B, to_engine: ConsensusEngineHandle, mode: MiningMode, payload_builder: PayloadBuilderHandle, ) -> Self { - let latest_header = + let last_header = provider.sealed_header(provider.best_block_number().unwrap()).unwrap().unwrap(); Self { @@ -134,8 +137,8 @@ where to_engine, mode, payload_builder, - last_timestamp: latest_header.timestamp(), - last_block_hashes: VecDeque::from([latest_header.hash()]), + last_block_hashes: VecDeque::from([last_header.hash()]), + last_header, } } @@ -193,19 +196,11 @@ where /// Generates payload attributes for a new block, passes them to FCU and inserts built payload /// through newPayload. async fn advance(&mut self) -> eyre::Result<()> { - let timestamp = std::cmp::max( - self.last_timestamp.saturating_add(1), - std::time::SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("cannot be earlier than UNIX_EPOCH") - .as_secs(), - ); - let res = self .to_engine .fork_choice_updated( self.forkchoice_state(), - Some(self.payload_attributes_builder.build(timestamp)), + Some(self.payload_attributes_builder.build(&self.last_header)), EngineApiMessageVersion::default(), ) .await?; @@ -222,8 +217,7 @@ where eyre::bail!("No payload") }; - let block = payload.block(); - + let header = payload.block().sealed_header().clone(); let payload = T::block_to_payload(payload.block().clone()); let res = self.to_engine.new_payload(payload).await?; @@ -231,8 +225,8 @@ where eyre::bail!("Invalid payload") } - self.last_timestamp = timestamp; - self.last_block_hashes.push_back(block.hash()); + self.last_block_hashes.push_back(header.hash()); + self.last_header = header; // ensure we keep at most 64 blocks if self.last_block_hashes.len() > 64 { self.last_block_hashes.pop_front(); diff --git a/crates/engine/local/src/payload.rs b/crates/engine/local/src/payload.rs index 34deaf3e10..dc3be02f17 100644 --- a/crates/engine/local/src/payload.rs +++ b/crates/engine/local/src/payload.rs @@ -1,10 +1,12 @@ //! The implementation of the [`PayloadAttributesBuilder`] for the //! [`LocalMiner`](super::LocalMiner). +use alloy_consensus::BlockHeader; use alloy_primitives::{Address, B256}; -use reth_chainspec::EthereumHardforks; +use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_ethereum_engine_primitives::EthPayloadAttributes; use reth_payload_primitives::PayloadAttributesBuilder; +use reth_primitives_traits::SealedHeader; use std::sync::Arc; /// The attributes builder for local Ethereum payload. @@ -13,21 +15,36 @@ use std::sync::Arc; pub struct LocalPayloadAttributesBuilder { /// The chainspec pub chain_spec: Arc, + + /// Whether to enforce increasing timestamp. + pub enforce_increasing_timestamp: bool, } impl LocalPayloadAttributesBuilder { /// Creates a new instance of the builder. pub const fn new(chain_spec: Arc) -> Self { - Self { chain_spec } + Self { chain_spec, enforce_increasing_timestamp: true } + } + + /// Creates a new instance of the builder without enforcing increasing timestamps. + pub fn without_increasing_timestamp(self) -> Self { + Self { enforce_increasing_timestamp: false, ..self } } } -impl PayloadAttributesBuilder +impl PayloadAttributesBuilder for LocalPayloadAttributesBuilder where - ChainSpec: Send + Sync + EthereumHardforks + 'static, + ChainSpec: EthChainSpec + EthereumHardforks + 'static, { - fn build(&self, timestamp: u64) -> EthPayloadAttributes { + fn build(&self, parent: &SealedHeader) -> EthPayloadAttributes { + let mut timestamp = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs(); + + if self.enforce_increasing_timestamp { + timestamp = std::cmp::max(parent.timestamp().saturating_add(1), timestamp); + } + EthPayloadAttributes { timestamp, prev_randao: B256::random(), @@ -45,14 +62,18 @@ where } #[cfg(feature = "op")] -impl PayloadAttributesBuilder +impl + PayloadAttributesBuilder for LocalPayloadAttributesBuilder where - ChainSpec: Send + Sync + EthereumHardforks + 'static, + ChainSpec: EthChainSpec + EthereumHardforks + 'static, { - fn build(&self, timestamp: u64) -> op_alloy_rpc_types_engine::OpPayloadAttributes { + fn build( + &self, + parent: &SealedHeader, + ) -> op_alloy_rpc_types_engine::OpPayloadAttributes { op_alloy_rpc_types_engine::OpPayloadAttributes { - payload_attributes: self.build(timestamp), + payload_attributes: self.build(parent), // Add dummy system transaction transactions: Some(vec![ reth_optimism_chainspec::constants::TX_SET_L1_BLOCK_OP_MAINNET_BLOCK_124665056 diff --git a/crates/node/builder/src/launch/debug.rs b/crates/node/builder/src/launch/debug.rs index f5e9745cdd..a623a825ad 100644 --- a/crates/node/builder/src/launch/debug.rs +++ b/crates/node/builder/src/launch/debug.rs @@ -7,7 +7,7 @@ use reth_chainspec::EthChainSpec; use reth_consensus_debug_client::{DebugConsensusClient, EtherscanBlockProvider, RpcBlockProvider}; use reth_engine_local::LocalMiner; use reth_node_api::{ - BlockTy, FullNodeComponents, PayloadAttrTy, PayloadAttributesBuilder, PayloadTypes, + BlockTy, FullNodeComponents, HeaderTy, PayloadAttrTy, PayloadAttributesBuilder, PayloadTypes, }; use std::{ future::{Future, IntoFuture}, @@ -73,9 +73,7 @@ pub trait DebugNode: Node { /// be constructed during local mining. fn local_payload_attributes_builder( chain_spec: &Self::ChainSpec, - ) -> impl PayloadAttributesBuilder< - <::Payload as PayloadTypes>::PayloadAttributes, - >; + ) -> impl PayloadAttributesBuilder<::PayloadAttributes, HeaderTy>; } /// Node launcher with support for launching various debugging utilities. @@ -120,7 +118,7 @@ where inner: L, target: Target, local_payload_attributes_builder: - Option>>>, + Option, HeaderTy>>>, map_attributes: Option) -> PayloadAttrTy + Send + Sync>>, } @@ -133,7 +131,7 @@ where { pub fn with_payload_attributes_builder( self, - builder: impl PayloadAttributesBuilder>, + builder: impl PayloadAttributesBuilder, HeaderTy>, ) -> Self { Self { inner: self.inner, @@ -229,7 +227,7 @@ where } else { let local = N::Types::local_payload_attributes_builder(&chain_spec); let builder = if let Some(f) = map_attributes { - Either::Left(move |block_number| f(local.build(block_number))) + Either::Left(move |parent| f(local.build(&parent))) } else { Either::Right(local) }; diff --git a/crates/payload/primitives/Cargo.toml b/crates/payload/primitives/Cargo.toml index ce8e07fc8e..32a5476a80 100644 --- a/crates/payload/primitives/Cargo.toml +++ b/crates/payload/primitives/Cargo.toml @@ -21,6 +21,7 @@ reth-execution-types.workspace = true reth-trie-common.workspace = true # alloy +alloy-consensus.workspace = true alloy-eips.workspace = true alloy-primitives.workspace = true alloy-rpc-types-engine = { workspace = true, features = ["serde"] } @@ -50,6 +51,7 @@ std = [ "thiserror/std", "reth-primitives-traits/std", "either/std", + "alloy-consensus/std", ] op = [ "dep:op-alloy-rpc-types-engine", diff --git a/crates/payload/primitives/src/traits.rs b/crates/payload/primitives/src/traits.rs index a95a7209e9..51390a7b84 100644 --- a/crates/payload/primitives/src/traits.rs +++ b/crates/payload/primitives/src/traits.rs @@ -197,40 +197,44 @@ impl PayloadAttributes for op_alloy_rpc_types_engine::OpPayloadAttributes { /// /// Enables different strategies for generating payload attributes based on /// contextual information. Useful for testing and specialized building. -pub trait PayloadAttributesBuilder: Send + Sync + 'static { +pub trait PayloadAttributesBuilder: + Send + Sync + 'static +{ /// Constructs new payload attributes for the given timestamp. - fn build(&self, timestamp: u64) -> Attributes; + fn build(&self, parent: &SealedHeader
) -> Attributes; } -impl PayloadAttributesBuilder for F +impl PayloadAttributesBuilder for F where - F: Fn(u64) -> Attributes + Send + Sync + 'static, + Header: Clone, + F: Fn(SealedHeader
) -> Attributes + Send + Sync + 'static, { - fn build(&self, timestamp: u64) -> Attributes { - self(timestamp) + fn build(&self, parent: &SealedHeader
) -> Attributes { + self(parent.clone()) } } -impl PayloadAttributesBuilder for Either +impl PayloadAttributesBuilder for Either where - L: PayloadAttributesBuilder, - R: PayloadAttributesBuilder, + L: PayloadAttributesBuilder, + R: PayloadAttributesBuilder, { - fn build(&self, timestamp: u64) -> Attributes { + fn build(&self, parent: &SealedHeader
) -> Attributes { match self { - Self::Left(l) => l.build(timestamp), - Self::Right(r) => r.build(timestamp), + Self::Left(l) => l.build(parent), + Self::Right(r) => r.build(parent), } } } -impl PayloadAttributesBuilder - for Box> +impl PayloadAttributesBuilder + for Box> where + Header: 'static, Attributes: 'static, { - fn build(&self, timestamp: u64) -> Attributes { - self.as_ref().build(timestamp) + fn build(&self, parent: &SealedHeader
) -> Attributes { + self.as_ref().build(parent) } } From 7f40013cf6aca02c7cc8dd4ee8071c87cf2c72d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Fri, 21 Nov 2025 23:05:43 +0200 Subject: [PATCH 16/23] feat(tracing-otlp): make trace id ratio sample customizable with `--tracing-otlp.sample-ratio` arg (#19438) --- Cargo.lock | 1 - crates/node/core/src/args/trace.rs | 44 ++++++--- crates/tracing-otlp/src/lib.rs | 90 ++++++++++++++++--- crates/tracing/Cargo.toml | 3 +- crates/tracing/src/layers.rs | 13 +-- docs/vocs/docs/pages/cli/reth.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/config.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/db.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/db/checksum.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/db/clear.mdx | 9 ++ .../docs/pages/cli/reth/db/clear/mdbx.mdx | 9 ++ .../pages/cli/reth/db/clear/static-file.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/db/diff.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/db/drop.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/db/get.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/db/get/mdbx.mdx | 9 ++ .../pages/cli/reth/db/get/static-file.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/db/list.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/db/path.mdx | 9 ++ .../docs/pages/cli/reth/db/repair-trie.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/db/settings.mdx | 9 ++ .../docs/pages/cli/reth/db/settings/get.mdx | 9 ++ .../docs/pages/cli/reth/db/settings/set.mdx | 9 ++ .../settings/set/receipts_in_static_files.mdx | 9 ++ .../transaction_senders_in_static_files.mdx | 9 ++ .../pages/cli/reth/db/static-file-header.mdx | 9 ++ .../cli/reth/db/static-file-header/block.mdx | 9 ++ .../cli/reth/db/static-file-header/path.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/db/stats.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/db/version.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/download.mdx | 9 ++ .../vocs/docs/pages/cli/reth/dump-genesis.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/export-era.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/import-era.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/import.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/init-state.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/init.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/node.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/p2p.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/p2p/body.mdx | 9 ++ .../vocs/docs/pages/cli/reth/p2p/bootnode.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/p2p/header.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/p2p/rlpx.mdx | 9 ++ .../docs/pages/cli/reth/p2p/rlpx/ping.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/prune.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/re-execute.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/stage.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/stage/drop.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/stage/dump.mdx | 9 ++ .../cli/reth/stage/dump/account-hashing.mdx | 9 ++ .../pages/cli/reth/stage/dump/execution.mdx | 9 ++ .../docs/pages/cli/reth/stage/dump/merkle.mdx | 9 ++ .../cli/reth/stage/dump/storage-hashing.mdx | 9 ++ docs/vocs/docs/pages/cli/reth/stage/run.mdx | 9 ++ .../vocs/docs/pages/cli/reth/stage/unwind.mdx | 9 ++ .../cli/reth/stage/unwind/num-blocks.mdx | 9 ++ .../pages/cli/reth/stage/unwind/to-block.mdx | 9 ++ 57 files changed, 584 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a729aa53e4..448e638b64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10745,7 +10745,6 @@ dependencies = [ "tracing-journald", "tracing-logfmt", "tracing-subscriber 0.3.20", - "url", ] [[package]] diff --git a/crates/node/core/src/args/trace.rs b/crates/node/core/src/args/trace.rs index f3b739eb2d..341971bd31 100644 --- a/crates/node/core/src/args/trace.rs +++ b/crates/node/core/src/args/trace.rs @@ -3,7 +3,7 @@ use clap::Parser; use eyre::WrapErr; use reth_tracing::{tracing_subscriber::EnvFilter, Layers}; -use reth_tracing_otlp::OtlpProtocol; +use reth_tracing_otlp::{OtlpConfig, OtlpProtocol}; use url::Url; /// CLI arguments for configuring `Opentelemetry` trace and span export. @@ -78,6 +78,23 @@ pub struct TraceArgs { help_heading = "Tracing" )] pub service_name: String, + + /// Trace sampling ratio to control the percentage of traces to export. + /// + /// Valid range: 0.0 to 1.0 + /// - 1.0, default: Sample all traces + /// - 0.01: Sample 1% of traces + /// - 0.0: Disable sampling + /// + /// Example: --tracing-otlp.sample-ratio=0.0. + #[arg( + long = "tracing-otlp.sample-ratio", + env = "OTEL_TRACES_SAMPLER_ARG", + global = true, + value_name = "RATIO", + help_heading = "Tracing" + )] + pub sample_ratio: Option, } impl Default for TraceArgs { @@ -86,6 +103,7 @@ impl Default for TraceArgs { otlp: None, protocol: OtlpProtocol::Http, otlp_filter: EnvFilter::from_default_env(), + sample_ratio: None, service_name: "reth".to_string(), } } @@ -102,22 +120,24 @@ impl TraceArgs { /// Note: even though this function is async, it does not actually perform any async operations. /// It's needed only to be able to initialize the gRPC transport of OTLP tracing that needs to /// be called inside a tokio runtime context. - pub async fn init_otlp_tracing( - &mut self, - _layers: &mut Layers, - ) -> eyre::Result { + pub async fn init_otlp_tracing(&mut self, layers: &mut Layers) -> eyre::Result { if let Some(endpoint) = self.otlp.as_mut() { self.protocol.validate_endpoint(endpoint)?; #[cfg(feature = "otlp")] { - _layers.with_span_layer( - self.service_name.clone(), - endpoint.clone(), - self.otlp_filter.clone(), - self.protocol, - )?; - Ok(OtlpInitStatus::Started(endpoint.clone())) + { + let config = OtlpConfig::new( + self.service_name.clone(), + endpoint.clone(), + self.protocol, + self.sample_ratio, + )?; + + layers.with_span_layer(config.clone(), self.otlp_filter.clone())?; + + Ok(OtlpInitStatus::Started(config.endpoint().clone())) + } } #[cfg(not(feature = "otlp"))] { diff --git a/crates/tracing-otlp/src/lib.rs b/crates/tracing-otlp/src/lib.rs index 2cfd332a40..c7af074ad1 100644 --- a/crates/tracing-otlp/src/lib.rs +++ b/crates/tracing-otlp/src/lib.rs @@ -12,7 +12,7 @@ use opentelemetry::{global, trace::TracerProvider, KeyValue, Value}; use opentelemetry_otlp::{SpanExporter, WithExportConfig}; use opentelemetry_sdk::{ propagation::TraceContextPropagator, - trace::{SdkTracer, SdkTracerProvider}, + trace::{Sampler, SdkTracer, SdkTracerProvider}, Resource, }; use opentelemetry_semantic_conventions::{attribute::SERVICE_VERSION, SCHEMA_URL}; @@ -29,36 +29,92 @@ const HTTP_TRACE_ENDPOINT: &str = "/v1/traces"; /// /// This layer can be added to a [`tracing_subscriber::Registry`] to enable `OpenTelemetry` tracing /// with OTLP export to an url. -pub fn span_layer( - service_name: impl Into, - endpoint: &Url, - protocol: OtlpProtocol, -) -> eyre::Result> +pub fn span_layer(otlp_config: OtlpConfig) -> eyre::Result> where for<'span> S: Subscriber + LookupSpan<'span>, { global::set_text_map_propagator(TraceContextPropagator::new()); - let resource = build_resource(service_name); + let resource = build_resource(otlp_config.service_name.clone()); let span_builder = SpanExporter::builder(); - let span_exporter = match protocol { - OtlpProtocol::Http => span_builder.with_http().with_endpoint(endpoint.as_str()).build()?, - OtlpProtocol::Grpc => span_builder.with_tonic().with_endpoint(endpoint.as_str()).build()?, + let span_exporter = match otlp_config.protocol { + OtlpProtocol::Http => { + span_builder.with_http().with_endpoint(otlp_config.endpoint.as_str()).build()? + } + OtlpProtocol::Grpc => { + span_builder.with_tonic().with_endpoint(otlp_config.endpoint.as_str()).build()? + } }; + let sampler = build_sampler(otlp_config.sample_ratio)?; + let tracer_provider = SdkTracerProvider::builder() .with_resource(resource) + .with_sampler(sampler) .with_batch_exporter(span_exporter) .build(); global::set_tracer_provider(tracer_provider.clone()); - let tracer = tracer_provider.tracer("reth"); + let tracer = tracer_provider.tracer(otlp_config.service_name); Ok(tracing_opentelemetry::layer().with_tracer(tracer)) } +/// Configuration for OTLP trace export. +#[derive(Debug, Clone)] +pub struct OtlpConfig { + /// Service name for trace identification + service_name: String, + /// Otlp endpoint URL + endpoint: Url, + /// Transport protocol, HTTP or gRPC + protocol: OtlpProtocol, + /// Optional sampling ratio, from 0.0 to 1.0 + sample_ratio: Option, +} + +impl OtlpConfig { + /// Creates a new OTLP configuration. + pub fn new( + service_name: impl Into, + endpoint: Url, + protocol: OtlpProtocol, + sample_ratio: Option, + ) -> eyre::Result { + if let Some(ratio) = sample_ratio { + ensure!( + (0.0..=1.0).contains(&ratio), + "Sample ratio must be between 0.0 and 1.0, got: {}", + ratio + ); + } + + Ok(Self { service_name: service_name.into(), endpoint, protocol, sample_ratio }) + } + + /// Returns the service name. + pub fn service_name(&self) -> &str { + &self.service_name + } + + /// Returns the OTLP endpoint URL. + pub const fn endpoint(&self) -> &Url { + &self.endpoint + } + + /// Returns the transport protocol. + pub const fn protocol(&self) -> OtlpProtocol { + self.protocol + } + + /// Returns the sampling ratio. + pub const fn sample_ratio(&self) -> Option { + self.sample_ratio + } +} + // Builds OTLP resource with service information. fn build_resource(service_name: impl Into) -> Resource { Resource::builder() @@ -67,6 +123,18 @@ fn build_resource(service_name: impl Into) -> Resource { .build() } +/// Builds the appropriate sampler based on the sample ratio. +fn build_sampler(sample_ratio: Option) -> eyre::Result { + match sample_ratio { + // Default behavior: sample all traces + None | Some(1.0) => Ok(Sampler::ParentBased(Box::new(Sampler::AlwaysOn))), + // Don't sample anything + Some(0.0) => Ok(Sampler::ParentBased(Box::new(Sampler::AlwaysOff))), + // Sample based on trace ID ratio + Some(ratio) => Ok(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(ratio)))), + } +} + /// OTLP transport protocol type #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] pub enum OtlpProtocol { diff --git a/crates/tracing/Cargo.toml b/crates/tracing/Cargo.toml index 8cf83e138c..f15ad8d650 100644 --- a/crates/tracing/Cargo.toml +++ b/crates/tracing/Cargo.toml @@ -26,8 +26,7 @@ tracing-logfmt.workspace = true clap = { workspace = true, features = ["derive"] } eyre.workspace = true rolling-file.workspace = true -url = { workspace = true, optional = true } [features] default = ["otlp"] -otlp = ["reth-tracing-otlp", "dep:url"] +otlp = ["reth-tracing-otlp"] diff --git a/crates/tracing/src/layers.rs b/crates/tracing/src/layers.rs index 33f8c90ada..44462ca7c6 100644 --- a/crates/tracing/src/layers.rs +++ b/crates/tracing/src/layers.rs @@ -1,4 +1,6 @@ use crate::formatter::LogFormat; +#[cfg(feature = "otlp")] +use reth_tracing_otlp::{span_layer, OtlpConfig}; use rolling_file::{RollingConditionBasic, RollingFileAppender}; use std::{ fmt, @@ -6,11 +8,6 @@ use std::{ }; use tracing_appender::non_blocking::WorkerGuard; use tracing_subscriber::{filter::Directive, EnvFilter, Layer, Registry}; -#[cfg(feature = "otlp")] -use { - reth_tracing_otlp::{span_layer, OtlpProtocol}, - url::Url, -}; /// A worker guard returned by the file layer. /// @@ -137,14 +134,12 @@ impl Layers { #[cfg(feature = "otlp")] pub fn with_span_layer( &mut self, - service_name: String, - endpoint_exporter: Url, + otlp_config: OtlpConfig, filter: EnvFilter, - otlp_protocol: OtlpProtocol, ) -> eyre::Result<()> { // Create the span provider - let span_layer = span_layer(service_name, &endpoint_exporter, otlp_protocol) + let span_layer = span_layer(otlp_config) .map_err(|e| eyre::eyre!("Failed to build OTLP span exporter {}", e))? .with_filter(filter); diff --git a/docs/vocs/docs/pages/cli/reth.mdx b/docs/vocs/docs/pages/cli/reth.mdx index c35216d6b5..0287f2e47f 100644 --- a/docs/vocs/docs/pages/cli/reth.mdx +++ b/docs/vocs/docs/pages/cli/reth.mdx @@ -146,4 +146,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/config.mdx b/docs/vocs/docs/pages/cli/reth/config.mdx index 6b3c9e4b65..07d09a16da 100644 --- a/docs/vocs/docs/pages/cli/reth/config.mdx +++ b/docs/vocs/docs/pages/cli/reth/config.mdx @@ -132,4 +132,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db.mdx b/docs/vocs/docs/pages/cli/reth/db.mdx index 95ea29b7bc..64c3cde45b 100644 --- a/docs/vocs/docs/pages/cli/reth/db.mdx +++ b/docs/vocs/docs/pages/cli/reth/db.mdx @@ -242,4 +242,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/checksum.mdx b/docs/vocs/docs/pages/cli/reth/db/checksum.mdx index 4b8b8ca2cc..21ce752c42 100644 --- a/docs/vocs/docs/pages/cli/reth/db/checksum.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/checksum.mdx @@ -149,4 +149,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/clear.mdx b/docs/vocs/docs/pages/cli/reth/db/clear.mdx index 1548558fe3..3483b71f46 100644 --- a/docs/vocs/docs/pages/cli/reth/db/clear.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/clear.mdx @@ -141,4 +141,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/clear/mdbx.mdx b/docs/vocs/docs/pages/cli/reth/db/clear/mdbx.mdx index b48ba18098..9fbfb1e825 100644 --- a/docs/vocs/docs/pages/cli/reth/db/clear/mdbx.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/clear/mdbx.mdx @@ -140,4 +140,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx b/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx index b9ad2a0cc7..61ad643752 100644 --- a/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx @@ -144,4 +144,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/diff.mdx b/docs/vocs/docs/pages/cli/reth/db/diff.mdx index 6032a93b20..745211efa4 100644 --- a/docs/vocs/docs/pages/cli/reth/db/diff.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/diff.mdx @@ -192,4 +192,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/drop.mdx b/docs/vocs/docs/pages/cli/reth/db/drop.mdx index c778320f2d..7529225295 100644 --- a/docs/vocs/docs/pages/cli/reth/db/drop.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/drop.mdx @@ -139,4 +139,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/get.mdx b/docs/vocs/docs/pages/cli/reth/db/get.mdx index dfcfcac188..abb95ab6b4 100644 --- a/docs/vocs/docs/pages/cli/reth/db/get.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/get.mdx @@ -141,4 +141,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/get/mdbx.mdx b/docs/vocs/docs/pages/cli/reth/db/get/mdbx.mdx index 981d0c9f9a..b909d0b0bc 100644 --- a/docs/vocs/docs/pages/cli/reth/db/get/mdbx.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/get/mdbx.mdx @@ -149,4 +149,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx b/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx index 36fd873bcc..3db4a946c4 100644 --- a/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx @@ -150,4 +150,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/list.mdx b/docs/vocs/docs/pages/cli/reth/db/list.mdx index 3be1cd183b..0c1f4bc857 100644 --- a/docs/vocs/docs/pages/cli/reth/db/list.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/list.mdx @@ -182,4 +182,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/path.mdx b/docs/vocs/docs/pages/cli/reth/db/path.mdx index a954093dd5..b0b2c3c754 100644 --- a/docs/vocs/docs/pages/cli/reth/db/path.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/path.mdx @@ -136,4 +136,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/repair-trie.mdx b/docs/vocs/docs/pages/cli/reth/db/repair-trie.mdx index 6436afc213..c649522bcc 100644 --- a/docs/vocs/docs/pages/cli/reth/db/repair-trie.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/repair-trie.mdx @@ -139,4 +139,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/settings.mdx b/docs/vocs/docs/pages/cli/reth/db/settings.mdx index 90ab3a3c8b..19d707370e 100644 --- a/docs/vocs/docs/pages/cli/reth/db/settings.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/settings.mdx @@ -141,4 +141,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/settings/get.mdx b/docs/vocs/docs/pages/cli/reth/db/settings/get.mdx index b96290cf0a..9df0654725 100644 --- a/docs/vocs/docs/pages/cli/reth/db/settings/get.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/settings/get.mdx @@ -136,4 +136,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/settings/set.mdx b/docs/vocs/docs/pages/cli/reth/db/settings/set.mdx index d1d4e2f216..e4412b6178 100644 --- a/docs/vocs/docs/pages/cli/reth/db/settings/set.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/settings/set.mdx @@ -141,4 +141,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/settings/set/receipts_in_static_files.mdx b/docs/vocs/docs/pages/cli/reth/db/settings/set/receipts_in_static_files.mdx index 22cc15b1b2..e9dadf91ef 100644 --- a/docs/vocs/docs/pages/cli/reth/db/settings/set/receipts_in_static_files.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/settings/set/receipts_in_static_files.mdx @@ -140,4 +140,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/settings/set/transaction_senders_in_static_files.mdx b/docs/vocs/docs/pages/cli/reth/db/settings/set/transaction_senders_in_static_files.mdx index 18115a5094..7dd4bb91af 100644 --- a/docs/vocs/docs/pages/cli/reth/db/settings/set/transaction_senders_in_static_files.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/settings/set/transaction_senders_in_static_files.mdx @@ -140,4 +140,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/static-file-header.mdx b/docs/vocs/docs/pages/cli/reth/db/static-file-header.mdx index 5d4764a2e6..0343097545 100644 --- a/docs/vocs/docs/pages/cli/reth/db/static-file-header.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/static-file-header.mdx @@ -141,4 +141,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/static-file-header/block.mdx b/docs/vocs/docs/pages/cli/reth/db/static-file-header/block.mdx index 8f18e10c7f..edb3fb58dd 100644 --- a/docs/vocs/docs/pages/cli/reth/db/static-file-header/block.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/static-file-header/block.mdx @@ -149,4 +149,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/static-file-header/path.mdx b/docs/vocs/docs/pages/cli/reth/db/static-file-header/path.mdx index 2c8dc766cb..404f711082 100644 --- a/docs/vocs/docs/pages/cli/reth/db/static-file-header/path.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/static-file-header/path.mdx @@ -140,4 +140,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/stats.mdx b/docs/vocs/docs/pages/cli/reth/db/stats.mdx index 5d985385e6..36ebe7938f 100644 --- a/docs/vocs/docs/pages/cli/reth/db/stats.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/stats.mdx @@ -152,4 +152,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/version.mdx b/docs/vocs/docs/pages/cli/reth/db/version.mdx index c87496d910..5e983356be 100644 --- a/docs/vocs/docs/pages/cli/reth/db/version.mdx +++ b/docs/vocs/docs/pages/cli/reth/db/version.mdx @@ -136,4 +136,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/download.mdx b/docs/vocs/docs/pages/cli/reth/download.mdx index ffdbccd669..785213182a 100644 --- a/docs/vocs/docs/pages/cli/reth/download.mdx +++ b/docs/vocs/docs/pages/cli/reth/download.mdx @@ -237,4 +237,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/dump-genesis.mdx b/docs/vocs/docs/pages/cli/reth/dump-genesis.mdx index 7aeaa8db49..905f2fbf4a 100644 --- a/docs/vocs/docs/pages/cli/reth/dump-genesis.mdx +++ b/docs/vocs/docs/pages/cli/reth/dump-genesis.mdx @@ -135,4 +135,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/export-era.mdx b/docs/vocs/docs/pages/cli/reth/export-era.mdx index bab9b07966..02ac02965d 100644 --- a/docs/vocs/docs/pages/cli/reth/export-era.mdx +++ b/docs/vocs/docs/pages/cli/reth/export-era.mdx @@ -243,4 +243,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/import-era.mdx b/docs/vocs/docs/pages/cli/reth/import-era.mdx index a535ce750a..ece330c642 100644 --- a/docs/vocs/docs/pages/cli/reth/import-era.mdx +++ b/docs/vocs/docs/pages/cli/reth/import-era.mdx @@ -238,4 +238,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/import.mdx b/docs/vocs/docs/pages/cli/reth/import.mdx index 8b57f761fe..25f96ac32a 100644 --- a/docs/vocs/docs/pages/cli/reth/import.mdx +++ b/docs/vocs/docs/pages/cli/reth/import.mdx @@ -239,4 +239,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/init-state.mdx b/docs/vocs/docs/pages/cli/reth/init-state.mdx index 80c3701988..9819039691 100644 --- a/docs/vocs/docs/pages/cli/reth/init-state.mdx +++ b/docs/vocs/docs/pages/cli/reth/init-state.mdx @@ -259,4 +259,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/init.mdx b/docs/vocs/docs/pages/cli/reth/init.mdx index 61f3756af0..d1465158f5 100644 --- a/docs/vocs/docs/pages/cli/reth/init.mdx +++ b/docs/vocs/docs/pages/cli/reth/init.mdx @@ -227,4 +227,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index c39d1b1f43..76a3bc2f9c 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -1116,4 +1116,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/p2p.mdx b/docs/vocs/docs/pages/cli/reth/p2p.mdx index 7b37fdfdaa..9ceba951c1 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p.mdx @@ -133,4 +133,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/p2p/body.mdx b/docs/vocs/docs/pages/cli/reth/p2p/body.mdx index 8eca4be487..04c32973ae 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/body.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/body.mdx @@ -372,4 +372,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/p2p/bootnode.mdx b/docs/vocs/docs/pages/cli/reth/p2p/bootnode.mdx index 324b01daac..79335dfd92 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/bootnode.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/bootnode.mdx @@ -144,4 +144,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/p2p/header.mdx b/docs/vocs/docs/pages/cli/reth/p2p/header.mdx index ada244eca4..10c1004fce 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/header.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/header.mdx @@ -372,4 +372,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/p2p/rlpx.mdx b/docs/vocs/docs/pages/cli/reth/p2p/rlpx.mdx index a8ac7fbd0d..ee5d70b5fa 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/rlpx.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/rlpx.mdx @@ -130,4 +130,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/p2p/rlpx/ping.mdx b/docs/vocs/docs/pages/cli/reth/p2p/rlpx/ping.mdx index 2d13663029..3bf3599145 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/rlpx/ping.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/rlpx/ping.mdx @@ -130,4 +130,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/prune.mdx b/docs/vocs/docs/pages/cli/reth/prune.mdx index 1d18691922..92aa365b8f 100644 --- a/docs/vocs/docs/pages/cli/reth/prune.mdx +++ b/docs/vocs/docs/pages/cli/reth/prune.mdx @@ -227,4 +227,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/re-execute.mdx b/docs/vocs/docs/pages/cli/reth/re-execute.mdx index c3ef601ccd..2743f4b491 100644 --- a/docs/vocs/docs/pages/cli/reth/re-execute.mdx +++ b/docs/vocs/docs/pages/cli/reth/re-execute.mdx @@ -243,4 +243,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage.mdx b/docs/vocs/docs/pages/cli/reth/stage.mdx index 006c6c7434..67ca5866f4 100644 --- a/docs/vocs/docs/pages/cli/reth/stage.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage.mdx @@ -133,4 +133,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/drop.mdx b/docs/vocs/docs/pages/cli/reth/stage/drop.mdx index 76e6383d10..35a051f859 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/drop.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/drop.mdx @@ -242,4 +242,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/dump.mdx b/docs/vocs/docs/pages/cli/reth/stage/dump.mdx index 4089b1510d..0ab8ab2d33 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/dump.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/dump.mdx @@ -234,4 +234,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/dump/account-hashing.mdx b/docs/vocs/docs/pages/cli/reth/stage/dump/account-hashing.mdx index 70fad94ea3..80348194ce 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/dump/account-hashing.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/dump/account-hashing.mdx @@ -148,4 +148,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/dump/execution.mdx b/docs/vocs/docs/pages/cli/reth/stage/dump/execution.mdx index bed5d33329..a48c7e65db 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/dump/execution.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/dump/execution.mdx @@ -148,4 +148,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/dump/merkle.mdx b/docs/vocs/docs/pages/cli/reth/stage/dump/merkle.mdx index 3bada103c8..203751e12f 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/dump/merkle.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/dump/merkle.mdx @@ -148,4 +148,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/dump/storage-hashing.mdx b/docs/vocs/docs/pages/cli/reth/stage/dump/storage-hashing.mdx index 723a54e927..1431798792 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/dump/storage-hashing.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/dump/storage-hashing.mdx @@ -148,4 +148,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/run.mdx b/docs/vocs/docs/pages/cli/reth/stage/run.mdx index 2c391268ce..a10aac3c02 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/run.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/run.mdx @@ -484,4 +484,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx b/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx index c1447e4cb0..1460b4c81d 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx @@ -235,4 +235,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/unwind/num-blocks.mdx b/docs/vocs/docs/pages/cli/reth/stage/unwind/num-blocks.mdx index b04e1920b7..adad84db51 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/unwind/num-blocks.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/unwind/num-blocks.mdx @@ -140,4 +140,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/unwind/to-block.mdx b/docs/vocs/docs/pages/cli/reth/stage/unwind/to-block.mdx index 2c22f8127c..133c0b0124 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/unwind/to-block.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/unwind/to-block.mdx @@ -140,4 +140,13 @@ Tracing: Defaults to TRACE if not specified. [default: debug] + + --tracing-otlp.sample-ratio + Trace sampling ratio to control the percentage of traces to export. + + Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling + + Example: --tracing-otlp.sample-ratio=0.0. + + [env: OTEL_TRACES_SAMPLER_ARG=] ``` \ No newline at end of file From a83ac8cc636389401689f16f0e5b4eb821bd047e Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Sun, 23 Nov 2025 16:05:48 +0400 Subject: [PATCH 17/23] refactor(e2e): relax bounds (#19913) --- crates/e2e-test-utils/src/lib.rs | 18 ++------- crates/e2e-test-utils/src/setup_builder.rs | 9 +---- crates/e2e-test-utils/src/testsuite/mod.rs | 8 +--- crates/e2e-test-utils/src/testsuite/setup.rs | 42 ++++---------------- 4 files changed, 13 insertions(+), 64 deletions(-) diff --git a/crates/e2e-test-utils/src/lib.rs b/crates/e2e-test-utils/src/lib.rs index 57d03f70fa..f5a2d1b030 100644 --- a/crates/e2e-test-utils/src/lib.rs +++ b/crates/e2e-test-utils/src/lib.rs @@ -3,13 +3,12 @@ use node::NodeTestContext; use reth_chainspec::ChainSpec; use reth_db::{test_utils::TempDatabase, DatabaseEnv}; -use reth_engine_local::LocalPayloadAttributesBuilder; use reth_network_api::test_utils::PeersHandleProvider; use reth_node_builder::{ components::NodeComponentsBuilder, rpc::{EngineValidatorAddOn, RethRpcAddOns}, FullNodeTypesAdapter, Node, NodeAdapter, NodeComponents, NodeTypes, NodeTypesWithDBAdapter, - PayloadAttributesBuilder, PayloadTypes, + PayloadTypes, }; use reth_provider::providers::{BlockchainProvider, NodeTypesForProvider}; use reth_tasks::TaskManager; @@ -54,8 +53,6 @@ pub async fn setup( ) -> eyre::Result<(Vec>, TaskManager, Wallet)> where N: NodeBuilderHelper, - LocalPayloadAttributesBuilder: - PayloadAttributesBuilder<<::Payload as PayloadTypes>::PayloadAttributes>, { E2ETestSetupBuilder::new(num_nodes, chain_spec, attributes_generator) .with_node_config_modifier(move |config| config.set_dev(is_dev)) @@ -77,8 +74,6 @@ pub async fn setup_engine( )> where N: NodeBuilderHelper, - LocalPayloadAttributesBuilder: - PayloadAttributesBuilder<::PayloadAttributes>, { setup_engine_with_connection::( num_nodes, @@ -106,8 +101,6 @@ pub async fn setup_engine_with_connection( )> where N: NodeBuilderHelper, - LocalPayloadAttributesBuilder: - PayloadAttributesBuilder<::PayloadAttributes>, { E2ETestSetupBuilder::new(num_nodes, chain_spec, attributes_generator) .with_tree_config_modifier(move |_| tree_config.clone()) @@ -160,13 +153,10 @@ where >, ChainSpec: From + Clone, >, - LocalPayloadAttributesBuilder: - PayloadAttributesBuilder<::PayloadAttributes>, { } -impl NodeBuilderHelper for T -where +impl NodeBuilderHelper for T where Self: Default + NodeTypesForProvider< Payload: PayloadTypes< @@ -187,8 +177,6 @@ where Adapter>>, >, ChainSpec: From + Clone, - >, - LocalPayloadAttributesBuilder: - PayloadAttributesBuilder<::PayloadAttributes>, + > { } diff --git a/crates/e2e-test-utils/src/setup_builder.rs b/crates/e2e-test-utils/src/setup_builder.rs index c1eabece9f..4b9cf8a95a 100644 --- a/crates/e2e-test-utils/src/setup_builder.rs +++ b/crates/e2e-test-utils/src/setup_builder.rs @@ -6,10 +6,9 @@ use crate::{node::NodeTestContext, wallet::Wallet, NodeBuilderHelper, NodeHelperType, TmpDB}; use futures_util::future::TryJoinAll; use reth_chainspec::EthChainSpec; -use reth_engine_local::LocalPayloadAttributesBuilder; use reth_node_builder::{ EngineNodeLauncher, NodeBuilder, NodeConfig, NodeHandle, NodeTypes, NodeTypesWithDBAdapter, - PayloadAttributesBuilder, PayloadTypes, + PayloadTypes, }; use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs}; use reth_provider::providers::BlockchainProvider; @@ -38,8 +37,6 @@ where + Sync + Copy + 'static, - LocalPayloadAttributesBuilder: - PayloadAttributesBuilder<::PayloadAttributes>, { num_nodes: usize, chain_spec: Arc, @@ -57,8 +54,6 @@ where + Sync + Copy + 'static, - LocalPayloadAttributesBuilder: - PayloadAttributesBuilder<::PayloadAttributes>, { /// Creates a new builder with the required parameters. pub fn new(num_nodes: usize, chain_spec: Arc, attributes_generator: F) -> Self { @@ -202,8 +197,6 @@ where + Sync + Copy + 'static, - LocalPayloadAttributesBuilder: - PayloadAttributesBuilder<::PayloadAttributes>, { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("E2ETestSetupBuilder") diff --git a/crates/e2e-test-utils/src/testsuite/mod.rs b/crates/e2e-test-utils/src/testsuite/mod.rs index 79e906ef59..17d04f64ea 100644 --- a/crates/e2e-test-utils/src/testsuite/mod.rs +++ b/crates/e2e-test-utils/src/testsuite/mod.rs @@ -2,13 +2,12 @@ use crate::{ testsuite::actions::{Action, ActionBox}, - NodeBuilderHelper, PayloadAttributesBuilder, + NodeBuilderHelper, }; use alloy_primitives::B256; use eyre::Result; use jsonrpsee::http_client::HttpClient; -use reth_engine_local::LocalPayloadAttributesBuilder; -use reth_node_api::{EngineTypes, NodeTypes, PayloadTypes}; +use reth_node_api::{EngineTypes, PayloadTypes}; use reth_payload_builder::PayloadId; use std::{collections::HashMap, marker::PhantomData}; pub mod actions; @@ -349,9 +348,6 @@ where pub async fn run(mut self) -> Result<()> where N: NodeBuilderHelper, - LocalPayloadAttributesBuilder: PayloadAttributesBuilder< - <::Payload as PayloadTypes>::PayloadAttributes, - >, { let mut setup = self.setup.take(); diff --git a/crates/e2e-test-utils/src/testsuite/setup.rs b/crates/e2e-test-utils/src/testsuite/setup.rs index 94f661753b..e7a57e7075 100644 --- a/crates/e2e-test-utils/src/testsuite/setup.rs +++ b/crates/e2e-test-utils/src/testsuite/setup.rs @@ -1,15 +1,11 @@ //! Test setup utilities for configuring the initial state. -use crate::{ - setup_engine_with_connection, testsuite::Environment, NodeBuilderHelper, - PayloadAttributesBuilder, -}; +use crate::{setup_engine_with_connection, testsuite::Environment, NodeBuilderHelper}; use alloy_eips::BlockNumberOrTag; use alloy_primitives::B256; use alloy_rpc_types_engine::{ForkchoiceState, PayloadAttributes}; use eyre::{eyre, Result}; use reth_chainspec::ChainSpec; -use reth_engine_local::LocalPayloadAttributesBuilder; use reth_ethereum_primitives::Block; use reth_network_p2p::sync::{NetworkSyncUpdater, SyncState}; use reth_node_api::{EngineTypes, NodeTypes, PayloadTypes, TreeConfig}; @@ -138,28 +134,19 @@ where ) -> Result<()> where N: NodeBuilderHelper, - LocalPayloadAttributesBuilder: PayloadAttributesBuilder< - <::Payload as PayloadTypes>::PayloadAttributes, - >, { // Note: this future is quite large so we box it - Box::pin(self.apply_with_import_::(env, rlp_path)).await + Box::pin(self.apply_with_import_(env, rlp_path)).await } /// Apply setup using pre-imported chain data from RLP file - async fn apply_with_import_( + async fn apply_with_import_( &mut self, env: &mut Environment, rlp_path: &Path, - ) -> Result<()> - where - N: NodeBuilderHelper, - LocalPayloadAttributesBuilder: PayloadAttributesBuilder< - <::Payload as PayloadTypes>::PayloadAttributes, - >, - { + ) -> Result<()> { // Create nodes with imported chain data - let import_result = self.create_nodes_with_import::(rlp_path).await?; + let import_result = self.create_nodes_with_import(rlp_path).await?; // Extract node clients let mut node_clients = Vec::new(); @@ -186,9 +173,6 @@ where pub async fn apply(&mut self, env: &mut Environment) -> Result<()> where N: NodeBuilderHelper, - LocalPayloadAttributesBuilder: PayloadAttributesBuilder< - <::Payload as PayloadTypes>::PayloadAttributes, - >, { // Note: this future is quite large so we box it Box::pin(self.apply_::(env)).await @@ -198,9 +182,6 @@ where async fn apply_(&mut self, env: &mut Environment) -> Result<()> where N: NodeBuilderHelper, - LocalPayloadAttributesBuilder: PayloadAttributesBuilder< - <::Payload as PayloadTypes>::PayloadAttributes, - >, { // If import_rlp_path is set, use apply_with_import instead if let Some(rlp_path) = self.import_rlp_path.take() { @@ -259,16 +240,10 @@ where /// Note: Currently this only supports `EthereumNode` due to the import process /// being Ethereum-specific. The generic parameter N is kept for consistency /// with other methods but is not used. - async fn create_nodes_with_import( + async fn create_nodes_with_import( &self, rlp_path: &Path, - ) -> Result - where - N: NodeBuilderHelper, - LocalPayloadAttributesBuilder: PayloadAttributesBuilder< - <::Payload as PayloadTypes>::PayloadAttributes, - >, - { + ) -> Result { let chain_spec = self.chain_spec.clone().ok_or_else(|| eyre!("Chain specification is required"))?; @@ -301,9 +276,6 @@ where + use where N: NodeBuilderHelper, - LocalPayloadAttributesBuilder: PayloadAttributesBuilder< - <::Payload as PayloadTypes>::PayloadAttributes, - >, { move |timestamp| { let attributes = PayloadAttributes { From 32f0a7446281564799c5966d91b555db56dac487 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 23 Nov 2025 11:50:58 +0000 Subject: [PATCH 18/23] chore(deps): weekly `cargo update` (#19917) Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com> --- Cargo.lock | 91 +++++++++++++++++++++++++++--------------------------- 1 file changed, 46 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 448e638b64..af56449cae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,9 +97,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "alloy-chains" -version = "0.2.18" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfaa9ea039a6f9304b4a593d780b1f23e1ae183acdee938b11b38795acacc9f1" +checksum = "4bc32535569185cbcb6ad5fa64d989a47bccb9a08e27284b1f2a3ccf16e6d010" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -426,8 +426,8 @@ dependencies = [ "derive_more", "foldhash 0.2.0", "getrandom 0.3.4", - "hashbrown 0.16.0", - "indexmap 2.12.0", + "hashbrown 0.16.1", + "indexmap 2.12.1", "itoa", "k256", "keccak-asm", @@ -790,7 +790,7 @@ dependencies = [ "alloy-sol-macro-input", "const-hex", "heck", - "indexmap 2.12.0", + "indexmap 2.12.1", "proc-macro-error2", "proc-macro2", "quote", @@ -1719,7 +1719,7 @@ dependencies = [ "boa_interner", "boa_macros", "boa_string", - "indexmap 2.12.0", + "indexmap 2.12.1", "num-bigint", "rustc-hash", ] @@ -1749,9 +1749,9 @@ dependencies = [ "futures-channel", "futures-concurrency", "futures-lite 2.6.1", - "hashbrown 0.16.0", + "hashbrown 0.16.1", "icu_normalizer", - "indexmap 2.12.0", + "indexmap 2.12.1", "intrusive-collections", "itertools 0.14.0", "num-bigint", @@ -1784,7 +1784,7 @@ checksum = "f1179f690cbfcbe5364cceee5f1cb577265bb6f07b0be6f210aabe270adcf9da" dependencies = [ "boa_macros", "boa_string", - "hashbrown 0.16.0", + "hashbrown 0.16.1", "thin-vec", ] @@ -1796,8 +1796,8 @@ checksum = "9626505d33dc63d349662437297df1d3afd9d5fc4a2b3ad34e5e1ce879a78848" dependencies = [ "boa_gc", "boa_macros", - "hashbrown 0.16.0", - "indexmap 2.12.0", + "hashbrown 0.16.1", + "indexmap 2.12.1", "once_cell", "phf", "rustc-hash", @@ -2161,9 +2161,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.51" +version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c26d721170e0295f191a69bd9a1f93efcdb0aff38684b61ab5750468972e5f5" +checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" dependencies = [ "clap_builder", "clap_derive", @@ -2171,9 +2171,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.51" +version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75835f0c7bf681bfd05abe44e965760fea999a5286c6eb2d59883634fd02011a" +checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" dependencies = [ "anstream", "anstyle", @@ -4363,7 +4363,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.12.0", + "indexmap 2.12.1", "slab", "tokio", "tokio-util", @@ -4421,14 +4421,15 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", "foldhash 0.2.0", "serde", + "serde_core", ] [[package]] @@ -4924,13 +4925,13 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "arbitrary", "equivalent", - "hashbrown 0.16.0", + "hashbrown 0.16.1", "serde", "serde_core", ] @@ -4982,9 +4983,9 @@ dependencies = [ [[package]] name = "insta" -version = "1.43.2" +version = "1.44.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fdb647ebde000f43b5b53f773c30cf9b0cb4300453208713fa38b2c70935a0" +checksum = "e8732d3774162a0851e3f2b150eb98f31a9885dd75985099421d393385a01dfd" dependencies = [ "console", "once_cell", @@ -5702,7 +5703,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034" dependencies = [ "base64 0.22.1", - "indexmap 2.12.0", + "indexmap 2.12.1", "metrics", "metrics-util", "quanta", @@ -5734,7 +5735,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.15.5", - "indexmap 2.12.0", + "indexmap 2.12.1", "metrics", "ordered-float", "quanta", @@ -6161,9 +6162,9 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "op-alloy" -version = "0.22.1" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5da980f0756a505bf7f2ab9f0be886c8b3213b9c23a19fff6c1c6afa53803c3a" +checksum = "c3b13412d297c1f9341f678b763750b120a73ffe998fa54a94d6eda98449e7ca" dependencies = [ "op-alloy-consensus", "op-alloy-network", @@ -6216,9 +6217,9 @@ dependencies = [ [[package]] name = "op-alloy-provider" -version = "0.22.1" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0594687a750dff379f16219c7441bfa8979f11aeda440981e32e65a9e28c3c6" +checksum = "a71456699aa256dc20119736422ad9a44da8b9585036117afb936778122093b9" dependencies = [ "alloy-network", "alloy-primitives", @@ -6532,9 +6533,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.3" +version = "2.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989e7521a040efde50c3ab6bbadafbe15ab6dc042686926be59ac35d74607df4" +checksum = "cbcfd20a6d4eeba40179f05735784ad32bdaef05ce8e8af05f180d45bb3e7e22" dependencies = [ "memchr", "ucd-trie", @@ -7300,7 +7301,7 @@ version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2057b2325e68a893284d1538021ab90279adac1139957ca2a74426c6f118fb48" dependencies = [ - "hashbrown 0.16.0", + "hashbrown 0.16.1", "memchr", ] @@ -7355,9 +7356,9 @@ dependencies = [ [[package]] name = "resolv-conf" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b3789b30bd25ba102de4beabd95d21ac45b69b1be7d14522bab988c526d6799" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "reth" @@ -11816,7 +11817,7 @@ version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.12.1", "itoa", "memchr", "ryu", @@ -11866,7 +11867,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.12.0", + "indexmap 2.12.1", "schemars 0.9.0", "schemars 1.1.0", "serde_core", @@ -11986,9 +11987,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.6" +version = "1.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" dependencies = [ "libc", ] @@ -12735,7 +12736,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.12.1", "serde", "serde_spanned", "toml_datetime 0.6.11", @@ -12749,7 +12750,7 @@ version = "0.23.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.12.1", "toml_datetime 0.7.3", "toml_parser", "winnow", @@ -12816,7 +12817,7 @@ dependencies = [ "futures-core", "futures-util", "hdrhistogram", - "indexmap 2.12.0", + "indexmap 2.12.1", "pin-project-lite", "slab", "sync_wrapper", @@ -14240,18 +14241,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.27" +version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +checksum = "43fa6694ed34d6e57407afbccdeecfa268c470a7d2a5b0cf49ce9fcc345afb90" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.27" +version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +checksum = "c640b22cd9817fae95be82f0d2f90b11f7605f6c319d16705c459b27ac2cbc26" dependencies = [ "proc-macro2", "quote", From 17021070286da608acd3d2268f5ac1d557b3408a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sun, 23 Nov 2025 14:07:59 +0100 Subject: [PATCH 19/23] feat: make txpool notify fns pub (#19918) --- crates/transaction-pool/src/pool/mod.rs | 27 ++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 50d959a475..74622b7189 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -637,7 +637,14 @@ where } /// Notify all listeners about a new pending transaction. - fn on_new_pending_transaction(&self, pending: &AddedPendingTransaction) { + /// + /// See also [`Self::add_pending_listener`] + /// + /// CAUTION: This function is only intended to be used manually in order to use this type's + /// pending transaction receivers when manually implementing the + /// [`TransactionPool`](crate::TransactionPool) trait for a custom pool implementation + /// [`TransactionPool::pending_transactions_listener_for`](crate::TransactionPool). + pub fn on_new_pending_transaction(&self, pending: &AddedPendingTransaction) { let propagate_allowed = pending.is_propagate_allowed(); let mut transaction_listeners = self.pending_transaction_listener.lock(); @@ -654,7 +661,14 @@ where } /// Notify all listeners about a newly inserted pending transaction. - fn on_new_transaction(&self, event: NewTransactionEvent) { + /// + /// See also [`Self::add_new_transaction_listener`] + /// + /// CAUTION: This function is only intended to be used manually in order to use this type's + /// transaction receivers when manually implementing the + /// [`TransactionPool`](crate::TransactionPool) trait for a custom pool implementation + /// [`TransactionPool::new_transactions_listener_for`](crate::TransactionPool). + pub fn on_new_transaction(&self, event: NewTransactionEvent) { let mut transaction_listeners = self.transaction_listener.lock(); transaction_listeners.retain_mut(|listener| { if listener.kind.is_propagate_only() && !event.transaction.propagate { @@ -728,7 +742,14 @@ where } /// Fire events for the newly added transaction if there are any. - fn notify_event_listeners(&self, tx: &AddedTransaction) { + /// + /// See also [`Self::add_transaction_event_listener`]. + /// + /// CAUTION: This function is only intended to be used manually in order to use this type's + /// [`TransactionEvents`] receivers when manually implementing the + /// [`TransactionPool`](crate::TransactionPool) trait for a custom pool implementation + /// [`TransactionPool::transaction_event_listener`](crate::TransactionPool). + pub fn notify_event_listeners(&self, tx: &AddedTransaction) { let mut listener = self.event_listener.write(); if listener.is_empty() { // nothing to notify From ee63c7d6b49a209bee587178f2ada3162c891fbb Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Sun, 23 Nov 2025 23:06:10 +0400 Subject: [PATCH 20/23] refactor: simplify rpc state provider traits (#19920) --- crates/optimism/rpc/src/eth/mod.rs | 2 +- crates/rpc/rpc-eth-api/src/helpers/call.rs | 49 +++++------ crates/rpc/rpc-eth-api/src/helpers/trace.rs | 67 +++++---------- crates/rpc/rpc-eth-types/src/cache/db.rs | 90 ++++----------------- crates/rpc/rpc/src/debug.rs | 78 ++++++------------ crates/rpc/rpc/src/eth/bundle.rs | 6 +- crates/rpc/rpc/src/eth/helpers/trace.rs | 2 +- crates/rpc/rpc/src/eth/sim_bundle.rs | 5 +- crates/rpc/rpc/src/trace.rs | 15 +--- 9 files changed, 90 insertions(+), 224 deletions(-) diff --git a/crates/optimism/rpc/src/eth/mod.rs b/crates/optimism/rpc/src/eth/mod.rs index f9b6195133..b25aac10c6 100644 --- a/crates/optimism/rpc/src/eth/mod.rs +++ b/crates/optimism/rpc/src/eth/mod.rs @@ -332,7 +332,7 @@ impl Trace for OpEthApi where N: RpcNodeCore, OpEthApiError: FromEvmError, - Rpc: RpcConvert, + Rpc: RpcConvert, { } diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index bec79920ec..d79b18765d 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -28,12 +28,12 @@ use reth_primitives_traits::Recovered; use reth_revm::{database::StateProviderDatabase, db::State}; use reth_rpc_convert::{RpcConvert, RpcTxReq}; use reth_rpc_eth_types::{ - cache::db::{StateCacheDbRefMutWrapper, StateProviderTraitObjWrapper}, + cache::db::StateProviderTraitObjWrapper, error::FromEthApiError, simulate::{self, EthSimulateError}, EthApiError, StateCacheDb, }; -use reth_storage_api::{BlockIdReader, ProviderTx}; +use reth_storage_api::{BlockIdReader, ProviderTx, StateProvider}; use revm::{ context::Block, context_interface::{result::ResultAndState, Transaction}, @@ -89,10 +89,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA self.recovered_block(block).await?.ok_or(EthApiError::HeaderNotFound(block))?; let mut parent = base_block.sealed_header().clone(); - let this = self.clone(); - self.spawn_with_state_at_block(block, move |state| { - let mut db = - State::builder().with_database(StateProviderDatabase::new(state)).build(); + self.spawn_with_state_at_block(block, move |this, mut db| { let mut blocks: Vec>> = Vec::with_capacity(block_state_calls.len()); for block in block_state_calls { @@ -277,11 +274,8 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA replay_block_txs = false; } - let this = self.clone(); - self.spawn_with_state_at_block(at.into(), move |state| { + self.spawn_with_state_at_block(at, move |this, mut db| { let mut all_results = Vec::with_capacity(bundles.len()); - let mut db = - State::builder().with_database(StateProviderDatabase::new(state)).build(); if replay_block_txs { // only need to replay the transactions in the block if not all transactions are @@ -487,13 +481,11 @@ pub trait Call: ) -> impl Future> + Send where R: Send + 'static, - F: FnOnce(Self, StateProviderTraitObjWrapper<'_>) -> Result - + Send - + 'static, + F: FnOnce(Self, &dyn StateProvider) -> Result + Send + 'static, { self.spawn_blocking_io_fut(move |this| async move { let state = this.state_at_block_id(at).await?; - f(this, StateProviderTraitObjWrapper(&state)) + f(this, &state) }) } @@ -552,16 +544,20 @@ pub trait Call: /// Executes the closure with the state that corresponds to the given [`BlockId`] on a new task fn spawn_with_state_at_block( &self, - at: BlockId, + at: impl Into, f: F, ) -> impl Future> + Send where - F: FnOnce(StateProviderTraitObjWrapper<'_>) -> Result + Send + 'static, + F: FnOnce(Self, StateCacheDb) -> Result + Send + 'static, R: Send + 'static, { + let at = at.into(); self.spawn_blocking_io_fut(move |this| async move { let state = this.state_at_block_id(at).await?; - f(StateProviderTraitObjWrapper(&state)) + let db = State::builder() + .with_database(StateProviderDatabase::new(StateProviderTraitObjWrapper(state))) + .build(); + f(this, db) }) } @@ -590,7 +586,7 @@ pub trait Call: where Self: LoadPendingBlock, F: FnOnce( - StateCacheDbRefMutWrapper<'_, '_>, + &mut StateCacheDb, EvmEnvFor, TxEnvFor, ) -> Result @@ -600,17 +596,11 @@ pub trait Call: { async move { let (evm_env, at) = self.evm_env_at(at).await?; - let this = self.clone(); - self.spawn_blocking_io_fut(move |_| async move { - let state = this.state_at_block_id(at).await?; - let mut db = State::builder() - .with_database(StateProviderDatabase::new(StateProviderTraitObjWrapper(&state))) - .build(); - + self.spawn_with_state_at_block(at, move |this, mut db| { let (evm_env, tx_env) = this.prepare_call_env(evm_env, request, &mut db, overrides)?; - f(StateCacheDbRefMutWrapper(&mut db), evm_env, tx_env) + f(&mut db, evm_env, tx_env) }) .await } @@ -635,7 +625,7 @@ pub trait Call: F: FnOnce( TransactionInfo, ResultAndState>, - StateCacheDb<'_>, + StateCacheDb, ) -> Result + Send + 'static, @@ -654,10 +644,7 @@ pub trait Call: // block the transaction is included in let parent_block = block.parent_hash(); - let this = self.clone(); - self.spawn_with_state_at_block(parent_block.into(), move |state| { - let mut db = - State::builder().with_database(StateProviderDatabase::new(state)).build(); + self.spawn_with_state_at_block(parent_block, move |this, mut db| { let block_txs = block.transactions_recovered(); // replay all transactions prior to the targeted transaction diff --git a/crates/rpc/rpc-eth-api/src/helpers/trace.rs b/crates/rpc/rpc-eth-api/src/helpers/trace.rs index 30ba12165e..20440725a8 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/trace.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/trace.rs @@ -1,6 +1,6 @@ //! Loads a pending block from database. Helper trait for `eth_` call and trace RPC methods. -use super::{Call, LoadBlock, LoadPendingBlock, LoadState, LoadTransaction}; +use super::{Call, LoadBlock, LoadState, LoadTransaction}; use crate::FromEvmError; use alloy_consensus::{transaction::TxHashRef, BlockHeader}; use alloy_primitives::B256; @@ -14,17 +14,14 @@ use reth_evm::{ }; use reth_primitives_traits::{BlockBody, Recovered, RecoveredBlock}; use reth_revm::{database::StateProviderDatabase, db::State}; -use reth_rpc_eth_types::{ - cache::db::{StateCacheDb, StateCacheDbRefMutWrapper, StateProviderTraitObjWrapper}, - EthApiError, -}; +use reth_rpc_eth_types::{cache::db::StateCacheDb, EthApiError}; use reth_storage_api::{ProviderBlock, ProviderTx}; use revm::{context::Block, context_interface::result::ResultAndState, DatabaseCommit}; use revm_inspectors::tracing::{TracingInspector, TracingInspectorConfig}; use std::sync::Arc; /// Executes CPU heavy tasks. -pub trait Trace: LoadState> { +pub trait Trace: LoadState> + Call { /// Executes the [`TxEnvFor`] with [`reth_evm::EvmEnv`] against the given [Database] without /// committing state changes. fn inspect( @@ -58,7 +55,6 @@ pub trait Trace: LoadState> { f: F, ) -> impl Future> + Send where - Self: Call, R: Send + 'static, F: FnOnce( TracingInspector, @@ -91,19 +87,16 @@ pub trait Trace: LoadState> { f: F, ) -> impl Future> + Send where - Self: LoadPendingBlock + Call, F: FnOnce( TracingInspector, ResultAndState>, - StateCacheDb<'_>, + StateCacheDb, ) -> Result + Send + 'static, R: Send + 'static, { - let this = self.clone(); - self.spawn_with_state_at_block(at, move |state| { - let mut db = State::builder().with_database(StateProviderDatabase::new(state)).build(); + self.spawn_with_state_at_block(at, move |this, mut db| { let mut inspector = TracingInspector::new(config); let res = this.inspect(&mut db, evm_env, tx_env, &mut inspector)?; f(inspector, res, db) @@ -126,12 +119,12 @@ pub trait Trace: LoadState> { f: F, ) -> impl Future, Self::Error>> + Send where - Self: LoadPendingBlock + LoadTransaction + Call, + Self: LoadTransaction, F: FnOnce( TransactionInfo, TracingInspector, ResultAndState>, - StateCacheDb<'_>, + StateCacheDb, ) -> Result + Send + 'static, @@ -156,17 +149,16 @@ pub trait Trace: LoadState> { f: F, ) -> impl Future, Self::Error>> + Send where - Self: LoadPendingBlock + LoadTransaction + Call, + Self: LoadTransaction, F: FnOnce( TransactionInfo, Insp, ResultAndState>, - StateCacheDb<'_>, + StateCacheDb, ) -> Result + Send + 'static, - Insp: - for<'a, 'b> InspectorFor> + Send + 'static, + Insp: for<'a> InspectorFor + Send + 'static, R: Send + 'static, { async move { @@ -182,10 +174,7 @@ pub trait Trace: LoadState> { // block the transaction is included in let parent_block = block.parent_hash(); - let this = self.clone(); - self.spawn_with_state_at_block(parent_block.into(), move |state| { - let mut db = - State::builder().with_database(StateProviderDatabase::new(state)).build(); + self.spawn_with_state_at_block(parent_block, move |this, mut db| { let block_txs = block.transactions_recovered(); this.apply_pre_execution_changes(&block, &mut db, &evm_env)?; @@ -194,12 +183,7 @@ pub trait Trace: LoadState> { this.replay_transactions_until(&mut db, evm_env.clone(), block_txs, *tx.tx_hash())?; let tx_env = this.evm_config().tx_env(tx); - let res = this.inspect( - StateCacheDbRefMutWrapper(&mut db), - evm_env, - tx_env, - &mut inspector, - )?; + let res = this.inspect(&mut db, evm_env, tx_env, &mut inspector)?; f(tx_info, inspector, res, db) }) .await @@ -228,7 +212,7 @@ pub trait Trace: LoadState> { TracingCtx< '_, Recovered<&ProviderTx>, - EvmFor, TracingInspector>, + EvmFor, >, ) -> Result + Send @@ -269,13 +253,13 @@ pub trait Trace: LoadState> { TracingCtx< '_, Recovered<&ProviderTx>, - EvmFor, Insp>, + EvmFor, >, ) -> Result + Send + 'static, Setup: FnMut() -> Insp + Send + 'static, - Insp: Clone + for<'a, 'b> InspectorFor>, + Insp: Clone + for<'a> InspectorFor, R: Send + 'static, { async move { @@ -296,21 +280,14 @@ pub trait Trace: LoadState> { } // replay all transactions of the block - self.spawn_blocking_io_fut(move |this| async move { - // we need to get the state of the parent block because we're replaying this block - // on top of its parent block's state - let state_at = block.parent_hash(); + // we need to get the state of the parent block because we're replaying this block + // on top of its parent block's state + self.spawn_with_state_at_block(block.parent_hash(), move |this, mut db| { let block_hash = block.hash(); let block_number = evm_env.block_env.number().saturating_to(); let base_fee = evm_env.block_env.basefee(); - // now get the state - let state = this.state_at_block_id(state_at.into()).await?; - let mut db = State::builder() - .with_database(StateProviderDatabase::new(StateProviderTraitObjWrapper(&state))) - .build(); - this.apply_pre_execution_changes(&block, &mut db, &evm_env)?; // prepare transactions, we do everything upfront to reduce time spent with open @@ -328,7 +305,7 @@ pub trait Trace: LoadState> { let results = this .evm_config() .evm_factory() - .create_tracer(StateCacheDbRefMutWrapper(&mut db), evm_env, inspector_setup()) + .create_tracer(&mut db, evm_env, inspector_setup()) .try_trace_many(block.transactions_recovered().take(max_transactions), |ctx| { let tx_info = TransactionInfo { hash: Some(*ctx.tx.tx_hash()), @@ -375,7 +352,7 @@ pub trait Trace: LoadState> { TracingCtx< '_, Recovered<&ProviderTx>, - EvmFor, TracingInspector>, + EvmFor, >, ) -> Result + Send @@ -415,13 +392,13 @@ pub trait Trace: LoadState> { TracingCtx< '_, Recovered<&ProviderTx>, - EvmFor, Insp>, + EvmFor, >, ) -> Result + Send + 'static, Setup: FnMut() -> Insp + Send + 'static, - Insp: Clone + for<'a, 'b> InspectorFor>, + Insp: Clone + for<'a> InspectorFor, R: Send + 'static, { self.trace_block_until_with_inspector(block_id, block, None, insp_setup, f) diff --git a/crates/rpc/rpc-eth-types/src/cache/db.rs b/crates/rpc/rpc-eth-types/src/cache/db.rs index 8209af0fa5..09e1b3db3c 100644 --- a/crates/rpc/rpc-eth-types/src/cache/db.rs +++ b/crates/rpc/rpc-eth-types/src/cache/db.rs @@ -4,25 +4,24 @@ use alloy_primitives::{Address, B256, U256}; use reth_errors::ProviderResult; -use reth_revm::{database::StateProviderDatabase, DatabaseRef}; -use reth_storage_api::{BytecodeReader, HashedPostStateProvider, StateProvider}; +use reth_revm::database::StateProviderDatabase; +use reth_storage_api::{BytecodeReader, HashedPostStateProvider, StateProvider, StateProviderBox}; use reth_trie::{HashedStorage, MultiProofTargets}; -use revm::{ - database::{BundleState, State}, - primitives::HashMap, - state::{AccountInfo, Bytecode}, - Database, DatabaseCommit, -}; +use revm::database::{BundleState, State}; /// Helper alias type for the state's [`State`] -pub type StateCacheDb<'a> = State>>; +pub type StateCacheDb = State>; /// Hack to get around 'higher-ranked lifetime error', see /// +/// +/// Apparently, when dealing with our RPC code, compiler is struggling to prove lifetimes around +/// [`StateProvider`] trait objects. This type is a workaround which should help the compiler to +/// understand that there are no lifetimes involved. #[expect(missing_debug_implementations)] -pub struct StateProviderTraitObjWrapper<'a>(pub &'a dyn StateProvider); +pub struct StateProviderTraitObjWrapper(pub StateProviderBox); -impl reth_storage_api::StateRootProvider for StateProviderTraitObjWrapper<'_> { +impl reth_storage_api::StateRootProvider for StateProviderTraitObjWrapper { fn state_root( &self, hashed_state: reth_trie::HashedPostState, @@ -52,7 +51,7 @@ impl reth_storage_api::StateRootProvider for StateProviderTraitObjWrapper<'_> { } } -impl reth_storage_api::StorageRootProvider for StateProviderTraitObjWrapper<'_> { +impl reth_storage_api::StorageRootProvider for StateProviderTraitObjWrapper { fn storage_root( &self, address: Address, @@ -80,7 +79,7 @@ impl reth_storage_api::StorageRootProvider for StateProviderTraitObjWrapper<'_> } } -impl reth_storage_api::StateProofProvider for StateProviderTraitObjWrapper<'_> { +impl reth_storage_api::StateProofProvider for StateProviderTraitObjWrapper { fn proof( &self, input: reth_trie::TrieInput, @@ -107,7 +106,7 @@ impl reth_storage_api::StateProofProvider for StateProviderTraitObjWrapper<'_> { } } -impl reth_storage_api::AccountReader for StateProviderTraitObjWrapper<'_> { +impl reth_storage_api::AccountReader for StateProviderTraitObjWrapper { fn basic_account( &self, address: &Address, @@ -116,7 +115,7 @@ impl reth_storage_api::AccountReader for StateProviderTraitObjWrapper<'_> { } } -impl reth_storage_api::BlockHashReader for StateProviderTraitObjWrapper<'_> { +impl reth_storage_api::BlockHashReader for StateProviderTraitObjWrapper { fn block_hash( &self, block_number: alloy_primitives::BlockNumber, @@ -140,13 +139,13 @@ impl reth_storage_api::BlockHashReader for StateProviderTraitObjWrapper<'_> { } } -impl HashedPostStateProvider for StateProviderTraitObjWrapper<'_> { +impl HashedPostStateProvider for StateProviderTraitObjWrapper { fn hashed_post_state(&self, bundle_state: &BundleState) -> reth_trie::HashedPostState { self.0.hashed_post_state(bundle_state) } } -impl StateProvider for StateProviderTraitObjWrapper<'_> { +impl StateProvider for StateProviderTraitObjWrapper { fn storage( &self, account: Address, @@ -171,7 +170,7 @@ impl StateProvider for StateProviderTraitObjWrapper<'_> { } } -impl BytecodeReader for StateProviderTraitObjWrapper<'_> { +impl BytecodeReader for StateProviderTraitObjWrapper { fn bytecode_by_hash( &self, code_hash: &B256, @@ -179,58 +178,3 @@ impl BytecodeReader for StateProviderTraitObjWrapper<'_> { self.0.bytecode_by_hash(code_hash) } } - -/// Hack to get around 'higher-ranked lifetime error', see -/// -pub struct StateCacheDbRefMutWrapper<'a, 'b>(pub &'b mut StateCacheDb<'a>); - -impl<'a, 'b> core::fmt::Debug for StateCacheDbRefMutWrapper<'a, 'b> { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("StateCacheDbRefMutWrapper").finish_non_exhaustive() - } -} - -impl<'a> Database for StateCacheDbRefMutWrapper<'a, '_> { - type Error = as Database>::Error; - fn basic(&mut self, address: Address) -> Result, Self::Error> { - self.0.basic(address) - } - - fn code_by_hash(&mut self, code_hash: B256) -> Result { - self.0.code_by_hash(code_hash) - } - - fn storage(&mut self, address: Address, index: U256) -> Result { - self.0.storage(address, index) - } - - fn block_hash(&mut self, number: u64) -> Result { - self.0.block_hash(number) - } -} - -impl<'a> DatabaseRef for StateCacheDbRefMutWrapper<'a, '_> { - type Error = as Database>::Error; - - fn basic_ref(&self, address: Address) -> Result, Self::Error> { - self.0.basic_ref(address) - } - - fn code_by_hash_ref(&self, code_hash: B256) -> Result { - self.0.code_by_hash_ref(code_hash) - } - - fn storage_ref(&self, address: Address, index: U256) -> Result { - self.0.storage_ref(address, index) - } - - fn block_hash_ref(&self, number: u64) -> Result { - self.0.block_hash_ref(number) - } -} - -impl DatabaseCommit for StateCacheDbRefMutWrapper<'_, '_> { - fn commit(&mut self, changes: HashMap) { - self.0.commit(changes) - } -} diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 8ecfdb2900..b74c686dac 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -22,7 +22,7 @@ use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks}; use reth_errors::RethError; use reth_evm::{execute::Executor, ConfigureEvm, EvmEnvFor, TxEnvFor}; use reth_primitives_traits::{Block as _, BlockBody, ReceiptWithBloom, RecoveredBlock}; -use reth_revm::{database::StateProviderDatabase, db::State, witness::ExecutionWitnessRecord}; +use reth_revm::{db::State, witness::ExecutionWitnessRecord}; use reth_rpc_api::DebugApiServer; use reth_rpc_convert::RpcTxReq; use reth_rpc_eth_api::{ @@ -91,22 +91,20 @@ where evm_env: EvmEnvFor, opts: GethDebugTracingOptions, ) -> Result, Eth::Error> { - // replay all transactions of the block let this = self.clone(); + // replay all transactions of the block self.eth_api() - .spawn_with_state_at_block(block.parent_hash().into(), move |state| { + .spawn_with_state_at_block(block.parent_hash(), move |eth_api, mut db| { let mut results = Vec::with_capacity(block.body().transactions().len()); - let mut db = - State::builder().with_database(StateProviderDatabase::new(state)).build(); - this.eth_api().apply_pre_execution_changes(&block, &mut db, &evm_env)?; + eth_api.apply_pre_execution_changes(&block, &mut db, &evm_env)?; let mut transactions = block.transactions_recovered().enumerate().peekable(); let mut inspector = None; while let Some((index, tx)) = transactions.next() { let tx_hash = *tx.tx_hash(); - let tx_env = this.eth_api().evm_config().tx_env(tx); + let tx_env = eth_api.evm_config().tx_env(tx); let (result, state_changes) = this.trace_transaction( &opts, @@ -221,26 +219,23 @@ where let this = self.clone(); self.eth_api() - .spawn_with_state_at_block(state_at, move |state| { + .spawn_with_state_at_block(state_at, move |eth_api, mut db| { let block_txs = block.transactions_recovered(); // configure env for the target transaction let tx = transaction.into_recovered(); - let mut db = - State::builder().with_database(StateProviderDatabase::new(state)).build(); - - this.eth_api().apply_pre_execution_changes(&block, &mut db, &evm_env)?; + eth_api.apply_pre_execution_changes(&block, &mut db, &evm_env)?; // replay all transactions prior to the targeted transaction - let index = this.eth_api().replay_transactions_until( + let index = eth_api.replay_transactions_until( &mut db, evm_env.clone(), block_txs, *tx.tx_hash(), )?; - let tx_env = this.eth_api().evm_config().tx_env(&tx); + let tx_env = eth_api.evm_config().tx_env(&tx); this.trace_transaction( &opts, @@ -343,10 +338,6 @@ where let frame = self .eth_api() .spawn_with_call_at(call, at, overrides, move |db, evm_env, tx_env| { - // wrapper is hack to get around 'higher-ranked lifetime error', - // see - let db = db.0; - let gas_limit = tx_env.gas_limit(); let res = this.eth_api().inspect( &mut *db, @@ -377,10 +368,6 @@ where .inner .eth_api .spawn_with_call_at(call, at, overrides, move |db, evm_env, tx_env| { - // wrapper is hack to get around 'higher-ranked lifetime error', see - // - let db = db.0; - let tx_info = TransactionInfo { block_number: Some(evm_env.block_env.number().saturating_to()), base_fee: Some(evm_env.block_env.basefee()), @@ -448,10 +435,6 @@ where let res = self .eth_api() .spawn_with_call_at(call, at, overrides, move |db, evm_env, tx_env| { - // wrapper is hack to get around 'higher-ranked lifetime error', see - // - let db = db.0; - let mut inspector = revm_inspectors::tracing::js::JsInspector::new(code, config) .map_err(Eth::Error::from_eth_err)?; @@ -531,27 +514,24 @@ where let (evm_env, _) = self.eth_api().evm_env_at(block.hash().into()).await?; // execute after the parent block, replaying `tx_index` transactions - let state_at = block.parent_hash().into(); + let state_at = block.parent_hash(); let this = self.clone(); self.eth_api() - .spawn_with_state_at_block(state_at, move |state| { - let mut db = - State::builder().with_database(StateProviderDatabase::new(state)).build(); - + .spawn_with_state_at_block(state_at, move |eth_api, mut db| { // 1. apply pre-execution changes - this.eth_api().apply_pre_execution_changes(&block, &mut db, &evm_env)?; + eth_api.apply_pre_execution_changes(&block, &mut db, &evm_env)?; // 2. replay the required number of transactions for tx in block.transactions_recovered().take(tx_index) { - let tx_env = this.eth_api().evm_config().tx_env(tx); - let res = this.eth_api().transact(&mut db, evm_env.clone(), tx_env)?; + let tx_env = eth_api.evm_config().tx_env(tx); + let res = eth_api.transact(&mut db, evm_env.clone(), tx_env)?; db.commit(res.state); } // 3. now execute the trace call on this state let (call_evm_env, call_tx_env) = - this.eth_api().prepare_call_env(evm_env, call, &mut db, overrides)?; + eth_api.prepare_call_env(evm_env, call, &mut db, overrides)?; // Execute the trace call using trace_transaction let (trace, _) = this.trace_transaction( @@ -613,11 +593,9 @@ where let this = self.clone(); self.eth_api() - .spawn_with_state_at_block(at.into(), move |state| { + .spawn_with_state_at_block(at, move |eth_api, mut db| { // the outer vec for the bundles let mut all_bundles = Vec::with_capacity(bundles.len()); - let mut db = - State::builder().with_database(StateProviderDatabase::new(state)).build(); if replay_block_txs { // only need to replay the transactions in the block if not all transactions are @@ -626,8 +604,8 @@ where // Execute all transactions until index for tx in transactions { - let tx_env = this.eth_api().evm_config().tx_env(tx); - let res = this.eth_api().transact(&mut db, evm_env.clone(), tx_env)?; + let tx_env = eth_api.evm_config().tx_env(tx); + let res = eth_api.transact(&mut db, evm_env.clone(), tx_env)?; db.commit(res.state); } } @@ -647,12 +625,8 @@ where let state_overrides = state_overrides.take(); let overrides = EvmOverrides::new(state_overrides, block_overrides.clone()); - let (evm_env, tx_env) = this.eth_api().prepare_call_env( - evm_env.clone(), - tx, - &mut db, - overrides, - )?; + let (evm_env, tx_env) = + eth_api.prepare_call_env(evm_env.clone(), tx, &mut db, overrides)?; let (trace, state) = this.trace_transaction( &tracing_options, @@ -722,14 +696,12 @@ where &self, block: Arc>>, ) -> Result { - let this = self.clone(); let block_number = block.header().number(); let (mut exec_witness, lowest_block_number) = self .eth_api() - .spawn_with_state_at_block(block.parent_hash().into(), move |state_provider| { - let db = StateProviderDatabase::new(&state_provider); - let block_executor = this.eth_api().evm_config().executor(db); + .spawn_with_state_at_block(block.parent_hash(), move |eth_api, mut db| { + let block_executor = eth_api.evm_config().executor(&mut db); let mut witness_record = ExecutionWitnessRecord::default(); @@ -742,7 +714,9 @@ where let ExecutionWitnessRecord { hashed_state, codes, keys, lowest_block_number } = witness_record; - let state = state_provider + let state = db + .database + .0 .witness(Default::default(), hashed_state) .map_err(EthApiError::from)?; Ok(( @@ -812,7 +786,7 @@ where opts: &GethDebugTracingOptions, evm_env: EvmEnvFor, tx_env: TxEnvFor, - db: &mut StateCacheDb<'_>, + db: &mut StateCacheDb, transaction_context: Option, fused_inspector: &mut Option, ) -> Result<(GethTrace, EvmState), Eth::Error> { diff --git a/crates/rpc/rpc/src/eth/bundle.rs b/crates/rpc/rpc/src/eth/bundle.rs index d49b5486d3..a3ddeda581 100644 --- a/crates/rpc/rpc/src/eth/bundle.rs +++ b/crates/rpc/rpc/src/eth/bundle.rs @@ -8,7 +8,6 @@ use alloy_rpc_types_mev::{EthCallBundle, EthCallBundleResponse, EthCallBundleTra use jsonrpsee::core::RpcResult; use reth_chainspec::{ChainSpecProvider, EthChainSpec}; use reth_evm::{ConfigureEvm, Evm}; -use reth_revm::{database::StateProviderDatabase, State}; use reth_rpc_eth_api::{ helpers::{Call, EthTransactions, LoadPendingBlock}, EthCallBundleApiServer, FromEthApiError, FromEvmError, @@ -144,13 +143,10 @@ where // use the block number of the request evm_env.block_env.inner_mut().number = U256::from(block_number); - let eth_api = self.eth_api().clone(); - self.eth_api() - .spawn_with_state_at_block(at, move |state| { + .spawn_with_state_at_block(at, move |eth_api, db| { let coinbase = evm_env.block_env.beneficiary(); let basefee = evm_env.block_env.basefee(); - let db = State::builder().with_database(StateProviderDatabase::new(state)).build(); let initial_coinbase = db .basic_ref(coinbase) diff --git a/crates/rpc/rpc/src/eth/helpers/trace.rs b/crates/rpc/rpc/src/eth/helpers/trace.rs index 3e00f2df0c..55b1604eea 100644 --- a/crates/rpc/rpc/src/eth/helpers/trace.rs +++ b/crates/rpc/rpc/src/eth/helpers/trace.rs @@ -10,6 +10,6 @@ impl Trace for EthApi where N: RpcNodeCore, EthApiError: FromEvmError, - Rpc: RpcConvert, + Rpc: RpcConvert, { } diff --git a/crates/rpc/rpc/src/eth/sim_bundle.rs b/crates/rpc/rpc/src/eth/sim_bundle.rs index fa3fd46e45..523d2bef5e 100644 --- a/crates/rpc/rpc/src/eth/sim_bundle.rs +++ b/crates/rpc/rpc/src/eth/sim_bundle.rs @@ -12,7 +12,6 @@ use alloy_rpc_types_mev::{ use jsonrpsee::core::RpcResult; use reth_evm::{ConfigureEvm, Evm}; use reth_primitives_traits::Recovered; -use reth_revm::{database::StateProviderDatabase, State}; use reth_rpc_api::MevSimApiServer; use reth_rpc_eth_api::{ helpers::{block::LoadBlock, Call, EthTransactions}, @@ -241,13 +240,11 @@ where let sim_response = self .inner .eth_api - .spawn_with_state_at_block(current_block_id, move |state| { + .spawn_with_state_at_block(current_block_id, move |_, mut db| { // Setup environment let current_block_number = current_block.number(); let coinbase = evm_env.block_env.beneficiary(); let basefee = evm_env.block_env.basefee(); - let mut db = - State::builder().with_database(StateProviderDatabase::new(state)).build(); // apply overrides apply_block_overrides(block_overrides, &mut db, evm_env.block_env.inner_mut()); diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index e1e6bc2654..5eb9a0f73c 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -20,7 +20,6 @@ use jsonrpsee::core::RpcResult; use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardfork, MAINNET, SEPOLIA}; use reth_evm::ConfigureEvm; use reth_primitives_traits::{BlockBody, BlockHeader}; -use reth_revm::{database::StateProviderDatabase, State}; use reth_rpc_api::TraceApiServer; use reth_rpc_convert::RpcTxReq; use reth_rpc_eth_api::{ @@ -102,10 +101,6 @@ where let this = self.clone(); self.eth_api() .spawn_with_call_at(trace_request.call, at, overrides, move |db, evm_env, tx_env| { - // wrapper is hack to get around 'higher-ranked lifetime error', see - // - let db = db.0; - let res = this.eth_api().inspect(&mut *db, evm_env, tx_env, &mut inspector)?; let trace_res = inspector .into_parity_builder() @@ -153,18 +148,14 @@ where let at = block_id.unwrap_or(BlockId::pending()); let (evm_env, at) = self.eth_api().evm_env_at(at).await?; - let this = self.clone(); // execute all transactions on top of each other and record the traces self.eth_api() - .spawn_with_state_at_block(at, move |state| { + .spawn_with_state_at_block(at, move |eth_api, mut db| { let mut results = Vec::with_capacity(calls.len()); - let mut db = - State::builder().with_database(StateProviderDatabase::new(state)).build(); - let mut calls = calls.into_iter().peekable(); while let Some((call, trace_types)) = calls.next() { - let (evm_env, tx_env) = this.eth_api().prepare_call_env( + let (evm_env, tx_env) = eth_api.prepare_call_env( evm_env.clone(), call, &mut db, @@ -172,7 +163,7 @@ where )?; let config = TracingInspectorConfig::from_parity_config(&trace_types); let mut inspector = TracingInspector::new(config); - let res = this.eth_api().inspect(&mut db, evm_env, tx_env, &mut inspector)?; + let res = eth_api.inspect(&mut db, evm_env, tx_env, &mut inspector)?; let trace_res = inspector .into_parity_builder() From e03c9da85cf8ec7b9864c46d0bdcafff6689ef04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90=E1=BA=A1t=20Nguy=E1=BB=85n?= Date: Mon, 24 Nov 2025 13:53:10 +0700 Subject: [PATCH 21/23] refactor: remove unused add_transactions_with_origins trait (#19824) --- crates/transaction-pool/src/lib.rs | 29 ------------------------- crates/transaction-pool/src/noop.rs | 13 ----------- crates/transaction-pool/src/pool/mod.rs | 27 +++++------------------ crates/transaction-pool/src/traits.rs | 10 --------- 4 files changed, 6 insertions(+), 73 deletions(-) diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index 7f3fa4a117..eca3814d92 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -384,23 +384,6 @@ where self.pool.validator().validate_transactions_with_origin(origin, transactions).await } - /// Validates all transactions with their individual origins. - /// - /// This returns the validated transactions in the same order as input. - async fn validate_all_with_origins( - &self, - transactions: Vec<(TransactionOrigin, V::Transaction)>, - ) -> Vec<(TransactionOrigin, TransactionValidationOutcome)> { - if transactions.len() == 1 { - let (origin, tx) = transactions.into_iter().next().unwrap(); - let res = self.pool.validator().validate_transaction(origin, tx).await; - return vec![(origin, res)] - } - let origins: Vec<_> = transactions.iter().map(|(origin, _)| *origin).collect(); - let tx_outcomes = self.pool.validator().validate_transactions(transactions).await; - origins.into_iter().zip(tx_outcomes).collect() - } - /// Number of transactions in the entire pool pub fn len(&self) -> usize { self.pool.len() @@ -516,18 +499,6 @@ where self.pool.add_transactions(origin, validated.into_iter()) } - async fn add_transactions_with_origins( - &self, - transactions: Vec<(TransactionOrigin, Self::Transaction)>, - ) -> Vec> { - if transactions.is_empty() { - return Vec::new() - } - let validated = self.validate_all_with_origins(transactions).await; - - self.pool.add_transactions_with_origins(validated) - } - fn transaction_event_listener(&self, tx_hash: TxHash) -> Option { self.pool.add_transaction_event_listener(tx_hash) } diff --git a/crates/transaction-pool/src/noop.rs b/crates/transaction-pool/src/noop.rs index dc5bb9c307..a553ea6e87 100644 --- a/crates/transaction-pool/src/noop.rs +++ b/crates/transaction-pool/src/noop.rs @@ -98,19 +98,6 @@ impl TransactionPool for NoopTransactionPool { .collect() } - async fn add_transactions_with_origins( - &self, - transactions: Vec<(TransactionOrigin, Self::Transaction)>, - ) -> Vec> { - transactions - .into_iter() - .map(|(_, transaction)| { - let hash = *transaction.hash(); - Err(PoolError::other(hash, Box::new(NoopInsertError::new(transaction)))) - }) - .collect() - } - fn transaction_event_listener(&self, _tx_hash: TxHash) -> Option { None } diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 74622b7189..02fc1ac753 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -571,24 +571,22 @@ where Ok(listener) } - /// Adds all transactions in the iterator to the pool, each with its individual origin, - /// returning a list of results. + /// Adds all transactions in the iterator to the pool, returning a list of results. /// /// Note: A large batch may lock the pool for a long time that blocks important operations /// like updating the pool on canonical state changes. The caller should consider having /// a max batch size to balance transaction insertions with other updates. - pub fn add_transactions_with_origins( + pub fn add_transactions( &self, - transactions: impl IntoIterator< - Item = (TransactionOrigin, TransactionValidationOutcome), - >, + origin: TransactionOrigin, + transactions: impl IntoIterator>, ) -> Vec> { // Process all transactions in one write lock, maintaining individual origins let (mut added, discarded) = { let mut pool = self.pool.write(); let added = transactions .into_iter() - .map(|(origin, tx)| self.add_transaction(&mut pool, origin, tx)) + .map(|tx| self.add_transaction(&mut pool, origin, tx)) .collect::>(); // Enforce the pool size limits if at least one transaction was added successfully @@ -618,24 +616,11 @@ where *res = Err(PoolError::new(*hash, PoolErrorKind::DiscardedOnInsert)) } } - } + }; added } - /// Adds all transactions in the iterator to the pool, returning a list of results. - /// - /// Note: A large batch may lock the pool for a long time that blocks important operations - /// like updating the pool on canonical state changes. The caller should consider having - /// a max batch size to balance transaction insertions with other updates. - pub fn add_transactions( - &self, - origin: TransactionOrigin, - transactions: impl IntoIterator>, - ) -> Vec> { - self.add_transactions_with_origins(transactions.into_iter().map(|tx| (origin, tx))) - } - /// Notify all listeners about a new pending transaction. /// /// See also [`Self::add_pending_listener`] diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 2b9d8bae8a..e5f9de0e60 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -177,16 +177,6 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { transactions: Vec, ) -> impl Future>> + Send; - /// Adds multiple _unvalidated_ transactions with individual origins. - /// - /// Each transaction can have its own [`TransactionOrigin`]. - /// - /// Consumer: RPC - fn add_transactions_with_origins( - &self, - transactions: Vec<(TransactionOrigin, Self::Transaction)>, - ) -> impl Future>> + Send; - /// Submit a consensus transaction directly to the pool fn add_consensus_transaction( &self, From d278b75c3ac71ddc7a027d3e93f0854a2b31385a Mon Sep 17 00:00:00 2001 From: Maximilian Hubert <64627729+gap-editor@users.noreply.github.com> Date: Mon, 24 Nov 2025 09:47:54 +0100 Subject: [PATCH 22/23] chore(stages): fix naming and simplify add_stages implementation (#19923) --- crates/stages/api/src/pipeline/builder.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/crates/stages/api/src/pipeline/builder.rs b/crates/stages/api/src/pipeline/builder.rs index 29da16dffe..818b037da7 100644 --- a/crates/stages/api/src/pipeline/builder.rs +++ b/crates/stages/api/src/pipeline/builder.rs @@ -35,11 +35,9 @@ impl PipelineBuilder { /// [`builder`][StageSet::builder] on the set which will convert it to a /// [`StageSetBuilder`][crate::StageSetBuilder]. pub fn add_stages>(mut self, set: Set) -> Self { - let states = set.builder().build(); - self.stages.reserve_exact(states.len()); - for stage in states { - self.stages.push(stage); - } + let stages = set.builder().build(); + self.stages.reserve(stages.len()); + self.stages.extend(stages); self } From f1fc979116754df919f1febe507fe4cdf43b000d Mon Sep 17 00:00:00 2001 From: YK Date: Mon, 24 Nov 2025 17:39:27 +0800 Subject: [PATCH 23/23] feat(reth-bench-compare): add standard deviation metrics to comparison report (#19928) --- bin/reth-bench-compare/src/comparison.rs | 39 +++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/bin/reth-bench-compare/src/comparison.rs b/bin/reth-bench-compare/src/comparison.rs index ad6d801e7c..544d17dd84 100644 --- a/bin/reth-bench-compare/src/comparison.rs +++ b/bin/reth-bench-compare/src/comparison.rs @@ -99,6 +99,8 @@ pub(crate) struct RefInfo { /// - `per_block_latency_change_mean_percent` / `per_block_latency_change_median_percent` are the /// mean and median of per-block percent deltas (feature vs baseline), capturing block-level /// drift. +/// - `per_block_latency_change_std_dev_percent`: standard deviation of per-block percent changes, +/// measuring consistency of performance changes across blocks. /// - `new_payload_total_latency_change_percent` is the percent change of the total newPayload time /// across the run. /// @@ -107,6 +109,7 @@ pub(crate) struct RefInfo { pub(crate) struct ComparisonSummary { pub per_block_latency_change_mean_percent: f64, pub per_block_latency_change_median_percent: f64, + pub per_block_latency_change_std_dev_percent: f64, pub new_payload_total_latency_change_percent: f64, pub new_payload_latency_p50_change_percent: f64, pub new_payload_latency_p90_change_percent: f64, @@ -384,6 +387,10 @@ impl ComparisonGenerator { } }; + // Calculate per-block statistics. "Per-block" means: for each block, compute the percent + // change (feature - baseline) / baseline * 100, then calculate statistics across those + // per-block percent changes. This captures how consistently the feature performs relative + // to baseline across all blocks. let per_block_percent_changes: Vec = per_block_comparisons.iter().map(|c| c.new_payload_latency_change_percent).collect(); let per_block_latency_change_mean_percent = if per_block_percent_changes.is_empty() { @@ -394,10 +401,12 @@ impl ComparisonGenerator { let per_block_latency_change_median_percent = if per_block_percent_changes.is_empty() { 0.0 } else { - let mut sorted = per_block_percent_changes; + let mut sorted = per_block_percent_changes.clone(); sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal)); percentile(&sorted, 0.5) }; + let per_block_latency_change_std_dev_percent = + calculate_std_dev(&per_block_percent_changes, per_block_latency_change_mean_percent); let baseline_total_latency_ms = baseline.mean_new_payload_latency_ms * baseline.total_blocks as f64; @@ -409,6 +418,7 @@ impl ComparisonGenerator { Ok(ComparisonSummary { per_block_latency_change_mean_percent, per_block_latency_change_median_percent, + per_block_latency_change_std_dev_percent, new_payload_total_latency_change_percent, new_payload_latency_p50_change_percent: calc_percent_change( baseline.median_new_payload_latency_ms, @@ -532,6 +542,10 @@ impl ComparisonGenerator { " NewPayload Latency per-block median change: {:+.2}%", summary.per_block_latency_change_median_percent ); + println!( + " NewPayload Latency per-block std dev: {:.2}%", + summary.per_block_latency_change_std_dev_percent + ); println!( " Total newPayload time change: {:+.2}%", summary.new_payload_total_latency_change_percent @@ -618,6 +632,29 @@ impl ComparisonGenerator { } } +/// Calculate standard deviation from a set of values and their mean. +/// +/// Computes the population standard deviation using the formula: +/// `sqrt(sum((x - mean)²) / n)` +/// +/// Returns 0.0 for empty input. +fn calculate_std_dev(values: &[f64], mean: f64) -> f64 { + if values.is_empty() { + return 0.0; + } + + let variance = values + .iter() + .map(|x| { + let diff = x - mean; + diff * diff + }) + .sum::() / + values.len() as f64; + + variance.sqrt() +} + /// Calculate percentile using linear interpolation on a sorted slice. /// /// Computes `rank = percentile × (n - 1)` where n is the array length. If the rank falls