github/reth
1
1
Fork 0
mirror of https://github.com/paradigmxyz/reth.git synced 2026-04-30 03:01:58 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into yk/async-trie

Browse Source
This commit is contained in:
Yong Kang
2025-11-24 10:09:09 +00:00
parent a5f19d3a65 f1fc979116
commit 8e0012da90
144 changed files with 3989 additions and 1599 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.github/workflows/lint.yml vendored
View File

@@ -92,7 +92,12 @@ jobs:
run: .github/assets/check_rv32imac.sh
crate-checks:
name: crate-checks (${{ matrix.partition }}/${{ matrix.total_partitions }})
runs-on: ubuntu-latest
strategy:
matrix:
partition: [1, 2]
total_partitions: [2]
timeout-minutes: 30
steps:
- uses: actions/checkout@v5
@@ -102,7 +107,7 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- run: cargo hack check --workspace
- run: cargo hack check --workspace --partition ${{ matrix.partition }}/${{ matrix.total_partitions }}
msrv:
name: MSRV

252
Cargo.lock generated
View File

@@ -97,9 +97,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
[[package]]
name = "alloy-chains"
version = "0.2.18"
version = "0.2.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfaa9ea039a6f9304b4a593d780b1f23e1ae183acdee938b11b38795acacc9f1"
checksum = "4bc32535569185cbcb6ad5fa64d989a47bccb9a08e27284b1f2a3ccf16e6d010"
dependencies = [
"alloy-primitives",
"alloy-rlp",
@@ -112,9 +112,9 @@ dependencies = [
[[package]]
name = "alloy-consensus"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad704069c12f68d0c742d0cad7e0a03882b42767350584627fbf8a47b1bf1846"
checksum = "8b6440213a22df93a87ed512d2f668e7dc1d62a05642d107f82d61edc9e12370"
dependencies = [
"alloy-eips",
"alloy-primitives",
@@ -140,9 +140,9 @@ dependencies = [
[[package]]
name = "alloy-consensus-any"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc374f640a5062224d7708402728e3d6879a514ba10f377da62e7dfb14c673e6"
checksum = "15d0bea09287942405c4f9d2a4f22d1e07611c2dbd9d5bf94b75366340f9e6e0"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -155,9 +155,9 @@ dependencies = [
[[package]]
name = "alloy-contract"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15c493b2812943f7b58191063a8d13ea97c76099900869c08231e8eba3bf2f92"
checksum = "d69af404f1d00ddb42f2419788fa87746a4cd13bab271916d7726fda6c792d94"
dependencies = [
"alloy-consensus",
"alloy-dyn-abi",
@@ -240,9 +240,9 @@ dependencies = [
[[package]]
name = "alloy-eips"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e867b5fd52ed0372a95016f3a37cbff95a9d5409230fbaef2d8ea00e8618098"
checksum = "4bd2c7ae05abcab4483ce821f12f285e01c0b33804e6883dd9ca1569a87ee2be"
dependencies = [
"alloy-eip2124",
"alloy-eip2930",
@@ -288,9 +288,9 @@ dependencies = [
[[package]]
name = "alloy-genesis"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b90be17e9760a6ba6d13cebdb049cea405ebc8bf57d90664ed708cc5bc348342"
checksum = "fc47eaae86488b07ea8e20236184944072a78784a1f4993f8ec17b3aa5d08c21"
dependencies = [
"alloy-eips",
"alloy-primitives",
@@ -329,9 +329,9 @@ dependencies = [
[[package]]
name = "alloy-json-rpc"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcab4c51fb1273e3b0f59078e0cdf8aa99f697925b09f0d2055c18be46b4d48c"
checksum = "003f46c54f22854a32b9cc7972660a476968008ad505427eabab49225309ec40"
dependencies = [
"alloy-primitives",
"alloy-sol-types",
@@ -344,9 +344,9 @@ dependencies = [
[[package]]
name = "alloy-network"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "196d7fd3f5d414f7bbd5886a628b7c42bd98d1b126f9a7cff69dbfd72007b39c"
checksum = "4f4029954d9406a40979f3a3b46950928a0fdcfe3ea8a9b0c17490d57e8aa0e3"
dependencies = [
"alloy-consensus",
"alloy-consensus-any",
@@ -370,9 +370,9 @@ dependencies = [
[[package]]
name = "alloy-network-primitives"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d3ae2777e900a7a47ad9e3b8ab58eff3d93628265e73bbdee09acf90bf68f75"
checksum = "7805124ad69e57bbae7731c9c344571700b2a18d351bda9e0eba521c991d1bcb"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -426,8 +426,8 @@ dependencies = [
"derive_more",
"foldhash 0.2.0",
"getrandom 0.3.4",
"hashbrown 0.16.0",
"indexmap 2.12.0",
"hashbrown 0.16.1",
"indexmap 2.12.1",
"itoa",
"k256",
"keccak-asm",
@@ -444,9 +444,9 @@ dependencies = [
[[package]]
name = "alloy-provider"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f9bf40c9b2a90c7677f9c39bccd9f06af457f35362439c0497a706f16557703"
checksum = "d369e12c92870d069e0c9dc5350377067af8a056e29e3badf8446099d7e00889"
dependencies = [
"alloy-chains",
"alloy-consensus",
@@ -489,9 +489,9 @@ dependencies = [
[[package]]
name = "alloy-pubsub"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acfdbe41e2ef1a7e79b5ea115baa750f9381ac9088fb600f4cedc731cf04a151"
checksum = "f77d20cdbb68a614c7a86b3ffef607b37d087bb47a03c58f4c3f8f99bc3ace3b"
dependencies = [
"alloy-json-rpc",
"alloy-primitives",
@@ -533,9 +533,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-client"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7c2630fde9ff6033a780635e1af6ef40e92d74a9cacb8af3defc1b15cfebca5"
checksum = "31c89883fe6b7381744cbe80fef638ac488ead4f1956a4278956a1362c71cd2e"
dependencies = [
"alloy-json-rpc",
"alloy-primitives",
@@ -559,9 +559,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad098153a12382c22a597e865530033f5e644473742d6c733562d448125e02a2"
checksum = "64e279e6d40ee40fe8f76753b678d8d5d260cb276dc6c8a8026099b16d2b43f4"
dependencies = [
"alloy-primitives",
"alloy-rpc-types-engine",
@@ -572,9 +572,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-admin"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7604c415f725bd776d46dae44912c276cc3d8af37f37811e5675389791aa0c6"
checksum = "2bcf50ccb65d29b8599f8f5e23dcac685f1d79459654c830cba381345760e901"
dependencies = [
"alloy-genesis",
"alloy-primitives",
@@ -584,9 +584,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-anvil"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "214d9d1033c173ab8fa32edd8a4655cd784447c820b0b66cd0d5167e049567d6"
checksum = "5e176c26fdd87893b6afeb5d92099d8f7e7a1fe11d6f4fe0883d6e33ac5f31ba"
dependencies = [
"alloy-primitives",
"alloy-rpc-types-eth",
@@ -596,9 +596,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-any"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50b8429b5b62d21bf3691eb1ae12aaae9bb496894d5a114e3cc73e27e6800ec8"
checksum = "b43c1622aac2508d528743fd4cfdac1dea92d5a8fa894038488ff7edd0af0b32"
dependencies = [
"alloy-consensus-any",
"alloy-rpc-types-eth",
@@ -607,9 +607,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-beacon"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f67f8269e8b5193a5328dd3ef4d60f93524071e53a993776e290581a59aa15fa"
checksum = "1786681640d4c60f22b6b8376b0f3fa200360bf1c3c2cb913e6c97f51928eb1b"
dependencies = [
"alloy-eips",
"alloy-primitives",
@@ -627,9 +627,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-debug"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01731601ea631bd825c652a225701ab466c09457f446b8d8129368a095389c5d"
checksum = "1b2ca3a434a6d49910a7e8e51797eb25db42ef8a5578c52d877fcb26d0afe7bc"
dependencies = [
"alloy-primitives",
"derive_more",
@@ -639,9 +639,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-engine"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9981491bb98e76099983f516ec7de550db0597031f5828c994961eb4bb993cce"
checksum = "d9c4c53a8b0905d931e7921774a1830609713bd3e8222347963172b03a3ecc68"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -660,9 +660,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-eth"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29031a6bf46177d65efce661f7ab37829ca09dd341bc40afb5194e97600655cc"
checksum = "ed5fafb741c19b3cca4cdd04fa215c89413491f9695a3e928dee2ae5657f607e"
dependencies = [
"alloy-consensus",
"alloy-consensus-any",
@@ -682,9 +682,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-mev"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5c5c78bdd2c72c47e66ab977af420fb4a10279707d4edbd2575693c47aa54a2"
checksum = "49a97bfc6d9b411c85bb08e1174ddd3e5d61b10d3bd13f529d6609f733cb2f6f"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -697,9 +697,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-trace"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01b842f5aac6676ff4b2e328262d03bdf49807eaec3fe3a4735c45c97388518b"
checksum = "c55324323aa634b01bdecb2d47462a8dce05f5505b14a6e5db361eef16eda476"
dependencies = [
"alloy-primitives",
"alloy-rpc-types-eth",
@@ -711,9 +711,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-txpool"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fa12c608873beeb7afa392944dce8829fa8a50c487f266863bb2dd6b743c4a2"
checksum = "96b1aa28effb6854be356ce92ed64cea3b323acd04c3f8bfb5126e2839698043"
dependencies = [
"alloy-primitives",
"alloy-rpc-types-eth",
@@ -723,9 +723,9 @@ dependencies = [
[[package]]
name = "alloy-serde"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01e856112bfa0d9adc85bd7c13db03fad0e71d1d6fb4c2010e475b6718108236"
checksum = "a6f180c399ca7c1e2fe17ea58343910cad0090878a696ff5a50241aee12fc529"
dependencies = [
"alloy-primitives",
"arbitrary",
@@ -735,9 +735,9 @@ dependencies = [
[[package]]
name = "alloy-signer"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66a4f629da632d5279bbc5731634f0f5c9484ad9c4cad0cd974d9669dc1f46d6"
checksum = "ecc39ad2c0a3d2da8891f4081565780703a593f090f768f884049aa3aa929cbc"
dependencies = [
"alloy-primitives",
"async-trait",
@@ -750,9 +750,9 @@ dependencies = [
[[package]]
name = "alloy-signer-local"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76c8950810dc43660c0f22883659c4218e090a5c75dce33fa4ca787715997b7b"
checksum = "930e17cb1e46446a193a593a3bfff8d0ecee4e510b802575ebe300ae2e43ef75"
dependencies = [
"alloy-consensus",
"alloy-network",
@@ -790,7 +790,7 @@ dependencies = [
"alloy-sol-macro-input",
"const-hex",
"heck",
"indexmap 2.12.0",
"indexmap 2.12.1",
"proc-macro-error2",
"proc-macro2",
"quote",
@@ -839,9 +839,9 @@ dependencies = [
[[package]]
name = "alloy-transport"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe215a2f9b51d5f1aa5c8cf22c8be8cdb354934de09c9a4e37aefb79b77552fd"
checksum = "cae82426d98f8bc18f53c5223862907cac30ab8fc5e4cd2bb50808e6d3ab43d8"
dependencies = [
"alloy-json-rpc",
"auto_impl",
@@ -862,9 +862,9 @@ dependencies = [
[[package]]
name = "alloy-transport-http"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc1b37b1a30d23deb3a8746e882c70b384c574d355bc2bbea9ea918b0c31366e"
checksum = "90aa6825760905898c106aba9c804b131816a15041523e80b6d4fe7af6380ada"
dependencies = [
"alloy-json-rpc",
"alloy-transport",
@@ -877,9 +877,9 @@ dependencies = [
[[package]]
name = "alloy-transport-ipc"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52c81a4deeaa0d4b022095db17b286188d731e29ea141d4ec765e166732972e4"
checksum = "6ace83a4a6bb896e5894c3479042e6ba78aa5271dde599aa8c36a021d49cc8cc"
dependencies = [
"alloy-json-rpc",
"alloy-pubsub",
@@ -897,9 +897,9 @@ dependencies = [
[[package]]
name = "alloy-transport-ws"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e9d6f5f304e8943afede2680e5fc7008780d4fc49387eafd53192ad95e20091"
checksum = "86c9ab4c199e3a8f3520b60ba81aa67bb21fed9ed0d8304e0569094d0758a56f"
dependencies = [
"alloy-pubsub",
"alloy-transport",
@@ -935,9 +935,9 @@ dependencies = [
[[package]]
name = "alloy-tx-macros"
version = "1.1.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ccf423f6de62e8ce1d6c7a11fb7508ae3536d02e0d68aaeb05c8669337d0937"
checksum = "ae109e33814b49fc0a62f2528993aa8a2dd346c26959b151f05441dc0b9da292"
dependencies = [
"darling 0.21.3",
"proc-macro2",
@@ -1719,7 +1719,7 @@ dependencies = [
"boa_interner",
"boa_macros",
"boa_string",
"indexmap 2.12.0",
"indexmap 2.12.1",
"num-bigint",
"rustc-hash",
]
@@ -1749,9 +1749,9 @@ dependencies = [
"futures-channel",
"futures-concurrency",
"futures-lite 2.6.1",
"hashbrown 0.16.0",
"hashbrown 0.16.1",
"icu_normalizer",
"indexmap 2.12.0",
"indexmap 2.12.1",
"intrusive-collections",
"itertools 0.14.0",
"num-bigint",
@@ -1784,7 +1784,7 @@ checksum = "f1179f690cbfcbe5364cceee5f1cb577265bb6f07b0be6f210aabe270adcf9da"
dependencies = [
"boa_macros",
"boa_string",
"hashbrown 0.16.0",
"hashbrown 0.16.1",
"thin-vec",
]
@@ -1796,8 +1796,8 @@ checksum = "9626505d33dc63d349662437297df1d3afd9d5fc4a2b3ad34e5e1ce879a78848"
dependencies = [
"boa_gc",
"boa_macros",
"hashbrown 0.16.0",
"indexmap 2.12.0",
"hashbrown 0.16.1",
"indexmap 2.12.1",
"once_cell",
"phf",
"rustc-hash",
@@ -2161,9 +2161,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.51"
version = "4.5.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c26d721170e0295f191a69bd9a1f93efcdb0aff38684b61ab5750468972e5f5"
checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8"
dependencies = [
"clap_builder",
"clap_derive",
@@ -2171,9 +2171,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.51"
version = "4.5.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75835f0c7bf681bfd05abe44e965760fea999a5286c6eb2d59883634fd02011a"
checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00"
dependencies = [
"anstream",
"anstyle",
@@ -3049,7 +3049,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users 0.5.2",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3379,7 +3379,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -4363,7 +4363,7 @@ dependencies = [
"futures-core",
"futures-sink",
"http",
"indexmap 2.12.0",
"indexmap 2.12.1",
"slab",
"tokio",
"tokio-util",
@@ -4421,14 +4421,15 @@ dependencies = [
[[package]]
name = "hashbrown"
version = "0.16.0"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash 0.2.0",
"serde",
"serde_core",
]
[[package]]
@@ -4728,7 +4729,7 @@ dependencies = [
"js-sys",
"log",
"wasm-bindgen",
"windows-core 0.57.0",
"windows-core 0.62.2",
]
[[package]]
@@ -4924,13 +4925,13 @@ dependencies = [
[[package]]
name = "indexmap"
version = "2.12.0"
version = "2.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f"
checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2"
dependencies = [
"arbitrary",
"equivalent",
"hashbrown 0.16.0",
"hashbrown 0.16.1",
"serde",
"serde_core",
]
@@ -4982,9 +4983,9 @@ dependencies = [
[[package]]
name = "insta"
version = "1.43.2"
version = "1.44.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46fdb647ebde000f43b5b53f773c30cf9b0cb4300453208713fa38b2c70935a0"
checksum = "e8732d3774162a0851e3f2b150eb98f31a9885dd75985099421d393385a01dfd"
dependencies = [
"console",
"once_cell",
@@ -5073,7 +5074,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -5702,7 +5703,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034"
dependencies = [
"base64 0.22.1",
"indexmap 2.12.0",
"indexmap 2.12.1",
"metrics",
"metrics-util",
"quanta",
@@ -5734,7 +5735,7 @@ dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
"hashbrown 0.15.5",
"indexmap 2.12.0",
"indexmap 2.12.1",
"metrics",
"ordered-float",
"quanta",
@@ -5982,7 +5983,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -6161,9 +6162,9 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
[[package]]
name = "op-alloy"
version = "0.22.1"
version = "0.22.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5da980f0756a505bf7f2ab9f0be886c8b3213b9c23a19fff6c1c6afa53803c3a"
checksum = "c3b13412d297c1f9341f678b763750b120a73ffe998fa54a94d6eda98449e7ca"
dependencies = [
"op-alloy-consensus",
"op-alloy-network",
@@ -6174,9 +6175,9 @@ dependencies = [
[[package]]
name = "op-alloy-consensus"
version = "0.22.1"
version = "0.22.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0d7ec388eb83a3e6c71774131dbbb2ba9c199b6acac7dce172ed8de2f819e91"
checksum = "726da827358a547be9f1e37c2a756b9e3729cb0350f43408164794b370cad8ae"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -6200,9 +6201,9 @@ checksum = "a79f352fc3893dcd670172e615afef993a41798a1d3fc0db88a3e60ef2e70ecc"
[[package]]
name = "op-alloy-network"
version = "0.22.1"
version = "0.22.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "979fe768bbb571d1d0bd7f84bc35124243b4db17f944b94698872a4701e743a0"
checksum = "f63f27e65be273ec8fcb0b6af0fd850b550979465ab93423705ceb3dfddbd2ab"
dependencies = [
"alloy-consensus",
"alloy-network",
@@ -6216,9 +6217,9 @@ dependencies = [
[[package]]
name = "op-alloy-provider"
version = "0.22.1"
version = "0.22.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0594687a750dff379f16219c7441bfa8979f11aeda440981e32e65a9e28c3c6"
checksum = "a71456699aa256dc20119736422ad9a44da8b9585036117afb936778122093b9"
dependencies = [
"alloy-network",
"alloy-primitives",
@@ -6231,9 +6232,9 @@ dependencies = [
[[package]]
name = "op-alloy-rpc-jsonrpsee"
version = "0.22.1"
version = "0.22.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7bdbb3c0453fe2605fb008851ea0b45f3f2ba607722c9f2e4ffd7198958ce501"
checksum = "8ef9114426b16172254555aad34a8ea96c01895e40da92f5d12ea680a1baeaa7"
dependencies = [
"alloy-primitives",
"jsonrpsee",
@@ -6241,9 +6242,9 @@ dependencies = [
[[package]]
name = "op-alloy-rpc-types"
version = "0.22.1"
version = "0.22.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc252b5fa74dbd33aa2f9a40e5ff9cfe34ed2af9b9b235781bc7cc8ec7d6aca8"
checksum = "562dd4462562c41f9fdc4d860858c40e14a25df7f983ae82047f15f08fce4d19"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -6261,9 +6262,9 @@ dependencies = [
[[package]]
name = "op-alloy-rpc-types-engine"
version = "0.22.1"
version = "0.22.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1abe694cd6718b8932da3f824f46778be0f43289e4103c88abc505c63533a04"
checksum = "d8f24b8cb66e4b33e6c9e508bf46b8ecafc92eadd0b93fedd306c0accb477657"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -6532,9 +6533,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "pest"
version = "2.8.3"
version = "2.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "989e7521a040efde50c3ab6bbadafbe15ab6dc042686926be59ac35d74607df4"
checksum = "cbcfd20a6d4eeba40179f05735784ad32bdaef05ce8e8af05f180d45bb3e7e22"
dependencies = [
"memchr",
"ucd-trie",
@@ -7013,7 +7014,7 @@ dependencies = [
"once_cell",
"socket2 0.6.1",
"tracing",
"windows-sys 0.52.0",
"windows-sys 0.60.2",
]
[[package]]
@@ -7300,7 +7301,7 @@ version = "0.10.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2057b2325e68a893284d1538021ab90279adac1139957ca2a74426c6f118fb48"
dependencies = [
"hashbrown 0.16.0",
"hashbrown 0.16.1",
"memchr",
]
@@ -7355,9 +7356,9 @@ dependencies = [
[[package]]
name = "resolv-conf"
version = "0.7.5"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b3789b30bd25ba102de4beabd95d21ac45b69b1be7d14522bab988c526d6799"
checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7"
[[package]]
name = "reth"
@@ -8101,6 +8102,7 @@ dependencies = [
"reth-optimism-chainspec",
"reth-payload-builder",
"reth-payload-primitives",
"reth-primitives-traits",
"reth-storage-api",
"reth-transaction-pool",
"tokio",
@@ -9802,6 +9804,7 @@ dependencies = [
name = "reth-payload-primitives"
version = "1.9.3"
dependencies = [
"alloy-consensus",
"alloy-eips",
"alloy-primitives",
"alloy-rpc-types-engine",
@@ -10744,7 +10747,6 @@ dependencies = [
"tracing-journald",
"tracing-logfmt",
"tracing-subscriber 0.3.20",
"url",
]
[[package]]
@@ -11464,7 +11466,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.11.0",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -11816,7 +11818,7 @@ version = "1.0.145"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"
dependencies = [
"indexmap 2.12.0",
"indexmap 2.12.1",
"itoa",
"memchr",
"ryu",
@@ -11866,7 +11868,7 @@ dependencies = [
"chrono",
"hex",
"indexmap 1.9.3",
"indexmap 2.12.0",
"indexmap 2.12.1",
"schemars 0.9.0",
"schemars 1.1.0",
"serde_core",
@@ -11986,9 +11988,9 @@ dependencies = [
[[package]]
name = "signal-hook-registry"
version = "1.4.6"
version = "1.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b"
checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad"
dependencies = [
"libc",
]
@@ -12355,7 +12357,7 @@ dependencies = [
"getrandom 0.3.4",
"once_cell",
"rustix 1.1.2",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -12735,7 +12737,7 @@ version = "0.22.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
dependencies = [
"indexmap 2.12.0",
"indexmap 2.12.1",
"serde",
"serde_spanned",
"toml_datetime 0.6.11",
@@ -12749,7 +12751,7 @@ version = "0.23.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d"
dependencies = [
"indexmap 2.12.0",
"indexmap 2.12.1",
"toml_datetime 0.7.3",
"toml_parser",
"winnow",
@@ -12816,7 +12818,7 @@ dependencies = [
"futures-core",
"futures-util",
"hdrhistogram",
"indexmap 2.12.0",
"indexmap 2.12.1",
"pin-project-lite",
"slab",
"sync_wrapper",
@@ -13570,7 +13572,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.48.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -14240,18 +14242,18 @@ dependencies = [
[[package]]
name = "zerocopy"
version = "0.8.27"
version = "0.8.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c"
checksum = "43fa6694ed34d6e57407afbccdeecfa268c470a7d2a5b0cf49ce9fcc345afb90"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.27"
version = "0.8.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831"
checksum = "c640b22cd9817fae95be82f0d2f90b11f7605f6c319d16705c459b27ac2cbc26"
dependencies = [
"proc-macro2",
"quote",

64
Cargo.toml
View File

@@ -489,42 +489,42 @@ alloy-trie = { version = "0.9.1", default-features = false }
alloy-hardforks = "0.4.4"
alloy-consensus = { version = "1.0.41", default-features = false }
alloy-contract = { version = "1.0.41", default-features = false }
alloy-eips = { version = "1.0.41", default-features = false }
alloy-genesis = { version = "1.0.41", default-features = false }
alloy-json-rpc = { version = "1.0.41", default-features = false }
alloy-network = { version = "1.0.41", default-features = false }
alloy-network-primitives = { version = "1.0.41", default-features = false }
alloy-provider = { version = "1.0.41", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.0.41", default-features = false }
alloy-rpc-client = { version = "1.0.41", default-features = false }
alloy-rpc-types = { version = "1.0.41", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.0.41", default-features = false }
alloy-rpc-types-anvil = { version = "1.0.41", default-features = false }
alloy-rpc-types-beacon = { version = "1.0.41", default-features = false }
alloy-rpc-types-debug = { version = "1.0.41", default-features = false }
alloy-rpc-types-engine = { version = "1.0.41", default-features = false }
alloy-rpc-types-eth = { version = "1.0.41", default-features = false }
alloy-rpc-types-mev = { version = "1.0.41", default-features = false }
alloy-rpc-types-trace = { version = "1.0.41", default-features = false }
alloy-rpc-types-txpool = { version = "1.0.41", default-features = false }
alloy-serde = { version = "1.0.41", default-features = false }
alloy-signer = { version = "1.0.41", default-features = false }
alloy-signer-local = { version = "1.0.41", default-features = false }
alloy-transport = { version = "1.0.41" }
alloy-transport-http = { version = "1.0.41", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.0.41", default-features = false }
alloy-transport-ws = { version = "1.0.41", default-features = false }
alloy-consensus = { version = "1.1.2", default-features = false }
alloy-contract = { version = "1.1.2", default-features = false }
alloy-eips = { version = "1.1.2", default-features = false }
alloy-genesis = { version = "1.1.2", default-features = false }
alloy-json-rpc = { version = "1.1.2", default-features = false }
alloy-network = { version = "1.1.2", default-features = false }
alloy-network-primitives = { version = "1.1.2", default-features = false }
alloy-provider = { version = "1.1.2", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.1.2", default-features = false }
alloy-rpc-client = { version = "1.1.2", default-features = false }
alloy-rpc-types = { version = "1.1.2", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.1.2", default-features = false }
alloy-rpc-types-anvil = { version = "1.1.2", default-features = false }
alloy-rpc-types-beacon = { version = "1.1.2", default-features = false }
alloy-rpc-types-debug = { version = "1.1.2", default-features = false }
alloy-rpc-types-engine = { version = "1.1.2", default-features = false }
alloy-rpc-types-eth = { version = "1.1.2", default-features = false }
alloy-rpc-types-mev = { version = "1.1.2", default-features = false }
alloy-rpc-types-trace = { version = "1.1.2", default-features = false }
alloy-rpc-types-txpool = { version = "1.1.2", default-features = false }
alloy-serde = { version = "1.1.2", default-features = false }
alloy-signer = { version = "1.1.2", default-features = false }
alloy-signer-local = { version = "1.1.2", default-features = false }
alloy-transport = { version = "1.1.2" }
alloy-transport-http = { version = "1.1.2", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.1.2", default-features = false }
alloy-transport-ws = { version = "1.1.2", default-features = false }
# op
alloy-op-evm = { version = "0.24.1", default-features = false }
alloy-op-hardforks = "0.4.4"
op-alloy-rpc-types = { version = "0.22.1", default-features = false }
op-alloy-rpc-types-engine = { version = "0.22.1", default-features = false }
op-alloy-network = { version = "0.22.1", default-features = false }
op-alloy-consensus = { version = "0.22.1", default-features = false }
op-alloy-rpc-jsonrpsee = { version = "0.22.1", default-features = false }
op-alloy-rpc-types = { version = "0.22.4", default-features = false }
op-alloy-rpc-types-engine = { version = "0.22.4", default-features = false }
op-alloy-network = { version = "0.22.4", default-features = false }
op-alloy-consensus = { version = "0.22.4", default-features = false }
op-alloy-rpc-jsonrpsee = { version = "0.22.4", default-features = false }
op-alloy-flz = { version = "0.13.1", default-features = false }
# misc

214
bin/reth-bench-compare/src/comparison.rs
View File

@@ -6,6 +6,7 @@ use csv::Reader;
use eyre::{eyre, Result, WrapErr};
use serde::{Deserialize, Serialize};
use std::{
cmp::Ordering,
collections::HashMap,
fs,
path::{Path, PathBuf},
@@ -50,13 +51,21 @@ pub(crate) struct TotalGasRow {
pub time: u128,
}
/// Summary statistics for a benchmark run
/// Summary statistics for a benchmark run.
///
/// Latencies are derived from per-block `engine_newPayload` timings (converted from µs to ms):
/// - `mean_new_payload_latency_ms`: arithmetic mean latency across blocks.
/// - `median_new_payload_latency_ms`: p50 latency across blocks.
/// - `p90_new_payload_latency_ms` / `p99_new_payload_latency_ms`: tail latencies across blocks.
#[derive(Debug, Clone, Serialize)]
pub(crate) struct BenchmarkSummary {
pub total_blocks: u64,
pub total_gas_used: u64,
pub total_duration_ms: u128,
pub avg_new_payload_latency_ms: f64,
pub mean_new_payload_latency_ms: f64,
pub median_new_payload_latency_ms: f64,
pub p90_new_payload_latency_ms: f64,
pub p99_new_payload_latency_ms: f64,
pub gas_per_second: f64,
pub blocks_per_second: f64,
pub min_block_number: u64,
@@ -82,10 +91,29 @@ pub(crate) struct RefInfo {
pub end_timestamp: Option<DateTime<Utc>>,
}
/// Summary of the comparison between references
/// Summary of the comparison between references.
///
/// Percent deltas are `(feature - baseline) / baseline * 100`:
/// - `new_payload_latency_p50_change_percent` / p90 / p99: percent changes of the respective
/// per-block percentiles.
/// - `per_block_latency_change_mean_percent` / `per_block_latency_change_median_percent` are the
/// mean and median of per-block percent deltas (feature vs baseline), capturing block-level
/// drift.
/// - `per_block_latency_change_std_dev_percent`: standard deviation of per-block percent changes,
/// measuring consistency of performance changes across blocks.
/// - `new_payload_total_latency_change_percent` is the percent change of the total newPayload time
/// across the run.
///
/// Positive means slower/higher; negative means faster/lower.
#[derive(Debug, Serialize)]
pub(crate) struct ComparisonSummary {
pub new_payload_latency_change_percent: f64,
pub per_block_latency_change_mean_percent: f64,
pub per_block_latency_change_median_percent: f64,
pub per_block_latency_change_std_dev_percent: f64,
pub new_payload_total_latency_change_percent: f64,
pub new_payload_latency_p50_change_percent: f64,
pub new_payload_latency_p90_change_percent: f64,
pub new_payload_latency_p99_change_percent: f64,
pub gas_per_second_change_percent: f64,
pub blocks_per_second_change_percent: f64,
}
@@ -188,10 +216,12 @@ impl ComparisonGenerator {
let feature =
self.feature_results.as_ref().ok_or_else(|| eyre!("Feature results not loaded"))?;
// Generate comparison
let comparison_summary =
self.calculate_comparison_summary(&baseline.summary, &feature.summary)?;
let per_block_comparisons = self.calculate_per_block_comparisons(baseline, feature)?;
let comparison_summary = self.calculate_comparison_summary(
&baseline.summary,
&feature.summary,
&per_block_comparisons,
)?;
let report = ComparisonReport {
timestamp: self.timestamp.clone(),
@@ -281,7 +311,11 @@ impl ComparisonGenerator {
Ok(rows)
}
/// Calculate summary statistics for a benchmark run
/// Calculate summary statistics for a benchmark run.
///
/// Computes latency statistics from per-block `new_payload_latency` values in `combined_data`
/// (converting from µs to ms), and throughput metrics using the total run duration from
/// `total_gas_data`. Percentiles (p50/p90/p99) use linear interpolation on sorted latencies.
fn calculate_summary(
&self,
combined_data: &[CombinedLatencyRow],
@@ -296,9 +330,16 @@ impl ComparisonGenerator {
let total_duration_ms = total_gas_data.last().unwrap().time / 1000; // Convert microseconds to milliseconds
let avg_new_payload_latency_ms: f64 =
combined_data.iter().map(|r| r.new_payload_latency as f64 / 1000.0).sum::<f64>() /
total_blocks as f64;
let latencies_ms: Vec<f64> =
combined_data.iter().map(|r| r.new_payload_latency as f64 / 1000.0).collect();
let mean_new_payload_latency_ms: f64 =
latencies_ms.iter().sum::<f64>() / total_blocks as f64;
let mut sorted_latencies_ms = latencies_ms;
sorted_latencies_ms.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal));
let median_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.5);
let p90_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.9);
let p99_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.99);
let total_duration_seconds = total_duration_ms as f64 / 1000.0;
let gas_per_second = if total_duration_seconds > f64::EPSILON {
@@ -320,7 +361,10 @@ impl ComparisonGenerator {
total_blocks,
total_gas_used,
total_duration_ms,
avg_new_payload_latency_ms,
mean_new_payload_latency_ms,
median_new_payload_latency_ms,
p90_new_payload_latency_ms,
p99_new_payload_latency_ms,
gas_per_second,
blocks_per_second,
min_block_number,
@@ -333,6 +377,7 @@ impl ComparisonGenerator {
&self,
baseline: &BenchmarkSummary,
feature: &BenchmarkSummary,
per_block_comparisons: &[BlockComparison],
) -> Result<ComparisonSummary> {
let calc_percent_change = |baseline: f64, feature: f64| -> f64 {
if baseline.abs() > f64::EPSILON {
@@ -342,10 +387,50 @@ impl ComparisonGenerator {
}
};
// Calculate per-block statistics. "Per-block" means: for each block, compute the percent
// change (feature - baseline) / baseline * 100, then calculate statistics across those
// per-block percent changes. This captures how consistently the feature performs relative
// to baseline across all blocks.
let per_block_percent_changes: Vec<f64> =
per_block_comparisons.iter().map(|c| c.new_payload_latency_change_percent).collect();
let per_block_latency_change_mean_percent = if per_block_percent_changes.is_empty() {
0.0
} else {
per_block_percent_changes.iter().sum::<f64>() / per_block_percent_changes.len() as f64
};
let per_block_latency_change_median_percent = if per_block_percent_changes.is_empty() {
0.0
} else {
let mut sorted = per_block_percent_changes.clone();
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal));
percentile(&sorted, 0.5)
};
let per_block_latency_change_std_dev_percent =
calculate_std_dev(&per_block_percent_changes, per_block_latency_change_mean_percent);
let baseline_total_latency_ms =
baseline.mean_new_payload_latency_ms * baseline.total_blocks as f64;
let feature_total_latency_ms =
feature.mean_new_payload_latency_ms * feature.total_blocks as f64;
let new_payload_total_latency_change_percent =
calc_percent_change(baseline_total_latency_ms, feature_total_latency_ms);
Ok(ComparisonSummary {
new_payload_latency_change_percent: calc_percent_change(
baseline.avg_new_payload_latency_ms,
feature.avg_new_payload_latency_ms,
per_block_latency_change_mean_percent,
per_block_latency_change_median_percent,
per_block_latency_change_std_dev_percent,
new_payload_total_latency_change_percent,
new_payload_latency_p50_change_percent: calc_percent_change(
baseline.median_new_payload_latency_ms,
feature.median_new_payload_latency_ms,
),
new_payload_latency_p90_change_percent: calc_percent_change(
baseline.p90_new_payload_latency_ms,
feature.p90_new_payload_latency_ms,
),
new_payload_latency_p99_change_percent: calc_percent_change(
baseline.p99_new_payload_latency_ms,
feature.p99_new_payload_latency_ms,
),
gas_per_second_change_percent: calc_percent_change(
baseline.gas_per_second,
@@ -450,15 +535,39 @@ impl ComparisonGenerator {
println!("Performance Changes:");
println!(
" NewPayload Latency: {:+.2}% (total avg change)",
summary.new_payload_latency_change_percent
" NewPayload Latency per-block mean change: {:+.2}%",
summary.per_block_latency_change_mean_percent
);
println!(
" Gas/Second: {:+.2}% (total avg change)",
" NewPayload Latency per-block median change: {:+.2}%",
summary.per_block_latency_change_median_percent
);
println!(
" NewPayload Latency per-block std dev: {:.2}%",
summary.per_block_latency_change_std_dev_percent
);
println!(
" Total newPayload time change: {:+.2}%",
summary.new_payload_total_latency_change_percent
);
println!(
" NewPayload Latency p50: {:+.2}%",
summary.new_payload_latency_p50_change_percent
);
println!(
" NewPayload Latency p90: {:+.2}%",
summary.new_payload_latency_p90_change_percent
);
println!(
" NewPayload Latency p99: {:+.2}%",
summary.new_payload_latency_p99_change_percent
);
println!(
" Gas/Second: {:+.2}%",
summary.gas_per_second_change_percent
);
println!(
" Blocks/Second: {:+.2}% (total avg change)",
" Blocks/Second: {:+.2}%",
summary.blocks_per_second_change_percent
);
println!();
@@ -473,7 +582,14 @@ impl ComparisonGenerator {
baseline.total_gas_used,
baseline.total_duration_ms as f64 / 1000.0
);
println!(" Avg NewPayload: {:.2}ms", baseline.avg_new_payload_latency_ms);
println!(" NewPayload latency (ms):");
println!(
" mean: {:.2}, p50: {:.2}, p90: {:.2}, p99: {:.2}",
baseline.mean_new_payload_latency_ms,
baseline.median_new_payload_latency_ms,
baseline.p90_new_payload_latency_ms,
baseline.p99_new_payload_latency_ms
);
if let (Some(start), Some(end)) =
(&report.baseline.start_timestamp, &report.baseline.end_timestamp)
{
@@ -495,7 +611,14 @@ impl ComparisonGenerator {
feature.total_gas_used,
feature.total_duration_ms as f64 / 1000.0
);
println!(" Avg NewPayload: {:.2}ms", feature.avg_new_payload_latency_ms);
println!(" NewPayload latency (ms):");
println!(
" mean: {:.2}, p50: {:.2}, p90: {:.2}, p99: {:.2}",
feature.mean_new_payload_latency_ms,
feature.median_new_payload_latency_ms,
feature.p90_new_payload_latency_ms,
feature.p99_new_payload_latency_ms
);
if let (Some(start), Some(end)) =
(&report.feature.start_timestamp, &report.feature.end_timestamp)
{
@@ -508,3 +631,52 @@ impl ComparisonGenerator {
println!();
}
}
/// Calculate standard deviation from a set of values and their mean.
///
/// Computes the population standard deviation using the formula:
/// `sqrt(sum((x - mean)²) / n)`
///
/// Returns 0.0 for empty input.
fn calculate_std_dev(values: &[f64], mean: f64) -> f64 {
if values.is_empty() {
return 0.0;
}
let variance = values
.iter()
.map(|x| {
let diff = x - mean;
diff * diff
})
.sum::<f64>() /
values.len() as f64;
variance.sqrt()
}
/// Calculate percentile using linear interpolation on a sorted slice.
///
/// Computes `rank = percentile × (n - 1)` where n is the array length. If the rank falls
/// between two indices, linearly interpolates between those values. For example, with 100 values,
/// p90 computes rank = 0.9 × 99 = 89.1, then returns `values[89] × 0.9 + values[90] × 0.1`.
///
/// Returns 0.0 for empty input.
fn percentile(sorted_values: &[f64], percentile: f64) -> f64 {
if sorted_values.is_empty() {
return 0.0;
}
let clamped = percentile.clamp(0.0, 1.0);
let max_index = sorted_values.len() - 1;
let rank = clamped * max_index as f64;
let lower = rank.floor() as usize;
let upper = rank.ceil() as usize;
if lower == upper {
sorted_values[lower]
} else {
let weight = rank - lower as f64;
sorted_values[lower].mul_add(1.0 - weight, sorted_values[upper] * weight)
}
}

12
crates/cli/commands/src/db/get.rs
View File

@@ -3,6 +3,7 @@ use clap::Parser;
use reth_db::{
static_file::{
ColumnSelectorOne, ColumnSelectorTwo, HeaderWithHashMask, ReceiptMask, TransactionMask,
TransactionSenderMask,
},
RawDupSort,
};
@@ -75,6 +76,10 @@ impl Command {
StaticFileSegment::Receipts => {
(table_key::<tables::Receipts>(&key)?, <ReceiptMask<ReceiptTy<N>>>::MASK)
}
StaticFileSegment::TransactionSenders => (
table_key::<tables::TransactionSenders>(&key)?,
<TransactionSenderMask>::MASK,
),
};
let content = tool
@@ -114,6 +119,13 @@ impl Command {
)?;
println!("{}", serde_json::to_string_pretty(&receipt)?);
}
StaticFileSegment::TransactionSenders => {
let sender =
<<tables::TransactionSenders as Table>::Value>::decompress(
content[0].as_slice(),
)?;
println!("{}", serde_json::to_string_pretty(&sender)?);
}
}
}
}

19
crates/cli/commands/src/db/settings.rs
View File

@@ -44,6 +44,11 @@ pub enum SetCommand {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store transaction senders in static files instead of the database
TransactionSendersInStaticFiles {
#[clap(action(ArgAction::Set))]
value: bool,
},
}
impl Command {
@@ -83,8 +88,10 @@ impl Command {
println!("No storage settings found, creating new settings.");
}
let mut settings @ StorageSettings { receipts_in_static_files: _ } =
settings.unwrap_or_default();
let mut settings @ StorageSettings {
receipts_in_static_files: _,
transaction_senders_in_static_files: _,
} = settings.unwrap_or_else(StorageSettings::legacy);
// Update the setting based on the key
match cmd {
@@ -96,6 +103,14 @@ impl Command {
settings.receipts_in_static_files = value;
println!("Set receipts_in_static_files = {}", value);
}
SetCommand::TransactionSendersInStaticFiles { value } => {
if settings.transaction_senders_in_static_files == value {
println!("transaction_senders_in_static_files is already set to {}", value);
return Ok(());
}
settings.transaction_senders_in_static_files = value;
println!("Set transaction_senders_in_static_files = {}", value);
}
}
// Write updated settings

12
crates/cli/commands/src/stage/drop.rs
View File

@@ -45,6 +45,7 @@ impl<C: ChainSpecParser> Command<C> {
StageEnum::Headers => Some(StaticFileSegment::Headers),
StageEnum::Bodies => Some(StaticFileSegment::Transactions),
StageEnum::Execution => Some(StaticFileSegment::Receipts),
StageEnum::Senders => Some(StaticFileSegment::TransactionSenders),
_ => None,
};
@@ -68,17 +69,24 @@ impl<C: ChainSpecParser> Command<C> {
StaticFileSegment::Transactions => {
let to_delete = static_file_provider
.get_highest_static_file_tx(static_file_segment)
.map(|tx| tx + 1)
.map(|tx_num| tx_num + 1)
.unwrap_or_default();
writer.prune_transactions(to_delete, 0)?;
}
StaticFileSegment::Receipts => {
let to_delete = static_file_provider
.get_highest_static_file_tx(static_file_segment)
.map(|receipt| receipt + 1)
.map(|tx_num| tx_num + 1)
.unwrap_or_default();
writer.prune_receipts(to_delete, 0)?;
}
StaticFileSegment::TransactionSenders => {
let to_delete = static_file_provider
.get_highest_static_file_tx(static_file_segment)
.map(|tx_num| tx_num + 1)
.unwrap_or_default();
writer.prune_transaction_senders(to_delete, 0)?;
}
}
}
}

13
crates/config/src/config.rs
View File

@@ -436,6 +436,8 @@ pub struct BlocksPerFileConfig {
pub transactions: Option<u64>,
/// Number of blocks per file for the receipts segment.
pub receipts: Option<u64>,
/// Number of blocks per file for the transaction senders segment.
pub transaction_senders: Option<u64>,
}
impl StaticFilesConfig {
@@ -443,7 +445,8 @@ impl StaticFilesConfig {
///
/// Returns an error if any blocks per file value is zero.
pub fn validate(&self) -> eyre::Result<()> {
let BlocksPerFileConfig { headers, transactions, receipts } = self.blocks_per_file;
let BlocksPerFileConfig { headers, transactions, receipts, transaction_senders } =
self.blocks_per_file;
eyre::ensure!(headers != Some(0), "Headers segment blocks per file must be greater than 0");
eyre::ensure!(
transactions != Some(0),
@@ -453,12 +456,17 @@ impl StaticFilesConfig {
receipts != Some(0),
"Receipts segment blocks per file must be greater than 0"
);
eyre::ensure!(
transaction_senders != Some(0),
"Transaction senders segment blocks per file must be greater than 0"
);
Ok(())
}
/// Converts the blocks per file configuration into a [`HashMap`] per segment.
pub fn as_blocks_per_file_map(&self) -> HashMap<StaticFileSegment, u64> {
let BlocksPerFileConfig { headers, transactions, receipts } = self.blocks_per_file;
let BlocksPerFileConfig { headers, transactions, receipts, transaction_senders } =
self.blocks_per_file;
let mut map = HashMap::new();
// Iterating over all possible segments allows us to do an exhaustive match here,
@@ -468,6 +476,7 @@ impl StaticFilesConfig {
StaticFileSegment::Headers => headers,
StaticFileSegment::Transactions => transactions,
StaticFileSegment::Receipts => receipts,
StaticFileSegment::TransactionSenders => transaction_senders,
};
if let Some(blocks_per_file) = blocks_per_file {

18
crates/e2e-test-utils/src/lib.rs
View File

@@ -3,13 +3,12 @@
use node::NodeTestContext;
use reth_chainspec::ChainSpec;
use reth_db::{test_utils::TempDatabase, DatabaseEnv};
use reth_engine_local::LocalPayloadAttributesBuilder;
use reth_network_api::test_utils::PeersHandleProvider;
use reth_node_builder::{
components::NodeComponentsBuilder,
rpc::{EngineValidatorAddOn, RethRpcAddOns},
FullNodeTypesAdapter, Node, NodeAdapter, NodeComponents, NodeTypes, NodeTypesWithDBAdapter,
PayloadAttributesBuilder, PayloadTypes,
PayloadTypes,
};
use reth_provider::providers::{BlockchainProvider, NodeTypesForProvider};
use reth_tasks::TaskManager;
@@ -54,8 +53,6 @@ pub async fn setup<N>(
) -> eyre::Result<(Vec<NodeHelperType<N>>, TaskManager, Wallet)>
where
N: NodeBuilderHelper,
LocalPayloadAttributesBuilder<N::ChainSpec>:
PayloadAttributesBuilder<<<N as NodeTypes>::Payload as PayloadTypes>::PayloadAttributes>,
{
E2ETestSetupBuilder::new(num_nodes, chain_spec, attributes_generator)
.with_node_config_modifier(move |config| config.set_dev(is_dev))
@@ -77,8 +74,6 @@ pub async fn setup_engine<N>(
)>
where
N: NodeBuilderHelper,
LocalPayloadAttributesBuilder<N::ChainSpec>:
PayloadAttributesBuilder<<N::Payload as PayloadTypes>::PayloadAttributes>,
{
setup_engine_with_connection::<N>(
num_nodes,
@@ -106,8 +101,6 @@ pub async fn setup_engine_with_connection<N>(
)>
where
N: NodeBuilderHelper,
LocalPayloadAttributesBuilder<N::ChainSpec>:
PayloadAttributesBuilder<<N::Payload as PayloadTypes>::PayloadAttributes>,
{
E2ETestSetupBuilder::new(num_nodes, chain_spec, attributes_generator)
.with_tree_config_modifier(move |_| tree_config.clone())
@@ -160,13 +153,10 @@ where
>,
ChainSpec: From<ChainSpec> + Clone,
>,
LocalPayloadAttributesBuilder<Self::ChainSpec>:
PayloadAttributesBuilder<<Self::Payload as PayloadTypes>::PayloadAttributes>,
{
}
impl<T> NodeBuilderHelper for T
where
impl<T> NodeBuilderHelper for T where
Self: Default
+ NodeTypesForProvider<
Payload: PayloadTypes<
@@ -187,8 +177,6 @@ where
Adapter<Self, BlockchainProvider<NodeTypesWithDBAdapter<Self, TmpDB>>>,
>,
ChainSpec: From<ChainSpec> + Clone,
>,
LocalPayloadAttributesBuilder<Self::ChainSpec>:
PayloadAttributesBuilder<<Self::Payload as PayloadTypes>::PayloadAttributes>,
>
{
}

113
crates/e2e-test-utils/src/setup_builder.rs
View File

@@ -4,18 +4,18 @@
//! configurations through closures that modify `NodeConfig` and `TreeConfig`.
use crate::{node::NodeTestContext, wallet::Wallet, NodeBuilderHelper, NodeHelperType, TmpDB};
use futures_util::future::TryJoinAll;
use reth_chainspec::EthChainSpec;
use reth_engine_local::LocalPayloadAttributesBuilder;
use reth_node_builder::{
EngineNodeLauncher, NodeBuilder, NodeConfig, NodeHandle, NodeTypes, NodeTypesWithDBAdapter,
PayloadAttributesBuilder, PayloadTypes,
PayloadTypes,
};
use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs};
use reth_provider::providers::BlockchainProvider;
use reth_rpc_server_types::RpcModuleSelection;
use reth_tasks::TaskManager;
use std::sync::Arc;
use tracing::{span, Level};
use tracing::{span, Instrument, Level};
/// Type alias for tree config modifier closure
type TreeConfigModifier =
@@ -37,8 +37,6 @@ where
+ Sync
+ Copy
+ 'static,
LocalPayloadAttributesBuilder<N::ChainSpec>:
PayloadAttributesBuilder<<N::Payload as PayloadTypes>::PayloadAttributes>,
{
num_nodes: usize,
chain_spec: Arc<N::ChainSpec>,
@@ -56,8 +54,6 @@ where
+ Sync
+ Copy
+ 'static,
LocalPayloadAttributesBuilder<N::ChainSpec>:
PayloadAttributesBuilder<<N::Payload as PayloadTypes>::PayloadAttributes>,
{
/// Creates a new builder with the required parameters.
pub fn new(num_nodes: usize, chain_spec: Arc<N::ChainSpec>, attributes_generator: F) -> Self {
@@ -122,66 +118,71 @@ where
reth_node_api::TreeConfig::default()
};
let mut nodes: Vec<NodeTestContext<_, _>> = Vec::with_capacity(self.num_nodes);
let mut nodes = (0..self.num_nodes)
.map(async |idx| {
// Create base node config
let base_config = NodeConfig::new(self.chain_spec.clone())
.with_network(network_config.clone())
.with_unused_ports()
.with_rpc(
RpcServerArgs::default()
.with_unused_ports()
.with_http()
.with_http_api(RpcModuleSelection::All),
);
// Apply node config modifier if present
let node_config = if let Some(modifier) = &self.node_config_modifier {
modifier(base_config)
} else {
base_config
};
let span = span!(Level::INFO, "node", idx);
let node = N::default();
let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config)
.testing_node(exec.clone())
.with_types_and_provider::<N, BlockchainProvider<_>>()
.with_components(node.components_builder())
.with_add_ons(node.add_ons())
.launch_with_fn(|builder| {
let launcher = EngineNodeLauncher::new(
builder.task_executor().clone(),
builder.config().datadir(),
tree_config.clone(),
);
builder.launch_with(launcher)
})
.instrument(span)
.await?;
let node = NodeTestContext::new(node, self.attributes_generator).await?;
let genesis = node.block_hash(0);
node.update_forkchoice(genesis, genesis).await?;
eyre::Ok(node)
})
.collect::<TryJoinAll<_>>()
.await?;
for idx in 0..self.num_nodes {
// Create base node config
let base_config = NodeConfig::new(self.chain_spec.clone())
.with_network(network_config.clone())
.with_unused_ports()
.with_rpc(
RpcServerArgs::default()
.with_unused_ports()
.with_http()
.with_http_api(RpcModuleSelection::All),
);
// Apply node config modifier if present
let node_config = if let Some(modifier) = &self.node_config_modifier {
modifier(base_config)
} else {
base_config
};
let span = span!(Level::INFO, "node", idx);
let _enter = span.enter();
let node = N::default();
let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config)
.testing_node(exec.clone())
.with_types_and_provider::<N, BlockchainProvider<_>>()
.with_components(node.components_builder())
.with_add_ons(node.add_ons())
.launch_with_fn(|builder| {
let launcher = EngineNodeLauncher::new(
builder.task_executor().clone(),
builder.config().datadir(),
tree_config.clone(),
);
builder.launch_with(launcher)
})
.await?;
let mut node = NodeTestContext::new(node, self.attributes_generator).await?;
let genesis = node.block_hash(0);
node.update_forkchoice(genesis, genesis).await?;
let (prev, current) = nodes.split_at_mut(idx);
let current = current.first_mut().unwrap();
// Connect nodes if requested
if self.connect_nodes {
if let Some(previous_node) = nodes.last_mut() {
previous_node.connect(&mut node).await;
if let Some(prev_idx) = idx.checked_sub(1) {
prev[prev_idx].connect(current).await;
}
// Connect last node with the first if there are more than two
if idx + 1 == self.num_nodes &&
self.num_nodes > 2 &&
let Some(first_node) = nodes.first_mut()
let Some(first) = prev.first_mut()
{
node.connect(first_node).await;
current.connect(first).await;
}
}
nodes.push(node);
}
Ok((nodes, tasks, Wallet::default().with_chain_id(self.chain_spec.chain().into())))
@@ -196,8 +197,6 @@ where
+ Sync
+ Copy
+ 'static,
LocalPayloadAttributesBuilder<N::ChainSpec>:
PayloadAttributesBuilder<<N::Payload as PayloadTypes>::PayloadAttributes>,
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("E2ETestSetupBuilder")

8
crates/e2e-test-utils/src/testsuite/mod.rs
View File

@@ -2,13 +2,12 @@
use crate::{
testsuite::actions::{Action, ActionBox},
NodeBuilderHelper, PayloadAttributesBuilder,
NodeBuilderHelper,
};
use alloy_primitives::B256;
use eyre::Result;
use jsonrpsee::http_client::HttpClient;
use reth_engine_local::LocalPayloadAttributesBuilder;
use reth_node_api::{EngineTypes, NodeTypes, PayloadTypes};
use reth_node_api::{EngineTypes, PayloadTypes};
use reth_payload_builder::PayloadId;
use std::{collections::HashMap, marker::PhantomData};
pub mod actions;
@@ -349,9 +348,6 @@ where
pub async fn run<N>(mut self) -> Result<()>
where
N: NodeBuilderHelper<Payload = I>,
LocalPayloadAttributesBuilder<N::ChainSpec>: PayloadAttributesBuilder<
<<N as NodeTypes>::Payload as PayloadTypes>::PayloadAttributes,
>,
{
let mut setup = self.setup.take();

42
crates/e2e-test-utils/src/testsuite/setup.rs
View File

@@ -1,15 +1,11 @@
//! Test setup utilities for configuring the initial state.
use crate::{
setup_engine_with_connection, testsuite::Environment, NodeBuilderHelper,
PayloadAttributesBuilder,
};
use crate::{setup_engine_with_connection, testsuite::Environment, NodeBuilderHelper};
use alloy_eips::BlockNumberOrTag;
use alloy_primitives::B256;
use alloy_rpc_types_engine::{ForkchoiceState, PayloadAttributes};
use eyre::{eyre, Result};
use reth_chainspec::ChainSpec;
use reth_engine_local::LocalPayloadAttributesBuilder;
use reth_ethereum_primitives::Block;
use reth_network_p2p::sync::{NetworkSyncUpdater, SyncState};
use reth_node_api::{EngineTypes, NodeTypes, PayloadTypes, TreeConfig};
@@ -138,28 +134,19 @@ where
) -> Result<()>
where
N: NodeBuilderHelper<Payload = I>,
LocalPayloadAttributesBuilder<N::ChainSpec>: PayloadAttributesBuilder<
<<N as NodeTypes>::Payload as PayloadTypes>::PayloadAttributes,
>,
{
// Note: this future is quite large so we box it
Box::pin(self.apply_with_import_::<N>(env, rlp_path)).await
Box::pin(self.apply_with_import_(env, rlp_path)).await
}
/// Apply setup using pre-imported chain data from RLP file
async fn apply_with_import_<N>(
async fn apply_with_import_(
&mut self,
env: &mut Environment<I>,
rlp_path: &Path,
) -> Result<()>
where
N: NodeBuilderHelper<Payload = I>,
LocalPayloadAttributesBuilder<N::ChainSpec>: PayloadAttributesBuilder<
<<N as NodeTypes>::Payload as PayloadTypes>::PayloadAttributes,
>,
{
) -> Result<()> {
// Create nodes with imported chain data
let import_result = self.create_nodes_with_import::<N>(rlp_path).await?;
let import_result = self.create_nodes_with_import(rlp_path).await?;
// Extract node clients
let mut node_clients = Vec::new();
@@ -186,9 +173,6 @@ where
pub async fn apply<N>(&mut self, env: &mut Environment<I>) -> Result<()>
where
N: NodeBuilderHelper<Payload = I>,
LocalPayloadAttributesBuilder<N::ChainSpec>: PayloadAttributesBuilder<
<<N as NodeTypes>::Payload as PayloadTypes>::PayloadAttributes,
>,
{
// Note: this future is quite large so we box it
Box::pin(self.apply_::<N>(env)).await
@@ -198,9 +182,6 @@ where
async fn apply_<N>(&mut self, env: &mut Environment<I>) -> Result<()>
where
N: NodeBuilderHelper<Payload = I>,
LocalPayloadAttributesBuilder<N::ChainSpec>: PayloadAttributesBuilder<
<<N as NodeTypes>::Payload as PayloadTypes>::PayloadAttributes,
>,
{
// If import_rlp_path is set, use apply_with_import instead
if let Some(rlp_path) = self.import_rlp_path.take() {
@@ -259,16 +240,10 @@ where
/// Note: Currently this only supports `EthereumNode` due to the import process
/// being Ethereum-specific. The generic parameter N is kept for consistency
/// with other methods but is not used.
async fn create_nodes_with_import<N>(
async fn create_nodes_with_import(
&self,
rlp_path: &Path,
) -> Result<crate::setup_import::ChainImportResult>
where
N: NodeBuilderHelper<Payload = I>,
LocalPayloadAttributesBuilder<N::ChainSpec>: PayloadAttributesBuilder<
<<N as NodeTypes>::Payload as PayloadTypes>::PayloadAttributes,
>,
{
) -> Result<crate::setup_import::ChainImportResult> {
let chain_spec =
self.chain_spec.clone().ok_or_else(|| eyre!("Chain specification is required"))?;
@@ -301,9 +276,6 @@ where
+ use<N, I>
where
N: NodeBuilderHelper<Payload = I>,
LocalPayloadAttributesBuilder<N::ChainSpec>: PayloadAttributesBuilder<
<<N as NodeTypes>::Payload as PayloadTypes>::PayloadAttributes,
>,
{
move |timestamp| {
let attributes = PayloadAttributes {

2
crates/engine/local/Cargo.toml
View File

@@ -15,6 +15,7 @@ reth-engine-primitives = { workspace = true, features = ["std"] }
reth-ethereum-engine-primitives.workspace = true
reth-payload-builder.workspace = true
reth-payload-primitives.workspace = true
reth-primitives-traits.workspace = true
reth-storage-api.workspace = true
reth-transaction-pool.workspace = true
@@ -43,4 +44,5 @@ op = [
"dep:op-alloy-rpc-types-engine",
"dep:reth-optimism-chainspec",
"reth-payload-primitives/op",
"reth-primitives-traits/op",
]

38
crates/engine/local/src/miner.rs
View File

@@ -1,6 +1,5 @@
//! Contains the implementation of the mining mode for the local engine.
use alloy_consensus::BlockHeader;
use alloy_primitives::{TxHash, B256};
use alloy_rpc_types_engine::ForkchoiceState;
use eyre::OptionExt;
@@ -10,6 +9,7 @@ use reth_payload_builder::PayloadBuilderHandle;
use reth_payload_primitives::{
BuiltPayload, EngineApiMessageVersion, PayloadAttributesBuilder, PayloadKind, PayloadTypes,
};
use reth_primitives_traits::{HeaderTy, SealedHeaderFor};
use reth_storage_api::BlockReader;
use reth_transaction_pool::TransactionPool;
use std::{
@@ -17,7 +17,7 @@ use std::{
future::Future,
pin::Pin,
task::{Context, Poll},
time::{Duration, UNIX_EPOCH},
time::Duration,
};
use tokio::time::Interval;
use tokio_stream::wrappers::ReceiverStream;
@@ -106,8 +106,8 @@ pub struct LocalMiner<T: PayloadTypes, B, Pool: TransactionPool + Unpin> {
mode: MiningMode<Pool>,
/// The payload builder for the engine
payload_builder: PayloadBuilderHandle<T>,
/// Timestamp for the next block.
last_timestamp: u64,
/// Latest block in the chain so far.
last_header: SealedHeaderFor<<T::BuiltPayload as BuiltPayload>::Primitives>,
/// Stores latest mined blocks.
last_block_hashes: VecDeque<B256>,
}
@@ -115,18 +115,21 @@ pub struct LocalMiner<T: PayloadTypes, B, Pool: TransactionPool + Unpin> {
impl<T, B, Pool> LocalMiner<T, B, Pool>
where
T: PayloadTypes,
B: PayloadAttributesBuilder<<T as PayloadTypes>::PayloadAttributes>,
B: PayloadAttributesBuilder<
T::PayloadAttributes,
HeaderTy<<T::BuiltPayload as BuiltPayload>::Primitives>,
>,
Pool: TransactionPool + Unpin,
{
/// Spawns a new [`LocalMiner`] with the given parameters.
pub fn new(
provider: impl BlockReader,
provider: impl BlockReader<Header = HeaderTy<<T::BuiltPayload as BuiltPayload>::Primitives>>,
payload_attributes_builder: B,
to_engine: ConsensusEngineHandle<T>,
mode: MiningMode<Pool>,
payload_builder: PayloadBuilderHandle<T>,
) -> Self {
let latest_header =
let last_header =
provider.sealed_header(provider.best_block_number().unwrap()).unwrap().unwrap();
Self {
@@ -134,8 +137,8 @@ where
to_engine,
mode,
payload_builder,
last_timestamp: latest_header.timestamp(),
last_block_hashes: VecDeque::from([latest_header.hash()]),
last_block_hashes: VecDeque::from([last_header.hash()]),
last_header,
}
}
@@ -193,19 +196,11 @@ where
/// Generates payload attributes for a new block, passes them to FCU and inserts built payload
/// through newPayload.
async fn advance(&mut self) -> eyre::Result<()> {
let timestamp = std::cmp::max(
self.last_timestamp.saturating_add(1),
std::time::SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("cannot be earlier than UNIX_EPOCH")
.as_secs(),
);
let res = self
.to_engine
.fork_choice_updated(
self.forkchoice_state(),
Some(self.payload_attributes_builder.build(timestamp)),
Some(self.payload_attributes_builder.build(&self.last_header)),
EngineApiMessageVersion::default(),
)
.await?;
@@ -222,8 +217,7 @@ where
eyre::bail!("No payload")
};
let block = payload.block();
let header = payload.block().sealed_header().clone();
let payload = T::block_to_payload(payload.block().clone());
let res = self.to_engine.new_payload(payload).await?;
@@ -231,8 +225,8 @@ where
eyre::bail!("Invalid payload")
}
self.last_timestamp = timestamp;
self.last_block_hashes.push_back(block.hash());
self.last_block_hashes.push_back(header.hash());
self.last_header = header;
// ensure we keep at most 64 blocks
if self.last_block_hashes.len() > 64 {
self.last_block_hashes.pop_front();

39
crates/engine/local/src/payload.rs
View File

@@ -1,10 +1,12 @@
//! The implementation of the [`PayloadAttributesBuilder`] for the
//! [`LocalMiner`](super::LocalMiner).
use alloy_consensus::BlockHeader;
use alloy_primitives::{Address, B256};
use reth_chainspec::EthereumHardforks;
use reth_chainspec::{EthChainSpec, EthereumHardforks};
use reth_ethereum_engine_primitives::EthPayloadAttributes;
use reth_payload_primitives::PayloadAttributesBuilder;
use reth_primitives_traits::SealedHeader;
use std::sync::Arc;
/// The attributes builder for local Ethereum payload.
@@ -13,21 +15,36 @@ use std::sync::Arc;
pub struct LocalPayloadAttributesBuilder<ChainSpec> {
/// The chainspec
pub chain_spec: Arc<ChainSpec>,
/// Whether to enforce increasing timestamp.
pub enforce_increasing_timestamp: bool,
}
impl<ChainSpec> LocalPayloadAttributesBuilder<ChainSpec> {
/// Creates a new instance of the builder.
pub const fn new(chain_spec: Arc<ChainSpec>) -> Self {
Self { chain_spec }
Self { chain_spec, enforce_increasing_timestamp: true }
}
/// Creates a new instance of the builder without enforcing increasing timestamps.
pub fn without_increasing_timestamp(self) -> Self {
Self { enforce_increasing_timestamp: false, ..self }
}
}
impl<ChainSpec> PayloadAttributesBuilder<EthPayloadAttributes>
impl<ChainSpec> PayloadAttributesBuilder<EthPayloadAttributes, ChainSpec::Header>
for LocalPayloadAttributesBuilder<ChainSpec>
where
ChainSpec: Send + Sync + EthereumHardforks + 'static,
ChainSpec: EthChainSpec + EthereumHardforks + 'static,
{
fn build(&self, timestamp: u64) -> EthPayloadAttributes {
fn build(&self, parent: &SealedHeader<ChainSpec::Header>) -> EthPayloadAttributes {
let mut timestamp =
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs();
if self.enforce_increasing_timestamp {
timestamp = std::cmp::max(parent.timestamp().saturating_add(1), timestamp);
}
EthPayloadAttributes {
timestamp,
prev_randao: B256::random(),
@@ -45,14 +62,18 @@ where
}
#[cfg(feature = "op")]
impl<ChainSpec> PayloadAttributesBuilder<op_alloy_rpc_types_engine::OpPayloadAttributes>
impl<ChainSpec>
PayloadAttributesBuilder<op_alloy_rpc_types_engine::OpPayloadAttributes, ChainSpec::Header>
for LocalPayloadAttributesBuilder<ChainSpec>
where
ChainSpec: Send + Sync + EthereumHardforks + 'static,
ChainSpec: EthChainSpec + EthereumHardforks + 'static,
{
fn build(&self, timestamp: u64) -> op_alloy_rpc_types_engine::OpPayloadAttributes {
fn build(
&self,
parent: &SealedHeader<ChainSpec::Header>,
) -> op_alloy_rpc_types_engine::OpPayloadAttributes {
op_alloy_rpc_types_engine::OpPayloadAttributes {
payload_attributes: self.build(timestamp),
payload_attributes: self.build(parent),
// Add dummy system transaction
transactions: Some(vec![
reth_optimism_chainspec::constants::TX_SET_L1_BLOCK_OP_MAINNET_BLOCK_124665056

2
crates/era-utils/src/history.rs
View File

@@ -9,7 +9,7 @@ use reth_db_api::{
RawKey, RawTable, RawValue,
};
use reth_era::{
common::{decode::DecodeCompressed, file_ops::StreamReader},
common::{decode::DecodeCompressedRlp, file_ops::StreamReader},
e2s::error::E2sError,
era1::{
file::{BlockTupleIterator, Era1Reader},

2
crates/era/src/common/decode.rs
View File

@@ -5,7 +5,7 @@ use alloy_rlp::Decodable;
use ssz::Decode;
/// Extension trait for generic decoding from compressed data
pub trait DecodeCompressed {
pub trait DecodeCompressedRlp {
/// Decompress and decode the data into the given type
fn decode<T: Decodable>(&self) -> Result<T, E2sError>;
}

342
crates/era/src/era/file.rs Normal file
View File

@@ -0,0 +1,342 @@
//! Represents a complete Era file
//!
//! The structure of an Era file follows the specification:
//! `Version | block* | era-state | other-entries* | slot-index(block)? | slot-index(state)`
//!
//! See also <https://github.com/eth-clients/e2store-format-specs/blob/main/formats/era.md>.
use crate::{
common::file_ops::{EraFileFormat, FileReader, StreamReader, StreamWriter},
e2s::{
error::E2sError,
file::{E2StoreReader, E2StoreWriter},
types::{Entry, IndexEntry, Version, SLOT_INDEX},
},
era::types::{
consensus::{
CompressedBeaconState, CompressedSignedBeaconBlock, COMPRESSED_BEACON_STATE,
COMPRESSED_SIGNED_BEACON_BLOCK,
},
group::{EraGroup, EraId, SlotIndex},
},
};
use std::{
fs::File,
io::{Read, Seek, Write},
};
/// Era file interface
#[derive(Debug)]
pub struct EraFile {
/// Version record, must be the first record in the file
pub version: Version,
/// Main content group of the Era file
pub group: EraGroup,
/// File identifier
pub id: EraId,
}
impl EraFileFormat for EraFile {
type EraGroup = EraGroup;
type Id = EraId;
/// Create a new [`EraFile`]
fn new(group: EraGroup, id: EraId) -> Self {
Self { version: Version, group, id }
}
fn version(&self) -> &Version {
&self.version
}
fn group(&self) -> &Self::EraGroup {
&self.group
}
fn id(&self) -> &Self::Id {
&self.id
}
}
/// Reader for era files that builds on top of [`E2StoreReader`]
#[derive(Debug)]
pub struct EraReader<R: Read> {
reader: E2StoreReader<R>,
}
/// An iterator of [`BeaconBlockIterator`] streaming from [`E2StoreReader`].
#[derive(Debug)]
pub struct BeaconBlockIterator<R: Read> {
reader: E2StoreReader<R>,
state: Option<CompressedBeaconState>,
other_entries: Vec<Entry>,
block_slot_index: Option<SlotIndex>,
state_slot_index: Option<SlotIndex>,
}
impl<R: Read> BeaconBlockIterator<R> {
fn new(reader: E2StoreReader<R>) -> Self {
Self {
reader,
state: None,
other_entries: Default::default(),
block_slot_index: None,
state_slot_index: None,
}
}
}
impl<R: Read + Seek> Iterator for BeaconBlockIterator<R> {
type Item = Result<CompressedSignedBeaconBlock, E2sError>;
fn next(&mut self) -> Option<Self::Item> {
self.next_result().transpose()
}
}
impl<R: Read + Seek> BeaconBlockIterator<R> {
fn next_result(&mut self) -> Result<Option<CompressedSignedBeaconBlock>, E2sError> {
loop {
let Some(entry) = self.reader.read_next_entry()? else {
return Ok(None);
};
match entry.entry_type {
COMPRESSED_SIGNED_BEACON_BLOCK => {
let block = CompressedSignedBeaconBlock::from_entry(&entry)?;
return Ok(Some(block));
}
COMPRESSED_BEACON_STATE => {
if self.state.is_some() {
return Err(E2sError::Ssz("Multiple state entries found".to_string()));
}
self.state = Some(CompressedBeaconState::from_entry(&entry)?);
}
SLOT_INDEX => {
let slot_index = SlotIndex::from_entry(&entry)?;
// if we haven't seen the state yet, the slot index is for blocks,
// if we have seen the state, the slot index is for the state
if self.state.is_none() {
self.block_slot_index = Some(slot_index);
} else {
self.state_slot_index = Some(slot_index);
}
}
_ => {
self.other_entries.push(entry);
}
}
}
}
}
impl<R: Read + Seek> StreamReader<R> for EraReader<R> {
type File = EraFile;
type Iterator = BeaconBlockIterator<R>;
/// Create a new [`EraReader`]
fn new(reader: R) -> Self {
Self { reader: E2StoreReader::new(reader) }
}
/// Returns an iterator of [`BeaconBlockIterator`] streaming from `reader`.
fn iter(self) -> BeaconBlockIterator<R> {
BeaconBlockIterator::new(self.reader)
}
fn read(self, network_name: String) -> Result<Self::File, E2sError> {
self.read_and_assemble(network_name)
}
}
impl<R: Read + Seek> EraReader<R> {
/// Reads and parses an era file from the underlying reader, assembling all components
/// into a complete [`EraFile`] with an [`EraId`] that includes the provided network name.
pub fn read_and_assemble(mut self, network_name: String) -> Result<EraFile, E2sError> {
// Validate version entry
let _version_entry = match self.reader.read_version()? {
Some(entry) if entry.is_version() => entry,
Some(_) => return Err(E2sError::Ssz("First entry is not a Version entry".to_string())),
None => return Err(E2sError::Ssz("Empty Era file".to_string())),
};
let mut iter = self.iter();
let blocks = (&mut iter).collect::<Result<Vec<_>, _>>()?;
let BeaconBlockIterator {
state, other_entries, block_slot_index, state_slot_index, ..
} = iter;
let state =
state.ok_or_else(|| E2sError::Ssz("Era file missing state entry".to_string()))?;
let state_slot_index = state_slot_index
.ok_or_else(|| E2sError::Ssz("Era file missing state slot index".to_string()))?;
// Create appropriate `EraGroup`, genesis vs non-genesis
let mut group = if let Some(block_index) = block_slot_index {
EraGroup::with_block_index(blocks, state, block_index, state_slot_index)
} else {
EraGroup::new(blocks, state, state_slot_index)
};
// Add other entries
for entry in other_entries {
group.add_entry(entry);
}
let (start_slot, slot_count) = group.slot_range();
let id = EraId::new(network_name, start_slot, slot_count);
Ok(EraFile::new(group, id))
}
}
impl FileReader for EraReader<File> {}
/// Writer for Era files that builds on top of [`E2StoreWriter`]
#[derive(Debug)]
pub struct EraWriter<W: Write> {
writer: E2StoreWriter<W>,
has_written_version: bool,
has_written_state: bool,
has_written_block_slot_index: bool,
has_written_state_slot_index: bool,
}
impl<W: Write> StreamWriter<W> for EraWriter<W> {
type File = EraFile;
/// Create a new [`EraWriter`]
fn new(writer: W) -> Self {
Self {
writer: E2StoreWriter::new(writer),
has_written_version: false,
has_written_state: false,
has_written_block_slot_index: false,
has_written_state_slot_index: false,
}
}
/// Write the version entry
fn write_version(&mut self) -> Result<(), E2sError> {
if self.has_written_version {
return Ok(());
}
self.writer.write_version()?;
self.has_written_version = true;
Ok(())
}
fn write_file(&mut self, file: &Self::File) -> Result<(), E2sError> {
// Write version
self.write_version()?;
// Write all blocks
for block in &file.group.blocks {
self.write_beacon_block(block)?;
}
// Write state
self.write_beacon_state(&file.group.era_state)?;
// Write other entries
for entry in &file.group.other_entries {
self.writer.write_entry(entry)?;
}
// Write slot index
if let Some(ref block_index) = file.group.slot_index {
self.write_block_slot_index(block_index)?;
}
// Write state index
self.write_state_slot_index(&file.group.state_slot_index)?;
self.writer.flush()?;
Ok(())
}
/// Flush any buffered data to the underlying writer
fn flush(&mut self) -> Result<(), E2sError> {
self.writer.flush()
}
}
impl<W: Write> EraWriter<W> {
/// Write beacon block
pub fn write_beacon_block(
&mut self,
block: &CompressedSignedBeaconBlock,
) -> Result<(), E2sError> {
self.ensure_version_written()?;
// Ensure blocks are written before state/indices
if self.has_written_state ||
self.has_written_block_slot_index ||
self.has_written_state_slot_index
{
return Err(E2sError::Ssz("Cannot write blocks after state or indices".to_string()));
}
let entry = block.to_entry();
self.writer.write_entry(&entry)?;
Ok(())
}
// Write beacon state
fn write_beacon_state(&mut self, state: &CompressedBeaconState) -> Result<(), E2sError> {
self.ensure_version_written()?;
if self.has_written_state {
return Err(E2sError::Ssz("State already written".to_string()));
}
let entry = state.to_entry();
self.writer.write_entry(&entry)?;
self.has_written_state = true;
Ok(())
}
/// Write the block slot index
pub fn write_block_slot_index(&mut self, slot_index: &SlotIndex) -> Result<(), E2sError> {
self.ensure_version_written()?;
if self.has_written_block_slot_index {
return Err(E2sError::Ssz("Block slot index already written".to_string()));
}
let entry = slot_index.to_entry();
self.writer.write_entry(&entry)?;
self.has_written_block_slot_index = true;
Ok(())
}
/// Write the state slot index
pub fn write_state_slot_index(&mut self, slot_index: &SlotIndex) -> Result<(), E2sError> {
self.ensure_version_written()?;
if self.has_written_state_slot_index {
return Err(E2sError::Ssz("State slot index already written".to_string()));
}
let entry = slot_index.to_entry();
self.writer.write_entry(&entry)?;
self.has_written_state_slot_index = true;
Ok(())
}
/// Helper to ensure version is written before any data
fn ensure_version_written(&mut self) -> Result<(), E2sError> {
if !self.has_written_version {
self.write_version()?;
}
Ok(())
}
}

1
crates/era/src/era/mod.rs
View File

@@ -1,3 +1,4 @@
//! Core era primitives.
pub mod file;
pub mod types;

113
crates/era/src/era/types/group.rs
View File

@@ -3,10 +3,14 @@
//! See also <https://github.com/eth-clients/e2store-format-specs/blob/main/formats/era.md>
use crate::{
common::file_ops::EraFileId,
e2s::types::{Entry, IndexEntry, SLOT_INDEX},
era::types::consensus::{CompressedBeaconState, CompressedSignedBeaconBlock},
};
/// Number of slots per historical root in ERA files
pub const SLOTS_PER_HISTORICAL_ROOT: u64 = 8192;
/// Era file content group
///
/// Format: `Version | block* | era-state | other-entries* | slot-index(block)? | slot-index(state)`
@@ -64,6 +68,28 @@ impl EraGroup {
pub fn add_entry(&mut self, entry: Entry) {
self.other_entries.push(entry);
}
/// Get the starting slot and slot count.
pub const fn slot_range(&self) -> (u64, u32) {
if let Some(ref block_index) = self.slot_index {
// Non-genesis era: use block slot index
(block_index.starting_slot, block_index.slot_count() as u32)
} else {
// Genesis era: use state slot index, it should be slot 0
// Genesis has only the genesis state, no blocks
(self.state_slot_index.starting_slot, 0)
}
}
/// Get the starting slot number
pub const fn starting_slot(&self) -> u64 {
self.slot_range().0
}
/// Get the number of slots
pub const fn slot_count(&self) -> u32 {
self.slot_range().1
}
}
/// [`SlotIndex`] records store offsets to data at specific slots
@@ -122,6 +148,93 @@ impl IndexEntry for SlotIndex {
}
}
/// Era file identifier
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct EraId {
/// Network configuration name
pub network_name: String,
/// First slot number in file
pub start_slot: u64,
/// Number of slots in the file
pub slot_count: u32,
/// Optional hash identifier for this file
/// First 4 bytes of the last historical root in the last state in the era file
pub hash: Option<[u8; 4]>,
}
impl EraId {
/// Create a new [`EraId`]
pub fn new(network_name: impl Into<String>, start_slot: u64, slot_count: u32) -> Self {
Self { network_name: network_name.into(), start_slot, slot_count, hash: None }
}
/// Add a hash identifier to [`EraId`]
pub const fn with_hash(mut self, hash: [u8; 4]) -> Self {
self.hash = Some(hash);
self
}
/// Calculate which era number the file starts at
pub const fn era_number(&self) -> u64 {
self.start_slot / SLOTS_PER_HISTORICAL_ROOT
}
// Helper function to calculate the number of eras per era1 file,
// If the user can decide how many blocks per era1 file there are, we need to calculate it.
// Most of the time it should be 1, but it can never be more than 2 eras per file
// as there is a maximum of 8192 blocks per era1 file.
const fn calculate_era_count(&self) -> u64 {
if self.slot_count == 0 {
return 0;
}
let first_era = self.era_number();
// Calculate the actual last slot number in the range
let last_slot = self.start_slot + self.slot_count as u64 - 1;
// Find which era the last block belongs to
let last_era = last_slot / SLOTS_PER_HISTORICAL_ROOT;
// Count how many eras we span
last_era - first_era + 1
}
}
impl EraFileId for EraId {
fn network_name(&self) -> &str {
&self.network_name
}
fn start_number(&self) -> u64 {
self.start_slot
}
fn count(&self) -> u32 {
self.slot_count
}
/// Convert to file name following the era file naming:
/// `<config-name>-<era-number>-<era-count>-<short-historical-root>.era`
/// <https://github.com/eth-clients/e2store-format-specs/blob/main/formats/era.md#file-name>
/// See also <https://github.com/eth-clients/e2store-format-specs/blob/main/formats/era.md>
fn to_file_name(&self) -> String {
let era_number = self.era_number();
let era_count = self.calculate_era_count();
if let Some(hash) = self.hash {
format!(
"{}-{:05}-{:05}-{:02x}{:02x}{:02x}{:02x}.era",
self.network_name, era_number, era_count, hash[0], hash[1], hash[2], hash[3]
)
} else {
// era spec format with placeholder hash when no hash available
// Format: `<config-name>-<era-number>-<era-count>-00000000.era`
format!("{}-{:05}-{:05}-00000000.era", self.network_name, era_number, era_count)
}
}
}
#[cfg(test)]
mod tests {
use super::*;

16
crates/era/src/era1/types/execution.rs
View File

@@ -16,7 +16,7 @@
//!
//! ```rust
//! use alloy_consensus::Header;
//! use reth_era::{common::decode::DecodeCompressed, era1::types::execution::CompressedHeader};
//! use reth_era::{common::decode::DecodeCompressedRlp, era1::types::execution::CompressedHeader};
//!
//! let header = Header { number: 100, ..Default::default() };
//! // Compress the header: rlp encoding and Snappy compression
@@ -32,7 +32,7 @@
//! ```rust
//! use alloy_consensus::{BlockBody, Header};
//! use alloy_primitives::Bytes;
//! use reth_era::{common::decode::DecodeCompressed, era1::types::execution::CompressedBody};
//! use reth_era::{common::decode::DecodeCompressedRlp, era1::types::execution::CompressedBody};
//! use reth_ethereum_primitives::TransactionSigned;
//!
//! let body: BlockBody<Bytes> = BlockBody {
@@ -53,7 +53,9 @@
//!
//! ```rust
//! use alloy_consensus::{Eip658Value, Receipt, ReceiptEnvelope, ReceiptWithBloom};
//! use reth_era::{common::decode::DecodeCompressed, era1::types::execution::CompressedReceipts};
//! use reth_era::{
//! common::decode::DecodeCompressedRlp, era1::types::execution::CompressedReceipts,
//! };
//!
//! let receipt =
//! Receipt { status: Eip658Value::Eip658(true), cumulative_gas_used: 21000, logs: vec![] };
@@ -68,7 +70,7 @@
//! ``````
use crate::{
common::decode::DecodeCompressed,
common::decode::DecodeCompressedRlp,
e2s::{error::E2sError, types::Entry},
};
use alloy_consensus::{Block, BlockBody, Header};
@@ -223,7 +225,7 @@ impl CompressedHeader {
}
}
impl DecodeCompressed for CompressedHeader {
impl DecodeCompressedRlp for CompressedHeader {
fn decode<T: Decodable>(&self) -> Result<T, E2sError> {
let decoder = SnappyRlpCodec::<T>::new();
decoder.decode(&self.data)
@@ -310,7 +312,7 @@ impl CompressedBody {
}
}
impl DecodeCompressed for CompressedBody {
impl DecodeCompressedRlp for CompressedBody {
fn decode<T: Decodable>(&self) -> Result<T, E2sError> {
let decoder = SnappyRlpCodec::<T>::new();
decoder.decode(&self.data)
@@ -401,7 +403,7 @@ impl CompressedReceipts {
}
}
impl DecodeCompressed for CompressedReceipts {
impl DecodeCompressedRlp for CompressedReceipts {
fn decode<T: Decodable>(&self) -> Result<T, E2sError> {
let decoder = SnappyRlpCodec::<T>::new();
decoder.decode(&self.data)

2
crates/era/src/era1/types/group.rs
View File

@@ -174,7 +174,7 @@ impl EraFileId for Era1Id {
mod tests {
use super::*;
use crate::{
common::decode::DecodeCompressed,
common::decode::DecodeCompressedRlp,
test_utils::{create_sample_block, create_test_block_with_compressed_data},
};
use alloy_consensus::ReceiptWithBloom;

190
crates/era/tests/it/era/dd.rs Normal file
View File

@@ -0,0 +1,190 @@
//! Simple decoding and decompressing tests
//! for mainnet era files
use reth_era::{
common::file_ops::{StreamReader, StreamWriter},
era::file::{EraReader, EraWriter},
};
use std::io::Cursor;
use crate::{EraTestDownloader, HOODI};
// Helper function to test decompression and decoding for a specific era file
async fn test_era_file_decompression_and_decoding(
downloader: &EraTestDownloader,
filename: &str,
network: &str,
) -> eyre::Result<()> {
println!("\nTesting file: {filename}");
let file = downloader.open_era_file(filename, network).await?;
// Handle genesis era separately
if file.group.is_genesis() {
// Genesis has no blocks
assert_eq!(file.group.blocks.len(), 0, "Genesis should have no blocks");
assert!(file.group.slot_index.is_none(), "Genesis should not have block slot index");
// Test genesis state decompression
let state_data = file.group.era_state.decompress()?;
assert!(!state_data.is_empty(), "Genesis state should decompress to non-empty data");
// Verify state slot index
assert_eq!(
file.group.state_slot_index.slot_count(),
1,
"Genesis state index should have count of 1"
);
let mut buffer = Vec::new();
{
let mut writer = EraWriter::new(&mut buffer);
writer.write_file(&file)?;
}
let reader = EraReader::new(Cursor::new(&buffer));
let read_back_file = reader.read(file.id.network_name.clone())?;
assert_eq!(
file.group.era_state.decompress()?,
read_back_file.group.era_state.decompress()?,
"Genesis state data should be identical"
);
println!("Genesis era verified successfully");
return Ok(());
}
// Non-genesis era - test beacon blocks
println!(
" Non-genesis era with {} beacon blocks, starting at slot {}",
file.group.blocks.len(),
file.group.starting_slot()
);
// Test beacon block decompression across different positions
let test_block_indices = [
0, // First block
file.group.blocks.len() / 2, // Middle block
file.group.blocks.len() - 1, // Last block
];
for &block_idx in &test_block_indices {
let block = &file.group.blocks[block_idx];
let slot = file.group.starting_slot() + block_idx as u64;
println!(
"\n Testing beacon block at slot {}, compressed size: {} bytes",
slot,
block.data.len()
);
// Test beacon block decompression
let block_data = block.decompress()?;
assert!(
!block_data.is_empty(),
"Beacon block at slot {slot} decompression should produce non-empty data"
);
}
// Test era state decompression
let state_data = file.group.era_state.decompress()?;
assert!(!state_data.is_empty(), "Era state decompression should produce non-empty data");
println!(" Era state decompressed: {} bytes", state_data.len());
// Verify slot indices
if let Some(ref block_slot_index) = file.group.slot_index {
println!(
" Block slot index: starting_slot={}, count={}",
block_slot_index.starting_slot,
block_slot_index.slot_count()
);
// Check for empty slots
let empty_slots: Vec<usize> = (0..block_slot_index.slot_count())
.filter(|&i| !block_slot_index.has_data_at_slot(i))
.collect();
if !empty_slots.is_empty() {
println!(
" Found {} empty slots (first few): {:?}",
empty_slots.len(),
&empty_slots[..empty_slots.len().min(5)]
);
}
}
// Test round-trip serialization
let mut buffer = Vec::new();
{
let mut writer = EraWriter::new(&mut buffer);
writer.write_file(&file)?;
}
// Read back from buffer
let reader = EraReader::new(Cursor::new(&buffer));
let read_back_file = reader.read(file.id.network_name.clone())?;
// Verify basic properties are preserved
assert_eq!(file.id.network_name, read_back_file.id.network_name);
assert_eq!(file.id.start_slot, read_back_file.id.start_slot);
assert_eq!(file.id.slot_count, read_back_file.id.slot_count);
assert_eq!(file.group.blocks.len(), read_back_file.group.blocks.len());
// Test data preservation for beacon blocks
for &idx in &test_block_indices {
let original_block = &file.group.blocks[idx];
let read_back_block = &read_back_file.group.blocks[idx];
let slot = file.group.starting_slot() + idx as u64;
// Test that decompressed data is identical
assert_eq!(
original_block.decompress()?,
read_back_block.decompress()?,
"Beacon block data should be identical for slot {slot}"
);
}
// Test state data preservation
assert_eq!(
file.group.era_state.decompress()?,
read_back_file.group.era_state.decompress()?,
"Era state data should be identical"
);
// Test slot indices preservation
if let (Some(original_index), Some(read_index)) =
(&file.group.slot_index, &read_back_file.group.slot_index)
{
assert_eq!(
original_index.starting_slot, read_index.starting_slot,
"Block slot index starting slot should match"
);
assert_eq!(
original_index.offsets, read_index.offsets,
"Block slot index offsets should match"
);
}
assert_eq!(
file.group.state_slot_index.starting_slot,
read_back_file.group.state_slot_index.starting_slot,
"State slot index starting slot should match"
);
assert_eq!(
file.group.state_slot_index.offsets, read_back_file.group.state_slot_index.offsets,
"State slot index offsets should match"
);
Ok(())
}
#[test_case::test_case("hoodi-00000-212f13fc.era"; "era_dd_hoodi_0")]
#[test_case::test_case("hoodi-00021-857e418b.era"; "era_dd_hoodi_21")]
#[test_case::test_case("hoodi-00175-202aaa6d.era"; "era_dd_hoodi_175")]
#[test_case::test_case("hoodi-00201-0d521fc8.era"; "era_dd_hoodi_201")]
#[tokio::test(flavor = "multi_thread")]
#[ignore = "download intensive"]
async fn test_hoodi_era1_file_decompression_and_decoding(filename: &str) -> eyre::Result<()> {
let downloader = EraTestDownloader::new().await?;
test_era_file_decompression_and_decoding(&downloader, filename, HOODI).await
}

37
crates/era/tests/it/era/genesis.rs Normal file
View File

@@ -0,0 +1,37 @@
//! Genesis block tests for `era1` files.
//!
//! These tests verify proper decompression and decoding of genesis blocks
//! from different networks.
use crate::{EraTestDownloader, ERA_HOODI_FILES_NAMES, HOODI};
#[tokio::test(flavor = "multi_thread")]
#[ignore = "download intensive"]
async fn test_hoodi_genesis_era_decompression() -> eyre::Result<()> {
let downloader = EraTestDownloader::new().await?;
let file = downloader.open_era_file(ERA_HOODI_FILES_NAMES[0], HOODI).await?;
// Verify this is genesis era
assert!(file.group.is_genesis(), "First file should be genesis era");
assert_eq!(file.group.starting_slot(), 0, "Genesis should start at slot 0");
// Genesis era has no blocks
assert_eq!(file.group.blocks.len(), 0, "Genesis era should have no blocks");
// Genesis should not have block slot index
assert!(file.group.slot_index.is_none(), "Genesis should not have block slot index");
// Test state decompression
let state_data = file.group.era_state.decompress()?;
assert!(!state_data.is_empty(), "Decompressed state should not be empty");
// Verify state slot index
assert_eq!(
file.group.state_slot_index.slot_count(),
1,
"Genesis state index should have count of 1"
);
Ok(())
}

2
crates/era/tests/it/era/mod.rs Normal file
View File

@@ -0,0 +1,2 @@
mod dd;
mod genesis;

8
crates/era/tests/it/dd.rs → crates/era/tests/it/era1/dd.rs
View File

@@ -14,11 +14,11 @@ use reth_era::{
use reth_ethereum_primitives::TransactionSigned;
use std::io::Cursor;
use crate::{Era1TestDownloader, MAINNET};
use crate::{EraTestDownloader, MAINNET};
// Helper function to test decompression and decoding for a specific file
// Helper function to test decompression and decoding for a specific era1 file
async fn test_file_decompression(
downloader: &Era1TestDownloader,
downloader: &EraTestDownloader,
filename: &str,
) -> eyre::Result<()> {
println!("\nTesting file: {filename}");
@@ -154,6 +154,6 @@ async fn test_file_decompression(
#[tokio::test(flavor = "multi_thread")]
#[ignore = "download intensive"]
async fn test_mainnet_era1_file_decompression_and_decoding(filename: &str) -> eyre::Result<()> {
let downloader = Era1TestDownloader::new().await?;
let downloader = EraTestDownloader::new().await?;
test_file_decompression(&downloader, filename).await
}

6
crates/era/tests/it/genesis.rs → crates/era/tests/it/era1/genesis.rs
View File

@@ -4,7 +4,7 @@
//! from different networks.
use crate::{
Era1TestDownloader, ERA1_MAINNET_FILES_NAMES, ERA1_SEPOLIA_FILES_NAMES, MAINNET, SEPOLIA,
EraTestDownloader, ERA1_MAINNET_FILES_NAMES, ERA1_SEPOLIA_FILES_NAMES, MAINNET, SEPOLIA,
};
use alloy_consensus::{BlockBody, Header};
use reth_era::{e2s::types::IndexEntry, era1::types::execution::CompressedBody};
@@ -13,7 +13,7 @@ use reth_ethereum_primitives::TransactionSigned;
#[tokio::test(flavor = "multi_thread")]
#[ignore = "download intensive"]
async fn test_mainnet_genesis_block_decompression() -> eyre::Result<()> {
let downloader = Era1TestDownloader::new().await?;
let downloader = EraTestDownloader::new().await?;
let file = downloader.open_era1_file(ERA1_MAINNET_FILES_NAMES[0], MAINNET).await?;
@@ -65,7 +65,7 @@ async fn test_mainnet_genesis_block_decompression() -> eyre::Result<()> {
#[tokio::test(flavor = "multi_thread")]
#[ignore = "download intensive"]
async fn test_sepolia_genesis_block_decompression() -> eyre::Result<()> {
let downloader = Era1TestDownloader::new().await?;
let downloader = EraTestDownloader::new().await?;
let file = downloader.open_era1_file(ERA1_SEPOLIA_FILES_NAMES[0], SEPOLIA).await?;

3
crates/era/tests/it/era1/mod.rs Normal file
View File

@@ -0,0 +1,3 @@
mod dd;
mod genesis;
mod roundtrip;

8
crates/era/tests/it/roundtrip.rs → crates/era/tests/it/era1/roundtrip.rs
View File

@@ -24,11 +24,11 @@ use reth_era::{
use reth_ethereum_primitives::TransactionSigned;
use std::io::Cursor;
use crate::{Era1TestDownloader, MAINNET, SEPOLIA};
use crate::{EraTestDownloader, MAINNET, SEPOLIA};
// Helper function to test roundtrip compression/encoding for a specific file
async fn test_file_roundtrip(
downloader: &Era1TestDownloader,
downloader: &EraTestDownloader,
filename: &str,
network: &str,
) -> eyre::Result<()> {
@@ -259,7 +259,7 @@ async fn test_file_roundtrip(
#[tokio::test(flavor = "multi_thread")]
#[ignore = "download intensive"]
async fn test_roundtrip_compression_encoding_mainnet(filename: &str) -> eyre::Result<()> {
let downloader = Era1TestDownloader::new().await?;
let downloader = EraTestDownloader::new().await?;
test_file_roundtrip(&downloader, filename, MAINNET).await
}
@@ -270,7 +270,7 @@ async fn test_roundtrip_compression_encoding_mainnet(filename: &str) -> eyre::Re
#[tokio::test(flavor = "multi_thread")]
#[ignore = "download intensive"]
async fn test_roundtrip_compression_encoding_sepolia(filename: &str) -> eyre::Result<()> {
let downloader = Era1TestDownloader::new().await?;
let downloader = EraTestDownloader::new().await?;
test_file_roundtrip(&downloader, filename, SEPOLIA).await?;

118
crates/era/tests/it/main.rs
View File

@@ -8,8 +8,9 @@
use reqwest::{Client, Url};
use reth_era::{
common::file_ops::FileReader,
common::file_ops::{EraFileType, FileReader},
e2s::error::E2sError,
era::file::{EraFile, EraReader},
era1::file::{Era1File, Era1Reader},
};
use reth_era_downloader::EraClient;
@@ -23,9 +24,8 @@ use std::{
use eyre::{eyre, Result};
use tempfile::TempDir;
mod dd;
mod genesis;
mod roundtrip;
mod era;
mod era1;
const fn main() {}
@@ -33,7 +33,7 @@ const fn main() {}
const MAINNET: &str = "mainnet";
/// Default mainnet url
/// for downloading mainnet `.era1` files
const MAINNET_URL: &str = "https://era.ithaca.xyz/era1/";
const ERA1_MAINNET_URL: &str = "https://era.ithaca.xyz/era1/";
/// Succinct list of mainnet files we want to download
/// from <https://era.ithaca.xyz/era1/>
@@ -54,7 +54,7 @@ const SEPOLIA: &str = "sepolia";
/// Default sepolia url
/// for downloading sepolia `.era1` files
const SEPOLIA_URL: &str = "https://era.ithaca.xyz/sepolia-era1/";
const ERA1_SEPOLIA_URL: &str = "https://era.ithaca.xyz/sepolia-era1/";
/// Succinct list of sepolia files we want to download
/// from <https://era.ithaca.xyz/sepolia-era1/>
@@ -66,18 +66,50 @@ const ERA1_SEPOLIA_FILES_NAMES: [&str; 4] = [
"sepolia-00182-a4f0a8a1.era1",
];
const HOODI: &str = "hoodi";
/// Default hoodi url
/// for downloading hoodi `.era` files
/// TODO: to replace with internal era files hosting url
const ERA_HOODI_URL: &str = "https://hoodi.era.nimbus.team/";
/// Succinct list of hoodi files we want to download
/// from <https://hoodi.era.nimbus.team/> //TODO: to replace with internal era files hosting url
/// for testing purposes
const ERA_HOODI_FILES_NAMES: [&str; 4] = [
"hoodi-00000-212f13fc.era",
"hoodi-00021-857e418b.era",
"hoodi-00175-202aaa6d.era",
"hoodi-00201-0d521fc8.era",
];
/// Default mainnet url
/// for downloading mainnet `.era` files
//TODO: to replace with internal era files hosting url
const ERA_MAINNET_URL: &str = "https://mainnet.era.nimbus.team/";
/// Succinct list of mainnet files we want to download
/// from <https://mainnet.era.nimbus.team/> //TODO: to replace with internal era files hosting url
/// for testing purposes
const ERA_MAINNET_FILES_NAMES: [&str; 4] = [
"mainnet-00000-4b363db9.era",
"mainnet-00518-4e267a3a.era",
"mainnet-01140-f70d4869.era",
"mainnet-01581-82073d28.era",
];
/// Utility for downloading `.era1` files for tests
/// in a temporary directory
/// and caching them in memory
#[derive(Debug)]
struct Era1TestDownloader {
struct EraTestDownloader {
/// Temporary directory for storing downloaded files
temp_dir: TempDir,
/// Cache mapping file names to their paths
file_cache: Arc<Mutex<HashMap<String, PathBuf>>>,
}
impl Era1TestDownloader {
impl EraTestDownloader {
/// Create a new downloader instance with a temporary directory
async fn new() -> Result<Self> {
let temp_dir =
@@ -97,29 +129,9 @@ impl Era1TestDownloader {
}
// check if the filename is supported
if !ERA1_MAINNET_FILES_NAMES.contains(&filename) &&
!ERA1_SEPOLIA_FILES_NAMES.contains(&filename)
{
return Err(eyre!(
"Unknown file: {}. Only the following files are supported: {:?} or {:?}",
filename,
ERA1_MAINNET_FILES_NAMES,
ERA1_SEPOLIA_FILES_NAMES
));
}
// initialize the client and build url config
let url = match network {
MAINNET => MAINNET_URL,
SEPOLIA => SEPOLIA_URL,
_ => {
return Err(eyre!(
"Unknown network: {}. Only mainnet and sepolia are supported.",
network
));
}
};
self.validate_filename(filename, network)?;
let (url, _): (&str, &[&str]) = self.get_network_config(filename, network)?;
let final_url = Url::from_str(url).map_err(|e| eyre!("Failed to parse URL: {}", e))?;
let folder = self.temp_dir.path();
@@ -142,6 +154,7 @@ impl Era1TestDownloader {
.download_to_file(file_url)
.await
.map_err(|e| eyre!("Failed to download file: {}", e))?;
// update the cache
{
let mut cache = self.file_cache.lock().unwrap();
@@ -151,9 +164,54 @@ impl Era1TestDownloader {
Ok(downloaded_path.to_path_buf())
}
/// Validate that filename is in the supported list for the network
fn validate_filename(&self, filename: &str, network: &str) -> Result<()> {
let (_, supported_files) = self.get_network_config(filename, network)?;
if !supported_files.contains(&filename) {
return Err(eyre!(
"Unknown file: '{}' for network '{}'. Supported files: {:?}",
filename,
network,
supported_files
));
}
Ok(())
}
/// Get network configuration, URL and supported files, based on network and file type
fn get_network_config(
&self,
filename: &str,
network: &str,
) -> Result<(&'static str, &'static [&'static str])> {
let file_type = EraFileType::from_filename(filename)
.ok_or_else(|| eyre!("Unknown file extension for: {}", filename))?;
match (network, file_type) {
(MAINNET, EraFileType::Era1) => Ok((ERA1_MAINNET_URL, &ERA1_MAINNET_FILES_NAMES[..])),
(MAINNET, EraFileType::Era) => Ok((ERA_MAINNET_URL, &ERA_MAINNET_FILES_NAMES[..])),
(SEPOLIA, EraFileType::Era1) => Ok((ERA1_SEPOLIA_URL, &ERA1_SEPOLIA_FILES_NAMES[..])),
(HOODI, EraFileType::Era) => Ok((ERA_HOODI_URL, &ERA_HOODI_FILES_NAMES[..])),
_ => Err(eyre!(
"Unsupported combination: network '{}' with file type '{:?}'",
network,
file_type
)),
}
}
/// open .era1 file, downloading it if necessary
async fn open_era1_file(&self, filename: &str, network: &str) -> Result<Era1File> {
let path = self.download_file(filename, network).await?;
Era1Reader::open(&path, network).map_err(|e| eyre!("Failed to open Era1 file: {e}"))
}
/// open .era file, downloading it if necessary
#[allow(dead_code)]
async fn open_era_file(&self, filename: &str, network: &str) -> Result<EraFile> {
let path = self.download_file(filename, network).await?;
EraReader::open(&path, network).map_err(|e| eyre!("Failed to open Era1 file: {e}"))
}
}

22
crates/ethereum/cli/src/app.rs
View File

@@ -10,13 +10,11 @@ use reth_cli_runner::CliRunner;
use reth_db::DatabaseEnv;
use reth_node_api::NodePrimitives;
use reth_node_builder::{NodeBuilder, WithLaunchContext};
use reth_node_core::args::OtlpInitStatus;
use reth_node_ethereum::{consensus::EthBeaconConsensus, EthEvmConfig, EthereumNode};
use reth_node_metrics::recorder::install_prometheus_recorder;
use reth_rpc_server_types::RpcModuleValidator;
use reth_tracing::{FileWorkerGuard, Layers};
use std::{fmt, sync::Arc};
use tracing::{info, warn};
/// A wrapper around a parsed CLI that handles command execution.
#[derive(Debug)]
@@ -107,26 +105,12 @@ where
/// Initializes tracing with the configured options.
///
/// If file logging is enabled, this function stores guard to the struct.
/// For gRPC OTLP, it requires tokio runtime context.
/// See [`Cli::init_tracing`] for more information.
pub fn init_tracing(&mut self, runner: &CliRunner) -> Result<()> {
if self.guard.is_none() {
let mut layers = self.layers.take().unwrap_or_default();
let otlp_status = runner.block_on(self.cli.traces.init_otlp_tracing(&mut layers))?;
self.guard = self.cli.logs.init_tracing_with_layers(layers)?;
info!(target: "reth::cli", "Initialized tracing, debug log directory: {}", self.cli.logs.log_file_directory);
match otlp_status {
OtlpInitStatus::Started(endpoint) => {
info!(target: "reth::cli", "Started OTLP {:?} tracing export to {endpoint}", self.cli.traces.protocol);
}
OtlpInitStatus::NoFeature => {
warn!(target: "reth::cli", "Provided OTLP tracing arguments do not have effect, compile with the `otlp` feature")
}
OtlpInitStatus::Disabled => {}
}
self.guard = self.cli.init_tracing(runner, self.layers.take().unwrap_or_default())?;
}
Ok(())
}
}

29
crates/ethereum/cli/src/interface.rs
View File

@@ -19,14 +19,14 @@ use reth_db::DatabaseEnv;
use reth_node_api::NodePrimitives;
use reth_node_builder::{NodeBuilder, WithLaunchContext};
use reth_node_core::{
args::{LogArgs, TraceArgs},
args::{LogArgs, OtlpInitStatus, TraceArgs},
version::version_metadata,
};
use reth_node_metrics::recorder::install_prometheus_recorder;
use reth_rpc_server_types::{DefaultRpcModuleValidator, RpcModuleValidator};
use reth_tracing::FileWorkerGuard;
use reth_tracing::{FileWorkerGuard, Layers};
use std::{ffi::OsString, fmt, future::Future, marker::PhantomData, sync::Arc};
use tracing::info;
use tracing::{info, warn};
/// The main reth cli interface.
///
@@ -205,8 +205,7 @@ impl<C: ChainSpecParser, Ext: clap::Args + fmt::Debug, Rpc: RpcModuleValidator>
self.logs.log_file_directory =
self.logs.log_file_directory.join(chain_spec.chain().to_string());
}
let _guard = self.init_tracing()?;
info!(target: "reth::cli", "Initialized tracing, debug log directory: {}", self.logs.log_file_directory);
let _guard = self.init_tracing(&runner, Layers::new())?;
// Install the prometheus recorder to be sure to record all metrics
let _ = install_prometheus_recorder();
@@ -219,11 +218,27 @@ impl<C: ChainSpecParser, Ext: clap::Args + fmt::Debug, Rpc: RpcModuleValidator>
///
/// If file logging is enabled, this function returns a guard that must be kept alive to ensure
/// that all logs are flushed to disk.
///
/// If an OTLP endpoint is specified, it will export metrics to the configured collector.
pub fn init_tracing(&self) -> eyre::Result<Option<FileWorkerGuard>> {
let layers = reth_tracing::Layers::new();
pub fn init_tracing(
&mut self,
runner: &CliRunner,
mut layers: Layers,
) -> eyre::Result<Option<FileWorkerGuard>> {
let otlp_status = runner.block_on(self.traces.init_otlp_tracing(&mut layers))?;
let guard = self.logs.init_tracing_with_layers(layers)?;
info!(target: "reth::cli", "Initialized tracing, debug log directory: {}", self.logs.log_file_directory);
match otlp_status {
OtlpInitStatus::Started(endpoint) => {
info!(target: "reth::cli", "Started OTLP {:?} tracing export to {endpoint}", self.traces.protocol);
}
OtlpInitStatus::NoFeature => {
warn!(target: "reth::cli", "Provided OTLP tracing arguments do not have effect, compile with the `otlp` feature")
}
OtlpInitStatus::Disabled => {}
}
Ok(guard)
}
}

105
crates/ethereum/node/tests/e2e/p2p.rs
View File

@@ -1,10 +1,18 @@
use crate::utils::{advance_with_random_transactions, eth_payload_attributes};
use alloy_consensus::{SignableTransaction, TxEip1559, TxEnvelope};
use alloy_eips::Encodable2718;
use alloy_network::TxSignerSync;
use alloy_provider::{Provider, ProviderBuilder};
use rand::{rngs::StdRng, Rng, SeedableRng};
use futures::future::JoinAll;
use rand::{rngs::StdRng, seq::IndexedRandom, Rng, SeedableRng};
use reth_chainspec::{ChainSpecBuilder, MAINNET};
use reth_e2e_test_utils::{setup, setup_engine, transaction::TransactionTestContext};
use reth_e2e_test_utils::{
setup, setup_engine, setup_engine_with_connection, transaction::TransactionTestContext,
wallet::Wallet,
};
use reth_node_ethereum::EthereumNode;
use std::sync::Arc;
use reth_rpc_api::EthApiServer;
use std::{sync::Arc, time::Duration};
#[tokio::test]
async fn can_sync() -> eyre::Result<()> {
@@ -195,3 +203,94 @@ async fn test_reorg_through_backfill() -> eyre::Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn test_tx_propagation() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
.cancun_activated()
.prague_activated()
.build(),
);
// Setup wallet
let chain_id = chain_spec.chain().into();
let wallet = Wallet::new(1).inner;
let mut nonce = 0;
let mut build_tx = || {
let mut tx = TxEip1559 {
chain_id,
max_priority_fee_per_gas: 1_000_000_000,
max_fee_per_gas: 1_000_000_000,
gas_limit: 100_000,
nonce,
..Default::default()
};
nonce += 1;
let signature = wallet.sign_transaction_sync(&mut tx).unwrap();
TxEnvelope::Eip1559(tx.into_signed(signature))
};
// Setup 10 nodes
let (mut nodes, _tasks, _) = setup_engine_with_connection::<EthereumNode>(
10,
chain_spec.clone(),
false,
Default::default(),
eth_payload_attributes,
false,
)
.await?;
// Connect all nodes to the first one
let (first, rest) = nodes.split_at_mut(1);
for node in rest {
node.connect(&mut first[0]).await;
}
// Advance all nodes for 1 block so that they don't consider themselves unsynced
let tx = build_tx();
nodes[0].rpc.inject_tx(tx.encoded_2718().into()).await?;
let payload = nodes[0].advance_block().await?;
nodes[1..]
.iter_mut()
.map(|node| async {
node.submit_payload(payload.clone()).await.unwrap();
node.sync_to(payload.block().hash()).await.unwrap();
})
.collect::<JoinAll<_>>()
.await;
// Build and send transaction to first node
let tx = build_tx();
let tx_hash = *tx.tx_hash();
let _ = nodes[0].rpc.inject_tx(tx.encoded_2718().into()).await?;
tokio::time::sleep(Duration::from_millis(100)).await;
// Assert that all nodes have the transaction
for (i, node) in nodes.iter().enumerate() {
assert!(
node.rpc.inner.eth_api().transaction_by_hash(tx_hash).await?.is_some(),
"Node {i} should have the transaction"
);
}
// Build and send one more transaction to a random node
let tx = build_tx();
let tx_hash = *tx.tx_hash();
let _ = nodes.choose(&mut rand::rng()).unwrap().rpc.inject_tx(tx.encoded_2718().into()).await?;
tokio::time::sleep(Duration::from_millis(100)).await;
// Assert that all nodes have the transaction
for node in nodes {
assert!(node.rpc.inner.eth_api().transaction_by_hash(tx_hash).await?.is_some());
}
Ok(())
}

16
crates/net/eth-wire-types/src/capability.rs
View File

@@ -158,6 +158,15 @@ pub struct Capabilities {
}
impl Capabilities {
/// Create a new instance from the given vec.
pub fn new(value: Vec<Capability>) -> Self {
Self {
eth_66: value.iter().any(Capability::is_eth_v66),
eth_67: value.iter().any(Capability::is_eth_v67),
eth_68: value.iter().any(Capability::is_eth_v68),
inner: value,
}
}
/// Returns all capabilities.
#[inline]
pub fn capabilities(&self) -> &[Capability] {
@@ -197,12 +206,7 @@ impl Capabilities {
impl From<Vec<Capability>> for Capabilities {
fn from(value: Vec<Capability>) -> Self {
Self {
eth_66: value.iter().any(Capability::is_eth_v66),
eth_67: value.iter().any(Capability::is_eth_v67),
eth_68: value.iter().any(Capability::is_eth_v68),
inner: value,
}
Self::new(value)
}
}

39
crates/net/network/src/error.rs
View File

@@ -113,7 +113,22 @@ impl SessionError for EthStreamError {
P2PHandshakeError::HelloNotInHandshake |
P2PHandshakeError::NonHelloMessageInHandshake,
)) => true,
Self::EthHandshakeError(err) => !matches!(err, EthHandshakeError::NoResponse),
Self::EthHandshakeError(err) => {
#[allow(clippy::match_same_arms)]
match err {
EthHandshakeError::NoResponse => {
// this happens when the conn simply stalled
false
}
EthHandshakeError::InvalidFork(_) => {
// this can occur when the remote or our node is running an outdated client,
// we shouldn't treat this as fatal, because the node can come back online
// with an updated version any time
false
}
_ => true,
}
}
_ => false,
}
}
@@ -144,7 +159,22 @@ impl SessionError for EthStreamError {
P2PStreamError::MismatchedProtocolVersion { .. }
)
}
Self::EthHandshakeError(err) => !matches!(err, EthHandshakeError::NoResponse),
Self::EthHandshakeError(err) => {
#[allow(clippy::match_same_arms)]
match err {
EthHandshakeError::NoResponse => {
// this happens when the conn simply stalled
false
}
EthHandshakeError::InvalidFork(_) => {
// this can occur when the remote or our node is running an outdated client,
// we shouldn't treat this as fatal, because the node can come back online
// with an updated version any time
false
}
_ => true,
}
}
_ => false,
}
}
@@ -196,6 +226,11 @@ impl SessionError for EthStreamError {
P2PStreamError::PingerError(_) |
P2PStreamError::Snap(_),
) => Some(BackoffKind::Medium),
Self::EthHandshakeError(EthHandshakeError::InvalidFork(_)) => {
// the remote can come back online after updating client version, so we can back off
// for a bit
Some(BackoffKind::Medium)
}
_ => None,
}
}

492
crates/net/network/src/fetch/mod.rs
View File

@@ -139,8 +139,9 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
/// Returns the _next_ idle peer that's ready to accept a request,
/// prioritizing those with the lowest timeout/latency and those that recently responded with
/// adequate data.
fn next_best_peer(&self) -> Option<PeerId> {
/// adequate data. Additionally, if full blocks are required this prioritizes peers that have
/// full history available
fn next_best_peer(&self, requirement: BestPeerRequirements) -> Option<PeerId> {
let mut idle = self.peers.iter().filter(|(_, peer)| peer.state.is_idle());
let mut best_peer = idle.next()?;
@@ -152,7 +153,13 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
continue
}
// replace best peer if this peer has better rtt
// replace best peer if this peer meets the requirements better
if maybe_better.1.is_better(best_peer.1, &requirement) {
best_peer = maybe_better;
continue
}
// replace best peer if this peer has better rtt and both have same range quality
if maybe_better.1.timeout() < best_peer.1.timeout() &&
!maybe_better.1.last_response_likely_bad
{
@@ -170,9 +177,13 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
return PollAction::NoRequests
}
let Some(peer_id) = self.next_best_peer() else { return PollAction::NoPeersAvailable };
let request = self.queued_requests.pop_front().expect("not empty");
let Some(peer_id) = self.next_best_peer(request.best_peer_requirements()) else {
// need to put back the the request
self.queued_requests.push_front(request);
return PollAction::NoPeersAvailable
};
let request = self.prepare_block_request(peer_id, request);
PollAction::Ready(FetchAction::BlockRequest { peer_id, request })
@@ -358,7 +369,6 @@ struct Peer {
/// lowest timeout.
last_response_likely_bad: bool,
/// Tracks the range info for the peer.
#[allow(dead_code)]
range_info: Option<BlockRangeInfo>,
}
@@ -366,6 +376,74 @@ impl Peer {
fn timeout(&self) -> u64 {
self.timeout.load(Ordering::Relaxed)
}
/// Returns the earliest block number available from the peer.
fn earliest(&self) -> u64 {
self.range_info.as_ref().map_or(0, |info| info.earliest())
}
/// Returns true if the peer has the full history available.
fn has_full_history(&self) -> bool {
self.earliest() == 0
}
fn range(&self) -> Option<RangeInclusive<u64>> {
self.range_info.as_ref().map(|info| info.range())
}
/// Returns true if this peer has a better range than the other peer for serving the requested
/// range.
///
/// A peer has a "better range" if:
/// 1. It can fully cover the requested range while the other cannot
/// 2. None can fully cover the range, but this peer has lower start value
/// 3. If a peer doesnt announce a range we assume it has full history, but check the other's
/// range and treat that as better if it can cover the range
fn has_better_range(&self, other: &Self, range: RangeInclusive<u64>) -> bool {
let self_range = self.range();
let other_range = other.range();
match (self_range, other_range) {
(Some(self_r), Some(other_r)) => {
// Check if each peer can fully cover the requested range
let self_covers = self_r.contains(range.start()) && self_r.contains(range.end());
let other_covers = other_r.contains(range.start()) && other_r.contains(range.end());
#[allow(clippy::match_same_arms)]
match (self_covers, other_covers) {
(true, false) => true, // Only self covers the range
(false, true) => false, // Only other covers the range
(true, true) => false, // Both cover
(false, false) => {
// neither covers - prefer if peer has lower (better) start range
self_r.start() < other_r.start()
}
}
}
(Some(self_r), None) => {
// Self has range info, other doesn't (treated as full history with unknown latest)
// Self is better only if it covers the range
self_r.contains(range.start()) && self_r.contains(range.end())
}
(None, Some(other_r)) => {
// Self has no range info (full history), other has range info
// Self is better only if other doesn't cover the range
!(other_r.contains(range.start()) && other_r.contains(range.end()))
}
(None, None) => false, // Neither has range info - no one is better
}
}
/// Returns true if this peer is better than the other peer based on the given requirements.
fn is_better(&self, other: &Self, requirement: &BestPeerRequirements) -> bool {
match requirement {
BestPeerRequirements::None => false,
BestPeerRequirements::FullBlockRange(range) => {
self.has_better_range(other, range.clone())
}
BestPeerRequirements::FullBlock => self.has_full_history() && !other.has_full_history(),
}
}
}
/// Tracks the state of an individual peer
@@ -427,7 +505,6 @@ pub(crate) enum DownloadRequest<N: NetworkPrimitives> {
request: Vec<B256>,
response: oneshot::Sender<PeerRequestResult<Vec<N::BlockBody>>>,
priority: Priority,
#[allow(dead_code)]
range_hint: Option<RangeInclusive<u64>>,
},
}
@@ -456,6 +533,20 @@ impl<N: NetworkPrimitives> DownloadRequest<N> {
const fn is_normal_priority(&self) -> bool {
self.get_priority().is_normal()
}
/// Returns the best peer requirements for this request.
fn best_peer_requirements(&self) -> BestPeerRequirements {
match self {
Self::GetBlockHeaders { .. } => BestPeerRequirements::None,
Self::GetBlockBodies { range_hint, .. } => {
if let Some(range) = range_hint {
BestPeerRequirements::FullBlockRange(range.clone())
} else {
BestPeerRequirements::FullBlock
}
}
}
}
}
/// An action the syncer can emit.
@@ -480,6 +571,16 @@ pub(crate) enum BlockResponseOutcome {
BadResponse(PeerId, ReputationChangeKind),
}
/// Additional requirements for how to rank peers during selection.
enum BestPeerRequirements {
/// No additional requirements
None,
/// Peer must have this block range available.
FullBlockRange(RangeInclusive<u64>),
/// Peer must have full range.
FullBlock,
}
#[cfg(test)]
mod tests {
use super::*;
@@ -536,17 +637,17 @@ mod tests {
None,
);
let first_peer = fetcher.next_best_peer().unwrap();
let first_peer = fetcher.next_best_peer(BestPeerRequirements::None).unwrap();
assert!(first_peer == peer1 || first_peer == peer2);
// Pending disconnect for first_peer
fetcher.on_pending_disconnect(&first_peer);
// first_peer now isn't idle, so we should get other peer
let second_peer = fetcher.next_best_peer().unwrap();
let second_peer = fetcher.next_best_peer(BestPeerRequirements::None).unwrap();
assert!(first_peer == peer1 || first_peer == peer2);
assert_ne!(first_peer, second_peer);
// without idle peers, returns None
fetcher.on_pending_disconnect(&second_peer);
assert_eq!(fetcher.next_best_peer(), None);
assert_eq!(fetcher.next_best_peer(BestPeerRequirements::None), None);
}
#[tokio::test]
@@ -588,13 +689,13 @@ mod tests {
);
// Must always get peer1 (lowest timeout)
assert_eq!(fetcher.next_best_peer(), Some(peer1));
assert_eq!(fetcher.next_best_peer(), Some(peer1));
assert_eq!(fetcher.next_best_peer(BestPeerRequirements::None), Some(peer1));
assert_eq!(fetcher.next_best_peer(BestPeerRequirements::None), Some(peer1));
// peer2's timeout changes below peer1's
peer2_timeout.store(10, Ordering::Relaxed);
// Then we get peer 2 always (now lowest)
assert_eq!(fetcher.next_best_peer(), Some(peer2));
assert_eq!(fetcher.next_best_peer(), Some(peer2));
assert_eq!(fetcher.next_best_peer(BestPeerRequirements::None), Some(peer2));
assert_eq!(fetcher.next_best_peer(BestPeerRequirements::None), Some(peer2));
}
#[tokio::test]
@@ -684,4 +785,367 @@ mod tests {
assert!(fetcher.peers[&peer_id].state.is_idle());
}
#[test]
fn test_peer_is_better_none_requirement() {
let peer1 = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: Some(BlockRangeInfo::new(0, 100, B256::random())),
};
let peer2 = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 50,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(20)),
last_response_likely_bad: false,
range_info: None,
};
// With None requirement, is_better should always return false
assert!(!peer1.is_better(&peer2, &BestPeerRequirements::None));
assert!(!peer2.is_better(&peer1, &BestPeerRequirements::None));
}
#[test]
fn test_peer_is_better_full_block_requirement() {
// Peer with full history (earliest = 0)
let peer_full = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: Some(BlockRangeInfo::new(0, 100, B256::random())),
};
// Peer without full history (earliest = 50)
let peer_partial = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: Some(BlockRangeInfo::new(50, 100, B256::random())),
};
// Peer without range info (treated as full history)
let peer_no_range = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: None,
};
// Peer with full history is better than peer without
assert!(peer_full.is_better(&peer_partial, &BestPeerRequirements::FullBlock));
assert!(!peer_partial.is_better(&peer_full, &BestPeerRequirements::FullBlock));
// Peer without range info (full history) is better than partial
assert!(peer_no_range.is_better(&peer_partial, &BestPeerRequirements::FullBlock));
assert!(!peer_partial.is_better(&peer_no_range, &BestPeerRequirements::FullBlock));
// Both have full history - no improvement
assert!(!peer_full.is_better(&peer_no_range, &BestPeerRequirements::FullBlock));
assert!(!peer_no_range.is_better(&peer_full, &BestPeerRequirements::FullBlock));
}
#[test]
fn test_peer_is_better_full_block_range_requirement() {
let range = RangeInclusive::new(40, 60);
// Peer that covers the requested range
let peer_covers = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: Some(BlockRangeInfo::new(0, 100, B256::random())),
};
// Peer that doesn't cover the range (earliest too high)
let peer_no_cover = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: Some(BlockRangeInfo::new(70, 100, B256::random())),
};
// Peer that covers the requested range is better than one that doesn't
assert!(peer_covers
.is_better(&peer_no_cover, &BestPeerRequirements::FullBlockRange(range.clone())));
assert!(
!peer_no_cover.is_better(&peer_covers, &BestPeerRequirements::FullBlockRange(range))
);
}
#[test]
fn test_peer_is_better_both_cover_range() {
let range = RangeInclusive::new(30, 50);
// Peer with full history that covers the range
let peer_full = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: Some(BlockRangeInfo::new(0, 50, B256::random())),
};
// Peer without full history that also covers the range
let peer_partial = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: Some(BlockRangeInfo::new(30, 50, B256::random())),
};
// When both cover the range, prefer none
assert!(!peer_full
.is_better(&peer_partial, &BestPeerRequirements::FullBlockRange(range.clone())));
assert!(!peer_partial.is_better(&peer_full, &BestPeerRequirements::FullBlockRange(range)));
}
#[test]
fn test_peer_is_better_lower_start() {
let range = RangeInclusive::new(30, 60);
// Peer with full history that covers the range
let peer_full = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: Some(BlockRangeInfo::new(0, 50, B256::random())),
};
// Peer without full history that also covers the range
let peer_partial = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: Some(BlockRangeInfo::new(30, 50, B256::random())),
};
// When both cover the range, prefer lower start value
assert!(peer_full
.is_better(&peer_partial, &BestPeerRequirements::FullBlockRange(range.clone())));
assert!(!peer_partial.is_better(&peer_full, &BestPeerRequirements::FullBlockRange(range)));
}
#[test]
fn test_peer_is_better_neither_covers_range() {
let range = RangeInclusive::new(40, 60);
// Peer with full history that doesn't cover the range (latest too low)
let peer_full = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 30,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: Some(BlockRangeInfo::new(0, 30, B256::random())),
};
// Peer without full history that also doesn't cover the range
let peer_partial = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 30,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: Some(BlockRangeInfo::new(10, 30, B256::random())),
};
// When neither covers the range, prefer full history
assert!(peer_full
.is_better(&peer_partial, &BestPeerRequirements::FullBlockRange(range.clone())));
assert!(!peer_partial.is_better(&peer_full, &BestPeerRequirements::FullBlockRange(range)));
}
#[test]
fn test_peer_is_better_no_range_info() {
let range = RangeInclusive::new(40, 60);
// Peer with range info
let peer_with_range = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: Some(BlockRangeInfo::new(30, 100, B256::random())),
};
// Peer without range info
let peer_no_range = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: None,
};
// Peer without range info is not better (we prefer peers with known ranges)
assert!(!peer_no_range
.is_better(&peer_with_range, &BestPeerRequirements::FullBlockRange(range.clone())));
// Peer with range info is better than peer without
assert!(
peer_with_range.is_better(&peer_no_range, &BestPeerRequirements::FullBlockRange(range))
);
}
#[test]
fn test_peer_is_better_one_peer_no_range_covers() {
let range = RangeInclusive::new(40, 60);
// Peer with range info that covers the requested range
let peer_with_range_covers = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: Some(BlockRangeInfo::new(30, 100, B256::random())),
};
// Peer without range info (treated as full history with unknown latest)
let peer_no_range = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: None,
};
// Peer with range that covers is better than peer without range info
assert!(peer_with_range_covers
.is_better(&peer_no_range, &BestPeerRequirements::FullBlockRange(range.clone())));
// Peer without range info is not better when other covers
assert!(!peer_no_range
.is_better(&peer_with_range_covers, &BestPeerRequirements::FullBlockRange(range)));
}
#[test]
fn test_peer_is_better_one_peer_no_range_doesnt_cover() {
let range = RangeInclusive::new(40, 60);
// Peer with range info that does NOT cover the requested range (too high)
let peer_with_range_no_cover = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: Some(BlockRangeInfo::new(70, 100, B256::random())),
};
// Peer without range info (treated as full history)
let peer_no_range = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: None,
};
// Peer with range that doesn't cover is not better
assert!(!peer_with_range_no_cover
.is_better(&peer_no_range, &BestPeerRequirements::FullBlockRange(range.clone())));
// Peer without range info (full history) is better when other doesn't cover
assert!(peer_no_range
.is_better(&peer_with_range_no_cover, &BestPeerRequirements::FullBlockRange(range)));
}
#[test]
fn test_peer_is_better_edge_cases() {
// Test exact range boundaries
let range = RangeInclusive::new(50, 100);
// Peer that exactly covers the range
let peer_exact = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: Some(BlockRangeInfo::new(50, 100, B256::random())),
};
// Peer that's one block short at the start
let peer_short_start = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: Some(BlockRangeInfo::new(51, 100, B256::random())),
};
// Peer that's one block short at the end
let peer_short_end = Peer {
state: PeerState::Idle,
best_hash: B256::random(),
best_number: 100,
capabilities: Arc::new(Capabilities::new(vec![])),
timeout: Arc::new(AtomicU64::new(10)),
last_response_likely_bad: false,
range_info: Some(BlockRangeInfo::new(50, 99, B256::random())),
};
// Exact coverage is better than short coverage
assert!(peer_exact
.is_better(&peer_short_start, &BestPeerRequirements::FullBlockRange(range.clone())));
assert!(peer_exact
.is_better(&peer_short_end, &BestPeerRequirements::FullBlockRange(range.clone())));
// Short coverage is not better than exact coverage
assert!(!peer_short_start
.is_better(&peer_exact, &BestPeerRequirements::FullBlockRange(range.clone())));
assert!(
!peer_short_end.is_better(&peer_exact, &BestPeerRequirements::FullBlockRange(range))
);
}
}

11
crates/net/network/src/session/types.rs
View File

@@ -11,7 +11,7 @@ use std::{
},
};
/// Information about the range of blocks available from a peer.
/// Information about the range of full blocks available from a peer.
///
/// This represents the announced `eth69`
/// [`BlockRangeUpdate`] of a peer.
@@ -45,12 +45,12 @@ impl BlockRangeInfo {
RangeInclusive::new(earliest, latest)
}
/// Returns the earliest block number available from the peer.
/// Returns the earliest full block number available from the peer.
pub fn earliest(&self) -> u64 {
self.inner.earliest.load(Ordering::Relaxed)
}
/// Returns the latest block number available from the peer.
/// Returns the latest full block number available from the peer.
pub fn latest(&self) -> u64 {
self.inner.latest.load(Ordering::Relaxed)
}
@@ -60,6 +60,11 @@ impl BlockRangeInfo {
*self.inner.latest_hash.read()
}
/// Returns true if the peer has the full history available.
pub fn has_full_history(&self) -> bool {
self.earliest() == 0
}
/// Updates the range information.
pub fn update(&self, earliest: u64, latest: u64, latest_hash: B256) {
self.inner.earliest.store(earliest, Ordering::Relaxed);

39
crates/net/network/src/transactions/mod.rs
View File

@@ -1554,25 +1554,6 @@ where
this.on_new_pending_transactions(new_txs);
}
// Advance inflight fetch requests (flush transaction fetcher and queue for
// import to pool).
//
// The smallest decodable transaction is an empty legacy transaction, 10 bytes
// (2 MiB / 10 bytes > 200k transactions).
//
// Since transactions aren't validated until they are inserted into the pool,
// this can potentially queue >200k transactions for insertion to pool. More
// if the message size is bigger than the soft limit on a `PooledTransactions`
// response which is 2 MiB.
let maybe_more_tx_fetch_events = metered_poll_nested_stream_with_budget!(
poll_durations.acc_fetch_events,
"net::tx",
"Transaction fetch events stream",
DEFAULT_BUDGET_TRY_DRAIN_STREAM,
this.transaction_fetcher.poll_next_unpin(cx),
|event| this.on_fetch_event(event),
);
// Advance incoming transaction events (stream new txns/announcements from
// network manager and queue for import to pool/fetch txns).
//
@@ -1596,6 +1577,25 @@ where
|event| this.on_network_tx_event(event),
);
// Advance inflight fetch requests (flush transaction fetcher and queue for
// import to pool).
//
// The smallest decodable transaction is an empty legacy transaction, 10 bytes
// (2 MiB / 10 bytes > 200k transactions).
//
// Since transactions aren't validated until they are inserted into the pool,
// this can potentially queue >200k transactions for insertion to pool. More
// if the message size is bigger than the soft limit on a `PooledTransactions`
// response which is 2 MiB.
let mut maybe_more_tx_fetch_events = metered_poll_nested_stream_with_budget!(
poll_durations.acc_fetch_events,
"net::tx",
"Transaction fetch events stream",
DEFAULT_BUDGET_TRY_DRAIN_STREAM,
this.transaction_fetcher.poll_next_unpin(cx),
|event| this.on_fetch_event(event),
);
// Advance pool imports (flush txns to pool).
//
// Note, this is done in batches. A batch is filled from one `Transactions`
@@ -1627,6 +1627,7 @@ where
{
if this.has_capacity_for_fetching_pending_hashes() {
this.on_fetch_hashes_pending_fetch();
maybe_more_tx_fetch_events = true;
}
},
poll_durations.acc_pending_fetch

12
crates/node/builder/src/launch/debug.rs
View File

@@ -7,7 +7,7 @@ use reth_chainspec::EthChainSpec;
use reth_consensus_debug_client::{DebugConsensusClient, EtherscanBlockProvider, RpcBlockProvider};
use reth_engine_local::LocalMiner;
use reth_node_api::{
BlockTy, FullNodeComponents, PayloadAttrTy, PayloadAttributesBuilder, PayloadTypes,
BlockTy, FullNodeComponents, HeaderTy, PayloadAttrTy, PayloadAttributesBuilder, PayloadTypes,
};
use std::{
future::{Future, IntoFuture},
@@ -73,9 +73,7 @@ pub trait DebugNode<N: FullNodeComponents>: Node<N> {
/// be constructed during local mining.
fn local_payload_attributes_builder(
chain_spec: &Self::ChainSpec,
) -> impl PayloadAttributesBuilder<
<<Self as reth_node_api::NodeTypes>::Payload as PayloadTypes>::PayloadAttributes,
>;
) -> impl PayloadAttributesBuilder<<Self::Payload as PayloadTypes>::PayloadAttributes, HeaderTy<Self>>;
}
/// Node launcher with support for launching various debugging utilities.
@@ -120,7 +118,7 @@ where
inner: L,
target: Target,
local_payload_attributes_builder:
Option<Box<dyn PayloadAttributesBuilder<PayloadAttrTy<N::Types>>>>,
Option<Box<dyn PayloadAttributesBuilder<PayloadAttrTy<N::Types>, HeaderTy<N::Types>>>>,
map_attributes:
Option<Box<dyn Fn(PayloadAttrTy<N::Types>) -> PayloadAttrTy<N::Types> + Send + Sync>>,
}
@@ -133,7 +131,7 @@ where
{
pub fn with_payload_attributes_builder(
self,
builder: impl PayloadAttributesBuilder<PayloadAttrTy<N::Types>>,
builder: impl PayloadAttributesBuilder<PayloadAttrTy<N::Types>, HeaderTy<N::Types>>,
) -> Self {
Self {
inner: self.inner,
@@ -229,7 +227,7 @@ where
} else {
let local = N::Types::local_payload_attributes_builder(&chain_spec);
let builder = if let Some(f) = map_attributes {
Either::Left(move |block_number| f(local.build(block_number)))
Either::Left(move |parent| f(local.build(&parent)))
} else {
Either::Right(local)
};

25
crates/node/core/src/args/static_files.rs
View File

@@ -20,6 +20,10 @@ pub struct StaticFilesArgs {
#[arg(long = "static-files.blocks-per-file.receipts")]
pub blocks_per_file_receipts: Option<u64>,
/// Number of blocks per file for the transaction senders segment.
#[arg(long = "static-files.blocks-per-file.transaction-senders")]
pub blocks_per_file_transaction_senders: Option<u64>,
/// Store receipts in static files instead of the database.
///
/// When enabled, receipts will be written to static files on disk instead of the database.
@@ -28,6 +32,16 @@ pub struct StaticFilesArgs {
/// the node has been initialized, changing this flag requires re-syncing from scratch.
#[arg(long = "static-files.receipts")]
pub receipts: bool,
/// Store transaction senders in static files instead of the database.
///
/// When enabled, transaction senders will be written to static files on disk instead of the
/// database.
///
/// Note: This setting can only be configured at genesis initialization. Once
/// the node has been initialized, changing this flag requires re-syncing from scratch.
#[arg(long = "static-files.transaction-senders")]
pub transaction_senders: bool,
}
impl StaticFilesArgs {
@@ -41,16 +55,17 @@ impl StaticFilesArgs {
.blocks_per_file_transactions
.or(config.blocks_per_file.transactions),
receipts: self.blocks_per_file_receipts.or(config.blocks_per_file.receipts),
transaction_senders: self
.blocks_per_file_transaction_senders
.or(config.blocks_per_file.transaction_senders),
},
}
}
/// Converts the static files arguments into [`StorageSettings`].
pub const fn to_settings(&self) -> StorageSettings {
if self.receipts {
StorageSettings::new().with_receipts_in_static_files()
} else {
StorageSettings::legacy()
}
StorageSettings::legacy()
.with_receipts_in_static_files(self.receipts)
.with_transaction_senders_in_static_files(self.transaction_senders)
}
}

44
crates/node/core/src/args/trace.rs
View File

@@ -3,7 +3,7 @@
use clap::Parser;
use eyre::WrapErr;
use reth_tracing::{tracing_subscriber::EnvFilter, Layers};
use reth_tracing_otlp::OtlpProtocol;
use reth_tracing_otlp::{OtlpConfig, OtlpProtocol};
use url::Url;
/// CLI arguments for configuring `Opentelemetry` trace and span export.
@@ -78,6 +78,23 @@ pub struct TraceArgs {
help_heading = "Tracing"
)]
pub service_name: String,
/// Trace sampling ratio to control the percentage of traces to export.
///
/// Valid range: 0.0 to 1.0
/// - 1.0, default: Sample all traces
/// - 0.01: Sample 1% of traces
/// - 0.0: Disable sampling
///
/// Example: --tracing-otlp.sample-ratio=0.0.
#[arg(
long = "tracing-otlp.sample-ratio",
env = "OTEL_TRACES_SAMPLER_ARG",
global = true,
value_name = "RATIO",
help_heading = "Tracing"
)]
pub sample_ratio: Option<f64>,
}
impl Default for TraceArgs {
@@ -86,6 +103,7 @@ impl Default for TraceArgs {
otlp: None,
protocol: OtlpProtocol::Http,
otlp_filter: EnvFilter::from_default_env(),
sample_ratio: None,
service_name: "reth".to_string(),
}
}
@@ -102,22 +120,24 @@ impl TraceArgs {
/// Note: even though this function is async, it does not actually perform any async operations.
/// It's needed only to be able to initialize the gRPC transport of OTLP tracing that needs to
/// be called inside a tokio runtime context.
pub async fn init_otlp_tracing(
&mut self,
_layers: &mut Layers,
) -> eyre::Result<OtlpInitStatus> {
pub async fn init_otlp_tracing(&mut self, layers: &mut Layers) -> eyre::Result<OtlpInitStatus> {
if let Some(endpoint) = self.otlp.as_mut() {
self.protocol.validate_endpoint(endpoint)?;
#[cfg(feature = "otlp")]
{
_layers.with_span_layer(
self.service_name.clone(),
endpoint.clone(),
self.otlp_filter.clone(),
self.protocol,
)?;
Ok(OtlpInitStatus::Started(endpoint.clone()))
{
let config = OtlpConfig::new(
self.service_name.clone(),
endpoint.clone(),
self.protocol,
self.sample_ratio,
)?;
layers.with_span_layer(config.clone(), self.otlp_filter.clone())?;
Ok(OtlpInitStatus::Started(config.endpoint().clone()))
}
}
#[cfg(not(feature = "otlp"))]
{

27
crates/optimism/primitives/Cargo.toml
View File

@@ -14,8 +14,6 @@ workspace = true
[dependencies]
# reth
reth-primitives-traits = { workspace = true, features = ["op"] }
reth-codecs = { workspace = true, optional = true, features = ["op"] }
reth-zstd-compressors = { workspace = true, optional = true }
# ethereum
alloy-primitives.workspace = true
@@ -27,17 +25,15 @@ alloy-rlp.workspace = true
op-alloy-consensus.workspace = true
# codec
bytes = { workspace = true, optional = true }
modular-bitfield = { workspace = true, optional = true }
serde = { workspace = true, optional = true }
serde_with = { workspace = true, optional = true }
# test
arbitrary = { workspace = true, features = ["derive"], optional = true }
[dev-dependencies]
reth-codecs = { workspace = true, features = ["test-utils", "op"] }
bytes.workspace = true
modular-bitfield.workspace = true
reth-zstd-compressors.workspace = true
rand.workspace = true
arbitrary.workspace = true
rstest.workspace = true
@@ -53,41 +49,35 @@ secp256k1 = { workspace = true, features = ["rand"] }
default = ["std"]
std = [
"reth-primitives-traits/std",
"reth-codecs?/std",
"alloy-consensus/std",
"alloy-primitives/std",
"serde?/std",
"bytes?/std",
"alloy-rlp/std",
"reth-zstd-compressors?/std",
"op-alloy-consensus/std",
"serde_json/std",
"serde_with?/std",
"alloy-eips/std",
"secp256k1/std",
"bytes/std",
"reth-zstd-compressors/std",
]
alloy-compat = ["op-alloy-consensus/alloy-compat"]
reth-codec = [
"dep:reth-codecs",
"std",
"reth-primitives-traits/reth-codec",
"reth-codecs?/op",
"dep:bytes",
"dep:modular-bitfield",
"dep:reth-zstd-compressors",
]
serde = [
"dep:serde",
"reth-primitives-traits/serde",
"alloy-primitives/serde",
"alloy-consensus/serde",
"bytes?/serde",
"reth-codecs?/serde",
"op-alloy-consensus/serde",
"alloy-eips/serde",
"rand/serde",
"rand_08/serde",
"secp256k1/serde",
"bytes/serde",
"reth-codecs/serde",
]
serde-bincode-compat = [
"serde",
@@ -99,11 +89,10 @@ serde-bincode-compat = [
]
arbitrary = [
"std",
"dep:arbitrary",
"reth-primitives-traits/arbitrary",
"reth-codecs?/arbitrary",
"op-alloy-consensus/arbitrary",
"alloy-consensus/arbitrary",
"alloy-primitives/arbitrary",
"alloy-eips/arbitrary",
"reth-codecs/arbitrary",
]

7
crates/optimism/primitives/src/lib.rs
View File

@@ -20,7 +20,8 @@ pub mod transaction;
pub use transaction::*;
mod receipt;
pub use receipt::{DepositReceipt, OpReceipt};
pub use op_alloy_consensus::OpReceipt;
pub use receipt::DepositReceipt;
/// Optimism-specific block type.
pub type OpBlock = alloy_consensus::Block<OpTransactionSigned>;
@@ -44,6 +45,6 @@ impl reth_primitives_traits::NodePrimitives for OpPrimitives {
/// Bincode-compatible serde implementations.
#[cfg(feature = "serde-bincode-compat")]
pub mod serde_bincode_compat {
pub use super::receipt::serde_bincode_compat::*;
pub use op_alloy_consensus::serde_bincode_compat::*;
pub use super::receipt::serde_bincode_compat::OpReceipt as LocalOpReceipt;
pub use op_alloy_consensus::serde_bincode_compat::OpReceipt;
}

516
crates/optimism/primitives/src/receipt.rs
View File

@@ -9,409 +9,9 @@ use alloy_eips::{
};
use alloy_primitives::{Bloom, Log};
use alloy_rlp::{BufMut, Decodable, Encodable, Header};
use op_alloy_consensus::{OpDepositReceipt, OpTxType};
use op_alloy_consensus::{OpDepositReceipt, OpReceipt, OpTxType};
use reth_primitives_traits::InMemorySize;
/// Typed ethereum transaction receipt.
/// Receipt containing result of transaction execution.
#[derive(Clone, Debug, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
#[cfg_attr(feature = "reth-codec", reth_codecs::add_arbitrary_tests(rlp))]
pub enum OpReceipt {
/// Legacy receipt
Legacy(Receipt),
/// EIP-2930 receipt
Eip2930(Receipt),
/// EIP-1559 receipt
Eip1559(Receipt),
/// EIP-7702 receipt
Eip7702(Receipt),
/// Deposit receipt
Deposit(OpDepositReceipt),
}
impl OpReceipt {
/// Returns [`OpTxType`] of the receipt.
pub const fn tx_type(&self) -> OpTxType {
match self {
Self::Legacy(_) => OpTxType::Legacy,
Self::Eip2930(_) => OpTxType::Eip2930,
Self::Eip1559(_) => OpTxType::Eip1559,
Self::Eip7702(_) => OpTxType::Eip7702,
Self::Deposit(_) => OpTxType::Deposit,
}
}
/// Returns inner [`Receipt`],
pub const fn as_receipt(&self) -> &Receipt {
match self {
Self::Legacy(receipt) |
Self::Eip2930(receipt) |
Self::Eip1559(receipt) |
Self::Eip7702(receipt) => receipt,
Self::Deposit(receipt) => &receipt.inner,
}
}
/// Returns a mutable reference to the inner [`Receipt`],
pub const fn as_receipt_mut(&mut self) -> &mut Receipt {
match self {
Self::Legacy(receipt) |
Self::Eip2930(receipt) |
Self::Eip1559(receipt) |
Self::Eip7702(receipt) => receipt,
Self::Deposit(receipt) => &mut receipt.inner,
}
}
/// Consumes this and returns the inner [`Receipt`].
pub fn into_receipt(self) -> Receipt {
match self {
Self::Legacy(receipt) |
Self::Eip2930(receipt) |
Self::Eip1559(receipt) |
Self::Eip7702(receipt) => receipt,
Self::Deposit(receipt) => receipt.inner,
}
}
/// Returns length of RLP-encoded receipt fields with the given [`Bloom`] without an RLP header.
pub fn rlp_encoded_fields_length(&self, bloom: &Bloom) -> usize {
match self {
Self::Legacy(receipt) |
Self::Eip2930(receipt) |
Self::Eip1559(receipt) |
Self::Eip7702(receipt) => receipt.rlp_encoded_fields_length_with_bloom(bloom),
Self::Deposit(receipt) => receipt.rlp_encoded_fields_length_with_bloom(bloom),
}
}
/// RLP-encodes receipt fields with the given [`Bloom`] without an RLP header.
pub fn rlp_encode_fields(&self, bloom: &Bloom, out: &mut dyn BufMut) {
match self {
Self::Legacy(receipt) |
Self::Eip2930(receipt) |
Self::Eip1559(receipt) |
Self::Eip7702(receipt) => receipt.rlp_encode_fields_with_bloom(bloom, out),
Self::Deposit(receipt) => receipt.rlp_encode_fields_with_bloom(bloom, out),
}
}
/// Returns RLP header for inner encoding.
pub fn rlp_header_inner(&self, bloom: &Bloom) -> Header {
Header { list: true, payload_length: self.rlp_encoded_fields_length(bloom) }
}
/// Returns RLP header for inner encoding without bloom.
pub fn rlp_header_inner_without_bloom(&self) -> Header {
Header { list: true, payload_length: self.rlp_encoded_fields_length_without_bloom() }
}
/// RLP-decodes the receipt from the provided buffer. This does not expect a type byte or
/// network header.
pub fn rlp_decode_inner(
buf: &mut &[u8],
tx_type: OpTxType,
) -> alloy_rlp::Result<ReceiptWithBloom<Self>> {
match tx_type {
OpTxType::Legacy => {
let ReceiptWithBloom { receipt, logs_bloom } =
RlpDecodableReceipt::rlp_decode_with_bloom(buf)?;
Ok(ReceiptWithBloom { receipt: Self::Legacy(receipt), logs_bloom })
}
OpTxType::Eip2930 => {
let ReceiptWithBloom { receipt, logs_bloom } =
RlpDecodableReceipt::rlp_decode_with_bloom(buf)?;
Ok(ReceiptWithBloom { receipt: Self::Eip2930(receipt), logs_bloom })
}
OpTxType::Eip1559 => {
let ReceiptWithBloom { receipt, logs_bloom } =
RlpDecodableReceipt::rlp_decode_with_bloom(buf)?;
Ok(ReceiptWithBloom { receipt: Self::Eip1559(receipt), logs_bloom })
}
OpTxType::Eip7702 => {
let ReceiptWithBloom { receipt, logs_bloom } =
RlpDecodableReceipt::rlp_decode_with_bloom(buf)?;
Ok(ReceiptWithBloom { receipt: Self::Eip7702(receipt), logs_bloom })
}
OpTxType::Deposit => {
let ReceiptWithBloom { receipt, logs_bloom } =
RlpDecodableReceipt::rlp_decode_with_bloom(buf)?;
Ok(ReceiptWithBloom { receipt: Self::Deposit(receipt), logs_bloom })
}
}
}
/// RLP-encodes receipt fields without an RLP header.
pub fn rlp_encode_fields_without_bloom(&self, out: &mut dyn BufMut) {
match self {
Self::Legacy(receipt) |
Self::Eip2930(receipt) |
Self::Eip1559(receipt) |
Self::Eip7702(receipt) => {
receipt.status.encode(out);
receipt.cumulative_gas_used.encode(out);
receipt.logs.encode(out);
}
Self::Deposit(receipt) => {
receipt.inner.status.encode(out);
receipt.inner.cumulative_gas_used.encode(out);
receipt.inner.logs.encode(out);
if let Some(nonce) = receipt.deposit_nonce {
nonce.encode(out);
}
if let Some(version) = receipt.deposit_receipt_version {
version.encode(out);
}
}
}
}
/// Returns length of RLP-encoded receipt fields without an RLP header.
pub fn rlp_encoded_fields_length_without_bloom(&self) -> usize {
match self {
Self::Legacy(receipt) |
Self::Eip2930(receipt) |
Self::Eip1559(receipt) |
Self::Eip7702(receipt) => {
receipt.status.length() +
receipt.cumulative_gas_used.length() +
receipt.logs.length()
}
Self::Deposit(receipt) => {
receipt.inner.status.length() +
receipt.inner.cumulative_gas_used.length() +
receipt.inner.logs.length() +
receipt.deposit_nonce.map_or(0, |nonce| nonce.length()) +
receipt.deposit_receipt_version.map_or(0, |version| version.length())
}
}
}
/// RLP-decodes the receipt from the provided buffer without bloom.
pub fn rlp_decode_inner_without_bloom(
buf: &mut &[u8],
tx_type: OpTxType,
) -> alloy_rlp::Result<Self> {
let header = Header::decode(buf)?;
if !header.list {
return Err(alloy_rlp::Error::UnexpectedString);
}
let remaining = buf.len();
let status = Decodable::decode(buf)?;
let cumulative_gas_used = Decodable::decode(buf)?;
let logs = Decodable::decode(buf)?;
let mut deposit_nonce = None;
let mut deposit_receipt_version = None;
// For deposit receipts, try to decode nonce and version if they exist
if tx_type == OpTxType::Deposit && buf.len() + header.payload_length > remaining {
deposit_nonce = Some(Decodable::decode(buf)?);
if buf.len() + header.payload_length > remaining {
deposit_receipt_version = Some(Decodable::decode(buf)?);
}
}
if buf.len() + header.payload_length != remaining {
return Err(alloy_rlp::Error::UnexpectedLength);
}
match tx_type {
OpTxType::Legacy => Ok(Self::Legacy(Receipt { status, cumulative_gas_used, logs })),
OpTxType::Eip2930 => Ok(Self::Eip2930(Receipt { status, cumulative_gas_used, logs })),
OpTxType::Eip1559 => Ok(Self::Eip1559(Receipt { status, cumulative_gas_used, logs })),
OpTxType::Eip7702 => Ok(Self::Eip7702(Receipt { status, cumulative_gas_used, logs })),
OpTxType::Deposit => Ok(Self::Deposit(OpDepositReceipt {
inner: Receipt { status, cumulative_gas_used, logs },
deposit_nonce,
deposit_receipt_version,
})),
}
}
}
impl Eip2718EncodableReceipt for OpReceipt {
fn eip2718_encoded_length_with_bloom(&self, bloom: &Bloom) -> usize {
!self.tx_type().is_legacy() as usize + self.rlp_header_inner(bloom).length_with_payload()
}
fn eip2718_encode_with_bloom(&self, bloom: &Bloom, out: &mut dyn BufMut) {
if !self.tx_type().is_legacy() {
out.put_u8(self.tx_type() as u8);
}
self.rlp_header_inner(bloom).encode(out);
self.rlp_encode_fields(bloom, out);
}
}
impl RlpEncodableReceipt for OpReceipt {
fn rlp_encoded_length_with_bloom(&self, bloom: &Bloom) -> usize {
let mut len = self.eip2718_encoded_length_with_bloom(bloom);
if !self.tx_type().is_legacy() {
len += Header {
list: false,
payload_length: self.eip2718_encoded_length_with_bloom(bloom),
}
.length();
}
len
}
fn rlp_encode_with_bloom(&self, bloom: &Bloom, out: &mut dyn BufMut) {
if !self.tx_type().is_legacy() {
Header { list: false, payload_length: self.eip2718_encoded_length_with_bloom(bloom) }
.encode(out);
}
self.eip2718_encode_with_bloom(bloom, out);
}
}
impl RlpDecodableReceipt for OpReceipt {
fn rlp_decode_with_bloom(buf: &mut &[u8]) -> alloy_rlp::Result<ReceiptWithBloom<Self>> {
let header_buf = &mut &**buf;
let header = Header::decode(header_buf)?;
// Legacy receipt, reuse initial buffer without advancing
if header.list {
return Self::rlp_decode_inner(buf, OpTxType::Legacy)
}
// Otherwise, advance the buffer and try decoding type flag followed by receipt
*buf = *header_buf;
let remaining = buf.len();
let tx_type = OpTxType::decode(buf)?;
let this = Self::rlp_decode_inner(buf, tx_type)?;
if buf.len() + header.payload_length != remaining {
return Err(alloy_rlp::Error::UnexpectedLength);
}
Ok(this)
}
}
impl Encodable2718 for OpReceipt {
fn encode_2718_len(&self) -> usize {
!self.tx_type().is_legacy() as usize +
self.rlp_header_inner_without_bloom().length_with_payload()
}
fn encode_2718(&self, out: &mut dyn BufMut) {
if !self.tx_type().is_legacy() {
out.put_u8(self.tx_type() as u8);
}
self.rlp_header_inner_without_bloom().encode(out);
self.rlp_encode_fields_without_bloom(out);
}
}
impl Decodable2718 for OpReceipt {
fn typed_decode(ty: u8, buf: &mut &[u8]) -> Eip2718Result<Self> {
Ok(Self::rlp_decode_inner_without_bloom(buf, OpTxType::try_from(ty)?)?)
}
fn fallback_decode(buf: &mut &[u8]) -> Eip2718Result<Self> {
Ok(Self::rlp_decode_inner_without_bloom(buf, OpTxType::Legacy)?)
}
}
impl Encodable for OpReceipt {
fn encode(&self, out: &mut dyn BufMut) {
self.network_encode(out);
}
fn length(&self) -> usize {
self.network_len()
}
}
impl Decodable for OpReceipt {
fn decode(buf: &mut &[u8]) -> alloy_rlp::Result<Self> {
Ok(Self::network_decode(buf)?)
}
}
impl TxReceipt for OpReceipt {
type Log = Log;
fn status_or_post_state(&self) -> Eip658Value {
self.as_receipt().status_or_post_state()
}
fn status(&self) -> bool {
self.as_receipt().status()
}
fn bloom(&self) -> Bloom {
self.as_receipt().bloom()
}
fn cumulative_gas_used(&self) -> u64 {
self.as_receipt().cumulative_gas_used()
}
fn logs(&self) -> &[Log] {
self.as_receipt().logs()
}
fn into_logs(self) -> Vec<Self::Log> {
match self {
Self::Legacy(receipt) |
Self::Eip2930(receipt) |
Self::Eip1559(receipt) |
Self::Eip7702(receipt) => receipt.logs,
Self::Deposit(receipt) => receipt.inner.logs,
}
}
}
impl Typed2718 for OpReceipt {
fn ty(&self) -> u8 {
self.tx_type().into()
}
}
impl IsTyped2718 for OpReceipt {
fn is_type(type_id: u8) -> bool {
<OpTxType as IsTyped2718>::is_type(type_id)
}
}
impl InMemorySize for OpReceipt {
fn size(&self) -> usize {
self.as_receipt().size()
}
}
impl From<op_alloy_consensus::OpReceiptEnvelope> for OpReceipt {
fn from(envelope: op_alloy_consensus::OpReceiptEnvelope) -> Self {
match envelope {
op_alloy_consensus::OpReceiptEnvelope::Legacy(receipt) => Self::Legacy(receipt.receipt),
op_alloy_consensus::OpReceiptEnvelope::Eip2930(receipt) => {
Self::Eip2930(receipt.receipt)
}
op_alloy_consensus::OpReceiptEnvelope::Eip1559(receipt) => {
Self::Eip1559(receipt.receipt)
}
op_alloy_consensus::OpReceiptEnvelope::Eip7702(receipt) => {
Self::Eip7702(receipt.receipt)
}
op_alloy_consensus::OpReceiptEnvelope::Deposit(receipt) => {
Self::Deposit(OpDepositReceipt {
deposit_nonce: receipt.receipt.deposit_nonce,
deposit_receipt_version: receipt.receipt.deposit_receipt_version,
inner: receipt.receipt.inner,
})
}
}
}
}
/// Trait for deposit receipt.
pub trait DepositReceipt: reth_primitives_traits::Receipt {
/// Converts a `Receipt` into a mutable Optimism deposit receipt.
@@ -437,100 +37,6 @@ impl DepositReceipt for OpReceipt {
}
}
#[cfg(feature = "reth-codec")]
mod compact {
use super::*;
use alloc::borrow::Cow;
use reth_codecs::Compact;
#[derive(reth_codecs::CompactZstd)]
#[reth_zstd(
compressor = reth_zstd_compressors::RECEIPT_COMPRESSOR,
decompressor = reth_zstd_compressors::RECEIPT_DECOMPRESSOR
)]
struct CompactOpReceipt<'a> {
tx_type: OpTxType,
success: bool,
cumulative_gas_used: u64,
#[expect(clippy::owned_cow)]
logs: Cow<'a, Vec<Log>>,
deposit_nonce: Option<u64>,
deposit_receipt_version: Option<u64>,
}
impl<'a> From<&'a OpReceipt> for CompactOpReceipt<'a> {
fn from(receipt: &'a OpReceipt) -> Self {
Self {
tx_type: receipt.tx_type(),
success: receipt.status(),
cumulative_gas_used: receipt.cumulative_gas_used(),
logs: Cow::Borrowed(&receipt.as_receipt().logs),
deposit_nonce: if let OpReceipt::Deposit(receipt) = receipt {
receipt.deposit_nonce
} else {
None
},
deposit_receipt_version: if let OpReceipt::Deposit(receipt) = receipt {
receipt.deposit_receipt_version
} else {
None
},
}
}
}
impl From<CompactOpReceipt<'_>> for OpReceipt {
fn from(receipt: CompactOpReceipt<'_>) -> Self {
let CompactOpReceipt {
tx_type,
success,
cumulative_gas_used,
logs,
deposit_nonce,
deposit_receipt_version,
} = receipt;
let inner =
Receipt { status: success.into(), cumulative_gas_used, logs: logs.into_owned() };
match tx_type {
OpTxType::Legacy => Self::Legacy(inner),
OpTxType::Eip2930 => Self::Eip2930(inner),
OpTxType::Eip1559 => Self::Eip1559(inner),
OpTxType::Eip7702 => Self::Eip7702(inner),
OpTxType::Deposit => Self::Deposit(OpDepositReceipt {
inner,
deposit_nonce,
deposit_receipt_version,
}),
}
}
}
impl Compact for OpReceipt {
fn to_compact<B>(&self, buf: &mut B) -> usize
where
B: bytes::BufMut + AsMut<[u8]>,
{
CompactOpReceipt::from(self).to_compact(buf)
}
fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) {
let (receipt, buf) = CompactOpReceipt::from_compact(buf, len);
(receipt.into(), buf)
}
}
#[cfg(test)]
#[test]
fn test_ensure_backwards_compatibility() {
use reth_codecs::{test_utils::UnusedBits, validate_bitflag_backwards_compat};
assert_eq!(CompactOpReceipt::bitflag_encoded_bytes(), 2);
validate_bitflag_backwards_compat!(CompactOpReceipt<'_>, UnusedBits::NotZero);
}
}
#[cfg(all(feature = "serde", feature = "serde-bincode-compat"))]
pub(super) mod serde_bincode_compat {
use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -540,17 +46,21 @@ pub(super) mod serde_bincode_compat {
///
/// Intended to use with the [`serde_with::serde_as`] macro in the following way:
/// ```rust
/// use reth_optimism_primitives::{serde_bincode_compat, OpReceipt};
/// use reth_optimism_primitives::OpReceipt;
/// use reth_primitives_traits::serde_bincode_compat::SerdeBincodeCompat;
/// use serde::{de::DeserializeOwned, Deserialize, Serialize};
/// use serde_with::serde_as;
///
/// #[serde_as]
/// #[derive(Serialize, Deserialize)]
/// struct Data {
/// #[serde_as(as = "serde_bincode_compat::OpReceipt<'_>")]
/// #[serde_as(
/// as = "reth_primitives_traits::serde_bincode_compat::BincodeReprFor<'_, OpReceipt>"
/// )]
/// receipt: OpReceipt,
/// }
/// ```
#[allow(rustdoc::private_doc_tests)]
#[derive(Debug, Serialize, Deserialize)]
pub enum OpReceipt<'a> {
/// Legacy receipt
@@ -609,18 +119,6 @@ pub(super) mod serde_bincode_compat {
}
}
impl reth_primitives_traits::serde_bincode_compat::SerdeBincodeCompat for super::OpReceipt {
type BincodeRepr<'a> = OpReceipt<'a>;
fn as_repr(&self) -> Self::BincodeRepr<'_> {
self.into()
}
fn from_repr(repr: Self::BincodeRepr<'_>) -> Self {
repr.into()
}
}
#[cfg(test)]
mod tests {
use crate::{receipt::serde_bincode_compat, OpReceipt};

2
crates/optimism/rpc/src/eth/mod.rs
View File

@@ -332,7 +332,7 @@ impl<N, Rpc> Trace for OpEthApi<N, Rpc>
where
N: RpcNodeCore,
OpEthApiError: FromEvmError<N::Evm>,
Rpc: RpcConvert<Primitives = N::Primitives>,
Rpc: RpcConvert<Primitives = N::Primitives, Error = OpEthApiError, Evm = N::Evm>,
{
}

2
crates/payload/primitives/Cargo.toml
View File

@@ -21,6 +21,7 @@ reth-execution-types.workspace = true
reth-trie-common.workspace = true
# alloy
alloy-consensus.workspace = true
alloy-eips.workspace = true
alloy-primitives.workspace = true
alloy-rpc-types-engine = { workspace = true, features = ["serde"] }
@@ -50,6 +51,7 @@ std = [
"thiserror/std",
"reth-primitives-traits/std",
"either/std",
"alloy-consensus/std",
]
op = [
"dep:op-alloy-rpc-types-engine",

36
crates/payload/primitives/src/traits.rs
View File

@@ -202,40 +202,44 @@ impl PayloadAttributes for op_alloy_rpc_types_engine::OpPayloadAttributes {
///
/// Enables different strategies for generating payload attributes based on
/// contextual information. Useful for testing and specialized building.
pub trait PayloadAttributesBuilder<Attributes>: Send + Sync + 'static {
pub trait PayloadAttributesBuilder<Attributes, Header = alloy_consensus::Header>:
Send + Sync + 'static
{
/// Constructs new payload attributes for the given timestamp.
fn build(&self, timestamp: u64) -> Attributes;
fn build(&self, parent: &SealedHeader<Header>) -> Attributes;
}
impl<Attributes, F> PayloadAttributesBuilder<Attributes> for F
impl<Attributes, Header, F> PayloadAttributesBuilder<Attributes, Header> for F
where
F: Fn(u64) -> Attributes + Send + Sync + 'static,
Header: Clone,
F: Fn(SealedHeader<Header>) -> Attributes + Send + Sync + 'static,
{
fn build(&self, timestamp: u64) -> Attributes {
self(timestamp)
fn build(&self, parent: &SealedHeader<Header>) -> Attributes {
self(parent.clone())
}
}
impl<Attributes, L, R> PayloadAttributesBuilder<Attributes> for Either<L, R>
impl<Attributes, Header, L, R> PayloadAttributesBuilder<Attributes, Header> for Either<L, R>
where
L: PayloadAttributesBuilder<Attributes>,
R: PayloadAttributesBuilder<Attributes>,
L: PayloadAttributesBuilder<Attributes, Header>,
R: PayloadAttributesBuilder<Attributes, Header>,
{
fn build(&self, timestamp: u64) -> Attributes {
fn build(&self, parent: &SealedHeader<Header>) -> Attributes {
match self {
Self::Left(l) => l.build(timestamp),
Self::Right(r) => r.build(timestamp),
Self::Left(l) => l.build(parent),
Self::Right(r) => r.build(parent),
}
}
}
impl<Attributes> PayloadAttributesBuilder<Attributes>
for Box<dyn PayloadAttributesBuilder<Attributes>>
impl<Attributes, Header> PayloadAttributesBuilder<Attributes, Header>
for Box<dyn PayloadAttributesBuilder<Attributes, Header>>
where
Header: 'static,
Attributes: 'static,
{
fn build(&self, timestamp: u64) -> Attributes {
self.as_ref().build(timestamp)
fn build(&self, parent: &SealedHeader<Header>) -> Attributes {
self.as_ref().build(parent)
}
}

13
crates/primitives-traits/src/serde_bincode_compat.rs
View File

@@ -345,4 +345,17 @@ mod block_bincode {
repr.into()
}
}
#[cfg(feature = "op")]
impl super::SerdeBincodeCompat for op_alloy_consensus::OpReceipt {
type BincodeRepr<'a> = op_alloy_consensus::serde_bincode_compat::OpReceipt<'a>;
fn as_repr(&self) -> Self::BincodeRepr<'_> {
self.into()
}
fn from_repr(repr: Self::BincodeRepr<'_>) -> Self {
repr.into()
}
}
}

12
crates/primitives-traits/src/size.rs
View File

@@ -148,6 +148,18 @@ mod op {
}
}
impl InMemorySize for op_alloy_consensus::OpReceipt {
fn size(&self) -> usize {
match self {
Self::Legacy(receipt) |
Self::Eip2930(receipt) |
Self::Eip1559(receipt) |
Self::Eip7702(receipt) => receipt.size(),
Self::Deposit(receipt) => receipt.size(),
}
}
}
impl InMemorySize for op_alloy_consensus::OpTypedTransaction {
fn size(&self) -> usize {
match self {

6
crates/rpc/rpc-eth-api/src/core.rs
View File

@@ -18,7 +18,7 @@ use alloy_serde::JsonStorageKey;
use jsonrpsee::{core::RpcResult, proc_macros::rpc};
use reth_primitives_traits::TxTy;
use reth_rpc_convert::RpcTxReq;
use reth_rpc_eth_types::FillTransactionResult;
use reth_rpc_eth_types::FillTransaction;
use reth_rpc_server_types::{result::internal_rpc_err, ToRpcResult};
use tracing::trace;
@@ -242,7 +242,7 @@ pub trait EthApi<
/// Fills the defaults on a given unsigned transaction.
#[method(name = "fillTransaction")]
async fn fill_transaction(&self, request: TxReq) -> RpcResult<FillTransactionResult<RawTx>>;
async fn fill_transaction(&self, request: TxReq) -> RpcResult<FillTransaction<RawTx>>;
/// Simulate arbitrary number of transactions at an arbitrary blockchain index, with the
/// optionality of state overrides
@@ -703,7 +703,7 @@ where
async fn fill_transaction(
&self,
request: RpcTxReq<T::NetworkTypes>,
) -> RpcResult<FillTransactionResult<TxTy<T::Primitives>>> {
) -> RpcResult<FillTransaction<TxTy<T::Primitives>>> {
trace!(target: "rpc::eth", ?request, "Serving eth_fillTransaction");
Ok(EthTransactions::fill_transaction(self, request).await?)
}

49
crates/rpc/rpc-eth-api/src/helpers/call.rs
View File

@@ -28,12 +28,12 @@ use reth_primitives_traits::Recovered;
use reth_revm::{database::StateProviderDatabase, db::State};
use reth_rpc_convert::{RpcConvert, RpcTxReq};
use reth_rpc_eth_types::{
cache::db::{StateCacheDbRefMutWrapper, StateProviderTraitObjWrapper},
cache::db::StateProviderTraitObjWrapper,
error::FromEthApiError,
simulate::{self, EthSimulateError},
EthApiError, StateCacheDb,
};
use reth_storage_api::{BlockIdReader, ProviderTx};
use reth_storage_api::{BlockIdReader, ProviderTx, StateProvider};
use revm::{
context::Block,
context_interface::{result::ResultAndState, Transaction},
@@ -89,10 +89,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA
self.recovered_block(block).await?.ok_or(EthApiError::HeaderNotFound(block))?;
let mut parent = base_block.sealed_header().clone();
let this = self.clone();
self.spawn_with_state_at_block(block, move |state| {
let mut db =
State::builder().with_database(StateProviderDatabase::new(state)).build();
self.spawn_with_state_at_block(block, move |this, mut db| {
let mut blocks: Vec<SimulatedBlock<RpcBlock<Self::NetworkTypes>>> =
Vec::with_capacity(block_state_calls.len());
for block in block_state_calls {
@@ -277,11 +274,8 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA
replay_block_txs = false;
}
let this = self.clone();
self.spawn_with_state_at_block(at.into(), move |state| {
self.spawn_with_state_at_block(at, move |this, mut db| {
let mut all_results = Vec::with_capacity(bundles.len());
let mut db =
State::builder().with_database(StateProviderDatabase::new(state)).build();
if replay_block_txs {
// only need to replay the transactions in the block if not all transactions are
@@ -487,13 +481,11 @@ pub trait Call:
) -> impl Future<Output = Result<R, Self::Error>> + Send
where
R: Send + 'static,
F: FnOnce(Self, StateProviderTraitObjWrapper<'_>) -> Result<R, Self::Error>
+ Send
+ 'static,
F: FnOnce(Self, &dyn StateProvider) -> Result<R, Self::Error> + Send + 'static,
{
self.spawn_blocking_io_fut(move |this| async move {
let state = this.state_at_block_id(at).await?;
f(this, StateProviderTraitObjWrapper(&state))
f(this, &state)
})
}
@@ -552,16 +544,20 @@ pub trait Call:
/// Executes the closure with the state that corresponds to the given [`BlockId`] on a new task
fn spawn_with_state_at_block<F, R>(
&self,
at: BlockId,
at: impl Into<BlockId>,
f: F,
) -> impl Future<Output = Result<R, Self::Error>> + Send
where
F: FnOnce(StateProviderTraitObjWrapper<'_>) -> Result<R, Self::Error> + Send + 'static,
F: FnOnce(Self, StateCacheDb) -> Result<R, Self::Error> + Send + 'static,
R: Send + 'static,
{
let at = at.into();
self.spawn_blocking_io_fut(move |this| async move {
let state = this.state_at_block_id(at).await?;
f(StateProviderTraitObjWrapper(&state))
let db = State::builder()
.with_database(StateProviderDatabase::new(StateProviderTraitObjWrapper(state)))
.build();
f(this, db)
})
}
@@ -590,7 +586,7 @@ pub trait Call:
where
Self: LoadPendingBlock,
F: FnOnce(
StateCacheDbRefMutWrapper<'_, '_>,
&mut StateCacheDb,
EvmEnvFor<Self::Evm>,
TxEnvFor<Self::Evm>,
) -> Result<R, Self::Error>
@@ -600,17 +596,11 @@ pub trait Call:
{
async move {
let (evm_env, at) = self.evm_env_at(at).await?;
let this = self.clone();
self.spawn_blocking_io_fut(move |_| async move {
let state = this.state_at_block_id(at).await?;
let mut db = State::builder()
.with_database(StateProviderDatabase::new(StateProviderTraitObjWrapper(&state)))
.build();
self.spawn_with_state_at_block(at, move |this, mut db| {
let (evm_env, tx_env) =
this.prepare_call_env(evm_env, request, &mut db, overrides)?;
f(StateCacheDbRefMutWrapper(&mut db), evm_env, tx_env)
f(&mut db, evm_env, tx_env)
})
.await
}
@@ -635,7 +625,7 @@ pub trait Call:
F: FnOnce(
TransactionInfo,
ResultAndState<HaltReasonFor<Self::Evm>>,
StateCacheDb<'_>,
StateCacheDb,
) -> Result<R, Self::Error>
+ Send
+ 'static,
@@ -654,10 +644,7 @@ pub trait Call:
// block the transaction is included in
let parent_block = block.parent_hash();
let this = self.clone();
self.spawn_with_state_at_block(parent_block.into(), move |state| {
let mut db =
State::builder().with_database(StateProviderDatabase::new(state)).build();
self.spawn_with_state_at_block(parent_block, move |this, mut db| {
let block_txs = block.transactions_recovered();
// replay all transactions prior to the targeted transaction

67
crates/rpc/rpc-eth-api/src/helpers/trace.rs
View File

@@ -1,6 +1,6 @@
//! Loads a pending block from database. Helper trait for `eth_` call and trace RPC methods.
use super::{Call, LoadBlock, LoadPendingBlock, LoadState, LoadTransaction};
use super::{Call, LoadBlock, LoadState, LoadTransaction};
use crate::FromEvmError;
use alloy_consensus::{transaction::TxHashRef, BlockHeader};
use alloy_primitives::B256;
@@ -14,17 +14,14 @@ use reth_evm::{
};
use reth_primitives_traits::{BlockBody, Recovered, RecoveredBlock};
use reth_revm::{database::StateProviderDatabase, db::State};
use reth_rpc_eth_types::{
cache::db::{StateCacheDb, StateCacheDbRefMutWrapper, StateProviderTraitObjWrapper},
EthApiError,
};
use reth_rpc_eth_types::{cache::db::StateCacheDb, EthApiError};
use reth_storage_api::{ProviderBlock, ProviderTx};
use revm::{context::Block, context_interface::result::ResultAndState, DatabaseCommit};
use revm_inspectors::tracing::{TracingInspector, TracingInspectorConfig};
use std::sync::Arc;
/// Executes CPU heavy tasks.
pub trait Trace: LoadState<Error: FromEvmError<Self::Evm>> {
pub trait Trace: LoadState<Error: FromEvmError<Self::Evm>> + Call {
/// Executes the [`TxEnvFor`] with [`reth_evm::EvmEnv`] against the given [Database] without
/// committing state changes.
fn inspect<DB, I>(
@@ -58,7 +55,6 @@ pub trait Trace: LoadState<Error: FromEvmError<Self::Evm>> {
f: F,
) -> impl Future<Output = Result<R, Self::Error>> + Send
where
Self: Call,
R: Send + 'static,
F: FnOnce(
TracingInspector,
@@ -91,19 +87,16 @@ pub trait Trace: LoadState<Error: FromEvmError<Self::Evm>> {
f: F,
) -> impl Future<Output = Result<R, Self::Error>> + Send
where
Self: LoadPendingBlock + Call,
F: FnOnce(
TracingInspector,
ResultAndState<HaltReasonFor<Self::Evm>>,
StateCacheDb<'_>,
StateCacheDb,
) -> Result<R, Self::Error>
+ Send
+ 'static,
R: Send + 'static,
{
let this = self.clone();
self.spawn_with_state_at_block(at, move |state| {
let mut db = State::builder().with_database(StateProviderDatabase::new(state)).build();
self.spawn_with_state_at_block(at, move |this, mut db| {
let mut inspector = TracingInspector::new(config);
let res = this.inspect(&mut db, evm_env, tx_env, &mut inspector)?;
f(inspector, res, db)
@@ -126,12 +119,12 @@ pub trait Trace: LoadState<Error: FromEvmError<Self::Evm>> {
f: F,
) -> impl Future<Output = Result<Option<R>, Self::Error>> + Send
where
Self: LoadPendingBlock + LoadTransaction + Call,
Self: LoadTransaction,
F: FnOnce(
TransactionInfo,
TracingInspector,
ResultAndState<HaltReasonFor<Self::Evm>>,
StateCacheDb<'_>,
StateCacheDb,
) -> Result<R, Self::Error>
+ Send
+ 'static,
@@ -156,17 +149,16 @@ pub trait Trace: LoadState<Error: FromEvmError<Self::Evm>> {
f: F,
) -> impl Future<Output = Result<Option<R>, Self::Error>> + Send
where
Self: LoadPendingBlock + LoadTransaction + Call,
Self: LoadTransaction,
F: FnOnce(
TransactionInfo,
Insp,
ResultAndState<HaltReasonFor<Self::Evm>>,
StateCacheDb<'_>,
StateCacheDb,
) -> Result<R, Self::Error>
+ Send
+ 'static,
Insp:
for<'a, 'b> InspectorFor<Self::Evm, StateCacheDbRefMutWrapper<'a, 'b>> + Send + 'static,
Insp: for<'a> InspectorFor<Self::Evm, &'a mut StateCacheDb> + Send + 'static,
R: Send + 'static,
{
async move {
@@ -182,10 +174,7 @@ pub trait Trace: LoadState<Error: FromEvmError<Self::Evm>> {
// block the transaction is included in
let parent_block = block.parent_hash();
let this = self.clone();
self.spawn_with_state_at_block(parent_block.into(), move |state| {
let mut db =
State::builder().with_database(StateProviderDatabase::new(state)).build();
self.spawn_with_state_at_block(parent_block, move |this, mut db| {
let block_txs = block.transactions_recovered();
this.apply_pre_execution_changes(&block, &mut db, &evm_env)?;
@@ -194,12 +183,7 @@ pub trait Trace: LoadState<Error: FromEvmError<Self::Evm>> {
this.replay_transactions_until(&mut db, evm_env.clone(), block_txs, *tx.tx_hash())?;
let tx_env = this.evm_config().tx_env(tx);
let res = this.inspect(
StateCacheDbRefMutWrapper(&mut db),
evm_env,
tx_env,
&mut inspector,
)?;
let res = this.inspect(&mut db, evm_env, tx_env, &mut inspector)?;
f(tx_info, inspector, res, db)
})
.await
@@ -228,7 +212,7 @@ pub trait Trace: LoadState<Error: FromEvmError<Self::Evm>> {
TracingCtx<
'_,
Recovered<&ProviderTx<Self::Provider>>,
EvmFor<Self::Evm, StateCacheDbRefMutWrapper<'_, '_>, TracingInspector>,
EvmFor<Self::Evm, &mut StateCacheDb, TracingInspector>,
>,
) -> Result<R, Self::Error>
+ Send
@@ -269,13 +253,13 @@ pub trait Trace: LoadState<Error: FromEvmError<Self::Evm>> {
TracingCtx<
'_,
Recovered<&ProviderTx<Self::Provider>>,
EvmFor<Self::Evm, StateCacheDbRefMutWrapper<'_, '_>, Insp>,
EvmFor<Self::Evm, &mut StateCacheDb, Insp>,
>,
) -> Result<R, Self::Error>
+ Send
+ 'static,
Setup: FnMut() -> Insp + Send + 'static,
Insp: Clone + for<'a, 'b> InspectorFor<Self::Evm, StateCacheDbRefMutWrapper<'a, 'b>>,
Insp: Clone + for<'a> InspectorFor<Self::Evm, &'a mut StateCacheDb>,
R: Send + 'static,
{
async move {
@@ -296,21 +280,14 @@ pub trait Trace: LoadState<Error: FromEvmError<Self::Evm>> {
}
// replay all transactions of the block
self.spawn_blocking_io_fut(move |this| async move {
// we need to get the state of the parent block because we're replaying this block
// on top of its parent block's state
let state_at = block.parent_hash();
// we need to get the state of the parent block because we're replaying this block
// on top of its parent block's state
self.spawn_with_state_at_block(block.parent_hash(), move |this, mut db| {
let block_hash = block.hash();
let block_number = evm_env.block_env.number().saturating_to();
let base_fee = evm_env.block_env.basefee();
// now get the state
let state = this.state_at_block_id(state_at.into()).await?;
let mut db = State::builder()
.with_database(StateProviderDatabase::new(StateProviderTraitObjWrapper(&state)))
.build();
this.apply_pre_execution_changes(&block, &mut db, &evm_env)?;
// prepare transactions, we do everything upfront to reduce time spent with open
@@ -328,7 +305,7 @@ pub trait Trace: LoadState<Error: FromEvmError<Self::Evm>> {
let results = this
.evm_config()
.evm_factory()
.create_tracer(StateCacheDbRefMutWrapper(&mut db), evm_env, inspector_setup())
.create_tracer(&mut db, evm_env, inspector_setup())
.try_trace_many(block.transactions_recovered().take(max_transactions), |ctx| {
let tx_info = TransactionInfo {
hash: Some(*ctx.tx.tx_hash()),
@@ -375,7 +352,7 @@ pub trait Trace: LoadState<Error: FromEvmError<Self::Evm>> {
TracingCtx<
'_,
Recovered<&ProviderTx<Self::Provider>>,
EvmFor<Self::Evm, StateCacheDbRefMutWrapper<'_, '_>, TracingInspector>,
EvmFor<Self::Evm, &mut StateCacheDb, TracingInspector>,
>,
) -> Result<R, Self::Error>
+ Send
@@ -415,13 +392,13 @@ pub trait Trace: LoadState<Error: FromEvmError<Self::Evm>> {
TracingCtx<
'_,
Recovered<&ProviderTx<Self::Provider>>,
EvmFor<Self::Evm, StateCacheDbRefMutWrapper<'_, '_>, Insp>,
EvmFor<Self::Evm, &mut StateCacheDb, Insp>,
>,
) -> Result<R, Self::Error>
+ Send
+ 'static,
Setup: FnMut() -> Insp + Send + 'static,
Insp: Clone + for<'a, 'b> InspectorFor<Self::Evm, StateCacheDbRefMutWrapper<'a, 'b>>,
Insp: Clone + for<'a> InspectorFor<Self::Evm, &'a mut StateCacheDb>,
R: Send + 'static,
{
self.trace_block_until_with_inspector(block_id, block, None, insp_setup, f)

6
crates/rpc/rpc-eth-api/src/helpers/transaction.rs
View File

@@ -24,7 +24,7 @@ use reth_rpc_convert::{transaction::RpcConvert, RpcTxReq};
use reth_rpc_eth_types::{
utils::{binary_search, recover_raw_transaction},
EthApiError::{self, TransactionConfirmationTimeout},
FillTransactionResult, SignError, TransactionSource,
FillTransaction, SignError, TransactionSource,
};
use reth_storage_api::{
BlockNumReader, BlockReaderIdExt, ProviderBlock, ProviderReceipt, ProviderTx, ReceiptProvider,
@@ -450,7 +450,7 @@ pub trait EthTransactions: LoadTransaction<Provider: BlockReaderIdExt> {
fn fill_transaction(
&self,
mut request: RpcTxReq<Self::NetworkTypes>,
) -> impl Future<Output = Result<FillTransactionResult<TxTy<Self::Primitives>>, Self::Error>> + Send
) -> impl Future<Output = Result<FillTransaction<TxTy<Self::Primitives>>, Self::Error>> + Send
where
Self: EthApiSpec + LoadBlock + EstimateCall + LoadFee,
{
@@ -511,7 +511,7 @@ pub trait EthTransactions: LoadTransaction<Provider: BlockReaderIdExt> {
let raw = tx.encoded_2718().into();
Ok(FillTransactionResult { raw, tx })
Ok(FillTransaction { raw, tx })
}
}

90
crates/rpc/rpc-eth-types/src/cache/db.rs vendored
View File

@@ -4,25 +4,24 @@
use alloy_primitives::{Address, B256, U256};
use reth_errors::ProviderResult;
use reth_revm::{database::StateProviderDatabase, DatabaseRef};
use reth_storage_api::{BytecodeReader, HashedPostStateProvider, StateProvider};
use reth_revm::database::StateProviderDatabase;
use reth_storage_api::{BytecodeReader, HashedPostStateProvider, StateProvider, StateProviderBox};
use reth_trie::{HashedStorage, MultiProofTargets};
use revm::{
database::{BundleState, State},
primitives::HashMap,
state::{AccountInfo, Bytecode},
Database, DatabaseCommit,
};
use revm::database::{BundleState, State};
/// Helper alias type for the state's [`State`]
pub type StateCacheDb<'a> = State<StateProviderDatabase<StateProviderTraitObjWrapper<'a>>>;
pub type StateCacheDb = State<StateProviderDatabase<StateProviderTraitObjWrapper>>;
/// Hack to get around 'higher-ranked lifetime error', see
/// <https://github.com/rust-lang/rust/issues/100013>
///
/// Apparently, when dealing with our RPC code, compiler is struggling to prove lifetimes around
/// [`StateProvider`] trait objects. This type is a workaround which should help the compiler to
/// understand that there are no lifetimes involved.
#[expect(missing_debug_implementations)]
pub struct StateProviderTraitObjWrapper<'a>(pub &'a dyn StateProvider);
pub struct StateProviderTraitObjWrapper(pub StateProviderBox);
impl reth_storage_api::StateRootProvider for StateProviderTraitObjWrapper<'_> {
impl reth_storage_api::StateRootProvider for StateProviderTraitObjWrapper {
fn state_root(
&self,
hashed_state: reth_trie::HashedPostState,
@@ -52,7 +51,7 @@ impl reth_storage_api::StateRootProvider for StateProviderTraitObjWrapper<'_> {
}
}
impl reth_storage_api::StorageRootProvider for StateProviderTraitObjWrapper<'_> {
impl reth_storage_api::StorageRootProvider for StateProviderTraitObjWrapper {
fn storage_root(
&self,
address: Address,
@@ -80,7 +79,7 @@ impl reth_storage_api::StorageRootProvider for StateProviderTraitObjWrapper<'_>
}
}
impl reth_storage_api::StateProofProvider for StateProviderTraitObjWrapper<'_> {
impl reth_storage_api::StateProofProvider for StateProviderTraitObjWrapper {
fn proof(
&self,
input: reth_trie::TrieInput,
@@ -107,7 +106,7 @@ impl reth_storage_api::StateProofProvider for StateProviderTraitObjWrapper<'_> {
}
}
impl reth_storage_api::AccountReader for StateProviderTraitObjWrapper<'_> {
impl reth_storage_api::AccountReader for StateProviderTraitObjWrapper {
fn basic_account(
&self,
address: &Address,
@@ -116,7 +115,7 @@ impl reth_storage_api::AccountReader for StateProviderTraitObjWrapper<'_> {
}
}
impl reth_storage_api::BlockHashReader for StateProviderTraitObjWrapper<'_> {
impl reth_storage_api::BlockHashReader for StateProviderTraitObjWrapper {
fn block_hash(
&self,
block_number: alloy_primitives::BlockNumber,
@@ -140,13 +139,13 @@ impl reth_storage_api::BlockHashReader for StateProviderTraitObjWrapper<'_> {
}
}
impl HashedPostStateProvider for StateProviderTraitObjWrapper<'_> {
impl HashedPostStateProvider for StateProviderTraitObjWrapper {
fn hashed_post_state(&self, bundle_state: &BundleState) -> reth_trie::HashedPostState {
self.0.hashed_post_state(bundle_state)
}
}
impl StateProvider for StateProviderTraitObjWrapper<'_> {
impl StateProvider for StateProviderTraitObjWrapper {
fn storage(
&self,
account: Address,
@@ -171,7 +170,7 @@ impl StateProvider for StateProviderTraitObjWrapper<'_> {
}
}
impl BytecodeReader for StateProviderTraitObjWrapper<'_> {
impl BytecodeReader for StateProviderTraitObjWrapper {
fn bytecode_by_hash(
&self,
code_hash: &B256,
@@ -179,58 +178,3 @@ impl BytecodeReader for StateProviderTraitObjWrapper<'_> {
self.0.bytecode_by_hash(code_hash)
}
}
/// Hack to get around 'higher-ranked lifetime error', see
/// <https://github.com/rust-lang/rust/issues/100013>
pub struct StateCacheDbRefMutWrapper<'a, 'b>(pub &'b mut StateCacheDb<'a>);
impl<'a, 'b> core::fmt::Debug for StateCacheDbRefMutWrapper<'a, 'b> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("StateCacheDbRefMutWrapper").finish_non_exhaustive()
}
}
impl<'a> Database for StateCacheDbRefMutWrapper<'a, '_> {
type Error = <StateCacheDb<'a> as Database>::Error;
fn basic(&mut self, address: Address) -> Result<Option<AccountInfo>, Self::Error> {
self.0.basic(address)
}
fn code_by_hash(&mut self, code_hash: B256) -> Result<Bytecode, Self::Error> {
self.0.code_by_hash(code_hash)
}
fn storage(&mut self, address: Address, index: U256) -> Result<U256, Self::Error> {
self.0.storage(address, index)
}
fn block_hash(&mut self, number: u64) -> Result<B256, Self::Error> {
self.0.block_hash(number)
}
}
impl<'a> DatabaseRef for StateCacheDbRefMutWrapper<'a, '_> {
type Error = <StateCacheDb<'a> as Database>::Error;
fn basic_ref(&self, address: Address) -> Result<Option<AccountInfo>, Self::Error> {
self.0.basic_ref(address)
}
fn code_by_hash_ref(&self, code_hash: B256) -> Result<Bytecode, Self::Error> {
self.0.code_by_hash_ref(code_hash)
}
fn storage_ref(&self, address: Address, index: U256) -> Result<U256, Self::Error> {
self.0.storage_ref(address, index)
}
fn block_hash_ref(&self, number: u64) -> Result<B256, Self::Error> {
self.0.block_hash_ref(number)
}
}
impl DatabaseCommit for StateCacheDbRefMutWrapper<'_, '_> {
fn commit(&mut self, changes: HashMap<Address, revm::state::Account>) {
self.0.commit(changes)
}
}

3
crates/rpc/rpc-eth-types/src/lib.rs
View File

@@ -23,6 +23,7 @@ pub mod transaction;
pub mod tx_forward;
pub mod utils;
pub use alloy_rpc_types_eth::FillTransaction;
pub use builder::config::{EthConfig, EthFilterConfig};
pub use cache::{
config::EthStateCacheConfig, db::StateCacheDb, multi_consumer::MultiConsumerLruCache,
@@ -35,5 +36,5 @@ pub use gas_oracle::{
};
pub use id_provider::EthSubscriptionIdProvider;
pub use pending_block::{PendingBlock, PendingBlockEnv, PendingBlockEnvOrigin};
pub use transaction::{FillTransactionResult, TransactionSource};
pub use transaction::TransactionSource;
pub use tx_forward::ForwardConfig;

12
crates/rpc/rpc-eth-types/src/transaction.rs
View File

@@ -2,21 +2,11 @@
//!
//! Transaction wrapper that labels transaction with its origin.
use alloy_primitives::{Bytes, B256};
use alloy_primitives::B256;
use alloy_rpc_types_eth::TransactionInfo;
use reth_ethereum_primitives::TransactionSigned;
use reth_primitives_traits::{NodePrimitives, Recovered, SignedTransaction};
use reth_rpc_convert::{RpcConvert, RpcTransaction};
use serde::{Deserialize, Serialize};
/// Response type for `eth_fillTransaction` RPC method.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FillTransactionResult<T> {
/// RLP-encoded transaction bytes
pub raw: Bytes,
/// Filled transaction object
pub tx: T,
}
/// Represents from where a transaction was fetched.
#[derive(Debug, Clone, Eq, PartialEq)]

78
crates/rpc/rpc/src/debug.rs
View File

@@ -22,7 +22,7 @@ use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks};
use reth_errors::RethError;
use reth_evm::{execute::Executor, ConfigureEvm, EvmEnvFor, TxEnvFor};
use reth_primitives_traits::{Block as _, BlockBody, ReceiptWithBloom, RecoveredBlock};
use reth_revm::{database::StateProviderDatabase, db::State, witness::ExecutionWitnessRecord};
use reth_revm::{db::State, witness::ExecutionWitnessRecord};
use reth_rpc_api::DebugApiServer;
use reth_rpc_convert::RpcTxReq;
use reth_rpc_eth_api::{
@@ -91,22 +91,20 @@ where
evm_env: EvmEnvFor<Eth::Evm>,
opts: GethDebugTracingOptions,
) -> Result<Vec<TraceResult>, Eth::Error> {
// replay all transactions of the block
let this = self.clone();
// replay all transactions of the block
self.eth_api()
.spawn_with_state_at_block(block.parent_hash().into(), move |state| {
.spawn_with_state_at_block(block.parent_hash(), move |eth_api, mut db| {
let mut results = Vec::with_capacity(block.body().transactions().len());
let mut db =
State::builder().with_database(StateProviderDatabase::new(state)).build();
this.eth_api().apply_pre_execution_changes(&block, &mut db, &evm_env)?;
eth_api.apply_pre_execution_changes(&block, &mut db, &evm_env)?;
let mut transactions = block.transactions_recovered().enumerate().peekable();
let mut inspector = None;
while let Some((index, tx)) = transactions.next() {
let tx_hash = *tx.tx_hash();
let tx_env = this.eth_api().evm_config().tx_env(tx);
let tx_env = eth_api.evm_config().tx_env(tx);
let (result, state_changes) = this.trace_transaction(
&opts,
@@ -221,26 +219,23 @@ where
let this = self.clone();
self.eth_api()
.spawn_with_state_at_block(state_at, move |state| {
.spawn_with_state_at_block(state_at, move |eth_api, mut db| {
let block_txs = block.transactions_recovered();
// configure env for the target transaction
let tx = transaction.into_recovered();
let mut db =
State::builder().with_database(StateProviderDatabase::new(state)).build();
this.eth_api().apply_pre_execution_changes(&block, &mut db, &evm_env)?;
eth_api.apply_pre_execution_changes(&block, &mut db, &evm_env)?;
// replay all transactions prior to the targeted transaction
let index = this.eth_api().replay_transactions_until(
let index = eth_api.replay_transactions_until(
&mut db,
evm_env.clone(),
block_txs,
*tx.tx_hash(),
)?;
let tx_env = this.eth_api().evm_config().tx_env(&tx);
let tx_env = eth_api.evm_config().tx_env(&tx);
this.trace_transaction(
&opts,
@@ -343,10 +338,6 @@ where
let frame = self
.eth_api()
.spawn_with_call_at(call, at, overrides, move |db, evm_env, tx_env| {
// wrapper is hack to get around 'higher-ranked lifetime error',
// see <https://github.com/rust-lang/rust/issues/100013>
let db = db.0;
let gas_limit = tx_env.gas_limit();
let res = this.eth_api().inspect(
&mut *db,
@@ -377,10 +368,6 @@ where
.inner
.eth_api
.spawn_with_call_at(call, at, overrides, move |db, evm_env, tx_env| {
// wrapper is hack to get around 'higher-ranked lifetime error', see
// <https://github.com/rust-lang/rust/issues/100013>
let db = db.0;
let tx_info = TransactionInfo {
block_number: Some(evm_env.block_env.number().saturating_to()),
base_fee: Some(evm_env.block_env.basefee()),
@@ -448,10 +435,6 @@ where
let res = self
.eth_api()
.spawn_with_call_at(call, at, overrides, move |db, evm_env, tx_env| {
// wrapper is hack to get around 'higher-ranked lifetime error', see
// <https://github.com/rust-lang/rust/issues/100013>
let db = db.0;
let mut inspector =
revm_inspectors::tracing::js::JsInspector::new(code, config)
.map_err(Eth::Error::from_eth_err)?;
@@ -531,27 +514,24 @@ where
let (evm_env, _) = self.eth_api().evm_env_at(block.hash().into()).await?;
// execute after the parent block, replaying `tx_index` transactions
let state_at = block.parent_hash().into();
let state_at = block.parent_hash();
let this = self.clone();
self.eth_api()
.spawn_with_state_at_block(state_at, move |state| {
let mut db =
State::builder().with_database(StateProviderDatabase::new(state)).build();
.spawn_with_state_at_block(state_at, move |eth_api, mut db| {
// 1. apply pre-execution changes
this.eth_api().apply_pre_execution_changes(&block, &mut db, &evm_env)?;
eth_api.apply_pre_execution_changes(&block, &mut db, &evm_env)?;
// 2. replay the required number of transactions
for tx in block.transactions_recovered().take(tx_index) {
let tx_env = this.eth_api().evm_config().tx_env(tx);
let res = this.eth_api().transact(&mut db, evm_env.clone(), tx_env)?;
let tx_env = eth_api.evm_config().tx_env(tx);
let res = eth_api.transact(&mut db, evm_env.clone(), tx_env)?;
db.commit(res.state);
}
// 3. now execute the trace call on this state
let (call_evm_env, call_tx_env) =
this.eth_api().prepare_call_env(evm_env, call, &mut db, overrides)?;
eth_api.prepare_call_env(evm_env, call, &mut db, overrides)?;
// Execute the trace call using trace_transaction
let (trace, _) = this.trace_transaction(
@@ -613,11 +593,9 @@ where
let this = self.clone();
self.eth_api()
.spawn_with_state_at_block(at.into(), move |state| {
.spawn_with_state_at_block(at, move |eth_api, mut db| {
// the outer vec for the bundles
let mut all_bundles = Vec::with_capacity(bundles.len());
let mut db =
State::builder().with_database(StateProviderDatabase::new(state)).build();
if replay_block_txs {
// only need to replay the transactions in the block if not all transactions are
@@ -626,8 +604,8 @@ where
// Execute all transactions until index
for tx in transactions {
let tx_env = this.eth_api().evm_config().tx_env(tx);
let res = this.eth_api().transact(&mut db, evm_env.clone(), tx_env)?;
let tx_env = eth_api.evm_config().tx_env(tx);
let res = eth_api.transact(&mut db, evm_env.clone(), tx_env)?;
db.commit(res.state);
}
}
@@ -647,12 +625,8 @@ where
let state_overrides = state_overrides.take();
let overrides = EvmOverrides::new(state_overrides, block_overrides.clone());
let (evm_env, tx_env) = this.eth_api().prepare_call_env(
evm_env.clone(),
tx,
&mut db,
overrides,
)?;
let (evm_env, tx_env) =
eth_api.prepare_call_env(evm_env.clone(), tx, &mut db, overrides)?;
let (trace, state) = this.trace_transaction(
&tracing_options,
@@ -722,14 +696,12 @@ where
&self,
block: Arc<RecoveredBlock<ProviderBlock<Eth::Provider>>>,
) -> Result<ExecutionWitness, Eth::Error> {
let this = self.clone();
let block_number = block.header().number();
let (mut exec_witness, lowest_block_number) = self
.eth_api()
.spawn_with_state_at_block(block.parent_hash().into(), move |state_provider| {
let db = StateProviderDatabase::new(&state_provider);
let block_executor = this.eth_api().evm_config().executor(db);
.spawn_with_state_at_block(block.parent_hash(), move |eth_api, mut db| {
let block_executor = eth_api.evm_config().executor(&mut db);
let mut witness_record = ExecutionWitnessRecord::default();
@@ -742,7 +714,9 @@ where
let ExecutionWitnessRecord { hashed_state, codes, keys, lowest_block_number } =
witness_record;
let state = state_provider
let state = db
.database
.0
.witness(Default::default(), hashed_state)
.map_err(EthApiError::from)?;
Ok((
@@ -812,7 +786,7 @@ where
opts: &GethDebugTracingOptions,
evm_env: EvmEnvFor<Eth::Evm>,
tx_env: TxEnvFor<Eth::Evm>,
db: &mut StateCacheDb<'_>,
db: &mut StateCacheDb,
transaction_context: Option<TransactionContext>,
fused_inspector: &mut Option<TracingInspector>,
) -> Result<(GethTrace, EvmState), Eth::Error> {

6
crates/rpc/rpc/src/eth/bundle.rs
View File

@@ -8,7 +8,6 @@ use alloy_rpc_types_mev::{EthCallBundle, EthCallBundleResponse, EthCallBundleTra
use jsonrpsee::core::RpcResult;
use reth_chainspec::{ChainSpecProvider, EthChainSpec};
use reth_evm::{ConfigureEvm, Evm};
use reth_revm::{database::StateProviderDatabase, State};
use reth_rpc_eth_api::{
helpers::{Call, EthTransactions, LoadPendingBlock},
EthCallBundleApiServer, FromEthApiError, FromEvmError,
@@ -144,13 +143,10 @@ where
// use the block number of the request
evm_env.block_env.inner_mut().number = U256::from(block_number);
let eth_api = self.eth_api().clone();
self.eth_api()
.spawn_with_state_at_block(at, move |state| {
.spawn_with_state_at_block(at, move |eth_api, db| {
let coinbase = evm_env.block_env.beneficiary();
let basefee = evm_env.block_env.basefee();
let db = State::builder().with_database(StateProviderDatabase::new(state)).build();
let initial_coinbase = db
.basic_ref(coinbase)

2
crates/rpc/rpc/src/eth/helpers/trace.rs
View File

@@ -10,6 +10,6 @@ impl<N, Rpc> Trace for EthApi<N, Rpc>
where
N: RpcNodeCore,
EthApiError: FromEvmError<N::Evm>,
Rpc: RpcConvert<Primitives = N::Primitives>,
Rpc: RpcConvert<Primitives = N::Primitives, Error = EthApiError, Evm = N::Evm>,
{
}

5
crates/rpc/rpc/src/eth/sim_bundle.rs
View File

@@ -12,7 +12,6 @@ use alloy_rpc_types_mev::{
use jsonrpsee::core::RpcResult;
use reth_evm::{ConfigureEvm, Evm};
use reth_primitives_traits::Recovered;
use reth_revm::{database::StateProviderDatabase, State};
use reth_rpc_api::MevSimApiServer;
use reth_rpc_eth_api::{
helpers::{block::LoadBlock, Call, EthTransactions},
@@ -241,13 +240,11 @@ where
let sim_response = self
.inner
.eth_api
.spawn_with_state_at_block(current_block_id, move |state| {
.spawn_with_state_at_block(current_block_id, move |_, mut db| {
// Setup environment
let current_block_number = current_block.number();
let coinbase = evm_env.block_env.beneficiary();
let basefee = evm_env.block_env.basefee();
let mut db =
State::builder().with_database(StateProviderDatabase::new(state)).build();
// apply overrides
apply_block_overrides(block_overrides, &mut db, evm_env.block_env.inner_mut());

15
crates/rpc/rpc/src/trace.rs
View File

@@ -20,7 +20,6 @@ use jsonrpsee::core::RpcResult;
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardfork, MAINNET, SEPOLIA};
use reth_evm::ConfigureEvm;
use reth_primitives_traits::{BlockBody, BlockHeader};
use reth_revm::{database::StateProviderDatabase, State};
use reth_rpc_api::TraceApiServer;
use reth_rpc_convert::RpcTxReq;
use reth_rpc_eth_api::{
@@ -102,10 +101,6 @@ where
let this = self.clone();
self.eth_api()
.spawn_with_call_at(trace_request.call, at, overrides, move |db, evm_env, tx_env| {
// wrapper is hack to get around 'higher-ranked lifetime error', see
// <https://github.com/rust-lang/rust/issues/100013>
let db = db.0;
let res = this.eth_api().inspect(&mut *db, evm_env, tx_env, &mut inspector)?;
let trace_res = inspector
.into_parity_builder()
@@ -153,18 +148,14 @@ where
let at = block_id.unwrap_or(BlockId::pending());
let (evm_env, at) = self.eth_api().evm_env_at(at).await?;
let this = self.clone();
// execute all transactions on top of each other and record the traces
self.eth_api()
.spawn_with_state_at_block(at, move |state| {
.spawn_with_state_at_block(at, move |eth_api, mut db| {
let mut results = Vec::with_capacity(calls.len());
let mut db =
State::builder().with_database(StateProviderDatabase::new(state)).build();
let mut calls = calls.into_iter().peekable();
while let Some((call, trace_types)) = calls.next() {
let (evm_env, tx_env) = this.eth_api().prepare_call_env(
let (evm_env, tx_env) = eth_api.prepare_call_env(
evm_env.clone(),
call,
&mut db,
@@ -172,7 +163,7 @@ where
)?;
let config = TracingInspectorConfig::from_parity_config(&trace_types);
let mut inspector = TracingInspector::new(config);
let res = this.eth_api().inspect(&mut db, evm_env, tx_env, &mut inspector)?;
let res = eth_api.inspect(&mut db, evm_env, tx_env, &mut inspector)?;
let trace_res = inspector
.into_parity_builder()

8
crates/stages/api/src/pipeline/builder.rs
View File

@@ -35,11 +35,9 @@ impl<Provider> PipelineBuilder<Provider> {
/// [`builder`][StageSet::builder] on the set which will convert it to a
/// [`StageSetBuilder`][crate::StageSetBuilder].
pub fn add_stages<Set: StageSet<Provider>>(mut self, set: Set) -> Self {
let states = set.builder().build();
self.stages.reserve_exact(states.len());
for stage in states {
self.stages.push(stage);
}
let stages = set.builder().build();
self.stages.reserve(stages.len());
self.stages.extend(stages);
self
}

5
crates/stages/stages/src/stages/bodies.rs
View File

@@ -772,8 +772,9 @@ mod tests {
*range.start()..*range.end() + 1,
|cursor, number| cursor.get_two::<HeaderWithHashMask<Header>>(number.into()),
)? {
let (header, hash) = header?;
self.headers.push_back(SealedHeader::new(header, hash));
if let Some((header, hash)) = header? {
self.headers.push_back(SealedHeader::new(header, hash));
}
}
Ok(())

9
crates/stages/stages/src/stages/execution.rs
View File

@@ -653,8 +653,9 @@ where
*range.start()..*range.end() + 1,
|cursor, number| cursor.get_one::<HeaderMask<N::BlockHeader>>(number.into()),
)? {
let entry = entry?;
gas_total += entry.gas_used();
if let Some(entry) = entry? {
gas_total += entry.gas_used();
}
}
let duration = start.elapsed();
@@ -1252,7 +1253,9 @@ mod tests {
// but no receipt data is written.
let factory = create_test_provider_factory();
factory.set_storage_settings_cache(StorageSettings::new().with_receipts_in_static_files());
factory.set_storage_settings_cache(
StorageSettings::legacy().with_receipts_in_static_files(true),
);
// Setup with block 1
let provider_rw = factory.database_provider_rw().unwrap();

68
crates/stages/stages/src/stages/sender_recovery.rs
View File

@@ -1,4 +1,4 @@
use alloy_primitives::{Address, TxNumber};
use alloy_primitives::{Address, BlockNumber, TxNumber};
use reth_config::config::SenderRecoveryConfig;
use reth_consensus::ConsensusError;
use reth_db::static_file::TransactionMask;
@@ -11,8 +11,8 @@ use reth_db_api::{
};
use reth_primitives_traits::{GotExpected, NodePrimitives, SignedTransaction};
use reth_provider::{
BlockReader, DBProvider, HeaderProvider, ProviderError, PruneCheckpointReader,
StaticFileProviderFactory, StatsReader,
BlockReader, DBProvider, EitherWriter, HeaderProvider, ProviderError, PruneCheckpointReader,
StaticFileProviderFactory, StatsReader, StorageSettingsCache, TransactionsProvider,
};
use reth_prune_types::PruneSegment;
use reth_stages_api::{
@@ -20,7 +20,7 @@ use reth_stages_api::{
StageId, UnwindInput, UnwindOutput,
};
use reth_static_file_types::StaticFileSegment;
use std::{fmt::Debug, ops::Range, sync::mpsc};
use std::{fmt::Debug, ops::Range, sync::mpsc, time::Instant};
use thiserror::Error;
use tracing::*;
@@ -64,7 +64,8 @@ where
+ BlockReader
+ StaticFileProviderFactory<Primitives: NodePrimitives<SignedTx: Value + SignedTransaction>>
+ StatsReader
+ PruneCheckpointReader,
+ PruneCheckpointReader
+ StorageSettingsCache,
{
/// Return the id of the stage
fn id(&self) -> StageId {
@@ -74,7 +75,8 @@ where
/// Retrieve the range of transactions to iterate over by querying
/// [`BlockBodyIndices`][reth_db_api::tables::BlockBodyIndices],
/// collect transactions within that range, recover signer for each transaction and store
/// entries in the [`TransactionSenders`][reth_db_api::tables::TransactionSenders] table.
/// entries in the [`TransactionSenders`][reth_db_api::tables::TransactionSenders] table or
/// static files depending on configuration.
fn execute(&mut self, provider: &Provider, input: ExecInput) -> Result<ExecOutput, StageError> {
if input.target_reached() {
return Ok(ExecOutput::done(input.checkpoint()))
@@ -84,6 +86,14 @@ where
input.next_block_range_with_transaction_threshold(provider, self.commit_threshold)?
else {
info!(target: "sync::stages::sender_recovery", "No transaction senders to recover");
EitherWriter::new_senders(
provider,
provider
.static_file_provider()
.get_highest_static_file_block(StaticFileSegment::TransactionSenders)
.unwrap_or_default(),
)?
.ensure_at_block(input.target())?;
return Ok(ExecOutput {
checkpoint: StageCheckpoint::new(input.target())
.with_entities_stage_checkpoint(stage_checkpoint(provider)?),
@@ -92,8 +102,7 @@ where
};
let end_block = *range_output.block_range.end();
// Acquire the cursor for inserting elements
let mut senders_cursor = provider.tx_ref().cursor_write::<tables::TransactionSenders>()?;
let mut writer = EitherWriter::new_senders(provider, *range_output.block_range.start())?;
info!(target: "sync::stages::sender_recovery", tx_range = ?range_output.tx_range, "Recovering senders");
@@ -107,8 +116,28 @@ where
let tx_batch_sender = setup_range_recovery(provider);
let start = Instant::now();
let block_body_indices =
provider.block_body_indices_range(range_output.block_range.clone())?;
let block_body_indices_elapsed = start.elapsed();
let mut blocks_with_indices = range_output.block_range.zip(block_body_indices).peekable();
for range in batch {
recover_range(range, provider, tx_batch_sender.clone(), &mut senders_cursor)?;
// Pair each transaction number with its block number
let start = Instant::now();
let block_numbers = range.clone().fold(Vec::new(), |mut block_numbers, tx| {
while let Some((block, index)) = blocks_with_indices.peek() {
if index.contains_tx(tx) {
block_numbers.push(*block);
return block_numbers
}
blocks_with_indices.next();
}
block_numbers
});
let fold_elapsed = start.elapsed();
debug!(target: "sync::stages::sender_recovery", ?block_body_indices_elapsed, ?fold_elapsed, len = block_numbers.len(), "Calculated block numbers");
recover_range(range, block_numbers, provider, tx_batch_sender.clone(), &mut writer)?;
}
Ok(ExecOutput {
@@ -141,15 +170,22 @@ where
}
fn recover_range<Provider, CURSOR>(
tx_range: Range<u64>,
tx_range: Range<TxNumber>,
block_numbers: Vec<BlockNumber>,
provider: &Provider,
tx_batch_sender: mpsc::Sender<Vec<(Range<u64>, RecoveryResultSender)>>,
senders_cursor: &mut CURSOR,
writer: &mut EitherWriter<'_, CURSOR, Provider::Primitives>,
) -> Result<(), StageError>
where
Provider: DBProvider + HeaderProvider + StaticFileProviderFactory,
Provider: DBProvider + HeaderProvider + TransactionsProvider + StaticFileProviderFactory,
CURSOR: DbCursorRW<tables::TransactionSenders>,
{
debug_assert_eq!(
tx_range.clone().count(),
block_numbers.len(),
"Transaction range and block numbers count mismatch"
);
debug!(target: "sync::stages::sender_recovery", ?tx_range, "Sending batch for processing");
// Preallocate channels for each chunks in the batch
@@ -171,6 +207,7 @@ where
debug!(target: "sync::stages::sender_recovery", ?tx_range, "Appending recovered senders to the database");
let mut processed_transactions = 0;
let mut block_numbers = block_numbers.into_iter();
for channel in receivers {
while let Ok(recovered) = channel.recv() {
let (tx_id, sender) = match recovered {
@@ -208,7 +245,12 @@ where
}
}
};
senders_cursor.append(tx_id, &sender)?;
let new_block_number = block_numbers
.next()
.expect("block numbers iterator has the same length as the number of transactions");
writer.ensure_at_block(new_block_number)?;
writer.append_sender(tx_id, &sender)?;
processed_transactions += 1;
}
}

24
crates/static-file/types/src/segment.rs
View File

@@ -2,7 +2,6 @@ use crate::{BlockNumber, Compression};
use alloc::{format, string::String};
use alloy_primitives::TxNumber;
use core::{ops::RangeInclusive, str::FromStr};
use derive_more::Display;
use serde::{Deserialize, Serialize};
use strum::{EnumIs, EnumString};
@@ -18,7 +17,7 @@ use strum::{EnumIs, EnumString};
Deserialize,
Serialize,
EnumString,
Display,
derive_more::Display,
EnumIs,
)]
#[strum(serialize_all = "kebab-case")]
@@ -32,10 +31,12 @@ pub enum StaticFileSegment {
Transactions,
/// Static File segment responsible for the `Receipts` table.
Receipts,
/// Static File segment responsible for the `TransactionSenders` table.
TransactionSenders,
}
impl StaticFileSegment {
/// Returns the segment as a string.
/// Returns a string representation of the segment.
pub const fn as_str(&self) -> &'static str {
// `strum` doesn't generate a doc comment for `into_str` when using `IntoStaticStr` derive
// macro, so we need to manually implement it.
@@ -46,13 +47,14 @@ impl StaticFileSegment {
Self::Headers => "headers",
Self::Transactions => "transactions",
Self::Receipts => "receipts",
Self::TransactionSenders => "transaction-senders",
}
}
/// Returns an iterator over all segments.
pub fn iter() -> impl Iterator<Item = Self> {
// The order of segments is significant and must be maintained to ensure correctness.
[Self::Headers, Self::Transactions, Self::Receipts].into_iter()
[Self::Headers, Self::Transactions, Self::Receipts, Self::TransactionSenders].into_iter()
}
/// Returns the default configuration of the segment.
@@ -64,7 +66,7 @@ impl StaticFileSegment {
pub const fn columns(&self) -> usize {
match self {
Self::Headers => 3,
Self::Transactions | Self::Receipts => 1,
Self::Transactions | Self::Receipts | Self::TransactionSenders => 1,
}
}
@@ -125,7 +127,7 @@ impl StaticFileSegment {
/// Returns `true` if a segment row is linked to a transaction.
pub const fn is_tx_based(&self) -> bool {
match self {
Self::Receipts | Self::Transactions => true,
Self::Receipts | Self::Transactions | Self::TransactionSenders => true,
Self::Headers => false,
}
}
@@ -134,7 +136,7 @@ impl StaticFileSegment {
pub const fn is_block_based(&self) -> bool {
match self {
Self::Headers => true,
Self::Receipts | Self::Transactions => false,
Self::Receipts | Self::Transactions | Self::TransactionSenders => false,
}
}
}
@@ -450,6 +452,12 @@ mod tests {
tx_range: Some(SegmentRangeInclusive::new(0, 300)),
segment: StaticFileSegment::Receipts,
},
SegmentHeader {
expected_block_range: SegmentRangeInclusive::new(0, 200),
block_range: Some(SegmentRangeInclusive::new(0, 100)),
tx_range: Some(SegmentRangeInclusive::new(0, 300)),
segment: StaticFileSegment::TransactionSenders,
},
];
// Check that we test all segments
assert_eq!(
@@ -481,6 +489,7 @@ mod tests {
StaticFileSegment::Headers => "headers",
StaticFileSegment::Transactions => "transactions",
StaticFileSegment::Receipts => "receipts",
StaticFileSegment::TransactionSenders => "transaction-senders",
};
assert_eq!(static_str, expected_str);
}
@@ -497,6 +506,7 @@ mod tests {
StaticFileSegment::Headers => "Headers",
StaticFileSegment::Transactions => "Transactions",
StaticFileSegment::Receipts => "Receipts",
StaticFileSegment::TransactionSenders => "TransactionSenders",
};
assert_eq!(ser, format!("\"{expected_str}\""));
}

5
crates/static-file/types/src/snapshots/reth_static_file_types__segment__tests__TransactionSenders.snap Normal file
View File

@@ -0,0 +1,5 @@
---
source: crates/static-file/types/src/segment.rs
expression: "Bytes::from(serialized)"
---
0x01000000000000000000000000000000c80000000000000001000000000000000064000000000000000100000000000000002c010000000000000300000001000000000000000000000000000000000000000000000000

3
crates/storage/codecs/src/alloy/mod.rs
View File

@@ -24,6 +24,9 @@ cond_mod!(
withdrawal
);
#[cfg(all(feature = "op", feature = "std"))]
pub mod optimism;
pub mod transaction;
#[cfg(test)]

97
crates/storage/codecs/src/alloy/optimism.rs Normal file
View File

@@ -0,0 +1,97 @@
//! Compact implementations for Optimism types.
use crate::Compact;
use alloc::{borrow::Cow, vec::Vec};
use alloy_consensus::{Receipt, TxReceipt};
use alloy_primitives::Log;
use op_alloy_consensus::{OpDepositReceipt, OpReceipt, OpTxType};
use reth_codecs_derive::CompactZstd;
#[derive(CompactZstd)]
#[reth_codecs(crate = "crate")]
#[reth_zstd(
compressor = reth_zstd_compressors::RECEIPT_COMPRESSOR,
decompressor = reth_zstd_compressors::RECEIPT_DECOMPRESSOR
)]
struct CompactOpReceipt<'a> {
tx_type: OpTxType,
success: bool,
cumulative_gas_used: u64,
#[expect(clippy::owned_cow)]
logs: Cow<'a, Vec<Log>>,
deposit_nonce: Option<u64>,
deposit_receipt_version: Option<u64>,
}
impl<'a> From<&'a OpReceipt> for CompactOpReceipt<'a> {
fn from(receipt: &'a OpReceipt) -> Self {
Self {
tx_type: receipt.tx_type(),
success: receipt.status(),
cumulative_gas_used: receipt.cumulative_gas_used(),
logs: Cow::Borrowed(&receipt.as_receipt().logs),
deposit_nonce: if let OpReceipt::Deposit(receipt) = receipt {
receipt.deposit_nonce
} else {
None
},
deposit_receipt_version: if let OpReceipt::Deposit(receipt) = receipt {
receipt.deposit_receipt_version
} else {
None
},
}
}
}
impl From<CompactOpReceipt<'_>> for OpReceipt {
fn from(receipt: CompactOpReceipt<'_>) -> Self {
let CompactOpReceipt {
tx_type,
success,
cumulative_gas_used,
logs,
deposit_nonce,
deposit_receipt_version,
} = receipt;
let inner =
Receipt { status: success.into(), cumulative_gas_used, logs: logs.into_owned() };
match tx_type {
OpTxType::Legacy => Self::Legacy(inner),
OpTxType::Eip2930 => Self::Eip2930(inner),
OpTxType::Eip1559 => Self::Eip1559(inner),
OpTxType::Eip7702 => Self::Eip7702(inner),
OpTxType::Deposit => {
Self::Deposit(OpDepositReceipt { inner, deposit_nonce, deposit_receipt_version })
}
}
}
}
impl Compact for OpReceipt {
fn to_compact<B>(&self, buf: &mut B) -> usize
where
B: bytes::BufMut + AsMut<[u8]>,
{
CompactOpReceipt::from(self).to_compact(buf)
}
fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) {
let (receipt, buf) = CompactOpReceipt::from_compact(buf, len);
(receipt.into(), buf)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{test_utils::UnusedBits, validate_bitflag_backwards_compat};
#[test]
fn test_ensure_backwards_compatibility() {
assert_eq!(CompactOpReceipt::bitflag_encoded_bytes(), 2);
validate_bitflag_backwards_compat!(CompactOpReceipt<'_>, UnusedBits::NotZero);
}
}

30
crates/storage/db-api/src/models/metadata.rs
View File

@@ -7,33 +7,39 @@ use serde::{Deserialize, Serialize};
///
/// These should be set during `init_genesis` or `init_db` depending on whether we want dictate
/// behaviour of new or old nodes respectively.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, Compact)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Compact)]
#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
#[add_arbitrary_tests(compact)]
pub struct StorageSettings {
/// Whether this node always writes receipts to static files.
///
/// If this is set to FALSE AND receipt pruning IS ENABLED, all receipts should be written to DB. Otherwise, they should be written to static files. This ensures that older nodes do not need to migrate their current DB tables to static files. For more, read: <https://github.com/paradigmxyz/reth/issues/18890#issuecomment-3457760097>
#[serde(default)]
pub receipts_in_static_files: bool,
/// Whether this node always writes transaction senders to static files.
#[serde(default)]
pub transaction_senders_in_static_files: bool,
}
impl StorageSettings {
/// Creates a new `StorageSettings` with default values.
pub const fn new() -> Self {
Self { receipts_in_static_files: false }
}
/// Creates `StorageSettings` for legacy nodes.
///
/// This explicitly sets `receipts_in_static_files` to `false`, ensuring older nodes
/// continue writing receipts to the database when receipt pruning is enabled.
/// This explicitly sets `receipts_in_static_files` and `transaction_senders_in_static_files` to
/// `false`, ensuring older nodes continue writing receipts and transaction senders to the
/// database when receipt pruning is enabled.
pub const fn legacy() -> Self {
Self { receipts_in_static_files: false }
Self { receipts_in_static_files: false, transaction_senders_in_static_files: false }
}
/// Sets the `receipts_static_files` flag to true.
pub const fn with_receipts_in_static_files(mut self) -> Self {
self.receipts_in_static_files = true;
/// Sets the `receipts_in_static_files` flag to the provided value.
pub const fn with_receipts_in_static_files(mut self, value: bool) -> Self {
self.receipts_in_static_files = value;
self
}
/// Sets the `transaction_senders_in_static_files` flag to the provided value.
pub const fn with_transaction_senders_in_static_files(mut self, value: bool) -> Self {
self.transaction_senders_in_static_files = value;
self
}
}

3
crates/storage/db-api/src/transaction.rs
View File

@@ -5,6 +5,9 @@ use crate::{
};
use std::fmt::Debug;
/// Helper adapter type for accessing [`DbTx`] cursor.
pub type CursorTy<TX, T> = <TX as DbTx>::Cursor<T>;
/// Helper adapter type for accessing [`DbTxMut`] mutable cursor.
pub type CursorMutTy<TX, T> = <TX as DbTxMut>::CursorMut<T>;

5
crates/storage/db-models/src/blocks.rs
View File

@@ -64,6 +64,11 @@ impl StoredBlockBodyIndices {
pub const fn tx_count(&self) -> NumTransactions {
self.tx_count
}
/// Returns true if the block contains a transaction with the given number.
pub const fn contains_tx(&self, tx_num: TxNumber) -> bool {
tx_num >= self.first_tx_num && tx_num < self.next_tx_num()
}
}
/// The storage representation of block withdrawals.

12
crates/storage/db/src/static_file/masks.rs
View File

@@ -3,7 +3,7 @@ use crate::{
static_file::mask::{ColumnSelectorOne, ColumnSelectorTwo},
HeaderTerminalDifficulties,
};
use alloy_primitives::BlockHash;
use alloy_primitives::{Address, BlockHash};
use reth_db_api::table::Table;
// HEADER MASKS
@@ -33,12 +33,18 @@ add_static_file_mask! {
// RECEIPT MASKS
add_static_file_mask! {
#[doc = "Mask for selecting a single receipt from Receipts static file segment"]
#[doc = "Mask for selecting a single receipt from `Receipts` static file segment"]
ReceiptMask<R>, R, 0b1
}
// TRANSACTION MASKS
add_static_file_mask! {
#[doc = "Mask for selecting a single transaction from Transactions static file segment"]
#[doc = "Mask for selecting a single transaction from `Transactions` static file segment"]
TransactionMask<T>, T, 0b1
}
// TRANSACTION SENDER MASKS
add_static_file_mask! {
#[doc = "Mask for selecting a single transaction sender from `TransactionSenders` static file segment"]
TransactionSenderMask, Address, 0b1
}

314
crates/storage/provider/src/either_writer.rs
View File

@@ -1,18 +1,31 @@
//! Generic writer abstraction for writing to either database tables or static files.
//! Generic reader and writer abstractions for interacting with either database tables or static
//! files.
use crate::{providers::StaticFileProviderRWRefMut, StaticFileProviderFactory};
use alloy_primitives::{BlockNumber, TxNumber};
use std::ops::Range;
use crate::{
providers::{StaticFileProvider, StaticFileProviderRWRefMut},
StaticFileProviderFactory,
};
use alloy_primitives::{map::HashMap, Address, BlockNumber, TxNumber};
use reth_db::{
cursor::DbCursorRO,
static_file::TransactionSenderMask,
table::Value,
transaction::{CursorMutTy, DbTxMut},
transaction::{CursorMutTy, CursorTy, DbTx, DbTxMut},
};
use reth_db_api::{cursor::DbCursorRW, tables};
use reth_errors::ProviderError;
use reth_node_types::NodePrimitives;
use reth_primitives_traits::ReceiptTy;
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::{DBProvider, NodePrimitivesProvider, StorageSettingsCache};
use reth_storage_errors::provider::ProviderResult;
use strum::EnumIs;
use strum::{Display, EnumIs};
/// Type alias for [`EitherReader`] constructors.
type EitherReaderTy<P, T> =
EitherReader<CursorTy<<P as DBProvider>::Tx, T>, <P as NodePrimitivesProvider>::Primitives>;
/// Type alias for [`EitherWriter`] constructors.
type EitherWriterTy<'a, P, T> = EitherWriter<
@@ -22,7 +35,7 @@ type EitherWriterTy<'a, P, T> = EitherWriter<
>;
/// Represents a destination for writing data, either to database or static files.
#[derive(Debug)]
#[derive(Debug, Display)]
pub enum EitherWriter<'a, CURSOR, N> {
/// Write to database table via cursor
Database(CURSOR),
@@ -41,11 +54,7 @@ impl<'a> EitherWriter<'a, (), ()> {
P::Tx: DbTxMut,
ReceiptTy<P::Primitives>: Value,
{
// Write receipts to static files only if they're explicitly enabled or we don't have
// receipts pruning
if provider.cached_storage_settings().receipts_in_static_files ||
!provider.prune_modes_ref().has_receipts_pruning()
{
if Self::receipts_destination(provider).is_static_file() {
Ok(EitherWriter::StaticFile(
provider.get_static_file_writer(block_number, StaticFileSegment::Receipts)?,
))
@@ -55,35 +64,7 @@ impl<'a> EitherWriter<'a, (), ()> {
))
}
}
}
impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N> {
/// Increment the block number.
///
/// Relevant only for [`Self::StaticFile`]. It is a no-op for [`Self::Database`].
pub fn increment_block(&mut self, expected_block_number: BlockNumber) -> ProviderResult<()> {
match self {
Self::Database(_) => Ok(()),
Self::StaticFile(writer) => writer.increment_block(expected_block_number),
}
}
}
impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N>
where
N::Receipt: Value,
CURSOR: DbCursorRW<tables::Receipts<N::Receipt>>,
{
/// Append a transaction receipt.
pub fn append_receipt(&mut self, tx_num: TxNumber, receipt: &N::Receipt) -> ProviderResult<()> {
match self {
Self::Database(cursor) => Ok(cursor.append(tx_num, receipt)?),
Self::StaticFile(writer) => writer.append_receipt(tx_num, receipt),
}
}
}
impl EitherWriter<'_, (), ()> {
/// Returns the destination for writing receipts.
///
/// The rules are as follows:
@@ -107,11 +88,264 @@ impl EitherWriter<'_, (), ()> {
EitherWriterDestination::StaticFile
}
}
/// Creates a new [`EitherWriter`] for senders based on storage settings.
pub fn new_senders<P>(
provider: &'a P,
block_number: BlockNumber,
) -> ProviderResult<EitherWriterTy<'a, P, tables::TransactionSenders>>
where
P: DBProvider + NodePrimitivesProvider + StorageSettingsCache + StaticFileProviderFactory,
P::Tx: DbTxMut,
{
if EitherWriterDestination::senders(provider).is_static_file() {
Ok(EitherWriter::StaticFile(
provider
.get_static_file_writer(block_number, StaticFileSegment::TransactionSenders)?,
))
} else {
Ok(EitherWriter::Database(
provider.tx_ref().cursor_write::<tables::TransactionSenders>()?,
))
}
}
}
impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N> {
/// Increment the block number.
///
/// Relevant only for [`Self::StaticFile`]. It is a no-op for [`Self::Database`].
pub fn increment_block(&mut self, expected_block_number: BlockNumber) -> ProviderResult<()> {
match self {
Self::Database(_) => Ok(()),
Self::StaticFile(writer) => writer.increment_block(expected_block_number),
}
}
/// Ensures that the writer is positioned at the specified block number.
///
/// If the writer is positioned at a greater block number than the specified one, the writer
/// will NOT be unwound and the error will be returned.
///
/// Relevant only for [`Self::StaticFile`]. It is a no-op for [`Self::Database`].
pub fn ensure_at_block(&mut self, block_number: BlockNumber) -> ProviderResult<()> {
match self {
Self::Database(_) => Ok(()),
Self::StaticFile(writer) => writer.ensure_at_block(block_number),
}
}
}
impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N>
where
N::Receipt: Value,
CURSOR: DbCursorRW<tables::Receipts<N::Receipt>>,
{
/// Append a transaction receipt.
pub fn append_receipt(&mut self, tx_num: TxNumber, receipt: &N::Receipt) -> ProviderResult<()> {
match self {
Self::Database(cursor) => Ok(cursor.append(tx_num, receipt)?),
Self::StaticFile(writer) => writer.append_receipt(tx_num, receipt),
}
}
}
impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N>
where
CURSOR: DbCursorRW<tables::TransactionSenders>,
{
/// Append a transaction sender to the destination
pub fn append_sender(&mut self, tx_num: TxNumber, sender: &Address) -> ProviderResult<()> {
match self {
Self::Database(cursor) => Ok(cursor.append(tx_num, sender)?),
Self::StaticFile(writer) => writer.append_transaction_sender(tx_num, sender),
}
}
/// Append transaction senders to the destination
pub fn append_senders<I>(&mut self, senders: I) -> ProviderResult<()>
where
I: Iterator<Item = (TxNumber, Address)>,
{
match self {
Self::Database(cursor) => {
for (tx_num, sender) in senders {
cursor.append(tx_num, &sender)?;
}
Ok(())
}
Self::StaticFile(writer) => writer.append_transaction_senders(senders),
}
}
/// Removes all transaction senders above the given transaction number, and stops at the given
/// block number.
pub fn prune_senders(
&mut self,
unwind_tx_from: TxNumber,
block: BlockNumber,
) -> ProviderResult<()>
where
CURSOR: DbCursorRO<tables::TransactionSenders>,
{
match self {
Self::Database(cursor) => {
let mut walker = cursor.walk_range(unwind_tx_from..)?;
while walker.next().transpose()?.is_some() {
walker.delete_current()?;
}
}
Self::StaticFile(writer) => {
let static_file_transaction_sender_num = writer
.reader()
.get_highest_static_file_tx(StaticFileSegment::TransactionSenders);
let to_delete = static_file_transaction_sender_num
.map(|static_num| (static_num + 1).saturating_sub(unwind_tx_from))
.unwrap_or_default();
writer.prune_transaction_senders(to_delete, block)?;
}
}
Ok(())
}
}
/// Represents a source for reading data, either from database or static files.
#[derive(Debug, Display)]
pub enum EitherReader<CURSOR, N> {
/// Read from database table via cursor
Database(CURSOR),
/// Read from static file
StaticFile(StaticFileProvider<N>),
}
impl EitherReader<(), ()> {
/// Creates a new [`EitherReader`] for senders based on storage settings.
pub fn new_senders<P>(
provider: &P,
) -> ProviderResult<EitherReaderTy<P, tables::TransactionSenders>>
where
P: DBProvider + NodePrimitivesProvider + StorageSettingsCache + StaticFileProviderFactory,
P::Tx: DbTx,
{
if EitherWriterDestination::senders(provider).is_static_file() {
Ok(EitherReader::StaticFile(provider.static_file_provider()))
} else {
Ok(EitherReader::Database(
provider.tx_ref().cursor_read::<tables::TransactionSenders>()?,
))
}
}
}
impl<CURSOR, N: NodePrimitives> EitherReader<CURSOR, N>
where
CURSOR: DbCursorRO<tables::TransactionSenders>,
{
/// Fetches the senders for a range of transactions.
pub fn senders_by_tx_range(
&mut self,
range: Range<TxNumber>,
) -> ProviderResult<HashMap<TxNumber, Address>> {
match self {
Self::Database(cursor) => cursor
.walk_range(range)?
.map(|result| result.map_err(ProviderError::from))
.collect::<ProviderResult<HashMap<_, _>>>(),
Self::StaticFile(provider) => range
.clone()
.zip(provider.fetch_range_iter(
StaticFileSegment::TransactionSenders,
range,
|cursor, number| cursor.get_one::<TransactionSenderMask>(number.into()),
)?)
.filter_map(|(tx_num, sender)| {
let result = sender.transpose()?;
Some(result.map(|sender| (tx_num, sender)))
})
.collect::<ProviderResult<HashMap<_, _>>>(),
}
}
}
/// Destination for writing data.
#[derive(Debug, EnumIs)]
#[allow(missing_docs)]
pub enum EitherWriterDestination {
/// Write to database table
Database,
/// Write to static file
StaticFile,
}
impl EitherWriterDestination {
/// Returns the destination for writing senders based on storage settings.
pub fn senders<P>(provider: &P) -> Self
where
P: StorageSettingsCache,
{
// Write senders to static files only if they're explicitly enabled
if provider.cached_storage_settings().transaction_senders_in_static_files {
Self::StaticFile
} else {
Self::Database
}
}
}
#[cfg(test)]
mod tests {
use crate::test_utils::create_test_provider_factory;
use super::*;
use alloy_primitives::Address;
use reth_storage_api::{DatabaseProviderFactory, StorageSettings};
#[test]
fn test_reader_senders_by_tx_range() {
let factory = create_test_provider_factory();
// Insert senders only from 1 to 4, but we will query from 0 to 5.
let senders = [
(1, Address::random()),
(2, Address::random()),
(3, Address::random()),
(4, Address::random()),
];
for transaction_senders_in_static_files in [false, true] {
factory.set_storage_settings_cache(
StorageSettings::legacy()
.with_transaction_senders_in_static_files(transaction_senders_in_static_files),
);
let provider = factory.database_provider_rw().unwrap();
let mut writer = EitherWriter::new_senders(&provider, 0).unwrap();
if transaction_senders_in_static_files {
assert!(matches!(writer, EitherWriter::StaticFile(_)));
} else {
assert!(matches!(writer, EitherWriter::Database(_)));
}
writer.increment_block(0).unwrap();
writer.append_senders(senders.iter().copied()).unwrap();
drop(writer);
provider.commit().unwrap();
let provider = factory.database_provider_ro().unwrap();
let mut reader = EitherReader::new_senders(&provider).unwrap();
if transaction_senders_in_static_files {
assert!(matches!(reader, EitherReader::StaticFile(_)));
} else {
assert!(matches!(reader, EitherReader::Database(_)));
}
assert_eq!(
reader.senders_by_tx_range(0..6).unwrap(),
senders.iter().copied().collect::<HashMap<_, _>>(),
"{reader}"
);
}
}
}

10
crates/storage/provider/src/providers/database/metrics.rs
View File

@@ -45,6 +45,8 @@ pub(crate) enum Action {
InsertBlockBodyIndices,
InsertTransactionBlocks,
GetNextTxNum,
InsertTransactionSenders,
InsertTransactionHashNumbers,
}
/// Database provider metrics
@@ -70,6 +72,10 @@ struct DatabaseProviderMetrics {
insert_tx_blocks: Histogram,
/// Duration of get next tx num
get_next_tx_num: Histogram,
/// Duration of insert transaction senders
insert_transaction_senders: Histogram,
/// Duration of insert transaction hash numbers
insert_transaction_hash_numbers: Histogram,
}
impl DatabaseProviderMetrics {
@@ -85,6 +91,10 @@ impl DatabaseProviderMetrics {
Action::InsertBlockBodyIndices => self.insert_block_body_indices.record(duration),
Action::InsertTransactionBlocks => self.insert_tx_blocks.record(duration),
Action::GetNextTxNum => self.get_next_tx_num.record(duration),
Action::InsertTransactionSenders => self.insert_transaction_senders.record(duration),
Action::InsertTransactionHashNumbers => {
self.insert_transaction_hash_numbers.record(duration)
}
}
}
}

19
crates/storage/provider/src/providers/database/mod.rs
View File

@@ -6,9 +6,10 @@ use crate::{
to_range,
traits::{BlockSource, ReceiptProvider},
BlockHashReader, BlockNumReader, BlockReader, ChainSpecProvider, DatabaseProviderFactory,
HashedPostStateProvider, HeaderProvider, HeaderSyncGapProvider, MetadataProvider,
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProviderBox,
StaticFileProviderFactory, StaticFileWriter, TransactionVariant, TransactionsProvider,
EitherWriterDestination, HashedPostStateProvider, HeaderProvider, HeaderSyncGapProvider,
MetadataProvider, ProviderError, PruneCheckpointReader, StageCheckpointReader,
StateProviderBox, StaticFileProviderFactory, StaticFileWriter, TransactionVariant,
TransactionsProvider,
};
use alloy_consensus::transaction::TransactionMeta;
use alloy_eips::BlockHashOrNumber;
@@ -467,11 +468,19 @@ impl<N: ProviderNodeTypes> TransactionsProvider for ProviderFactory<N> {
&self,
range: impl RangeBounds<TxNumber>,
) -> ProviderResult<Vec<Address>> {
self.provider()?.senders_by_tx_range(range)
if EitherWriterDestination::senders(self).is_static_file() {
self.static_file_provider.senders_by_tx_range(range)
} else {
self.provider()?.senders_by_tx_range(range)
}
}
fn transaction_sender(&self, id: TxNumber) -> ProviderResult<Option<Address>> {
self.provider()?.transaction_sender(id)
if EitherWriterDestination::senders(self).is_static_file() {
self.static_file_provider.transaction_sender(id)
} else {
self.provider()?.transaction_sender(id)
}
}
}

74
crates/storage/provider/src/providers/database/provider.rs
View File

@@ -13,12 +13,13 @@ use crate::{
},
AccountReader, BlockBodyWriter, BlockExecutionWriter, BlockHashReader, BlockNumReader,
BlockReader, BlockWriter, BundleStateInit, ChainStateBlockReader, ChainStateBlockWriter,
DBProvider, EitherWriter, HashingWriter, HeaderProvider, HeaderSyncGapProvider,
HistoricalStateProvider, HistoricalStateProviderRef, HistoryWriter, LatestStateProvider,
LatestStateProviderRef, OriginalValuesKnown, ProviderError, PruneCheckpointReader,
PruneCheckpointWriter, RevertsInit, StageCheckpointReader, StateProviderBox, StateWriter,
StaticFileProviderFactory, StatsReader, StorageReader, StorageTrieWriter, TransactionVariant,
TransactionsProvider, TransactionsProviderExt, TrieReader, TrieWriter,
DBProvider, EitherReader, EitherWriter, EitherWriterDestination, HashingWriter, HeaderProvider,
HeaderSyncGapProvider, HistoricalStateProvider, HistoricalStateProviderRef, HistoryWriter,
LatestStateProvider, LatestStateProviderRef, OriginalValuesKnown, ProviderError,
PruneCheckpointReader, PruneCheckpointWriter, RevertsInit, StageCheckpointReader,
StateProviderBox, StateWriter, StaticFileProviderFactory, StatsReader, StorageReader,
StorageTrieWriter, TransactionVariant, TransactionsProvider, TransactionsProviderExt,
TrieReader, TrieWriter,
};
use alloy_consensus::{
transaction::{SignerRecoverable, TransactionMeta, TxHashRef},
@@ -679,17 +680,12 @@ impl<TX: DbTx + 'static, N: NodeTypesForProvider> DatabaseProvider<TX, N> {
HF: Fn(RangeInclusive<BlockNumber>) -> ProviderResult<Vec<H>>,
BF: Fn(H, BodyTy<N>, Vec<Address>) -> ProviderResult<B>,
{
let mut senders_cursor = self.tx.cursor_read::<tables::TransactionSenders>()?;
self.block_range(range, headers_range, |header, body, tx_range| {
let senders = if tx_range.is_empty() {
Vec::new()
} else {
// fetch senders from the senders table
let known_senders =
senders_cursor
.walk_range(tx_range.clone())?
.collect::<Result<HashMap<_, _>, _>>()?;
let known_senders: HashMap<TxNumber, Address> =
EitherReader::new_senders(self)?.senders_by_tx_range(tx_range.clone())?;
let mut senders = Vec::with_capacity(body.transactions().len());
for (tx_num, tx) in tx_range.zip(body.transactions()) {
@@ -1342,11 +1338,19 @@ impl<TX: DbTx + 'static, N: NodeTypesForProvider> TransactionsProvider for Datab
&self,
range: impl RangeBounds<TxNumber>,
) -> ProviderResult<Vec<Address>> {
self.cursor_read_collect::<tables::TransactionSenders>(range)
if EitherWriterDestination::senders(self).is_static_file() {
self.static_file_provider.senders_by_tx_range(range)
} else {
self.cursor_read_collect::<tables::TransactionSenders>(range)
}
}
fn transaction_sender(&self, id: TxNumber) -> ProviderResult<Option<Address>> {
Ok(self.tx.get::<tables::TransactionSenders>(id)?)
if EitherWriterDestination::senders(self).is_static_file() {
self.static_file_provider.transaction_sender(id)
} else {
Ok(self.tx.get::<tables::TransactionSenders>(id)?)
}
}
}
@@ -2822,8 +2826,9 @@ impl<TX: DbTxMut + DbTx + 'static, N: NodeTypesForProvider + 'static> BlockWrite
/// If withdrawals are not empty, this will modify
/// [`BlockWithdrawals`](tables::BlockWithdrawals).
///
/// If the provider has __not__ configured full sender pruning, this will modify
/// [`TransactionSenders`](tables::TransactionSenders).
/// If the provider has __not__ configured full sender pruning, this will modify either:
/// * [`StaticFileSegment::TransactionSenders`] if senders are written to static files
/// * [`tables::TransactionSenders`] if senders are written to the database
///
/// If the provider has __not__ configured full transaction lookup pruning, this will modify
/// [`TransactionHashNumbers`](tables::TransactionHashNumbers).
@@ -2832,6 +2837,7 @@ impl<TX: DbTxMut + DbTx + 'static, N: NodeTypesForProvider + 'static> BlockWrite
block: RecoveredBlock<Self::Block>,
) -> ProviderResult<StoredBlockBodyIndices> {
let block_number = block.number();
let tx_count = block.body().transaction_count() as u64;
let mut durations_recorder = metrics::DurationsRecorder::default();
@@ -2842,29 +2848,30 @@ impl<TX: DbTxMut + DbTx + 'static, N: NodeTypesForProvider + 'static> BlockWrite
self.tx.put::<tables::HeaderNumbers>(block.hash(), block_number)?;
durations_recorder.record_relative(metrics::Action::InsertHeaderNumbers);
let mut next_tx_num = self
let first_tx_num = self
.tx
.cursor_read::<tables::TransactionBlocks>()?
.last()?
.map(|(n, _)| n + 1)
.unwrap_or_default();
durations_recorder.record_relative(metrics::Action::GetNextTxNum);
let first_tx_num = next_tx_num;
let tx_count = block.body().transaction_count() as u64;
let tx_nums_iter = std::iter::successors(Some(first_tx_num), |n| Some(n + 1));
// Ensures we have all the senders for the block's transactions.
for (transaction, sender) in block.body().transactions_iter().zip(block.senders_iter()) {
let hash = transaction.tx_hash();
if self.prune_modes.sender_recovery.as_ref().is_none_or(|m| !m.is_full()) {
let mut senders_writer = EitherWriter::new_senders(self, block.number())?;
senders_writer.increment_block(block.number())?;
senders_writer
.append_senders(tx_nums_iter.clone().zip(block.senders_iter().copied()))?;
durations_recorder.record_relative(metrics::Action::InsertTransactionSenders);
}
if self.prune_modes.sender_recovery.as_ref().is_none_or(|m| !m.is_full()) {
self.tx.put::<tables::TransactionSenders>(next_tx_num, *sender)?;
if self.prune_modes.transaction_lookup.is_none_or(|m| !m.is_full()) {
for (tx_num, transaction) in tx_nums_iter.zip(block.body().transactions_iter()) {
let hash = transaction.tx_hash();
self.tx.put::<tables::TransactionHashNumbers>(*hash, tx_num)?;
}
if self.prune_modes.transaction_lookup.is_none_or(|m| !m.is_full()) {
self.tx.put::<tables::TransactionHashNumbers>(*hash, next_tx_num)?;
}
next_tx_num += 1;
durations_recorder.record_relative(metrics::Action::InsertTransactionHashNumbers);
}
self.append_block_bodies(vec![(block_number, Some(block.into_body()))])?;
@@ -2932,8 +2939,9 @@ impl<TX: DbTxMut + DbTx + 'static, N: NodeTypesForProvider + 'static> BlockWrite
}
fn remove_blocks_above(&self, block: BlockNumber) -> ProviderResult<()> {
let last_block_number = self.last_block_number()?;
// Clean up HeaderNumbers for blocks being removed, we must clear all indexes from MDBX.
for hash in self.canonical_hashes_range(block + 1, self.last_block_number()? + 1)? {
for hash in self.canonical_hashes_range(block + 1, last_block_number + 1)? {
self.tx.delete::<tables::HeaderNumbers>(hash, None)?;
}
@@ -2975,7 +2983,7 @@ impl<TX: DbTxMut + DbTx + 'static, N: NodeTypesForProvider + 'static> BlockWrite
}
}
self.remove::<tables::TransactionSenders>(unwind_tx_from..)?;
EitherWriter::new_senders(self, last_block_number)?.prune_senders(unwind_tx_from, block)?;
self.remove_bodies_above(block)?;
@@ -4752,7 +4760,7 @@ mod tests {
// Static files mode
{
let factory = create_test_provider_factory();
let storage_settings = StorageSettings::new().with_receipts_in_static_files();
let storage_settings = StorageSettings::legacy().with_receipts_in_static_files(true);
factory.set_storage_settings_cache(storage_settings);
let factory = factory.with_prune_modes(PruneModes {
receipts: Some(PruneMode::Before(2)),

48
crates/storage/provider/src/providers/static_file/jar.rs
View File

@@ -6,16 +6,18 @@ use crate::{
to_range, BlockHashReader, BlockNumReader, HeaderProvider, ReceiptProvider,
TransactionsProvider,
};
use alloy_consensus::transaction::{SignerRecoverable, TransactionMeta};
use alloy_consensus::transaction::TransactionMeta;
use alloy_eips::{eip2718::Encodable2718, BlockHashOrNumber};
use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256};
use reth_chainspec::ChainInfo;
use reth_db::static_file::{
BlockHashMask, HeaderMask, HeaderWithHashMask, ReceiptMask, StaticFileCursor, TransactionMask,
TransactionSenderMask,
};
use reth_db_api::table::{Decompress, Value};
use reth_node_types::NodePrimitives;
use reth_primitives_traits::{SealedHeader, SignedTransaction};
use reth_storage_api::range_size_hint;
use reth_storage_errors::provider::{ProviderError, ProviderResult};
use std::{
fmt::Debug,
@@ -104,12 +106,10 @@ impl<N: NodePrimitives<BlockHeader: Value>> HeaderProvider for StaticFileJarProv
&self,
range: impl RangeBounds<BlockNumber>,
) -> ProviderResult<Vec<Self::Header>> {
let range = to_range(range);
let mut cursor = self.cursor()?;
let mut headers = Vec::with_capacity((range.end - range.start) as usize);
let mut headers = Vec::with_capacity(range_size_hint(&range).unwrap_or(1024));
for num in range {
for num in to_range(range) {
if let Some(header) = cursor.get_one::<HeaderMask<Self::Header>>(num.into())? {
headers.push(header);
}
@@ -133,12 +133,10 @@ impl<N: NodePrimitives<BlockHeader: Value>> HeaderProvider for StaticFileJarProv
range: impl RangeBounds<BlockNumber>,
mut predicate: impl FnMut(&SealedHeader<Self::Header>) -> bool,
) -> ProviderResult<Vec<SealedHeader<Self::Header>>> {
let range = to_range(range);
let mut cursor = self.cursor()?;
let mut headers = Vec::with_capacity((range.end - range.start) as usize);
let mut headers = Vec::with_capacity(range_size_hint(&range).unwrap_or(1024));
for number in range {
for number in to_range(range) {
if let Some((header, hash)) =
cursor.get_two::<HeaderWithHashMask<Self::Header>>(number.into())?
{
@@ -258,31 +256,34 @@ impl<N: NodePrimitives<SignedTx: Decompress + SignedTransaction>> TransactionsPr
&self,
range: impl RangeBounds<TxNumber>,
) -> ProviderResult<Vec<Self::Transaction>> {
let range = to_range(range);
let mut cursor = self.cursor()?;
let mut txes = Vec::with_capacity((range.end - range.start) as usize);
let mut txs = Vec::with_capacity(range_size_hint(&range).unwrap_or(1024));
for num in range {
for num in to_range(range) {
if let Some(tx) = cursor.get_one::<TransactionMask<Self::Transaction>>(num.into())? {
txes.push(tx)
txs.push(tx)
}
}
Ok(txes)
Ok(txs)
}
fn senders_by_tx_range(
&self,
range: impl RangeBounds<TxNumber>,
) -> ProviderResult<Vec<Address>> {
let txs = self.transactions_by_tx_range(range)?;
Ok(reth_primitives_traits::transaction::recover::recover_signers(&txs)?)
let mut cursor = self.cursor()?;
let mut senders = Vec::with_capacity(range_size_hint(&range).unwrap_or(1024));
for num in to_range(range) {
if let Some(tx) = cursor.get_one::<TransactionSenderMask>(num.into())? {
senders.push(tx)
}
}
Ok(senders)
}
fn transaction_sender(&self, num: TxNumber) -> ProviderResult<Option<Address>> {
Ok(self
.cursor()?
.get_one::<TransactionMask<Self::Transaction>>(num.into())?
.and_then(|tx| tx.recover_signer().ok()))
fn transaction_sender(&self, id: TxNumber) -> ProviderResult<Option<Address>> {
self.cursor()?.get_one::<TransactionSenderMask>(id.into())
}
}
@@ -317,11 +318,10 @@ impl<N: NodePrimitives<SignedTx: Decompress + SignedTransaction, Receipt: Decomp
&self,
range: impl RangeBounds<TxNumber>,
) -> ProviderResult<Vec<Self::Receipt>> {
let range = to_range(range);
let mut cursor = self.cursor()?;
let mut receipts = Vec::with_capacity((range.end - range.start) as usize);
let mut receipts = Vec::with_capacity(range_size_hint(&range).unwrap_or(1024));
for num in range {
for num in to_range(range) {
if let Some(tx) = cursor.get_one::<ReceiptMask<Self::Receipt>>(num.into())? {
receipts.push(tx)
}

119
crates/storage/provider/src/providers/static_file/manager.rs
View File

@@ -4,13 +4,10 @@ use super::{
};
use crate::{
to_range, BlockHashReader, BlockNumReader, BlockReader, BlockSource, EitherWriter,
HeaderProvider, ReceiptProvider, StageCheckpointReader, StatsReader, TransactionVariant,
TransactionsProvider, TransactionsProviderExt,
};
use alloy_consensus::{
transaction::{SignerRecoverable, TransactionMeta},
Header,
EitherWriterDestination, HeaderProvider, ReceiptProvider, StageCheckpointReader, StatsReader,
TransactionVariant, TransactionsProvider, TransactionsProviderExt,
};
use alloy_consensus::{transaction::TransactionMeta, Header};
use alloy_eips::{eip2718::Encodable2718, BlockHashOrNumber};
use alloy_primitives::{b256, keccak256, Address, BlockHash, BlockNumber, TxHash, TxNumber, B256};
use dashmap::DashMap;
@@ -21,7 +18,7 @@ use reth_db::{
lockfile::StorageLock,
static_file::{
iter_static_files, BlockHashMask, HeaderMask, HeaderWithHashMask, ReceiptMask,
StaticFileCursor, TransactionMask,
StaticFileCursor, TransactionMask, TransactionSenderMask,
},
};
use reth_db_api::{
@@ -461,12 +458,37 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
pub fn get_segment_provider(
&self,
segment: StaticFileSegment,
start: u64,
number: u64,
) -> ProviderResult<StaticFileJarProvider<'_, N>> {
if segment.is_block_based() {
self.get_segment_provider_for_block(segment, start, None)
self.get_segment_provider_for_block(segment, number, None)
} else {
self.get_segment_provider_for_transaction(segment, start, None)
self.get_segment_provider_for_transaction(segment, number, None)
}
}
/// Gets the [`StaticFileJarProvider`] of the requested segment and start index that can be
/// either block or transaction.
///
/// If the segment is not found, returns [`None`].
pub fn get_maybe_segment_provider(
&self,
segment: StaticFileSegment,
number: u64,
) -> ProviderResult<Option<StaticFileJarProvider<'_, N>>> {
let provider = if segment.is_block_based() {
self.get_segment_provider_for_block(segment, number, None)
} else {
self.get_segment_provider_for_transaction(segment, number, None)
};
match provider {
Ok(provider) => Ok(Some(provider)),
Err(
ProviderError::MissingStaticFileBlock(_, _) |
ProviderError::MissingStaticFileTx(_, _),
) => Ok(None),
Err(err) => Err(err),
}
}
@@ -1022,6 +1044,11 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
continue;
}
}
StaticFileSegment::TransactionSenders => {
if EitherWriterDestination::senders(provider).is_database() {
continue
}
}
}
let initial_highest_block = self.get_highest_static_file_block(segment);
@@ -1124,6 +1151,13 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
highest_tx,
highest_block,
)?,
StaticFileSegment::TransactionSenders => self
.ensure_invariants::<_, tables::TransactionSenders>(
provider,
segment,
highest_tx,
highest_block,
)?,
} {
debug!(target: "reth::providers::static_file", ?segment, unwind_target=unwind, "Invariants check returned unwind target");
update_unwind_target(unwind);
@@ -1224,6 +1258,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
StaticFileSegment::Headers => StageId::Headers,
StaticFileSegment::Transactions => StageId::Bodies,
StaticFileSegment::Receipts => StageId::Execution,
StaticFileSegment::TransactionSenders => StageId::SenderRecovery,
};
let checkpoint_block_number =
provider.get_stage_checkpoint(stage_id)?.unwrap_or_default().block_number;
@@ -1260,7 +1295,9 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
// TODO(joshie): is_block_meta
writer.prune_headers(prune_count)?;
}
StaticFileSegment::Transactions | StaticFileSegment::Receipts => {
StaticFileSegment::Transactions |
StaticFileSegment::Receipts |
StaticFileSegment::TransactionSenders => {
if let Some(block) = provider.block_body_indices(checkpoint_block_number)? {
let number = highest_static_file_entry - block.last_tx_num();
debug!(target: "reth::providers::static_file", ?segment, prune_count = number, checkpoint_block_number, "Pruning transaction based segment");
@@ -1272,6 +1309,9 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
StaticFileSegment::Receipts => {
writer.prune_receipts(number, checkpoint_block_number)?
}
StaticFileSegment::TransactionSenders => {
writer.prune_transaction_senders(number, checkpoint_block_number)?
}
StaticFileSegment::Headers => unreachable!(),
}
} else {
@@ -1447,29 +1487,37 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
/// Fetches data within a specified range across multiple static files.
///
/// Returns an iterator over the data
/// Returns an iterator over the data. Yields [`None`] if the data for the specified number is
/// not found.
pub fn fetch_range_iter<'a, T, F>(
&'a self,
segment: StaticFileSegment,
range: Range<u64>,
get_fn: F,
) -> ProviderResult<impl Iterator<Item = ProviderResult<T>> + 'a>
) -> ProviderResult<impl Iterator<Item = ProviderResult<Option<T>>> + 'a>
where
F: Fn(&mut StaticFileCursor<'_>, u64) -> ProviderResult<Option<T>> + 'a,
T: std::fmt::Debug,
{
let mut provider = Some(self.get_segment_provider(segment, range.start)?);
Ok(range.filter_map(move |number| {
match get_fn(&mut provider.as_ref().expect("qed").cursor().ok()?, number).transpose() {
Some(result) => Some(result),
let mut provider = self.get_maybe_segment_provider(segment, range.start)?;
Ok(range.map(move |number| {
match provider
.as_ref()
.map(|provider| get_fn(&mut provider.cursor()?, number))
.and_then(|result| result.transpose())
{
Some(result) => result.map(Some),
None => {
// There is a very small chance of hitting a deadlock if two consecutive static
// files share the same bucket in the internal dashmap and
// we don't drop the current provider before requesting the
// next one.
// There is a very small chance of hitting a deadlock if two consecutive
// static files share the same bucket in the internal dashmap and we don't drop
// the current provider before requesting the next one.
provider.take();
provider = Some(self.get_segment_provider(segment, number).ok()?);
get_fn(&mut provider.as_ref().expect("qed").cursor().ok()?, number).transpose()
provider = self.get_maybe_segment_provider(segment, number)?;
provider
.as_ref()
.map(|provider| get_fn(&mut provider.cursor()?, number))
.and_then(|result| result.transpose())
.transpose()
}
}
}))
@@ -1993,15 +2041,24 @@ impl<N: NodePrimitives<SignedTx: Decompress + SignedTransaction>> TransactionsPr
&self,
range: impl RangeBounds<TxNumber>,
) -> ProviderResult<Vec<Address>> {
let txes = self.transactions_by_tx_range(range)?;
Ok(reth_primitives_traits::transaction::recover::recover_signers(&txes)?)
self.fetch_range_with_predicate(
StaticFileSegment::TransactionSenders,
to_range(range),
|cursor, number| cursor.get_one::<TransactionSenderMask>(number.into()),
|_| true,
)
}
fn transaction_sender(&self, id: TxNumber) -> ProviderResult<Option<Address>> {
match self.transaction_by_id_unhashed(id)? {
Some(tx) => Ok(tx.recover_signer().ok()),
None => Ok(None),
}
self.get_segment_provider_for_transaction(StaticFileSegment::TransactionSenders, id, None)
.and_then(|provider| provider.transaction_sender(id))
.or_else(|err| {
if let ProviderError::MissingStaticFileTx(_, _) = err {
Ok(None)
} else {
Err(err)
}
})
}
}
@@ -2133,6 +2190,10 @@ impl<N: NodePrimitives> StatsReader for StaticFileProvider<N> {
.map(|txs| txs + 1)
.unwrap_or_default()
as usize),
tables::TransactionSenders::NAME => Ok(self
.get_highest_static_file_tx(StaticFileSegment::TransactionSenders)
.map(|txs| txs + 1)
.unwrap_or_default() as usize),
_ => Err(ProviderError::UnsupportedProvider),
}
}

17
crates/storage/provider/src/providers/static_file/mod.rs
View File

@@ -61,7 +61,7 @@ mod tests {
test_utils::create_test_provider_factory, HeaderProvider, StaticFileProviderFactory,
};
use alloy_consensus::{Header, SignableTransaction, Transaction, TxLegacy};
use alloy_primitives::{BlockHash, Signature, TxNumber, B256};
use alloy_primitives::{Address, BlockHash, Signature, TxNumber, B256, U160};
use rand::seq::SliceRandom;
use reth_db::test_utils::create_test_static_files_dir;
use reth_db_api::{transaction::DbTxMut, CanonicalHeaders, HeaderNumbers, Headers};
@@ -328,6 +328,11 @@ mod tests {
receipt.cumulative_gas_used = *next_tx_num;
writer.append_receipt(*next_tx_num, &receipt).unwrap();
}
StaticFileSegment::TransactionSenders => {
// Used as ID for validation
let sender = Address::from(U160::from(*next_tx_num));
writer.append_transaction_sender(*next_tx_num, &sender).unwrap();
}
}
*next_tx_num += 1;
tx_count -= 1;
@@ -430,6 +435,9 @@ mod tests {
writer.prune_transactions(prune_count, last_block)?
}
StaticFileSegment::Receipts => writer.prune_receipts(prune_count, last_block)?,
StaticFileSegment::TransactionSenders => {
writer.prune_transaction_senders(prune_count, last_block)?
}
}
writer.commit()?;
@@ -456,6 +464,13 @@ mod tests {
sf_rw.receipt(id)?.map(|r| r.cumulative_gas_used),
"receipt mismatch",
)?,
StaticFileSegment::TransactionSenders => assert_eyre(
expected_tx_tip,
sf_rw
.transaction_sender(id)?
.map(|s| u64::try_from(U160::from_be_bytes(s.0.into())).unwrap()),
"sender mismatch",
)?,
}
}

176
crates/storage/provider/src/providers/static_file/writer.rs
View File

@@ -13,6 +13,7 @@ use reth_static_file_types::{SegmentHeader, SegmentRangeInclusive, StaticFileSeg
use reth_storage_errors::provider::{ProviderError, ProviderResult, StaticFileWriterError};
use std::{
borrow::Borrow,
cmp::Ordering,
fmt::Debug,
path::{Path, PathBuf},
sync::{Arc, Weak},
@@ -29,6 +30,7 @@ pub(crate) struct StaticFileWriters<N> {
headers: RwLock<Option<StaticFileProviderRW<N>>>,
transactions: RwLock<Option<StaticFileProviderRW<N>>>,
receipts: RwLock<Option<StaticFileProviderRW<N>>>,
transaction_senders: RwLock<Option<StaticFileProviderRW<N>>>,
}
impl<N> Default for StaticFileWriters<N> {
@@ -37,6 +39,7 @@ impl<N> Default for StaticFileWriters<N> {
headers: Default::default(),
transactions: Default::default(),
receipts: Default::default(),
transaction_senders: Default::default(),
}
}
}
@@ -51,6 +54,7 @@ impl<N: NodePrimitives> StaticFileWriters<N> {
StaticFileSegment::Headers => self.headers.write(),
StaticFileSegment::Transactions => self.transactions.write(),
StaticFileSegment::Receipts => self.receipts.write(),
StaticFileSegment::TransactionSenders => self.transaction_senders.write(),
};
if write_guard.is_none() {
@@ -63,7 +67,9 @@ impl<N: NodePrimitives> StaticFileWriters<N> {
pub(crate) fn commit(&self) -> ProviderResult<()> {
debug!(target: "provider::static_file", "Committing all static file segments");
for writer_lock in [&self.headers, &self.transactions, &self.receipts] {
for writer_lock in
[&self.headers, &self.transactions, &self.receipts, &self.transaction_senders]
{
let mut writer = writer_lock.write();
if let Some(writer) = writer.as_mut() {
writer.commit()?;
@@ -75,7 +81,9 @@ impl<N: NodePrimitives> StaticFileWriters<N> {
}
pub(crate) fn has_unwind_queued(&self) -> bool {
for writer_lock in [&self.headers, &self.transactions, &self.receipts] {
for writer_lock in
[&self.headers, &self.transactions, &self.receipts, &self.transaction_senders]
{
let writer = writer_lock.read();
if let Some(writer) = writer.as_ref() &&
writer.will_prune_on_commit()
@@ -262,6 +270,10 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
StaticFileSegment::Receipts => {
self.prune_receipt_data(to_delete, last_block_number.expect("should exist"))?
}
StaticFileSegment::TransactionSenders => self.prune_transaction_sender_data(
to_delete,
last_block_number.expect("should exist"),
)?,
}
}
@@ -358,10 +370,39 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
self.reader().update_index(self.writer.user_header().segment(), segment_max_block)
}
/// Ensures that the writer is positioned at the specified block number.
///
/// If the writer is positioned at a greater block number than the specified one, the writer
/// will NOT be unwound and the error will be returned.
pub fn ensure_at_block(&mut self, advance_to: BlockNumber) -> ProviderResult<()> {
let current_block = if let Some(current_block_number) = self.current_block_number() {
current_block_number
} else {
self.increment_block(0)?;
0
};
match current_block.cmp(&advance_to) {
Ordering::Less => {
for block in current_block + 1..=advance_to {
self.increment_block(block)?;
}
}
Ordering::Equal => {}
Ordering::Greater => {
return Err(ProviderError::UnexpectedStaticFileBlockNumber(
self.writer.user_header().segment(),
current_block,
advance_to,
));
}
}
Ok(())
}
/// Allows to increment the [`SegmentHeader`] end block. It will commit the current static file,
/// and create the next one if we are past the end range.
///
/// Returns the current [`BlockNumber`] as seen in the static file.
pub fn increment_block(&mut self, expected_block_number: BlockNumber) -> ProviderResult<()> {
let segment = self.writer.user_header().segment();
@@ -401,6 +442,11 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
Ok(())
}
/// Returns the current block number of the static file writer.
pub fn current_block_number(&self) -> Option<u64> {
self.writer.user_header().block_end()
}
/// Returns a block number that is one next to the current tip of static files.
pub fn next_block_number(&self) -> u64 {
// The next static file block number can be found by checking the one after block_end.
@@ -529,8 +575,6 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
}
/// Appends to tx number-based static file.
///
/// Returns the current [`TxNumber`] as seen in the static file.
fn append_with_tx_number<V: Compact>(
&mut self,
tx_num: TxNumber,
@@ -559,8 +603,6 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
///
/// It **CALLS** `increment_block()` since the number of headers is equal to the number of
/// blocks.
///
/// Returns the current [`BlockNumber`] as seen in the static file.
pub fn append_header(&mut self, header: &N::BlockHeader, hash: &BlockHash) -> ProviderResult<()>
where
N::BlockHeader: Compact,
@@ -572,8 +614,6 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
///
/// It **CALLS** `increment_block()` since the number of headers is equal to the number of
/// blocks.
///
/// Returns the current [`BlockNumber`] as seen in the static file.
pub fn append_header_with_td(
&mut self,
header: &N::BlockHeader,
@@ -609,8 +649,6 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
///
/// It **DOES NOT CALL** `increment_block()`, it should be handled elsewhere. There might be
/// empty blocks and this function wouldn't be called.
///
/// Returns the current [`TxNumber`] as seen in the static file.
pub fn append_transaction(&mut self, tx_num: TxNumber, tx: &N::SignedTx) -> ProviderResult<()>
where
N::SignedTx: Compact,
@@ -636,8 +674,6 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
///
/// It **DOES NOT** call `increment_block()`, it should be handled elsewhere. There might be
/// empty blocks and this function wouldn't be called.
///
/// Returns the current [`TxNumber`] as seen in the static file.
pub fn append_receipt(&mut self, tx_num: TxNumber, receipt: &N::Receipt) -> ProviderResult<()>
where
N::Receipt: Compact,
@@ -660,9 +696,7 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
}
/// Appends multiple receipts to the static file.
///
/// Returns the current [`TxNumber`] as seen in the static file, if any.
pub fn append_receipts<I, R>(&mut self, receipts: I) -> ProviderResult<Option<TxNumber>>
pub fn append_receipts<I, R>(&mut self, receipts: I) -> ProviderResult<()>
where
I: Iterator<Item = Result<(TxNumber, R), ProviderError>>,
R: Borrow<N::Receipt>,
@@ -673,20 +707,18 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
let mut receipts_iter = receipts.into_iter().peekable();
// If receipts are empty, we can simply return None
if receipts_iter.peek().is_none() {
return Ok(None);
return Ok(());
}
let start = Instant::now();
self.ensure_no_queued_prune()?;
// At this point receipts contains at least one receipt, so this would be overwritten.
let mut tx_number = 0;
let mut count: u64 = 0;
for receipt_result in receipts_iter {
let (tx_num, receipt) = receipt_result?;
self.append_with_tx_number(tx_num, receipt.borrow())?;
tx_number = tx_num;
count += 1;
}
@@ -699,7 +731,68 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
);
}
Ok(Some(tx_number))
Ok(())
}
/// Appends transaction sender to static file.
///
/// It **DOES NOT** call `increment_block()`, it should be handled elsewhere. There might be
/// empty blocks and this function wouldn't be called.
pub fn append_transaction_sender(
&mut self,
tx_num: TxNumber,
sender: &alloy_primitives::Address,
) -> ProviderResult<()> {
let start = Instant::now();
self.ensure_no_queued_prune()?;
debug_assert!(self.writer.user_header().segment() == StaticFileSegment::TransactionSenders);
self.append_with_tx_number(tx_num, sender)?;
if let Some(metrics) = &self.metrics {
metrics.record_segment_operation(
StaticFileSegment::TransactionSenders,
StaticFileProviderOperation::Append,
Some(start.elapsed()),
);
}
Ok(())
}
/// Appends multiple transaction senders to the static file.
pub fn append_transaction_senders<I>(&mut self, senders: I) -> ProviderResult<()>
where
I: Iterator<Item = (TxNumber, alloy_primitives::Address)>,
{
debug_assert!(self.writer.user_header().segment() == StaticFileSegment::TransactionSenders);
let mut senders_iter = senders.into_iter().peekable();
// If senders are empty, we can simply return
if senders_iter.peek().is_none() {
return Ok(());
}
let start = Instant::now();
self.ensure_no_queued_prune()?;
// At this point senders contains at least one sender, so this would be overwritten.
let mut count: u64 = 0;
for (tx_num, sender) in senders_iter {
self.append_with_tx_number(tx_num, sender)?;
count += 1;
}
if let Some(metrics) = &self.metrics {
metrics.record_segment_operations(
StaticFileSegment::TransactionSenders,
StaticFileProviderOperation::Append,
count,
Some(start.elapsed()),
);
}
Ok(())
}
/// Adds an instruction to prune `to_delete` transactions during commit.
@@ -726,6 +819,21 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
self.queue_prune(to_delete, Some(last_block))
}
/// Adds an instruction to prune `to_delete` transaction senders during commit.
///
/// Note: `last_block` refers to the block the unwinds ends at.
pub fn prune_transaction_senders(
&mut self,
to_delete: u64,
last_block: BlockNumber,
) -> ProviderResult<()> {
debug_assert_eq!(
self.writer.user_header().segment(),
StaticFileSegment::TransactionSenders
);
self.queue_prune(to_delete, Some(last_block))
}
/// Adds an instruction to prune `to_delete` headers during commit.
pub fn prune_headers(&mut self, to_delete: u64) -> ProviderResult<()> {
debug_assert_eq!(self.writer.user_header().segment(), StaticFileSegment::Headers);
@@ -802,6 +910,29 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
Ok(())
}
/// Prunes the last `to_delete` transaction senders from the data file.
fn prune_transaction_sender_data(
&mut self,
to_delete: u64,
last_block: BlockNumber,
) -> ProviderResult<()> {
let start = Instant::now();
debug_assert!(self.writer.user_header().segment() == StaticFileSegment::TransactionSenders);
self.truncate(to_delete, Some(last_block))?;
if let Some(metrics) = &self.metrics {
metrics.record_segment_operation(
StaticFileSegment::TransactionSenders,
StaticFileProviderOperation::Prune,
Some(start.elapsed()),
);
}
Ok(())
}
/// Prunes the last `to_delete` headers from the data file.
fn prune_header_data(&mut self, to_delete: u64) -> ProviderResult<()> {
let start = Instant::now();
@@ -821,7 +952,8 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
Ok(())
}
fn reader(&self) -> StaticFileProvider<N> {
/// Returns a [`StaticFileProvider`] associated with this writer.
pub fn reader(&self) -> StaticFileProvider<N> {
Self::upgrade_provider_to_strong_reference(&self.reader)
}

3
crates/storage/storage-api/src/database_provider.rs
View File

@@ -183,7 +183,8 @@ where
}
}
fn range_size_hint(range: &impl RangeBounds<u64>) -> Option<usize> {
/// Returns the length of the range if the range has a bounded end.
pub fn range_size_hint(range: &impl RangeBounds<u64>) -> Option<usize> {
let start = match range.start_bound().cloned() {
Bound::Included(start) => start,
Bound::Excluded(start) => start.checked_add(1)?,

90
crates/tracing-otlp/src/lib.rs
View File

@@ -12,7 +12,7 @@ use opentelemetry::{global, trace::TracerProvider, KeyValue, Value};
use opentelemetry_otlp::{SpanExporter, WithExportConfig};
use opentelemetry_sdk::{
propagation::TraceContextPropagator,
trace::{SdkTracer, SdkTracerProvider},
trace::{Sampler, SdkTracer, SdkTracerProvider},
Resource,
};
use opentelemetry_semantic_conventions::{attribute::SERVICE_VERSION, SCHEMA_URL};
@@ -29,36 +29,92 @@ const HTTP_TRACE_ENDPOINT: &str = "/v1/traces";
///
/// This layer can be added to a [`tracing_subscriber::Registry`] to enable `OpenTelemetry` tracing
/// with OTLP export to an url.
pub fn span_layer<S>(
service_name: impl Into<Value>,
endpoint: &Url,
protocol: OtlpProtocol,
) -> eyre::Result<OpenTelemetryLayer<S, SdkTracer>>
pub fn span_layer<S>(otlp_config: OtlpConfig) -> eyre::Result<OpenTelemetryLayer<S, SdkTracer>>
where
for<'span> S: Subscriber + LookupSpan<'span>,
{
global::set_text_map_propagator(TraceContextPropagator::new());
let resource = build_resource(service_name);
let resource = build_resource(otlp_config.service_name.clone());
let span_builder = SpanExporter::builder();
let span_exporter = match protocol {
OtlpProtocol::Http => span_builder.with_http().with_endpoint(endpoint.as_str()).build()?,
OtlpProtocol::Grpc => span_builder.with_tonic().with_endpoint(endpoint.as_str()).build()?,
let span_exporter = match otlp_config.protocol {
OtlpProtocol::Http => {
span_builder.with_http().with_endpoint(otlp_config.endpoint.as_str()).build()?
}
OtlpProtocol::Grpc => {
span_builder.with_tonic().with_endpoint(otlp_config.endpoint.as_str()).build()?
}
};
let sampler = build_sampler(otlp_config.sample_ratio)?;
let tracer_provider = SdkTracerProvider::builder()
.with_resource(resource)
.with_sampler(sampler)
.with_batch_exporter(span_exporter)
.build();
global::set_tracer_provider(tracer_provider.clone());
let tracer = tracer_provider.tracer("reth");
let tracer = tracer_provider.tracer(otlp_config.service_name);
Ok(tracing_opentelemetry::layer().with_tracer(tracer))
}
/// Configuration for OTLP trace export.
#[derive(Debug, Clone)]
pub struct OtlpConfig {
/// Service name for trace identification
service_name: String,
/// Otlp endpoint URL
endpoint: Url,
/// Transport protocol, HTTP or gRPC
protocol: OtlpProtocol,
/// Optional sampling ratio, from 0.0 to 1.0
sample_ratio: Option<f64>,
}
impl OtlpConfig {
/// Creates a new OTLP configuration.
pub fn new(
service_name: impl Into<String>,
endpoint: Url,
protocol: OtlpProtocol,
sample_ratio: Option<f64>,
) -> eyre::Result<Self> {
if let Some(ratio) = sample_ratio {
ensure!(
(0.0..=1.0).contains(&ratio),
"Sample ratio must be between 0.0 and 1.0, got: {}",
ratio
);
}
Ok(Self { service_name: service_name.into(), endpoint, protocol, sample_ratio })
}
/// Returns the service name.
pub fn service_name(&self) -> &str {
&self.service_name
}
/// Returns the OTLP endpoint URL.
pub const fn endpoint(&self) -> &Url {
&self.endpoint
}
/// Returns the transport protocol.
pub const fn protocol(&self) -> OtlpProtocol {
self.protocol
}
/// Returns the sampling ratio.
pub const fn sample_ratio(&self) -> Option<f64> {
self.sample_ratio
}
}
// Builds OTLP resource with service information.
fn build_resource(service_name: impl Into<Value>) -> Resource {
Resource::builder()
@@ -67,6 +123,18 @@ fn build_resource(service_name: impl Into<Value>) -> Resource {
.build()
}
/// Builds the appropriate sampler based on the sample ratio.
fn build_sampler(sample_ratio: Option<f64>) -> eyre::Result<Sampler> {
match sample_ratio {
// Default behavior: sample all traces
None | Some(1.0) => Ok(Sampler::ParentBased(Box::new(Sampler::AlwaysOn))),
// Don't sample anything
Some(0.0) => Ok(Sampler::ParentBased(Box::new(Sampler::AlwaysOff))),
// Sample based on trace ID ratio
Some(ratio) => Ok(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(ratio)))),
}
}
/// OTLP transport protocol type
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum OtlpProtocol {

3
crates/tracing/Cargo.toml
View File

@@ -26,8 +26,7 @@ tracing-logfmt.workspace = true
clap = { workspace = true, features = ["derive"] }
eyre.workspace = true
rolling-file.workspace = true
url = { workspace = true, optional = true }
[features]
default = ["otlp"]
otlp = ["reth-tracing-otlp", "dep:url"]
otlp = ["reth-tracing-otlp"]

13
crates/tracing/src/layers.rs
View File

@@ -1,4 +1,6 @@
use crate::formatter::LogFormat;
#[cfg(feature = "otlp")]
use reth_tracing_otlp::{span_layer, OtlpConfig};
use rolling_file::{RollingConditionBasic, RollingFileAppender};
use std::{
fmt,
@@ -6,11 +8,6 @@ use std::{
};
use tracing_appender::non_blocking::WorkerGuard;
use tracing_subscriber::{filter::Directive, EnvFilter, Layer, Registry};
#[cfg(feature = "otlp")]
use {
reth_tracing_otlp::{span_layer, OtlpProtocol},
url::Url,
};
/// A worker guard returned by the file layer.
///
@@ -137,14 +134,12 @@ impl Layers {
#[cfg(feature = "otlp")]
pub fn with_span_layer(
&mut self,
service_name: String,
endpoint_exporter: Url,
otlp_config: OtlpConfig,
filter: EnvFilter,
otlp_protocol: OtlpProtocol,
) -> eyre::Result<()> {
// Create the span provider
let span_layer = span_layer(service_name, &endpoint_exporter, otlp_protocol)
let span_layer = span_layer(otlp_config)
.map_err(|e| eyre::eyre!("Failed to build OTLP span exporter {}", e))?
.with_filter(filter);

29
crates/transaction-pool/src/lib.rs
View File

@@ -384,23 +384,6 @@ where
self.pool.validator().validate_transactions_with_origin(origin, transactions).await
}
/// Validates all transactions with their individual origins.
///
/// This returns the validated transactions in the same order as input.
async fn validate_all_with_origins(
&self,
transactions: Vec<(TransactionOrigin, V::Transaction)>,
) -> Vec<(TransactionOrigin, TransactionValidationOutcome<V::Transaction>)> {
if transactions.len() == 1 {
let (origin, tx) = transactions.into_iter().next().unwrap();
let res = self.pool.validator().validate_transaction(origin, tx).await;
return vec![(origin, res)]
}
let origins: Vec<_> = transactions.iter().map(|(origin, _)| *origin).collect();
let tx_outcomes = self.pool.validator().validate_transactions(transactions).await;
origins.into_iter().zip(tx_outcomes).collect()
}
/// Number of transactions in the entire pool
pub fn len(&self) -> usize {
self.pool.len()
@@ -516,18 +499,6 @@ where
self.pool.add_transactions(origin, validated.into_iter())
}
async fn add_transactions_with_origins(
&self,
transactions: Vec<(TransactionOrigin, Self::Transaction)>,
) -> Vec<PoolResult<AddedTransactionOutcome>> {
if transactions.is_empty() {
return Vec::new()
}
let validated = self.validate_all_with_origins(transactions).await;
self.pool.add_transactions_with_origins(validated)
}
fn transaction_event_listener(&self, tx_hash: TxHash) -> Option<TransactionEvents> {
self.pool.add_transaction_event_listener(tx_hash)
}

13
crates/transaction-pool/src/noop.rs
View File

@@ -98,19 +98,6 @@ impl<T: EthPoolTransaction> TransactionPool for NoopTransactionPool<T> {
.collect()
}
async fn add_transactions_with_origins(
&self,
transactions: Vec<(TransactionOrigin, Self::Transaction)>,
) -> Vec<PoolResult<AddedTransactionOutcome>> {
transactions
.into_iter()
.map(|(_, transaction)| {
let hash = *transaction.hash();
Err(PoolError::other(hash, Box::new(NoopInsertError::new(transaction))))
})
.collect()
}
fn transaction_event_listener(&self, _tx_hash: TxHash) -> Option<TransactionEvents> {
None
}

54
crates/transaction-pool/src/pool/mod.rs
View File

@@ -571,24 +571,22 @@ where
Ok(listener)
}
/// Adds all transactions in the iterator to the pool, each with its individual origin,
/// returning a list of results.
/// Adds all transactions in the iterator to the pool, returning a list of results.
///
/// Note: A large batch may lock the pool for a long time that blocks important operations
/// like updating the pool on canonical state changes. The caller should consider having
/// a max batch size to balance transaction insertions with other updates.
pub fn add_transactions_with_origins(
pub fn add_transactions(
&self,
transactions: impl IntoIterator<
Item = (TransactionOrigin, TransactionValidationOutcome<T::Transaction>),
>,
origin: TransactionOrigin,
transactions: impl IntoIterator<Item = TransactionValidationOutcome<T::Transaction>>,
) -> Vec<PoolResult<AddedTransactionOutcome>> {
// Process all transactions in one write lock, maintaining individual origins
let (mut added, discarded) = {
let mut pool = self.pool.write();
let added = transactions
.into_iter()
.map(|(origin, tx)| self.add_transaction(&mut pool, origin, tx))
.map(|tx| self.add_transaction(&mut pool, origin, tx))
.collect::<Vec<_>>();
// Enforce the pool size limits if at least one transaction was added successfully
@@ -618,26 +616,20 @@ where
*res = Err(PoolError::new(*hash, PoolErrorKind::DiscardedOnInsert))
}
}
}
};
added
}
/// Adds all transactions in the iterator to the pool, returning a list of results.
///
/// Note: A large batch may lock the pool for a long time that blocks important operations
/// like updating the pool on canonical state changes. The caller should consider having
/// a max batch size to balance transaction insertions with other updates.
pub fn add_transactions(
&self,
origin: TransactionOrigin,
transactions: impl IntoIterator<Item = TransactionValidationOutcome<T::Transaction>>,
) -> Vec<PoolResult<AddedTransactionOutcome>> {
self.add_transactions_with_origins(transactions.into_iter().map(|tx| (origin, tx)))
}
/// Notify all listeners about a new pending transaction.
fn on_new_pending_transaction(&self, pending: &AddedPendingTransaction<T::Transaction>) {
///
/// See also [`Self::add_pending_listener`]
///
/// CAUTION: This function is only intended to be used manually in order to use this type's
/// pending transaction receivers when manually implementing the
/// [`TransactionPool`](crate::TransactionPool) trait for a custom pool implementation
/// [`TransactionPool::pending_transactions_listener_for`](crate::TransactionPool).
pub fn on_new_pending_transaction(&self, pending: &AddedPendingTransaction<T::Transaction>) {
let propagate_allowed = pending.is_propagate_allowed();
let mut transaction_listeners = self.pending_transaction_listener.lock();
@@ -654,7 +646,14 @@ where
}
/// Notify all listeners about a newly inserted pending transaction.
fn on_new_transaction(&self, event: NewTransactionEvent<T::Transaction>) {
///
/// See also [`Self::add_new_transaction_listener`]
///
/// CAUTION: This function is only intended to be used manually in order to use this type's
/// transaction receivers when manually implementing the
/// [`TransactionPool`](crate::TransactionPool) trait for a custom pool implementation
/// [`TransactionPool::new_transactions_listener_for`](crate::TransactionPool).
pub fn on_new_transaction(&self, event: NewTransactionEvent<T::Transaction>) {
let mut transaction_listeners = self.transaction_listener.lock();
transaction_listeners.retain_mut(|listener| {
if listener.kind.is_propagate_only() && !event.transaction.propagate {
@@ -728,7 +727,14 @@ where
}
/// Fire events for the newly added transaction if there are any.
fn notify_event_listeners(&self, tx: &AddedTransaction<T::Transaction>) {
///
/// See also [`Self::add_transaction_event_listener`].
///
/// CAUTION: This function is only intended to be used manually in order to use this type's
/// [`TransactionEvents`] receivers when manually implementing the
/// [`TransactionPool`](crate::TransactionPool) trait for a custom pool implementation
/// [`TransactionPool::transaction_event_listener`](crate::TransactionPool).
pub fn notify_event_listeners(&self, tx: &AddedTransaction<T::Transaction>) {
let mut listener = self.event_listener.write();
if listener.is_empty() {
// nothing to notify

10
crates/transaction-pool/src/traits.rs
View File

@@ -177,16 +177,6 @@ pub trait TransactionPool: Clone + Debug + Send + Sync {
transactions: Vec<Self::Transaction>,
) -> impl Future<Output = Vec<PoolResult<AddedTransactionOutcome>>> + Send;
/// Adds multiple _unvalidated_ transactions with individual origins.
///
/// Each transaction can have its own [`TransactionOrigin`].
///
/// Consumer: RPC
fn add_transactions_with_origins(
&self,
transactions: Vec<(TransactionOrigin, Self::Transaction)>,
) -> impl Future<Output = Vec<PoolResult<AddedTransactionOutcome>>> + Send;
/// Submit a consensus transaction directly to the pool
fn add_consensus_transaction(
&self,

10
crates/trie/parallel/src/proof_task.rs
View File

@@ -1365,11 +1365,6 @@ where
match ctx.missed_leaves_storage_roots.entry(hashed_address) {
dashmap::Entry::Occupied(occ) => *occ.get(),
dashmap::Entry::Vacant(vac) => {
let _guard = debug_span!(
target: "trie::proof_task",
"Waiting on missed leaf storage proof computation",
?hashed_address,
);
let root =
StorageProof::new_hashed(provider, provider, hashed_address)
.with_prefix_set_mut(Default::default())
@@ -1413,11 +1408,6 @@ where
// Consume remaining storage proof receivers for accounts not encountered during trie walk.
for (hashed_address, receiver) in storage_proof_receivers {
let _guard = debug_span!(
target: "trie::proof_task",
"Blocking on final storage proof",
?hashed_address,
);
if let Ok(proof_msg) = receiver.recv() {
// Extract storage proof from the result
if let Ok(ProofResult::StorageProof { proof, .. }) = proof_msg.result {

1
docs/vocs/docs/pages/cli/SUMMARY.mdx
View File

@@ -28,6 +28,7 @@
- [`reth db settings get`](/cli/reth/db/settings/get)
- [`reth db settings set`](/cli/reth/db/settings/set)
- [`reth db settings set receipts_in_static_files`](/cli/reth/db/settings/set/receipts_in_static_files)
- [`reth db settings set transaction_senders_in_static_files`](/cli/reth/db/settings/set/transaction_senders_in_static_files)
- [`reth download`](/cli/reth/download)
- [`reth stage`](/cli/reth/stage)
- [`reth stage run`](/cli/reth/stage/run)

9
docs/vocs/docs/pages/cli/reth.mdx
View File

@@ -146,4 +146,13 @@ Tracing:
Defaults to TRACE if not specified.
[default: debug]
--tracing-otlp.sample-ratio <RATIO>
Trace sampling ratio to control the percentage of traces to export.
Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
Example: --tracing-otlp.sample-ratio=0.0.
[env: OTEL_TRACES_SAMPLER_ARG=]
```

9
docs/vocs/docs/pages/cli/reth/config.mdx
View File

@@ -132,4 +132,13 @@ Tracing:
Defaults to TRACE if not specified.
[default: debug]
--tracing-otlp.sample-ratio <RATIO>
Trace sampling ratio to control the percentage of traces to export.
Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
Example: --tracing-otlp.sample-ratio=0.0.
[env: OTEL_TRACES_SAMPLER_ARG=]
```

19
docs/vocs/docs/pages/cli/reth/db.mdx
View File

@@ -111,6 +111,9 @@ Static Files:
--static-files.blocks-per-file.receipts <BLOCKS_PER_FILE_RECEIPTS>
Number of blocks per file for the receipts segment
--static-files.blocks-per-file.transaction-senders <BLOCKS_PER_FILE_TRANSACTION_SENDERS>
Number of blocks per file for the transaction senders segment
--static-files.receipts
Store receipts in static files instead of the database.
@@ -118,6 +121,13 @@ Static Files:
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
--static-files.transaction-senders
Store transaction senders in static files instead of the database.
When enabled, transaction senders will be written to static files on disk instead of the database.
Note: This setting can only be configured at genesis initialization. Once the node has been initialized, changing this flag requires re-syncing from scratch.
Logging:
--log.stdout.format <FORMAT>
The format to use for logs written to stdout
@@ -232,4 +242,13 @@ Tracing:
Defaults to TRACE if not specified.
[default: debug]
--tracing-otlp.sample-ratio <RATIO>
Trace sampling ratio to control the percentage of traces to export.
Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
Example: --tracing-otlp.sample-ratio=0.0.
[env: OTEL_TRACES_SAMPLER_ARG=]
```

9
docs/vocs/docs/pages/cli/reth/db/checksum.mdx
View File

@@ -149,4 +149,13 @@ Tracing:
Defaults to TRACE if not specified.
[default: debug]
--tracing-otlp.sample-ratio <RATIO>
Trace sampling ratio to control the percentage of traces to export.
Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
Example: --tracing-otlp.sample-ratio=0.0.
[env: OTEL_TRACES_SAMPLER_ARG=]
```

9
docs/vocs/docs/pages/cli/reth/db/clear.mdx
View File

@@ -141,4 +141,13 @@ Tracing:
Defaults to TRACE if not specified.
[default: debug]
--tracing-otlp.sample-ratio <RATIO>
Trace sampling ratio to control the percentage of traces to export.
Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
Example: --tracing-otlp.sample-ratio=0.0.
[env: OTEL_TRACES_SAMPLER_ARG=]
```

9
docs/vocs/docs/pages/cli/reth/db/clear/mdbx.mdx
View File

@@ -140,4 +140,13 @@ Tracing:
Defaults to TRACE if not specified.
[default: debug]
--tracing-otlp.sample-ratio <RATIO>
Trace sampling ratio to control the percentage of traces to export.
Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
Example: --tracing-otlp.sample-ratio=0.0.
[env: OTEL_TRACES_SAMPLER_ARG=]
```

16
docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx
View File

@@ -11,9 +11,10 @@ Usage: reth db clear static-file [OPTIONS] <SEGMENT>
Arguments:
<SEGMENT>
Possible values:
- headers: Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTerminalDifficulties` tables
- transactions: Static File segment responsible for the `Transactions` table
- receipts: Static File segment responsible for the `Receipts` table
- headers: Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTerminalDifficulties` tables
- transactions: Static File segment responsible for the `Transactions` table
- receipts: Static File segment responsible for the `Receipts` table
- transaction-senders: Static File segment responsible for the `TransactionSenders` table
Options:
-h, --help
@@ -143,4 +144,13 @@ Tracing:
Defaults to TRACE if not specified.
[default: debug]
--tracing-otlp.sample-ratio <RATIO>
Trace sampling ratio to control the percentage of traces to export.
Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
Example: --tracing-otlp.sample-ratio=0.0.
[env: OTEL_TRACES_SAMPLER_ARG=]
```

9
docs/vocs/docs/pages/cli/reth/db/diff.mdx
View File

@@ -192,4 +192,13 @@ Tracing:
Defaults to TRACE if not specified.
[default: debug]
--tracing-otlp.sample-ratio <RATIO>
Trace sampling ratio to control the percentage of traces to export.
Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
Example: --tracing-otlp.sample-ratio=0.0.
[env: OTEL_TRACES_SAMPLER_ARG=]
```

9
docs/vocs/docs/pages/cli/reth/db/drop.mdx
View File

@@ -139,4 +139,13 @@ Tracing:
Defaults to TRACE if not specified.
[default: debug]
--tracing-otlp.sample-ratio <RATIO>
Trace sampling ratio to control the percentage of traces to export.
Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
Example: --tracing-otlp.sample-ratio=0.0.
[env: OTEL_TRACES_SAMPLER_ARG=]
```

Some files were not shown because too many files have changed in this diff Show More