Compare commits

..

3 Commits

Author SHA1 Message Date
yongkangc
e10c071279 fix: clippy and fmt issues
- Add backticks to RocksDB and MDBX in doc comments
- Make with_assume_history_complete const fn
2026-01-22 14:56:13 +00:00
yongkangc
03a89ddaf1 fix: reintroduce pruning awareness in history_info softening
When deciding whether to soften NotYetWritten -> MaybeInPlainState,
we must consider both:
1. assume_history_complete flag (for hybrid storage)
2. lowest_available_block_number (for pruned history)

We only return NotYetWritten when history is complete AND not pruned.
This prevents incorrectly treating pruned entries as 'never written'.
2026-01-22 13:38:01 +00:00
yongkangc
d9c9960fd7 fix(rocksdb): return MaybeInPlainState for missing history entries
RocksDB only has history for blocks AFTER it was enabled. For accounts
that existed before RocksDB was enabled, returning NotYetWritten
incorrectly treats them as non-existent (nonce 0). We now return
MaybeInPlainState to trigger a plain state lookup.

Added assume_history_complete flag to RocksTx that:
- When false (default): returns MaybeInPlainState for hybrid storage
- When true: returns NotYetWritten to match MDBX semantics for tests

This preserves the correct hybrid storage behavior in production while
allowing tests with identical data to verify semantic equivalence.
2026-01-22 12:58:35 +00:00
180 changed files with 3005 additions and 6468 deletions

2
.github/CODEOWNERS vendored
View File

@@ -37,7 +37,7 @@ crates/storage/db/ @joshieDo
crates/storage/errors/ @joshieDo
crates/storage/libmdbx-rs/ @shekhirin
crates/storage/nippy-jar/ @joshieDo @shekhirin
crates/storage/provider/ @joshieDo @shekhirin @yongkangc
crates/storage/provider/ @joshieDo @shekhirin
crates/storage/storage-api/ @joshieDo
crates/tasks/ @mattsse
crates/tokio-util/ @mattsse

View File

@@ -60,6 +60,7 @@ jobs:
tail -50 Cargo.toml
- name: Check workspace
run: cargo clippy --workspace --lib --examples --tests --benches --all-features --locked
env:
RUSTFLAGS: -D warnings
run: cargo check --workspace --all-features
- name: Check Optimism
run: cargo check -p reth-optimism-node --all-features

267
Cargo.lock generated
View File

@@ -121,9 +121,9 @@ dependencies = [
[[package]]
name = "alloy-consensus"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed1958f0294ecc05ebe7b3c9a8662a3e221c2523b7f2bcd94c7a651efbd510bf"
checksum = "5c3a590d13de3944675987394715f37537b50b856e3b23a0e66e97d963edbf38"
dependencies = [
"alloy-eips",
"alloy-primitives",
@@ -149,9 +149,9 @@ dependencies = [
[[package]]
name = "alloy-consensus-any"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f752e99497ddc39e22d547d7dfe516af10c979405a034ed90e69b914b7dddeae"
checksum = "0f28f769d5ea999f0d8a105e434f483456a15b4e1fcb08edbbbe1650a497ff6d"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -164,9 +164,9 @@ dependencies = [
[[package]]
name = "alloy-contract"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2140796bc79150b1b7375daeab99750f0ff5e27b1f8b0aa81ccde229c7f02a2"
checksum = "990fa65cd132a99d3c3795a82b9f93ec82b81c7de3bab0bf26ca5c73286f7186"
dependencies = [
"alloy-consensus",
"alloy-dyn-abi",
@@ -255,21 +255,19 @@ checksum = "6adac476434bf024279164dcdca299309f0c7d1e3557024eb7a83f8d9d01c6b5"
dependencies = [
"alloy-primitives",
"alloy-rlp",
"arbitrary",
"borsh",
"serde",
]
[[package]]
name = "alloy-eips"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "813a67f87e56b38554d18b182616ee5006e8e2bf9df96a0df8bf29dff1d52e3f"
checksum = "09535cbc646b0e0c6fcc12b7597eaed12cf86dff4c4fba9507a61e71b94f30eb"
dependencies = [
"alloy-eip2124",
"alloy-eip2930",
"alloy-eip7702",
"alloy-eip7928",
"alloy-primitives",
"alloy-rlp",
"alloy-serde",
@@ -289,9 +287,9 @@ dependencies = [
[[package]]
name = "alloy-evm"
version = "0.27.0"
version = "0.26.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1582933a9fc27c0953220eb4f18f6492ff577822e9a8d848890ff59f6b4f5beb"
checksum = "a96827207397445a919a8adc49289b53cc74e48e460411740bba31cec2fc307d"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -311,9 +309,9 @@ dependencies = [
[[package]]
name = "alloy-genesis"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05864eef929c4d28895ae4b4d8ac9c6753c4df66e873b9c8fafc8089b59c1502"
checksum = "1005520ccf89fa3d755e46c1d992a9e795466c2e7921be2145ef1f749c5727de"
dependencies = [
"alloy-eips",
"alloy-primitives",
@@ -352,9 +350,9 @@ dependencies = [
[[package]]
name = "alloy-json-rpc"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2dd146b3de349a6ffaa4e4e319ab3a90371fb159fb0bddeb1c7bbe8b1792eff"
checksum = "72b626409c98ba43aaaa558361bca21440c88fd30df7542c7484b9c7a1489cdb"
dependencies = [
"alloy-primitives",
"alloy-sol-types",
@@ -367,9 +365,9 @@ dependencies = [
[[package]]
name = "alloy-network"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c12278ffbb8872dfba3b2f17d8ea5e8503c2df5155d9bc5ee342794bde505c3"
checksum = "89924fdcfeee0e0fa42b1f10af42f92802b5d16be614a70897382565663bf7cf"
dependencies = [
"alloy-consensus",
"alloy-consensus-any",
@@ -393,9 +391,9 @@ dependencies = [
[[package]]
name = "alloy-network-primitives"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "833037c04917bc2031541a60e8249e4ab5500e24c637c1c62e95e963a655d66f"
checksum = "0f0dbe56ff50065713ff8635d8712a0895db3ad7f209db9793ad8fcb6b1734aa"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -406,9 +404,9 @@ dependencies = [
[[package]]
name = "alloy-op-evm"
version = "0.27.0"
version = "0.26.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f19214adae08ea95600c3ede76bcbf0c40b36a263534a8f441a4c732f60e868"
checksum = "54dc5c46a92fc7267055a174d30efb34e2599a0047102a4d38a025ae521435ba"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -469,9 +467,9 @@ dependencies = [
[[package]]
name = "alloy-provider"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eafa840b0afe01c889a3012bb2fde770a544f74eab2e2870303eb0a5fb869c48"
checksum = "8b56f7a77513308a21a2ba0e9d57785a9d9d2d609e77f4e71a78a1192b83ff2d"
dependencies = [
"alloy-chains",
"alloy-consensus",
@@ -514,9 +512,9 @@ dependencies = [
[[package]]
name = "alloy-pubsub"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57b3a3b3e4efc9f4d30e3326b6bd6811231d16ef94837e18a802b44ca55119e6"
checksum = "94813abbd7baa30c700ea02e7f92319dbcb03bff77aeea92a3a9af7ba19c5c70"
dependencies = [
"alloy-json-rpc",
"alloy-primitives",
@@ -558,9 +556,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-client"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12768ae6303ec764905a8a7cd472aea9072f9f9c980d18151e26913da8ae0123"
checksum = "ff01723afc25ec4c5b04de399155bef7b6a96dfde2475492b1b7b4e7a4f46445"
dependencies = [
"alloy-json-rpc",
"alloy-primitives",
@@ -584,9 +582,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0622d8bcac2f16727590aa33f4c3f05ea98130e7e4b4924bce8be85da5ad0dae"
checksum = "f91bf006bb06b7d812591b6ac33395cb92f46c6a65cda11ee30b348338214f0f"
dependencies = [
"alloy-primitives",
"alloy-rpc-types-engine",
@@ -597,9 +595,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-admin"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c38c5ac70457ecc74e87fe1a5a19f936419224ded0eb0636241452412ca92733"
checksum = "b934c3bcdc6617563b45deb36a40881c8230b94d0546ea739dff7edb3aa2f6fd"
dependencies = [
"alloy-genesis",
"alloy-primitives",
@@ -609,9 +607,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-anvil"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae8eb0e5d6c48941b61ab76fabab4af66f7d88309a98aa14ad3dec7911c1eba3"
checksum = "7e82145856df8abb1fefabef58cdec0f7d9abf337d4abd50c1ed7e581634acdd"
dependencies = [
"alloy-primitives",
"alloy-rpc-types-eth",
@@ -621,9 +619,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-any"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1cf5a093e437dfd62df48e480f24e1a3807632358aad6816d7a52875f1c04aa"
checksum = "212ca1c1dab27f531d3858f8b1a2d6bfb2da664be0c1083971078eb7b71abe4b"
dependencies = [
"alloy-consensus-any",
"alloy-rpc-types-eth",
@@ -632,9 +630,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-beacon"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e07949e912479ef3b848e1cf8db54b534bdd7bc58e6c23f28ea9488960990c8c"
checksum = "6d92a9b4b268fac505ef7fb1dac9bb129d4fd7de7753f22a5b6e9f666f7f7de6"
dependencies = [
"alloy-eips",
"alloy-primitives",
@@ -652,9 +650,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-debug"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "925ff0f48c2169c050f0ae7a82769bdf3f45723d6742ebb6a5efb4ed2f491b26"
checksum = "bab1ebed118b701c497e6541d2d11dfa6f3c6ae31a3c52999daa802fcdcc16b7"
dependencies = [
"alloy-primitives",
"derive_more",
@@ -664,9 +662,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-engine"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "336ef381c7409f23c69f6e79bddc1917b6e832cff23e7a5cf84b9381d53582e6"
checksum = "232f00fcbcd3ee3b9399b96223a8fc884d17742a70a44f9d7cef275f93e6e872"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -685,9 +683,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-eth"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28e97603095020543a019ab133e0e3dc38cd0819f19f19bdd70c642404a54751"
checksum = "5715d0bf7efbd360873518bd9f6595762136b5327a9b759a8c42ccd9b5e44945"
dependencies = [
"alloy-consensus",
"alloy-consensus-any",
@@ -707,9 +705,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-mev"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2805153975e25d38e37ee100880e642d5b24e421ed3014a7d2dae1d9be77562e"
checksum = "c7b61941d2add2ee64646612d3eda92cbbde8e6c933489760b6222c8898c79be"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -722,9 +720,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-trace"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1aec4e1c66505d067933ea1a949a4fb60a19c4cfc2f109aa65873ea99e62ea8"
checksum = "9763cc931a28682bd4b9a68af90057b0fbe80e2538a82251afd69d7ae00bbebf"
dependencies = [
"alloy-primitives",
"alloy-rpc-types-eth",
@@ -736,9 +734,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-txpool"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25b73c1d6e4f1737a20d246dad5a0abd6c1b76ec4c3d153684ef8c6f1b6bb4f4"
checksum = "359a8caaa98cb49eed62d03f5bc511dd6dd5dee292238e8627a6e5690156df0f"
dependencies = [
"alloy-primitives",
"alloy-rpc-types-eth",
@@ -748,9 +746,9 @@ dependencies = [
[[package]]
name = "alloy-serde"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "946a0d413dbb5cd9adba0de5f8a1a34d5b77deda9b69c1d7feed8fc875a1aa26"
checksum = "5ed8531cae8d21ee1c6571d0995f8c9f0652a6ef6452fde369283edea6ab7138"
dependencies = [
"alloy-primitives",
"arbitrary",
@@ -760,9 +758,9 @@ dependencies = [
[[package]]
name = "alloy-signer"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f7481dc8316768f042495eaf305d450c32defbc9bce09d8bf28afcd956895bb"
checksum = "fb10ccd49d0248df51063fce6b716f68a315dd912d55b32178c883fd48b4021d"
dependencies = [
"alloy-primitives",
"async-trait",
@@ -775,9 +773,9 @@ dependencies = [
[[package]]
name = "alloy-signer-local"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1259dac1f534a4c66c1d65237c89915d0010a2a91d6c3b0bada24dc5ee0fb917"
checksum = "f4d992d44e6c414ece580294abbadb50e74cfd4eaa69787350a4dfd4b20eaa1b"
dependencies = [
"alloy-consensus",
"alloy-network",
@@ -864,9 +862,9 @@ dependencies = [
[[package]]
name = "alloy-transport"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78f169b85eb9334871db986e7eaf59c58a03d86a30cc68b846573d47ed0656bb"
checksum = "3f50a9516736d22dd834cc2240e5bf264f338667cc1d9e514b55ec5a78b987ca"
dependencies = [
"alloy-json-rpc",
"auto_impl",
@@ -887,9 +885,9 @@ dependencies = [
[[package]]
name = "alloy-transport-http"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "019821102e70603e2c141954418255bec539ef64ac4117f8e84fb493769acf73"
checksum = "0a18b541a6197cf9a084481498a766fdf32fefda0c35ea6096df7d511025e9f1"
dependencies = [
"alloy-json-rpc",
"alloy-transport",
@@ -902,9 +900,9 @@ dependencies = [
[[package]]
name = "alloy-transport-ipc"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e574ca2f490fb5961d2cdd78188897392c46615cd88b35c202d34bbc31571a81"
checksum = "8075911680ebc537578cacf9453464fd394822a0f68614884a9c63f9fbaf5e89"
dependencies = [
"alloy-json-rpc",
"alloy-pubsub",
@@ -922,9 +920,9 @@ dependencies = [
[[package]]
name = "alloy-transport-ws"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b92dea6996269769f74ae56475570e3586910661e037b7b52d50c9641f76c68f"
checksum = "921d37a57e2975e5215f7dd0f28873ed5407c7af630d4831a4b5c737de4b0b8b"
dependencies = [
"alloy-pubsub",
"alloy-transport",
@@ -959,9 +957,9 @@ dependencies = [
[[package]]
name = "alloy-tx-macros"
version = "1.5.2"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45ceac797eb8a56bdf5ab1fab353072c17d472eab87645ca847afe720db3246d"
checksum = "b2289a842d02fe63f8c466db964168bb2c7a9fdfb7b24816dbb17d45520575fb"
dependencies = [
"darling 0.21.3",
"proc-macro2",
@@ -7909,7 +7907,6 @@ dependencies = [
"reth-stages-types",
"reth-static-file",
"reth-static-file-types",
"reth-storage-api",
"reth-tasks",
"reth-trie",
"reth-trie-common",
@@ -10628,7 +10625,6 @@ dependencies = [
"jsonrpsee-core",
"jsonrpsee-types",
"metrics",
"parking_lot",
"reth-chainspec",
"reth-engine-primitives",
"reth-ethereum-engine-primitives",
@@ -14641,3 +14637,138 @@ dependencies = [
"cc",
"pkg-config",
]
[[patch.unused]]
name = "alloy-consensus"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-contract"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-eips"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-genesis"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-json-rpc"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-network"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-network-primitives"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-provider"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-pubsub"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-rpc-client"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-rpc-types"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-rpc-types-admin"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-rpc-types-anvil"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-rpc-types-beacon"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-rpc-types-debug"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-rpc-types-engine"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-rpc-types-eth"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-rpc-types-mev"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-rpc-types-trace"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-rpc-types-txpool"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-serde"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-signer"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-signer-local"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-transport"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-transport-http"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-transport-ipc"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
[[patch.unused]]
name = "alloy-transport-ws"
version = "1.5.1"
source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"

View File

@@ -485,10 +485,10 @@ revm-inspectors = "0.34.0"
# eth
alloy-chains = { version = "0.2.5", default-features = false }
alloy-dyn-abi = "1.5.2"
alloy-dyn-abi = "1.4.3"
alloy-eip2124 = { version = "0.2.0", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-evm = { version = "0.27.0", default-features = false }
alloy-evm = { version = "0.26.3", default-features = false }
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
alloy-sol-macro = "1.5.0"
@@ -497,36 +497,36 @@ alloy-trie = { version = "0.9.1", default-features = false }
alloy-hardforks = "0.4.5"
alloy-consensus = { version = "1.5.2", default-features = false }
alloy-contract = { version = "1.5.2", default-features = false }
alloy-eips = { version = "1.5.2", default-features = false }
alloy-genesis = { version = "1.5.2", default-features = false }
alloy-json-rpc = { version = "1.5.2", default-features = false }
alloy-network = { version = "1.5.2", default-features = false }
alloy-network-primitives = { version = "1.5.2", default-features = false }
alloy-provider = { version = "1.5.2", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.5.2", default-features = false }
alloy-rpc-client = { version = "1.5.2", default-features = false }
alloy-rpc-types = { version = "1.5.2", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.5.2", default-features = false }
alloy-rpc-types-anvil = { version = "1.5.2", default-features = false }
alloy-rpc-types-beacon = { version = "1.5.2", default-features = false }
alloy-rpc-types-debug = { version = "1.5.2", default-features = false }
alloy-rpc-types-engine = { version = "1.5.2", default-features = false }
alloy-rpc-types-eth = { version = "1.5.2", default-features = false }
alloy-rpc-types-mev = { version = "1.5.2", default-features = false }
alloy-rpc-types-trace = { version = "1.5.2", default-features = false }
alloy-rpc-types-txpool = { version = "1.5.2", default-features = false }
alloy-serde = { version = "1.5.2", default-features = false }
alloy-signer = { version = "1.5.2", default-features = false }
alloy-signer-local = { version = "1.5.2", default-features = false }
alloy-transport = { version = "1.5.2" }
alloy-transport-http = { version = "1.5.2", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.5.2", default-features = false }
alloy-transport-ws = { version = "1.5.2", default-features = false }
alloy-consensus = { version = "1.4.3", default-features = false }
alloy-contract = { version = "1.4.3", default-features = false }
alloy-eips = { version = "1.4.3", default-features = false }
alloy-genesis = { version = "1.4.3", default-features = false }
alloy-json-rpc = { version = "1.4.3", default-features = false }
alloy-network = { version = "1.4.3", default-features = false }
alloy-network-primitives = { version = "1.4.3", default-features = false }
alloy-provider = { version = "1.4.3", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.4.3", default-features = false }
alloy-rpc-client = { version = "1.4.3", default-features = false }
alloy-rpc-types = { version = "1.4.3", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.4.3", default-features = false }
alloy-rpc-types-anvil = { version = "1.4.3", default-features = false }
alloy-rpc-types-beacon = { version = "1.4.3", default-features = false }
alloy-rpc-types-debug = { version = "1.4.3", default-features = false }
alloy-rpc-types-engine = { version = "1.4.3", default-features = false }
alloy-rpc-types-eth = { version = "1.4.3", default-features = false }
alloy-rpc-types-mev = { version = "1.4.3", default-features = false }
alloy-rpc-types-trace = { version = "1.4.3", default-features = false }
alloy-rpc-types-txpool = { version = "1.4.3", default-features = false }
alloy-serde = { version = "1.4.3", default-features = false }
alloy-signer = { version = "1.4.3", default-features = false }
alloy-signer-local = { version = "1.4.3", default-features = false }
alloy-transport = { version = "1.4.3" }
alloy-transport-http = { version = "1.4.3", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.4.3", default-features = false }
alloy-transport-ws = { version = "1.4.3", default-features = false }
# op
alloy-op-evm = { version = "0.27.0", default-features = false }
alloy-op-evm = { version = "0.26.3", default-features = false }
alloy-op-hardforks = "0.4.4"
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
@@ -790,10 +790,39 @@ ipnet = "2.11"
# jsonrpsee-http-client = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# jsonrpsee-types = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "3020ea8" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# Patched by patch-alloy.sh
alloy-consensus = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-contract = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-eips = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-genesis = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-network = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-provider = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-serde = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-signer = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", branch = "main" }

View File

@@ -32,7 +32,7 @@ alloy-eips.workspace = true
alloy-json-rpc.workspace = true
alloy-consensus.workspace = true
alloy-network.workspace = true
alloy-primitives = { workspace = true, features = ["rand"] }
alloy-primitives.workspace = true
alloy-provider = { workspace = true, features = ["engine-api", "pubsub", "reqwest-rustls-tls"], default-features = false }
alloy-pubsub.workspace = true
alloy-rpc-client = { workspace = true, features = ["pubsub"] }

View File

@@ -3,7 +3,7 @@
use crate::{
authenticated_transport::AuthenticatedTransportConnect,
bench::{
helpers::{build_payload, parse_gas_limit, prepare_payload_request, rpc_block_to_header},
helpers::{build_payload, prepare_payload_request, rpc_block_to_header},
output::GasRampPayloadFile,
},
valid_payload::{call_forkchoice_updated, call_new_payload, payload_to_new_payload},
@@ -22,6 +22,29 @@ use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIM
use std::{path::PathBuf, time::Instant};
use tracing::info;
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
///
/// Examples: "30000000", "30M", "1G", "2G"
fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
let s = s.trim();
if s.is_empty() {
return Err(eyre::eyre!("empty value"));
}
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
(prefix, 1_000_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
(prefix, 1_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
(prefix, 1_000u64)
} else {
(s, 1u64)
};
let base: u64 = num_str.trim().parse()?;
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
}
/// `reth benchmark gas-limit-ramp` command.
#[derive(Debug, Parser)]
pub struct Command {
@@ -214,3 +237,50 @@ const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u
RampMode::TargetGasLimit(target) => current_gas_limit >= target,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_gas_limit_plain_number() {
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("1").unwrap(), 1);
assert_eq!(parse_gas_limit("0").unwrap(), 0);
}
#[test]
fn test_parse_gas_limit_k_suffix() {
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
}
#[test]
fn test_parse_gas_limit_m_suffix() {
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
}
#[test]
fn test_parse_gas_limit_g_suffix() {
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
}
#[test]
fn test_parse_gas_limit_with_whitespace() {
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
}
#[test]
fn test_parse_gas_limit_errors() {
assert!(parse_gas_limit("").is_err());
assert!(parse_gas_limit("abc").is_err());
assert!(parse_gas_limit("G").is_err());
assert!(parse_gas_limit("-1G").is_err());
}
}

View File

@@ -3,9 +3,7 @@
//! This command fetches transactions from existing blocks and packs them into a single
//! large block using the `testing_buildBlockV1` RPC endpoint.
use crate::{
authenticated_transport::AuthenticatedTransportConnect, bench::helpers::parse_gas_limit,
};
use crate::authenticated_transport::AuthenticatedTransportConnect;
use alloy_eips::{BlockNumberOrTag, Typed2718};
use alloy_primitives::{Bytes, B256};
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
@@ -204,9 +202,7 @@ pub struct Command {
jwt_secret: std::path::PathBuf,
/// Target gas to pack into the block.
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 1G = 1
/// billion).
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000", value_parser = parse_gas_limit)]
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000")]
target_gas: u64,
/// Starting block number to fetch transactions from.

View File

@@ -1,29 +1,6 @@
//! Common helpers for reth-bench commands.
use crate::valid_payload::call_forkchoice_updated;
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
///
/// Examples: "30000000", "30M", "1G", "2G"
pub(crate) fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
let s = s.trim();
if s.is_empty() {
return Err(eyre::eyre!("empty value"));
}
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
(prefix, 1_000_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
(prefix, 1_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
(prefix, 1_000u64)
} else {
(s, 1u64)
};
let base: u64 = num_str.trim().parse()?;
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
}
use alloy_consensus::Header;
use alloy_eips::eip4844::kzg_to_versioned_hash;
use alloy_primitives::{Address, B256};
@@ -217,50 +194,3 @@ pub(crate) async fn get_payload_with_sidecar(
_ => panic!("This tool does not support getPayload versions past v5"),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_gas_limit_plain_number() {
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("1").unwrap(), 1);
assert_eq!(parse_gas_limit("0").unwrap(), 0);
}
#[test]
fn test_parse_gas_limit_k_suffix() {
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
}
#[test]
fn test_parse_gas_limit_m_suffix() {
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
}
#[test]
fn test_parse_gas_limit_g_suffix() {
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
}
#[test]
fn test_parse_gas_limit_with_whitespace() {
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
}
#[test]
fn test_parse_gas_limit_errors() {
assert!(parse_gas_limit("").is_err());
assert!(parse_gas_limit("abc").is_err());
assert!(parse_gas_limit("G").is_err());
assert!(parse_gas_limit("-1G").is_err());
}
}

View File

@@ -16,7 +16,6 @@ mod new_payload_fcu;
mod new_payload_only;
mod output;
mod replay_payloads;
mod send_invalid_payload;
mod send_payload;
/// `reth bench` command
@@ -75,18 +74,6 @@ pub enum Subcommands {
/// `reth-bench replay-payloads --payload-dir ./payloads --engine-rpc-url
/// http://localhost:8551 --jwt-secret ~/.local/share/reth/mainnet/jwt.hex`
ReplayPayloads(replay_payloads::Command),
/// Generate and send an invalid `engine_newPayload` request for testing.
///
/// Takes a valid block and modifies fields to make it invalid, allowing you to test
/// Engine API rejection behavior. Block hash is recalculated after modifications
/// unless `--invalid-block-hash` or `--skip-hash-recalc` is used.
///
/// Example:
///
/// `cast block latest --full --json | reth-bench send-invalid-payload --rpc-url localhost:5000
/// --jwt-secret $(cat ~/.local/share/reth/mainnet/jwt.hex) --invalid-state-root`
SendInvalidPayload(Box<send_invalid_payload::Command>),
}
impl BenchmarkCommand {
@@ -102,7 +89,6 @@ impl BenchmarkCommand {
Subcommands::SendPayload(command) => command.execute(ctx).await,
Subcommands::GenerateBigBlock(command) => command.execute(ctx).await,
Subcommands::ReplayPayloads(command) => command.execute(ctx).await,
Subcommands::SendInvalidPayload(command) => (*command).execute(ctx).await,
}
}

View File

@@ -1,219 +0,0 @@
use alloy_eips::eip4895::Withdrawal;
use alloy_primitives::{Address, Bloom, Bytes, B256, U256};
use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3};
/// Configuration for invalidating payload fields
#[derive(Debug, Default)]
pub(super) struct InvalidationConfig {
// Explicit value overrides (Option<T>)
pub(super) parent_hash: Option<B256>,
pub(super) fee_recipient: Option<Address>,
pub(super) state_root: Option<B256>,
pub(super) receipts_root: Option<B256>,
pub(super) logs_bloom: Option<Bloom>,
pub(super) prev_randao: Option<B256>,
pub(super) block_number: Option<u64>,
pub(super) gas_limit: Option<u64>,
pub(super) gas_used: Option<u64>,
pub(super) timestamp: Option<u64>,
pub(super) extra_data: Option<Bytes>,
pub(super) base_fee_per_gas: Option<u64>,
pub(super) block_hash: Option<B256>,
pub(super) blob_gas_used: Option<u64>,
pub(super) excess_blob_gas: Option<u64>,
// Auto-invalidation flags
pub(super) invalidate_parent_hash: bool,
pub(super) invalidate_state_root: bool,
pub(super) invalidate_receipts_root: bool,
pub(super) invalidate_gas_used: bool,
pub(super) invalidate_block_number: bool,
pub(super) invalidate_timestamp: bool,
pub(super) invalidate_base_fee: bool,
pub(super) invalidate_transactions: bool,
pub(super) invalidate_block_hash: bool,
pub(super) invalidate_withdrawals: bool,
pub(super) invalidate_blob_gas_used: bool,
pub(super) invalidate_excess_blob_gas: bool,
}
impl InvalidationConfig {
/// Returns true if `block_hash` is being explicitly set or auto-invalidated.
/// When true, the caller should skip recalculating the block hash since it will be overwritten.
pub(super) const fn should_skip_hash_recalc(&self) -> bool {
self.block_hash.is_some() || self.invalidate_block_hash
}
/// Applies invalidations to a V1 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v1(&self, payload: &mut ExecutionPayloadV1) -> Vec<String> {
let mut changes = Vec::new();
// Explicit value overrides
if let Some(parent_hash) = self.parent_hash {
payload.parent_hash = parent_hash;
changes.push(format!("parent_hash = {parent_hash}"));
}
if let Some(fee_recipient) = self.fee_recipient {
payload.fee_recipient = fee_recipient;
changes.push(format!("fee_recipient = {fee_recipient}"));
}
if let Some(state_root) = self.state_root {
payload.state_root = state_root;
changes.push(format!("state_root = {state_root}"));
}
if let Some(receipts_root) = self.receipts_root {
payload.receipts_root = receipts_root;
changes.push(format!("receipts_root = {receipts_root}"));
}
if let Some(logs_bloom) = self.logs_bloom {
payload.logs_bloom = logs_bloom;
changes.push("logs_bloom = <custom>".to_string());
}
if let Some(prev_randao) = self.prev_randao {
payload.prev_randao = prev_randao;
changes.push(format!("prev_randao = {prev_randao}"));
}
if let Some(block_number) = self.block_number {
payload.block_number = block_number;
changes.push(format!("block_number = {block_number}"));
}
if let Some(gas_limit) = self.gas_limit {
payload.gas_limit = gas_limit;
changes.push(format!("gas_limit = {gas_limit}"));
}
if let Some(gas_used) = self.gas_used {
payload.gas_used = gas_used;
changes.push(format!("gas_used = {gas_used}"));
}
if let Some(timestamp) = self.timestamp {
payload.timestamp = timestamp;
changes.push(format!("timestamp = {timestamp}"));
}
if let Some(ref extra_data) = self.extra_data {
payload.extra_data = extra_data.clone();
changes.push(format!("extra_data = {} bytes", extra_data.len()));
}
if let Some(base_fee_per_gas) = self.base_fee_per_gas {
payload.base_fee_per_gas = U256::from_limbs([base_fee_per_gas, 0, 0, 0]);
changes.push(format!("base_fee_per_gas = {base_fee_per_gas}"));
}
if let Some(block_hash) = self.block_hash {
payload.block_hash = block_hash;
changes.push(format!("block_hash = {block_hash}"));
}
// Auto-invalidation flags
if self.invalidate_parent_hash {
let random_hash = B256::random();
payload.parent_hash = random_hash;
changes.push(format!("parent_hash = {random_hash} (auto-invalidated: random)"));
}
if self.invalidate_state_root {
payload.state_root = B256::ZERO;
changes.push("state_root = ZERO (auto-invalidated: empty trie root)".to_string());
}
if self.invalidate_receipts_root {
payload.receipts_root = B256::ZERO;
changes.push("receipts_root = ZERO (auto-invalidated)".to_string());
}
if self.invalidate_gas_used {
let invalid_gas = payload.gas_limit + 1;
payload.gas_used = invalid_gas;
changes.push(format!("gas_used = {invalid_gas} (auto-invalidated: exceeds gas_limit)"));
}
if self.invalidate_block_number {
let invalid_number = payload.block_number + 999;
payload.block_number = invalid_number;
changes.push(format!("block_number = {invalid_number} (auto-invalidated: huge gap)"));
}
if self.invalidate_timestamp {
payload.timestamp = 0;
changes.push("timestamp = 0 (auto-invalidated: impossibly old)".to_string());
}
if self.invalidate_base_fee {
payload.base_fee_per_gas = U256::ZERO;
changes
.push("base_fee_per_gas = 0 (auto-invalidated: invalid post-London)".to_string());
}
if self.invalidate_transactions {
let invalid_tx = Bytes::from_static(&[0xff, 0xff, 0xff]);
payload.transactions.insert(0, invalid_tx);
changes.push("transactions = prepended invalid RLP (auto-invalidated)".to_string());
}
if self.invalidate_block_hash {
let random_hash = B256::random();
payload.block_hash = random_hash;
changes.push(format!("block_hash = {random_hash} (auto-invalidated: random)"));
}
changes
}
/// Applies invalidations to a V2 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v2(&self, payload: &mut ExecutionPayloadV2) -> Vec<String> {
let mut changes = self.apply_to_payload_v1(&mut payload.payload_inner);
// Handle withdrawals invalidation (V2+)
if self.invalidate_withdrawals {
let fake_withdrawal = Withdrawal {
index: u64::MAX,
validator_index: u64::MAX,
address: Address::ZERO,
amount: u64::MAX,
};
payload.withdrawals.push(fake_withdrawal);
changes.push("withdrawals = added fake withdrawal (auto-invalidated)".to_string());
}
changes
}
/// Applies invalidations to a V3 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v3(&self, payload: &mut ExecutionPayloadV3) -> Vec<String> {
let mut changes = self.apply_to_payload_v2(&mut payload.payload_inner);
// Explicit overrides for V3 fields
if let Some(blob_gas_used) = self.blob_gas_used {
payload.blob_gas_used = blob_gas_used;
changes.push(format!("blob_gas_used = {blob_gas_used}"));
}
if let Some(excess_blob_gas) = self.excess_blob_gas {
payload.excess_blob_gas = excess_blob_gas;
changes.push(format!("excess_blob_gas = {excess_blob_gas}"));
}
// Auto-invalidation for V3 fields
if self.invalidate_blob_gas_used {
payload.blob_gas_used = u64::MAX;
changes.push("blob_gas_used = MAX (auto-invalidated)".to_string());
}
if self.invalidate_excess_blob_gas {
payload.excess_blob_gas = u64::MAX;
changes.push("excess_blob_gas = MAX (auto-invalidated)".to_string());
}
changes
}
}

View File

@@ -1,367 +0,0 @@
//! Command for sending invalid payloads to test Engine API rejection.
mod invalidation;
use invalidation::InvalidationConfig;
use alloy_primitives::{Address, B256};
use alloy_provider::network::AnyRpcBlock;
use alloy_rpc_types_engine::ExecutionPayload;
use clap::Parser;
use eyre::{OptionExt, Result};
use op_alloy_consensus::OpTxEnvelope;
use reth_cli_runner::CliContext;
use std::io::{BufReader, Read, Write};
/// Command for generating and sending an invalid `engine_newPayload` request.
///
/// Takes a valid block and modifies fields to make it invalid for testing
/// Engine API rejection behavior. Block hash is recalculated after modifications
/// unless `--invalidate-block-hash` or `--skip-hash-recalc` is used.
#[derive(Debug, Parser)]
pub struct Command {
// ==================== Input Options ====================
/// Path to the JSON file containing the block. If not specified, stdin will be used.
#[arg(short, long, help_heading = "Input Options")]
path: Option<String>,
/// The engine RPC URL to use.
#[arg(
short,
long,
help_heading = "Input Options",
required_if_eq_any([("mode", "execute"), ("mode", "cast")]),
required_unless_present("mode")
)]
rpc_url: Option<String>,
/// The JWT secret to use. Can be either a path to a file containing the secret or the secret
/// itself.
#[arg(short, long, help_heading = "Input Options")]
jwt_secret: Option<String>,
/// The newPayload version to use (3 or 4).
#[arg(long, default_value_t = 3, help_heading = "Input Options")]
new_payload_version: u8,
/// The output mode to use.
#[arg(long, value_enum, default_value = "execute", help_heading = "Input Options")]
mode: Mode,
// ==================== Explicit Value Overrides ====================
/// Override the parent hash with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
parent_hash: Option<B256>,
/// Override the fee recipient (coinbase) with a specific address.
#[arg(long, value_name = "ADDR", help_heading = "Explicit Value Overrides")]
fee_recipient: Option<Address>,
/// Override the state root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
state_root: Option<B256>,
/// Override the receipts root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
receipts_root: Option<B256>,
/// Override the block number with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
block_number: Option<u64>,
/// Override the gas limit with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
gas_limit: Option<u64>,
/// Override the gas used with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
gas_used: Option<u64>,
/// Override the timestamp with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
timestamp: Option<u64>,
/// Override the base fee per gas with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
base_fee_per_gas: Option<u64>,
/// Override the block hash with a specific value (skips hash recalculation).
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
block_hash: Option<B256>,
/// Override the blob gas used with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
blob_gas_used: Option<u64>,
/// Override the excess blob gas with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
excess_blob_gas: Option<u64>,
/// Override the parent beacon block root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
parent_beacon_block_root: Option<B256>,
/// Override the requests hash with a specific value (EIP-7685).
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
requests_hash: Option<B256>,
// ==================== Auto-Invalidation Flags ====================
/// Invalidate the parent hash by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_parent_hash: bool,
/// Invalidate the state root by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_state_root: bool,
/// Invalidate the receipts root by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_receipts_root: bool,
/// Invalidate the gas used by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_gas_used: bool,
/// Invalidate the block number by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_block_number: bool,
/// Invalidate the timestamp by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_timestamp: bool,
/// Invalidate the base fee by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_base_fee: bool,
/// Invalidate the transactions by modifying them.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_transactions: bool,
/// Invalidate the block hash by not recalculating it after modifications.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_block_hash: bool,
/// Invalidate the withdrawals by modifying them.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_withdrawals: bool,
/// Invalidate the blob gas used by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_blob_gas_used: bool,
/// Invalidate the excess blob gas by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_excess_blob_gas: bool,
/// Invalidate the requests hash by setting it to a random value (EIP-7685).
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_requests_hash: bool,
// ==================== Meta Flags ====================
/// Skip block hash recalculation after modifications.
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
skip_hash_recalc: bool,
/// Print what would be done without actually sending the payload.
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
dry_run: bool,
}
#[derive(Debug, Clone, clap::ValueEnum)]
enum Mode {
/// Execute the `cast` command. This works with blocks of any size, because it pipes the
/// payload into the `cast` command.
Execute,
/// Print the `cast` command. Caution: this may not work with large blocks because of the
/// command length limit.
Cast,
/// Print the JSON payload. Can be piped into `cast` command if the block is small enough.
Json,
}
impl Command {
/// Read input from either a file or stdin
fn read_input(&self) -> Result<String> {
Ok(match &self.path {
Some(path) => reth_fs_util::read_to_string(path)?,
None => String::from_utf8(
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
)?,
})
}
/// Load JWT secret from either a file or use the provided string directly
fn load_jwt_secret(&self) -> Result<Option<String>> {
match &self.jwt_secret {
Some(secret) => match std::fs::read_to_string(secret) {
Ok(contents) => Ok(Some(contents.trim().to_string())),
Err(_) => Ok(Some(secret.clone())),
},
None => Ok(None),
}
}
/// Build `InvalidationConfig` from command flags
const fn build_invalidation_config(&self) -> InvalidationConfig {
InvalidationConfig {
parent_hash: self.parent_hash,
fee_recipient: self.fee_recipient,
state_root: self.state_root,
receipts_root: self.receipts_root,
logs_bloom: None,
prev_randao: None,
block_number: self.block_number,
gas_limit: self.gas_limit,
gas_used: self.gas_used,
timestamp: self.timestamp,
extra_data: None,
base_fee_per_gas: self.base_fee_per_gas,
block_hash: self.block_hash,
blob_gas_used: self.blob_gas_used,
excess_blob_gas: self.excess_blob_gas,
invalidate_parent_hash: self.invalidate_parent_hash,
invalidate_state_root: self.invalidate_state_root,
invalidate_receipts_root: self.invalidate_receipts_root,
invalidate_gas_used: self.invalidate_gas_used,
invalidate_block_number: self.invalidate_block_number,
invalidate_timestamp: self.invalidate_timestamp,
invalidate_base_fee: self.invalidate_base_fee,
invalidate_transactions: self.invalidate_transactions,
invalidate_block_hash: self.invalidate_block_hash,
invalidate_withdrawals: self.invalidate_withdrawals,
invalidate_blob_gas_used: self.invalidate_blob_gas_used,
invalidate_excess_blob_gas: self.invalidate_excess_blob_gas,
}
}
/// Execute the command
pub async fn execute(self, _ctx: CliContext) -> Result<()> {
let block_json = self.read_input()?;
let jwt_secret = self.load_jwt_secret()?;
let block = serde_json::from_str::<AnyRpcBlock>(&block_json)?
.into_inner()
.map_header(|header| header.map(|h| h.into_header_with_defaults()))
.try_map_transactions(|tx| tx.try_into_either::<OpTxEnvelope>())?
.into_consensus();
let config = self.build_invalidation_config();
let parent_beacon_block_root =
self.parent_beacon_block_root.or(block.header.parent_beacon_block_root);
let blob_versioned_hashes =
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
let use_v4 = block.header.requests_hash.is_some();
let requests_hash = self.requests_hash.or(block.header.requests_hash);
let mut execution_payload = ExecutionPayload::from_block_slow(&block).0;
let changes = match &mut execution_payload {
ExecutionPayload::V1(p) => config.apply_to_payload_v1(p),
ExecutionPayload::V2(p) => config.apply_to_payload_v2(p),
ExecutionPayload::V3(p) => config.apply_to_payload_v3(p),
};
let skip_recalc = self.skip_hash_recalc || config.should_skip_hash_recalc();
if !skip_recalc {
let new_hash = match execution_payload.clone().into_block_raw() {
Ok(block) => block.header.hash_slow(),
Err(e) => {
eprintln!(
"Warning: Could not recalculate block hash: {e}. Using original hash."
);
match &execution_payload {
ExecutionPayload::V1(p) => p.block_hash,
ExecutionPayload::V2(p) => p.payload_inner.block_hash,
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash,
}
}
};
match &mut execution_payload {
ExecutionPayload::V1(p) => p.block_hash = new_hash,
ExecutionPayload::V2(p) => p.payload_inner.block_hash = new_hash,
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash = new_hash,
}
}
if self.dry_run {
println!("=== Dry Run ===");
println!("Changes that would be applied:");
for change in &changes {
println!(" - {}", change);
}
if changes.is_empty() {
println!(" (no changes)");
}
if skip_recalc {
println!(" - Block hash recalculation: SKIPPED");
} else {
println!(" - Block hash recalculation: PERFORMED");
}
println!("\nResulting payload JSON:");
let json = serde_json::to_string_pretty(&execution_payload)?;
println!("{}", json);
return Ok(());
}
let json_request = if use_v4 {
serde_json::to_string(&(
execution_payload,
blob_versioned_hashes,
parent_beacon_block_root,
requests_hash.unwrap_or_default(),
))?
} else {
serde_json::to_string(&(
execution_payload,
blob_versioned_hashes,
parent_beacon_block_root,
))?
};
match self.mode {
Mode::Execute => {
let mut command = std::process::Command::new("cast");
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
command.arg("rpc").arg(method).arg("--raw");
if let Some(rpc_url) = self.rpc_url {
command.arg("--rpc-url").arg(rpc_url);
}
if let Some(secret) = &jwt_secret {
command.arg("--jwt-secret").arg(secret);
}
let mut process = command.stdin(std::process::Stdio::piped()).spawn()?;
process
.stdin
.take()
.ok_or_eyre("stdin not available")?
.write_all(json_request.as_bytes())?;
process.wait()?;
}
Mode::Cast => {
let mut cmd = format!(
"cast rpc engine_newPayloadV{} --raw '{}'",
self.new_payload_version, json_request
);
if let Some(rpc_url) = self.rpc_url {
cmd += &format!(" --rpc-url {rpc_url}");
}
if let Some(secret) = &jwt_secret {
cmd += &format!(" --jwt-secret {secret}");
}
println!("{cmd}");
}
Mode::Json => {
println!("{json_request}");
}
}
Ok(())
}
}

View File

@@ -25,8 +25,8 @@
//! - `jemalloc-unprefixed`: Uses unprefixed jemalloc symbols.
//! - `tracy-allocator`: Enables [Tracy](https://github.com/wolfpld/tracy) profiler allocator
//! integration for memory profiling.
//! - `snmalloc`: Uses [snmalloc](https://github.com/microsoft/snmalloc) as the global allocator.
//! Use `--no-default-features` when enabling this, as jemalloc takes precedence.
//! - `snmalloc`: Uses [snmalloc](https://github.com/snmalloc/snmalloc) as the global allocator. Use
//! `--no-default-features` when enabling this, as jemalloc takes precedence.
//! - `snmalloc-native`: Uses snmalloc with native CPU optimizations. Use `--no-default-features`
//! when enabling this.
//!

View File

@@ -50,7 +50,6 @@ reth-stages-types = { workspace = true, optional = true }
reth-static-file-types = { workspace = true, features = ["clap"] }
reth-static-file.workspace = true
reth-tasks.workspace = true
reth-storage-api.workspace = true
reth-trie = { workspace = true, features = ["metrics"] }
reth-trie-db = { workspace = true, features = ["metrics"] }
reth-trie-common.workspace = true

View File

@@ -21,7 +21,6 @@ use reth_node_builder::NodeTypesWithDB;
use reth_primitives_traits::ValueWithSubKey;
use reth_provider::{providers::ProviderNodeTypes, ChangeSetReader, StaticFileProviderFactory};
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::StorageChangeSetReader;
use tracing::error;
/// The arguments for the `reth db get` command
@@ -83,33 +82,6 @@ impl Command {
table.view(&GetValueViewer { tool, key, subkey, end_key, end_subkey, raw })?
}
Subcommand::StaticFile { segment, key, subkey, raw } => {
if let StaticFileSegment::StorageChangeSets = segment {
let storage_key =
table_subkey::<tables::StorageChangeSets>(subkey.as_deref()).ok();
let key = table_key::<tables::StorageChangeSets>(&key)?;
let provider = tool.provider_factory.static_file_provider();
if let Some(storage_key) = storage_key {
let entry = provider.get_storage_before_block(
key.block_number(),
key.address(),
storage_key,
)?;
if let Some(entry) = entry {
println!("{}", serde_json::to_string_pretty(&entry)?);
} else {
error!(target: "reth::cli", "No content for the given table key.");
}
return Ok(());
}
let changesets = provider.storage_changeset(key.block_number())?;
println!("{}", serde_json::to_string_pretty(&changesets)?);
return Ok(());
}
let (key, subkey, mask): (u64, _, _) = match segment {
StaticFileSegment::Headers => (
table_key::<tables::Headers>(&key)?,
@@ -140,9 +112,6 @@ impl Command {
AccountChangesetMask::MASK,
)
}
StaticFileSegment::StorageChangeSets => {
unreachable!("storage changesets handled above");
}
};
// handle account changesets differently if a subkey is provided.
@@ -221,9 +190,6 @@ impl Command {
StaticFileSegment::AccountChangeSets => {
unreachable!("account changeset static files are special cased before this match")
}
StaticFileSegment::StorageChangeSets => {
unreachable!("storage changeset static files are special cased before this match")
}
}
}
}

View File

@@ -61,21 +61,19 @@ impl Command {
}
/// Generate [`ListFilter`] from command.
pub fn list_filter(&self) -> eyre::Result<ListFilter> {
let search = match self.search.as_deref() {
Some(search) => {
pub fn list_filter(&self) -> ListFilter {
let search = self
.search
.as_ref()
.map(|search| {
if let Some(search) = search.strip_prefix("0x") {
hex::decode(search).wrap_err(
"Invalid hex content after 0x prefix in --search (expected valid hex like 0xdeadbeef).",
)?
} else {
search.as_bytes().to_vec()
return hex::decode(search).unwrap()
}
}
None => Vec::new(),
};
search.as_bytes().to_vec()
})
.unwrap_or_default();
Ok(ListFilter {
ListFilter {
skip: self.skip,
len: self.len,
search,
@@ -84,7 +82,7 @@ impl Command {
min_value_size: self.min_value_size,
reverse: self.reverse,
only_count: self.count,
})
}
}
}
@@ -117,7 +115,7 @@ impl<N: NodeTypes> TableViewer<()> for ListTableViewer<'_, N> {
}
let list_filter = self.args.list_filter()?;
let list_filter = self.args.list_filter();
if self.args.json || self.args.count {
let (list, count) = self.tool.list::<T>(&list_filter)?;

View File

@@ -162,7 +162,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
let access_rights =
if command.dry_run { AccessRights::RO } else { AccessRights::RW };
db_exec!(self.env, tool, N, access_rights, {
command.execute(&tool, ctx.task_executor, &data_dir)?;
command.execute(&tool, ctx.task_executor.clone(), &data_dir)?;
});
}
Subcommands::StaticFileHeader(command) => {

View File

@@ -69,11 +69,6 @@ pub enum SetCommand {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store storage changesets in static files instead of the database
StorageChangesets {
#[clap(action(ArgAction::Set))]
value: bool,
},
}
impl Command {
@@ -120,7 +115,6 @@ impl Command {
transaction_hash_numbers_in_rocksdb: _,
account_history_in_rocksdb: _,
account_changesets_in_static_files: _,
storage_changesets_in_static_files: _,
} = settings.unwrap_or_else(StorageSettings::legacy);
// Update the setting based on the key
@@ -173,14 +167,6 @@ impl Command {
settings.account_history_in_rocksdb = value;
println!("Set account_history_in_rocksdb = {}", value);
}
SetCommand::StorageChangesets { value } => {
if settings.storage_changesets_in_static_files == value {
println!("storage_changesets_in_static_files is already set to {}", value);
return Ok(());
}
settings.storage_changesets_in_static_files = value;
println!("Set storage_changesets_in_static_files = {}", value);
}
}
// Write updated settings

View File

@@ -1,34 +0,0 @@
//! Enode identifier command
use clap::Parser;
use reth_cli_util::get_secret_key;
use reth_network_peers::NodeRecord;
use std::{
net::{IpAddr, Ipv4Addr, SocketAddr},
path::PathBuf,
};
/// Print the enode identifier for a given secret key.
#[derive(Parser, Debug)]
pub struct Command {
/// Path to the secret key file for discovery.
pub discovery_secret: PathBuf,
/// Optional IP address to include in the enode URL.
///
/// If not provided, defaults to 0.0.0.0.
#[arg(long)]
pub ip: Option<IpAddr>,
}
impl Command {
/// Execute the enode command.
pub fn execute(self) -> eyre::Result<()> {
let sk = get_secret_key(&self.discovery_secret)?;
let ip = self.ip.unwrap_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED));
let addr = SocketAddr::new(ip, 30303);
let enr = NodeRecord::from_secret_key(addr, &sk);
println!("{enr}");
Ok(())
}
}

View File

@@ -18,7 +18,6 @@ use reth_node_core::{
};
pub mod bootnode;
pub mod enode;
pub mod rlpx;
/// `reth p2p` command
@@ -86,9 +85,6 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
Subcommands::Bootnode(command) => {
command.execute().await?;
}
Subcommands::Enode(command) => {
command.execute()?;
}
}
Ok(())
@@ -103,7 +99,6 @@ impl<C: ChainSpecParser> Command<C> {
Subcommands::Body { args, .. } => Some(&args.chain),
Subcommands::Rlpx(_) => None,
Subcommands::Bootnode(_) => None,
Subcommands::Enode(_) => None,
}
}
}
@@ -131,8 +126,6 @@ pub enum Subcommands<C: ChainSpecParser> {
Rlpx(rlpx::Command),
/// Bootnode command
Bootnode(bootnode::Command),
/// Print enode identifier
Enode(enode::Command),
}
#[derive(Debug, Clone, Parser)]
@@ -232,16 +225,4 @@ mod tests {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "body", "--chain", "mainnet", "1000"]);
}
#[test]
fn parse_enode_cmd() {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "enode", "/tmp/secret"]);
}
#[test]
fn parse_enode_cmd_with_ip() {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "enode", "/tmp/secret", "--ip", "192.168.1.1"]);
}
}

View File

@@ -15,8 +15,7 @@ use reth_db_common::{
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
use reth_node_core::args::StageEnum;
use reth_provider::{
DBProvider, RocksDBProviderFactory, StaticFileProviderFactory, StaticFileWriter,
StorageSettingsCache,
DBProvider, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter,
};
use reth_prune::PruneSegment;
use reth_stages::StageId;
@@ -91,14 +90,11 @@ impl<C: ChainSpecParser> Command<C> {
StaticFileSegment::AccountChangeSets => {
writer.prune_account_changesets(highest_block)?;
}
StaticFileSegment::StorageChangeSets => {
writer.prune_storage_changesets(highest_block)?;
}
}
}
}
let provider_rw = tool.provider_factory.unwind_provider_rw()?;
let provider_rw = tool.provider_factory.database_provider_rw()?;
let tx = provider_rw.tx_ref();
match self.stage {
@@ -172,20 +168,8 @@ impl<C: ChainSpecParser> Command<C> {
)?;
}
StageEnum::AccountHistory | StageEnum::StorageHistory => {
let settings = provider_rw.cached_storage_settings();
let rocksdb = tool.provider_factory.rocksdb_provider();
if settings.account_history_in_rocksdb {
rocksdb.clear::<tables::AccountsHistory>()?;
} else {
tx.clear::<tables::AccountsHistory>()?;
}
if settings.storages_history_in_rocksdb {
rocksdb.clear::<tables::StoragesHistory>()?;
} else {
tx.clear::<tables::StoragesHistory>()?;
}
tx.clear::<tables::AccountsHistory>()?;
tx.clear::<tables::StoragesHistory>()?;
reset_stage_checkpoint(tx, StageId::IndexAccountHistory)?;
reset_stage_checkpoint(tx, StageId::IndexStorageHistory)?;
@@ -193,14 +177,7 @@ impl<C: ChainSpecParser> Command<C> {
insert_genesis_history(&provider_rw, self.env.chain.genesis().alloc.iter())?;
}
StageEnum::TxLookup => {
if provider_rw.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
tool.provider_factory
.rocksdb_provider()
.clear::<tables::TransactionHashNumbers>()?;
} else {
tx.clear::<tables::TransactionHashNumbers>()?;
}
tx.clear::<tables::TransactionHashNumbers>()?;
reset_prune_checkpoint(tx, PruneSegment::TransactionLookup)?;
reset_stage_checkpoint(tx, StageId::TransactionLookup)?;

View File

@@ -438,8 +438,6 @@ pub struct BlocksPerFileConfig {
pub transaction_senders: Option<u64>,
/// Number of blocks per file for the account changesets segment.
pub account_change_sets: Option<u64>,
/// Number of blocks per file for the storage changesets segment.
pub storage_change_sets: Option<u64>,
}
impl StaticFilesConfig {
@@ -453,7 +451,6 @@ impl StaticFilesConfig {
receipts,
transaction_senders,
account_change_sets,
storage_change_sets,
} = self.blocks_per_file;
eyre::ensure!(headers != Some(0), "Headers segment blocks per file must be greater than 0");
eyre::ensure!(
@@ -472,10 +469,6 @@ impl StaticFilesConfig {
account_change_sets != Some(0),
"Account changesets segment blocks per file must be greater than 0"
);
eyre::ensure!(
storage_change_sets != Some(0),
"Storage changesets segment blocks per file must be greater than 0"
);
Ok(())
}
@@ -487,7 +480,6 @@ impl StaticFilesConfig {
receipts,
transaction_senders,
account_change_sets,
storage_change_sets,
} = self.blocks_per_file;
let mut map = StaticFileMap::default();
@@ -500,7 +492,6 @@ impl StaticFilesConfig {
StaticFileSegment::Receipts => receipts,
StaticFileSegment::TransactionSenders => transaction_senders,
StaticFileSegment::AccountChangeSets => account_change_sets,
StaticFileSegment::StorageChangeSets => storage_change_sets,
};
if let Some(blocks_per_file) = blocks_per_file {

View File

@@ -34,11 +34,6 @@ fn default_account_worker_count() -> usize {
/// The size of proof targets chunk to spawn in one multiproof calculation.
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE: usize = 60;
/// The size of proof targets chunk to spawn in one multiproof calculation when V2 proofs are
/// enabled. This is 4x the default chunk size to take advantage of more efficient V2 proof
/// computation.
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2: usize = DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE * 4;
/// Default number of reserved CPU cores for non-reth processes.
///
/// This will be deducted from the thread count of main reth global threadpool.
@@ -272,17 +267,6 @@ impl TreeConfig {
self.multiproof_chunk_size
}
/// Return the multiproof task chunk size, using the V2 default if V2 proofs are enabled
/// and the chunk size is at the default value.
pub const fn effective_multiproof_chunk_size(&self) -> usize {
if self.enable_proof_v2 && self.multiproof_chunk_size == DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE
{
DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2
} else {
self.multiproof_chunk_size
}
}
/// Return the number of reserved CPU cores for non-reth processes
pub const fn reserved_cpu_cores(&self) -> usize {
self.reserved_cpu_cores

View File

@@ -62,8 +62,7 @@ pub trait EngineTypes:
+ TryInto<Self::ExecutionPayloadEnvelopeV2>
+ TryInto<Self::ExecutionPayloadEnvelopeV3>
+ TryInto<Self::ExecutionPayloadEnvelopeV4>
+ TryInto<Self::ExecutionPayloadEnvelopeV5>
+ TryInto<Self::ExecutionPayloadEnvelopeV6>,
+ TryInto<Self::ExecutionPayloadEnvelopeV5>,
> + DeserializeOwned
+ Serialize
{
@@ -107,14 +106,6 @@ pub trait EngineTypes:
+ Send
+ Sync
+ 'static;
/// Execution Payload V6 envelope type.
type ExecutionPayloadEnvelopeV6: DeserializeOwned
+ Serialize
+ Clone
+ Unpin
+ Send
+ Sync
+ 'static;
}
/// Type that validates the payloads processed by the engine API.

View File

@@ -32,8 +32,7 @@ use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedBlock, Sealed
use reth_provider::{
BlockExecutionOutput, BlockExecutionResult, BlockNumReader, BlockReader, ChangeSetReader,
DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader,
StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader,
TransactionVariant,
StateProviderBox, StateProviderFactory, StateReader, TransactionVariant,
};
use reth_revm::database::StateProviderDatabase;
use reth_stages_api::ControlFlow;
@@ -85,12 +84,6 @@ pub mod state;
/// backfill this gap.
pub(crate) const MIN_BLOCKS_FOR_PIPELINE_RUN: u64 = EPOCH_SLOTS;
/// The minimum number of blocks to retain in the changeset cache after eviction.
///
/// This ensures that recent trie changesets are kept in memory for potential reorgs,
/// even when the finalized block is not set (e.g., on L2s like Optimism).
const CHANGESET_CACHE_RETENTION_BLOCKS: u64 = 64;
/// A builder for creating state providers that can be used across threads.
#[derive(Clone, Debug)]
pub struct StateProviderBuilder<N: NodePrimitives, P> {
@@ -324,7 +317,6 @@ where
<P as DatabaseProviderFactory>::Provider: BlockReader<Block = N::Block, Header = N::BlockHeader>
+ StageCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader,
C: ConfigureEvm<Primitives = N> + 'static,
T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>,
@@ -1384,27 +1376,19 @@ where
debug!(target: "engine::tree", ?last_persisted_block_hash, ?last_persisted_block_number, elapsed=?start_time.elapsed(), "Finished persisting, calling finish");
self.persistence_state.finish(last_persisted_block_hash, last_persisted_block_number);
// Evict trie changesets for blocks below the eviction threshold.
// Keep at least CHANGESET_CACHE_RETENTION_BLOCKS from the persisted tip, and also respect
// the finalized block if set.
let min_threshold =
last_persisted_block_number.saturating_sub(CHANGESET_CACHE_RETENTION_BLOCKS);
let eviction_threshold =
if let Some(finalized) = self.canonical_in_memory_state.get_finalized_num_hash() {
// Use the minimum of finalized block and retention threshold to be conservative
finalized.number.min(min_threshold)
} else {
// When finalized is not set (e.g., on L2s), use the retention threshold
min_threshold
};
debug!(
target: "engine::tree",
last_persisted = last_persisted_block_number,
finalized_number = ?self.canonical_in_memory_state.get_finalized_num_hash().map(|f| f.number),
eviction_threshold,
"Evicting changesets below threshold"
);
self.changeset_cache.evict(eviction_threshold);
// Evict trie changesets for blocks below the finalized block, but keep at least 64 blocks
if let Some(finalized) = self.canonical_in_memory_state.get_finalized_num_hash() {
let min_threshold = last_persisted_block_number.saturating_sub(64);
let eviction_threshold = finalized.number.min(min_threshold);
debug!(
target: "engine::tree",
last_persisted = last_persisted_block_number,
finalized_number = finalized.number,
eviction_threshold,
"Evicting changesets below threshold"
);
self.changeset_cache.evict(eviction_threshold);
}
self.on_new_persisted_block()?;
Ok(())

View File

@@ -15,7 +15,7 @@ use crate::tree::{
};
use alloy_eip7928::BlockAccessList;
use alloy_eips::eip1898::BlockWithParent;
use alloy_evm::block::StateChangeSource;
use alloy_evm::{block::StateChangeSource, ToTxEnv};
use alloy_primitives::B256;
use crossbeam_channel::Sender as CrossbeamSender;
use executor::WorkloadExecutor;
@@ -25,7 +25,6 @@ use parking_lot::RwLock;
use prewarm::PrewarmMetrics;
use rayon::prelude::*;
use reth_evm::{
block::ExecutableTxParts,
execute::{ExecutableTxFor, WithTxEnv},
ConfigureEvm, EvmEnvFor, ExecutableTxIterator, ExecutableTxTuple, OnStateHook, SpecFor,
TxEnvFor,
@@ -102,7 +101,7 @@ pub const SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY: usize = 1_000_000;
/// Type alias for [`PayloadHandle`] returned by payload processor spawn methods.
type IteratorPayloadHandle<Evm, I, N> = PayloadHandle<
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxIterator<Evm>>::Recovered>,
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxTuple>::Tx>,
<I as ExecutableTxTuple>::Error,
<N as NodePrimitives>::Receipt,
>;
@@ -248,9 +247,6 @@ where
let (to_sparse_trie, sparse_trie_rx) = channel();
let (to_multi_proof, from_multi_proof) = crossbeam_channel::unbounded();
// Extract V2 proofs flag early so we can pass it to prewarm
let v2_proofs_enabled = config.enable_proof_v2();
// Handle BAL-based optimization if available
let prewarm_handle = if let Some(bal) = bal {
// When BAL is present, use BAL prewarming and send BAL to multiproof
@@ -267,7 +263,6 @@ where
provider_builder.clone(),
None, // Don't send proof targets when BAL is present
Some(bal),
v2_proofs_enabled,
)
} else {
// Normal path: spawn with transaction prewarming
@@ -278,7 +273,6 @@ where
provider_builder.clone(),
Some(to_multi_proof.clone()),
None,
v2_proofs_enabled,
)
};
@@ -286,6 +280,7 @@ where
let task_ctx = ProofTaskCtx::new(multiproof_provider_factory);
let storage_worker_count = config.storage_worker_count();
let account_worker_count = config.account_worker_count();
let v2_proofs_enabled = config.enable_proof_v2();
let proof_handle = ProofWorkerHandle::new(
self.executor.handle().clone(),
task_ctx,
@@ -297,13 +292,10 @@ where
let multi_proof_task = MultiProofTask::new(
proof_handle.clone(),
to_sparse_trie,
config
.multiproof_chunking_enabled()
.then_some(config.effective_multiproof_chunk_size()),
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size()),
to_multi_proof.clone(),
from_multi_proof,
)
.with_v2_proofs_enabled(v2_proofs_enabled);
);
// spawn multi-proof task
let parent_span = span.clone();
@@ -352,9 +344,8 @@ where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
{
let (prewarm_rx, execution_rx, size_hint) = self.spawn_tx_iterator(transactions);
// This path doesn't use multiproof, so V2 proofs flag doesn't matter
let prewarm_handle =
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal, false);
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal);
PayloadHandle {
to_multi_proof: None,
prewarm_handle,
@@ -370,8 +361,8 @@ where
&self,
transactions: I,
) -> (
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Recovered>>,
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Recovered>, I::Error>>,
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Tx>>,
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Tx>, I::Error>>,
usize,
) {
let (transactions, convert) = transactions.into();
@@ -386,10 +377,7 @@ where
self.executor.spawn_blocking(move || {
transactions.enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
let tx = convert(tx);
let tx = tx.map(|tx| {
let (tx_env, tx) = tx.into_parts();
WithTxEnv { tx_env, tx: Arc::new(tx) }
});
let tx = tx.map(|tx| WithTxEnv { tx_env: tx.to_tx_env(), tx: Arc::new(tx) });
// Only send Ok(_) variants to prewarming task.
if let Ok(tx) = &tx {
let _ = prewarm_tx.send(tx.clone());
@@ -424,7 +412,6 @@ where
}
/// Spawn prewarming optionally wired to the multiproof task for target updates.
#[expect(clippy::too_many_arguments)]
fn spawn_caching_with<P>(
&self,
env: ExecutionEnv<Evm>,
@@ -433,7 +420,6 @@ where
provider_builder: StateProviderBuilder<N, P>,
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
bal: Option<Arc<BlockAccessList>>,
v2_proofs_enabled: bool,
) -> CacheTaskHandle<N::Receipt>
where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
@@ -456,7 +442,6 @@ where
terminate_execution: Arc::new(AtomicBool::new(false)),
precompile_cache_disabled: self.precompile_cache_disabled,
precompile_cache_map: self.precompile_cache_map.clone(),
v2_proofs_enabled,
};
let (prewarm_task, to_prewarm_task) = PrewarmCacheTask::new(

View File

@@ -11,18 +11,14 @@ use reth_metrics::Metrics;
use reth_provider::AccountReader;
use reth_revm::state::EvmState;
use reth_trie::{
added_removed_keys::MultiAddedRemovedKeys, proof_v2, HashedPostState, HashedStorage,
added_removed_keys::MultiAddedRemovedKeys, DecodedMultiProof, HashedPostState, HashedStorage,
MultiProofTargets,
};
#[cfg(test)]
use reth_trie_parallel::stats::ParallelTrieTracker;
use reth_trie_parallel::{
proof::ParallelProof,
proof_task::{
AccountMultiproofInput, ProofResult, ProofResultContext, ProofResultMessage,
ProofWorkerHandle,
AccountMultiproofInput, ProofResultContext, ProofResultMessage, ProofWorkerHandle,
},
targets_v2::{ChunkedMultiProofTargetsV2, MultiProofTargetsV2},
};
use revm_primitives::map::{hash_map, B256Map};
use std::{collections::BTreeMap, sync::Arc, time::Instant};
@@ -67,12 +63,12 @@ const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300;
/// A trie update that can be applied to sparse trie alongside the proofs for touched parts of the
/// state.
#[derive(Debug)]
#[derive(Default, Debug)]
pub struct SparseTrieUpdate {
/// The state update that was used to calculate the proof
pub(crate) state: HashedPostState,
/// The calculated multiproof
pub(crate) multiproof: ProofResult,
pub(crate) multiproof: DecodedMultiProof,
}
impl SparseTrieUpdate {
@@ -84,11 +80,7 @@ impl SparseTrieUpdate {
/// Construct update from multiproof.
#[cfg(test)]
pub(super) fn from_multiproof(multiproof: reth_trie::MultiProof) -> alloy_rlp::Result<Self> {
let stats = ParallelTrieTracker::default().finish();
Ok(Self {
state: HashedPostState::default(),
multiproof: ProofResult::Legacy(multiproof.try_into()?, stats),
})
Ok(Self { multiproof: multiproof.try_into()?, ..Default::default() })
}
/// Extend update with contents of the other.
@@ -102,7 +94,7 @@ impl SparseTrieUpdate {
#[derive(Debug)]
pub(super) enum MultiProofMessage {
/// Prefetch proof targets
PrefetchProofs(VersionedMultiProofTargets),
PrefetchProofs(MultiProofTargets),
/// New state update from transaction execution with its source
StateUpdate(Source, EvmState),
/// State update that can be applied to the sparse trie without any new proofs.
@@ -231,155 +223,12 @@ pub(crate) fn evm_state_to_hashed_post_state(update: EvmState) -> HashedPostStat
hashed_state
}
/// Extends a `MultiProofTargets` with the contents of a `VersionedMultiProofTargets`,
/// regardless of which variant the latter is.
fn extend_multiproof_targets(dest: &mut MultiProofTargets, src: &VersionedMultiProofTargets) {
match src {
VersionedMultiProofTargets::Legacy(targets) => {
dest.extend_ref(targets);
}
VersionedMultiProofTargets::V2(targets) => {
// Add all account targets
for target in &targets.account_targets {
dest.entry(target.key()).or_default();
}
// Add all storage targets
for (hashed_address, slots) in &targets.storage_targets {
let slot_set = dest.entry(*hashed_address).or_default();
for slot in slots {
slot_set.insert(slot.key());
}
}
}
}
}
/// A set of multiproof targets which can be either in the legacy or V2 representations.
#[derive(Debug)]
pub(super) enum VersionedMultiProofTargets {
/// Legacy targets
Legacy(MultiProofTargets),
/// V2 targets
V2(MultiProofTargetsV2),
}
impl VersionedMultiProofTargets {
/// Returns true if there are no account or storage targets.
fn is_empty(&self) -> bool {
match self {
Self::Legacy(targets) => targets.is_empty(),
Self::V2(targets) => targets.is_empty(),
}
}
/// Returns the number of account targets in the multiproof target
fn account_targets_len(&self) -> usize {
match self {
Self::Legacy(targets) => targets.len(),
Self::V2(targets) => targets.account_targets.len(),
}
}
/// Returns the number of storage targets in the multiproof target
fn storage_targets_len(&self) -> usize {
match self {
Self::Legacy(targets) => targets.values().map(|slots| slots.len()).sum::<usize>(),
Self::V2(targets) => {
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
}
}
}
/// Returns the number of accounts in the multiproof targets.
fn len(&self) -> usize {
match self {
Self::Legacy(targets) => targets.len(),
Self::V2(targets) => targets.account_targets.len(),
}
}
/// Returns the total storage slot count across all accounts.
fn storage_count(&self) -> usize {
match self {
Self::Legacy(targets) => targets.values().map(|slots| slots.len()).sum(),
Self::V2(targets) => targets.storage_targets.values().map(|slots| slots.len()).sum(),
}
}
/// Returns the number of items that will be considered during chunking.
fn chunking_length(&self) -> usize {
match self {
Self::Legacy(targets) => targets.chunking_length(),
Self::V2(targets) => {
// For V2, count accounts + storage slots
targets.account_targets.len() +
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
}
}
}
/// Retains the targets representing the difference with another `MultiProofTargets`.
/// Removes all targets that are already present in `other`.
fn retain_difference(&mut self, other: &MultiProofTargets) {
match self {
Self::Legacy(targets) => {
targets.retain_difference(other);
}
Self::V2(targets) => {
// Remove account targets that exist in other
targets.account_targets.retain(|target| !other.contains_key(&target.key()));
// For each account in storage_targets, remove slots that exist in other
targets.storage_targets.retain(|hashed_address, slots| {
if let Some(other_slots) = other.get(hashed_address) {
slots.retain(|slot| !other_slots.contains(&slot.key()));
!slots.is_empty()
} else {
true
}
});
}
}
}
/// Extends this `VersionedMultiProofTargets` with the contents of another.
///
/// Panics if the variants do not match.
fn extend(&mut self, other: Self) {
match (self, other) {
(Self::Legacy(dest), Self::Legacy(src)) => {
dest.extend(src);
}
(Self::V2(dest), Self::V2(src)) => {
dest.account_targets.extend(src.account_targets);
for (addr, slots) in src.storage_targets {
dest.storage_targets.entry(addr).or_default().extend(slots);
}
}
_ => panic!("Cannot extend VersionedMultiProofTargets with mismatched variants"),
}
}
/// Chunks this `VersionedMultiProofTargets` into smaller chunks of the given size.
fn chunks(self, chunk_size: usize) -> Box<dyn Iterator<Item = Self>> {
match self {
Self::Legacy(targets) => {
Box::new(MultiProofTargets::chunks(targets, chunk_size).map(Self::Legacy))
}
Self::V2(targets) => {
Box::new(ChunkedMultiProofTargetsV2::new(targets, chunk_size).map(Self::V2))
}
}
}
}
/// Input parameters for dispatching a multiproof calculation.
#[derive(Debug)]
struct MultiproofInput {
source: Option<Source>,
hashed_state_update: HashedPostState,
proof_targets: VersionedMultiProofTargets,
proof_targets: MultiProofTargets,
proof_sequence_number: u64,
state_root_message_sender: CrossbeamSender<MultiProofMessage>,
multi_added_removed_keys: Option<Arc<MultiAddedRemovedKeys>>,
@@ -414,6 +263,8 @@ pub struct MultiproofManager {
proof_result_tx: CrossbeamSender<ProofResultMessage>,
/// Metrics
metrics: MultiProofTaskMetrics,
/// Whether to use V2 storage proofs
v2_proofs_enabled: bool,
}
impl MultiproofManager {
@@ -427,7 +278,9 @@ impl MultiproofManager {
metrics.max_storage_workers.set(proof_worker_handle.total_storage_workers() as f64);
metrics.max_account_workers.set(proof_worker_handle.total_account_workers() as f64);
Self { metrics, proof_worker_handle, proof_result_tx }
let v2_proofs_enabled = proof_worker_handle.v2_proofs_enabled();
Self { metrics, proof_worker_handle, proof_result_tx, v2_proofs_enabled }
}
/// Dispatches a new multiproof calculation to worker pools.
@@ -472,48 +325,41 @@ impl MultiproofManager {
multi_added_removed_keys,
} = multiproof_input;
let account_targets = proof_targets.len();
let storage_targets = proof_targets.values().map(|slots| slots.len()).sum::<usize>();
trace!(
target: "engine::tree::payload_processor::multiproof",
proof_sequence_number,
?proof_targets,
account_targets = proof_targets.account_targets_len(),
storage_targets = proof_targets.storage_targets_len(),
account_targets,
storage_targets,
?source,
"Dispatching multiproof to workers"
);
let start = Instant::now();
// Workers will send ProofResultMessage directly to proof_result_rx
let proof_result_sender = ProofResultContext::new(
self.proof_result_tx.clone(),
proof_sequence_number,
hashed_state_update,
start,
);
let input = match proof_targets {
VersionedMultiProofTargets::Legacy(proof_targets) => {
// Extend prefix sets with targets
let frozen_prefix_sets = ParallelProof::extend_prefix_sets_with_targets(
&Default::default(),
&proof_targets,
);
AccountMultiproofInput::Legacy {
targets: proof_targets,
prefix_sets: frozen_prefix_sets,
collect_branch_node_masks: true,
multi_added_removed_keys,
proof_result_sender,
}
}
VersionedMultiProofTargets::V2(proof_targets) => {
AccountMultiproofInput::V2 { targets: proof_targets, proof_result_sender }
}
};
// Extend prefix sets with targets
let frozen_prefix_sets =
ParallelProof::extend_prefix_sets_with_targets(&Default::default(), &proof_targets);
// Dispatch account multiproof to worker pool with result sender
let input = AccountMultiproofInput {
targets: proof_targets,
prefix_sets: frozen_prefix_sets,
collect_branch_node_masks: true,
multi_added_removed_keys,
// Workers will send ProofResultMessage directly to proof_result_rx
proof_result_sender: ProofResultContext::new(
self.proof_result_tx.clone(),
proof_sequence_number,
hashed_state_update,
start,
),
v2_proofs_enabled: self.v2_proofs_enabled,
};
if let Err(e) = self.proof_worker_handle.dispatch_account_multiproof(input) {
error!(target: "engine::tree::payload_processor::multiproof", ?e, "Failed to dispatch account multiproof");
return;
@@ -715,9 +561,6 @@ pub(super) struct MultiProofTask {
/// there are any active workers and force chunking across workers. This is to prevent tasks
/// which are very long from hitting a single worker.
max_targets_for_chunking: usize,
/// Whether or not V2 proof calculation is enabled. If enabled then [`MultiProofTargetsV2`]
/// will be produced by state updates.
v2_proofs_enabled: bool,
}
impl MultiProofTask {
@@ -749,16 +592,9 @@ impl MultiProofTask {
),
metrics,
max_targets_for_chunking: DEFAULT_MAX_TARGETS_FOR_CHUNKING,
v2_proofs_enabled: false,
}
}
/// Enables V2 proof target generation on state updates.
pub(super) const fn with_v2_proofs_enabled(mut self, v2_proofs_enabled: bool) -> Self {
self.v2_proofs_enabled = v2_proofs_enabled;
self
}
/// Handles request for proof prefetch.
///
/// Returns how many multiproof tasks were dispatched for the prefetch request.
@@ -766,29 +602,37 @@ impl MultiProofTask {
level = "debug",
target = "engine::tree::payload_processor::multiproof",
skip_all,
fields(accounts = targets.account_targets_len(), chunks = 0)
fields(accounts = targets.len(), chunks = 0)
)]
fn on_prefetch_proof(&mut self, mut targets: VersionedMultiProofTargets) -> u64 {
fn on_prefetch_proof(&mut self, mut targets: MultiProofTargets) -> u64 {
// Remove already fetched proof targets to avoid redundant work.
targets.retain_difference(&self.fetched_proof_targets);
extend_multiproof_targets(&mut self.fetched_proof_targets, &targets);
self.fetched_proof_targets.extend_ref(&targets);
// For Legacy multiproofs, make sure all target accounts have an `AddedRemovedKeySet` in the
// Make sure all target accounts have an `AddedRemovedKeySet` in the
// [`MultiAddedRemovedKeys`]. Even if there are not any known removed keys for the account,
// we still want to optimistically fetch extension children for the leaf addition case.
// V2 multiproofs don't need this.
let multi_added_removed_keys =
if let VersionedMultiProofTargets::Legacy(legacy_targets) = &targets {
self.multi_added_removed_keys.touch_accounts(legacy_targets.keys().copied());
Some(Arc::new(self.multi_added_removed_keys.clone()))
} else {
None
};
self.multi_added_removed_keys.touch_accounts(targets.keys().copied());
// Clone+Arc MultiAddedRemovedKeys for sharing with the dispatched multiproof tasks
let multi_added_removed_keys = Arc::new(MultiAddedRemovedKeys {
account: self.multi_added_removed_keys.account.clone(),
storages: targets
.keys()
.filter_map(|account| {
self.multi_added_removed_keys
.storages
.get(account)
.cloned()
.map(|keys| (*account, keys))
})
.collect(),
});
self.metrics.prefetch_proof_targets_accounts_histogram.record(targets.len() as f64);
self.metrics
.prefetch_proof_targets_storages_histogram
.record(targets.storage_count() as f64);
.record(targets.values().map(|slots| slots.len()).sum::<usize>() as f64);
let chunking_len = targets.chunking_length();
let available_account_workers =
@@ -802,7 +646,7 @@ impl MultiProofTask {
self.max_targets_for_chunking,
available_account_workers,
available_storage_workers,
VersionedMultiProofTargets::chunks,
MultiProofTargets::chunks,
|proof_targets| {
self.multiproof_manager.dispatch(MultiproofInput {
source: None,
@@ -810,7 +654,7 @@ impl MultiProofTask {
proof_targets,
proof_sequence_number: self.proof_sequencer.next_sequence(),
state_root_message_sender: self.tx.clone(),
multi_added_removed_keys: multi_added_removed_keys.clone(),
multi_added_removed_keys: Some(multi_added_removed_keys.clone()),
});
},
);
@@ -913,7 +757,6 @@ impl MultiProofTask {
self.multiproof_manager.proof_worker_handle.available_account_workers();
let available_storage_workers =
self.multiproof_manager.proof_worker_handle.available_storage_workers();
let num_chunks = dispatch_with_chunking(
not_fetched_state_update,
chunking_len,
@@ -927,9 +770,8 @@ impl MultiProofTask {
&hashed_state_update,
&self.fetched_proof_targets,
&multi_added_removed_keys,
self.v2_proofs_enabled,
);
extend_multiproof_targets(&mut spawned_proof_targets, &proof_targets);
spawned_proof_targets.extend_ref(&proof_targets);
self.multiproof_manager.dispatch(MultiproofInput {
source: Some(source),
@@ -1029,10 +871,7 @@ impl MultiProofTask {
batch_metrics.proofs_processed += 1;
if let Some(combined_update) = self.on_proof(
sequence_number,
SparseTrieUpdate {
state,
multiproof: ProofResult::empty(self.v2_proofs_enabled),
},
SparseTrieUpdate { state, multiproof: Default::default() },
) {
let _ = self.to_sparse_trie.send(combined_update);
}
@@ -1059,7 +898,8 @@ impl MultiProofTask {
}
let account_targets = merged_targets.len();
let storage_targets = merged_targets.storage_count();
let storage_targets =
merged_targets.values().map(|slots| slots.len()).sum::<usize>();
batch_metrics.prefetch_proofs_requested += self.on_prefetch_proof(merged_targets);
trace!(
target: "engine::tree::payload_processor::multiproof",
@@ -1163,10 +1003,7 @@ impl MultiProofTask {
if let Some(combined_update) = self.on_proof(
sequence_number,
SparseTrieUpdate {
state,
multiproof: ProofResult::empty(self.v2_proofs_enabled),
},
SparseTrieUpdate { state, multiproof: Default::default() },
) {
let _ = self.to_sparse_trie.send(combined_update);
}
@@ -1269,7 +1106,7 @@ impl MultiProofTask {
let update = SparseTrieUpdate {
state: proof_result.state,
multiproof: proof_result_data,
multiproof: proof_result_data.proof,
};
if let Some(combined_update) =
@@ -1359,7 +1196,7 @@ struct MultiproofBatchCtx {
/// received.
updates_finished_time: Option<Instant>,
/// Reusable buffer for accumulating prefetch targets during batching.
accumulated_prefetch_targets: Vec<VersionedMultiProofTargets>,
accumulated_prefetch_targets: Vec<MultiProofTargets>,
}
impl MultiproofBatchCtx {
@@ -1405,77 +1242,40 @@ fn get_proof_targets(
state_update: &HashedPostState,
fetched_proof_targets: &MultiProofTargets,
multi_added_removed_keys: &MultiAddedRemovedKeys,
v2_enabled: bool,
) -> VersionedMultiProofTargets {
if v2_enabled {
let mut targets = MultiProofTargetsV2::default();
) -> MultiProofTargets {
let mut targets = MultiProofTargets::default();
// first collect all new accounts (not previously fetched)
for &hashed_address in state_update.accounts.keys() {
if !fetched_proof_targets.contains_key(&hashed_address) {
targets.account_targets.push(hashed_address.into());
}
// first collect all new accounts (not previously fetched)
for hashed_address in state_update.accounts.keys() {
if !fetched_proof_targets.contains_key(hashed_address) {
targets.insert(*hashed_address, HashSet::default());
}
// then process storage slots for all accounts in the state update
for (hashed_address, storage) in &state_update.storages {
let fetched = fetched_proof_targets.get(hashed_address);
// If the storage is wiped, we still need to fetch the account proof.
if storage.wiped && fetched.is_none() {
targets.account_targets.push(Into::<proof_v2::Target>::into(*hashed_address));
continue
}
let changed_slots = storage
.storage
.keys()
.filter(|slot| !fetched.is_some_and(|f| f.contains(*slot)))
.map(|slot| Into::<proof_v2::Target>::into(*slot))
.collect::<Vec<_>>();
if !changed_slots.is_empty() {
targets.account_targets.push((*hashed_address).into());
targets.storage_targets.insert(*hashed_address, changed_slots);
}
}
VersionedMultiProofTargets::V2(targets)
} else {
let mut targets = MultiProofTargets::default();
// first collect all new accounts (not previously fetched)
for hashed_address in state_update.accounts.keys() {
if !fetched_proof_targets.contains_key(hashed_address) {
targets.insert(*hashed_address, HashSet::default());
}
}
// then process storage slots for all accounts in the state update
for (hashed_address, storage) in &state_update.storages {
let fetched = fetched_proof_targets.get(hashed_address);
let storage_added_removed_keys = multi_added_removed_keys.get_storage(hashed_address);
let mut changed_slots = storage
.storage
.keys()
.filter(|slot| {
!fetched.is_some_and(|f| f.contains(*slot)) ||
storage_added_removed_keys.is_some_and(|k| k.is_removed(slot))
})
.peekable();
// If the storage is wiped, we still need to fetch the account proof.
if storage.wiped && fetched.is_none() {
targets.entry(*hashed_address).or_default();
}
if changed_slots.peek().is_some() {
targets.entry(*hashed_address).or_default().extend(changed_slots);
}
}
VersionedMultiProofTargets::Legacy(targets)
}
// then process storage slots for all accounts in the state update
for (hashed_address, storage) in &state_update.storages {
let fetched = fetched_proof_targets.get(hashed_address);
let storage_added_removed_keys = multi_added_removed_keys.get_storage(hashed_address);
let mut changed_slots = storage
.storage
.keys()
.filter(|slot| {
!fetched.is_some_and(|f| f.contains(*slot)) ||
storage_added_removed_keys.is_some_and(|k| k.is_removed(slot))
})
.peekable();
// If the storage is wiped, we still need to fetch the account proof.
if storage.wiped && fetched.is_none() {
targets.entry(*hashed_address).or_default();
}
if changed_slots.peek().is_some() {
targets.entry(*hashed_address).or_default().extend(changed_slots);
}
}
targets
}
/// Dispatches work items as a single unit or in chunks based on target size and worker
@@ -1523,7 +1323,7 @@ mod tests {
use reth_provider::{
providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory,
BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, LatestStateProvider,
PruneCheckpointReader, StageCheckpointReader, StateProviderBox, StorageChangeSetReader,
PruneCheckpointReader, StageCheckpointReader, StateProviderBox,
};
use reth_trie::MultiProof;
use reth_trie_db::ChangesetCache;
@@ -1550,7 +1350,6 @@ mod tests {
+ StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader,
> + Clone
+ Send
@@ -1682,24 +1481,12 @@ mod tests {
state
}
fn unwrap_legacy_targets(targets: VersionedMultiProofTargets) -> MultiProofTargets {
match targets {
VersionedMultiProofTargets::Legacy(targets) => targets,
VersionedMultiProofTargets::V2(_) => panic!("Expected Legacy targets"),
}
}
#[test]
fn test_get_proof_targets_new_account_targets() {
let state = create_get_proof_targets_state();
let fetched = MultiProofTargets::default();
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
// should return all accounts as targets since nothing was fetched before
assert_eq!(targets.len(), state.accounts.len());
@@ -1713,12 +1500,7 @@ mod tests {
let state = create_get_proof_targets_state();
let fetched = MultiProofTargets::default();
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
// verify storage slots are included for accounts with storage
for (addr, storage) in &state.storages {
@@ -1746,12 +1528,7 @@ mod tests {
// mark the account as already fetched
fetched.insert(*fetched_addr, HashSet::default());
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
// should not include the already fetched account since it has no storage updates
assert!(!targets.contains_key(fetched_addr));
@@ -1771,12 +1548,7 @@ mod tests {
fetched_slots.insert(fetched_slot);
fetched.insert(*addr, fetched_slots);
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
// should not include the already fetched storage slot
let target_slots = &targets[addr];
@@ -1789,12 +1561,7 @@ mod tests {
let state = HashedPostState::default();
let fetched = MultiProofTargets::default();
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
assert!(targets.is_empty());
}
@@ -1821,12 +1588,7 @@ mod tests {
fetched_slots.insert(slot1);
fetched.insert(addr1, fetched_slots);
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
assert!(targets.contains_key(&addr2));
assert!(!targets[&addr1].contains(&slot1));
@@ -1852,12 +1614,7 @@ mod tests {
assert!(!state.accounts.contains_key(&addr));
assert!(!fetched.contains_key(&addr));
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
// verify that we still get the storage slots for the unmodified account
assert!(targets.contains_key(&addr));
@@ -1899,12 +1656,7 @@ mod tests {
removed_state.storages.insert(addr, removed_storage);
multi_added_removed_keys.update_with_state(&removed_state);
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&multi_added_removed_keys,
false,
));
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
// slot1 should be included despite being fetched, because it's marked as removed
assert!(targets.contains_key(&addr));
@@ -1931,12 +1683,7 @@ mod tests {
storage.storage.insert(slot1, U256::from(100));
state.storages.insert(addr, storage);
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&multi_added_removed_keys,
false,
));
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
// account should be included because storage is wiped and account wasn't fetched
assert!(targets.contains_key(&addr));
@@ -1979,12 +1726,7 @@ mod tests {
removed_state.storages.insert(addr, removed_storage);
multi_added_removed_keys.update_with_state(&removed_state);
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&multi_added_removed_keys,
false,
));
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
// only slots in the state update can be included, so slot3 should not appear
assert!(!targets.contains_key(&addr));
@@ -2011,12 +1753,9 @@ mod tests {
targets3.insert(addr3, HashSet::default());
let tx = task.tx.clone();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets1)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets2)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets3)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets1)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets2)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets3)).unwrap();
let proofs_requested =
if let Ok(MultiProofMessage::PrefetchProofs(targets)) = task.rx.recv() {
@@ -2030,12 +1769,11 @@ mod tests {
assert_eq!(num_batched, 3);
assert_eq!(merged_targets.len(), 3);
let legacy_targets = unwrap_legacy_targets(merged_targets);
assert!(legacy_targets.contains_key(&addr1));
assert!(legacy_targets.contains_key(&addr2));
assert!(legacy_targets.contains_key(&addr3));
assert!(merged_targets.contains_key(&addr1));
assert!(merged_targets.contains_key(&addr2));
assert!(merged_targets.contains_key(&addr3));
task.on_prefetch_proof(VersionedMultiProofTargets::Legacy(legacy_targets))
task.on_prefetch_proof(merged_targets)
} else {
panic!("Expected PrefetchProofs message");
};
@@ -2110,16 +1848,11 @@ mod tests {
// Queue: [PrefetchProofs1, PrefetchProofs2, StateUpdate1, StateUpdate2, PrefetchProofs3]
let tx = task.tx.clone();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets1)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets2)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets1)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets2)).unwrap();
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update1)).unwrap();
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update2)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(
targets3.clone(),
)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets3.clone())).unwrap();
// Step 1: Receive and batch PrefetchProofs (should get targets1 + targets2)
let mut pending_msg: Option<MultiProofMessage> = None;
@@ -2145,10 +1878,9 @@ mod tests {
// Should have batched exactly 2 PrefetchProofs (not 3!)
assert_eq!(num_batched, 2, "Should batch only until different message type");
assert_eq!(merged_targets.len(), 2);
let legacy_targets = unwrap_legacy_targets(merged_targets);
assert!(legacy_targets.contains_key(&addr1));
assert!(legacy_targets.contains_key(&addr2));
assert!(!legacy_targets.contains_key(&addr3), "addr3 should NOT be in first batch");
assert!(merged_targets.contains_key(&addr1));
assert!(merged_targets.contains_key(&addr2));
assert!(!merged_targets.contains_key(&addr3), "addr3 should NOT be in first batch");
} else {
panic!("Expected PrefetchProofs message");
}
@@ -2173,8 +1905,7 @@ mod tests {
match task.rx.try_recv() {
Ok(MultiProofMessage::PrefetchProofs(targets)) => {
assert_eq!(targets.len(), 1);
let legacy_targets = unwrap_legacy_targets(targets);
assert!(legacy_targets.contains_key(&addr3));
assert!(targets.contains_key(&addr3));
}
_ => panic!("PrefetchProofs3 was lost!"),
}
@@ -2220,13 +1951,9 @@ mod tests {
let source = StateChangeSource::Transaction(99);
let tx = task.tx.clone();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(prefetch1)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(prefetch1)).unwrap();
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(
prefetch2.clone(),
)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(prefetch2.clone())).unwrap();
let mut ctx = MultiproofBatchCtx::new(Instant::now());
let mut batch_metrics = MultiproofBatchMetrics::default();
@@ -2259,8 +1986,7 @@ mod tests {
match task.rx.try_recv() {
Ok(MultiProofMessage::PrefetchProofs(targets)) => {
assert_eq!(targets.len(), 1);
let legacy_targets = unwrap_legacy_targets(targets);
assert!(legacy_targets.contains_key(&prefetch_addr2));
assert!(targets.contains_key(&prefetch_addr2));
}
other => panic!("Expected PrefetchProofs2 in channel, got {:?}", other),
}

View File

@@ -16,7 +16,7 @@ use crate::tree::{
payload_processor::{
bal::{total_slots, BALSlotIter},
executor::WorkloadExecutor,
multiproof::{MultiProofMessage, VersionedMultiProofTargets},
multiproof::MultiProofMessage,
ExecutionCache as PayloadExecutionCache,
},
precompile_cache::{CachedPrecompile, PrecompileCacheMap},
@@ -29,7 +29,7 @@ use alloy_evm::Database;
use alloy_primitives::{keccak256, map::B256Set, B256};
use crossbeam_channel::Sender as CrossbeamSender;
use metrics::{Counter, Gauge, Histogram};
use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, RecoveredTx, SpecFor};
use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, SpecFor};
use reth_metrics::Metrics;
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
@@ -237,7 +237,7 @@ where
}
/// If configured and the tx returned proof targets, emit the targets the transaction produced
fn send_multi_proof_targets(&self, targets: Option<VersionedMultiProofTargets>) {
fn send_multi_proof_targets(&self, targets: Option<MultiProofTargets>) {
if self.is_execution_terminated() {
// if execution is already terminated then we dont need to send more proof fetch
// messages
@@ -484,8 +484,6 @@ where
pub(super) terminate_execution: Arc<AtomicBool>,
pub(super) precompile_cache_disabled: bool,
pub(super) precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
/// Whether V2 proof calculation is enabled.
pub(super) v2_proofs_enabled: bool,
}
impl<N, P, Evm> PrewarmContext<N, P, Evm>
@@ -494,12 +492,10 @@ where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
Evm: ConfigureEvm<Primitives = N> + 'static,
{
/// Splits this context into an evm, an evm config, metrics, the atomic bool for terminating
/// execution, and whether V2 proofs are enabled.
/// Splits this context into an evm, an evm config, metrics, and the atomic bool for terminating
/// execution.
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
fn evm_for_ctx(
self,
) -> Option<(EvmFor<Evm, impl Database>, PrewarmMetrics, Arc<AtomicBool>, bool)> {
fn evm_for_ctx(self) -> Option<(EvmFor<Evm, impl Database>, PrewarmMetrics, Arc<AtomicBool>)> {
let Self {
env,
evm_config,
@@ -509,7 +505,6 @@ where
terminate_execution,
precompile_cache_disabled,
precompile_cache_map,
v2_proofs_enabled,
} = self;
let mut state_provider = match provider.build() {
@@ -559,7 +554,7 @@ where
});
}
Some((evm, metrics, terminate_execution, v2_proofs_enabled))
Some((evm, metrics, terminate_execution))
}
/// Accepts an [`mpsc::Receiver`] of transactions and a handle to prewarm task. Executes
@@ -580,10 +575,7 @@ where
) where
Tx: ExecutableTxFor<Evm>,
{
let Some((mut evm, metrics, terminate_execution, v2_proofs_enabled)) = self.evm_for_ctx()
else {
return
};
let Some((mut evm, metrics, terminate_execution)) = self.evm_for_ctx() else { return };
while let Ok(IndexedTransaction { index, tx }) = {
let _enter = debug_span!(target: "engine::tree::payload_processor::prewarm", "recv tx")
@@ -609,8 +601,7 @@ where
break
}
let (tx_env, tx) = tx.into_parts();
let res = match evm.transact(tx_env) {
let res = match evm.transact(&tx) {
Ok(res) => res,
Err(err) => {
trace!(
@@ -647,8 +638,7 @@ where
let _enter =
debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm outcome", index, tx_hash=%tx.tx().tx_hash())
.entered();
let (targets, storage_targets) =
multiproof_targets_from_state(res.state, v2_proofs_enabled);
let (targets, storage_targets) = multiproof_targets_from_state(res.state);
metrics.prefetch_storage_targets.record(storage_targets as f64);
let _ = sender.send(PrewarmTaskEvent::Outcome { proof_targets: Some(targets) });
drop(_enter);
@@ -793,22 +783,9 @@ where
}
}
/// Returns a set of [`VersionedMultiProofTargets`] and the total amount of storage targets, based
/// on the given state.
fn multiproof_targets_from_state(
state: EvmState,
v2_enabled: bool,
) -> (VersionedMultiProofTargets, usize) {
if v2_enabled {
multiproof_targets_v2_from_state(state)
} else {
multiproof_targets_legacy_from_state(state)
}
}
/// Returns legacy [`MultiProofTargets`] and the total amount of storage targets, based on the
/// Returns a set of [`MultiProofTargets`] and the total amount of storage targets, based on the
/// given state.
fn multiproof_targets_legacy_from_state(state: EvmState) -> (VersionedMultiProofTargets, usize) {
fn multiproof_targets_from_state(state: EvmState) -> (MultiProofTargets, usize) {
let mut targets = MultiProofTargets::with_capacity(state.len());
let mut storage_targets = 0;
for (addr, account) in state {
@@ -838,50 +815,7 @@ fn multiproof_targets_legacy_from_state(state: EvmState) -> (VersionedMultiProof
targets.insert(keccak256(addr), storage_set);
}
(VersionedMultiProofTargets::Legacy(targets), storage_targets)
}
/// Returns V2 [`reth_trie_parallel::targets_v2::MultiProofTargetsV2`] and the total amount of
/// storage targets, based on the given state.
fn multiproof_targets_v2_from_state(state: EvmState) -> (VersionedMultiProofTargets, usize) {
use reth_trie::proof_v2;
use reth_trie_parallel::targets_v2::MultiProofTargetsV2;
let mut targets = MultiProofTargetsV2::default();
let mut storage_target_count = 0;
for (addr, account) in state {
// if the account was not touched, or if the account was selfdestructed, do not
// fetch proofs for it
//
// Since selfdestruct can only happen in the same transaction, we can skip
// prefetching proofs for selfdestructed accounts
//
// See: https://eips.ethereum.org/EIPS/eip-6780
if !account.is_touched() || account.is_selfdestructed() {
continue
}
let hashed_address = keccak256(addr);
targets.account_targets.push(hashed_address.into());
let mut storage_slots = Vec::with_capacity(account.storage.len());
for (key, slot) in account.storage {
// do nothing if unchanged
if !slot.is_changed() {
continue
}
let hashed_slot = keccak256(B256::new(key.to_be_bytes()));
storage_slots.push(proof_v2::Target::from(hashed_slot));
}
storage_target_count += storage_slots.len();
if !storage_slots.is_empty() {
targets.storage_targets.insert(hashed_address, storage_slots);
}
}
(VersionedMultiProofTargets::V2(targets), storage_target_count)
(targets, storage_targets)
}
/// The events the pre-warm task can handle.
@@ -906,7 +840,7 @@ pub(super) enum PrewarmTaskEvent<R> {
/// The outcome of a pre-warm task
Outcome {
/// The prepared proof targets based on the evm state outcome
proof_targets: Option<VersionedMultiProofTargets>,
proof_targets: Option<MultiProofTargets>,
},
/// Finished executing all transactions
FinishedTxExecution {

View File

@@ -4,7 +4,7 @@ use crate::tree::payload_processor::multiproof::{MultiProofTaskMetrics, SparseTr
use alloy_primitives::B256;
use rayon::iter::{ParallelBridge, ParallelIterator};
use reth_trie::{updates::TrieUpdates, Nibbles};
use reth_trie_parallel::{proof_task::ProofResult, root::ParallelStateRootError};
use reth_trie_parallel::root::ParallelStateRootError;
use reth_trie_sparse::{
errors::{SparseStateTrieResult, SparseTrieErrorKind},
provider::{TrieNodeProvider, TrieNodeProviderFactory},
@@ -97,8 +97,8 @@ where
debug!(
target: "engine::root",
num_updates,
account_proofs = update.multiproof.account_proofs_len(),
storage_proofs = update.multiproof.storage_proofs_len(),
account_proofs = update.multiproof.account_subtree.len(),
storage_proofs = update.multiproof.storages.len(),
"Updating sparse trie"
);
@@ -157,14 +157,7 @@ where
let started_at = Instant::now();
// Reveal new accounts and storage slots.
match multiproof {
ProofResult::Legacy(decoded, _) => {
trie.reveal_decoded_multiproof(decoded)?;
}
ProofResult::V2(decoded_v2) => {
trie.reveal_decoded_multiproof_v2(decoded_v2)?;
}
}
trie.reveal_decoded_multiproof(multiproof)?;
let reveal_multiproof_elapsed = started_at.elapsed();
trace!(
target: "engine::root::sparse",

View File

@@ -39,7 +39,7 @@ use reth_provider::{
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader,
ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider,
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
StateProviderFactory, StateReader, StorageChangeSetReader,
StateProviderFactory, StateReader,
};
use reth_revm::db::{states::bundle_state::BundleRetention, State};
use reth_trie::{updates::TrieUpdates, HashedPostState, StateRoot};
@@ -144,7 +144,6 @@ where
+ StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader,
> + BlockReader<Header = N::BlockHeader>
+ ChangeSetReader
@@ -1337,7 +1336,6 @@ where
+ StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader,
> + BlockReader<Header = N::BlockHeader>
+ StateProviderFactory

View File

@@ -17,11 +17,10 @@ pub use payload::{payload_id, BlobSidecars, EthBuiltPayload, EthPayloadBuilderAt
mod error;
pub use error::*;
use alloy_rpc_types_engine::{ExecutionData, ExecutionPayload};
use alloy_rpc_types_engine::{ExecutionData, ExecutionPayload, ExecutionPayloadEnvelopeV5};
pub use alloy_rpc_types_engine::{
ExecutionPayloadEnvelopeV2, ExecutionPayloadEnvelopeV3, ExecutionPayloadEnvelopeV4,
ExecutionPayloadEnvelopeV5, ExecutionPayloadEnvelopeV6, ExecutionPayloadV1,
PayloadAttributes as EthPayloadAttributes,
ExecutionPayloadV1, PayloadAttributes as EthPayloadAttributes,
};
use reth_engine_primitives::EngineTypes;
use reth_payload_primitives::{BuiltPayload, PayloadTypes};
@@ -67,15 +66,13 @@ where
+ TryInto<ExecutionPayloadEnvelopeV2>
+ TryInto<ExecutionPayloadEnvelopeV3>
+ TryInto<ExecutionPayloadEnvelopeV4>
+ TryInto<ExecutionPayloadEnvelopeV5>
+ TryInto<ExecutionPayloadEnvelopeV6>,
+ TryInto<ExecutionPayloadEnvelopeV5>,
{
type ExecutionPayloadEnvelopeV1 = ExecutionPayloadV1;
type ExecutionPayloadEnvelopeV2 = ExecutionPayloadEnvelopeV2;
type ExecutionPayloadEnvelopeV3 = ExecutionPayloadEnvelopeV3;
type ExecutionPayloadEnvelopeV4 = ExecutionPayloadEnvelopeV4;
type ExecutionPayloadEnvelopeV5 = ExecutionPayloadEnvelopeV5;
type ExecutionPayloadEnvelopeV6 = ExecutionPayloadEnvelopeV6;
}
/// A default payload type for [`EthEngineTypes`]

View File

@@ -11,8 +11,8 @@ use alloy_primitives::{Address, B256, U256};
use alloy_rlp::Encodable;
use alloy_rpc_types_engine::{
BlobsBundleV1, BlobsBundleV2, ExecutionPayloadEnvelopeV2, ExecutionPayloadEnvelopeV3,
ExecutionPayloadEnvelopeV4, ExecutionPayloadEnvelopeV5, ExecutionPayloadEnvelopeV6,
ExecutionPayloadFieldV2, ExecutionPayloadV1, ExecutionPayloadV3, PayloadAttributes, PayloadId,
ExecutionPayloadEnvelopeV4, ExecutionPayloadEnvelopeV5, ExecutionPayloadFieldV2,
ExecutionPayloadV1, ExecutionPayloadV3, PayloadAttributes, PayloadId,
};
use core::convert::Infallible;
use reth_ethereum_primitives::EthPrimitives;
@@ -160,13 +160,6 @@ impl EthBuiltPayload {
execution_requests: requests.unwrap_or_default(),
})
}
/// Try converting built payload into [`ExecutionPayloadEnvelopeV6`].
///
/// Note: Amsterdam fork is not yet implemented, so this conversion is not yet supported.
pub fn try_into_v6(self) -> Result<ExecutionPayloadEnvelopeV6, BuiltPayloadConversionError> {
unimplemented!("ExecutionPayloadEnvelopeV6 not yet supported")
}
}
impl<N: NodePrimitives> BuiltPayload for EthBuiltPayload<N> {
@@ -234,14 +227,6 @@ impl TryFrom<EthBuiltPayload> for ExecutionPayloadEnvelopeV5 {
}
}
impl TryFrom<EthBuiltPayload> for ExecutionPayloadEnvelopeV6 {
type Error = BuiltPayloadConversionError;
fn try_from(value: EthBuiltPayload) -> Result<Self, Self::Error> {
value.try_into_v6()
}
}
/// An enum representing blob transaction sidecars belonging to [`EthBuiltPayload`].
#[derive(Clone, Default, Debug)]
pub enum BlobSidecars {

View File

@@ -1,4 +1,3 @@
use alloy_consensus::TxType;
use alloy_evm::eth::receipt_builder::{ReceiptBuilder, ReceiptBuilderCtx};
use reth_ethereum_primitives::{Receipt, TransactionSigned};
use reth_evm::Evm;
@@ -13,10 +12,13 @@ impl ReceiptBuilder for RethReceiptBuilder {
type Transaction = TransactionSigned;
type Receipt = Receipt;
fn build_receipt<E: Evm>(&self, ctx: ReceiptBuilderCtx<'_, TxType, E>) -> Self::Receipt {
let ReceiptBuilderCtx { tx_type, result, cumulative_gas_used, .. } = ctx;
fn build_receipt<E: Evm>(
&self,
ctx: ReceiptBuilderCtx<'_, Self::Transaction, E>,
) -> Self::Receipt {
let ReceiptBuilderCtx { tx, result, cumulative_gas_used, .. } = ctx;
Receipt {
tx_type,
tx_type: tx.tx_type(),
// Success flag was added in `EIP-658: Embedding transaction status code in
// receipts`.
success: result.is_success(),

View File

@@ -1,6 +1,6 @@
use crate::EthEvmConfig;
use alloc::{boxed::Box, sync::Arc, vec, vec::Vec};
use alloy_consensus::{Header, TxType};
use alloy_consensus::Header;
use alloy_eips::eip7685::Requests;
use alloy_evm::precompiles::PrecompilesMap;
use alloy_primitives::Bytes;
@@ -11,14 +11,14 @@ use reth_evm::{
block::{
BlockExecutionError, BlockExecutor, BlockExecutorFactory, BlockExecutorFor, ExecutableTx,
},
eth::{EthBlockExecutionCtx, EthEvmContext, EthTxResult},
eth::{EthBlockExecutionCtx, EthEvmContext},
ConfigureEngineEvm, ConfigureEvm, Database, EthEvm, EthEvmFactory, Evm, EvmEnvFor, EvmFactory,
ExecutableTxIterator, ExecutionCtxFor, RecoveredTx,
ExecutableTxIterator, ExecutionCtxFor,
};
use reth_execution_types::{BlockExecutionResult, ExecutionOutcome};
use reth_primitives_traits::{BlockTy, SealedBlock, SealedHeader};
use revm::{
context::result::{ExecutionResult, HaltReason, Output, ResultAndState, SuccessReason},
context::result::{ExecutionResult, Output, ResultAndState, SuccessReason},
database::State,
Inspector,
};
@@ -90,7 +90,6 @@ impl<'a, DB: Database, I: Inspector<EthEvmContext<&'a mut State<DB>>>> BlockExec
type Evm = EthEvm<&'a mut State<DB>, I, PrecompilesMap>;
type Transaction = TransactionSigned;
type Receipt = Receipt;
type Result = EthTxResult<HaltReason, TxType>;
fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> {
Ok(())
@@ -102,25 +101,25 @@ impl<'a, DB: Database, I: Inspector<EthEvmContext<&'a mut State<DB>>>> BlockExec
fn execute_transaction_without_commit(
&mut self,
tx: impl ExecutableTx<Self>,
) -> Result<Self::Result, BlockExecutionError> {
Ok(EthTxResult {
result: ResultAndState::new(
ExecutionResult::Success {
reason: SuccessReason::Return,
gas_used: 0,
gas_refunded: 0,
logs: vec![],
output: Output::Call(Bytes::from(vec![])),
},
Default::default(),
),
tx_type: tx.into_parts().1.tx().tx_type(),
blob_gas_used: 0,
})
_tx: impl ExecutableTx<Self>,
) -> Result<ResultAndState<<Self::Evm as Evm>::HaltReason>, BlockExecutionError> {
Ok(ResultAndState::new(
ExecutionResult::Success {
reason: SuccessReason::Return,
gas_used: 0,
gas_refunded: 0,
logs: vec![],
output: Output::Call(Bytes::from(vec![])),
},
Default::default(),
))
}
fn commit_transaction(&mut self, _output: Self::Result) -> Result<u64, BlockExecutionError> {
fn commit_transaction(
&mut self,
_output: ResultAndState<<Self::Evm as Evm>::HaltReason>,
_tx: impl ExecutableTx<Self>,
) -> Result<u64, BlockExecutionError> {
Ok(0)
}

View File

@@ -18,7 +18,7 @@ use reth_evm::{
};
use reth_network::{primitives::BasicNetworkPrimitives, NetworkHandle, PeersInfo};
use reth_node_api::{
AddOnsContext, BlockTy, FullNodeComponents, HeaderTy, NodeAddOns, NodePrimitives,
AddOnsContext, FullNodeComponents, HeaderTy, NodeAddOns, NodePrimitives,
PayloadAttributesBuilder, PrimitivesTy, TxTy,
};
use reth_node_builder::{
@@ -53,8 +53,8 @@ use reth_rpc_eth_types::{error::FromEvmError, EthApiError};
use reth_rpc_server_types::RethRpcModule;
use reth_tracing::tracing::{debug, info};
use reth_transaction_pool::{
blobstore::DiskFileBlobStore, EthPooledTransaction, EthTransactionPool, PoolPooledTx,
PoolTransaction, TransactionPool, TransactionValidationTaskExecutor,
blobstore::DiskFileBlobStore, EthTransactionPool, PoolPooledTx, PoolTransaction,
TransactionPool, TransactionValidationTaskExecutor,
};
use revm::context::TxEnv;
use std::{marker::PhantomData, sync::Arc, time::SystemTime};
@@ -464,8 +464,7 @@ where
>,
Node: FullNodeTypes<Types = Types>,
{
type Pool =
EthTransactionPool<Node::Provider, DiskFileBlobStore, EthPooledTransaction, BlockTy<Types>>;
type Pool = EthTransactionPool<Node::Provider, DiskFileBlobStore>;
async fn build_pool(self, ctx: &BuilderContext<Node>) -> eyre::Result<Self::Pool> {
let pool_config = ctx.pool_config();

View File

@@ -1,357 +0,0 @@
//! Tests for handling invalid payloads via Engine API.
//!
//! This module tests the scenario where a node receives invalid payloads (e.g., with modified
//! state roots) before receiving valid ones, ensuring the node can recover and continue.
use crate::utils::eth_payload_attributes;
use alloy_primitives::B256;
use alloy_rpc_types_engine::{ExecutionPayloadV3, PayloadStatusEnum};
use rand::{rngs::StdRng, Rng, SeedableRng};
use reth_chainspec::{ChainSpecBuilder, MAINNET};
use reth_e2e_test_utils::{setup_engine, transaction::TransactionTestContext};
use reth_node_ethereum::EthereumNode;
use reth_rpc_api::EngineApiClient;
use std::sync::Arc;
/// Tests that a node can handle receiving an invalid payload (with wrong state root)
/// followed by the correct payload, and continue operating normally.
///
/// Setup:
/// - Node 1: Produces valid payloads and advances the chain
/// - Node 2: Receives payloads from node 1, but we also inject modified payloads with invalid state
/// roots in between to verify error handling
#[tokio::test]
async fn can_handle_invalid_payload_then_valid() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
let seed: [u8; 32] = rand::rng().random();
let mut rng = StdRng::from_seed(seed);
println!("Seed: {seed:?}");
let chain_spec = Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
.cancun_activated()
.build(),
);
let (mut nodes, _tasks, wallet) = setup_engine::<EthereumNode>(
2,
chain_spec.clone(),
false,
Default::default(),
eth_payload_attributes,
)
.await?;
let mut producer = nodes.pop().unwrap();
let receiver = nodes.pop().unwrap();
// Get engine API client for the receiver node
let receiver_engine = receiver.auth_server_handle().http_client();
// Inject a transaction to allow block building (advance_block waits for transactions)
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
producer.rpc.inject_tx(raw_tx).await?;
// Build a valid payload on the producer
let payload = producer.advance_block().await?;
let valid_block = payload.block().clone();
// Create valid payload first, then corrupt the state root
let mut invalid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
let original_state_root = invalid_payload.payload_inner.payload_inner.state_root;
invalid_payload.payload_inner.payload_inner.state_root = B256::random_with(&mut rng);
// Send the invalid payload to the receiver - should be rejected
let invalid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
invalid_payload.clone(),
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!(
"Invalid payload response: {:?} (state_root changed from {original_state_root} to {})",
invalid_result.status, invalid_payload.payload_inner.payload_inner.state_root
);
// The invalid payload should be rejected
assert!(
matches!(
invalid_result.status,
PayloadStatusEnum::Invalid { .. } | PayloadStatusEnum::Syncing
),
"Expected INVALID or SYNCING status for invalid payload, got {:?}",
invalid_result.status
);
// Now send the valid payload - should be accepted
let valid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
let valid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
valid_payload,
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!("Valid payload response: {:?}", valid_result.status);
// The valid payload should be accepted
assert!(
matches!(
valid_result.status,
PayloadStatusEnum::Valid | PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted
),
"Expected VALID/SYNCING/ACCEPTED status for valid payload, got {:?}",
valid_result.status
);
// Update forkchoice on receiver to the valid block
receiver.update_forkchoice(valid_block.hash(), valid_block.hash()).await?;
// Verify the receiver node is at the expected block
let receiver_head = receiver.block_hash(1);
let producer_head = producer.block_hash(1);
assert_eq!(
receiver_head, producer_head,
"Receiver should have synced to the same chain as producer"
);
println!(
"Test passed: Receiver successfully handled invalid payloads and synced to valid chain"
);
Ok(())
}
/// Tests that a node can handle multiple consecutive invalid payloads
/// before receiving a valid one.
#[tokio::test]
async fn can_handle_multiple_invalid_payloads() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
let seed: [u8; 32] = rand::rng().random();
let mut rng = StdRng::from_seed(seed);
println!("Seed: {seed:?}");
let chain_spec = Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
.cancun_activated()
.build(),
);
let (mut nodes, _tasks, wallet) = setup_engine::<EthereumNode>(
2,
chain_spec.clone(),
false,
Default::default(),
eth_payload_attributes,
)
.await?;
let mut producer = nodes.pop().unwrap();
let receiver = nodes.pop().unwrap();
let receiver_engine = receiver.auth_server_handle().http_client();
// Inject a transaction to allow block building
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
producer.rpc.inject_tx(raw_tx).await?;
// Produce a valid block
let payload = producer.advance_block().await?;
let valid_block = payload.block().clone();
// Send multiple invalid payloads with different corruptions
for i in 0..3 {
// Create valid payload first, then corrupt the state root
let mut invalid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
invalid_payload.payload_inner.payload_inner.state_root = B256::random_with(&mut rng);
let result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
invalid_payload,
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!("Invalid payload {i}: status = {:?}", result.status);
assert!(
matches!(result.status, PayloadStatusEnum::Invalid { .. } | PayloadStatusEnum::Syncing),
"Expected INVALID or SYNCING for invalid payload {i}, got {:?}",
result.status
);
}
// Now send the valid payload
let valid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
let valid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
valid_payload,
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!("Valid payload: status = {:?}", valid_result.status);
assert!(
matches!(
valid_result.status,
PayloadStatusEnum::Valid | PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted
),
"Expected valid status for correct payload, got {:?}",
valid_result.status
);
// Finalize the valid block
receiver.update_forkchoice(valid_block.hash(), valid_block.hash()).await?;
println!("Test passed: Receiver handled multiple invalid payloads and accepted valid one");
Ok(())
}
/// Tests invalid payload handling with blocks that contain transactions.
///
/// This test sends real transactions to node 1, produces blocks with those transactions,
/// then sends invalid (corrupted state root) and valid payloads to node 2.
#[tokio::test]
async fn can_handle_invalid_payload_with_transactions() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
let seed: [u8; 32] = rand::rng().random();
let mut rng = StdRng::from_seed(seed);
println!("Seed: {seed:?}");
let chain_spec = Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
.cancun_activated()
.build(),
);
let (mut nodes, _tasks, wallet) = setup_engine::<EthereumNode>(
2,
chain_spec.clone(),
false,
Default::default(),
eth_payload_attributes,
)
.await?;
let mut producer = nodes.pop().unwrap();
let receiver = nodes.pop().unwrap();
let receiver_engine = receiver.auth_server_handle().http_client();
// Create and send a transaction to the producer node
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
let tx_hash = producer.rpc.inject_tx(raw_tx).await?;
println!("Injected transaction {tx_hash}");
// Build a block containing the transaction
let payload = producer.advance_block().await?;
let valid_block = payload.block().clone();
// Verify the block contains a transaction
let tx_count = valid_block.body().transactions().count();
println!("Block contains {tx_count} transaction(s)");
assert!(tx_count > 0, "Block should contain at least one transaction");
// Create invalid payload by corrupting the state root
let mut invalid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
let original_state_root = invalid_payload.payload_inner.payload_inner.state_root;
invalid_payload.payload_inner.payload_inner.state_root = B256::random_with(&mut rng);
// Send invalid payload - should be rejected
let invalid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
invalid_payload.clone(),
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!(
"Invalid payload (with tx) response: {:?} (state_root changed from {original_state_root} to {})",
invalid_result.status,
invalid_payload.payload_inner.payload_inner.state_root
);
assert!(
matches!(
invalid_result.status,
PayloadStatusEnum::Invalid { .. } | PayloadStatusEnum::Syncing
),
"Expected INVALID or SYNCING for invalid payload with transactions, got {:?}",
invalid_result.status
);
// Send valid payload - should be accepted
let valid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
let valid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
valid_payload,
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!("Valid payload (with tx) response: {:?}", valid_result.status);
assert!(
matches!(
valid_result.status,
PayloadStatusEnum::Valid | PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted
),
"Expected valid status for correct payload with transactions, got {:?}",
valid_result.status
);
// Update forkchoice
receiver.update_forkchoice(valid_block.hash(), valid_block.hash()).await?;
// Verify both nodes are at the same head
let receiver_head = receiver.block_hash(1);
let producer_head = producer.block_hash(1);
assert_eq!(
receiver_head, producer_head,
"Receiver should have synced to the same chain as producer"
);
println!("Test passed: Receiver handled invalid payloads with transactions correctly");
Ok(())
}

View File

@@ -4,7 +4,6 @@ mod blobs;
mod custom_genesis;
mod dev;
mod eth;
mod invalid_payload;
mod p2p;
mod pool;
mod prestate;

View File

@@ -1,7 +1,5 @@
use crate::{execute::ExecutableTxFor, ConfigureEvm, EvmEnvFor, ExecutionCtxFor, TxEnvFor};
use alloy_evm::{block::ExecutableTxParts, RecoveredTx};
use crate::{execute::ExecutableTxFor, ConfigureEvm, EvmEnvFor, ExecutionCtxFor};
use rayon::prelude::*;
use reth_primitives_traits::TxTy;
/// [`ConfigureEvm`] extension providing methods for executing payloads.
pub trait ConfigureEngineEvm<ExecutionData>: ConfigureEvm {
@@ -63,16 +61,11 @@ where
/// Iterator over executable transactions.
pub trait ExecutableTxIterator<Evm: ConfigureEvm>:
ExecutableTxTuple<Tx: ExecutableTxFor<Evm, Recovered = Self::Recovered>>
ExecutableTxTuple<Tx: ExecutableTxFor<Evm>>
{
/// HACK: for some reason, this duplicated AT is the only way to enforce the inner Recovered:
/// Send + Sync bound. Effectively alias for `Self::Tx::Recovered`.
type Recovered: RecoveredTx<TxTy<Evm::Primitives>> + Send + Sync;
}
impl<T, Evm: ConfigureEvm> ExecutableTxIterator<Evm> for T
where
T: ExecutableTxTuple<Tx: ExecutableTxFor<Evm, Recovered: Send + Sync>>,
impl<T, Evm: ConfigureEvm> ExecutableTxIterator<Evm> for T where
T: ExecutableTxTuple<Tx: ExecutableTxFor<Evm>>
{
type Recovered = <T::Tx as ExecutableTxParts<TxEnvFor<Evm>, TxTy<Evm::Primitives>>>::Recovered;
}

View File

@@ -6,7 +6,7 @@ use alloy_consensus::{BlockHeader, Header};
use alloy_eips::eip2718::WithEncoded;
pub use alloy_evm::block::{BlockExecutor, BlockExecutorFactory};
use alloy_evm::{
block::{CommitChanges, ExecutableTxParts},
block::{CommitChanges, ExecutableTx},
Evm, EvmEnv, EvmFactory, RecoveredTx, ToTxEnv,
};
use alloy_primitives::{Address, B256};
@@ -401,31 +401,49 @@ where
/// Conversions for executable transactions.
pub trait ExecutorTx<Executor: BlockExecutor> {
/// Converts the transaction into a tuple of [`TxEnvFor`] and [`Recovered`].
fn into_parts(self) -> (<Executor::Evm as Evm>::Tx, Recovered<Executor::Transaction>);
/// Converts the transaction into [`ExecutableTx`].
fn as_executable(&self) -> impl ExecutableTx<Executor>;
/// Converts the transaction into [`Recovered`].
fn into_recovered(self) -> Recovered<Executor::Transaction>;
}
impl<Executor: BlockExecutor> ExecutorTx<Executor>
for WithEncoded<Recovered<Executor::Transaction>>
{
fn into_parts(self) -> (<Executor::Evm as Evm>::Tx, Recovered<Executor::Transaction>) {
(self.to_tx_env(), self.1)
fn as_executable(&self) -> impl ExecutableTx<Executor> {
self
}
fn into_recovered(self) -> Recovered<Executor::Transaction> {
self.1
}
}
impl<Executor: BlockExecutor> ExecutorTx<Executor> for Recovered<Executor::Transaction> {
fn into_parts(self) -> (<Executor::Evm as Evm>::Tx, Self) {
(self.to_tx_env(), self)
fn as_executable(&self) -> impl ExecutableTx<Executor> {
self
}
fn into_recovered(self) -> Self {
self
}
}
impl<Executor> ExecutorTx<Executor>
for WithTxEnv<<Executor::Evm as Evm>::Tx, Recovered<Executor::Transaction>>
impl<T, Executor> ExecutorTx<Executor>
for WithTxEnv<<<Executor as BlockExecutor>::Evm as Evm>::Tx, T>
where
Executor: BlockExecutor<Transaction: Clone>,
T: ExecutorTx<Executor> + Clone,
Executor: BlockExecutor,
<<Executor as BlockExecutor>::Evm as Evm>::Tx: Clone,
Self: RecoveredTx<Executor::Transaction>,
{
fn into_parts(self) -> (<Executor::Evm as Evm>::Tx, Recovered<Executor::Transaction>) {
(self.tx_env, Arc::unwrap_or_clone(self.tx))
fn as_executable(&self) -> impl ExecutableTx<Executor> {
self
}
fn into_recovered(self) -> Recovered<Executor::Transaction> {
Arc::unwrap_or_clone(self.tx).into_recovered()
}
}
@@ -461,11 +479,10 @@ where
&ExecutionResult<<<Self::Executor as BlockExecutor>::Evm as Evm>::HaltReason>,
) -> CommitChanges,
) -> Result<Option<u64>, BlockExecutionError> {
let (tx_env, tx) = tx.into_parts();
if let Some(gas_used) =
self.executor.execute_transaction_with_commit_condition((tx_env, &tx), f)?
self.executor.execute_transaction_with_commit_condition(tx.as_executable(), f)?
{
self.transactions.push(tx);
self.transactions.push(tx.into_recovered());
Ok(Some(gas_used))
} else {
Ok(None)
@@ -592,20 +609,20 @@ where
}
}
/// A helper trait marking a 'static type that can be converted into an [`ExecutableTxParts`] for
/// block executor.
/// A helper trait marking a 'static type that can be converted into an [`ExecutableTx`] for block
/// executor.
pub trait ExecutableTxFor<Evm: ConfigureEvm>:
ExecutableTxParts<TxEnvFor<Evm>, TxTy<Evm::Primitives>> + RecoveredTx<TxTy<Evm::Primitives>>
ToTxEnv<TxEnvFor<Evm>> + RecoveredTx<TxTy<Evm::Primitives>>
{
}
impl<T, Evm: ConfigureEvm> ExecutableTxFor<Evm> for T where
T: ExecutableTxParts<TxEnvFor<Evm>, TxTy<Evm::Primitives>> + RecoveredTx<TxTy<Evm::Primitives>>
T: ToTxEnv<TxEnvFor<Evm>> + RecoveredTx<TxTy<Evm::Primitives>>
{
}
/// A container for a transaction and a transaction environment.
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct WithTxEnv<TxEnv, T> {
/// The transaction environment for EVM.
pub tx_env: TxEnv,
@@ -613,12 +630,6 @@ pub struct WithTxEnv<TxEnv, T> {
pub tx: Arc<T>,
}
impl<TxEnv: Clone, T> Clone for WithTxEnv<TxEnv, T> {
fn clone(&self) -> Self {
Self { tx_env: self.tx_env.clone(), tx: self.tx.clone() }
}
}
impl<TxEnv, Tx, T: RecoveredTx<Tx>> RecoveredTx<Tx> for WithTxEnv<TxEnv, T> {
fn tx(&self) -> &Tx {
self.tx.tx()
@@ -629,11 +640,9 @@ impl<TxEnv, Tx, T: RecoveredTx<Tx>> RecoveredTx<Tx> for WithTxEnv<TxEnv, T> {
}
}
impl<TxEnv, T: RecoveredTx<Tx>, Tx> ExecutableTxParts<TxEnv, Tx> for WithTxEnv<TxEnv, T> {
type Recovered = Arc<T>;
fn into_parts(self) -> (TxEnv, Self::Recovered) {
(self.tx_env, self.tx)
impl<TxEnv: Clone, T> ToTxEnv<TxEnv> for WithTxEnv<TxEnv, T> {
fn to_tx_env(&self) -> TxEnv {
self.tx_env.clone()
}
}

View File

@@ -131,8 +131,6 @@ pub struct TransactionsManagerMetrics {
/// capacity. Note, this is not a limit to the number of inflight requests, but a health
/// measure.
pub(crate) capacity_pending_pool_imports: Counter,
/// Total number of transactions ignored because pending pool imports are at capacity.
pub(crate) skipped_transactions_pending_pool_imports_at_capacity: Counter,
/// The time it took to prepare transactions for import. This is mostly sender recovery.
pub(crate) pool_import_prepare_duration: Histogram,

View File

@@ -429,22 +429,11 @@ impl<Pool: TransactionPool, N: NetworkPrimitives> TransactionsManager<Pool, N> {
/// Returns `true` if [`TransactionsManager`] has capacity to request pending hashes. Returns
/// `false` if [`TransactionsManager`] is operating close to full capacity.
fn has_capacity_for_fetching_pending_hashes(&self) -> bool {
self.has_capacity_for_pending_pool_imports() &&
self.pending_pool_imports_info
.has_capacity(self.pending_pool_imports_info.max_pending_pool_imports) &&
self.transaction_fetcher.has_capacity_for_fetching_pending_hashes()
}
/// Returns `true` if [`TransactionsManager`] has capacity for more pending pool imports.
fn has_capacity_for_pending_pool_imports(&self) -> bool {
self.remaining_pool_import_capacity() > 0
}
/// Returns the remaining capacity for pending pool imports.
fn remaining_pool_import_capacity(&self) -> usize {
self.pending_pool_imports_info.max_pending_pool_imports.saturating_sub(
self.pending_pool_imports_info.pending_pool_imports.load(Ordering::Relaxed),
)
}
fn report_peer_bad_transactions(&self, peer_id: PeerId) {
self.report_peer(peer_id, ReputationChangeKind::BadTransactions);
self.metrics.reported_bad_transactions.increment(1);
@@ -1296,7 +1285,6 @@ where
trace!(target: "net::tx", peer_id=format!("{peer_id:#}"), policy=?self.config.ingress_policy, "Ignoring full transactions from peer blocked by ingress policy");
return;
}
// ensure we didn't receive any blob transactions as these are disallowed to be
// broadcasted in full
@@ -1347,13 +1335,7 @@ where
return
}
// Early return if we don't have capacity for any imports
if !self.has_capacity_for_pending_pool_imports() {
return
}
let Some(peer) = self.peers.get_mut(&peer_id) else { return };
let client_version = peer.client_version.clone();
let mut transactions = transactions.0;
let start = Instant::now();
@@ -1396,7 +1378,7 @@ where
trace!(target: "net::tx",
peer_id=format!("{peer_id:#}"),
hash=%tx.tx_hash(),
%client_version,
client_version=%peer.client_version,
"received a known bad transaction from peer"
);
has_bad_transactions = true;
@@ -1405,18 +1387,6 @@ where
true
});
// Truncate to remaining capacity before recovery to avoid wasting CPU on transactions
// that won't be imported anyway.
let capacity = self.remaining_pool_import_capacity();
if transactions.len() > capacity {
let skipped = transactions.len() - capacity;
transactions.truncate(capacity);
self.metrics
.skipped_transactions_pending_pool_imports_at_capacity
.increment(skipped as u64);
trace!(target: "net::tx", skipped, capacity, "Truncated transactions batch to capacity");
}
let txs_len = transactions.len();
let new_txs = transactions
@@ -1427,7 +1397,7 @@ where
trace!(target: "net::tx",
peer_id=format!("{peer_id:#}"),
hash=%badtx.tx_hash(),
client_version=%client_version,
client_version=%peer.client_version,
"failed ecrecovery for transaction"
);
None
@@ -1478,7 +1448,7 @@ where
self.metrics
.occurrences_of_transaction_already_seen_by_peer
.increment(num_already_seen_by_peer);
trace!(target: "net::tx", num_txs=%num_already_seen_by_peer, ?peer_id, client=%client_version, "Peer sent already seen transactions");
trace!(target: "net::tx", num_txs=%num_already_seen_by_peer, ?peer_id, client=?peer.client_version, "Peer sent already seen transactions");
}
if has_bad_transactions {

View File

@@ -4,7 +4,7 @@ use crate::{BuilderContext, FullNodeTypes};
use alloy_primitives::Address;
use reth_chain_state::CanonStateSubscriptions;
use reth_chainspec::EthereumHardforks;
use reth_node_api::{BlockTy, NodeTypes, TxTy};
use reth_node_api::{NodeTypes, TxTy};
use reth_transaction_pool::{
blobstore::DiskFileBlobStore, BlobStore, CoinbaseTipOrdering, PoolConfig, PoolTransaction,
SubPoolLimit, TransactionPool, TransactionValidationTaskExecutor, TransactionValidator,
@@ -129,7 +129,7 @@ impl<'a, Node: FullNodeTypes, V> TxPoolBuilder<'a, Node, V> {
impl<'a, Node, V> TxPoolBuilder<'a, Node, TransactionValidationTaskExecutor<V>>
where
Node: FullNodeTypes<Types: NodeTypes<ChainSpec: EthereumHardforks>>,
V: TransactionValidator<Block = BlockTy<Node::Types>> + 'static,
V: TransactionValidator + 'static,
V::Transaction:
PoolTransaction<Consensus = TxTy<Node::Types>> + reth_transaction_pool::EthPoolTransaction,
{
@@ -248,7 +248,7 @@ fn spawn_pool_maintenance_task<Node, Pool>(
) -> eyre::Result<()>
where
Node: FullNodeTypes<Types: NodeTypes<ChainSpec: EthereumHardforks>>,
Pool: reth_transaction_pool::TransactionPoolExt<Block = BlockTy<Node::Types>> + Clone + 'static,
Pool: reth_transaction_pool::TransactionPoolExt + Clone + 'static,
Pool::Transaction: PoolTransaction<Consensus = TxTy<Node::Types>>,
{
let chain_events = ctx.provider().canonical_state_stream();
@@ -280,7 +280,7 @@ pub fn spawn_maintenance_tasks<Node, Pool>(
) -> eyre::Result<()>
where
Node: FullNodeTypes<Types: NodeTypes<ChainSpec: EthereumHardforks>>,
Pool: reth_transaction_pool::TransactionPoolExt<Block = BlockTy<Node::Types>> + Clone + 'static,
Pool: reth_transaction_pool::TransactionPoolExt + Clone + 'static,
Pool::Transaction: PoolTransaction<Consensus = TxTy<Node::Types>>,
{
spawn_local_backup_task(ctx, pool.clone())?;

View File

@@ -4,13 +4,10 @@ use alloy_consensus::transaction::Either;
use alloy_provider::network::AnyNetwork;
use jsonrpsee::core::{DeserializeOwned, Serialize};
use reth_chainspec::EthChainSpec;
use reth_consensus_debug_client::{
BlockProvider, DebugConsensusClient, EtherscanBlockProvider, RpcBlockProvider,
};
use reth_consensus_debug_client::{DebugConsensusClient, EtherscanBlockProvider, RpcBlockProvider};
use reth_engine_local::LocalMiner;
use reth_node_api::{
BlockTy, FullNodeComponents, FullNodeTypes, HeaderTy, PayloadAttrTy, PayloadAttributesBuilder,
PayloadTypes,
BlockTy, FullNodeComponents, HeaderTy, PayloadAttrTy, PayloadAttributesBuilder, PayloadTypes,
};
use std::{
future::{Future, IntoFuture},
@@ -112,16 +109,9 @@ impl<L> DebugNodeLauncher<L> {
}
}
/// Type alias for the default debug block provider. We use etherscan provider to satisfy the
/// bounds.
pub type DefaultDebugBlockProvider<N> = EtherscanBlockProvider<
<<N as FullNodeTypes>::Types as DebugNode<N>>::RpcBlock,
BlockTy<<N as FullNodeTypes>::Types>,
>;
/// Future for the [`DebugNodeLauncher`].
#[expect(missing_debug_implementations, clippy::type_complexity)]
pub struct DebugNodeLauncherFuture<L, Target, N, B = DefaultDebugBlockProvider<N>>
pub struct DebugNodeLauncherFuture<L, Target, N>
where
N: FullNodeComponents<Types: DebugNode<N>>,
{
@@ -131,17 +121,14 @@ where
Option<Box<dyn PayloadAttributesBuilder<PayloadAttrTy<N::Types>, HeaderTy<N::Types>>>>,
map_attributes:
Option<Box<dyn Fn(PayloadAttrTy<N::Types>) -> PayloadAttrTy<N::Types> + Send + Sync>>,
debug_block_provider: Option<B>,
}
impl<L, Target, N, AddOns, B> DebugNodeLauncherFuture<L, Target, N, B>
impl<L, Target, N, AddOns> DebugNodeLauncherFuture<L, Target, N>
where
N: FullNodeComponents<Types: DebugNode<N>>,
AddOns: RethRpcAddOns<N>,
L: LaunchNode<Target, Node = NodeHandle<N, AddOns>>,
B: BlockProvider<Block = BlockTy<N::Types>> + Clone,
{
/// Sets a custom payload attributes builder for local mining in dev mode.
pub fn with_payload_attributes_builder(
self,
builder: impl PayloadAttributesBuilder<PayloadAttrTy<N::Types>, HeaderTy<N::Types>>,
@@ -151,11 +138,9 @@ where
target: self.target,
local_payload_attributes_builder: Some(Box::new(builder)),
map_attributes: None,
debug_block_provider: self.debug_block_provider,
}
}
/// Sets a function to map payload attributes before building.
pub fn map_debug_payload_attributes(
self,
f: impl Fn(PayloadAttrTy<N::Types>) -> PayloadAttrTy<N::Types> + Send + Sync + 'static,
@@ -165,58 +150,16 @@ where
target: self.target,
local_payload_attributes_builder: None,
map_attributes: Some(Box::new(f)),
debug_block_provider: self.debug_block_provider,
}
}
/// Sets a custom block provider for the debug consensus client.
///
/// When set, this provider will be used instead of creating an `EtherscanBlockProvider`
/// or `RpcBlockProvider` from CLI arguments.
pub fn with_debug_block_provider<B2>(
self,
provider: B2,
) -> DebugNodeLauncherFuture<L, Target, N, B2>
where
B2: BlockProvider<Block = BlockTy<N::Types>> + Clone,
{
DebugNodeLauncherFuture {
inner: self.inner,
target: self.target,
local_payload_attributes_builder: self.local_payload_attributes_builder,
map_attributes: self.map_attributes,
debug_block_provider: Some(provider),
}
}
async fn launch_node(self) -> eyre::Result<NodeHandle<N, AddOns>> {
let Self {
inner,
target,
local_payload_attributes_builder,
map_attributes,
debug_block_provider,
} = self;
let Self { inner, target, local_payload_attributes_builder, map_attributes } = self;
let handle = inner.launch_node(target).await?;
let config = &handle.node.config;
if let Some(provider) = debug_block_provider {
info!(target: "reth::cli", "Using custom debug block provider");
let rpc_consensus_client = DebugConsensusClient::new(
handle.node.add_ons_handle.beacon_engine_handle.clone(),
Arc::new(provider),
);
handle
.node
.task_executor
.spawn_critical("custom debug block provider consensus client", async move {
rpc_consensus_client.run().await
});
} else if let Some(url) = config.debug.rpc_consensus_url.clone() {
if let Some(url) = config.debug.rpc_consensus_url.clone() {
info!(target: "reth::cli", "Using RPC consensus client: {}", url);
let block_provider =
@@ -237,11 +180,14 @@ where
handle.node.task_executor.spawn_critical("rpc-ws consensus client", async move {
rpc_consensus_client.run().await
});
} else if let Some(maybe_custom_etherscan_url) = config.debug.etherscan.clone() {
}
if let Some(maybe_custom_etherscan_url) = config.debug.etherscan.clone() {
info!(target: "reth::cli", "Using etherscan as consensus client");
let chain = config.chain.chain();
let etherscan_url = maybe_custom_etherscan_url.map(Ok).unwrap_or_else(|| {
// If URL isn't provided, use default Etherscan URL for the chain if it is known
chain
.etherscan_urls()
.map(|urls| urls.0.to_string())
@@ -306,13 +252,12 @@ where
}
}
impl<L, Target, N, AddOns, B> IntoFuture for DebugNodeLauncherFuture<L, Target, N, B>
impl<L, Target, N, AddOns> IntoFuture for DebugNodeLauncherFuture<L, Target, N>
where
Target: Send + 'static,
N: FullNodeComponents<Types: DebugNode<N>>,
AddOns: RethRpcAddOns<N> + 'static,
L: LaunchNode<Target, Node = NodeHandle<N, AddOns>> + 'static,
B: BlockProvider<Block = BlockTy<N::Types>> + Clone + 'static,
{
type Output = eyre::Result<NodeHandle<N, AddOns>>;
type IntoFuture = Pin<Box<dyn Future<Output = eyre::Result<NodeHandle<N, AddOns>>> + Send>>;
@@ -328,7 +273,6 @@ where
N: FullNodeComponents<Types: DebugNode<N>>,
AddOns: RethRpcAddOns<N> + 'static,
L: LaunchNode<Target, Node = NodeHandle<N, AddOns>> + 'static,
DefaultDebugBlockProvider<N>: BlockProvider<Block = BlockTy<N::Types>> + Clone,
{
type Node = NodeHandle<N, AddOns>;
type Future = DebugNodeLauncherFuture<L, Target, N>;
@@ -339,7 +283,6 @@ where
target,
local_payload_attributes_builder: None,
map_attributes: None,
debug_block_provider: None,
}
}
}

View File

@@ -31,7 +31,7 @@ pub use builder::{add_ons::AddOns, *};
mod launch;
pub use launch::{
debug::{DebugNode, DebugNodeLauncher, DebugNodeLauncherFuture, DefaultDebugBlockProvider},
debug::{DebugNode, DebugNodeLauncher},
engine::EngineNodeLauncher,
*,
};

View File

@@ -2,7 +2,6 @@
use clap::Args;
use reth_config::config::{BlocksPerFileConfig, StaticFilesConfig};
use reth_storage_api::StorageSettings;
/// Blocks per static file when running in `--minimal` node.
///
@@ -41,10 +40,6 @@ pub struct StaticFilesArgs {
#[arg(long = "static-files.blocks-per-file.account-change-sets")]
pub blocks_per_file_account_change_sets: Option<u64>,
/// Number of blocks per file for the storage changesets segment.
#[arg(long = "static-files.blocks-per-file.storage-change-sets")]
pub blocks_per_file_storage_change_sets: Option<u64>,
/// Store receipts in static files instead of the database.
///
/// When enabled, receipts will be written to static files on disk instead of the database.
@@ -73,16 +68,6 @@ pub struct StaticFilesArgs {
/// the node has been initialized, changing this flag requires re-syncing from scratch.
#[arg(long = "static-files.account-change-sets", default_value_t = default_static_file_flag(), action = clap::ArgAction::Set)]
pub account_changesets: bool,
/// Store storage changesets in static files.
///
/// When enabled, storage changesets will be written to static files on disk instead of the
/// database.
///
/// Note: This setting can only be configured at genesis initialization. Once
/// the node has been initialized, changing this flag requires re-syncing from scratch.
#[arg(long = "static-files.storage-change-sets", default_value_t = default_static_file_flag(), action = clap::ArgAction::Set)]
pub storage_changesets: bool,
}
impl StaticFilesArgs {
@@ -113,25 +98,9 @@ impl StaticFilesArgs {
account_change_sets: self
.blocks_per_file_account_change_sets
.or(config.blocks_per_file.account_change_sets),
storage_change_sets: self
.blocks_per_file_storage_change_sets
.or(config.blocks_per_file.storage_change_sets),
},
}
}
/// Converts the static files arguments into [`StorageSettings`].
pub const fn to_settings(&self) -> StorageSettings {
#[cfg(feature = "edge")]
let base = StorageSettings::edge();
#[cfg(not(feature = "edge"))]
let base = StorageSettings::legacy();
base.with_receipts_in_static_files(self.receipts)
.with_transaction_senders_in_static_files(self.transaction_senders)
.with_account_changesets_in_static_files(self.account_changesets)
.with_storage_changesets_in_static_files(self.storage_changesets)
}
}
impl Default for StaticFilesArgs {
@@ -142,11 +111,9 @@ impl Default for StaticFilesArgs {
blocks_per_file_receipts: None,
blocks_per_file_transaction_senders: None,
blocks_per_file_account_change_sets: None,
blocks_per_file_storage_change_sets: None,
receipts: default_static_file_flag(),
transaction_senders: default_static_file_flag(),
account_changesets: default_static_file_flag(),
storage_changesets: default_static_file_flag(),
}
}
}

View File

@@ -363,7 +363,6 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
.with_receipts_in_static_files(self.static_files.receipts)
.with_transaction_senders_in_static_files(self.static_files.transaction_senders)
.with_account_changesets_in_static_files(self.static_files.account_changesets)
.with_storage_changesets_in_static_files(self.static_files.storage_changesets)
.with_transaction_hash_numbers_in_rocksdb(self.rocksdb.all || self.rocksdb.tx_hash)
.with_storages_history_in_rocksdb(self.rocksdb.all || self.rocksdb.storages_history)
.with_account_history_in_rocksdb(self.rocksdb.all || self.rocksdb.account_history)

View File

@@ -17,9 +17,9 @@ impl OpReceiptBuilder for OpRethReceiptBuilder {
fn build_receipt<'a, E: Evm>(
&self,
ctx: ReceiptBuilderCtx<'a, OpTxType, E>,
) -> Result<Self::Receipt, ReceiptBuilderCtx<'a, OpTxType, E>> {
match ctx.tx_type {
ctx: ReceiptBuilderCtx<'a, OpTransactionSigned, E>,
) -> Result<Self::Receipt, ReceiptBuilderCtx<'a, OpTransactionSigned, E>> {
match ctx.tx.tx_type() {
OpTxType::Deposit => Err(ctx),
ty => {
let receipt = Receipt {

View File

@@ -62,7 +62,6 @@ where
type ExecutionPayloadEnvelopeV3 = OpExecutionPayloadEnvelopeV3;
type ExecutionPayloadEnvelopeV4 = OpExecutionPayloadEnvelopeV4;
type ExecutionPayloadEnvelopeV5 = OpExecutionPayloadEnvelopeV4;
type ExecutionPayloadEnvelopeV6 = OpExecutionPayloadEnvelopeV4;
}
/// Validator for Optimism engine API.

View File

@@ -16,7 +16,7 @@ use reth_network::{
PeersInfo,
};
use reth_node_api::{
AddOnsContext, BlockTy, BuildNextEnv, EngineTypes, FullNodeComponents, HeaderTy, NodeAddOns,
AddOnsContext, BuildNextEnv, EngineTypes, FullNodeComponents, HeaderTy, NodeAddOns,
NodePrimitives, PayloadAttributesBuilder, PayloadTypes, PrimitivesTy, TxTy,
};
use reth_node_builder::{
@@ -962,7 +962,7 @@ where
Node: FullNodeTypes<Types: NodeTypes<ChainSpec: OpHardforks>>,
T: EthPoolTransaction<Consensus = TxTy<Node::Types>> + OpPooledTx,
{
type Pool = OpTransactionPool<Node::Provider, DiskFileBlobStore, T, BlockTy<Node::Types>>;
type Pool = OpTransactionPool<Node::Provider, DiskFileBlobStore, T>;
async fn build_pool(self, ctx: &BuilderContext<Node>) -> eyre::Result<Self::Pool> {
let Self { pool_config_overrides, .. } = self;

View File

@@ -9,7 +9,6 @@
#![cfg_attr(docsrs, feature(doc_cfg))]
mod validator;
use op_alloy_consensus::OpBlock;
pub use validator::{OpL1BlockInfo, OpTransactionValidator};
pub mod conditional;
@@ -25,8 +24,8 @@ pub mod estimated_da_size;
use reth_transaction_pool::{CoinbaseTipOrdering, Pool, TransactionValidationTaskExecutor};
/// Type alias for default optimism transaction pool
pub type OpTransactionPool<Client, S, T = OpPooledTransaction, B = OpBlock> = Pool<
TransactionValidationTaskExecutor<OpTransactionValidator<Client, T, B>>,
pub type OpTransactionPool<Client, S, T = OpPooledTransaction> = Pool<
TransactionValidationTaskExecutor<OpTransactionValidator<Client, T>>,
CoinbaseTipOrdering<T>,
S,
>;

View File

@@ -325,11 +325,10 @@ mod tests {
#[tokio::test]
async fn validate_optimism_transaction() {
let client = MockEthProvider::default().with_chain_spec(OP_MAINNET.clone());
let validator =
EthTransactionValidatorBuilder::new(client)
.no_shanghai()
.no_cancun()
.build::<_, _, reth_optimism_primitives::OpBlock>(InMemoryBlobStore::default());
let validator = EthTransactionValidatorBuilder::new(client)
.no_shanghai()
.no_cancun()
.build(InMemoryBlobStore::default());
let validator = OpTransactionValidator::new(validator);
let origin = TransactionOrigin::External;

View File

@@ -1,6 +1,5 @@
use crate::{supervisor::SupervisorClient, InvalidCrossTx, OpPooledTx};
use alloy_consensus::{BlockHeader, Transaction};
use op_alloy_consensus::OpBlock;
use op_revm::L1BlockInfo;
use parking_lot::RwLock;
use reth_chainspec::ChainSpecProvider;
@@ -40,9 +39,9 @@ impl OpL1BlockInfo {
/// Validator for Optimism transactions.
#[derive(Debug, Clone)]
pub struct OpTransactionValidator<Client, Tx, B = OpBlock> {
pub struct OpTransactionValidator<Client, Tx> {
/// The type that performs the actual validation.
inner: Arc<EthTransactionValidator<Client, Tx, B>>,
inner: Arc<EthTransactionValidator<Client, Tx>>,
/// Additional block info required for validation.
block_info: Arc<OpL1BlockInfo>,
/// If true, ensure that the transaction's sender has enough balance to cover the L1 gas fee
@@ -55,7 +54,7 @@ pub struct OpTransactionValidator<Client, Tx, B = OpBlock> {
fork_tracker: Arc<OpForkTracker>,
}
impl<Client, Tx, B: Block> OpTransactionValidator<Client, Tx, B> {
impl<Client, Tx> OpTransactionValidator<Client, Tx> {
/// Returns the configured chain spec
pub fn chain_spec(&self) -> Arc<Client::ChainSpec>
where
@@ -87,15 +86,14 @@ impl<Client, Tx, B: Block> OpTransactionValidator<Client, Tx, B> {
}
}
impl<Client, Tx, B> OpTransactionValidator<Client, Tx, B>
impl<Client, Tx> OpTransactionValidator<Client, Tx>
where
Client:
ChainSpecProvider<ChainSpec: OpHardforks> + StateProviderFactory + BlockReaderIdExt + Sync,
Tx: EthPoolTransaction + OpPooledTx,
B: Block,
{
/// Create a new [`OpTransactionValidator`].
pub fn new(inner: EthTransactionValidator<Client, Tx, B>) -> Self {
pub fn new(inner: EthTransactionValidator<Client, Tx>) -> Self {
let this = Self::with_block_info(inner, OpL1BlockInfo::default());
if let Ok(Some(block)) =
this.inner.client().block_by_number_or_tag(alloy_eips::BlockNumberOrTag::Latest)
@@ -114,7 +112,7 @@ where
/// Create a new [`OpTransactionValidator`] with the given [`OpL1BlockInfo`].
pub fn with_block_info(
inner: EthTransactionValidator<Client, Tx, B>,
inner: EthTransactionValidator<Client, Tx>,
block_info: OpL1BlockInfo,
) -> Self {
Self {
@@ -290,15 +288,13 @@ where
}
}
impl<Client, Tx, B> TransactionValidator for OpTransactionValidator<Client, Tx, B>
impl<Client, Tx> TransactionValidator for OpTransactionValidator<Client, Tx>
where
Client:
ChainSpecProvider<ChainSpec: OpHardforks> + StateProviderFactory + BlockReaderIdExt + Sync,
Tx: EthPoolTransaction + OpPooledTx,
B: Block,
{
type Transaction = Tx;
type Block = B;
async fn validate_transaction(
&self,
@@ -329,7 +325,10 @@ where
.await
}
fn on_new_head_block(&self, new_tip_block: &SealedBlock<Self::Block>) {
fn on_new_head_block<B>(&self, new_tip_block: &SealedBlock<B>)
where
B: Block,
{
self.inner.on_new_head_block(new_tip_block);
self.update_l1_block_info(
new_tip_block.header(),

View File

@@ -9,16 +9,6 @@ use std::{fmt::Debug, ops::RangeBounds};
use tracing::debug;
pub(crate) trait DbTxPruneExt: DbTxMut + DbTx {
/// Clear the entire table in a single operation.
///
/// This is much faster than iterating entry-by-entry for `PruneMode::Full`.
/// Returns the number of entries that were in the table.
fn clear_table<T: Table>(&self) -> Result<usize, DatabaseError> {
let count = self.entries::<T>()?;
<Self as DbTxMut>::clear::<T>(self)?;
Ok(count)
}
/// Prune the table for the specified pre-sorted key iterator.
///
/// Returns number of rows pruned.

View File

@@ -0,0 +1,119 @@
use crate::{
db_ext::DbTxPruneExt,
segments::{PruneInput, Segment},
PrunerError,
};
use alloy_primitives::B256;
use reth_db_api::{models::BlockNumberHashedAddress, table::Value, tables, transaction::DbTxMut};
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
errors::provider::ProviderResult, BlockReader, ChainStateBlockReader, DBProvider,
NodePrimitivesProvider, PruneCheckpointWriter, TransactionsProvider,
};
use reth_prune_types::{
PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint,
};
use reth_stages_types::StageId;
use tracing::{instrument, trace};
#[derive(Debug)]
pub struct MerkleChangeSets {
mode: PruneMode,
}
impl MerkleChangeSets {
pub const fn new(mode: PruneMode) -> Self {
Self { mode }
}
}
impl<Provider> Segment<Provider> for MerkleChangeSets
where
Provider: DBProvider<Tx: DbTxMut>
+ PruneCheckpointWriter
+ TransactionsProvider
+ BlockReader
+ ChainStateBlockReader
+ NodePrimitivesProvider<Primitives: NodePrimitives<Receipt: Value>>,
{
fn segment(&self) -> PruneSegment {
PruneSegment::MerkleChangeSets
}
fn mode(&self) -> Option<PruneMode> {
Some(self.mode)
}
fn purpose(&self) -> PrunePurpose {
PrunePurpose::User
}
fn required_stage(&self) -> Option<StageId> {
Some(StageId::MerkleChangeSets)
}
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
fn prune(&self, provider: &Provider, input: PruneInput) -> Result<SegmentOutput, PrunerError> {
let Some(block_range) = input.get_next_block_range() else {
trace!(target: "pruner", "No change sets to prune");
return Ok(SegmentOutput::done())
};
let block_range_end = *block_range.end();
let mut limiter = input.limiter;
// Create range for StoragesTrieChangeSets which uses BlockNumberHashedAddress as key
let storage_range_start: BlockNumberHashedAddress =
(*block_range.start(), B256::ZERO).into();
let storage_range_end: BlockNumberHashedAddress =
(*block_range.end() + 1, B256::ZERO).into();
let storage_range = storage_range_start..storage_range_end;
let mut last_storages_pruned_block = None;
let (storages_pruned, done) =
provider.tx_ref().prune_dupsort_table_with_range::<tables::StoragesTrieChangeSets>(
storage_range,
&mut limiter,
|(BlockNumberHashedAddress((block_number, _)), _)| {
last_storages_pruned_block = Some(block_number);
},
)?;
trace!(target: "pruner", %storages_pruned, %done, "Pruned storages change sets");
let mut last_accounts_pruned_block = block_range_end;
let last_storages_pruned_block = last_storages_pruned_block
// If there's more storage changesets to prune, set the checkpoint block number to
// previous, so we could finish pruning its storage changesets on the next run.
.map(|block_number| if done { block_number } else { block_number.saturating_sub(1) })
.unwrap_or(block_range_end);
let (accounts_pruned, done) =
provider.tx_ref().prune_dupsort_table_with_range::<tables::AccountsTrieChangeSets>(
block_range,
&mut limiter,
|row| last_accounts_pruned_block = row.0,
)?;
trace!(target: "pruner", %accounts_pruned, %done, "Pruned accounts change sets");
let progress = limiter.progress(done);
Ok(SegmentOutput {
progress,
pruned: accounts_pruned + storages_pruned,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: Some(last_storages_pruned_block.min(last_accounts_pruned_block)),
tx_number: None,
}),
})
}
fn save_checkpoint(
&self,
provider: &Provider,
checkpoint: PruneCheckpoint,
) -> ProviderResult<()> {
provider.save_prune_checkpoint(PruneSegment::MerkleChangeSets, checkpoint)
}
}

View File

@@ -6,7 +6,7 @@ use crate::{
use reth_db_api::{tables, transaction::DbTxMut};
use reth_provider::{BlockReader, DBProvider, TransactionsProvider};
use reth_prune_types::{
PruneMode, PruneProgress, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint,
PruneMode, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint,
};
use tracing::{instrument, trace};
@@ -48,25 +48,6 @@ where
};
let tx_range_end = *tx_range.end();
// For PruneMode::Full, clear the entire table in one operation
if self.mode.is_full() {
let pruned = provider.tx_ref().clear_table::<tables::TransactionSenders>()?;
trace!(target: "pruner", %pruned, "Cleared transaction senders table");
let last_pruned_block = provider
.block_by_transaction_id(tx_range_end)?
.ok_or(PrunerError::InconsistentData("Block for transaction is not found"))?;
return Ok(SegmentOutput {
progress: PruneProgress::Finished,
pruned,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: Some(last_pruned_block),
tx_number: Some(tx_range_end),
}),
});
}
let mut limiter = input.limiter;
let mut last_pruned_transaction = tx_range_end;

View File

@@ -8,7 +8,7 @@ use rayon::prelude::*;
use reth_db_api::{tables, transaction::DbTxMut};
use reth_provider::{BlockReader, DBProvider, PruneCheckpointReader, StaticFileProviderFactory};
use reth_prune_types::{
PruneCheckpoint, PruneMode, PruneProgress, PrunePurpose, PruneSegment, SegmentOutputCheckpoint,
PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment, SegmentOutputCheckpoint,
};
use reth_static_file_types::StaticFileSegment;
use tracing::{debug, instrument, trace};
@@ -82,26 +82,6 @@ where
}
}
.into_inner();
// For PruneMode::Full, clear the entire table in one operation
if self.mode.is_full() {
let pruned = provider.tx_ref().clear_table::<tables::TransactionHashNumbers>()?;
trace!(target: "pruner", %pruned, "Cleared transaction lookup table");
let last_pruned_block = provider
.block_by_transaction_id(end)?
.ok_or(PrunerError::InconsistentData("Block for transaction is not found"))?;
return Ok(SegmentOutput {
progress: PruneProgress::Finished,
pruned,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: Some(last_pruned_block),
tx_number: Some(end),
}),
});
}
let tx_range = start..=
Some(end)
.min(
@@ -116,17 +96,12 @@ where
let tx_range_end = *tx_range.end();
// Retrieve transactions in the range and calculate their hashes in parallel
let mut hashes = provider
let hashes = provider
.transactions_by_tx_range(tx_range.clone())?
.into_par_iter()
.map(|transaction| transaction.trie_hash())
.collect::<Vec<_>>();
// Sort hashes to enable efficient cursor traversal through the TransactionHashNumbers
// table, which is keyed by hash. Without sorting, each seek would be O(log n) random
// access; with sorting, the cursor advances sequentially through the B+tree.
hashes.sort_unstable();
// Number of transactions retrieved from the database should match the tx range count
let tx_count = tx_range.count();
if hashes.len() != tx_count {

View File

@@ -12,8 +12,7 @@ use alloy_json_rpc::RpcObject;
use alloy_primitives::{Address, BlockHash, Bytes, B256, U256, U64};
use alloy_rpc_types_engine::{
ClientVersionV1, ExecutionPayloadBodiesV1, ExecutionPayloadInputV2, ExecutionPayloadV1,
ExecutionPayloadV3, ExecutionPayloadV4, ForkchoiceState, ForkchoiceUpdated, PayloadId,
PayloadStatus,
ExecutionPayloadV3, ForkchoiceState, ForkchoiceUpdated, PayloadId, PayloadStatus,
};
use alloy_rpc_types_eth::{
state::StateOverride, BlockOverrides, EIP1186AccountProofResponse, Filter, Log, SyncStatus,
@@ -74,18 +73,6 @@ pub trait EngineApi<Engine: EngineTypes> {
execution_requests: RequestsOrHash,
) -> RpcResult<PayloadStatus>;
/// Post Amsterdam payload handler
///
/// See also <https://github.com/ethereum/execution-apis/blob/main/src/engine/amsterdam.md#engine_newpayloadv5>
#[method(name = "newPayloadV5")]
async fn new_payload_v5(
&self,
payload: ExecutionPayloadV4,
versioned_hashes: Vec<B256>,
parent_beacon_block_root: B256,
execution_requests: RequestsOrHash,
) -> RpcResult<PayloadStatus>;
/// See also <https://github.com/ethereum/execution-apis/blob/6709c2a795b707202e93c4f2867fa0bf2640a84f/src/engine/paris.md#engine_forkchoiceupdatedv1>
///
/// Caution: This should not accept the `withdrawals` field in the payload attributes.
@@ -191,19 +178,6 @@ pub trait EngineApi<Engine: EngineTypes> {
payload_id: PayloadId,
) -> RpcResult<Engine::ExecutionPayloadEnvelopeV5>;
/// Post Amsterdam payload handler.
///
/// See also <https://github.com/ethereum/execution-apis/blob/main/src/engine/amsterdam.md#engine_getpayloadv6>
///
/// Returns the most recent version of the payload that is available in the corresponding
/// payload build process at the time of receiving this call. Note:
/// > Provider software MAY stop the corresponding build process after serving this call.
#[method(name = "getPayloadV6")]
async fn get_payload_v6(
&self,
payload_id: PayloadId,
) -> RpcResult<Engine::ExecutionPayloadEnvelopeV6>;
/// See also <https://github.com/ethereum/execution-apis/blob/6452a6b194d7db269bf1dbd087a267251d3cc7f8/src/engine/shanghai.md#engine_getpayloadbodiesbyhashv1>
#[method(name = "getPayloadBodiesByHashV1")]
async fn get_payload_bodies_by_hash_v1(

View File

@@ -41,7 +41,6 @@ metrics.workspace = true
async-trait.workspace = true
jsonrpsee-core.workspace = true
jsonrpsee-types.workspace = true
parking_lot.workspace = true
serde.workspace = true
thiserror.workspace = true
tracing.workspace = true

View File

@@ -1,274 +0,0 @@
//! Block Access List (BAL) cache for EIP-7928.
//!
//! This module provides an in-memory cache for storing Block Access Lists received via
//! the Engine API. BALs are stored for valid payloads and can be retrieved via
//! `engine_getBALsByHashV1` and `engine_getBALsByRangeV1`.
//!
//! According to EIP-7928, the EL MUST retain BALs for at least the duration of the
//! weak subjectivity period (~3533 epochs) to support synchronization with re-execution.
//! This initial implementation uses a simple in-memory cache with configurable capacity.
use alloy_primitives::{BlockHash, BlockNumber, Bytes};
use parking_lot::RwLock;
use reth_metrics::{
metrics::{Counter, Gauge},
Metrics,
};
use std::{
collections::{BTreeMap, HashMap},
sync::Arc,
};
/// Default capacity for the BAL cache.
///
/// This is a conservative default - production deployments should configure based on
/// weak subjectivity period requirements (~3533 epochs ≈ 113,000 blocks).
const DEFAULT_BAL_CACHE_CAPACITY: u32 = 1024;
/// In-memory cache for Block Access Lists (BALs).
///
/// Provides O(1) lookups by block hash and O(log n) range queries by block number.
/// Evicts the oldest (lowest) block numbers when capacity is exceeded.
///
/// This type is cheaply cloneable as it wraps an `Arc` internally.
#[derive(Debug, Clone)]
pub struct BalCache {
inner: Arc<BalCacheInner>,
}
#[derive(Debug)]
struct BalCacheInner {
/// Maximum number of entries to store.
capacity: u32,
/// Mapping from block hash to BAL bytes.
entries: RwLock<HashMap<BlockHash, Bytes>>,
/// Index mapping block number to block hash for range queries.
/// Uses `BTreeMap` for efficient range iteration and eviction of oldest blocks.
block_index: RwLock<BTreeMap<BlockNumber, BlockHash>>,
/// Cache metrics.
metrics: BalCacheMetrics,
}
impl BalCache {
/// Creates a new BAL cache with the default capacity.
pub fn new() -> Self {
Self::with_capacity(DEFAULT_BAL_CACHE_CAPACITY)
}
/// Creates a new BAL cache with the specified capacity.
pub fn with_capacity(capacity: u32) -> Self {
Self {
inner: Arc::new(BalCacheInner {
capacity,
entries: RwLock::new(HashMap::new()),
block_index: RwLock::new(BTreeMap::new()),
metrics: BalCacheMetrics::default(),
}),
}
}
/// Inserts a BAL into the cache.
///
/// If a different hash already exists for this block number (reorg), the old entry
/// is removed first. If the cache is at capacity, the oldest block number is evicted.
pub fn insert(&self, block_hash: BlockHash, block_number: BlockNumber, bal: Bytes) {
let mut entries = self.inner.entries.write();
let mut block_index = self.inner.block_index.write();
// If this block number already has a different hash, remove the old entry
if let Some(old_hash) = block_index.get(&block_number) &&
*old_hash != block_hash
{
entries.remove(old_hash);
}
// Evict oldest block if at capacity and this is a new entry
if !entries.contains_key(&block_hash) &&
entries.len() as u32 >= self.inner.capacity &&
let Some((&oldest_num, &oldest_hash)) = block_index.first_key_value()
{
entries.remove(&oldest_hash);
block_index.remove(&oldest_num);
}
entries.insert(block_hash, bal);
block_index.insert(block_number, block_hash);
self.inner.metrics.inserts.increment(1);
self.inner.metrics.count.set(entries.len() as f64);
}
/// Retrieves BALs for the given block hashes.
///
/// Returns a vector with the same length as `block_hashes`, where each element
/// is `Some(bal)` if found or `None` if not in cache.
pub fn get_by_hashes(&self, block_hashes: &[BlockHash]) -> Vec<Option<Bytes>> {
let entries = self.inner.entries.read();
block_hashes
.iter()
.map(|hash| {
let result = entries.get(hash).cloned();
if result.is_some() {
self.inner.metrics.hits.increment(1);
} else {
self.inner.metrics.misses.increment(1);
}
result
})
.collect()
}
/// Retrieves BALs for a range of blocks starting at `start` for `count` blocks.
///
/// Returns a vector of contiguous BALs in block number order, stopping at the first
/// missing block. This ensures the caller knows the returned BALs correspond to
/// blocks `[start, start + len)`.
pub fn get_by_range(&self, start: BlockNumber, count: u64) -> Vec<Bytes> {
let entries = self.inner.entries.read();
let block_index = self.inner.block_index.read();
let mut result = Vec::new();
for block_num in start..start.saturating_add(count) {
let Some(hash) = block_index.get(&block_num) else {
break;
};
let Some(bal) = entries.get(hash) else {
break;
};
result.push(bal.clone());
}
result
}
/// Returns the number of entries in the cache.
#[cfg(test)]
fn len(&self) -> usize {
self.inner.entries.read().len()
}
}
impl Default for BalCache {
fn default() -> Self {
Self::new()
}
}
/// Metrics for the BAL cache.
#[derive(Metrics)]
#[metrics(scope = "engine.bal_cache")]
struct BalCacheMetrics {
/// The total number of BALs in the cache.
count: Gauge,
/// The number of cache inserts.
inserts: Counter,
/// The number of cache hits.
hits: Counter,
/// The number of cache misses.
misses: Counter,
}
#[cfg(test)]
mod tests {
use super::*;
use alloy_primitives::B256;
#[test]
fn test_insert_and_get_by_hash() {
let cache = BalCache::with_capacity(10);
let hash1 = B256::random();
let hash2 = B256::random();
let bal1 = Bytes::from_static(b"bal1");
let bal2 = Bytes::from_static(b"bal2");
cache.insert(hash1, 1, bal1.clone());
cache.insert(hash2, 2, bal2.clone());
let results = cache.get_by_hashes(&[hash1, hash2, B256::random()]);
assert_eq!(results.len(), 3);
assert_eq!(results[0], Some(bal1));
assert_eq!(results[1], Some(bal2));
assert_eq!(results[2], None);
}
#[test]
fn test_get_by_range() {
let cache = BalCache::with_capacity(10);
for i in 1..=5 {
let hash = B256::random();
let bal = Bytes::from(format!("bal{i}").into_bytes());
cache.insert(hash, i, bal);
}
let results = cache.get_by_range(2, 3);
assert_eq!(results.len(), 3);
}
#[test]
fn test_get_by_range_stops_at_gap() {
let cache = BalCache::with_capacity(10);
// Insert blocks 1, 2, 4, 5 (missing block 3)
for i in [1, 2, 4, 5] {
let hash = B256::random();
let bal = Bytes::from(format!("bal{i}").into_bytes());
cache.insert(hash, i, bal);
}
// Requesting range starting at 1 should stop at the gap (block 3)
let results = cache.get_by_range(1, 5);
assert_eq!(results.len(), 2); // Only blocks 1 and 2
// Requesting range starting at 4 should return 4 and 5
let results = cache.get_by_range(4, 3);
assert_eq!(results.len(), 2);
}
#[test]
fn test_eviction_oldest_first() {
let cache = BalCache::with_capacity(3);
// Insert blocks 10, 20, 30
for i in [10, 20, 30] {
let hash = B256::random();
cache.insert(hash, i, Bytes::from_static(b"bal"));
}
assert_eq!(cache.len(), 3);
// Insert block 40, should evict block 10 (oldest/lowest)
let hash40 = B256::random();
cache.insert(hash40, 40, Bytes::from_static(b"bal40"));
assert_eq!(cache.len(), 3);
// Block 10 should be gone, block 20 should still be there
let results = cache.get_by_range(10, 1);
assert_eq!(results.len(), 0);
let results = cache.get_by_range(20, 1);
assert_eq!(results.len(), 1);
}
#[test]
fn test_reorg_replaces_hash() {
let cache = BalCache::with_capacity(10);
let hash1 = B256::random();
let hash2 = B256::random();
let bal1 = Bytes::from_static(b"bal1");
let bal2 = Bytes::from_static(b"bal2");
// Insert block 100 with hash1
cache.insert(hash1, 100, bal1.clone());
assert_eq!(cache.get_by_hashes(&[hash1])[0], Some(bal1));
// Reorg: insert block 100 with hash2
cache.insert(hash2, 100, bal2.clone());
// hash1 should be gone, hash2 should be there
assert_eq!(cache.get_by_hashes(&[hash1])[0], None);
assert_eq!(cache.get_by_hashes(&[hash2])[0], Some(bal2));
assert_eq!(cache.len(), 1);
}
}

View File

@@ -1,20 +1,18 @@
use crate::{
bal_cache::BalCache, capabilities::EngineCapabilities, metrics::EngineApiMetrics,
EngineApiError, EngineApiResult,
capabilities::EngineCapabilities, metrics::EngineApiMetrics, EngineApiError, EngineApiResult,
};
use alloy_eips::{
eip1898::BlockHashOrNumber,
eip4844::{BlobAndProofV1, BlobAndProofV2},
eip4895::Withdrawals,
eip7685::RequestsOrHash,
BlockNumHash,
};
use alloy_primitives::{BlockHash, BlockNumber, Bytes, B256, U64};
use alloy_primitives::{BlockHash, BlockNumber, B256, U64};
use alloy_rpc_types_engine::{
CancunPayloadFields, ClientVersionV1, ExecutionData, ExecutionPayloadBodiesV1,
ExecutionPayloadBodyV1, ExecutionPayloadInputV2, ExecutionPayloadSidecar, ExecutionPayloadV1,
ExecutionPayloadV3, ExecutionPayloadV4, ForkchoiceState, ForkchoiceUpdated, PayloadId,
PayloadStatus, PraguePayloadFields,
ExecutionPayloadV3, ForkchoiceState, ForkchoiceUpdated, PayloadId, PayloadStatus,
PraguePayloadFields,
};
use async_trait::async_trait;
use jsonrpsee_core::{server::RpcModule, RpcResult};
@@ -23,7 +21,7 @@ use reth_engine_primitives::{ConsensusEngineHandle, EngineApiValidator, EngineTy
use reth_network_api::NetworkInfo;
use reth_payload_builder::PayloadStore;
use reth_payload_primitives::{
validate_payload_timestamp, EngineApiMessageVersion, ExecutionPayload, MessageValidationKind,
validate_payload_timestamp, EngineApiMessageVersion, MessageValidationKind,
PayloadOrAttributes, PayloadTypes,
};
use reth_primitives_traits::{Block, BlockBody};
@@ -98,38 +96,6 @@ where
validator: Validator,
accept_execution_requests_hash: bool,
network: impl NetworkInfo + 'static,
) -> Self {
Self::with_bal_cache(
provider,
chain_spec,
beacon_consensus,
payload_store,
tx_pool,
task_spawner,
client,
capabilities,
validator,
accept_execution_requests_hash,
network,
BalCache::new(),
)
}
/// Create new instance of [`EngineApi`] with a custom BAL cache.
#[expect(clippy::too_many_arguments)]
pub fn with_bal_cache(
provider: Provider,
chain_spec: Arc<ChainSpec>,
beacon_consensus: ConsensusEngineHandle<PayloadT>,
payload_store: PayloadStore<PayloadT>,
tx_pool: Pool,
task_spawner: Box<dyn TaskSpawner>,
client: ClientVersionV1,
capabilities: EngineCapabilities,
validator: Validator,
accept_execution_requests_hash: bool,
network: impl NetworkInfo + 'static,
bal_cache: BalCache,
) -> Self {
let is_syncing = Arc::new(move || network.is_syncing());
let inner = Arc::new(EngineApiInner {
@@ -145,25 +111,10 @@ where
validator,
accept_execution_requests_hash,
is_syncing,
bal_cache,
});
Self { inner }
}
/// Returns a reference to the BAL cache.
pub fn bal_cache(&self) -> &BalCache {
&self.inner.bal_cache
}
/// Caches the BAL if the status is valid.
fn maybe_cache_bal(&self, num_hash: BlockNumHash, bal: Option<Bytes>, status: &PayloadStatus) {
if status.is_valid() &&
let Some(bal) = bal
{
self.inner.bal_cache.insert(num_hash.hash, num_hash.number, bal);
}
}
/// Fetches the client version.
pub fn get_client_version_v1(
&self,
@@ -198,11 +149,7 @@ where
.validator
.validate_version_specific_fields(EngineApiMessageVersion::V1, payload_or_attrs)?;
let num_hash = payload.num_hash();
let bal = payload.block_access_list().cloned();
let status = self.inner.beacon_consensus.new_payload(payload).await?;
self.maybe_cache_bal(num_hash, bal, &status);
Ok(status)
Ok(self.inner.beacon_consensus.new_payload(payload).await?)
}
/// Metered version of `new_payload_v1`.
@@ -230,12 +177,7 @@ where
self.inner
.validator
.validate_version_specific_fields(EngineApiMessageVersion::V2, payload_or_attrs)?;
let num_hash = payload.num_hash();
let bal = payload.block_access_list().cloned();
let status = self.inner.beacon_consensus.new_payload(payload).await?;
self.maybe_cache_bal(num_hash, bal, &status);
Ok(status)
Ok(self.inner.beacon_consensus.new_payload(payload).await?)
}
/// Metered version of `new_payload_v2`.
@@ -264,11 +206,7 @@ where
.validator
.validate_version_specific_fields(EngineApiMessageVersion::V3, payload_or_attrs)?;
let num_hash = payload.num_hash();
let bal = payload.block_access_list().cloned();
let status = self.inner.beacon_consensus.new_payload(payload).await?;
self.maybe_cache_bal(num_hash, bal, &status);
Ok(status)
Ok(self.inner.beacon_consensus.new_payload(payload).await?)
}
/// Metrics version of `new_payload_v3`
@@ -298,11 +236,7 @@ where
.validator
.validate_version_specific_fields(EngineApiMessageVersion::V4, payload_or_attrs)?;
let num_hash = payload.num_hash();
let bal = payload.block_access_list().cloned();
let status = self.inner.beacon_consensus.new_payload(payload).await?;
self.maybe_cache_bal(num_hash, bal, &status);
Ok(status)
Ok(self.inner.beacon_consensus.new_payload(payload).await?)
}
/// Metrics version of `new_payload_v4`
@@ -947,22 +881,6 @@ where
res
}
/// Retrieves BALs for the given block hashes from the cache.
///
/// Returns the RLP-encoded BALs for blocks found in the cache.
/// Missing blocks are returned as empty bytes.
pub fn get_bals_by_hash(&self, block_hashes: Vec<BlockHash>) -> Vec<alloy_primitives::Bytes> {
let results = self.inner.bal_cache.get_by_hashes(&block_hashes);
results.into_iter().map(|opt| opt.unwrap_or_default()).collect()
}
/// Retrieves BALs for a range of blocks from the cache.
///
/// Returns the RLP-encoded BALs for blocks in the range `[start, start + count)`.
pub fn get_bals_by_range(&self, start: u64, count: u64) -> Vec<alloy_primitives::Bytes> {
self.inner.bal_cache.get_by_range(start, count)
}
}
// This is the concrete ethereum engine API implementation.
@@ -1045,24 +963,6 @@ where
Ok(self.new_payload_v4_metered(payload).await?)
}
/// Handler for `engine_newPayloadV5`
///
/// Post Amsterdam payload handler. Currently returns unsupported fork error.
///
/// See also <https://github.com/ethereum/execution-apis/blob/main/src/engine/amsterdam.md#engine_newpayloadv5>
async fn new_payload_v5(
&self,
_payload: ExecutionPayloadV4,
_versioned_hashes: Vec<B256>,
_parent_beacon_block_root: B256,
_execution_requests: RequestsOrHash,
) -> RpcResult<PayloadStatus> {
trace!(target: "rpc::engine", "Serving engine_newPayloadV5");
Err(EngineApiError::EngineObjectValidationError(
reth_payload_primitives::EngineObjectValidationError::UnsupportedFork,
))?
}
/// Handler for `engine_forkchoiceUpdatedV1`
/// See also <https://github.com/ethereum/execution-apis/blob/3d627c95a4d3510a8187dd02e0250ecb4331d27e/src/engine/paris.md#engine_forkchoiceupdatedv1>
///
@@ -1186,21 +1086,6 @@ where
Ok(self.get_payload_v5_metered(payload_id).await?)
}
/// Handler for `engine_getPayloadV6`
///
/// Post Amsterdam payload handler. Currently returns unsupported fork error.
///
/// See also <https://github.com/ethereum/execution-apis/blob/main/src/engine/amsterdam.md#engine_getpayloadv6>
async fn get_payload_v6(
&self,
_payload_id: PayloadId,
) -> RpcResult<EngineT::ExecutionPayloadEnvelopeV6> {
trace!(target: "rpc::engine", "Serving engine_getPayloadV6");
Err(EngineApiError::EngineObjectValidationError(
reth_payload_primitives::EngineObjectValidationError::UnsupportedFork,
))?
}
/// Handler for `engine_getPayloadBodiesByHashV1`
/// See also <https://github.com/ethereum/execution-apis/blob/6452a6b194d7db269bf1dbd087a267251d3cc7f8/src/engine/shanghai.md#engine_getpayloadbodiesbyhashv1>
async fn get_payload_bodies_by_hash_v1(
@@ -1287,10 +1172,12 @@ where
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
async fn get_bals_by_hash_v1(
&self,
block_hashes: Vec<BlockHash>,
_block_hashes: Vec<BlockHash>,
) -> RpcResult<Vec<alloy_primitives::Bytes>> {
trace!(target: "rpc::engine", "Serving engine_getBALsByHashV1");
Ok(self.get_bals_by_hash(block_hashes))
Err(EngineApiError::EngineObjectValidationError(
reth_payload_primitives::EngineObjectValidationError::UnsupportedFork,
))?
}
/// Handler for `engine_getBALsByRangeV1`
@@ -1298,11 +1185,13 @@ where
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
async fn get_bals_by_range_v1(
&self,
start: U64,
count: U64,
_start: U64,
_count: U64,
) -> RpcResult<Vec<alloy_primitives::Bytes>> {
trace!(target: "rpc::engine", "Serving engine_getBALsByRangeV1");
Ok(self.get_bals_by_range(start.to(), count.to()))
Err(EngineApiError::EngineObjectValidationError(
reth_payload_primitives::EngineObjectValidationError::UnsupportedFork,
))?
}
}
@@ -1362,8 +1251,6 @@ struct EngineApiInner<Provider, PayloadT: PayloadTypes, Pool, Validator, ChainSp
accept_execution_requests_hash: bool,
/// Returns `true` if the node is currently syncing.
is_syncing: Arc<dyn Fn() -> bool + Send + Sync>,
/// Cache for Block Access Lists (BALs) per EIP-7928.
bal_cache: BalCache,
}
#[cfg(test)]

View File

@@ -12,10 +12,6 @@
/// The Engine API implementation.
mod engine_api;
/// Block Access List (BAL) cache for EIP-7928.
mod bal_cache;
pub use bal_cache::BalCache;
/// Engine API capabilities.
pub mod capabilities;
pub use capabilities::EngineCapabilities;

View File

@@ -258,13 +258,11 @@ where
let call = match result {
ExecutionResult::Halt { reason, gas_used } => {
let error = Err::from_evm_halt(reason, tx.gas_limit());
#[allow(clippy::needless_update)]
SimCallResult {
return_data: Bytes::new(),
error: Some(SimulateError {
message: error.to_string(),
code: error.into().code(),
..SimulateError::invalid_params()
}),
gas_used,
logs: Vec::new(),
@@ -273,13 +271,11 @@ where
}
ExecutionResult::Revert { output, gas_used } => {
let error = Err::from_revert(output.clone());
#[allow(clippy::needless_update)]
SimCallResult {
return_data: output,
error: Some(SimulateError {
message: error.to_string(),
code: error.into().code(),
..SimulateError::invalid_params()
}),
gas_used,
status: false,

View File

@@ -483,12 +483,6 @@ where
.ok_or_else(|| ProviderError::HeaderNotFound(block_hash.into()))?
};
// Check if the block has been pruned (EIP-4444)
let earliest_block = self.provider().earliest_block_number()?;
if header.number() < earliest_block {
return Err(EthApiError::PrunedHistoryUnavailable.into());
}
let block_num_hash = BlockNumHash::new(header.number(), block_hash);
let mut all_logs = Vec::new();
@@ -571,12 +565,6 @@ where
let (from_block_number, to_block_number) =
logs_utils::get_filter_block_range(from, to, start_block, info)?;
// Check if the requested range overlaps with pruned history (EIP-4444)
let earliest_block = self.provider().earliest_block_number()?;
if from_block_number < earliest_block {
return Err(EthApiError::PrunedHistoryUnavailable.into());
}
self.get_logs_in_block_range(filter, from_block_number, to_block_number, limits)
.await
}

View File

@@ -316,7 +316,7 @@ impl<N: ProviderNodeTypes> Pipeline<N> {
let _locked_sf_producer = self.static_file_producer.lock();
let mut provider_rw =
self.provider_factory.unwind_provider_rw()?.disable_long_read_transaction_safety();
self.provider_factory.database_provider_rw()?.disable_long_read_transaction_safety();
for stage in unwind_pipeline {
let stage_id = stage.id();
@@ -396,7 +396,7 @@ impl<N: ProviderNodeTypes> Pipeline<N> {
stage.post_unwind_commit()?;
provider_rw = self.provider_factory.unwind_provider_rw()?;
provider_rw = self.provider_factory.database_provider_rw()?;
}
Err(err) => {
self.event_sender.notify(PipelineEvent::Error { stage_id });

View File

@@ -136,7 +136,7 @@ where
info!(target: "sync::stages::index_account_history::exec", "Loading indices into database");
provider.with_rocksdb_batch_auto_commit(|rocksdb_batch| {
provider.with_rocksdb_batch(|rocksdb_batch| {
let mut writer = EitherWriter::new_accounts_history(provider, rocksdb_batch)?;
load_account_history(collector, first_sync, &mut writer)
.map_err(|e| reth_provider::ProviderError::other(Box::new(e)))?;

View File

@@ -1,4 +1,4 @@
use super::{collect_history_indices, collect_storage_history_indices};
use super::collect_history_indices;
use crate::{stages::utils::load_storage_history, StageCheckpoint, StageId};
use reth_config::config::{EtlConfig, IndexHistoryConfig};
use reth_db_api::{
@@ -8,8 +8,7 @@ use reth_db_api::{
};
use reth_provider::{
DBProvider, EitherWriter, HistoryWriter, PruneCheckpointReader, PruneCheckpointWriter,
RocksDBProviderFactory, StaticFileProviderFactory, StorageChangeSetReader,
StorageSettingsCache,
RocksDBProviderFactory, StorageSettingsCache,
};
use reth_prune_types::{PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment};
use reth_stages_api::{ExecInput, ExecOutput, Stage, StageError, UnwindInput, UnwindOutput};
@@ -55,8 +54,6 @@ where
+ PruneCheckpointWriter
+ StorageSettingsCache
+ RocksDBProviderFactory
+ StorageChangeSetReader
+ StaticFileProviderFactory
+ reth_provider::NodePrimitivesProvider,
{
/// Return the id of the stage
@@ -124,9 +121,7 @@ where
}
info!(target: "sync::stages::index_storage_history::exec", ?first_sync, ?use_rocksdb, "Collecting indices");
let collector = if provider.cached_storage_settings().storage_changesets_in_static_files {
collect_storage_history_indices(provider, range.clone(), &self.etl_config)?
} else {
let collector =
collect_history_indices::<_, tables::StorageChangeSets, tables::StoragesHistory, _>(
provider,
BlockNumberAddress::range(range.clone()),
@@ -135,12 +130,11 @@ where
},
|(key, value)| (key.block_number(), AddressStorageKey((key.address(), value.key))),
&self.etl_config,
)?
};
)?;
info!(target: "sync::stages::index_storage_history::exec", "Loading indices into database");
provider.with_rocksdb_batch_auto_commit(|rocksdb_batch| {
provider.with_rocksdb_batch(|rocksdb_batch| {
let mut writer = EitherWriter::new_storages_history(provider, rocksdb_batch)?;
load_storage_history(collector, first_sync, &mut writer)
.map_err(|e| reth_provider::ProviderError::other(Box::new(e)))?;

View File

@@ -9,7 +9,7 @@ use reth_db_api::{
use reth_primitives_traits::{GotExpected, SealedHeader};
use reth_provider::{
ChangeSetReader, DBProvider, HeaderProvider, ProviderError, StageCheckpointReader,
StageCheckpointWriter, StatsReader, StorageChangeSetReader, TrieWriter,
StageCheckpointWriter, StatsReader, TrieWriter,
};
use reth_stages_api::{
BlockErrorKind, EntitiesCheckpoint, ExecInput, ExecOutput, MerkleCheckpoint, Stage,
@@ -159,7 +159,6 @@ where
+ StatsReader
+ HeaderProvider
+ ChangeSetReader
+ StorageChangeSetReader
+ StageCheckpointReader
+ StageCheckpointWriter,
{

View File

@@ -0,0 +1,447 @@
use crate::stages::merkle::INVALID_STATE_ROOT_ERROR_MESSAGE;
use alloy_consensus::BlockHeader;
use alloy_primitives::BlockNumber;
use reth_consensus::ConsensusError;
use reth_primitives_traits::{GotExpected, SealedHeader};
use reth_provider::{
BlockNumReader, ChainStateBlockReader, ChangeSetReader, DBProvider, HeaderProvider,
ProviderError, PruneCheckpointReader, PruneCheckpointWriter, StageCheckpointReader,
StageCheckpointWriter, TrieWriter,
};
use reth_prune_types::{
PruneCheckpoint, PruneMode, PruneSegment, MERKLE_CHANGESETS_RETENTION_BLOCKS,
};
use reth_stages_api::{
BlockErrorKind, ExecInput, ExecOutput, Stage, StageCheckpoint, StageError, StageId,
UnwindInput, UnwindOutput,
};
use reth_trie::{
updates::TrieUpdates, HashedPostStateSorted, KeccakKeyHasher, StateRoot, TrieInputSorted,
};
use reth_trie_db::{DatabaseHashedPostState, DatabaseStateRoot};
use std::{ops::Range, sync::Arc};
use tracing::{debug, error};
/// The `MerkleChangeSets` stage.
///
/// This stage processes and maintains trie changesets from the finalized block to the latest block.
#[derive(Debug, Clone)]
pub struct MerkleChangeSets {
/// The number of blocks to retain changesets for, used as a fallback when the finalized block
/// is not found. Defaults to [`MERKLE_CHANGESETS_RETENTION_BLOCKS`] (2 epochs in beacon
/// chain).
retention_blocks: u64,
}
impl MerkleChangeSets {
/// Creates a new `MerkleChangeSets` stage with the default retention blocks.
pub const fn new() -> Self {
Self { retention_blocks: MERKLE_CHANGESETS_RETENTION_BLOCKS }
}
/// Creates a new `MerkleChangeSets` stage with a custom finalized block height.
pub const fn with_retention_blocks(retention_blocks: u64) -> Self {
Self { retention_blocks }
}
/// Returns the range of blocks which are already computed. Will return an empty range if none
/// have been computed.
fn computed_range<Provider>(
provider: &Provider,
checkpoint: Option<StageCheckpoint>,
) -> Result<Range<BlockNumber>, StageError>
where
Provider: PruneCheckpointReader,
{
let to = checkpoint.map(|chk| chk.block_number).unwrap_or_default();
// Get the prune checkpoint for MerkleChangeSets to use as the lower bound. If there's no
// prune checkpoint or if the pruned block number is None, return empty range
let Some(from) = provider
.get_prune_checkpoint(PruneSegment::MerkleChangeSets)?
.and_then(|chk| chk.block_number)
// prune checkpoint indicates the last block pruned, so the block after is the start of
// the computed data
.map(|block_number| block_number + 1)
else {
return Ok(0..0)
};
Ok(from..to + 1)
}
/// Determines the target range for changeset computation based on the checkpoint and provider
/// state.
///
/// Returns the target range (exclusive end) to compute changesets for.
fn determine_target_range<Provider>(
&self,
provider: &Provider,
) -> Result<Range<BlockNumber>, StageError>
where
Provider: StageCheckpointReader + ChainStateBlockReader,
{
// Get merkle checkpoint which represents our target end block
let merkle_checkpoint = provider
.get_stage_checkpoint(StageId::MerkleExecute)?
.map(|checkpoint| checkpoint.block_number)
.unwrap_or(0);
let target_end = merkle_checkpoint + 1; // exclusive
// Calculate the target range based on the finalized block and the target block.
// We maintain changesets from the finalized block to the latest block.
let finalized_block = provider.last_finalized_block_number()?;
// Calculate the fallback start position based on retention blocks
let retention_based_start = merkle_checkpoint.saturating_sub(self.retention_blocks);
// If the finalized block was way in the past then we don't want to generate changesets for
// all of those past blocks; we only care about the recent history.
//
// Use maximum of finalized_block and retention_based_start if finalized_block exists,
// otherwise just use retention_based_start.
let mut target_start = finalized_block
.map(|finalized| finalized.saturating_add(1).max(retention_based_start))
.unwrap_or(retention_based_start);
// We cannot revert the genesis block; target_start must be >0
target_start = target_start.max(1);
Ok(target_start..target_end)
}
/// Calculates the trie updates given a [`TrieInputSorted`], asserting that the resulting state
/// root matches the expected one for the block.
fn calculate_block_trie_updates<Provider: DBProvider + HeaderProvider>(
provider: &Provider,
block_number: BlockNumber,
input: TrieInputSorted,
) -> Result<TrieUpdates, StageError> {
let (root, trie_updates) =
StateRoot::overlay_root_from_nodes_with_updates(provider.tx_ref(), input).map_err(
|e| {
error!(
target: "sync::stages::merkle_changesets",
%e,
?block_number,
"Incremental state root failed! {INVALID_STATE_ROOT_ERROR_MESSAGE}");
StageError::Fatal(Box::new(e))
},
)?;
let block = provider
.header_by_number(block_number)?
.ok_or_else(|| ProviderError::HeaderNotFound(block_number.into()))?;
let (got, expected) = (root, block.state_root());
if got != expected {
// Only seal the header when we need it for the error
let header = SealedHeader::seal_slow(block);
error!(
target: "sync::stages::merkle_changesets",
?block_number,
?got,
?expected,
"Failed to verify block state root! {INVALID_STATE_ROOT_ERROR_MESSAGE}",
);
return Err(StageError::Block {
error: BlockErrorKind::Validation(ConsensusError::BodyStateRootDiff(
GotExpected { got, expected }.into(),
)),
block: Box::new(header.block_with_parent()),
})
}
Ok(trie_updates)
}
fn populate_range<Provider>(
provider: &Provider,
target_range: Range<BlockNumber>,
) -> Result<(), StageError>
where
Provider: StageCheckpointReader
+ TrieWriter
+ DBProvider
+ HeaderProvider
+ ChainStateBlockReader
+ BlockNumReader
+ ChangeSetReader,
{
let target_start = target_range.start;
let target_end = target_range.end;
debug!(
target: "sync::stages::merkle_changesets",
?target_range,
"Starting trie changeset computation",
);
// We need to distinguish a cumulative revert and a per-block revert. A cumulative revert
// reverts changes starting at db tip all the way to a block. A per-block revert only
// reverts a block's changes.
//
// We need to calculate the cumulative HashedPostState reverts for every block in the
// target range. The cumulative HashedPostState revert for block N can be calculated as:
//
//
// ```
// // where `extend` overwrites any shared keys
// cumulative_state_revert(N) = cumulative_state_revert(N + 1).extend(get_block_state_revert(N))
// ```
//
// We need per-block reverts to calculate the prefix set for each individual block. By
// using the per-block reverts to calculate cumulative reverts on-the-fly we can save a
// bunch of memory.
debug!(
target: "sync::stages::merkle_changesets",
?target_range,
"Computing per-block state reverts",
);
let range_len = target_end - target_start;
let mut per_block_state_reverts = Vec::with_capacity(range_len as usize);
for block_number in target_range.clone() {
per_block_state_reverts.push(HashedPostStateSorted::from_reverts::<KeccakKeyHasher>(
provider,
block_number..=block_number,
)?);
}
// Helper to retrieve state revert data for a specific block from the pre-computed array
let get_block_state_revert = |block_number: BlockNumber| -> &HashedPostStateSorted {
let index = (block_number - target_start) as usize;
&per_block_state_reverts[index]
};
// Helper to accumulate state reverts from a given block to the target end
let compute_cumulative_state_revert = |block_number: BlockNumber| -> HashedPostStateSorted {
let mut cumulative_revert = HashedPostStateSorted::default();
for n in (block_number..target_end).rev() {
cumulative_revert.extend_ref_and_sort(get_block_state_revert(n))
}
cumulative_revert
};
// To calculate the changeset for a block, we first need the TrieUpdates which are
// generated as a result of processing the block. To get these we need:
// 1) The TrieUpdates which revert the db's trie to _prior_ to the block
// 2) The HashedPostStateSorted to revert the db's state to _after_ the block
//
// To get (1) for `target_start` we need to do a big state root calculation which takes
// into account all changes between that block and db tip. For each block after the
// `target_start` we can update (1) using the TrieUpdates which were output by the previous
// block, only targeting the state changes of that block.
debug!(
target: "sync::stages::merkle_changesets",
?target_start,
"Computing trie state at starting block",
);
let initial_state = compute_cumulative_state_revert(target_start);
let initial_prefix_sets = initial_state.construct_prefix_sets();
let initial_input =
TrieInputSorted::new(Arc::default(), Arc::new(initial_state), initial_prefix_sets);
// target_start will be >= 1, see `determine_target_range`.
let mut nodes = Arc::new(
Self::calculate_block_trie_updates(provider, target_start - 1, initial_input)?
.into_sorted(),
);
for block_number in target_range {
debug!(
target: "sync::stages::merkle_changesets",
?block_number,
"Computing trie updates for block",
);
// Revert the state so that this block has been just processed, meaning we take the
// cumulative revert of the subsequent block.
let state = Arc::new(compute_cumulative_state_revert(block_number + 1));
// Construct prefix sets from only this block's `HashedPostStateSorted`, because we only
// care about trie updates which occurred as a result of this block being processed.
let prefix_sets = get_block_state_revert(block_number).construct_prefix_sets();
let input = TrieInputSorted::new(Arc::clone(&nodes), state, prefix_sets);
// Calculate the trie updates for this block, then apply those updates to the reverts.
// We calculate the overlay which will be passed into the next step using the trie
// reverts prior to them being updated.
let this_trie_updates =
Self::calculate_block_trie_updates(provider, block_number, input)?.into_sorted();
let trie_overlay = Arc::clone(&nodes);
let mut nodes_mut = Arc::unwrap_or_clone(nodes);
nodes_mut.extend_ref_and_sort(&this_trie_updates);
nodes = Arc::new(nodes_mut);
// Write the changesets to the DB using the trie updates produced by the block, and the
// trie reverts as the overlay.
debug!(
target: "sync::stages::merkle_changesets",
?block_number,
"Writing trie changesets for block",
);
provider.write_trie_changesets(
block_number,
&this_trie_updates,
Some(&trie_overlay),
)?;
}
Ok(())
}
}
impl Default for MerkleChangeSets {
fn default() -> Self {
Self::new()
}
}
impl<Provider> Stage<Provider> for MerkleChangeSets
where
Provider: StageCheckpointReader
+ TrieWriter
+ DBProvider
+ HeaderProvider
+ ChainStateBlockReader
+ StageCheckpointWriter
+ PruneCheckpointReader
+ PruneCheckpointWriter
+ ChangeSetReader
+ BlockNumReader,
{
fn id(&self) -> StageId {
StageId::MerkleChangeSets
}
fn execute(&mut self, provider: &Provider, input: ExecInput) -> Result<ExecOutput, StageError> {
// Get merkle checkpoint and assert that the target is the same.
let merkle_checkpoint = provider
.get_stage_checkpoint(StageId::MerkleExecute)?
.map(|checkpoint| checkpoint.block_number)
.unwrap_or(0);
if input.target.is_none_or(|target| merkle_checkpoint != target) {
return Err(StageError::Fatal(eyre::eyre!("Cannot sync stage to block {:?} when MerkleExecute is at block {merkle_checkpoint:?}", input.target).into()))
}
let mut target_range = self.determine_target_range(provider)?;
// Get the previously computed range. This will be updated to reflect the populating of the
// target range.
let mut computed_range = Self::computed_range(provider, input.checkpoint)?;
debug!(
target: "sync::stages::merkle_changesets",
?computed_range,
?target_range,
"Got computed and target ranges",
);
// We want the target range to not include any data already computed previously, if
// possible, so we start the target range from the end of the computed range if that is
// greater.
//
// ------------------------------> Block #
// |------computed-----|
// |-----target-----|
// |--actual--|
//
// However, if the target start is less than the previously computed start, we don't want to
// do this, as it would leave a gap of data at `target_range.start..=computed_range.start`.
//
// ------------------------------> Block #
// |---computed---|
// |-------target-------|
// |-------actual-------|
//
if target_range.start >= computed_range.start {
target_range.start = target_range.start.max(computed_range.end);
}
// If target range is empty (target_start >= target_end), stage is already successfully
// executed.
if target_range.start >= target_range.end {
return Ok(ExecOutput::done(StageCheckpoint::new(target_range.end.saturating_sub(1))));
}
// If our target range is a continuation of the already computed range then we can keep the
// already computed data.
if target_range.start == computed_range.end {
// Clear from target_start onwards to ensure no stale data exists
provider.clear_trie_changesets_from(target_range.start)?;
computed_range.end = target_range.end;
} else {
// If our target range is not a continuation of the already computed range then we
// simply clear the computed data, to make sure there's no gaps or conflicts.
provider.clear_trie_changesets()?;
computed_range = target_range.clone();
}
// Populate the target range with changesets
Self::populate_range(provider, target_range)?;
// Update the prune checkpoint to reflect that all data before `computed_range.start`
// is not available.
provider.save_prune_checkpoint(
PruneSegment::MerkleChangeSets,
PruneCheckpoint {
block_number: Some(computed_range.start.saturating_sub(1)),
tx_number: None,
prune_mode: PruneMode::Before(computed_range.start),
},
)?;
// `computed_range.end` is exclusive.
let checkpoint = StageCheckpoint::new(computed_range.end.saturating_sub(1));
Ok(ExecOutput::done(checkpoint))
}
fn unwind(
&mut self,
provider: &Provider,
input: UnwindInput,
) -> Result<UnwindOutput, StageError> {
// Unwinding is trivial; just clear everything after the target block.
provider.clear_trie_changesets_from(input.unwind_to + 1)?;
let mut computed_range = Self::computed_range(provider, Some(input.checkpoint))?;
computed_range.end = input.unwind_to + 1;
if computed_range.start > computed_range.end {
computed_range.start = computed_range.end;
}
// If we've unwound so far that there are no longer enough trie changesets available then
// simply clear them and the checkpoints, so that on next pipeline startup they will be
// regenerated.
//
// We don't do this check if the target block is not greater than the retention threshold
// (which happens near genesis), as in that case would could still have all possible
// changesets even if the total count doesn't meet the threshold.
debug!(
target: "sync::stages::merkle_changesets",
?computed_range,
retention_blocks=?self.retention_blocks,
"Checking if computed range is over retention threshold",
);
if input.unwind_to > self.retention_blocks &&
computed_range.end - computed_range.start < self.retention_blocks
{
debug!(
target: "sync::stages::merkle_changesets",
?computed_range,
retention_blocks=?self.retention_blocks,
"Clearing checkpoints completely",
);
provider.clear_trie_changesets()?;
provider
.save_stage_checkpoint(StageId::MerkleChangeSets, StageCheckpoint::default())?;
return Ok(UnwindOutput { checkpoint: StageCheckpoint::default() })
}
// `computed_range.end` is exclusive
let checkpoint = StageCheckpoint::new(computed_range.end.saturating_sub(1));
Ok(UnwindOutput { checkpoint })
}
}

View File

@@ -1,6 +1,5 @@
/// The bodies stage.
mod bodies;
mod era;
/// The execution stage that generates state diff.
mod execution;
/// The finish stage
@@ -37,7 +36,9 @@ pub use prune::*;
pub use sender_recovery::*;
pub use tx_lookup::*;
mod era;
mod utils;
use utils::*;
#[cfg(test)]

View File

@@ -158,13 +158,15 @@ where
let append_only =
provider.count_entries::<tables::TransactionHashNumbers>()?.is_zero();
// Auto-commits on threshold; consistency check heals any crash.
// Create RocksDB batch if feature is enabled
#[cfg(all(unix, feature = "rocksdb"))]
let rocksdb = provider.rocksdb_provider();
#[cfg(all(unix, feature = "rocksdb"))]
let rocksdb_batch = rocksdb.batch_with_auto_commit();
let rocksdb_batch = rocksdb.batch();
#[cfg(not(all(unix, feature = "rocksdb")))]
let rocksdb_batch = ();
// Create writer that routes to either MDBX or RocksDB based on settings
let mut writer =
EitherWriter::new_transaction_hash_numbers(provider, rocksdb_batch)?;
@@ -215,12 +217,15 @@ where
) -> Result<UnwindOutput, StageError> {
let (range, unwind_to, _) = input.unwind_block_range_with_threshold(self.chunk_size);
// Create RocksDB batch if feature is enabled
#[cfg(all(unix, feature = "rocksdb"))]
let rocksdb = provider.rocksdb_provider();
#[cfg(all(unix, feature = "rocksdb"))]
let rocksdb_batch = rocksdb.batch();
#[cfg(not(all(unix, feature = "rocksdb")))]
let rocksdb_batch = ();
// Create writer that routes to either MDBX or RocksDB based on settings
let mut writer = EitherWriter::new_transaction_hash_numbers(provider, rocksdb_batch)?;
let static_file_provider = provider.static_file_provider();

View File

@@ -5,7 +5,7 @@ use reth_db_api::{
cursor::{DbCursorRO, DbCursorRW},
models::{
sharded_key::NUM_OF_INDICES_IN_SHARD, storage_sharded_key::StorageShardedKey,
AccountBeforeTx, AddressStorageKey, BlockNumberAddress, ShardedKey,
AccountBeforeTx, ShardedKey,
},
table::{Decode, Decompress, Table},
transaction::DbTx,
@@ -19,7 +19,7 @@ use reth_provider::{
};
use reth_stages_api::StageError;
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader};
use reth_storage_api::ChangeSetReader;
use std::{collections::HashMap, hash::Hash, ops::RangeBounds};
use tracing::info;
@@ -102,15 +102,15 @@ where
}
/// Allows collecting indices from a cache with a custom insert fn
fn collect_indices<K, F>(
cache: impl Iterator<Item = (K, Vec<u64>)>,
fn collect_indices<F>(
cache: impl Iterator<Item = (Address, Vec<u64>)>,
mut insert_fn: F,
) -> Result<(), StageError>
where
F: FnMut(K, Vec<u64>) -> Result<(), StageError>,
F: FnMut(Address, Vec<u64>) -> Result<(), StageError>,
{
for (key, indices) in cache {
insert_fn(key, indices)?
for (address, indices) in cache {
insert_fn(address, indices)?
}
Ok(())
}
@@ -174,62 +174,6 @@ where
Ok(collector)
}
/// Collects storage history indices using a provider that implements `StorageChangeSetReader`.
pub(crate) fn collect_storage_history_indices<Provider>(
provider: &Provider,
range: impl RangeBounds<BlockNumber>,
etl_config: &EtlConfig,
) -> Result<Collector<StorageShardedKey, BlockNumberList>, StageError>
where
Provider: DBProvider + StorageChangeSetReader + StaticFileProviderFactory,
{
let mut collector = Collector::new(etl_config.file_size, etl_config.dir.clone());
let mut cache: HashMap<AddressStorageKey, Vec<u64>> = HashMap::default();
let mut insert_fn = |key: AddressStorageKey, indices: Vec<u64>| {
let last = indices.last().expect("qed");
collector.insert(
StorageShardedKey::new(key.0 .0, key.0 .1, *last),
BlockNumberList::new_pre_sorted(indices.into_iter()),
)?;
Ok::<(), StageError>(())
};
let range = to_range(range);
let static_file_provider = provider.static_file_provider();
let total_changesets = static_file_provider.storage_changeset_count()?;
let interval = (total_changesets / 1000).max(1);
let walker = static_file_provider.walk_storage_changeset_range(range);
let mut flush_counter = 0;
let mut current_block_number = u64::MAX;
for (idx, changeset_result) in walker.enumerate() {
let (BlockNumberAddress((block_number, address)), storage) = changeset_result?;
cache.entry(AddressStorageKey((address, storage.key))).or_default().push(block_number);
if idx > 0 && idx % interval == 0 && total_changesets > 1000 {
info!(target: "sync::stages::index_history", progress = %format!("{:.4}%", (idx as f64 / total_changesets as f64) * 100.0), "Collecting indices");
}
if block_number != current_block_number {
current_block_number = block_number;
flush_counter += 1;
}
if flush_counter > DEFAULT_CACHE_THRESHOLD {
collect_indices(cache.drain(), &mut insert_fn)?;
flush_counter = 0;
}
}
collect_indices(cache.into_iter(), insert_fn)?;
Ok(collector)
}
/// Loads account history indices into the database via `EitherWriter`.
///
/// Works with [`EitherWriter`] to support both MDBX and `RocksDB` backends.

View File

@@ -12,10 +12,6 @@ pub enum StageId {
note = "Static Files are generated outside of the pipeline and do not require a separate stage"
)]
StaticFile,
#[deprecated(
note = "MerkleChangeSets stage has been removed; kept for DB checkpoint compatibility"
)]
MerkleChangeSets,
Era,
Headers,
Bodies,
@@ -79,8 +75,6 @@ impl StageId {
match self {
#[expect(deprecated)]
Self::StaticFile => "StaticFile",
#[expect(deprecated)]
Self::MerkleChangeSets => "MerkleChangeSets",
Self::Era => "Era",
Self::Headers => "Headers",
Self::Bodies => "Bodies",

View File

@@ -55,11 +55,6 @@ pub enum StaticFileSegment {
/// * address 0xbb, account info
/// * address 0xcc, account info
AccountChangeSets,
/// Static File segment responsible for the `StorageChangeSets` table.
///
/// Storage changeset static files append block-by-block changesets sorted by address and
/// storage slot.
StorageChangeSets,
}
impl StaticFileSegment {
@@ -76,7 +71,6 @@ impl StaticFileSegment {
Self::Receipts => "receipts",
Self::TransactionSenders => "transaction-senders",
Self::AccountChangeSets => "account-change-sets",
Self::StorageChangeSets => "storage-change-sets",
}
}
@@ -89,7 +83,6 @@ impl StaticFileSegment {
Self::Receipts,
Self::TransactionSenders,
Self::AccountChangeSets,
Self::StorageChangeSets,
]
.into_iter()
}
@@ -106,8 +99,7 @@ impl StaticFileSegment {
Self::Transactions |
Self::Receipts |
Self::TransactionSenders |
Self::AccountChangeSets |
Self::StorageChangeSets => 1,
Self::AccountChangeSets => 1,
}
}
@@ -169,14 +161,14 @@ impl StaticFileSegment {
pub const fn is_tx_based(&self) -> bool {
match self {
Self::Receipts | Self::Transactions | Self::TransactionSenders => true,
Self::Headers | Self::AccountChangeSets | Self::StorageChangeSets => false,
Self::Headers | Self::AccountChangeSets => false,
}
}
/// Returns `true` if the segment is change-based.
/// Returns `true` if the segment is [`StaticFileSegment::AccountChangeSets`]
pub const fn is_change_based(&self) -> bool {
match self {
Self::AccountChangeSets | Self::StorageChangeSets => true,
Self::AccountChangeSets => true,
Self::Receipts | Self::Transactions | Self::Headers | Self::TransactionSenders => false,
}
}
@@ -188,8 +180,7 @@ impl StaticFileSegment {
Self::Receipts |
Self::Transactions |
Self::TransactionSenders |
Self::AccountChangeSets |
Self::StorageChangeSets => false,
Self::AccountChangeSets => false,
}
}
@@ -268,10 +259,10 @@ impl<'de> Visitor<'de> for SegmentHeaderVisitor {
let tx_range =
seq.next_element()?.ok_or_else(|| serde::de::Error::invalid_length(2, &self))?;
let segment: StaticFileSegment =
let segment =
seq.next_element()?.ok_or_else(|| serde::de::Error::invalid_length(3, &self))?;
let changeset_offsets = if segment.is_change_based() {
let changeset_offsets = if segment == StaticFileSegment::AccountChangeSets {
// Try to read the 5th field (changeset_offsets)
// If it doesn't exist (old format), this will return None
match seq.next_element()? {
@@ -318,8 +309,8 @@ impl Serialize for SegmentHeader {
where
S: Serializer,
{
// We serialize an extra field, the changeset offsets, for change-based segments
let len = if self.segment.is_change_based() { 5 } else { 4 };
// We serialize an extra field, the changeset offsets, for account changesets
let len = if self.segment.is_account_change_sets() { 5 } else { 4 };
let mut state = serializer.serialize_struct("SegmentHeader", len)?;
state.serialize_field("expected_block_range", &self.expected_block_range)?;
@@ -327,7 +318,7 @@ impl Serialize for SegmentHeader {
state.serialize_field("tx_range", &self.tx_range)?;
state.serialize_field("segment", &self.segment)?;
if self.segment.is_change_based() {
if self.segment.is_account_change_sets() {
state.serialize_field("changeset_offsets", &self.changeset_offsets)?;
}
@@ -681,12 +672,6 @@ mod tests {
"static_file_account-change-sets_1123233_11223233",
None,
),
(
StaticFileSegment::StorageChangeSets,
1_123_233..=11_223_233,
"static_file_storage-change-sets_1123233_11223233",
None,
),
(
StaticFileSegment::Headers,
2..=30,
@@ -770,13 +755,6 @@ mod tests {
segment: StaticFileSegment::AccountChangeSets,
changeset_offsets: Some(vec![ChangesetOffset { offset: 1, num_changes: 1 }; 100]),
},
SegmentHeader {
expected_block_range: SegmentRangeInclusive::new(0, 200),
block_range: Some(SegmentRangeInclusive::new(0, 100)),
tx_range: None,
segment: StaticFileSegment::StorageChangeSets,
changeset_offsets: Some(vec![ChangesetOffset { offset: 1, num_changes: 1 }; 100]),
},
];
// Check that we test all segments
assert_eq!(
@@ -810,7 +788,6 @@ mod tests {
StaticFileSegment::Receipts => "receipts",
StaticFileSegment::TransactionSenders => "transaction-senders",
StaticFileSegment::AccountChangeSets => "account-change-sets",
StaticFileSegment::StorageChangeSets => "storage-change-sets",
};
assert_eq!(static_str, expected_str);
}
@@ -829,7 +806,6 @@ mod tests {
StaticFileSegment::Receipts => "Receipts",
StaticFileSegment::TransactionSenders => "TransactionSenders",
StaticFileSegment::AccountChangeSets => "AccountChangeSets",
StaticFileSegment::StorageChangeSets => "StorageChangeSets",
};
assert_eq!(ser, format!("\"{expected_str}\""));
}

View File

@@ -1,5 +0,0 @@
---
source: crates/static-file/types/src/segment.rs
expression: "Bytes::from(serialized)"
---
0x01000000000000000000000000000000c800000000000000010000000000000000640000000000000000050000000164000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000100000000000000010000000000000001000000000000000000000000000000000000000000000000

View File

@@ -5,7 +5,7 @@ use crate::{
table::{Decode, Encode},
DatabaseError,
};
use alloy_primitives::{Address, BlockNumber, StorageKey};
use alloy_primitives::{Address, BlockNumber, StorageKey, B256};
use serde::{Deserialize, Serialize};
use std::ops::{Bound, Range, RangeBounds, RangeInclusive};
@@ -108,6 +108,43 @@ impl<R: RangeBounds<BlockNumber>> From<R> for BlockNumberAddressRange {
}
}
/// [`BlockNumber`] concatenated with [`B256`] (hashed address).
///
/// Since it's used as a key, it isn't compressed when encoding it.
#[derive(
Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Ord, PartialOrd, Hash,
)]
pub struct BlockNumberHashedAddress(pub (BlockNumber, B256));
impl From<(BlockNumber, B256)> for BlockNumberHashedAddress {
fn from(tpl: (BlockNumber, B256)) -> Self {
Self(tpl)
}
}
impl Encode for BlockNumberHashedAddress {
type Encoded = [u8; 40];
fn encode(self) -> Self::Encoded {
let block_number = self.0 .0;
let hashed_address = self.0 .1;
let mut buf = [0u8; 40];
buf[..8].copy_from_slice(&block_number.to_be_bytes());
buf[8..].copy_from_slice(hashed_address.as_slice());
buf
}
}
impl Decode for BlockNumberHashedAddress {
fn decode(value: &[u8]) -> Result<Self, DatabaseError> {
let num = u64::from_be_bytes(value[..8].try_into().map_err(|_| DatabaseError::Decode)?);
let hash = B256::from_slice(&value[8..]);
Ok(Self((num, hash)))
}
}
/// [`Address`] concatenated with [`StorageKey`]. Used by `reth_etl` and history stages.
///
/// Since it's used as a key, it isn't compressed when encoding it.
@@ -139,7 +176,11 @@ impl Decode for AddressStorageKey {
}
}
impl_fixed_arbitrary!((BlockNumberAddress, 28), (AddressStorageKey, 52));
impl_fixed_arbitrary!(
(BlockNumberAddress, 28),
(BlockNumberHashedAddress, 40),
(AddressStorageKey, 52)
);
#[cfg(test)]
mod tests {
@@ -172,6 +213,31 @@ mod tests {
assert_eq!(bytes, Encode::encode(key));
}
#[test]
fn test_block_number_hashed_address() {
let num = 1u64;
let hash = B256::from_slice(&[0xba; 32]);
let key = BlockNumberHashedAddress((num, hash));
let mut bytes = [0u8; 40];
bytes[..8].copy_from_slice(&num.to_be_bytes());
bytes[8..].copy_from_slice(hash.as_slice());
let encoded = Encode::encode(key);
assert_eq!(encoded, bytes);
let decoded: BlockNumberHashedAddress = Decode::decode(&encoded).unwrap();
assert_eq!(decoded, key);
}
#[test]
fn test_block_number_hashed_address_rand() {
let mut bytes = [0u8; 40];
rng().fill(bytes.as_mut_slice());
let key = BlockNumberHashedAddress::arbitrary(&mut Unstructured::new(&bytes)).unwrap();
assert_eq!(bytes, Encode::encode(key));
}
#[test]
fn test_address_storage_key() {
let storage_key = StorageKey::random();

View File

@@ -31,9 +31,6 @@ pub struct StorageSettings {
/// Whether this node should read and write account changesets from static files.
#[serde(default)]
pub account_changesets_in_static_files: bool,
/// Whether this node should read and write storage changesets from static files.
#[serde(default)]
pub storage_changesets_in_static_files: bool,
}
impl StorageSettings {
@@ -62,7 +59,6 @@ impl StorageSettings {
receipts_in_static_files: true,
transaction_senders_in_static_files: true,
account_changesets_in_static_files: true,
storage_changesets_in_static_files: true,
storages_history_in_rocksdb: false,
transaction_hash_numbers_in_rocksdb: true,
account_history_in_rocksdb: false,
@@ -82,7 +78,6 @@ impl StorageSettings {
transaction_hash_numbers_in_rocksdb: false,
account_history_in_rocksdb: false,
account_changesets_in_static_files: false,
storage_changesets_in_static_files: false,
}
}
@@ -122,12 +117,6 @@ impl StorageSettings {
self
}
/// Sets the `storage_changesets_in_static_files` flag to the provided value.
pub const fn with_storage_changesets_in_static_files(mut self, value: bool) -> Self {
self.storage_changesets_in_static_files = value;
self
}
/// Returns `true` if any tables are configured to be stored in `RocksDB`.
pub const fn any_in_rocksdb(&self) -> bool {
self.transaction_hash_numbers_in_rocksdb ||

View File

@@ -12,7 +12,9 @@ use reth_ethereum_primitives::{Receipt, TransactionSigned, TxType};
use reth_primitives_traits::{Account, Bytecode, StorageEntry};
use reth_prune_types::{PruneCheckpoint, PruneSegment};
use reth_stages_types::StageCheckpoint;
use reth_trie_common::{StorageTrieEntry, StoredNibbles, StoredNibblesSubKey, *};
use reth_trie_common::{
StorageTrieEntry, StoredNibbles, StoredNibblesSubKey, TrieChangeSetsEntry, *,
};
use serde::{Deserialize, Serialize};
pub mod accounts;
@@ -27,8 +29,8 @@ pub use blocks::*;
pub use integer_list::IntegerList;
pub use metadata::*;
pub use reth_db_models::{
AccountBeforeTx, ClientVersion, StaticFileBlockWithdrawals, StorageBeforeTx,
StoredBlockBodyIndices, StoredBlockWithdrawals,
AccountBeforeTx, ClientVersion, StaticFileBlockWithdrawals, StoredBlockBodyIndices,
StoredBlockWithdrawals,
};
pub use sharded_key::ShardedKey;
@@ -218,6 +220,7 @@ impl_compression_for_compact!(
TxType,
StorageEntry,
BranchNodeCompact,
TrieChangeSetsEntry,
StoredNibbles,
StoredNibblesSubKey,
StorageTrieEntry,
@@ -227,7 +230,6 @@ impl_compression_for_compact!(
StaticFileBlockWithdrawals,
Bytecode,
AccountBeforeTx,
StorageBeforeTx,
TransactionSigned,
CompactU256,
StageCheckpoint,

View File

@@ -21,8 +21,8 @@ use crate::{
accounts::BlockNumberAddress,
blocks::{HeaderHash, StoredBlockOmmers},
storage_sharded_key::StorageShardedKey,
AccountBeforeTx, ClientVersion, CompactU256, IntegerList, ShardedKey,
StoredBlockBodyIndices, StoredBlockWithdrawals,
AccountBeforeTx, BlockNumberHashedAddress, ClientVersion, CompactU256, IntegerList,
ShardedKey, StoredBlockBodyIndices, StoredBlockWithdrawals,
},
table::{Decode, DupSort, Encode, Table, TableInfo},
};
@@ -32,7 +32,9 @@ use reth_ethereum_primitives::{Receipt, TransactionSigned};
use reth_primitives_traits::{Account, Bytecode, StorageEntry};
use reth_prune_types::{PruneCheckpoint, PruneSegment};
use reth_stages_types::StageCheckpoint;
use reth_trie_common::{BranchNodeCompact, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey};
use reth_trie_common::{
BranchNodeCompact, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey, TrieChangeSetsEntry,
};
use serde::{Deserialize, Serialize};
use std::fmt;
@@ -490,6 +492,20 @@ tables! {
type SubKey = StoredNibblesSubKey;
}
/// Stores the state of a node in the accounts trie prior to a particular block being executed.
table AccountsTrieChangeSets {
type Key = BlockNumber;
type Value = TrieChangeSetsEntry;
type SubKey = StoredNibblesSubKey;
}
/// Stores the state of a node in a storage trie prior to a particular block being executed.
table StoragesTrieChangeSets {
type Key = BlockNumberHashedAddress;
type Value = TrieChangeSetsEntry;
type SubKey = StoredNibblesSubKey;
}
/// Stores the transaction sender for each canonical transaction.
/// It is needed to speed up execution stage and allows fetching signer without doing
/// transaction signed recovery

View File

@@ -15,10 +15,9 @@ use reth_primitives_traits::{
use reth_provider::{
errors::provider::ProviderResult, providers::StaticFileWriter, BlockHashReader, BlockNumReader,
BundleStateInit, ChainSpecProvider, DBProvider, DatabaseProviderFactory, ExecutionOutcome,
HashingWriter, HeaderProvider, HistoryWriter, MetadataProvider, MetadataWriter,
OriginalValuesKnown, ProviderError, RevertsInit, StageCheckpointReader, StageCheckpointWriter,
StateWriteConfig, StateWriter, StaticFileProviderFactory, StorageSettings,
StorageSettingsCache, TrieWriter,
HashingWriter, HeaderProvider, HistoryWriter, MetadataWriter, OriginalValuesKnown,
ProviderError, RevertsInit, StageCheckpointReader, StageCheckpointWriter, StateWriteConfig,
StateWriter, StaticFileProviderFactory, StorageSettings, StorageSettingsCache, TrieWriter,
};
use reth_stages_types::{StageCheckpoint, StageId};
use reth_static_file_types::StaticFileSegment;
@@ -29,7 +28,7 @@ use reth_trie::{
use reth_trie_db::DatabaseStateRoot;
use serde::{Deserialize, Serialize};
use std::io::BufRead;
use tracing::{debug, error, info, trace, warn};
use tracing::{debug, error, info, trace};
/// Default soft limit for number of bytes to read from state dump file, before inserting into
/// database.
@@ -91,8 +90,7 @@ where
+ StaticFileProviderFactory<Primitives: NodePrimitives<BlockHeader: Compact>>
+ ChainSpecProvider
+ StageCheckpointReader
+ BlockNumReader
+ MetadataProvider
+ BlockHashReader
+ StorageSettingsCache,
PF::ProviderRW: StaticFileProviderFactory<Primitives = PF::Primitives>
+ StageCheckpointWriter
@@ -126,8 +124,7 @@ where
+ StaticFileProviderFactory<Primitives: NodePrimitives<BlockHeader: Compact>>
+ ChainSpecProvider
+ StageCheckpointReader
+ BlockNumReader
+ MetadataProvider
+ BlockHashReader
+ StorageSettingsCache,
PF::ProviderRW: StaticFileProviderFactory<Primitives = PF::Primitives>
+ StageCheckpointWriter
@@ -162,16 +159,6 @@ where
return Err(InitStorageError::UninitializedDatabase)
}
let stored = factory.storage_settings()?.unwrap_or_else(StorageSettings::legacy);
if stored != genesis_storage_settings {
warn!(
target: "reth::storage",
?stored,
requested = ?genesis_storage_settings,
"Storage settings mismatch detected"
);
}
debug!("Genesis already written, skipping.");
return Ok(hash)
}
@@ -910,30 +897,4 @@ mod tests {
)],
);
}
#[test]
fn warn_storage_settings_mismatch() {
let factory = create_test_provider_factory_with_chain_spec(MAINNET.clone());
init_genesis_with_settings(&factory, StorageSettings::legacy()).unwrap();
// Request different settings - should warn but succeed
let result = init_genesis_with_settings(
&factory,
StorageSettings::legacy().with_receipts_in_static_files(true),
);
// Should succeed (warning is logged, not an error)
assert!(result.is_ok());
}
#[test]
fn allow_same_storage_settings() {
let factory = create_test_provider_factory_with_chain_spec(MAINNET.clone());
let settings = StorageSettings::legacy().with_receipts_in_static_files(true);
init_genesis_with_settings(&factory, settings).unwrap();
let result = init_genesis_with_settings(&factory, settings);
assert!(result.is_ok());
}
}

View File

@@ -19,10 +19,6 @@ pub use accounts::AccountBeforeTx;
pub mod blocks;
pub use blocks::{StaticFileBlockWithdrawals, StoredBlockBodyIndices, StoredBlockWithdrawals};
/// Storage
pub mod storage;
pub use storage::StorageBeforeTx;
/// Client Version
pub mod client_version;
pub use client_version::ClientVersion;

View File

@@ -1,48 +0,0 @@
use alloy_primitives::{Address, B256, U256};
use reth_primitives_traits::ValueWithSubKey;
/// Storage entry as it is saved in the static files.
///
/// [`B256`] is the subkey.
#[derive(Debug, Default, Copy, Clone, Eq, PartialEq)]
#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[cfg_attr(any(test, feature = "reth-codec"), reth_codecs::add_arbitrary_tests(compact))]
pub struct StorageBeforeTx {
/// Address for the storage entry. Acts as `DupSort::SubKey` in static files.
pub address: Address,
/// Storage key.
pub key: B256,
/// Value on storage key.
pub value: U256,
}
impl ValueWithSubKey for StorageBeforeTx {
type SubKey = B256;
fn get_subkey(&self) -> Self::SubKey {
self.key
}
}
// NOTE: Removing reth_codec and manually encode subkey
// and compress second part of the value. If we have compression
// over whole value (Even SubKey) that would mess up fetching of values with seek_by_key_subkey
#[cfg(any(test, feature = "reth-codec"))]
impl reth_codecs::Compact for StorageBeforeTx {
fn to_compact<B>(&self, buf: &mut B) -> usize
where
B: bytes::BufMut + AsMut<[u8]>,
{
buf.put_slice(self.address.as_slice());
buf.put_slice(&self.key[..]);
self.value.to_compact(buf) + 52
}
fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) {
let address = Address::from_slice(&buf[..20]);
let key = B256::from_slice(&buf[20..52]);
let (value, out) = U256::from_compact(&buf[52..], len - 52);
(Self { address, key, value }, out)
}
}

View File

@@ -560,35 +560,6 @@ impl DatabaseEnv {
Ok(handles)
}
/// Drops an orphaned table by name.
///
/// This is used to clean up tables that are no longer defined in the schema but may still
/// exist on disk from previous versions.
///
/// Returns `Ok(true)` if the table existed and was dropped, `Ok(false)` if the table was not
/// found.
///
/// # Safety
/// This permanently deletes the table and all its data. Only use for tables that are
/// confirmed to be obsolete.
pub fn drop_orphan_table(&self, name: &str) -> Result<bool, DatabaseError> {
let tx = self.inner.begin_rw_txn().map_err(|e| DatabaseError::InitTx(e.into()))?;
match tx.open_db(Some(name)) {
Ok(db) => {
// SAFETY: We just opened the db handle and will commit immediately after dropping.
// No other cursors or handles exist for this table.
unsafe {
tx.drop_db(db.dbi()).map_err(|e| DatabaseError::Delete(e.into()))?;
}
tx.commit().map_err(|e| DatabaseError::Commit(e.into()))?;
Ok(true)
}
Err(reth_libmdbx::Error::NotFound) => Ok(false),
Err(e) => Err(DatabaseError::Open(e.into())),
}
}
/// Records version that accesses the database with write privileges.
pub fn record_client_version(&self, version: ClientVersion) -> Result<(), DatabaseError> {
if version.is_empty() {
@@ -675,46 +646,6 @@ mod tests {
create_test_db(DatabaseEnvKind::RW);
}
#[test]
fn db_drop_orphan_table() {
let path = tempfile::TempDir::new().expect(ERROR_TEMPDIR).keep();
let db = create_test_db_with_path(DatabaseEnvKind::RW, &path);
// Create an orphan table by manually creating it
let orphan_table_name = "OrphanTestTable";
{
let tx = db.inner.begin_rw_txn().expect(ERROR_INIT_TX);
tx.create_db(Some(orphan_table_name), DatabaseFlags::empty())
.expect("Failed to create orphan table");
tx.commit().expect(ERROR_COMMIT);
}
// Verify the table exists by opening it
{
let tx = db.inner.begin_ro_txn().expect(ERROR_INIT_TX);
assert!(tx.open_db(Some(orphan_table_name)).is_ok(), "Orphan table should exist");
}
// Drop the orphan table
let result = db.drop_orphan_table(orphan_table_name);
assert!(result.is_ok(), "drop_orphan_table should succeed");
assert!(result.unwrap(), "drop_orphan_table should return true for existing table");
// Verify the table no longer exists
{
let tx = db.inner.begin_ro_txn().expect(ERROR_INIT_TX);
assert!(
tx.open_db(Some(orphan_table_name)).is_err(),
"Orphan table should no longer exist"
);
}
// Dropping a non-existent table should return Ok(false)
let result = db.drop_orphan_table("NonExistentTable");
assert!(result.is_ok(), "drop_orphan_table should succeed for non-existent table");
assert!(!result.unwrap(), "drop_orphan_table should return false for non-existent table");
}
#[test]
fn db_manual_put_get() {
let env = create_test_db(DatabaseEnvKind::RW);

View File

@@ -2,16 +2,11 @@
use crate::{is_database_empty, TableSet, Tables};
use eyre::Context;
use reth_tracing::tracing::info;
use std::path::Path;
pub use crate::implementation::mdbx::*;
pub use reth_libmdbx::*;
/// Tables that have been removed from the schema but may still exist on disk from previous
/// versions. These will be dropped during database initialization.
const ORPHAN_TABLES: &[&str] = &["AccountsTrieChangeSets", "StoragesTrieChangeSets"];
/// Creates a new database at the specified path if it doesn't exist. Does NOT create tables. Check
/// [`init_db`].
pub fn create_db<P: AsRef<Path>>(path: P, args: DatabaseArguments) -> eyre::Result<DatabaseEnv> {
@@ -49,30 +44,9 @@ pub fn init_db_for<P: AsRef<Path>, TS: TableSet>(
let mut db = create_db(path, args)?;
db.create_and_track_tables_for::<TS>()?;
db.record_client_version(client_version)?;
drop_orphan_tables(&db);
Ok(db)
}
/// Drops orphaned tables that are no longer part of the schema.
fn drop_orphan_tables(db: &DatabaseEnv) {
for table_name in ORPHAN_TABLES {
match db.drop_orphan_table(table_name) {
Ok(true) => {
info!(target: "reth::db", table = %table_name, "Dropped orphaned database table");
}
Ok(false) => {}
Err(e) => {
reth_tracing::tracing::warn!(
target: "reth::db",
table = %table_name,
%e,
"Failed to drop orphaned database table"
);
}
}
}
}
/// Opens up an existing database. Read only mode. It doesn't create it or create tables if missing.
pub fn open_db_read_only(
path: impl AsRef<Path>,

View File

@@ -4,7 +4,7 @@ use crate::{
HeaderTerminalDifficulties,
};
use alloy_primitives::{Address, BlockHash};
use reth_db_api::{models::StorageBeforeTx, table::Table, AccountChangeSets};
use reth_db_api::{table::Table, AccountChangeSets};
// HEADER MASKS
add_static_file_mask! {
@@ -54,9 +54,3 @@ add_static_file_mask! {
#[doc = "Mask for selecting a single changeset from `AccountChangesets` static file segment"]
AccountChangesetMask, <AccountChangeSets as Table>::Value, 0b1
}
// STORAGE CHANGESET MASKS
add_static_file_mask! {
#[doc = "Mask for selecting a single changeset from `StorageChangesets` static file segment"]
StorageChangesetMask, StorageBeforeTx, 0b1
}

View File

@@ -412,16 +412,8 @@ impl Transaction<RW> {
/// Returns a buffer which can be used to write a value into the item at the
/// given key and with the given length. The buffer must be completely
/// filled by the caller.
///
/// This should not be used on dupsort tables.
///
/// # Safety
///
/// The caller must ensure that the returned buffer is not used after the transaction is
/// committed or aborted, or if another value is inserted. To be clear: the second call to
/// this function is not permitted while the returned slice is reachable.
#[allow(clippy::mut_from_ref)]
pub unsafe fn reserve(
pub fn reserve(
&self,
dbi: ffi::MDBX_dbi,
key: impl AsRef<[u8]>,

View File

@@ -105,11 +105,8 @@ fn test_reserve() {
let txn = env.begin_rw_txn().unwrap();
let dbi = txn.open_db(None).unwrap().dbi();
{
unsafe {
// SAFETY: the returned slice is used before the transaction is committed or aborted.
let mut writer = txn.reserve(dbi, b"key1", 4, WriteFlags::empty()).unwrap();
writer.write_all(b"val1").unwrap();
}
let mut writer = txn.reserve(dbi, b"key1", 4, WriteFlags::empty()).unwrap();
writer.write_all(b"val1").unwrap();
}
txn.commit().unwrap();

View File

@@ -1,12 +1,10 @@
//! Account/storage changeset iteration support for walking through historical state changes in
//! Account changeset iteration support for walking through historical account state changes in
//! static files.
use crate::ProviderResult;
use alloy_primitives::BlockNumber;
use reth_db::models::AccountBeforeTx;
use reth_db_api::models::BlockNumberAddress;
use reth_primitives_traits::StorageEntry;
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader};
use reth_storage_api::ChangeSetReader;
use std::ops::{Bound, RangeBounds};
/// Iterator that walks account changesets from static files in a block range.
@@ -99,78 +97,3 @@ where
None
}
}
/// Iterator that walks storage changesets from static files in a block range.
#[derive(Debug)]
pub struct StaticFileStorageChangesetWalker<P> {
/// Static file provider
provider: P,
/// End block (exclusive). `None` means iterate until exhausted.
end_block: Option<BlockNumber>,
/// Current block being processed
current_block: BlockNumber,
/// Changesets for current block
current_changesets: Vec<(BlockNumberAddress, StorageEntry)>,
/// Index within current block's changesets
changeset_index: usize,
}
impl<P> StaticFileStorageChangesetWalker<P> {
/// Create a new static file storage changeset walker.
pub fn new(provider: P, range: impl RangeBounds<BlockNumber>) -> Self {
let start = match range.start_bound() {
Bound::Included(&n) => n,
Bound::Excluded(&n) => n + 1,
Bound::Unbounded => 0,
};
let end_block = match range.end_bound() {
Bound::Included(&n) => Some(n + 1),
Bound::Excluded(&n) => Some(n),
Bound::Unbounded => None,
};
Self {
provider,
end_block,
current_block: start,
current_changesets: Vec::new(),
changeset_index: 0,
}
}
}
impl<P> Iterator for StaticFileStorageChangesetWalker<P>
where
P: StorageChangeSetReader,
{
type Item = ProviderResult<(BlockNumberAddress, StorageEntry)>;
fn next(&mut self) -> Option<Self::Item> {
if let Some(changeset) = self.current_changesets.get(self.changeset_index).copied() {
self.changeset_index += 1;
return Some(Ok(changeset));
}
if !self.current_changesets.is_empty() {
self.current_block += 1;
}
while self.end_block.is_none_or(|end| self.current_block < end) {
match self.provider.storage_changeset(self.current_block) {
Ok(changesets) if !changesets.is_empty() => {
self.current_changesets = changesets;
self.changeset_index = 1;
return Some(Ok(self.current_changesets[0]));
}
Ok(_) => self.current_block += 1,
Err(e) => {
self.current_block += 1;
return Some(Err(e));
}
}
}
None
}
}

View File

@@ -17,20 +17,20 @@ use alloy_primitives::{map::HashMap, Address, BlockNumber, TxHash, TxNumber, B25
use rayon::slice::ParallelSliceMut;
use reth_db::{
cursor::{DbCursorRO, DbDupCursorRW},
models::{AccountBeforeTx, StorageBeforeTx},
models::AccountBeforeTx,
static_file::TransactionSenderMask,
table::Value,
transaction::{CursorMutTy, CursorTy, DbTx, DbTxMut, DupCursorMutTy, DupCursorTy},
};
use reth_db_api::{
cursor::DbCursorRW,
models::{storage_sharded_key::StorageShardedKey, BlockNumberAddress, ShardedKey},
models::{storage_sharded_key::StorageShardedKey, ShardedKey},
tables,
tables::BlockNumberList,
};
use reth_errors::ProviderError;
use reth_node_types::NodePrimitives;
use reth_primitives_traits::{ReceiptTy, StorageEntry};
use reth_primitives_traits::ReceiptTy;
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::{ChangeSetReader, DBProvider, NodePrimitivesProvider, StorageSettingsCache};
use reth_storage_errors::provider::ProviderResult;
@@ -83,17 +83,14 @@ pub type RawRocksDBBatch = ();
/// Helper type for `RocksDB` transaction reference argument in reader constructors.
///
/// When `rocksdb` feature is enabled, this is an optional reference to a `RocksDB` transaction.
/// The `Option` allows callers to skip transaction creation when `RocksDB` isn't needed
/// (e.g., on legacy MDBX-only nodes).
/// When `rocksdb` feature is disabled, it's `()` (unit type) to allow the same API without
/// feature gates.
/// When `rocksdb` feature is enabled, this is a reference to a `RocksDB` transaction.
/// Otherwise, it's `()` (unit type) to allow the same API without feature gates.
#[cfg(all(unix, feature = "rocksdb"))]
pub type RocksTxRefArg<'a> = Option<&'a crate::providers::rocksdb::RocksTx<'a>>;
pub type RocksTxRefArg<'a> = &'a crate::providers::rocksdb::RocksTx<'a>;
/// Helper type for `RocksDB` transaction reference argument in reader constructors.
///
/// When `rocksdb` feature is disabled, it's `()` (unit type) to allow the same API without
/// feature gates.
/// When `rocksdb` feature is enabled, this is a reference to a `RocksDB` transaction.
/// Otherwise, it's `()` (unit type) to allow the same API without feature gates.
#[cfg(not(all(unix, feature = "rocksdb")))]
pub type RocksTxRefArg<'a> = ();
@@ -174,27 +171,6 @@ impl<'a> EitherWriter<'a, (), ()> {
}
}
/// Creates a new [`EitherWriter`] for storage changesets based on storage settings.
pub fn new_storage_changesets<P>(
provider: &'a P,
block_number: BlockNumber,
) -> ProviderResult<DupEitherWriterTy<'a, P, tables::StorageChangeSets>>
where
P: DBProvider + NodePrimitivesProvider + StorageSettingsCache + StaticFileProviderFactory,
P::Tx: DbTxMut,
{
if provider.cached_storage_settings().storage_changesets_in_static_files {
Ok(EitherWriter::StaticFile(
provider
.get_static_file_writer(block_number, StaticFileSegment::StorageChangeSets)?,
))
} else {
Ok(EitherWriter::Database(
provider.tx_ref().cursor_dup_write::<tables::StorageChangeSets>()?,
))
}
}
/// Returns the destination for writing receipts.
///
/// The rules are as follows:
@@ -232,19 +208,6 @@ impl<'a> EitherWriter<'a, (), ()> {
}
}
/// Returns the destination for writing storage changesets.
///
/// This determines the destination based solely on storage settings.
pub fn storage_changesets_destination<P: DBProvider + StorageSettingsCache>(
provider: &P,
) -> EitherWriterDestination {
if provider.cached_storage_settings().storage_changesets_in_static_files {
EitherWriterDestination::StaticFile
} else {
EitherWriterDestination::Database
}
}
/// Creates a new [`EitherWriter`] for storages history based on storage settings.
pub fn new_storages_history<P>(
provider: &P,
@@ -688,41 +651,6 @@ where
}
}
impl<'a, CURSOR, N: NodePrimitives> EitherWriter<'a, CURSOR, N>
where
CURSOR: DbDupCursorRW<tables::StorageChangeSets>,
{
/// Append storage changeset for a block.
///
/// NOTE: This _sorts_ the changesets by address and storage key before appending.
pub fn append_storage_changeset(
&mut self,
block_number: BlockNumber,
mut changeset: Vec<StorageBeforeTx>,
) -> ProviderResult<()> {
changeset.par_sort_by_key(|change| (change.address, change.key));
match self {
Self::Database(cursor) => {
for change in changeset {
let storage_id = BlockNumberAddress((block_number, change.address));
cursor.append_dup(
storage_id,
StorageEntry { key: change.key, value: change.value },
)?;
}
}
Self::StaticFile(writer) => {
writer.append_storage_changeset(changeset, block_number)?;
}
#[cfg(all(unix, feature = "rocksdb"))]
Self::RocksDB(_) => return Err(ProviderError::UnsupportedProvider),
}
Ok(())
}
}
/// Represents a source for reading data, either from database, static files, or `RocksDB`.
#[derive(Debug, Display)]
pub enum EitherReader<'a, CURSOR, N> {
@@ -765,9 +693,7 @@ impl<'a> EitherReader<'a, (), ()> {
{
#[cfg(all(unix, feature = "rocksdb"))]
if provider.cached_storage_settings().storages_history_in_rocksdb {
return Ok(EitherReader::RocksDB(
_rocksdb_tx.expect("storages_history_in_rocksdb requires rocksdb tx"),
));
return Ok(EitherReader::RocksDB(_rocksdb_tx));
}
Ok(EitherReader::Database(
@@ -787,9 +713,7 @@ impl<'a> EitherReader<'a, (), ()> {
{
#[cfg(all(unix, feature = "rocksdb"))]
if provider.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
return Ok(EitherReader::RocksDB(
_rocksdb_tx.expect("transaction_hash_numbers_in_rocksdb requires rocksdb tx"),
));
return Ok(EitherReader::RocksDB(_rocksdb_tx));
}
Ok(EitherReader::Database(
@@ -809,9 +733,7 @@ impl<'a> EitherReader<'a, (), ()> {
{
#[cfg(all(unix, feature = "rocksdb"))]
if provider.cached_storage_settings().account_history_in_rocksdb {
return Ok(EitherReader::RocksDB(
_rocksdb_tx.expect("account_history_in_rocksdb requires rocksdb tx"),
));
return Ok(EitherReader::RocksDB(_rocksdb_tx));
}
Ok(EitherReader::Database(
@@ -1065,19 +987,6 @@ impl EitherWriterDestination {
Self::Database
}
}
/// Returns the destination for writing storage changesets based on storage settings.
pub fn storage_changesets<P>(provider: &P) -> Self
where
P: StorageSettingsCache,
{
// Write storage changesets to static files only if they're explicitly enabled
if provider.cached_storage_settings().storage_changesets_in_static_files {
Self::StaticFile
} else {
Self::Database
}
}
}
#[cfg(test)]
@@ -1442,7 +1351,8 @@ mod rocksdb_tests {
// Run queries against both backends using EitherReader
let mdbx_ro = factory.database_provider_ro().unwrap();
let rocks_tx = rocks_provider.tx();
// Use `with_assume_history_complete()` since both backends have identical data
let rocks_tx = rocks_provider.tx().with_assume_history_complete();
for (i, query) in queries.iter().enumerate() {
// MDBX query via EitherReader
@@ -1534,7 +1444,8 @@ mod rocksdb_tests {
// Run queries against both backends using EitherReader
let mdbx_ro = factory.database_provider_ro().unwrap();
let rocks_tx = rocks_provider.tx();
// Use `with_assume_history_complete()` since both backends have identical data
let rocks_tx = rocks_provider.tx().with_assume_history_complete();
for (i, query) in queries.iter().enumerate() {
// MDBX query via EitherReader
@@ -1823,20 +1734,4 @@ mod rocksdb_tests {
"Data should be visible after provider.commit()"
);
}
/// Test that `EitherReader::new_accounts_history` panics when settings require
/// `RocksDB` but no tx is provided (`None`). This is an invariant violation that
/// indicates a bug - `with_rocksdb_tx` should always provide a tx when needed.
#[test]
#[should_panic(expected = "account_history_in_rocksdb requires rocksdb tx")]
fn test_settings_mismatch_panics() {
let factory = create_test_provider_factory();
factory.set_storage_settings_cache(
StorageSettings::legacy().with_account_history_in_rocksdb(true),
);
let provider = factory.database_provider_ro().unwrap();
let _ = EitherReader::<(), ()>::new_accounts_history(&provider, None);
}
}

View File

@@ -711,26 +711,6 @@ impl<N: ProviderNodeTypes> StorageChangeSetReader for BlockchainProvider<N> {
) -> ProviderResult<Vec<(BlockNumberAddress, StorageEntry)>> {
self.consistent_provider()?.storage_changeset(block_number)
}
fn get_storage_before_block(
&self,
block_number: BlockNumber,
address: Address,
storage_key: B256,
) -> ProviderResult<Option<StorageEntry>> {
self.consistent_provider()?.get_storage_before_block(block_number, address, storage_key)
}
fn storage_changesets_range(
&self,
range: RangeInclusive<BlockNumber>,
) -> ProviderResult<Vec<(BlockNumberAddress, StorageEntry)>> {
self.consistent_provider()?.storage_changesets_range(range)
}
fn storage_changeset_count(&self) -> ProviderResult<usize> {
self.consistent_provider()?.storage_changeset_count()
}
}
impl<N: ProviderNodeTypes> ChangeSetReader for BlockchainProvider<N> {
@@ -1697,11 +1677,14 @@ mod tests {
database_state.into_iter().map(|(address, (account, _))| {
(address, None, Some(account.into()), Default::default())
}),
database_changesets.iter().map(|block_changesets| {
block_changesets.iter().map(|(address, account, _)| {
(*address, Some(Some((*account).into())), [])
database_changesets
.iter()
.map(|block_changesets| {
block_changesets.iter().map(|(address, account, _)| {
(*address, Some(Some((*account).into())), [])
})
})
}),
.collect::<Vec<_>>(),
Vec::new(),
),
first_block: first_database_block,

View File

@@ -1347,138 +1347,6 @@ impl<N: ProviderNodeTypes> StorageChangeSetReader for ConsistentProvider<N> {
self.storage_provider.storage_changeset(block_number)
}
}
fn get_storage_before_block(
&self,
block_number: BlockNumber,
address: Address,
storage_key: B256,
) -> ProviderResult<Option<StorageEntry>> {
if let Some(state) =
self.head_block.as_ref().and_then(|b| b.block_on_chain(block_number.into()))
{
let changeset = state
.block_ref()
.execution_output
.state
.reverts
.clone()
.to_plain_state_reverts()
.storage
.into_iter()
.flatten()
.find_map(|revert: PlainStorageRevert| {
if revert.address != address {
return None
}
revert.storage_revert.into_iter().find_map(|(key, value)| {
let key = key.into();
(key == storage_key)
.then(|| StorageEntry { key, value: value.to_previous_value() })
})
});
Ok(changeset)
} else {
let storage_history_exists = self
.storage_provider
.get_prune_checkpoint(PruneSegment::StorageHistory)?
.and_then(|checkpoint| {
checkpoint.block_number.map(|checkpoint| block_number > checkpoint)
})
.unwrap_or(true);
if !storage_history_exists {
return Err(ProviderError::StateAtBlockPruned(block_number))
}
self.storage_provider.get_storage_before_block(block_number, address, storage_key)
}
}
fn storage_changesets_range(
&self,
range: RangeInclusive<BlockNumber>,
) -> ProviderResult<Vec<(BlockNumberAddress, StorageEntry)>> {
let range = to_range(range);
let mut changesets = Vec::new();
let database_start = range.start;
let mut database_end = range.end;
if let Some(head_block) = &self.head_block {
database_end = head_block.anchor().number;
let chain = head_block.chain().collect::<Vec<_>>();
for state in chain {
let block_changesets = state
.block_ref()
.execution_output
.state
.reverts
.clone()
.to_plain_state_reverts()
.storage
.into_iter()
.flatten()
.flat_map(|revert: PlainStorageRevert| {
revert.storage_revert.into_iter().map(move |(key, value)| {
(
BlockNumberAddress((state.number(), revert.address)),
StorageEntry { key: key.into(), value: value.to_previous_value() },
)
})
});
changesets.extend(block_changesets);
}
}
if database_start < database_end {
let storage_history_exists = self
.storage_provider
.get_prune_checkpoint(PruneSegment::StorageHistory)?
.and_then(|checkpoint| {
checkpoint.block_number.map(|checkpoint| database_start > checkpoint)
})
.unwrap_or(true);
if !storage_history_exists {
return Err(ProviderError::StateAtBlockPruned(database_start))
}
let db_changesets = self
.storage_provider
.storage_changesets_range(database_start..=database_end - 1)?;
changesets.extend(db_changesets);
}
changesets.sort_by_key(|(block_address, _)| block_address.block_number());
Ok(changesets)
}
fn storage_changeset_count(&self) -> ProviderResult<usize> {
let mut count = 0;
if let Some(head_block) = &self.head_block {
for state in head_block.chain() {
count += state
.block_ref()
.execution_output
.state
.reverts
.clone()
.to_plain_state_reverts()
.storage
.into_iter()
.flatten()
.map(|revert: PlainStorageRevert| revert.storage_revert.len())
.sum::<usize>();
}
}
count += self.storage_provider.storage_changeset_count()?;
Ok(count)
}
}
impl<N: ProviderNodeTypes> ChangeSetReader for ConsistentProvider<N> {

View File

@@ -44,9 +44,7 @@ use std::{
use tracing::trace;
mod provider;
pub use provider::{
CommitOrder, DatabaseProvider, DatabaseProviderRO, DatabaseProviderRW, SaveBlocksMode,
};
pub use provider::{DatabaseProvider, DatabaseProviderRO, DatabaseProviderRW, SaveBlocksMode};
use super::ProviderNodeTypes;
use reth_trie::KeccakKeyHasher;
@@ -232,25 +230,6 @@ impl<N: ProviderNodeTypes> ProviderFactory<N> {
)))
}
/// Returns a provider with a created `DbTxMut` inside, configured for unwind operations.
/// Uses unwind commit order (MDBX first, then `RocksDB`, then static files) to allow
/// recovery by truncating static files on restart if interrupted.
#[track_caller]
pub fn unwind_provider_rw(
&self,
) -> ProviderResult<DatabaseProvider<<N::DB as Database>::TXMut, N>> {
Ok(DatabaseProvider::new_unwind_rw(
self.db.tx_mut()?,
self.chain_spec.clone(),
self.static_file_provider.clone(),
self.prune_modes.clone(),
self.storage.clone(),
self.storage_settings.clone(),
self.rocksdb_provider.clone(),
self.changeset_cache.clone(),
))
}
/// State provider for latest block
#[track_caller]
pub fn latest(&self) -> ProviderResult<StateProviderBox> {

File diff suppressed because it is too large Load Diff

View File

@@ -93,18 +93,14 @@ const DEFAULT_MAX_BACKGROUND_JOBS: i32 = 6;
/// Default bytes per sync for `RocksDB` WAL writes (1 MB).
const DEFAULT_BYTES_PER_SYNC: u64 = 1_048_576;
/// Default bloom filter bits per key (~1% false positive rate).
const DEFAULT_BLOOM_FILTER_BITS: f64 = 10.0;
/// Default buffer capacity for compression in batches.
/// 4 KiB matches common block/page sizes and comfortably holds typical history values,
/// reducing the first few reallocations without over-allocating.
const DEFAULT_COMPRESS_BUF_CAPACITY: usize = 4096;
/// Default auto-commit threshold for batch writes (4 GiB).
///
/// When a batch exceeds this size, it is automatically committed to prevent OOM
/// during large bulk writes. The consistency check on startup heals any crash
/// that occurs between auto-commits.
const DEFAULT_AUTO_COMMIT_THRESHOLD: usize = 4 * 1024 * 1024 * 1024;
/// Builder for [`RocksDBProvider`].
pub struct RocksDBBuilder {
path: PathBuf,
@@ -149,6 +145,11 @@ impl RocksDBBuilder {
table_options.set_pin_l0_filter_and_index_blocks_in_cache(true);
// Shared block cache for all column families.
table_options.set_block_cache(cache);
// Bloom filter: 10 bits/key = ~1% false positive rate, full filter for better read
// performance. this setting is good trade off a little bit of memory for better
// point lookup performance. see https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#configuration-basics
table_options.set_bloom_filter(DEFAULT_BLOOM_FILTER_BITS, false);
table_options.set_optimize_filters_for_memory(true);
table_options
}
@@ -200,32 +201,6 @@ impl RocksDBBuilder {
cf_options
}
/// Creates optimized column family options for `TransactionHashNumbers`.
///
/// This table stores `B256 -> TxNumber` mappings where:
/// - Keys are incompressible 32-byte hashes (compression wastes CPU for zero benefit)
/// - Values are varint-encoded `u64` (a few bytes - too small to benefit from compression)
/// - Every lookup expects a hit (bloom filters only help when checking non-existent keys)
fn tx_hash_numbers_column_family_options(cache: &Cache) -> Options {
let mut table_options = BlockBasedOptions::default();
table_options.set_block_size(DEFAULT_BLOCK_SIZE);
table_options.set_cache_index_and_filter_blocks(true);
table_options.set_pin_l0_filter_and_index_blocks_in_cache(true);
table_options.set_block_cache(cache);
// Disable bloom filter: every lookup expects a hit, so bloom filters provide no benefit
// and waste memory
let mut cf_options = Options::default();
cf_options.set_block_based_table_factory(&table_options);
cf_options.set_level_compaction_dynamic_level_bytes(true);
// Disable compression: B256 keys are incompressible hashes, TxNumber values are
// varint-encoded u64 (a few bytes). Compression wastes CPU cycles for zero space savings.
cf_options.set_compression_type(DBCompressionType::None);
cf_options.set_bottommost_compression_type(DBCompressionType::None);
cf_options
}
/// Adds a column family for a specific table type.
pub fn with_table<T: Table>(mut self) -> Self {
self.column_families.push(T::NAME.to_string());
@@ -287,12 +262,10 @@ impl RocksDBBuilder {
.column_families
.iter()
.map(|name| {
let cf_options = if name == tables::TransactionHashNumbers::NAME {
Self::tx_hash_numbers_column_family_options(&self.block_cache)
} else {
Self::default_column_family_options(&self.block_cache)
};
ColumnFamilyDescriptor::new(name.clone(), cf_options)
ColumnFamilyDescriptor::new(
name.clone(),
Self::default_column_family_options(&self.block_cache),
)
})
.collect();
@@ -621,7 +594,7 @@ impl RocksDBProvider {
let write_options = WriteOptions::default();
let txn_options = OptimisticTransactionOptions::default();
let inner = self.0.db_rw().transaction_opt(&write_options, &txn_options);
RocksTx { inner, provider: self }
RocksTx { inner, provider: self, assume_history_complete: false }
}
/// Creates a new batch for atomic writes.
@@ -636,21 +609,6 @@ impl RocksDBProvider {
provider: self,
inner: WriteBatchWithTransaction::<true>::default(),
buf: Vec::with_capacity(DEFAULT_COMPRESS_BUF_CAPACITY),
auto_commit_threshold: None,
}
}
/// Creates a new batch with auto-commit enabled.
///
/// When the batch size exceeds the threshold (4 GiB), the batch is automatically
/// committed and reset. This prevents OOM during large bulk writes while maintaining
/// crash-safety via the consistency check on startup.
pub fn batch_with_auto_commit(&self) -> RocksDBBatch<'_> {
RocksDBBatch {
provider: self,
inner: WriteBatchWithTransaction::<true>::default(),
buf: Vec::with_capacity(DEFAULT_COMPRESS_BUF_CAPACITY),
auto_commit_threshold: Some(DEFAULT_AUTO_COMMIT_THRESHOLD),
}
}
@@ -1159,16 +1117,11 @@ impl RocksDBProvider {
/// Unlike [`RocksTx`], this does NOT support read-your-writes. Use for write-only flows
/// where you don't need to read back uncommitted data within the same operation
/// (e.g., history index writes).
///
/// When `auto_commit_threshold` is set, the batch will automatically commit and reset
/// when the batch size exceeds the threshold. This prevents OOM during large bulk writes.
#[must_use = "batch must be committed"]
pub struct RocksDBBatch<'a> {
provider: &'a RocksDBProvider,
inner: WriteBatchWithTransaction<true>,
buf: Vec<u8>,
/// If set, batch auto-commits when size exceeds this threshold (in bytes).
auto_commit_threshold: Option<usize>,
}
impl fmt::Debug for RocksDBBatch<'_> {
@@ -1187,16 +1140,12 @@ impl fmt::Debug for RocksDBBatch<'_> {
impl<'a> RocksDBBatch<'a> {
/// Puts a value into the batch.
///
/// If auto-commit is enabled and the batch exceeds the threshold, commits and resets.
pub fn put<T: Table>(&mut self, key: T::Key, value: &T::Value) -> ProviderResult<()> {
let encoded_key = key.encode();
self.put_encoded::<T>(&encoded_key, value)
}
/// Puts a value into the batch using pre-encoded key.
///
/// If auto-commit is enabled and the batch exceeds the threshold, commits and resets.
pub fn put_encoded<T: Table>(
&mut self,
key: &<T::Key as Encode>::Encoded,
@@ -1204,43 +1153,12 @@ impl<'a> RocksDBBatch<'a> {
) -> ProviderResult<()> {
let value_bytes = compress_to_buf_or_ref!(self.buf, value).unwrap_or(&self.buf);
self.inner.put_cf(self.provider.get_cf_handle::<T>()?, key, value_bytes);
self.maybe_auto_commit()?;
Ok(())
}
/// Deletes a value from the batch.
///
/// If auto-commit is enabled and the batch exceeds the threshold, commits and resets.
pub fn delete<T: Table>(&mut self, key: T::Key) -> ProviderResult<()> {
self.inner.delete_cf(self.provider.get_cf_handle::<T>()?, key.encode().as_ref());
self.maybe_auto_commit()?;
Ok(())
}
/// Commits and resets the batch if it exceeds the auto-commit threshold.
///
/// This is called after each `put` or `delete` operation to prevent unbounded memory growth.
/// Returns immediately if auto-commit is disabled or threshold not reached.
fn maybe_auto_commit(&mut self) -> ProviderResult<()> {
if let Some(threshold) = self.auto_commit_threshold &&
self.inner.size_in_bytes() >= threshold
{
tracing::debug!(
target: "providers::rocksdb",
batch_size = self.inner.size_in_bytes(),
threshold,
"Auto-committing RocksDB batch"
);
let old_batch = std::mem::take(&mut self.inner);
self.provider.0.db_rw().write_opt(old_batch, &WriteOptions::default()).map_err(
|e| {
ProviderError::Database(DatabaseError::Commit(DatabaseErrorInfo {
message: e.to_string().into(),
code: -1,
}))
},
)?;
}
Ok(())
}
@@ -1270,11 +1188,6 @@ impl<'a> RocksDBBatch<'a> {
self.inner.is_empty()
}
/// Returns the size of the batch in bytes.
pub fn size_in_bytes(&self) -> usize {
self.inner.size_in_bytes()
}
/// Returns a reference to the underlying `RocksDB` provider.
pub const fn provider(&self) -> &RocksDBProvider {
self.provider
@@ -1612,6 +1525,10 @@ impl<'a> RocksDBBatch<'a> {
pub struct RocksTx<'db> {
inner: Transaction<'db, OptimisticTransactionDB>,
provider: &'db RocksDBProvider,
/// When true, assume `RocksDB` has complete history (like `MDBX`) and return `NotYetWritten`
/// when querying before the first history entry. When false (default), return
/// `MaybeInPlainState` for hybrid storage safety.
assume_history_complete: bool,
}
impl fmt::Debug for RocksTx<'_> {
@@ -1621,6 +1538,16 @@ impl fmt::Debug for RocksTx<'_> {
}
impl<'db> RocksTx<'db> {
/// Sets the `assume_history_complete` flag to true.
///
/// When enabled, history queries will return `NotYetWritten` (like `MDBX`) instead of
/// `MaybeInPlainState` when querying before the first history entry. Use this in tests
/// where `RocksDB` and `MDBX` have identical data.
pub const fn with_assume_history_complete(mut self) -> Self {
self.assume_history_complete = true;
self
}
/// Gets a value from the specified table. Sees uncommitted writes in this transaction.
pub fn get<T: Table>(&self, key: T::Key) -> ProviderResult<Option<T::Value>> {
let encoded_key = key.encode();
@@ -1787,10 +1714,19 @@ impl<'db> RocksTx<'db> {
where
T: Table<Value = BlockNumberList>,
{
// History may be pruned if a lowest available block is set.
let is_maybe_pruned = lowest_available_block_number.is_some();
// Determines whether to soften NotYetWritten -> MaybeInPlainState.
//
// We soften when:
// 1. `assume_history_complete` is false (hybrid storage - RocksDB may not have full
// history)
// 2. OR history may be pruned (`lowest_available_block_number.is_some()`)
//
// We only return NotYetWritten when we're certain history is complete AND not pruned.
let should_soften_not_yet_written =
!self.assume_history_complete || lowest_available_block_number.is_some();
let fallback = || {
Ok(if is_maybe_pruned {
Ok(if should_soften_not_yet_written {
HistoryInfo::MaybeInPlainState
} else {
HistoryInfo::NotYetWritten
@@ -1844,11 +1780,19 @@ impl<'db> RocksTx<'db> {
false
};
Ok(HistoryInfo::from_lookup(
// Apply the same softening logic to `from_lookup` result.
let result = HistoryInfo::from_lookup(
found_block,
is_before_first_write,
lowest_available_block_number,
))
);
Ok(match result {
HistoryInfo::NotYetWritten if should_soften_not_yet_written => {
HistoryInfo::MaybeInPlainState
}
other => other,
})
}
/// Returns an error if the raw iterator is in an invalid state due to an I/O error.
@@ -2834,40 +2778,4 @@ mod tests {
assert_eq!(shards[1].0.highest_block_number, u64::MAX);
assert_eq!(shards[1].1.iter().collect::<Vec<_>>(), (51..=75).collect::<Vec<_>>());
}
#[test]
fn test_batch_auto_commit_on_threshold() {
let temp_dir = TempDir::new().unwrap();
let provider =
RocksDBBuilder::new(temp_dir.path()).with_table::<TestTable>().build().unwrap();
// Create batch with tiny threshold (1KB) to force auto-commits
let mut batch = RocksDBBatch {
provider: &provider,
inner: WriteBatchWithTransaction::<true>::default(),
buf: Vec::new(),
auto_commit_threshold: Some(1024), // 1KB
};
// Write entries until we exceed threshold multiple times
// Each entry is ~20 bytes, so 100 entries = ~2KB = 2 auto-commits
for i in 0..100u64 {
let value = format!("value_{i:04}").into_bytes();
batch.put::<TestTable>(i, &value).unwrap();
}
// Data should already be visible (auto-committed) even before final commit
// At least some entries should be readable
let first_visible = provider.get::<TestTable>(0).unwrap();
assert!(first_visible.is_some(), "Auto-committed data should be visible");
// Final commit for remaining batch
batch.commit().unwrap();
// All entries should now be visible
for i in 0..100u64 {
let value = format!("value_{i:04}").into_bytes();
assert_eq!(provider.get::<TestTable>(i).unwrap(), Some(value));
}
}
}

View File

@@ -81,13 +81,6 @@ impl RocksDBProvider {
pub const fn table_stats(&self) -> Vec<RocksDBTableStats> {
Vec::new()
}
/// Clears all entries from the specified table (stub implementation).
///
/// This is a no-op since there is no `RocksDB` when the feature is disabled.
pub const fn clear<T>(&self) -> ProviderResult<()> {
Ok(())
}
}
impl DatabaseMetrics for RocksDBProvider {

View File

@@ -14,7 +14,7 @@ use reth_db_api::{
use reth_primitives_traits::{Account, Bytecode};
use reth_storage_api::{
BlockNumReader, BytecodeReader, DBProvider, NodePrimitivesProvider, StateProofProvider,
StorageChangeSetReader, StorageRootProvider, StorageSettingsCache,
StorageRootProvider, StorageSettingsCache,
};
use reth_storage_errors::provider::ProviderResult;
use reth_trie::{
@@ -26,8 +26,8 @@ use reth_trie::{
TrieInputSorted,
};
use reth_trie_db::{
hashed_storage_from_reverts_with_provider, DatabaseHashedPostState, DatabaseProof,
DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot, DatabaseTrieWitness,
DatabaseHashedPostState, DatabaseHashedStorage, DatabaseProof, DatabaseStateRoot,
DatabaseStorageProof, DatabaseStorageRoot, DatabaseTrieWitness,
};
use std::fmt::Debug;
@@ -109,7 +109,7 @@ pub struct HistoricalStateProviderRef<'b, Provider> {
lowest_available_blocks: LowestAvailableBlocks,
}
impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumReader>
impl<'b, Provider: DBProvider + ChangeSetReader + BlockNumReader>
HistoricalStateProviderRef<'b, Provider>
{
/// Create new `StateProvider` for historical block number
@@ -210,7 +210,7 @@ impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + Block
);
}
hashed_storage_from_reverts_with_provider(self.provider, address, self.block_number)
Ok(HashedStorage::from_reverts(self.tx(), address, self.block_number)?)
}
/// Set the lowest block number at which the account history is available.
@@ -242,7 +242,6 @@ impl<
Provider: DBProvider
+ BlockNumReader
+ ChangeSetReader
+ StorageChangeSetReader
+ StorageSettingsCache
+ RocksDBProviderFactory
+ NodePrimitivesProvider,
@@ -286,8 +285,8 @@ impl<Provider: DBProvider + BlockNumReader + BlockHashReader> BlockHashReader
}
}
impl<Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumReader>
StateRootProvider for HistoricalStateProviderRef<'_, Provider>
impl<Provider: DBProvider + ChangeSetReader + BlockNumReader> StateRootProvider
for HistoricalStateProviderRef<'_, Provider>
{
fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult<B256> {
let mut revert_state = self.revert_state()?;
@@ -323,8 +322,8 @@ impl<Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumR
}
}
impl<Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumReader>
StorageRootProvider for HistoricalStateProviderRef<'_, Provider>
impl<Provider: DBProvider + ChangeSetReader + BlockNumReader> StorageRootProvider
for HistoricalStateProviderRef<'_, Provider>
{
fn storage_root(
&self,
@@ -362,8 +361,8 @@ impl<Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumR
}
}
impl<Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumReader>
StateProofProvider for HistoricalStateProviderRef<'_, Provider>
impl<Provider: DBProvider + ChangeSetReader + BlockNumReader> StateProofProvider
for HistoricalStateProviderRef<'_, Provider>
{
/// Get account and storage proofs.
fn proof(
@@ -406,7 +405,6 @@ impl<
+ BlockNumReader
+ BlockHashReader
+ ChangeSetReader
+ StorageChangeSetReader
+ StorageSettingsCache
+ RocksDBProviderFactory
+ NodePrimitivesProvider,
@@ -420,16 +418,18 @@ impl<
) -> ProviderResult<Option<StorageValue>> {
match self.storage_history_lookup(address, storage_key)? {
HistoryInfo::NotYetWritten => Ok(None),
HistoryInfo::InChangeset(changeset_block_number) => self
.provider
.get_storage_before_block(changeset_block_number, address, storage_key)?
.ok_or_else(|| ProviderError::StorageChangesetNotFound {
block_number: changeset_block_number,
address,
storage_key: Box::new(storage_key),
})
.map(|entry| entry.value)
.map(Some),
HistoryInfo::InChangeset(changeset_block_number) => Ok(Some(
self.tx()
.cursor_dup_read::<tables::StorageChangeSets>()?
.seek_by_key_subkey((changeset_block_number, address).into(), storage_key)?
.filter(|entry| entry.key == storage_key)
.ok_or_else(|| ProviderError::StorageChangesetNotFound {
block_number: changeset_block_number,
address,
storage_key: Box::new(storage_key),
})?
.value,
)),
HistoryInfo::InPlainState | HistoryInfo::MaybeInPlainState => Ok(self
.tx()
.cursor_dup_read::<tables::PlainStorageState>()?
@@ -462,9 +462,7 @@ pub struct HistoricalStateProvider<Provider> {
lowest_available_blocks: LowestAvailableBlocks,
}
impl<Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumReader>
HistoricalStateProvider<Provider>
{
impl<Provider: DBProvider + ChangeSetReader + BlockNumReader> HistoricalStateProvider<Provider> {
/// Create new `StateProvider` for historical block number
pub fn new(provider: Provider, block_number: BlockNumber) -> Self {
Self { provider, block_number, lowest_available_blocks: Default::default() }
@@ -500,7 +498,7 @@ impl<Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumR
}
// Delegates all provider impls to [HistoricalStateProviderRef]
reth_storage_api::macros::delegate_provider_impls!(HistoricalStateProvider<Provider> where [Provider: DBProvider + BlockNumReader + BlockHashReader + ChangeSetReader + StorageChangeSetReader + StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider]);
reth_storage_api::macros::delegate_provider_impls!(HistoricalStateProvider<Provider> where [Provider: DBProvider + BlockNumReader + BlockHashReader + ChangeSetReader + StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider]);
/// Lowest blocks at which different parts of the state are available.
/// They may be [Some] if pruning is enabled.
@@ -633,7 +631,7 @@ mod tests {
use reth_primitives_traits::{Account, StorageEntry};
use reth_storage_api::{
BlockHashReader, BlockNumReader, ChangeSetReader, DBProvider, DatabaseProviderFactory,
NodePrimitivesProvider, StorageChangeSetReader, StorageSettingsCache,
NodePrimitivesProvider, StorageSettingsCache,
};
use reth_storage_errors::provider::ProviderError;
@@ -649,7 +647,6 @@ mod tests {
+ BlockNumReader
+ BlockHashReader
+ ChangeSetReader
+ StorageChangeSetReader
+ StorageSettingsCache
+ RocksDBProviderFactory
+ NodePrimitivesProvider,

View File

@@ -10,7 +10,6 @@ use reth_stages_types::StageId;
use reth_storage_api::{
BlockNumReader, ChangeSetReader, DBProvider, DatabaseProviderFactory,
DatabaseProviderROFactory, PruneCheckpointReader, StageCheckpointReader,
StorageChangeSetReader,
};
use reth_trie::{
hashed_cursor::{HashedCursorFactory, HashedPostStateCursorFactory},
@@ -197,7 +196,6 @@ where
F::Provider: StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ DBProvider
+ BlockNumReader,
{
@@ -448,11 +446,7 @@ where
impl<F> DatabaseProviderROFactory for OverlayStateProviderFactory<F>
where
F: DatabaseProviderFactory,
F::Provider: StageCheckpointReader
+ PruneCheckpointReader
+ BlockNumReader
+ ChangeSetReader
+ StorageChangeSetReader,
F::Provider: StageCheckpointReader + PruneCheckpointReader + BlockNumReader + ChangeSetReader,
{
type Provider = OverlayStateProvider<F::Provider>;

View File

@@ -3,10 +3,10 @@ use super::{
StaticFileJarProvider, StaticFileProviderRW, StaticFileProviderRWRefMut,
};
use crate::{
changeset_walker::{StaticFileAccountChangesetWalker, StaticFileStorageChangesetWalker},
to_range, BlockHashReader, BlockNumReader, BlockReader, BlockSource, EitherWriter,
EitherWriterDestination, HeaderProvider, ReceiptProvider, StageCheckpointReader, StatsReader,
TransactionVariant, TransactionsProvider, TransactionsProviderExt,
changeset_walker::StaticFileAccountChangesetWalker, to_range, BlockHashReader, BlockNumReader,
BlockReader, BlockSource, EitherWriter, EitherWriterDestination, HeaderProvider,
ReceiptProvider, StageCheckpointReader, StatsReader, TransactionVariant, TransactionsProvider,
TransactionsProviderExt,
};
use alloy_consensus::{transaction::TransactionMeta, Header};
use alloy_eips::{eip2718::Encodable2718, BlockHashOrNumber};
@@ -20,12 +20,12 @@ use reth_db::{
lockfile::StorageLock,
static_file::{
iter_static_files, BlockHashMask, HeaderMask, HeaderWithHashMask, ReceiptMask,
StaticFileCursor, StorageChangesetMask, TransactionMask, TransactionSenderMask,
StaticFileCursor, TransactionMask, TransactionSenderMask,
},
};
use reth_db_api::{
cursor::DbCursorRO,
models::{AccountBeforeTx, BlockNumberAddress, StorageBeforeTx, StoredBlockBodyIndices},
models::{AccountBeforeTx, StoredBlockBodyIndices},
table::{Decompress, Table, Value},
tables,
transaction::DbTx,
@@ -35,7 +35,6 @@ use reth_nippy_jar::{NippyJar, NippyJarChecker, CONFIG_FILE_EXTENSION};
use reth_node_types::NodePrimitives;
use reth_primitives_traits::{
AlloyBlockHeader as _, BlockBody as _, RecoveredBlock, SealedHeader, SignedTransaction,
StorageEntry,
};
use reth_stages_types::{PipelineTarget, StageId};
use reth_static_file_types::{
@@ -43,8 +42,7 @@ use reth_static_file_types::{
StaticFileSegment, DEFAULT_BLOCKS_PER_STATIC_FILE,
};
use reth_storage_api::{
BlockBodyIndicesProvider, ChangeSetReader, DBProvider, StorageChangeSetReader,
StorageSettingsCache,
BlockBodyIndicesProvider, ChangeSetReader, DBProvider, StorageSettingsCache,
};
use reth_storage_errors::provider::{ProviderError, ProviderResult, StaticFileWriterError};
use std::{
@@ -94,8 +92,6 @@ pub struct StaticFileWriteCtx {
pub write_receipts: bool,
/// Whether account changesets should be written to static files.
pub write_account_changesets: bool,
/// Whether storage changesets should be written to static files.
pub write_storage_changesets: bool,
/// The current chain tip block number (for pruning).
pub tip: BlockNumber,
/// The prune mode for receipts, if any.
@@ -626,35 +622,6 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
Ok(())
}
/// Writes storage changesets for all blocks to the static file segment.
#[instrument(level = "debug", target = "providers::db", skip_all)]
fn write_storage_changesets(
w: &mut StaticFileProviderRWRefMut<'_, N>,
blocks: &[ExecutedBlock<N>],
) -> ProviderResult<()> {
for block in blocks {
let block_number = block.recovered_block().number();
let reverts = block.execution_outcome().state.reverts.to_plain_state_reverts();
for storage_block_reverts in reverts.storage {
let changeset = storage_block_reverts
.into_iter()
.flat_map(|revert| {
revert.storage_revert.into_iter().map(move |(key, revert_to_slot)| {
StorageBeforeTx {
address: revert.address,
key: B256::new(key.to_be_bytes()),
value: revert_to_slot.to_previous_value(),
}
})
})
.collect::<Vec<_>>();
w.append_storage_changeset(changeset, block_number)?;
}
}
Ok(())
}
/// Spawns a scoped thread that writes to a static file segment using the provided closure.
///
/// The closure receives a mutable reference to the segment writer. After the closure completes,
@@ -730,15 +697,6 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
)
});
let h_storage_changesets = ctx.write_storage_changesets.then(|| {
self.spawn_segment_writer(
s,
StaticFileSegment::StorageChangeSets,
first_block_number,
|w| Self::write_storage_changesets(w, blocks),
)
});
h_headers.join().map_err(|_| StaticFileWriterError::ThreadPanic("headers"))??;
h_txs.join().map_err(|_| StaticFileWriterError::ThreadPanic("transactions"))??;
if let Some(h) = h_senders {
@@ -751,10 +709,6 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
h.join()
.map_err(|_| StaticFileWriterError::ThreadPanic("account_changesets"))??;
}
if let Some(h) = h_storage_changesets {
h.join()
.map_err(|_| StaticFileWriterError::ThreadPanic("storage_changesets"))??;
}
Ok(())
})
}
@@ -1427,13 +1381,6 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
highest_tx,
highest_block,
)?,
StaticFileSegment::StorageChangeSets => self
.ensure_changeset_invariants_by_block::<_, tables::StorageChangeSets, _>(
provider,
segment,
highest_block,
|key| key.block_number(),
)?,
} {
debug!(target: "reth::providers::static_file", ?segment, unwind_target=unwind, "Invariants check returned unwind target");
update_unwind_target(unwind);
@@ -1515,13 +1462,6 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
}
true
}
StaticFileSegment::StorageChangeSets => {
if EitherWriter::storage_changesets_destination(provider).is_database() {
debug!(target: "reth::providers::static_file", ?segment, "Skipping storage changesets segment: changesets stored in database");
return false
}
true
}
}
}
@@ -1654,9 +1594,9 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
let stage_id = match segment {
StaticFileSegment::Headers => StageId::Headers,
StaticFileSegment::Transactions => StageId::Bodies,
StaticFileSegment::Receipts |
StaticFileSegment::AccountChangeSets |
StaticFileSegment::StorageChangeSets => StageId::Execution,
StaticFileSegment::Receipts | StaticFileSegment::AccountChangeSets => {
StageId::Execution
}
StaticFileSegment::TransactionSenders => StageId::SenderRecovery,
};
let checkpoint_block_number =
@@ -1711,9 +1651,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
StaticFileSegment::TransactionSenders => {
writer.prune_transaction_senders(number, checkpoint_block_number)?
}
StaticFileSegment::Headers |
StaticFileSegment::AccountChangeSets |
StaticFileSegment::StorageChangeSets => {
StaticFileSegment::Headers | StaticFileSegment::AccountChangeSets => {
unreachable!()
}
}
@@ -1724,9 +1662,6 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
StaticFileSegment::AccountChangeSets => {
writer.prune_account_changesets(checkpoint_block_number)?;
}
StaticFileSegment::StorageChangeSets => {
writer.prune_storage_changesets(checkpoint_block_number)?;
}
}
debug!(target: "reth::providers::static_file", ?segment, "Committing writer after pruning");
writer.commit()?;
@@ -1737,105 +1672,6 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
Ok(None)
}
fn ensure_changeset_invariants_by_block<Provider, T, F>(
&self,
provider: &Provider,
segment: StaticFileSegment,
highest_static_file_block: Option<BlockNumber>,
block_from_key: F,
) -> ProviderResult<Option<BlockNumber>>
where
Provider: DBProvider + BlockReader + StageCheckpointReader,
T: Table,
F: Fn(&T::Key) -> BlockNumber,
{
debug!(
target: "reth::providers::static_file",
?segment,
?highest_static_file_block,
"Ensuring changeset invariants"
);
let mut db_cursor = provider.tx_ref().cursor_read::<T>()?;
if let Some((db_first_key, _)) = db_cursor.first()? {
let db_first_block = block_from_key(&db_first_key);
if let Some(highest_block) = highest_static_file_block &&
!(db_first_block <= highest_block || highest_block + 1 == db_first_block)
{
info!(
target: "reth::providers::static_file",
?db_first_block,
?highest_block,
unwind_target = highest_block,
?segment,
"Setting unwind target."
);
return Ok(Some(highest_block))
}
if let Some((db_last_key, _)) = db_cursor.last()? &&
highest_static_file_block
.is_none_or(|highest_block| block_from_key(&db_last_key) > highest_block)
{
debug!(
target: "reth::providers::static_file",
?segment,
"Database has entries beyond static files, no unwind needed"
);
return Ok(None)
}
} else {
debug!(target: "reth::providers::static_file", ?segment, "No database entries found");
}
let highest_static_file_block = highest_static_file_block.unwrap_or_default();
let stage_id = match segment {
StaticFileSegment::Headers => StageId::Headers,
StaticFileSegment::Transactions => StageId::Bodies,
StaticFileSegment::Receipts |
StaticFileSegment::AccountChangeSets |
StaticFileSegment::StorageChangeSets => StageId::Execution,
StaticFileSegment::TransactionSenders => StageId::SenderRecovery,
};
let checkpoint_block_number =
provider.get_stage_checkpoint(stage_id)?.unwrap_or_default().block_number;
if checkpoint_block_number > highest_static_file_block {
info!(
target: "reth::providers::static_file",
checkpoint_block_number,
unwind_target = highest_static_file_block,
?segment,
"Setting unwind target."
);
return Ok(Some(highest_static_file_block))
}
if checkpoint_block_number < highest_static_file_block {
info!(
target: "reth::providers",
?segment,
from = highest_static_file_block,
to = checkpoint_block_number,
"Unwinding static file segment."
);
let mut writer = self.latest_writer(segment)?;
match segment {
StaticFileSegment::AccountChangeSets => {
writer.prune_account_changesets(checkpoint_block_number)?;
}
StaticFileSegment::StorageChangeSets => {
writer.prune_storage_changesets(checkpoint_block_number)?;
}
_ => unreachable!("invalid segment for changeset invariants"),
}
writer.commit()?;
}
Ok(None)
}
/// Returns the earliest available block number that has not been expired and is still
/// available.
///
@@ -2376,124 +2212,6 @@ impl<N: NodePrimitives> ChangeSetReader for StaticFileProvider<N> {
}
}
impl<N: NodePrimitives> StorageChangeSetReader for StaticFileProvider<N> {
fn storage_changeset(
&self,
block_number: BlockNumber,
) -> ProviderResult<Vec<(BlockNumberAddress, StorageEntry)>> {
let provider = match self.get_segment_provider_for_block(
StaticFileSegment::StorageChangeSets,
block_number,
None,
) {
Ok(provider) => provider,
Err(ProviderError::MissingStaticFileBlock(_, _)) => return Ok(Vec::new()),
Err(err) => return Err(err),
};
if let Some(offset) = provider.user_header().changeset_offset(block_number) {
let mut cursor = provider.cursor()?;
let mut changeset = Vec::with_capacity(offset.num_changes() as usize);
for i in offset.changeset_range() {
if let Some(change) = cursor.get_one::<StorageChangesetMask>(i.into())? {
let block_address = BlockNumberAddress((block_number, change.address));
let entry = StorageEntry { key: change.key, value: change.value };
changeset.push((block_address, entry));
}
}
Ok(changeset)
} else {
Ok(Vec::new())
}
}
fn get_storage_before_block(
&self,
block_number: BlockNumber,
address: Address,
storage_key: B256,
) -> ProviderResult<Option<StorageEntry>> {
let provider = match self.get_segment_provider_for_block(
StaticFileSegment::StorageChangeSets,
block_number,
None,
) {
Ok(provider) => provider,
Err(ProviderError::MissingStaticFileBlock(_, _)) => return Ok(None),
Err(err) => return Err(err),
};
let user_header = provider.user_header();
let Some(offset) = user_header.changeset_offset(block_number) else {
return Ok(None);
};
let mut cursor = provider.cursor()?;
let range = offset.changeset_range();
let mut low = range.start;
let mut high = range.end;
while low < high {
let mid = low + (high - low) / 2;
if let Some(change) = cursor.get_one::<StorageChangesetMask>(mid.into())? {
match (change.address, change.key).cmp(&(address, storage_key)) {
std::cmp::Ordering::Less => low = mid + 1,
_ => high = mid,
}
} else {
debug!(
target: "provider::static_file",
?low,
?mid,
?high,
?range,
?block_number,
?address,
?storage_key,
"Cannot continue binary search for storage changeset fetch"
);
low = range.end;
break;
}
}
if low < range.end &&
let Some(change) = cursor
.get_one::<StorageChangesetMask>(low.into())?
.filter(|change| change.address == address && change.key == storage_key)
{
return Ok(Some(StorageEntry { key: change.key, value: change.value }));
}
Ok(None)
}
fn storage_changesets_range(
&self,
range: RangeInclusive<BlockNumber>,
) -> ProviderResult<Vec<(BlockNumberAddress, StorageEntry)>> {
self.walk_storage_changeset_range(range).collect()
}
fn storage_changeset_count(&self) -> ProviderResult<usize> {
let mut count = 0;
let static_files = iter_static_files(&self.path).map_err(ProviderError::other)?;
if let Some(changeset_segments) = static_files.get(StaticFileSegment::StorageChangeSets) {
for (_, header) in changeset_segments {
if let Some(changeset_offsets) = header.changeset_offsets() {
for offset in changeset_offsets {
count += offset.num_changes() as usize;
}
}
}
}
Ok(count)
}
}
impl<N: NodePrimitives> StaticFileProvider<N> {
/// Creates an iterator for walking through account changesets in the specified block range.
///
@@ -2510,14 +2228,6 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
) -> StaticFileAccountChangesetWalker<Self> {
StaticFileAccountChangesetWalker::new(self.clone(), range)
}
/// Creates an iterator for walking through storage changesets in the specified block range.
pub fn walk_storage_changeset_range(
&self,
range: impl RangeBounds<BlockNumber>,
) -> StaticFileStorageChangesetWalker<Self> {
StaticFileStorageChangesetWalker::new(self.clone(), range)
}
}
impl<N: NodePrimitives<BlockHeader: Value>> HeaderProvider for StaticFileProvider<N> {

View File

@@ -69,19 +69,14 @@ mod tests {
use alloy_consensus::{Header, SignableTransaction, Transaction, TxLegacy};
use alloy_primitives::{Address, BlockHash, Signature, TxNumber, B256, U160, U256};
use rand::seq::SliceRandom;
use reth_db::{
models::{AccountBeforeTx, StorageBeforeTx},
test_utils::create_test_static_files_dir,
};
use reth_db::{models::AccountBeforeTx, test_utils::create_test_static_files_dir};
use reth_db_api::{transaction::DbTxMut, CanonicalHeaders, HeaderNumbers, Headers};
use reth_ethereum_primitives::{EthPrimitives, Receipt, TransactionSigned};
use reth_primitives_traits::Account;
use reth_static_file_types::{
find_fixed_range, SegmentRangeInclusive, DEFAULT_BLOCKS_PER_STATIC_FILE,
};
use reth_storage_api::{
ChangeSetReader, ReceiptProvider, StorageChangeSetReader, TransactionsProvider,
};
use reth_storage_api::{ChangeSetReader, ReceiptProvider, TransactionsProvider};
use reth_testing_utils::generators::{self, random_header_range};
use std::{collections::BTreeMap, fmt::Debug, fs, ops::Range, path::Path};
@@ -326,9 +321,7 @@ mod tests {
// Append transaction/receipt if there's still a transaction count to append
if tx_count > 0 {
match segment {
StaticFileSegment::Headers |
StaticFileSegment::AccountChangeSets |
StaticFileSegment::StorageChangeSets => {
StaticFileSegment::Headers | StaticFileSegment::AccountChangeSets => {
panic!("non tx based segment")
}
StaticFileSegment::Transactions => {
@@ -445,9 +438,7 @@ mod tests {
// Prune transactions or receipts based on the segment type
match segment {
StaticFileSegment::Headers |
StaticFileSegment::AccountChangeSets |
StaticFileSegment::StorageChangeSets => {
StaticFileSegment::Headers | StaticFileSegment::AccountChangeSets => {
panic!("non tx based segment")
}
StaticFileSegment::Transactions => {
@@ -472,9 +463,7 @@ mod tests {
// cumulative_gas_used & nonce as ids.
if let Some(id) = expected_tx_tip {
match segment {
StaticFileSegment::Headers |
StaticFileSegment::AccountChangeSets |
StaticFileSegment::StorageChangeSets => {
StaticFileSegment::Headers | StaticFileSegment::AccountChangeSets => {
panic!("non tx based segment")
}
StaticFileSegment::Transactions => assert_eyre(
@@ -1044,311 +1033,4 @@ mod tests {
}
}
}
#[test]
fn test_storage_changeset_static_files() {
let (static_dir, _) = create_test_static_files_dir();
let sf_rw = StaticFileProvider::<EthPrimitives>::read_write(&static_dir)
.expect("Failed to create static file provider");
// Test writing and reading storage changesets
{
let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
// Create test data for multiple blocks
let test_blocks = 10u64;
let entries_per_block = 5;
for block_num in 0..test_blocks {
let changeset = (0..entries_per_block)
.map(|i| {
let mut addr = Address::ZERO;
addr.0[0] = block_num as u8;
addr.0[1] = i as u8;
StorageBeforeTx {
address: addr,
key: B256::with_last_byte(i as u8),
value: U256::from(block_num * 1000 + i as u64),
}
})
.collect::<Vec<_>>();
writer.append_storage_changeset(changeset, block_num).unwrap();
}
writer.commit().unwrap();
}
// Verify data can be read back correctly
{
let provider = sf_rw
.get_segment_provider_for_block(StaticFileSegment::StorageChangeSets, 5, None)
.unwrap();
// Check that the segment header has changeset offsets
assert!(provider.user_header().changeset_offsets().is_some());
let offsets = provider.user_header().changeset_offsets().unwrap();
assert_eq!(offsets.len(), 10); // Should have 10 blocks worth of offsets
// Verify each block has the expected number of changes
for (i, offset) in offsets.iter().enumerate() {
assert_eq!(offset.num_changes(), 5, "Block {} should have 5 changes", i);
}
}
}
#[test]
fn test_get_storage_before_block() {
let (static_dir, _) = create_test_static_files_dir();
let sf_rw = StaticFileProvider::<EthPrimitives>::read_write(&static_dir)
.expect("Failed to create static file provider");
let test_address = Address::from([1u8; 20]);
let other_address = Address::from([2u8; 20]);
let missing_address = Address::from([3u8; 20]);
let test_key = B256::with_last_byte(1);
let other_key = B256::with_last_byte(2);
// Write changesets for multiple blocks
{
let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
// Block 0: test_address and other_address change
writer
.append_storage_changeset(
vec![
StorageBeforeTx { address: test_address, key: test_key, value: U256::ZERO },
StorageBeforeTx {
address: other_address,
key: other_key,
value: U256::from(5),
},
],
0,
)
.unwrap();
// Block 1: only other_address changes
writer
.append_storage_changeset(
vec![StorageBeforeTx {
address: other_address,
key: other_key,
value: U256::from(7),
}],
1,
)
.unwrap();
// Block 2: test_address changes again
writer
.append_storage_changeset(
vec![StorageBeforeTx {
address: test_address,
key: test_key,
value: U256::from(9),
}],
2,
)
.unwrap();
writer.commit().unwrap();
}
// Test get_storage_before_block
{
let result = sf_rw.get_storage_before_block(0, test_address, test_key).unwrap();
assert!(result.is_some());
let entry = result.unwrap();
assert_eq!(entry.key, test_key);
assert_eq!(entry.value, U256::ZERO);
let result = sf_rw.get_storage_before_block(2, test_address, test_key).unwrap();
assert!(result.is_some());
let entry = result.unwrap();
assert_eq!(entry.key, test_key);
assert_eq!(entry.value, U256::from(9));
let result = sf_rw.get_storage_before_block(1, test_address, test_key).unwrap();
assert!(result.is_none());
let result = sf_rw.get_storage_before_block(2, missing_address, test_key).unwrap();
assert!(result.is_none());
let result = sf_rw.get_storage_before_block(1, other_address, other_key).unwrap();
assert!(result.is_some());
let entry = result.unwrap();
assert_eq!(entry.key, other_key);
}
}
#[test]
fn test_storage_changeset_truncation() {
let (static_dir, _) = create_test_static_files_dir();
let blocks_per_file = 10;
let files_per_range = 3;
let file_set_count = 3;
let initial_file_count = files_per_range * file_set_count;
let tip = blocks_per_file * file_set_count - 1;
// Setup: Create storage changesets for multiple blocks
{
let sf_rw: StaticFileProvider<EthPrimitives> =
StaticFileProviderBuilder::read_write(&static_dir)
.with_blocks_per_file(blocks_per_file)
.build()
.expect("failed to create static file provider");
let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
for block_num in 0..=tip {
let num_changes = ((block_num % 5) + 1) as usize;
let mut changeset = Vec::with_capacity(num_changes);
for i in 0..num_changes {
let mut address = Address::ZERO;
address.0[0] = block_num as u8;
address.0[1] = i as u8;
changeset.push(StorageBeforeTx {
address,
key: B256::with_last_byte(i as u8),
value: U256::from(block_num * 1000 + i as u64),
});
}
writer.append_storage_changeset(changeset, block_num).unwrap();
}
writer.commit().unwrap();
}
fn validate_truncation(
sf_rw: &StaticFileProvider<EthPrimitives>,
static_dir: impl AsRef<Path>,
expected_tip: Option<u64>,
expected_file_count: u64,
) -> eyre::Result<()> {
let highest_block =
sf_rw.get_highest_static_file_block(StaticFileSegment::StorageChangeSets);
assert_eyre(highest_block, expected_tip, "block tip mismatch")?;
assert_eyre(
count_files_without_lockfile(static_dir)?,
expected_file_count as usize,
"file count mismatch",
)?;
if let Some(tip) = expected_tip {
let provider = sf_rw.get_segment_provider_for_block(
StaticFileSegment::StorageChangeSets,
tip,
None,
)?;
let offsets = provider.user_header().changeset_offsets();
assert!(offsets.is_some(), "Should have changeset offsets");
}
Ok(())
}
let sf_rw = StaticFileProviderBuilder::read_write(&static_dir)
.with_blocks_per_file(blocks_per_file)
.build()
.expect("failed to create static file provider");
sf_rw.initialize_index().expect("Failed to initialize index");
// Case 1: Truncate to block 20
{
let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
writer.prune_storage_changesets(20).unwrap();
writer.commit().unwrap();
validate_truncation(&sf_rw, &static_dir, Some(20), initial_file_count)
.expect("Truncation validation failed");
}
// Case 2: Truncate to block 9
{
let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
writer.prune_storage_changesets(9).unwrap();
writer.commit().unwrap();
validate_truncation(&sf_rw, &static_dir, Some(9), files_per_range)
.expect("Truncation validation failed");
}
// Case 3: Truncate all (should keep block 0)
{
let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
writer.prune_storage_changesets(0).unwrap();
writer.commit().unwrap();
validate_truncation(&sf_rw, &static_dir, Some(0), files_per_range)
.expect("Truncation validation failed");
}
}
#[test]
fn test_storage_changeset_binary_search() {
let (static_dir, _) = create_test_static_files_dir();
let sf_rw = StaticFileProvider::<EthPrimitives>::read_write(&static_dir)
.expect("Failed to create static file provider");
let block_num = 0u64;
let num_slots = 100;
let address = Address::from([4u8; 20]);
let mut keys: Vec<B256> = Vec::with_capacity(num_slots);
for i in 0..num_slots {
keys.push(B256::with_last_byte(i as u8));
}
{
let mut writer = sf_rw.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
let changeset = keys
.iter()
.enumerate()
.map(|(i, key)| StorageBeforeTx { address, key: *key, value: U256::from(i as u64) })
.collect::<Vec<_>>();
writer.append_storage_changeset(changeset, block_num).unwrap();
writer.commit().unwrap();
}
{
let result = sf_rw.get_storage_before_block(block_num, address, keys[0]).unwrap();
assert!(result.is_some());
let entry = result.unwrap();
assert_eq!(entry.key, keys[0]);
assert_eq!(entry.value, U256::from(0));
let result =
sf_rw.get_storage_before_block(block_num, address, keys[num_slots - 1]).unwrap();
assert!(result.is_some());
let entry = result.unwrap();
assert_eq!(entry.key, keys[num_slots - 1]);
let mid = num_slots / 2;
let result = sf_rw.get_storage_before_block(block_num, address, keys[mid]).unwrap();
assert!(result.is_some());
let entry = result.unwrap();
assert_eq!(entry.key, keys[mid]);
let missing_key = B256::with_last_byte(255);
let result = sf_rw.get_storage_before_block(block_num, address, missing_key).unwrap();
assert!(result.is_none());
for i in (0..num_slots).step_by(10) {
let result = sf_rw.get_storage_before_block(block_num, address, keys[i]).unwrap();
assert!(result.is_some());
assert_eq!(result.unwrap().key, keys[i]);
}
}
}
}

View File

@@ -6,7 +6,7 @@ use alloy_consensus::BlockHeader;
use alloy_primitives::{BlockHash, BlockNumber, TxNumber, U256};
use parking_lot::{lock_api::RwLockWriteGuard, RawRwLock, RwLock};
use reth_codecs::Compact;
use reth_db::models::{AccountBeforeTx, StorageBeforeTx};
use reth_db::models::AccountBeforeTx;
use reth_db_api::models::CompactU256;
use reth_nippy_jar::{NippyJar, NippyJarError, NippyJarWriter};
use reth_node_types::NodePrimitives;
@@ -56,11 +56,6 @@ enum PruneStrategy {
/// The target block number to prune to.
last_block: BlockNumber,
},
/// Prune storage changesets to a target block number.
StorageChangeSets {
/// The target block number to prune to.
last_block: BlockNumber,
},
}
/// Static file writers for every known [`StaticFileSegment`].
@@ -74,7 +69,6 @@ pub(crate) struct StaticFileWriters<N> {
receipts: RwLock<Option<StaticFileProviderRW<N>>>,
transaction_senders: RwLock<Option<StaticFileProviderRW<N>>>,
account_change_sets: RwLock<Option<StaticFileProviderRW<N>>>,
storage_change_sets: RwLock<Option<StaticFileProviderRW<N>>>,
}
impl<N> Default for StaticFileWriters<N> {
@@ -85,7 +79,6 @@ impl<N> Default for StaticFileWriters<N> {
receipts: Default::default(),
transaction_senders: Default::default(),
account_change_sets: Default::default(),
storage_change_sets: Default::default(),
}
}
}
@@ -102,7 +95,6 @@ impl<N: NodePrimitives> StaticFileWriters<N> {
StaticFileSegment::Receipts => self.receipts.write(),
StaticFileSegment::TransactionSenders => self.transaction_senders.write(),
StaticFileSegment::AccountChangeSets => self.account_change_sets.write(),
StaticFileSegment::StorageChangeSets => self.storage_change_sets.write(),
};
if write_guard.is_none() {
@@ -121,7 +113,6 @@ impl<N: NodePrimitives> StaticFileWriters<N> {
&self.receipts,
&self.transaction_senders,
&self.account_change_sets,
&self.storage_change_sets,
] {
let mut writer = writer_lock.write();
if let Some(writer) = writer.as_mut() {
@@ -140,7 +131,6 @@ impl<N: NodePrimitives> StaticFileWriters<N> {
&self.receipts,
&self.transaction_senders,
&self.account_change_sets,
&self.storage_change_sets,
] {
let writer = writer_lock.read();
if let Some(writer) = writer.as_ref() &&
@@ -165,7 +155,6 @@ impl<N: NodePrimitives> StaticFileWriters<N> {
&self.receipts,
&self.transaction_senders,
&self.account_change_sets,
&self.storage_change_sets,
] {
let mut writer = writer_lock.write();
if let Some(writer) = writer.as_mut() {
@@ -399,9 +388,6 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
PruneStrategy::AccountChangeSets { last_block } => {
self.prune_account_changeset_data(last_block)?
}
PruneStrategy::StorageChangeSets { last_block } => {
self.prune_storage_changeset_data(last_block)?
}
}
}
@@ -610,7 +596,7 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
/// Commits to the configuration file at the end
fn truncate_changesets(&mut self, last_block: u64) -> ProviderResult<()> {
let segment = self.writer.user_header().segment();
debug_assert!(segment.is_change_based());
debug_assert_eq!(segment, StaticFileSegment::AccountChangeSets);
// Get the current block range
let current_block_end = self
@@ -1090,41 +1076,6 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
Ok(())
}
/// Appends a block storage changeset to the static file.
///
/// It **CALLS** `increment_block()`.
pub fn append_storage_changeset(
&mut self,
mut changeset: Vec<StorageBeforeTx>,
block_number: u64,
) -> ProviderResult<()> {
debug_assert!(self.writer.user_header().segment() == StaticFileSegment::StorageChangeSets);
let start = Instant::now();
self.increment_block(block_number)?;
self.ensure_no_queued_prune()?;
// sort by address + storage key
changeset.sort_by_key(|change| (change.address, change.key));
let mut count: u64 = 0;
for change in changeset {
self.append_change(&change)?;
count += 1;
}
if let Some(metrics) = &self.metrics {
metrics.record_segment_operations(
StaticFileSegment::StorageChangeSets,
StaticFileProviderOperation::Append,
count,
Some(start.elapsed()),
);
}
Ok(())
}
/// Adds an instruction to prune `to_delete` transactions during commit.
///
/// Note: `last_block` refers to the block the unwinds ends at.
@@ -1176,12 +1127,6 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
self.queue_prune(PruneStrategy::AccountChangeSets { last_block })
}
/// Adds an instruction to prune storage changesets until the given block.
pub fn prune_storage_changesets(&mut self, last_block: u64) -> ProviderResult<()> {
debug_assert_eq!(self.writer.user_header().segment(), StaticFileSegment::StorageChangeSets);
self.queue_prune(PruneStrategy::StorageChangeSets { last_block })
}
/// Adds an instruction to prune elements during commit using the specified strategy.
fn queue_prune(&mut self, strategy: PruneStrategy) -> ProviderResult<()> {
self.ensure_no_queued_prune()?;
@@ -1241,25 +1186,6 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
Ok(())
}
/// Prunes the last storage changesets from the data file.
fn prune_storage_changeset_data(&mut self, last_block: BlockNumber) -> ProviderResult<()> {
let start = Instant::now();
debug_assert!(self.writer.user_header().segment() == StaticFileSegment::StorageChangeSets);
self.truncate_changesets(last_block)?;
if let Some(metrics) = &self.metrics {
metrics.record_segment_operation(
StaticFileSegment::StorageChangeSets,
StaticFileProviderOperation::Prune,
Some(start.elapsed()),
);
}
Ok(())
}
/// Prunes the last `to_delete` receipts from the data file.
fn prune_receipt_data(
&mut self,

View File

@@ -27,14 +27,14 @@ use reth_ethereum_primitives::EthPrimitives;
use reth_execution_types::ExecutionOutcome;
use reth_primitives_traits::{
Account, Block, BlockBody, Bytecode, GotExpected, NodePrimitives, RecoveredBlock, SealedHeader,
SignerRecoverable, StorageEntry,
SignerRecoverable,
};
use reth_prune_types::{PruneCheckpoint, PruneModes, PruneSegment};
use reth_stages_types::{StageCheckpoint, StageId};
use reth_storage_api::{
BlockBodyIndicesProvider, BytecodeReader, DBProvider, DatabaseProviderFactory,
HashedPostStateProvider, NodePrimitivesProvider, StageCheckpointReader, StateProofProvider,
StorageChangeSetReader, StorageRootProvider,
StorageRootProvider,
};
use reth_storage_errors::provider::{ConsistentViewError, ProviderError, ProviderResult};
use reth_trie::{
@@ -989,37 +989,6 @@ impl<T: NodePrimitives, ChainSpec: Send + Sync> ChangeSetReader for MockEthProvi
}
}
impl<T: NodePrimitives, ChainSpec: Send + Sync> StorageChangeSetReader
for MockEthProvider<T, ChainSpec>
{
fn storage_changeset(
&self,
_block_number: BlockNumber,
) -> ProviderResult<Vec<(reth_db_api::models::BlockNumberAddress, StorageEntry)>> {
Ok(Vec::default())
}
fn get_storage_before_block(
&self,
_block_number: BlockNumber,
_address: Address,
_storage_key: B256,
) -> ProviderResult<Option<StorageEntry>> {
Ok(None)
}
fn storage_changesets_range(
&self,
_range: RangeInclusive<BlockNumber>,
) -> ProviderResult<Vec<(reth_db_api::models::BlockNumberAddress, StorageEntry)>> {
Ok(Vec::default())
}
fn storage_changeset_count(&self) -> ProviderResult<usize> {
Ok(0)
}
}
impl<T: NodePrimitives, ChainSpec: Send + Sync> StateReader for MockEthProvider<T, ChainSpec> {
type Receipt = T::Receipt;

View File

@@ -10,18 +10,14 @@ use reth_chain_state::{
CanonStateSubscriptions, ForkChoiceSubscriptions, PersistedBlockSubscriptions,
};
use reth_node_types::{BlockTy, HeaderTy, NodeTypesWithDB, ReceiptTy, TxTy};
use reth_storage_api::{NodePrimitivesProvider, StorageChangeSetReader};
use reth_storage_api::NodePrimitivesProvider;
use std::fmt::Debug;
/// Helper trait to unify all provider traits for simplicity.
pub trait FullProvider<N: NodeTypesWithDB>:
DatabaseProviderFactory<
DB = N::DB,
Provider: BlockReader
+ StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader,
Provider: BlockReader + StageCheckpointReader + PruneCheckpointReader + ChangeSetReader,
> + NodePrimitivesProvider<Primitives = N::Primitives>
+ StaticFileProviderFactory<Primitives = N::Primitives>
+ RocksDBProviderFactory
@@ -36,7 +32,6 @@ pub trait FullProvider<N: NodeTypesWithDB>:
+ HashedPostStateProvider
+ ChainSpecProvider<ChainSpec = N::ChainSpec>
+ ChangeSetReader
+ StorageChangeSetReader
+ CanonStateSubscriptions
+ ForkChoiceSubscriptions<Header = HeaderTy<N>>
+ PersistedBlockSubscriptions
@@ -51,11 +46,7 @@ pub trait FullProvider<N: NodeTypesWithDB>:
impl<T, N: NodeTypesWithDB> FullProvider<N> for T where
T: DatabaseProviderFactory<
DB = N::DB,
Provider: BlockReader
+ StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader,
Provider: BlockReader + StageCheckpointReader + PruneCheckpointReader + ChangeSetReader,
> + NodePrimitivesProvider<Primitives = N::Primitives>
+ StaticFileProviderFactory<Primitives = N::Primitives>
+ RocksDBProviderFactory
@@ -70,7 +61,6 @@ impl<T, N: NodeTypesWithDB> FullProvider<N> for T where
+ HashedPostStateProvider
+ ChainSpecProvider<ChainSpec = N::ChainSpec>
+ ChangeSetReader
+ StorageChangeSetReader
+ CanonStateSubscriptions
+ ForkChoiceSubscriptions<Header = HeaderTy<N>>
+ PersistedBlockSubscriptions

Some files were not shown because too many files have changed in this diff Show More