Compare commits

..

1 Commits

Author SHA1 Message Date
Georgios Konstantopoulos
4bcd6e03db feat(trie): add trie_nodes_batch API to TrieNodeProvider trait
Adds batch node fetching capability to reduce O(n²) proof walks when
revealing multiple blinded nodes. Default implementation falls back to
sequential fetching.

Amp-Thread-ID: https://ampcode.com/threads/T-019bfe25-43f3-75ac-98f7-32bf937b69e1
Co-authored-by: Amp <amp@ampcode.com>
2026-01-27 07:34:41 +00:00
159 changed files with 1684 additions and 6492 deletions

View File

@@ -38,6 +38,6 @@ for pid in "${saving_pids[@]}"; do
done
# Make sure we don't rebuild images on the CI jobs
git apply ../.github/scripts/hive/no_sim_build.diff
git apply ../.github/assets/hive/no_sim_build.diff
go build .
mv ./hive ../hive_assets/

View File

@@ -44,24 +44,3 @@ jobs:
--exclude 'op-reth' \
--exclude 'reth' \
-E 'binary(e2e_testsuite)'
rocksdb:
name: e2e-rocksdb
runs-on: depot-ubuntu-latest-4
env:
RUST_BACKTRACE: 1
timeout-minutes: 60
steps:
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@stable
- uses: mozilla-actions/sccache-action@v0.0.9
- uses: taiki-e/install-action@nextest
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Run RocksDB e2e tests
run: |
cargo nextest run \
--locked --features "edge" \
-p reth-e2e-test-utils \
-E 'binary(rocksdb)'

View File

@@ -58,11 +58,11 @@ jobs:
uses: actions/cache@v5
with:
path: ./hive_assets
key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/scripts/hive/build_simulators.sh') }}
key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/assets/hive/build_simulators.sh') }}
- name: Build hive assets
if: steps.cache-hive.outputs.cache-hit != 'true'
run: .github/scripts/hive/build_simulators.sh
run: .github/assets/hive/build_simulators.sh
- name: Load cached Docker images
if: steps.cache-hive.outputs.cache-hit == 'true'
@@ -213,7 +213,7 @@ jobs:
path: /tmp
- name: Load Docker images
run: .github/scripts/hive/load_images.sh
run: .github/assets/hive/load_images.sh
- name: Move hive binary
run: |
@@ -241,11 +241,11 @@ jobs:
FILTER="/"
fi
echo "filter: $FILTER"
.github/scripts/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
.github/assets/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
- name: Parse hive output
run: |
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/scripts/hive/parse.py {} --exclusion .github/scripts/hive/expected_failures.yaml --ignored .github/scripts/hive/ignored_tests.yaml
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.yaml --ignored .github/assets/hive/ignored_tests.yaml
- name: Print simulator output
if: ${{ failure() }}

View File

@@ -22,7 +22,7 @@ concurrency:
jobs:
test:
name: test / ${{ matrix.network }} / ${{ matrix.storage }}
name: test / ${{ matrix.network }}
if: github.event_name != 'schedule'
runs-on: depot-ubuntu-latest-4
env:
@@ -30,17 +30,13 @@ jobs:
strategy:
matrix:
network: ["ethereum", "optimism"]
storage: ["stable", "edge"]
exclude:
- network: optimism
storage: edge
timeout-minutes: 60
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- name: Install Geth
run: .github/scripts/install_geth.sh
run: .github/assets/install_geth.sh
- uses: taiki-e/install-action@nextest
- uses: mozilla-actions/sccache-action@v0.0.9
- uses: Swatinem/rust-cache@v2
@@ -50,7 +46,7 @@ jobs:
name: Run tests
run: |
cargo nextest run \
--locked --features "asm-keccak ${{ matrix.network }} ${{ matrix.storage == 'edge' && 'edge' || '' }}" \
--locked --features "asm-keccak ${{ matrix.network }}" \
--workspace --exclude ef-tests \
-E "kind(test) and not binary(e2e_testsuite)"
- if: matrix.network == 'optimism'

View File

@@ -19,5 +19,5 @@ jobs:
uses: actions/github-script@v8
with:
script: |
const label_pr = require('./.github/scripts/label_pr.js')
const label_pr = require('./.github/assets/label_pr.js')
await label_pr({github, context})

View File

@@ -76,7 +76,7 @@ jobs:
- name: Run Wasm checks
run: |
sudo apt update && sudo apt install gcc-multilib
.github/scripts/check_wasm.sh
.github/assets/check_wasm.sh
riscv:
runs-on: depot-ubuntu-latest
@@ -94,7 +94,7 @@ jobs:
cache-on-failure: true
- uses: dcarbone/install-jq-action@v3
- name: Run RISC-V checks
run: .github/scripts/check_rv32imac.sh
run: .github/assets/check_rv32imac.sh
crate-checks:
name: crate-checks (${{ matrix.partition }}/${{ matrix.total_partitions }})

View File

@@ -43,7 +43,7 @@ jobs:
uses: docker/build-push-action@v6
with:
context: .
file: .github/scripts/hive/Dockerfile
file: .github/assets/hive/Dockerfile
tags: ${{ inputs.image_tag }}
outputs: type=docker,dest=./artifacts/reth_image.tar
build-args: |

157
Cargo.lock generated
View File

@@ -186,9 +186,9 @@ dependencies = [
[[package]]
name = "alloy-dyn-abi"
version = "1.5.4"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14ff5ee5f27aa305bda825c735f686ad71bb65508158f059f513895abe69b8c3"
checksum = "369f5707b958927176265e8a58627fc6195e5dfa5c55689396e68b241b3a72e6"
dependencies = [
"alloy-json-abi",
"alloy-primitives",
@@ -340,9 +340,9 @@ dependencies = [
[[package]]
name = "alloy-json-abi"
version = "1.5.4"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8708475665cc00e081c085886e68eada2f64cfa08fc668213a9231655093d4de"
checksum = "84e3cf01219c966f95a460c95f1d4c30e12f6c18150c21a30b768af2a2a29142"
dependencies = [
"alloy-primitives",
"alloy-sol-type-parser",
@@ -437,9 +437,9 @@ dependencies = [
[[package]]
name = "alloy-primitives"
version = "1.5.4"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b88cf92ed20685979ed1d8472422f0c6c2d010cec77caf63aaa7669cc1a7bc2"
checksum = "f6a0fb18dd5fb43ec5f0f6a20be1ce0287c79825827de5744afaa6c957737c33"
dependencies = [
"alloy-rlp",
"arbitrary",
@@ -464,6 +464,7 @@ dependencies = [
"rustc-hash",
"serde",
"sha3",
"tiny-keccak",
]
[[package]]
@@ -793,9 +794,9 @@ dependencies = [
[[package]]
name = "alloy-sol-macro"
version = "1.5.4"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5fa1ca7e617c634d2bd9fa71f9ec8e47c07106e248b9fcbd3eaddc13cabd625"
checksum = "09eb18ce0df92b4277291bbaa0ed70545d78b02948df756bbd3d6214bf39a218"
dependencies = [
"alloy-sol-macro-expander",
"alloy-sol-macro-input",
@@ -807,9 +808,9 @@ dependencies = [
[[package]]
name = "alloy-sol-macro-expander"
version = "1.5.4"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27c00c0c3a75150a9dc7c8c679ca21853a137888b4e1c5569f92d7e2b15b5102"
checksum = "95d9fa2daf21f59aa546d549943f10b5cce1ae59986774019fbedae834ffe01b"
dependencies = [
"alloy-sol-macro-input",
"const-hex",
@@ -818,16 +819,16 @@ dependencies = [
"proc-macro-error2",
"proc-macro2",
"quote",
"sha3",
"syn 2.0.114",
"syn-solidity",
"tiny-keccak",
]
[[package]]
name = "alloy-sol-macro-input"
version = "1.5.4"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "297db260eb4d67c105f68d6ba11b8874eec681caec5505eab8fbebee97f790bc"
checksum = "9396007fe69c26ee118a19f4dee1f5d1d6be186ea75b3881adf16d87f8444686"
dependencies = [
"const-hex",
"dunce",
@@ -841,9 +842,9 @@ dependencies = [
[[package]]
name = "alloy-sol-type-parser"
version = "1.5.4"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94b91b13181d3bcd23680fd29d7bc861d1f33fbe90fdd0af67162434aeba902d"
checksum = "af67a0b0dcebe14244fc92002cd8d96ecbf65db4639d479f5fcd5805755a4c27"
dependencies = [
"serde",
"winnow",
@@ -851,9 +852,9 @@ dependencies = [
[[package]]
name = "alloy-sol-types"
version = "1.5.4"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc442cc2a75207b708d481314098a0f8b6f7b58e3148dd8d8cc7407b0d6f9385"
checksum = "09aeea64f09a7483bdcd4193634c7e5cf9fd7775ee767585270cd8ce2d69dc95"
dependencies = [
"alloy-json-abi",
"alloy-primitives",
@@ -1039,15 +1040,6 @@ version = "1.0.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
[[package]]
name = "approx"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6"
dependencies = [
"num-traits",
]
[[package]]
name = "aquamarine"
version = "0.6.0"
@@ -2217,9 +2209,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.55"
version = "4.5.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e34525d5bbbd55da2bb745d34b36121baac88d07619a9a09cfcf4a6c0832785"
checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394"
dependencies = [
"clap_builder",
"clap_derive",
@@ -2227,9 +2219,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.55"
version = "4.5.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59a20016a20a3da95bef50ec7238dbd09baeef4311dcdd38ec15aba69812fb61"
checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00"
dependencies = [
"anstream",
"anstyle",
@@ -2239,9 +2231,9 @@ dependencies = [
[[package]]
name = "clap_derive"
version = "4.5.55"
version = "4.5.49"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5"
checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671"
dependencies = [
"heck",
"proc-macro2",
@@ -2266,42 +2258,35 @@ dependencies = [
[[package]]
name = "codspeed"
version = "4.3.0"
version = "2.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38c2eb3388ebe26b5a0ab6bf4969d9c4840143d7f6df07caa3cc851b0606cef6"
checksum = "93f4cce9c27c49c4f101fffeebb1826f41a9df2e7498b7cd4d95c0658b796c6c"
dependencies = [
"anyhow",
"cc",
"colored",
"getrandom 0.2.17",
"glob",
"libc",
"nix 0.30.1",
"serde",
"serde_json",
"statrs",
"uuid",
]
[[package]]
name = "codspeed-criterion-compat"
version = "4.3.0"
version = "2.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1e270597a1d1e183f86d1cc9f94f0133654ee3daf201c17903ee29363555dd7"
checksum = "c3c23d880a28a2aab52d38ca8481dd7a3187157d0a952196b6db1db3c8499725"
dependencies = [
"clap",
"codspeed",
"codspeed-criterion-compat-walltime",
"colored",
"futures",
"regex",
"tokio",
]
[[package]]
name = "codspeed-criterion-compat-walltime"
version = "4.3.0"
version = "2.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c2613d2fac930fe34456be76f9124ee0800bb9db2e7fd2d6c65b9ebe98a292"
checksum = "7b0a2f7365e347f4f22a67e9ea689bf7bc89900a354e22e26cf8a531a42c8fbb"
dependencies = [
"anes",
"cast",
@@ -3605,6 +3590,7 @@ version = "0.0.0"
dependencies = [
"alloy-eips",
"alloy-evm",
"alloy-sol-macro",
"alloy-sol-types",
"eyre",
"reth-ethereum",
@@ -4833,9 +4819,9 @@ dependencies = [
[[package]]
name = "iana-time-zone"
version = "0.1.65"
version = "0.1.64"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb"
dependencies = [
"android_system_properties",
"core-foundation-sys",
@@ -5503,9 +5489,9 @@ dependencies = [
[[package]]
name = "keccak-asm"
version = "0.1.5"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b646a74e746cd25045aa0fd42f4f7f78aa6d119380182c7e63a5593c4ab8df6f"
checksum = "505d1856a39b200489082f90d897c3f07c455563880bc5952e38eabf731c83b6"
dependencies = [
"digest 0.10.7",
"sha3-asm",
@@ -5993,9 +5979,9 @@ dependencies = [
[[package]]
name = "moka"
version = "0.12.13"
version = "0.12.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ac832c50ced444ef6be0767a008b02c106a909ba79d1d830501e94b96f6b7e"
checksum = "a3dec6bd31b08944e08b58fd99373893a6c17054d6f3ea5006cc894f4f4eee2a"
dependencies = [
"crossbeam-channel",
"crossbeam-epoch",
@@ -6109,12 +6095,9 @@ dependencies = [
[[package]]
name = "notify-types"
version = "2.1.0"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42b8cfee0e339a0337359f3c88165702ac6e600dc01c0cc9579a92d62b08477a"
dependencies = [
"bitflags 2.10.0",
]
checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d"
[[package]]
name = "ntapi"
@@ -8058,7 +8041,6 @@ dependencies = [
"derive_more",
"metrics",
"modular-bitfield",
"op-alloy-consensus",
"parity-scale-codec",
"proptest",
"proptest-arbitrary-interop",
@@ -8066,6 +8048,7 @@ dependencies = [
"reth-codecs",
"reth-db-models",
"reth-ethereum-primitives",
"reth-optimism-primitives",
"reth-primitives-traits",
"reth-prune-types",
"reth-stages-types",
@@ -8336,6 +8319,7 @@ dependencies = [
"reth-chainspec",
"reth-engine-primitives",
"reth-ethereum-engine-primitives",
"reth-optimism-chainspec",
"reth-payload-builder",
"reth-payload-primitives",
"reth-primitives-traits",
@@ -10264,7 +10248,6 @@ dependencies = [
"strum 0.27.2",
"thiserror 2.0.18",
"toml",
"tracing",
]
[[package]]
@@ -10542,7 +10525,9 @@ dependencies = [
"op-alloy-rpc-types",
"reth-ethereum-primitives",
"reth-evm",
"reth-optimism-primitives",
"reth-primitives-traits",
"reth-storage-api",
"serde_json",
"thiserror 2.0.18",
]
@@ -10732,7 +10717,6 @@ version = "1.10.2"
dependencies = [
"alloy-consensus",
"alloy-eips",
"alloy-genesis",
"alloy-primitives",
"alloy-rlp",
"assert_matches",
@@ -10752,7 +10736,6 @@ dependencies = [
"reth-consensus",
"reth-db",
"reth-db-api",
"reth-db-common",
"reth-downloaders",
"reth-era",
"reth-era-downloader",
@@ -10778,7 +10761,6 @@ dependencies = [
"reth-storage-api",
"reth-storage-errors",
"reth-testing-utils",
"reth-tracing",
"reth-trie",
"reth-trie-db",
"tempfile",
@@ -12221,9 +12203,9 @@ dependencies = [
[[package]]
name = "sha3-asm"
version = "0.1.5"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b31139435f327c93c6038ed350ae4588e2c70a13d50599509fee6349967ba35a"
checksum = "c28efc5e327c837aa837c59eae585fc250715ef939ac32881bcc11677cd02d46"
dependencies = [
"cc",
"cfg-if",
@@ -12335,9 +12317,9 @@ dependencies = [
[[package]]
name = "siphasher"
version = "1.0.2"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e"
checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
[[package]]
name = "sketches-ddsketch"
@@ -12453,16 +12435,6 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "statrs"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e"
dependencies = [
"approx",
"num-traits",
]
[[package]]
name = "strsim"
version = "0.11.1"
@@ -12542,9 +12514,9 @@ dependencies = [
[[package]]
name = "syn-solidity"
version = "1.5.4"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2379beea9476b89d0237078be761cf8e012d92d5ae4ae0c9a329f974838870fc"
checksum = "5f92d01b5de07eaf324f7fca61cc6bd3d82bbc1de5b6c963e6fe79e86f36580d"
dependencies = [
"paste",
"proc-macro2",
@@ -12852,6 +12824,15 @@ dependencies = [
"time-core",
]
[[package]]
name = "tiny-keccak"
version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
dependencies = [
"crunchy",
]
[[package]]
name = "tinystr"
version = "0.8.2"
@@ -13043,9 +13024,9 @@ checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
[[package]]
name = "tonic"
version = "0.14.3"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a286e33f82f8a1ee2df63f4fa35c0becf4a85a0cb03091a15fd7bf0b402dc94a"
checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203"
dependencies = [
"async-trait",
"base64 0.22.1",
@@ -13069,9 +13050,9 @@ dependencies = [
[[package]]
name = "tonic-prost"
version = "0.14.3"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6c55a2d6a14174563de34409c9f92ff981d006f56da9c6ecd40d9d4a31500b0"
checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67"
dependencies = [
"bytes",
"prost 0.14.3",
@@ -14458,18 +14439,18 @@ dependencies = [
[[package]]
name = "zerocopy"
version = "0.8.35"
version = "0.8.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdea86ddd5568519879b8187e1cf04e24fce28f7fe046ceecbce472ff19a2572"
checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.35"
version = "0.8.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c15e1b46eff7c6c91195752e0eeed8ef040e391cdece7c25376957d5f15df22"
checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1"
dependencies = [
"proc-macro2",
"quote",
@@ -14553,9 +14534,9 @@ dependencies = [
[[package]]
name = "zmij"
version = "1.0.17"
version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02aae0f83f69aafc94776e879363e9771d7ecbffe2c7fbb6c14c5e00dfe88439"
checksum = "dfcd145825aace48cff44a8844de64bf75feec3080e0aa5cdbde72961ae51a65"
[[package]]
name = "zstd"

View File

@@ -484,15 +484,15 @@ op-revm = { version = "15.0.0", default-features = false }
revm-inspectors = "0.34.1"
# eth
alloy-dyn-abi = "1.5.4"
alloy-primitives = { version = "1.5.4", default-features = false, features = ["map-foldhash"] }
alloy-sol-types = { version = "1.5.4", default-features = false }
alloy-chains = { version = "0.2.5", default-features = false }
alloy-dyn-abi = "1.5.2"
alloy-eip2124 = { version = "0.2.0", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-evm = { version = "0.27.0", default-features = false }
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
alloy-sol-macro = "1.5.0"
alloy-sol-types = { version = "1.5.0", default-features = false }
alloy-trie = { version = "0.9.1", default-features = false }
alloy-hardforks = "0.4.5"
@@ -671,7 +671,7 @@ tracing-opentelemetry = "0.32"
# misc-testing
arbitrary = "1.3"
assert_matches = "1.5.0"
criterion = { package = "codspeed-criterion-compat", version = "4.3" }
criterion = { package = "codspeed-criterion-compat", version = "2.7" }
insta = "1.41"
proptest = "1.7"
proptest-derive = "0.5"
@@ -733,7 +733,7 @@ snap = "1.1.1"
socket2 = { version = "0.5", default-features = false }
sysinfo = { version = "0.33", default-features = false }
tracing-journald = "0.3"
tracing-logfmt = "=0.3.5"
tracing-logfmt = "0.3.3"
tracing-samply = "0.1"
tracing-subscriber = { version = "0.3", default-features = false }
tracing-tracy = "0.11"

View File

@@ -5,7 +5,7 @@
# reth: --build-arg BINARY=reth
# op-reth: --build-arg BINARY=op-reth --build-arg MANIFEST_PATH=crates/optimism/bin
FROM rust:1 AS builder
FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef
WORKDIR /app
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
@@ -19,6 +19,14 @@ ENV RUSTC_WRAPPER=sccache
ENV SCCACHE_DIR=/sccache
ENV SCCACHE_WEBDAV_ENDPOINT=https://cache.depot.dev
# Builds a cargo-chef plan
FROM chef AS planner
COPY --exclude=.git . .
RUN cargo chef prepare --recipe-path recipe.json
FROM chef AS builder
COPY --from=planner /app/recipe.json recipe.json
# Binary to build (reth or op-reth)
ARG BINARY=reth
@@ -45,6 +53,13 @@ ENV VERGEN_GIT_SHA=$VERGEN_GIT_SHA
ENV VERGEN_GIT_DESCRIBE=$VERGEN_GIT_DESCRIBE
ENV VERGEN_GIT_DIRTY=$VERGEN_GIT_DIRTY
# Build dependencies
RUN --mount=type=secret,id=DEPOT_TOKEN,env=SCCACHE_WEBDAV_TOKEN \
--mount=type=cache,target=/usr/local/cargo/registry,sharing=shared \
--mount=type=cache,target=/usr/local/cargo/git,sharing=shared \
--mount=type=cache,target=$SCCACHE_DIR,sharing=shared \
cargo chef cook --profile $BUILD_PROFILE --features "$FEATURES" --locked --recipe-path recipe.json --manifest-path $MANIFEST_PATH/Cargo.toml
# Build application
COPY --exclude=.git . .
RUN --mount=type=secret,id=DEPOT_TOKEN,env=SCCACHE_WEBDAV_TOKEN \

View File

@@ -274,10 +274,10 @@ impl Args {
/// Get the default RPC URL for a given chain
const fn get_default_rpc_url(chain: &Chain) -> &'static str {
match chain.id() {
8453 => "https://base.reth.rs/rpc", // base
8453 => "https://base-mainnet.rpc.ithaca.xyz", // base
84532 => "https://base-sepolia.rpc.ithaca.xyz", // base-sepolia
27082 => "https://rpc.hoodi.ethpandaops.io", // hoodi
_ => "https://ethereum.reth.rs/rpc", // mainnet and fallback
_ => "https://reth-ethereum.ithaca.xyz/rpc", // mainnet and fallback
}
}

View File

@@ -132,24 +132,13 @@ impl<S: TransactionSource> TransactionCollector<S> {
/// Collect transactions starting from the given block number.
///
/// Skips blob transactions (type 3) and collects until target gas is reached.
/// Returns a `CollectionResult` with transactions, gas info, and next block.
pub async fn collect(&self, start_block: u64) -> eyre::Result<CollectionResult> {
self.collect_gas(start_block, self.target_gas).await
}
/// Collect transactions up to a specific gas target.
///
/// This is used both for initial collection and for retry top-ups.
pub async fn collect_gas(
&self,
start_block: u64,
gas_target: u64,
) -> eyre::Result<CollectionResult> {
let mut transactions: Vec<RawTransaction> = Vec::new();
/// Returns the collected raw transaction bytes, total gas used, and the next block number.
pub async fn collect(&self, start_block: u64) -> eyre::Result<(Vec<Bytes>, u64, u64)> {
let mut transactions: Vec<Bytes> = Vec::new();
let mut total_gas: u64 = 0;
let mut current_block = start_block;
while total_gas < gas_target {
while total_gas < self.target_gas {
let Some((block_txs, _)) = self.source.fetch_block_transactions(current_block).await?
else {
warn!(block = current_block, "Block not found, stopping");
@@ -162,12 +151,12 @@ impl<S: TransactionSource> TransactionCollector<S> {
continue;
}
if total_gas + tx.gas_used <= gas_target {
if total_gas + tx.gas_used <= self.target_gas {
transactions.push(tx.raw);
total_gas += tx.gas_used;
transactions.push(tx);
}
if total_gas >= gas_target {
if total_gas >= self.target_gas {
break;
}
}
@@ -175,7 +164,7 @@ impl<S: TransactionSource> TransactionCollector<S> {
current_block += 1;
// Stop early if remaining gas is under 1M (close enough to target)
let remaining_gas = gas_target.saturating_sub(total_gas);
let remaining_gas = self.target_gas.saturating_sub(total_gas);
if remaining_gas < 1_000_000 {
break;
}
@@ -183,12 +172,12 @@ impl<S: TransactionSource> TransactionCollector<S> {
info!(
total_txs = transactions.len(),
gas_sent = total_gas,
total_gas,
next_block = current_block,
"Finished collecting transactions"
);
Ok(CollectionResult { transactions, gas_sent: total_gas, next_block: current_block })
Ok((transactions, total_gas, current_block))
}
}
@@ -263,80 +252,6 @@ struct BuiltPayload {
envelope: ExecutionPayloadEnvelopeV4,
block_hash: B256,
timestamp: u64,
/// The actual gas used in the built block.
gas_used: u64,
}
/// Result of collecting transactions from blocks.
#[derive(Debug)]
pub struct CollectionResult {
/// Collected transactions with their gas info.
pub transactions: Vec<RawTransaction>,
/// Total gas sent (sum of historical `gas_used` for all collected txs).
pub gas_sent: u64,
/// Next block number to continue collecting from.
pub next_block: u64,
}
/// Constants for retry logic.
const MAX_BUILD_RETRIES: u32 = 5;
/// Maximum retries for fetching a transaction batch.
const MAX_FETCH_RETRIES: u32 = 5;
/// Tolerance: if `gas_used` is within 1M of target, don't retry.
const MIN_TARGET_SLACK: u64 = 1_000_000;
/// Maximum gas to request in retries (10x target as safety cap).
const MAX_ADDITIONAL_GAS_MULTIPLIER: u64 = 10;
/// Fetches a batch of transactions with retry logic.
///
/// Returns `None` if all retries are exhausted.
async fn fetch_batch_with_retry<S: TransactionSource>(
collector: &TransactionCollector<S>,
block: u64,
) -> Option<CollectionResult> {
for attempt in 1..=MAX_FETCH_RETRIES {
match collector.collect(block).await {
Ok(result) => return Some(result),
Err(e) => {
if attempt == MAX_FETCH_RETRIES {
warn!(attempt, error = %e, "Failed to fetch transactions after max retries");
return None;
}
warn!(attempt, error = %e, "Failed to fetch transactions, retrying...");
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
}
}
None
}
/// Outcome of a build attempt check.
enum RetryOutcome {
/// Payload is close enough to target gas.
Success,
/// Max retries reached, accept what we have.
MaxRetries,
/// Need more transactions with the specified gas amount.
NeedMore(u64),
}
/// Buffer for receiving transaction batches from the fetcher.
///
/// This abstracts over the channel to allow the main loop to request
/// batches on demand, including for retries.
struct TxBuffer {
receiver: mpsc::Receiver<CollectionResult>,
}
impl TxBuffer {
const fn new(receiver: mpsc::Receiver<CollectionResult>) -> Self {
Self { receiver }
}
/// Take the next available batch from the fetcher.
async fn take_batch(&mut self) -> Option<CollectionResult> {
self.receiver.recv().await
}
}
impl Command {
@@ -397,20 +312,19 @@ impl Command {
)
.await?;
} else {
// Single payload - collect transactions and build with retry
// Single payload - collect transactions and build
let tx_source = RpcTransactionSource::from_url(&self.rpc_url)?;
let collector = TransactionCollector::new(tx_source, self.target_gas);
let result = collector.collect(start_block).await?;
let (transactions, _total_gas, _next_block) = collector.collect(start_block).await?;
if result.transactions.is_empty() {
if transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected"));
}
self.execute_sequential_with_retry(
self.execute_sequential(
&auth_provider,
&testing_provider,
&collector,
result,
transactions,
parent_hash,
parent_timestamp,
)
@@ -421,34 +335,32 @@ impl Command {
Ok(())
}
/// Sequential execution path with retry logic for underfilled payloads.
async fn execute_sequential_with_retry<S: TransactionSource>(
/// Sequential execution path for single payload or no-execute mode.
async fn execute_sequential(
&self,
auth_provider: &RootProvider<AnyNetwork>,
testing_provider: &RootProvider<AnyNetwork>,
collector: &TransactionCollector<S>,
initial_result: CollectionResult,
transactions: Vec<Bytes>,
mut parent_hash: B256,
mut parent_timestamp: u64,
) -> eyre::Result<()> {
let mut current_result = initial_result;
for i in 0..self.count {
info!(
payload = i + 1,
total = self.count,
parent_hash = %parent_hash,
parent_timestamp = parent_timestamp,
"Building payload via testing_buildBlockV1"
);
let built = self
.build_with_retry(
testing_provider,
collector,
&mut current_result,
i,
parent_hash,
parent_timestamp,
)
.build_payload(testing_provider, &transactions, i, parent_hash, parent_timestamp)
.await?;
self.save_payload(&built)?;
if self.execute || self.count > 1 {
info!(payload = i + 1, block_hash = %built.block_hash, gas_used = built.gas_used, "Executing payload (newPayload + FCU)");
info!(payload = i + 1, block_hash = %built.block_hash, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, built.envelope, parent_hash).await?;
info!(payload = i + 1, "Payload executed successfully");
}
@@ -459,62 +371,7 @@ impl Command {
Ok(())
}
/// Build a payload with retry logic when `gas_used` is below target.
///
/// Uses the ratio of `gas_used/gas_sent` to estimate how many more transactions
/// are needed to hit the target gas.
async fn build_with_retry<S: TransactionSource>(
&self,
testing_provider: &RootProvider<AnyNetwork>,
collector: &TransactionCollector<S>,
result: &mut CollectionResult,
index: u64,
parent_hash: B256,
parent_timestamp: u64,
) -> eyre::Result<BuiltPayload> {
for attempt in 1..=MAX_BUILD_RETRIES {
let tx_bytes: Vec<Bytes> = result.transactions.iter().map(|t| t.raw.clone()).collect();
let gas_sent = result.gas_sent;
info!(
payload = index + 1,
attempt,
tx_count = tx_bytes.len(),
gas_sent,
parent_hash = %parent_hash,
"Building payload via testing_buildBlockV1"
);
let built = Self::build_payload_static(
testing_provider,
&tx_bytes,
index,
parent_hash,
parent_timestamp,
)
.await?;
match self.check_retry_outcome(&built, index, attempt, gas_sent) {
RetryOutcome::Success | RetryOutcome::MaxRetries => return Ok(built),
RetryOutcome::NeedMore(additional_gas) => {
let additional =
collector.collect_gas(result.next_block, additional_gas).await?;
result.transactions.extend(additional.transactions);
result.gas_sent = result.gas_sent.saturating_add(additional.gas_sent);
result.next_block = additional.next_block;
}
}
}
warn!(payload = index + 1, "Retry loop exited without returning a payload");
Err(eyre::eyre!("build_with_retry exhausted retries without result"))
}
/// Pipelined execution - fetches transactions in background, builds with retry.
///
/// The fetcher continuously produces transaction batches. The main loop consumes them,
/// builds payloads with retry logic (requesting more transactions if underfilled),
/// and executes each payload before moving to the next.
/// Pipelined execution - fetches transactions and builds payloads in background.
async fn execute_pipelined(
&self,
auth_provider: &RootProvider<AnyNetwork>,
@@ -523,220 +380,167 @@ impl Command {
initial_parent_hash: B256,
initial_parent_timestamp: u64,
) -> eyre::Result<()> {
// Create channel for transaction batches - fetcher sends CollectionResult
let (tx_sender, tx_receiver) = mpsc::channel::<CollectionResult>(self.prefetch_buffer);
// Create channel for transaction batches (one batch per payload)
let (tx_sender, mut tx_receiver) = mpsc::channel::<Vec<Bytes>>(self.prefetch_buffer);
// Spawn background task to continuously fetch transaction batches
let rpc_url = self.rpc_url.clone();
let target_gas = self.target_gas;
let count = self.count;
let fetcher_handle = tokio::spawn(async move {
let tx_source = match RpcTransactionSource::from_url(&rpc_url) {
Ok(source) => source,
Err(e) => {
warn!(error = %e, "Failed to create transaction source");
return None;
return;
}
};
let collector = TransactionCollector::new(tx_source, target_gas);
let mut current_block = start_block;
while let Some(batch) = fetch_batch_with_retry(&collector, current_block).await {
if batch.transactions.is_empty() {
info!(block = current_block, "Reached chain tip, stopping fetcher");
break;
}
for payload_idx in 0..count {
match collector.collect(current_block).await {
Ok((transactions, total_gas, next_block)) => {
info!(
payload = payload_idx + 1,
tx_count = transactions.len(),
total_gas,
blocks = format!("{}..{}", current_block, next_block),
"Fetched transactions"
);
current_block = next_block;
info!(
tx_count = batch.transactions.len(),
gas_sent = batch.gas_sent,
blocks = format!("{}..{}", current_block, batch.next_block),
"Fetched transaction batch"
);
current_block = batch.next_block;
if tx_sender.send(batch).await.is_err() {
break;
if tx_sender.send(transactions).await.is_err() {
break;
}
}
Err(e) => {
warn!(payload = payload_idx + 1, error = %e, "Failed to fetch transactions");
break;
}
}
}
Some(current_block)
});
// Transaction buffer: holds transactions from batches + any extras from retries
let mut tx_buffer = TxBuffer::new(tx_receiver);
let mut parent_hash = initial_parent_hash;
let mut parent_timestamp = initial_parent_timestamp;
let mut pending_build: Option<tokio::task::JoinHandle<eyre::Result<BuiltPayload>>> = None;
for i in 0..self.count {
// Get initial batch of transactions for this payload
let mut result = tx_buffer
.take_batch()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
let is_last = i == self.count - 1;
if result.transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 1));
}
// Get current payload (either from pending build or build now)
let current_payload = if let Some(handle) = pending_build.take() {
handle.await??
} else {
// First payload - wait for transactions and build synchronously
let transactions = tx_receiver
.recv()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
// Build with retry - may need to request more transactions
let built = self
.build_with_retry_buffered(
if transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 1));
}
info!(
payload = i + 1,
total = self.count,
parent_hash = %parent_hash,
parent_timestamp = parent_timestamp,
tx_count = transactions.len(),
"Building payload via testing_buildBlockV1"
);
self.build_payload(
testing_provider,
&mut tx_buffer,
&mut result,
&transactions,
i,
parent_hash,
parent_timestamp,
)
.await?;
.await?
};
self.save_payload(&built)?;
self.save_payload(&current_payload)?;
let current_block_hash = built.block_hash;
let current_timestamp = built.timestamp;
let current_block_hash = current_payload.block_hash;
let current_timestamp = current_payload.timestamp;
// Execute payload
info!(payload = i + 1, block_hash = %current_block_hash, gas_used = built.gas_used, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, built.envelope, parent_hash).await?;
// Execute current payload first
info!(payload = i + 1, block_hash = %current_block_hash, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, current_payload.envelope, parent_hash).await?;
info!(payload = i + 1, "Payload executed successfully");
// Start building next payload in background (if not last) - AFTER execution
if !is_last {
// Get transactions for next payload (should already be fetched or fetching)
let next_transactions = tx_receiver
.recv()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
if next_transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 2));
}
let testing_provider = testing_provider.clone();
let next_index = i + 1;
let total = self.count;
pending_build = Some(tokio::spawn(async move {
info!(
payload = next_index + 1,
total = total,
parent_hash = %current_block_hash,
parent_timestamp = current_timestamp,
tx_count = next_transactions.len(),
"Building payload via testing_buildBlockV1"
);
Self::build_payload_static(
&testing_provider,
&next_transactions,
next_index,
current_block_hash,
current_timestamp,
)
.await
}));
}
parent_hash = current_block_hash;
parent_timestamp = current_timestamp;
}
// Clean up the fetcher task
drop(tx_buffer);
drop(tx_receiver);
let _ = fetcher_handle.await;
Ok(())
}
/// Build a payload with retry logic, using the buffered transaction source.
async fn build_with_retry_buffered(
/// Build a single payload via `testing_buildBlockV1`.
async fn build_payload(
&self,
testing_provider: &RootProvider<AnyNetwork>,
tx_buffer: &mut TxBuffer,
result: &mut CollectionResult,
transactions: &[Bytes],
index: u64,
parent_hash: B256,
parent_timestamp: u64,
) -> eyre::Result<BuiltPayload> {
for attempt in 1..=MAX_BUILD_RETRIES {
let tx_bytes: Vec<Bytes> = result.transactions.iter().map(|t| t.raw.clone()).collect();
let gas_sent = result.gas_sent;
info!(
payload = index + 1,
attempt,
tx_count = tx_bytes.len(),
gas_sent,
parent_hash = %parent_hash,
"Building payload via testing_buildBlockV1"
);
let built = Self::build_payload_static(
testing_provider,
&tx_bytes,
index,
parent_hash,
parent_timestamp,
)
.await?;
match self.check_retry_outcome(&built, index, attempt, gas_sent) {
RetryOutcome::Success | RetryOutcome::MaxRetries => return Ok(built),
RetryOutcome::NeedMore(additional_gas) => {
let mut collected_gas = 0u64;
while collected_gas < additional_gas {
if let Some(batch) = tx_buffer.take_batch().await {
collected_gas += batch.gas_sent;
result.transactions.extend(batch.transactions);
result.gas_sent = result.gas_sent.saturating_add(batch.gas_sent);
result.next_block = batch.next_block;
} else {
warn!("Transaction fetcher exhausted, proceeding with available transactions");
break;
}
}
}
}
}
warn!(payload = index + 1, "Retry loop exited without returning a payload");
Err(eyre::eyre!("build_with_retry_buffered exhausted retries without result"))
Self::build_payload_static(
testing_provider,
transactions,
index,
parent_hash,
parent_timestamp,
)
.await
}
/// Determines the outcome of a build attempt.
fn check_retry_outcome(
&self,
built: &BuiltPayload,
index: u64,
attempt: u32,
gas_sent: u64,
) -> RetryOutcome {
let gas_used = built.gas_used;
if gas_used + MIN_TARGET_SLACK >= self.target_gas {
info!(
payload = index + 1,
gas_used,
target_gas = self.target_gas,
attempts = attempt,
"Payload built successfully"
);
return RetryOutcome::Success;
}
if attempt == MAX_BUILD_RETRIES {
warn!(
payload = index + 1,
gas_used,
target_gas = self.target_gas,
gas_sent,
"Underfilled after max retries, accepting payload"
);
return RetryOutcome::MaxRetries;
}
if gas_used == 0 {
warn!(
payload = index + 1,
"Zero gas used in payload, requesting fixed chunk of additional transactions"
);
return RetryOutcome::NeedMore(self.target_gas);
}
let gas_sent_needed_total =
(self.target_gas as u128 * gas_sent as u128).div_ceil(gas_used as u128) as u64;
let additional = gas_sent_needed_total.saturating_sub(gas_sent);
let additional = additional.min(self.target_gas * MAX_ADDITIONAL_GAS_MULTIPLIER);
if additional == 0 {
info!(
payload = index + 1,
gas_used,
target_gas = self.target_gas,
"No additional transactions needed based on ratio"
);
return RetryOutcome::Success;
}
let ratio = gas_used as f64 / gas_sent as f64;
info!(
payload = index + 1,
gas_used,
gas_sent,
ratio = format!("{:.4}", ratio),
additional_gas = additional,
"Underfilled, collecting more transactions for retry"
);
RetryOutcome::NeedMore(additional)
}
/// Build a single payload via `testing_buildBlockV1`.
/// Static version for use in spawned tasks.
async fn build_payload_static(
testing_provider: &RootProvider<AnyNetwork>,
transactions: &[Bytes],
@@ -774,9 +578,8 @@ impl Command {
let block_hash = inner.block_hash;
let block_number = inner.block_number;
let timestamp = inner.timestamp;
let gas_used = inner.gas_used;
Ok(BuiltPayload { block_number, envelope: v4_envelope, block_hash, timestamp, gas_used })
Ok(BuiltPayload { block_number, envelope: v4_envelope, block_hash, timestamp })
}
/// Save a payload to disk.

View File

@@ -1,33 +1,6 @@
//! Common helpers for reth-bench commands.
use crate::valid_payload::call_forkchoice_updated;
use eyre::Result;
use std::io::{BufReader, Read};
/// Read input from either a file path or stdin.
pub(crate) fn read_input(path: Option<&str>) -> Result<String> {
Ok(match path {
Some(path) => reth_fs_util::read_to_string(path)?,
None => String::from_utf8(
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
)?,
})
}
/// Load JWT secret from either a file or use the provided string directly.
pub(crate) fn load_jwt_secret(jwt_secret: Option<&str>) -> Result<Option<String>> {
match jwt_secret {
Some(secret) => {
// Try to read as file first
match std::fs::read_to_string(secret) {
Ok(contents) => Ok(Some(contents.trim().to_string())),
// If file read fails, use the string directly
Err(_) => Ok(Some(secret.to_string())),
}
}
None => Ok(None),
}
}
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
///

View File

@@ -15,7 +15,6 @@ pub use generate_big_block::{
mod new_payload_fcu;
mod new_payload_only;
mod output;
mod persistence_waiter;
mod replay_payloads;
mod send_invalid_payload;
mod send_payload;

View File

@@ -15,23 +15,28 @@ use crate::{
output::{
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
},
persistence_waiter::{
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
PERSISTENCE_CHECKPOINT_TIMEOUT,
},
},
valid_payload::{block_to_new_payload, call_forkchoice_updated, call_new_payload},
};
use alloy_provider::Provider;
use alloy_eips::BlockNumHash;
use alloy_network::Ethereum;
use alloy_provider::{Provider, RootProvider};
use alloy_pubsub::SubscriptionStream;
use alloy_rpc_client::RpcClient;
use alloy_rpc_types_engine::ForkchoiceState;
use alloy_transport_ws::WsConnect;
use clap::Parser;
use eyre::{Context, OptionExt};
use futures::StreamExt;
use humantime::parse_duration;
use reth_cli_runner::CliContext;
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
use reth_node_core::args::BenchmarkArgs;
use std::time::{Duration, Instant};
use tracing::{debug, info};
use url::Url;
const PERSISTENCE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(60);
/// `reth benchmark new-payload-fcu` command
#[derive(Debug, Parser)]
@@ -100,11 +105,7 @@ impl Command {
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
(Some(duration), _) => Some(PersistenceWaiter::with_duration(duration)),
(None, true) => {
let ws_url = derive_ws_rpc_url(
self.benchmark.ws_rpc_url.as_deref(),
&self.benchmark.engine_rpc_url,
)?;
let sub = setup_persistence_subscription(ws_url).await?;
let sub = self.setup_persistence_subscription().await?;
Some(PersistenceWaiter::with_subscription(
sub,
self.persistence_threshold,
@@ -244,22 +245,293 @@ impl Command {
results.into_iter().unzip();
if let Some(ref path) = self.benchmark.output {
write_benchmark_results(path, &gas_output_results, &combined_results)?;
write_benchmark_results(path, &gas_output_results, combined_results)?;
}
let gas_output =
TotalGasOutput::with_combined_results(gas_output_results, &combined_results)?;
let gas_output = TotalGasOutput::new(gas_output_results)?;
info!(
total_gas_used = gas_output.total_gas_used,
total_duration = ?gas_output.total_duration,
execution_duration = ?gas_output.execution_duration,
blocks_processed = gas_output.blocks_processed,
wall_clock_ggas_per_second = format_args!("{:.4}", gas_output.total_gigagas_per_second()),
execution_ggas_per_second = format_args!("{:.4}", gas_output.execution_gigagas_per_second()),
"Benchmark complete"
total_duration=?gas_output.total_duration,
total_gas_used=?gas_output.total_gas_used,
blocks_processed=?gas_output.blocks_processed,
"Total Ggas/s: {:.4}",
gas_output.total_gigagas_per_second()
);
Ok(())
}
/// Returns the websocket RPC URL used for the persistence subscription.
///
/// Preference:
/// - If `--ws-rpc-url` is provided, use it directly.
/// - Otherwise, derive a WS RPC URL from `--engine-rpc-url`.
///
/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
/// Since `BenchmarkArgs` only has the engine URL by default, we convert the scheme
/// (http→ws, https→wss) and force the port to 8546.
fn derive_ws_rpc_url(&self) -> eyre::Result<Url> {
if let Some(ref ws_url) = self.benchmark.ws_rpc_url {
let parsed: Url = ws_url
.parse()
.wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
Ok(parsed)
} else {
let derived = engine_url_to_ws_url(&self.benchmark.engine_rpc_url)?;
debug!(
target: "reth-bench",
engine_url = %self.benchmark.engine_rpc_url,
%derived,
"Derived WebSocket RPC URL from engine RPC URL"
);
Ok(derived)
}
}
/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
async fn setup_persistence_subscription(&self) -> eyre::Result<PersistenceSubscription> {
let ws_url = self.derive_ws_rpc_url()?;
info!("Connecting to WebSocket at {} for persistence subscription", ws_url);
let ws_connect = WsConnect::new(ws_url.to_string());
let client = RpcClient::connect_pubsub(ws_connect)
.await
.wrap_err("Failed to connect to WebSocket RPC endpoint")?;
let provider: RootProvider<Ethereum> = RootProvider::new(client);
let subscription = provider
.subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
.await
.wrap_err("Failed to subscribe to persistence notifications")?;
info!("Subscribed to persistence notifications");
Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
}
}
/// Converts an engine API URL to the default RPC websocket URL.
///
/// Transformations:
/// - `http` → `ws`
/// - `https` → `wss`
/// - `ws` / `wss` keep their scheme
/// - Port is always set to `8546`, reth's default RPC websocket port.
///
/// This is used when we only know the engine API URL (typically `:8551`) but
/// need to connect to the node's WS RPC endpoint for persistence events.
fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
let url: Url = engine_url
.parse()
.wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
let mut ws_url = url.clone();
match ws_url.scheme() {
"http" => ws_url
.set_scheme("ws")
.map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
"https" => ws_url
.set_scheme("wss")
.map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
"ws" | "wss" => {}
scheme => {
return Err(eyre::eyre!(
"Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
))
}
}
ws_url.set_port(Some(8546)).map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
Ok(ws_url)
}
/// Waits until the persistence subscription reports that `target` has been persisted.
///
/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
/// - the subscription stream ends unexpectedly, or
/// - `timeout` elapses before `target` is observed.
async fn wait_for_persistence(
stream: &mut SubscriptionStream<BlockNumHash>,
target: u64,
last_persisted: &mut u64,
timeout: Duration,
) -> eyre::Result<()> {
tokio::time::timeout(timeout, async {
while *last_persisted < target {
match stream.next().await {
Some(persisted) => {
*last_persisted = persisted.number;
debug!(
target: "reth-bench",
persisted_block = ?last_persisted,
"Received persistence notification"
);
}
None => {
return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
}
}
}
Ok(())
})
.await
.map_err(|_| {
eyre::eyre!(
"Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
target,
timeout,
last_persisted
)
})?
}
/// Wrapper that keeps both the subscription stream and the underlying provider alive.
/// The provider must be kept alive for the subscription to continue receiving events.
struct PersistenceSubscription {
_provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
}
impl PersistenceSubscription {
const fn new(
provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
) -> Self {
Self { _provider: provider, stream }
}
const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
&mut self.stream
}
}
/// Encapsulates the block waiting logic.
///
/// Provides a simple `on_block()` interface that handles both:
/// - Fixed duration waits (when `wait_time` is set)
/// - Persistence-based waits (when `subscription` is set)
///
/// For persistence mode, waits after every `(threshold + 1)` blocks.
struct PersistenceWaiter {
wait_time: Option<Duration>,
subscription: Option<PersistenceSubscription>,
blocks_sent: u64,
last_persisted: u64,
threshold: u64,
timeout: Duration,
}
impl PersistenceWaiter {
const fn with_duration(wait_time: Duration) -> Self {
Self {
wait_time: Some(wait_time),
subscription: None,
blocks_sent: 0,
last_persisted: 0,
threshold: 0,
timeout: Duration::ZERO,
}
}
const fn with_subscription(
subscription: PersistenceSubscription,
threshold: u64,
timeout: Duration,
) -> Self {
Self {
wait_time: None,
subscription: Some(subscription),
blocks_sent: 0,
last_persisted: 0,
threshold,
timeout,
}
}
/// Called once per block. Waits based on the configured mode.
#[allow(clippy::manual_is_multiple_of)]
async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
if let Some(wait_time) = self.wait_time {
tokio::time::sleep(wait_time).await;
return Ok(());
}
let Some(ref mut subscription) = self.subscription else {
return Ok(());
};
self.blocks_sent += 1;
if self.blocks_sent % (self.threshold + 1) == 0 {
debug!(
target: "reth-bench",
target_block = ?block_number,
last_persisted = self.last_persisted,
blocks_sent = self.blocks_sent,
"Waiting for persistence"
);
wait_for_persistence(
subscription.stream_mut(),
block_number,
&mut self.last_persisted,
self.timeout,
)
.await?;
debug!(
target: "reth-bench",
persisted = self.last_persisted,
"Persistence caught up"
);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_engine_url_to_ws_url() {
// http -> ws, always uses port 8546
let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
assert_eq!(result.as_str(), "ws://localhost:8546/");
// https -> wss
let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
assert_eq!(result.as_str(), "wss://localhost:8546/");
// Custom engine port still maps to 8546
let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
assert_eq!(result.port(), Some(8546));
// Already ws passthrough
let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
assert_eq!(result.scheme(), "ws");
// Invalid inputs
assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
assert!(engine_url_to_ws_url("not a valid url").is_err());
}
#[tokio::test]
async fn test_waiter_with_duration() {
let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
let start = Instant::now();
waiter.on_block(1).await.unwrap();
waiter.on_block(2).await.unwrap();
waiter.on_block(3).await.unwrap();
// Should have waited ~3ms total
assert!(start.elapsed() >= Duration::from_millis(3));
}
}

View File

@@ -6,7 +6,7 @@ use csv::Writer;
use eyre::OptionExt;
use reth_primitives_traits::constants::GIGAGAS;
use serde::{ser::SerializeStruct, Deserialize, Serialize};
use std::{fs, path::Path, time::Duration};
use std::{path::Path, time::Duration};
use tracing::info;
/// This is the suffix for gas output csv files.
@@ -158,58 +158,29 @@ pub(crate) struct TotalGasRow {
pub(crate) struct TotalGasOutput {
/// The total gas used in the benchmark.
pub(crate) total_gas_used: u64,
/// The total wall-clock duration of the benchmark (includes wait times).
/// The total duration of the benchmark.
pub(crate) total_duration: Duration,
/// The total execution-only duration (excludes wait times).
pub(crate) execution_duration: Duration,
/// The total gas used per second.
pub(crate) total_gas_per_second: f64,
/// The number of blocks processed.
pub(crate) blocks_processed: u64,
}
impl TotalGasOutput {
/// Create a new [`TotalGasOutput`] from gas rows only.
///
/// Use this when execution-only timing is not available (e.g., `new_payload_only`).
/// `execution_duration` will equal `total_duration`.
/// Create a new [`TotalGasOutput`] from a list of [`TotalGasRow`].
pub(crate) fn new(rows: Vec<TotalGasRow>) -> eyre::Result<Self> {
// the duration is obtained from the last row
let total_duration = rows.last().map(|row| row.time).ok_or_eyre("empty results")?;
let blocks_processed = rows.len() as u64;
let total_gas_used: u64 = rows.into_iter().map(|row| row.gas_used).sum();
let total_gas_per_second = total_gas_used as f64 / total_duration.as_secs_f64();
Ok(Self {
total_gas_used,
total_duration,
execution_duration: total_duration,
blocks_processed,
})
Ok(Self { total_gas_used, total_duration, total_gas_per_second, blocks_processed })
}
/// Create a new [`TotalGasOutput`] from gas rows and combined results.
///
/// - `rows`: Used for total gas and wall-clock duration
/// - `combined_results`: Used for execution-only duration (sum of `total_latency`)
pub(crate) fn with_combined_results(
rows: Vec<TotalGasRow>,
combined_results: &[CombinedResult],
) -> eyre::Result<Self> {
let total_duration = rows.last().map(|row| row.time).ok_or_eyre("empty results")?;
let blocks_processed = rows.len() as u64;
let total_gas_used: u64 = rows.into_iter().map(|row| row.gas_used).sum();
// Sum execution-only time from combined results
let execution_duration: Duration = combined_results.iter().map(|r| r.total_latency).sum();
Ok(Self { total_gas_used, total_duration, execution_duration, blocks_processed })
}
/// Return the total gigagas per second based on wall-clock time.
/// Return the total gigagas per second.
pub(crate) fn total_gigagas_per_second(&self) -> f64 {
self.total_gas_used as f64 / self.total_duration.as_secs_f64() / GIGAGAS as f64
}
/// Return the execution-only gigagas per second (excludes wait times).
pub(crate) fn execution_gigagas_per_second(&self) -> f64 {
self.total_gas_used as f64 / self.execution_duration.as_secs_f64() / GIGAGAS as f64
self.total_gas_per_second / GIGAGAS as f64
}
}
@@ -221,10 +192,8 @@ impl TotalGasOutput {
pub(crate) fn write_benchmark_results(
output_dir: &Path,
gas_results: &[TotalGasRow],
combined_results: &[CombinedResult],
combined_results: Vec<CombinedResult>,
) -> eyre::Result<()> {
fs::create_dir_all(output_dir)?;
let output_path = output_dir.join(COMBINED_OUTPUT_SUFFIX);
info!("Writing engine api call latency output to file: {:?}", output_path);
let mut writer = Writer::from_path(&output_path)?;

View File

@@ -1,304 +0,0 @@
//! Persistence waiting utilities for benchmarks.
//!
//! Provides waiting behavior to control benchmark pacing:
//! - **Fixed duration waits**: Sleep for a fixed time between blocks
//! - **Persistence-based waits**: Wait for blocks to be persisted using
//! `reth_subscribePersistedBlock` subscription
use alloy_eips::BlockNumHash;
use alloy_network::Ethereum;
use alloy_provider::{Provider, RootProvider};
use alloy_pubsub::SubscriptionStream;
use alloy_rpc_client::RpcClient;
use alloy_transport_ws::WsConnect;
use eyre::Context;
use futures::StreamExt;
use std::time::Duration;
use tracing::{debug, info};
/// Default `WebSocket` RPC port for reth.
const DEFAULT_WS_RPC_PORT: u16 = 8546;
use url::Url;
/// Default timeout for waiting on persistence.
pub(crate) const PERSISTENCE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(60);
/// Returns the websocket RPC URL used for the persistence subscription.
///
/// Preference:
/// - If `ws_rpc_url` is provided, use it directly.
/// - Otherwise, derive a WS RPC URL from `engine_rpc_url`.
///
/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
/// Since we may only have the engine URL by default, we convert the scheme
/// (http→ws, https→wss) and force the port to 8546.
pub(crate) fn derive_ws_rpc_url(
ws_rpc_url: Option<&str>,
engine_rpc_url: &str,
) -> eyre::Result<Url> {
if let Some(ws_url) = ws_rpc_url {
let parsed: Url = ws_url
.parse()
.wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
Ok(parsed)
} else {
let derived = engine_url_to_ws_url(engine_rpc_url)?;
debug!(
target: "reth-bench",
engine_url = %engine_rpc_url,
%derived,
"Derived WebSocket RPC URL from engine RPC URL"
);
Ok(derived)
}
}
/// Converts an engine API URL to the default RPC websocket URL.
///
/// Transformations:
/// - `http` → `ws`
/// - `https` → `wss`
/// - `ws` / `wss` keep their scheme
/// - Port is always set to `8546`, reth's default RPC websocket port.
///
/// This is used when we only know the engine API URL (typically `:8551`) but
/// need to connect to the node's WS RPC endpoint for persistence events.
fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
let url: Url = engine_url
.parse()
.wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
let mut ws_url = url.clone();
match ws_url.scheme() {
"http" => ws_url
.set_scheme("ws")
.map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
"https" => ws_url
.set_scheme("wss")
.map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
"ws" | "wss" => {}
scheme => {
return Err(eyre::eyre!(
"Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
))
}
}
ws_url
.set_port(Some(DEFAULT_WS_RPC_PORT))
.map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
Ok(ws_url)
}
/// Waits until the persistence subscription reports that `target` has been persisted.
///
/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
/// - the subscription stream ends unexpectedly, or
/// - `timeout` elapses before `target` is observed.
async fn wait_for_persistence(
stream: &mut SubscriptionStream<BlockNumHash>,
target: u64,
last_persisted: &mut u64,
timeout: Duration,
) -> eyre::Result<()> {
tokio::time::timeout(timeout, async {
while *last_persisted < target {
match stream.next().await {
Some(persisted) => {
*last_persisted = persisted.number;
debug!(
target: "reth-bench",
persisted_block = ?last_persisted,
"Received persistence notification"
);
}
None => {
return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
}
}
}
Ok(())
})
.await
.map_err(|_| {
eyre::eyre!(
"Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
target,
timeout,
last_persisted
)
})?
}
/// Wrapper that keeps both the subscription stream and the underlying provider alive.
/// The provider must be kept alive for the subscription to continue receiving events.
pub(crate) struct PersistenceSubscription {
_provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
}
impl PersistenceSubscription {
const fn new(
provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
) -> Self {
Self { _provider: provider, stream }
}
const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
&mut self.stream
}
}
/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
pub(crate) async fn setup_persistence_subscription(
ws_url: Url,
) -> eyre::Result<PersistenceSubscription> {
info!("Connecting to WebSocket at {} for persistence subscription", ws_url);
let ws_connect = WsConnect::new(ws_url.to_string());
let client = RpcClient::connect_pubsub(ws_connect)
.await
.wrap_err("Failed to connect to WebSocket RPC endpoint")?;
let provider: RootProvider<Ethereum> = RootProvider::new(client);
let subscription = provider
.subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
.await
.wrap_err("Failed to subscribe to persistence notifications")?;
info!("Subscribed to persistence notifications");
Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
}
/// Encapsulates the block waiting logic.
///
/// Provides a simple `on_block()` interface that handles both:
/// - Fixed duration waits (when `wait_time` is set)
/// - Persistence-based waits (when `subscription` is set)
///
/// For persistence mode, waits after every `(threshold + 1)` blocks.
pub(crate) struct PersistenceWaiter {
wait_time: Option<Duration>,
subscription: Option<PersistenceSubscription>,
blocks_sent: u64,
last_persisted: u64,
threshold: u64,
timeout: Duration,
}
impl PersistenceWaiter {
pub(crate) const fn with_duration(wait_time: Duration) -> Self {
Self {
wait_time: Some(wait_time),
subscription: None,
blocks_sent: 0,
last_persisted: 0,
threshold: 0,
timeout: Duration::ZERO,
}
}
pub(crate) const fn with_subscription(
subscription: PersistenceSubscription,
threshold: u64,
timeout: Duration,
) -> Self {
Self {
wait_time: None,
subscription: Some(subscription),
blocks_sent: 0,
last_persisted: 0,
threshold,
timeout,
}
}
/// Called once per block. Waits based on the configured mode.
#[allow(clippy::manual_is_multiple_of)]
pub(crate) async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
if let Some(wait_time) = self.wait_time {
tokio::time::sleep(wait_time).await;
return Ok(());
}
let Some(ref mut subscription) = self.subscription else {
return Ok(());
};
self.blocks_sent += 1;
if self.blocks_sent % (self.threshold + 1) == 0 {
debug!(
target: "reth-bench",
target_block = ?block_number,
last_persisted = self.last_persisted,
blocks_sent = self.blocks_sent,
"Waiting for persistence"
);
wait_for_persistence(
subscription.stream_mut(),
block_number,
&mut self.last_persisted,
self.timeout,
)
.await?;
debug!(
target: "reth-bench",
persisted = self.last_persisted,
"Persistence caught up"
);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Instant;
#[test]
fn test_engine_url_to_ws_url() {
// http -> ws, always uses port 8546
let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
assert_eq!(result.as_str(), "ws://localhost:8546/");
// https -> wss
let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
assert_eq!(result.as_str(), "wss://localhost:8546/");
// Custom engine port still maps to 8546
let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
assert_eq!(result.port(), Some(8546));
// Already ws passthrough
let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
assert_eq!(result.scheme(), "ws");
// Invalid inputs
assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
assert!(engine_url_to_ws_url("not a valid url").is_err());
}
#[tokio::test]
async fn test_waiter_with_duration() {
let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
let start = Instant::now();
waiter.on_block(1).await.unwrap();
waiter.on_block(2).await.unwrap();
waiter.on_block(3).await.unwrap();
// Should have waited ~3ms total
assert!(start.elapsed() >= Duration::from_millis(3));
}
}

View File

@@ -2,27 +2,10 @@
//!
//! This command reads `ExecutionPayloadEnvelopeV4` files from a directory and replays them
//! in sequence using `newPayload` followed by `forkchoiceUpdated`.
//!
//! Supports configurable waiting behavior:
//! - **`--wait-time`**: Fixed sleep interval between blocks.
//! - **`--wait-for-persistence`**: Waits for every Nth block to be persisted using the
//! `reth_subscribePersistedBlock` subscription, where N matches the engine's persistence
//! threshold. This ensures the benchmark doesn't outpace persistence.
//!
//! Both options can be used together or independently.
use crate::{
authenticated_transport::AuthenticatedTransportConnect,
bench::{
output::{
write_benchmark_results, CombinedResult, GasRampPayloadFile, NewPayloadResult,
TotalGasOutput, TotalGasRow,
},
persistence_waiter::{
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
PERSISTENCE_CHECKPOINT_TIMEOUT,
},
},
bench::output::GasRampPayloadFile,
valid_payload::{call_forkchoice_updated, call_new_payload},
};
use alloy_primitives::B256;
@@ -31,16 +14,11 @@ use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV4, ForkchoiceState, JwtSecret};
use clap::Parser;
use eyre::Context;
use humantime::parse_duration;
use reqwest::Url;
use reth_cli_runner::CliContext;
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
use reth_node_api::EngineApiMessageVersion;
use std::{
path::PathBuf,
time::{Duration, Instant},
};
use std::path::PathBuf;
use tracing::{debug, info};
use url::Url;
/// `reth bench replay-payloads` command
///
@@ -73,42 +51,6 @@ pub struct Command {
/// These are replayed before the main payloads to warm up the gas limit.
#[arg(long, value_name = "GAS_RAMP_DIR")]
gas_ramp_dir: Option<PathBuf>,
/// Optional output directory for benchmark results (CSV files).
#[arg(long, value_name = "OUTPUT")]
output: Option<PathBuf>,
/// How long to wait after a forkchoice update before sending the next payload.
#[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)]
wait_time: Option<Duration>,
/// Wait for blocks to be persisted before sending the next batch.
///
/// When enabled, waits for every Nth block to be persisted using the
/// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
/// doesn't outpace persistence.
///
/// The subscription uses the regular RPC websocket endpoint (no JWT required).
#[arg(long, default_value = "false", verbatim_doc_comment)]
wait_for_persistence: bool,
/// Engine persistence threshold used for deciding when to wait for persistence.
///
/// The benchmark waits after every `(threshold + 1)` blocks. By default this
/// matches the engine's `DEFAULT_PERSISTENCE_THRESHOLD` (2), so waits occur
/// at blocks 3, 6, 9, etc.
#[arg(
long = "persistence-threshold",
value_name = "PERSISTENCE_THRESHOLD",
default_value_t = DEFAULT_PERSISTENCE_THRESHOLD,
verbatim_doc_comment
)]
persistence_threshold: u64,
/// Optional `WebSocket` RPC URL for persistence subscription.
/// If not provided, derives from engine RPC URL by changing scheme to ws and port to 8546.
#[arg(long, value_name = "WS_RPC_URL", verbatim_doc_comment)]
ws_rpc_url: Option<String>,
}
/// A loaded payload ready for execution.
@@ -136,33 +78,6 @@ impl Command {
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
info!(payload_dir = %self.payload_dir.display(), "Replaying payloads");
// Log mode configuration
if let Some(duration) = self.wait_time {
info!("Using wait-time mode with {}ms delay between blocks", duration.as_millis());
}
if self.wait_for_persistence {
info!(
"Persistence waiting enabled (waits after every {} blocks to match engine gap > {} behavior)",
self.persistence_threshold + 1,
self.persistence_threshold
);
}
// Set up waiter based on configured options (duration takes precedence)
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
(Some(duration), _) => Some(PersistenceWaiter::with_duration(duration)),
(None, true) => {
let ws_url = derive_ws_rpc_url(self.ws_rpc_url.as_deref(), &self.engine_rpc_url)?;
let sub = setup_persistence_subscription(ws_url).await?;
Some(PersistenceWaiter::with_subscription(
sub,
self.persistence_threshold,
PERSISTENCE_CHECKPOINT_TIMEOUT,
))
}
(None, false) => None,
};
// Set up authenticated engine provider
let jwt =
std::fs::read_to_string(&self.jwt_secret).wrap_err("Failed to read JWT secret file")?;
@@ -229,11 +144,6 @@ impl Command {
call_forkchoice_updated(&auth_provider, payload.version, fcu_state, None).await?;
info!(gas_ramp_payload = i + 1, "Gas ramp payload executed successfully");
if let Some(w) = &mut waiter {
w.on_block(payload.block_number).await?;
}
parent_hash = payload.file.block_hash;
}
@@ -241,112 +151,22 @@ impl Command {
info!(count = gas_ramp_payloads.len(), "All gas ramp payloads replayed");
}
let mut results = Vec::new();
let total_benchmark_duration = Instant::now();
for (i, payload) in payloads.iter().enumerate() {
let envelope = &payload.envelope;
let block_hash = payload.block_hash;
let execution_payload = &envelope.envelope_inner.execution_payload;
let inner_payload = &execution_payload.payload_inner.payload_inner;
let gas_used = inner_payload.gas_used;
let gas_limit = inner_payload.gas_limit;
let block_number = inner_payload.block_number;
let transaction_count =
execution_payload.payload_inner.payload_inner.transactions.len() as u64;
debug!(
info!(
payload = i + 1,
total = payloads.len(),
index = payload.index,
block_hash = %block_hash,
block_hash = %payload.block_hash,
"Executing payload (newPayload + FCU)"
);
let start = Instant::now();
self.execute_payload_v4(&auth_provider, &payload.envelope, parent_hash).await?;
debug!(
method = "engine_newPayloadV4",
block_hash = %block_hash,
"Sending newPayload"
);
let status = auth_provider
.new_payload_v4(
execution_payload.clone(),
vec![],
B256::ZERO,
envelope.execution_requests.to_vec(),
)
.await?;
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
if !status.is_valid() {
return Err(eyre::eyre!("Payload rejected: {:?}", status));
}
let fcu_state = ForkchoiceState {
head_block_hash: block_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
};
debug!(method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
let fcu_result = auth_provider.fork_choice_updated_v3(fcu_state, None).await?;
let total_latency = start.elapsed();
let fcu_latency = total_latency - new_payload_result.latency;
let combined_result = CombinedResult {
block_number,
gas_limit,
transaction_count,
new_payload_result,
fcu_latency,
total_latency,
};
let current_duration = total_benchmark_duration.elapsed();
info!(%combined_result);
if let Some(w) = &mut waiter {
w.on_block(block_number).await?;
}
let gas_row =
TotalGasRow { block_number, transaction_count, gas_used, time: current_duration };
results.push((gas_row, combined_result));
debug!(?status, ?fcu_result, "Payload executed successfully");
parent_hash = block_hash;
info!(payload = i + 1, "Payload executed successfully");
parent_hash = payload.block_hash;
}
// Drop waiter - we don't need to wait for final blocks to persist
// since the benchmark goal is measuring Ggas/s of newPayload/FCU, not persistence.
drop(waiter);
let (gas_output_results, combined_results): (Vec<TotalGasRow>, Vec<CombinedResult>) =
results.into_iter().unzip();
if let Some(ref path) = self.output {
write_benchmark_results(path, &gas_output_results, &combined_results)?;
}
let gas_output =
TotalGasOutput::with_combined_results(gas_output_results, &combined_results)?;
info!(
total_gas_used = gas_output.total_gas_used,
total_duration = ?gas_output.total_duration,
execution_duration = ?gas_output.execution_duration,
blocks_processed = gas_output.blocks_processed,
wall_clock_ggas_per_second = format_args!("{:.4}", gas_output.total_gigagas_per_second()),
execution_ggas_per_second = format_args!("{:.4}", gas_output.execution_gigagas_per_second()),
"Benchmark complete"
);
info!(count = payloads.len(), "All payloads replayed successfully");
Ok(())
}
@@ -464,4 +284,49 @@ impl Command {
Ok(payloads)
}
async fn execute_payload_v4(
&self,
provider: &RootProvider<AnyNetwork>,
envelope: &ExecutionPayloadEnvelopeV4,
parent_hash: B256,
) -> eyre::Result<()> {
let block_hash =
envelope.envelope_inner.execution_payload.payload_inner.payload_inner.block_hash;
debug!(
method = "engine_newPayloadV4",
block_hash = %block_hash,
"Sending newPayload"
);
let status = provider
.new_payload_v4(
envelope.envelope_inner.execution_payload.clone(),
vec![],
B256::ZERO,
envelope.execution_requests.to_vec(),
)
.await?;
info!(?status, "newPayloadV4 response");
if !status.is_valid() {
return Err(eyre::eyre!("Payload rejected: {:?}", status));
}
let fcu_state = ForkchoiceState {
head_block_hash: block_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
};
debug!(method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
let fcu_result = provider.fork_choice_updated_v3(fcu_state, None).await?;
info!(?fcu_result, "forkchoiceUpdatedV3 response");
Ok(())
}
}

View File

@@ -3,7 +3,6 @@
mod invalidation;
use invalidation::InvalidationConfig;
use super::helpers::{load_jwt_secret, read_input};
use alloy_primitives::{Address, B256};
use alloy_provider::network::AnyRpcBlock;
use alloy_rpc_types_engine::ExecutionPayload;
@@ -11,7 +10,7 @@ use clap::Parser;
use eyre::{OptionExt, Result};
use op_alloy_consensus::OpTxEnvelope;
use reth_cli_runner::CliContext;
use std::io::Write;
use std::io::{BufReader, Read, Write};
/// Command for generating and sending an invalid `engine_newPayload` request.
///
@@ -181,6 +180,27 @@ enum Mode {
}
impl Command {
/// Read input from either a file or stdin
fn read_input(&self) -> Result<String> {
Ok(match &self.path {
Some(path) => reth_fs_util::read_to_string(path)?,
None => String::from_utf8(
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
)?,
})
}
/// Load JWT secret from either a file or use the provided string directly
fn load_jwt_secret(&self) -> Result<Option<String>> {
match &self.jwt_secret {
Some(secret) => match std::fs::read_to_string(secret) {
Ok(contents) => Ok(Some(contents.trim().to_string())),
Err(_) => Ok(Some(secret.clone())),
},
None => Ok(None),
}
}
/// Build `InvalidationConfig` from command flags
const fn build_invalidation_config(&self) -> InvalidationConfig {
InvalidationConfig {
@@ -216,8 +236,8 @@ impl Command {
/// Execute the command
pub async fn execute(self, _ctx: CliContext) -> Result<()> {
let block_json = read_input(self.path.as_deref())?;
let jwt_secret = load_jwt_secret(self.jwt_secret.as_deref())?;
let block_json = self.read_input()?;
let jwt_secret = self.load_jwt_secret()?;
let block = serde_json::from_str::<AnyRpcBlock>(&block_json)?
.into_inner()

View File

@@ -1,11 +1,10 @@
use super::helpers::{load_jwt_secret, read_input};
use alloy_provider::network::AnyRpcBlock;
use alloy_rpc_types_engine::ExecutionPayload;
use clap::Parser;
use eyre::{OptionExt, Result};
use op_alloy_consensus::OpTxEnvelope;
use reth_cli_runner::CliContext;
use std::io::Write;
use std::io::{BufReader, Read, Write};
/// Command for generating and sending an `engine_newPayload` request constructed from an RPC
/// block.
@@ -52,13 +51,38 @@ enum Mode {
}
impl Command {
/// Read input from either a file or stdin
fn read_input(&self) -> Result<String> {
Ok(match &self.path {
Some(path) => reth_fs_util::read_to_string(path)?,
None => String::from_utf8(
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
)?,
})
}
/// Load JWT secret from either a file or use the provided string directly
fn load_jwt_secret(&self) -> Result<Option<String>> {
match &self.jwt_secret {
Some(secret) => {
// Try to read as file first
match std::fs::read_to_string(secret) {
Ok(contents) => Ok(Some(contents.trim().to_string())),
// If file read fails, use the string directly
Err(_) => Ok(Some(secret.clone())),
}
}
None => Ok(None),
}
}
/// Execute the generate payload command
pub async fn execute(self, _ctx: CliContext) -> Result<()> {
// Load block
let block_json = read_input(self.path.as_deref())?;
let block_json = self.read_input()?;
// Load JWT secret
let jwt_secret = load_jwt_secret(self.jwt_secret.as_deref())?;
let jwt_secret = self.load_jwt_secret()?;
// Parse the block
let block = serde_json::from_str::<AnyRpcBlock>(&block_json)?

View File

@@ -260,9 +260,7 @@ pub(crate) async fn call_new_payload<N: Network, P: Provider<N>>(
while !status.is_valid() {
if status.is_invalid() {
error!(?status, ?params, "Invalid {method}",);
return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(std::io::Error::other(
format!("Invalid {method}: {status:?}"),
))))
panic!("Invalid {method}: {status:?}");
}
if status.is_syncing() {
return Err(alloy_json_rpc::RpcError::UnsupportedFeature(

View File

@@ -101,8 +101,8 @@ impl<N: NodeTypes> TableViewer<()> for ListTableViewer<'_, N> {
// We may be using the tui for a long time
tx.disable_long_read_transaction_safety();
let table_db = tx.inner().open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?;
let stats = tx.inner().db_stat(table_db.dbi()).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
let table_db = tx.inner.open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?;
let stats = tx.inner.db_stat(table_db.dbi()).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
let total_entries = stats.entries();
let final_entry_idx = total_entries.saturating_sub(1);
if self.args.skip > final_entry_idx {

View File

@@ -92,10 +92,10 @@ impl Command {
db_tables.sort();
let mut total_size = 0;
for db_table in db_tables {
let table_db = tx.inner().open_db(Some(db_table)).wrap_err("Could not open db.")?;
let table_db = tx.inner.open_db(Some(db_table)).wrap_err("Could not open db.")?;
let stats = tx
.inner()
.inner
.db_stat(table_db.dbi())
.wrap_err(format!("Could not find table: {db_table}"))?;
@@ -136,9 +136,9 @@ impl Command {
.add_cell(Cell::new(human_bytes(total_size as f64)));
table.add_row(row);
let freelist = tx.inner().env().freelist()?;
let freelist = tx.inner.env().freelist()?;
let pagesize =
tx.inner().db_stat(mdbx::Database::freelist_db().dbi())?.page_size() as usize;
tx.inner.db_stat(mdbx::Database::freelist_db().dbi())?.page_size() as usize;
let freelist_size = freelist * pagesize;
let mut row = Row::new();

View File

@@ -1,16 +1,8 @@
//! Command that runs pruning without any limits.
use crate::common::{AccessRights, CliNodeTypes, EnvironmentArgs};
use clap::Parser;
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks};
use reth_chainspec::{EthChainSpec, EthereumHardforks};
use reth_cli::chainspec::ChainSpecParser;
use reth_cli_runner::CliContext;
use reth_node_builder::common::metrics_hooks;
use reth_node_core::{args::MetricArgs, version::version_metadata};
use reth_node_metrics::{
chain::ChainSpecInfo,
server::{MetricServer, MetricServerConfig},
version::VersionInfo,
};
use reth_prune::PrunerBuilder;
use reth_static_file::StaticFileProducer;
use std::sync::Arc;
@@ -21,42 +13,14 @@ use tracing::info;
pub struct PruneCommand<C: ChainSpecParser> {
#[command(flatten)]
env: EnvironmentArgs<C>,
/// Prometheus metrics configuration.
#[command(flatten)]
metrics: MetricArgs,
}
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneCommand<C> {
/// Execute the `prune` command
pub async fn execute<N: CliNodeTypes<ChainSpec = C::ChainSpec>>(
self,
ctx: CliContext,
) -> eyre::Result<()> {
pub async fn execute<N: CliNodeTypes<ChainSpec = C::ChainSpec>>(self) -> eyre::Result<()> {
let env = self.env.init::<N>(AccessRights::RW)?;
let provider_factory = env.provider_factory;
let config = env.config.prune;
let data_dir = env.data_dir;
if let Some(listen_addr) = self.metrics.prometheus {
let config = MetricServerConfig::new(
listen_addr,
VersionInfo {
version: version_metadata().cargo_pkg_version.as_ref(),
build_timestamp: version_metadata().vergen_build_timestamp.as_ref(),
cargo_features: version_metadata().vergen_cargo_features.as_ref(),
git_sha: version_metadata().vergen_git_sha.as_ref(),
target_triple: version_metadata().vergen_cargo_target_triple.as_ref(),
build_profile: version_metadata().build_profile_name.as_ref(),
},
ChainSpecInfo { name: provider_factory.chain_spec().chain().to_string() },
ctx.task_executor,
metrics_hooks(&provider_factory),
data_dir.pprof_dumps(),
);
MetricServer::new(config).serve().await?;
}
// Copy data from database to static files
info!(target: "reth::cli", "Copying data from database to static files...");

View File

@@ -125,10 +125,7 @@ pub async fn setup_engine_with_chain_import(
db.clone(),
chain_spec.clone(),
reth_provider::providers::StaticFileProvider::read_write(static_files_path.clone())?,
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
.with_default_tables()
.build()
.unwrap(),
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path).build().unwrap(),
)?;
// Initialize genesis if needed
@@ -331,7 +328,6 @@ mod tests {
reth_provider::providers::StaticFileProvider::read_write(static_files_path.clone())
.unwrap(),
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path.clone())
.with_default_tables()
.build()
.unwrap(),
)
@@ -396,7 +392,6 @@ mod tests {
reth_provider::providers::StaticFileProvider::read_only(static_files_path, false)
.unwrap(),
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
.with_default_tables()
.build()
.unwrap(),
)
@@ -495,10 +490,7 @@ mod tests {
db.clone(),
chain_spec.clone(),
reth_provider::providers::StaticFileProvider::read_write(static_files_path).unwrap(),
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
.with_default_tables()
.build()
.unwrap(),
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path).build().unwrap(),
)
.expect("failed to create provider factory");

View File

@@ -10,10 +10,11 @@ use jsonrpsee::core::client::ClientT;
use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET};
use reth_db::tables;
use reth_e2e_test_utils::{transaction::TransactionTestContext, wallet, E2ETestSetupBuilder};
use reth_node_builder::NodeConfig;
use reth_node_core::args::RocksDbArgs;
use reth_node_ethereum::EthereumNode;
use reth_payload_builder::EthPayloadBuilderAttributes;
use reth_provider::{RocksDBProviderFactory, StorageSettings};
use reth_provider::RocksDBProviderFactory;
use std::{sync::Arc, time::Duration};
const ROCKSDB_POLL_TIMEOUT: Duration = Duration::from_secs(60);
@@ -96,24 +97,16 @@ fn test_attributes_generator(timestamp: u64) -> EthPayloadBuilderAttributes {
EthPayloadBuilderAttributes::new(B256::ZERO, attributes)
}
/// Verifies that `RocksDB` CLI defaults match `StorageSettings::base()`.
#[test]
fn test_rocksdb_defaults_match_storage_settings() {
let args = RocksDbArgs::default();
let settings = StorageSettings::base();
assert_eq!(
args.tx_hash, settings.transaction_hash_numbers_in_rocksdb,
"tx_hash default should match StorageSettings::base()"
);
assert_eq!(
args.storages_history, settings.storages_history_in_rocksdb,
"storages_history default should match StorageSettings::base()"
);
assert_eq!(
args.account_history, settings.account_history_in_rocksdb,
"account_history default should match StorageSettings::base()"
);
/// Enables `RocksDB` for `TransactionHashNumbers` table.
///
/// Note: Static file changesets are disabled because `persistence_threshold(0)` causes
/// a race where the static file writer expects sequential block numbers but receives
/// them out of order, resulting in `UnexpectedStaticFileBlockNumber` errors.
fn with_rocksdb_enabled<C>(mut config: NodeConfig<C>) -> NodeConfig<C> {
config.rocksdb = RocksDbArgs { tx_hash: true, ..Default::default() };
config.static_files.storage_changesets = false;
config.static_files.account_changesets = false;
config
}
/// Smoke test: node boots with `RocksDB` routing enabled.
@@ -125,6 +118,7 @@ async fn test_rocksdb_node_startup() -> Result<()> {
let (nodes, _tasks, _wallet) =
E2ETestSetupBuilder::<EthereumNode, _>::new(1, chain_spec, test_attributes_generator)
.with_node_config_modifier(with_rocksdb_enabled)
.build()
.await?;
@@ -152,6 +146,7 @@ async fn test_rocksdb_block_mining() -> Result<()> {
let (mut nodes, _tasks, _wallet) =
E2ETestSetupBuilder::<EthereumNode, _>::new(1, chain_spec, test_attributes_generator)
.with_node_config_modifier(with_rocksdb_enabled)
.build()
.await?;
@@ -208,6 +203,7 @@ async fn test_rocksdb_transaction_queries() -> Result<()> {
chain_spec.clone(),
test_attributes_generator,
)
.with_node_config_modifier(with_rocksdb_enabled)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
@@ -274,6 +270,7 @@ async fn test_rocksdb_multi_tx_same_block() -> Result<()> {
chain_spec.clone(),
test_attributes_generator,
)
.with_node_config_modifier(with_rocksdb_enabled)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
@@ -341,6 +338,7 @@ async fn test_rocksdb_txs_across_blocks() -> Result<()> {
chain_spec.clone(),
test_attributes_generator,
)
.with_node_config_modifier(with_rocksdb_enabled)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
@@ -425,6 +423,7 @@ async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> {
chain_spec.clone(),
test_attributes_generator,
)
.with_node_config_modifier(with_rocksdb_enabled)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
@@ -469,123 +468,3 @@ async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> {
Ok(())
}
/// Reorg with `RocksDB`: verifies that unwind correctly reads changesets from
/// storage-aware locations (static files vs MDBX) rather than directly from MDBX.
///
/// This test exercises `unwind_trie_state_from` which previously failed with
/// `UnsortedInput` errors because it read changesets directly from MDBX tables
/// instead of using storage-aware methods that check `storage_changesets_in_static_files`.
#[tokio::test]
async fn test_rocksdb_reorg_unwind() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
1,
chain_spec.clone(),
test_attributes_generator,
)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
assert_eq!(nodes.len(), 1);
// Use two separate wallets to avoid nonce conflicts during reorg
let wallets = wallet::Wallet::new(2).with_chain_id(chain_id).wallet_gen();
let signer1 = wallets[0].clone();
let signer2 = wallets[1].clone();
let client = nodes[0].rpc_client().expect("RPC client");
// Mine block 1 with a transaction from signer1
let raw_tx1 =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 0).await;
let tx_hash1 = nodes[0].rpc.inject_tx(raw_tx1).await?;
wait_for_pending_tx(&client, tx_hash1).await;
let payload1 = nodes[0].advance_block().await?;
let block1_hash = payload1.block().hash();
assert_eq!(payload1.block().number(), 1);
// Poll until tx1 appears in RocksDB (ensures persistence happened)
let tx_number1 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash1).await;
assert_eq!(tx_number1, 0, "First tx should have tx_number 0");
// Mine block 2 with transaction from signer1 (nonce 1)
let raw_tx2 =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 1).await;
let tx_hash2 = nodes[0].rpc.inject_tx(raw_tx2).await?;
wait_for_pending_tx(&client, tx_hash2).await;
let payload2 = nodes[0].advance_block().await?;
assert_eq!(payload2.block().number(), 2);
// Poll until tx2 appears in RocksDB
let tx_number2 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash2).await;
assert_eq!(tx_number2, 1, "Second tx should have tx_number 1");
// Mine block 3 with transaction from signer1 (nonce 2)
let raw_tx3 =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 2).await;
let tx_hash3 = nodes[0].rpc.inject_tx(raw_tx3).await?;
wait_for_pending_tx(&client, tx_hash3).await;
let payload3 = nodes[0].advance_block().await?;
assert_eq!(payload3.block().number(), 3);
// Poll until tx3 appears in RocksDB
let tx_number3 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash3).await;
assert_eq!(tx_number3, 2, "Third tx should have tx_number 2");
// Now create an alternate block 2 using signer2 (different wallet, avoids nonce conflict)
// Inject a tx from signer2 (nonce 0) before building the alternate block
let raw_alt_tx =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer2.clone(), 0).await;
let alt_tx_hash = nodes[0].rpc.inject_tx(raw_alt_tx).await?;
wait_for_pending_tx(&client, alt_tx_hash).await;
// Build an alternate payload (this builds on top of the current head, i.e., block 3)
// But we want to reorg back to block 1, so we'll use the payload and then FCU to it
let alt_payload = nodes[0].new_payload().await?;
let alt_block_hash = nodes[0].submit_payload(alt_payload.clone()).await?;
// Trigger reorg: make the alternate chain canonical by sending FCU pointing to block 1's hash
// as finalized, which should trigger an unwind of blocks 2 and 3
// The alt block becomes the new head
nodes[0].update_forkchoice(block1_hash, alt_block_hash).await?;
// Give time for the reorg to complete
tokio::time::sleep(Duration::from_millis(500)).await;
// Verify we can still query transactions and the chain is consistent
// If unwind_trie_state_from failed, this would have errored during reorg
let latest: Option<alloy_rpc_types_eth::Block> =
client.request("eth_getBlockByNumber", ("latest", false)).await?;
let latest = latest.expect("Latest block should exist");
// The alt block is at height 4 (on top of block 3)
assert!(latest.header.number >= 3, "Should be at height >= 3 after operation");
// tx1 from block 1 should still be there
let tx1: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash1]).await?;
assert!(tx1.is_some(), "tx1 from block 1 should still be queryable");
assert_eq!(tx1.unwrap().block_number, Some(1));
// Mine another block to verify the chain can continue
let raw_tx_final =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer2.clone(), 1).await;
let tx_hash_final = nodes[0].rpc.inject_tx(raw_tx_final).await?;
wait_for_pending_tx(&client, tx_hash_final).await;
let final_payload = nodes[0].advance_block().await?;
assert!(final_payload.block().number() > 3, "Should be able to mine block after reorg");
// Verify tx_final is included
let tx_final: Option<Transaction> =
client.request("eth_getTransactionByHash", [tx_hash_final]).await?;
assert!(tx_final.is_some(), "final tx should be in latest block");
Ok(())
}

View File

@@ -32,7 +32,9 @@ futures-util.workspace = true
# misc
eyre.workspace = true
tracing.workspace = true
op-alloy-rpc-types-engine = { workspace = true, optional = true }
reth-optimism-chainspec = { workspace = true, optional = true }
[lints]
workspace = true
@@ -40,6 +42,7 @@ workspace = true
[features]
op = [
"dep:op-alloy-rpc-types-engine",
"dep:reth-optimism-chainspec",
"reth-payload-primitives/op",
"reth-primitives-traits/op",
]

View File

@@ -72,18 +72,13 @@ where
&self,
parent: &SealedHeader<ChainSpec::Header>,
) -> op_alloy_rpc_types_engine::OpPayloadAttributes {
/// Dummy system transaction for dev mode.
/// OP Mainnet transaction at index 0 in block 124665056.
///
/// <https://optimistic.etherscan.io/tx/0x312e290cf36df704a2217b015d6455396830b0ce678b860ebfcc30f41403d7b1>
const TX_SET_L1_BLOCK_OP_MAINNET_BLOCK_124665056: [u8; 251] = alloy_primitives::hex!(
"7ef8f8a0683079df94aa5b9cf86687d739a60a9b4f0835e520ec4d664e2e415dca17a6df94deaddeaddeaddeaddeaddeaddeaddeaddead00019442000000000000000000000000000000000000158080830f424080b8a4440a5e200000146b000f79c500000000000000040000000066d052e700000000013ad8a3000000000000000000000000000000000000000000000000000000003ef1278700000000000000000000000000000000000000000000000000000000000000012fdf87b89884a61e74b322bbcf60386f543bfae7827725efaaf0ab1de2294a590000000000000000000000006887246668a3b87f54deb3b94ba47a6f63f32985"
);
op_alloy_rpc_types_engine::OpPayloadAttributes {
payload_attributes: self.build(parent),
// Add dummy system transaction
transactions: Some(vec![TX_SET_L1_BLOCK_OP_MAINNET_BLOCK_124665056.into()]),
transactions: Some(vec![
reth_optimism_chainspec::constants::TX_SET_L1_BLOCK_OP_MAINNET_BLOCK_124665056
.into(),
]),
no_tx_pool: None,
gas_limit: None,
eip_1559_params: None,

View File

@@ -148,12 +148,10 @@ pub struct TreeConfig {
storage_worker_count: usize,
/// Number of account proof worker threads.
account_worker_count: usize,
/// Whether to disable V2 storage proofs.
disable_proof_v2: bool,
/// Whether to enable V2 storage proofs.
enable_proof_v2: bool,
/// Whether to disable cache metrics recording (can be expensive with large cached state).
disable_cache_metrics: bool,
/// Whether to enable sparse trie as cache.
enable_sparse_trie_as_cache: bool,
}
impl Default for TreeConfig {
@@ -181,9 +179,8 @@ impl Default for TreeConfig {
allow_unwind_canonical_header: false,
storage_worker_count: default_storage_worker_count(),
account_worker_count: default_account_worker_count(),
disable_proof_v2: false,
enable_proof_v2: false,
disable_cache_metrics: false,
enable_sparse_trie_as_cache: false,
}
}
}
@@ -214,7 +211,7 @@ impl TreeConfig {
allow_unwind_canonical_header: bool,
storage_worker_count: usize,
account_worker_count: usize,
disable_proof_v2: bool,
enable_proof_v2: bool,
disable_cache_metrics: bool,
) -> Self {
Self {
@@ -240,9 +237,8 @@ impl TreeConfig {
allow_unwind_canonical_header,
storage_worker_count,
account_worker_count,
disable_proof_v2,
enable_proof_v2,
disable_cache_metrics,
enable_sparse_trie_as_cache: false,
}
}
@@ -284,8 +280,7 @@ impl TreeConfig {
/// Return the multiproof task chunk size, using the V2 default if V2 proofs are enabled
/// and the chunk size is at the default value.
pub const fn effective_multiproof_chunk_size(&self) -> usize {
if !self.disable_proof_v2 &&
self.multiproof_chunk_size == DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE
if self.enable_proof_v2 && self.multiproof_chunk_size == DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE
{
DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2
} else {
@@ -523,14 +518,14 @@ impl TreeConfig {
self
}
/// Return whether V2 storage proofs are disabled.
pub const fn disable_proof_v2(&self) -> bool {
self.disable_proof_v2
/// Return whether V2 storage proofs are enabled.
pub const fn enable_proof_v2(&self) -> bool {
self.enable_proof_v2
}
/// Setter for whether to disable V2 storage proofs.
pub const fn with_disable_proof_v2(mut self, disable_proof_v2: bool) -> Self {
self.disable_proof_v2 = disable_proof_v2;
/// Setter for whether to enable V2 storage proofs.
pub const fn with_enable_proof_v2(mut self, enable_proof_v2: bool) -> Self {
self.enable_proof_v2 = enable_proof_v2;
self
}
@@ -544,15 +539,4 @@ impl TreeConfig {
self.disable_cache_metrics = disable_cache_metrics;
self
}
/// Returns whether sparse trie as cache is enabled.
pub const fn enable_sparse_trie_as_cache(&self) -> bool {
self.enable_sparse_trie_as_cache
}
/// Setter for whether to enable sparse trie as cache.
pub const fn with_enable_sparse_trie_as_cache(mut self, value: bool) -> Self {
self.enable_sparse_trie_as_cache = value;
self
}
}

View File

@@ -101,7 +101,7 @@ where
let canonical_in_memory_state = blockchain_db.canonical_in_memory_state();
let (to_tree_tx, from_tree) = EngineApiTreeHandler::spawn_new(
let (to_tree_tx, from_tree) = EngineApiTreeHandler::<N::Primitives, _, _, _, _>::spawn_new(
blockchain_db,
consensus,
payload_validator,

View File

@@ -312,7 +312,14 @@ impl<S: AccountReader> AccountReader for CachedStateProvider<S> {
match self.caches.get_or_try_insert_account_with(*address, || {
self.state_provider.basic_account(address)
})? {
CachedStatus::NotCached(value) | CachedStatus::Cached(value) => Ok(value),
CachedStatus::NotCached(value) => {
self.metrics.account_cache_misses.increment(1);
Ok(value)
}
CachedStatus::Cached(value) => {
self.metrics.account_cache_hits.increment(1);
Ok(value)
}
}
} else if let Some(account) = self.caches.account_cache.get(address) {
self.metrics.account_cache_hits.increment(1);
@@ -343,7 +350,14 @@ impl<S: StateProvider> StateProvider for CachedStateProvider<S> {
match self.caches.get_or_try_insert_storage_with(account, storage_key, || {
self.state_provider.storage(account, storage_key).map(Option::unwrap_or_default)
})? {
CachedStatus::NotCached(value) | CachedStatus::Cached(value) => {
CachedStatus::NotCached(value) => {
self.metrics.storage_cache_misses.increment(1);
// The slot that was never written to is indistinguishable from a slot
// explicitly set to zero. We return `None` in both cases.
Ok(Some(value).filter(|v| !v.is_zero()))
}
CachedStatus::Cached(value) => {
self.metrics.storage_cache_hits.increment(1);
// The slot that was never written to is indistinguishable from a slot
// explicitly set to zero. We return `None` in both cases.
Ok(Some(value).filter(|v| !v.is_zero()))
@@ -365,7 +379,14 @@ impl<S: BytecodeReader> BytecodeReader for CachedStateProvider<S> {
match self.caches.get_or_try_insert_code_with(*code_hash, || {
self.state_provider.bytecode_by_hash(code_hash)
})? {
CachedStatus::NotCached(code) | CachedStatus::Cached(code) => Ok(code),
CachedStatus::NotCached(code) => {
self.metrics.code_cache_misses.increment(1);
Ok(code)
}
CachedStatus::Cached(code) => {
self.metrics.code_cache_hits.increment(1);
Ok(code)
}
}
} else if let Some(code) = self.caches.code_cache.get(code_hash) {
self.metrics.code_cache_hits.increment(1);

View File

@@ -1406,20 +1406,7 @@ where
);
self.changeset_cache.evict(eviction_threshold);
// Invalidate cached overlay since the anchor has changed
self.state.tree_state.invalidate_cached_overlay();
self.on_new_persisted_block()?;
// Re-prepare overlay for the current canonical head with the new anchor.
// Spawn a background task to trigger computation so it's ready when the next payload
// arrives.
if let Some(overlay) = self.state.tree_state.prepare_canonical_overlay() {
rayon::spawn(move || {
let _ = overlay.get();
});
}
Ok(())
}

View File

@@ -7,14 +7,14 @@ use crate::tree::{
prewarm::{PrewarmCacheTask, PrewarmContext, PrewarmMode, PrewarmTaskEvent},
sparse_trie::StateRootComputeOutcome,
},
sparse_trie::{SparseTrieCacheTask, SparseTrieTask},
sparse_trie::SparseTrieTask,
StateProviderBuilder, TreeConfig,
};
use alloy_eip7928::BlockAccessList;
use alloy_eips::eip1898::BlockWithParent;
use alloy_evm::block::StateChangeSource;
use alloy_primitives::B256;
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
use crossbeam_channel::Sender as CrossbeamSender;
use executor::WorkloadExecutor;
use metrics::Counter;
use multiproof::{SparseTrieUpdate, *};
@@ -39,7 +39,10 @@ use reth_trie_parallel::{
proof_task::{ProofTaskCtx, ProofWorkerHandle},
root::ParallelStateRootError,
};
use reth_trie_sparse::{ClearedSparseStateTrie, RevealableSparseTrie, SparseStateTrie};
use reth_trie_sparse::{
provider::{TrieNodeProvider, TrieNodeProviderFactory},
ClearedSparseStateTrie, RevealableSparseTrie, SparseStateTrie,
};
use reth_trie_sparse_parallel::{ParallelSparseTrie, ParallelismThresholds};
use std::{
collections::BTreeMap,
@@ -235,7 +238,7 @@ where
let (to_multi_proof, from_multi_proof) = crossbeam_channel::unbounded();
// Extract V2 proofs flag early so we can pass it to prewarm
let v2_proofs_enabled = !config.disable_proof_v2();
let v2_proofs_enabled = config.enable_proof_v2();
// Handle BAL-based optimization if available
let prewarm_handle = if let Some(bal) = bal {
@@ -280,45 +283,39 @@ where
v2_proofs_enabled,
);
if !config.enable_sparse_trie_as_cache() {
let multi_proof_task = MultiProofTask::new(
proof_handle.clone(),
to_sparse_trie,
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size()),
to_multi_proof.clone(),
from_multi_proof.clone(),
)
.with_v2_proofs_enabled(v2_proofs_enabled);
let multi_proof_task = MultiProofTask::new(
proof_handle.clone(),
to_sparse_trie,
config
.multiproof_chunking_enabled()
.then_some(config.effective_multiproof_chunk_size()),
to_multi_proof.clone(),
from_multi_proof,
)
.with_v2_proofs_enabled(v2_proofs_enabled);
// spawn multi-proof task
let parent_span = span.clone();
let saved_cache = prewarm_handle.saved_cache.clone();
self.executor.spawn_blocking(move || {
let _enter = parent_span.entered();
// Build a state provider for the multiproof task
let provider = provider_builder.build().expect("failed to build provider");
let provider = if let Some(saved_cache) = saved_cache {
let (cache, metrics, _disable_metrics) = saved_cache.split();
Box::new(CachedStateProvider::new(provider, cache, metrics))
as Box<dyn StateProvider>
} else {
Box::new(provider)
};
multi_proof_task.run(provider);
});
}
// spawn multi-proof task
let parent_span = span.clone();
let saved_cache = prewarm_handle.saved_cache.clone();
self.executor.spawn_blocking(move || {
let _enter = parent_span.entered();
// Build a state provider for the multiproof task
let provider = provider_builder.build().expect("failed to build provider");
let provider = if let Some(saved_cache) = saved_cache {
let (cache, metrics, _disable_metrics) = saved_cache.split();
Box::new(CachedStateProvider::new(provider, cache, metrics))
as Box<dyn StateProvider>
} else {
Box::new(provider)
};
multi_proof_task.run(provider);
});
// wire the sparse trie to the state root response receiver
let (state_root_tx, state_root_rx) = channel();
// Spawn the sparse trie task using any stored trie and parallel trie configuration.
self.spawn_sparse_trie_task(
sparse_trie_rx,
proof_handle,
state_root_tx,
from_multi_proof,
config,
);
self.spawn_sparse_trie_task(sparse_trie_rx, proof_handle, state_root_tx);
PayloadHandle {
to_multi_proof: Some(to_multi_proof),
@@ -498,18 +495,19 @@ where
/// Spawns the [`SparseTrieTask`] for this payload processor.
#[instrument(level = "debug", target = "engine::tree::payload_processor", skip_all)]
fn spawn_sparse_trie_task(
fn spawn_sparse_trie_task<BPF>(
&self,
sparse_trie_rx: mpsc::Receiver<SparseTrieUpdate>,
proof_worker_handle: ProofWorkerHandle,
proof_worker_handle: BPF,
state_root_tx: mpsc::Sender<Result<StateRootComputeOutcome, ParallelStateRootError>>,
from_multi_proof: CrossbeamReceiver<MultiProofMessage>,
config: &TreeConfig,
) {
) where
BPF: TrieNodeProviderFactory + Clone + Send + Sync + 'static,
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
{
let cleared_sparse_trie = Arc::clone(&self.sparse_state_trie);
let trie_metrics = self.trie_metrics.clone();
let span = Span::current();
let disable_sparse_trie_as_cache = !config.enable_sparse_trie_as_cache();
self.executor.spawn_blocking(move || {
let _enter = span.entered();
@@ -529,24 +527,15 @@ where
)
});
let (result, trie) = if disable_sparse_trie_as_cache {
SparseTrieTask::new_with_cleared_trie(
let task =
SparseTrieTask::<_, ParallelSparseTrie, ParallelSparseTrie>::new_with_cleared_trie(
sparse_trie_rx,
proof_worker_handle,
trie_metrics,
sparse_state_trie,
)
.run()
} else {
SparseTrieCacheTask::new_with_cleared_trie(
from_multi_proof,
proof_worker_handle,
trie_metrics,
sparse_state_trie,
)
.run()
};
);
let (result, trie) = task.run();
// Send state root computation result
let _ = state_root_tx.send(result);

View File

@@ -563,7 +563,6 @@ where
index,
tx_hash = %tx.tx().tx_hash(),
is_success = tracing::field::Empty,
gas_used = tracing::field::Empty,
)
.entered();

View File

@@ -77,22 +77,8 @@ impl<R: Receipt> ReceiptRootTaskHandle<R> {
receipt_with_bloom.encode_2718(&mut encode_buf);
aggregated_bloom |= *receipt_with_bloom.bloom_ref();
match builder.push(indexed_receipt.index, &encode_buf) {
Ok(()) => {
received_count += 1;
}
Err(err) => {
// If a duplicate or out-of-bounds index is streamed, skip it and
// fall back to computing the receipt root from the full receipts
// vector later.
tracing::error!(
target: "engine::tree::payload_processor",
index = indexed_receipt.index,
?err,
"Receipt root task received invalid receipt index, skipping"
);
}
}
builder.push_unchecked(indexed_receipt.index, &encode_buf);
received_count += 1;
}
let Ok(root) = builder.finalize() else {

View File

@@ -1,34 +1,15 @@
//! Sparse Trie task related functionality.
use crate::tree::{
multiproof::{evm_state_to_hashed_post_state, MultiProofMessage, VersionedMultiProofTargets},
payload_processor::multiproof::{MultiProofTaskMetrics, SparseTrieUpdate},
};
use crate::tree::payload_processor::multiproof::{MultiProofTaskMetrics, SparseTrieUpdate};
use alloy_primitives::B256;
use alloy_rlp::Decodable;
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
use rayon::iter::{ParallelBridge, ParallelIterator};
use reth_errors::ProviderError;
use reth_primitives_traits::Account;
use reth_revm::state::EvmState;
use reth_trie::{
proof_v2::Target, updates::TrieUpdates, HashedPostState, Nibbles, TrieAccount, EMPTY_ROOT_HASH,
};
use reth_trie_parallel::{
proof_task::{
AccountMultiproofInput, ProofResult, ProofResultContext, ProofResultMessage,
ProofWorkerHandle,
},
root::ParallelStateRootError,
targets_v2::MultiProofTargetsV2,
};
use reth_trie::{updates::TrieUpdates, Nibbles};
use reth_trie_parallel::{proof_task::ProofResult, root::ParallelStateRootError};
use reth_trie_sparse::{
errors::{SparseStateTrieResult, SparseTrieErrorKind},
provider::{TrieNodeProvider, TrieNodeProviderFactory},
ClearedSparseStateTrie, LeafUpdate, SerialSparseTrie, SparseStateTrie, SparseTrie,
SparseTrieExt,
ClearedSparseStateTrie, SerialSparseTrie, SparseStateTrie, SparseTrie,
};
use revm_primitives::{hash_map::Entry, B256Map};
use smallvec::SmallVec;
use std::{
sync::mpsc,
@@ -148,359 +129,6 @@ where
}
}
/// Sparse trie task implementation that uses in-memory sparse trie data to schedule proof fetching.
pub(super) struct SparseTrieCacheTask<A = SerialSparseTrie, S = SerialSparseTrie> {
/// Sender for proof results.
proof_result_tx: CrossbeamSender<ProofResultMessage>,
/// Receiver for proof results directly from workers.
proof_result_rx: CrossbeamReceiver<ProofResultMessage>,
/// Receives updates from execution and prewarming.
updates: CrossbeamReceiver<MultiProofMessage>,
/// `SparseStateTrie` used for computing the state root.
trie: SparseStateTrie<A, S>,
/// Handle to the proof worker pools (storage and account).
proof_worker_handle: ProofWorkerHandle,
/// Account trie updates.
account_updates: B256Map<LeafUpdate>,
/// Storage trie updates. hashed address -> slot -> update.
storage_updates: B256Map<B256Map<LeafUpdate>>,
/// Account updates that are blocked by storage root calculation or account reveal.
///
/// Those are being moved into `account_updates` once storage roots
/// are revealed and/or calculated.
///
/// Invariant: for each entry in `pending_account_updates` account must either be already
/// revealed in the trie or have an entry in `account_updates`.
///
/// Values can be either of:
/// - None: account had a storage update and is awaiting storage root calculation and/or
/// account node reveal to complete.
/// - Some(_): account was changed/destroyed and is awaiting storage root calculation/reveal
/// to complete.
pending_account_updates: B256Map<Option<Option<Account>>>,
/// Metrics for the sparse trie.
metrics: MultiProofTaskMetrics,
}
impl<A, S> SparseTrieCacheTask<A, S>
where
A: SparseTrieExt + Default,
S: SparseTrieExt + Default + Clone,
{
/// Creates a new sparse trie, pre-populating with a [`ClearedSparseStateTrie`].
pub(super) fn new_with_cleared_trie(
updates: CrossbeamReceiver<MultiProofMessage>,
proof_worker_handle: ProofWorkerHandle,
metrics: MultiProofTaskMetrics,
sparse_state_trie: ClearedSparseStateTrie<A, S>,
) -> Self {
let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded();
Self {
proof_result_tx,
proof_result_rx,
updates,
proof_worker_handle,
trie: sparse_state_trie.into_inner(),
account_updates: Default::default(),
storage_updates: Default::default(),
pending_account_updates: Default::default(),
metrics,
}
}
/// Runs the sparse trie task to completion.
///
/// This waits for new incoming [`MultiProofMessage`]s, applies updates to the trie and
/// schedules proof fetching when needed.
///
/// This concludes once the last state update has been received and processed.
///
/// # Returns
///
/// - State root computation outcome.
/// - `SparseStateTrie` that needs to be cleared and reused to avoid reallocations.
#[instrument(
level = "debug",
target = "engine::tree::payload_processor::sparse_trie",
skip_all
)]
pub(super) fn run(
mut self,
) -> (Result<StateRootComputeOutcome, ParallelStateRootError>, SparseStateTrie<A, S>) {
// run the main loop to completion
let result = self.run_inner();
(result, self.trie)
}
/// Inner function to run the sparse trie task to completion.
///
/// See [`Self::run`] for more information.
fn run_inner(&mut self) -> Result<StateRootComputeOutcome, ParallelStateRootError> {
let now = Instant::now();
let mut finished_state_updates = false;
loop {
crossbeam_channel::select_biased! {
recv(self.proof_result_rx) -> message => {
let Ok(result) = message else {
unreachable!("we own the sender half")
};
self.on_proof_result(result)?;
},
recv(self.updates) -> message => {
let update = match message {
Ok(m) => m,
Err(_) => {
break
}
};
match update {
MultiProofMessage::PrefetchProofs(targets) => {
self.on_prewarm_targets(targets);
}
MultiProofMessage::StateUpdate(_, state) => {
self.on_state_update(state);
}
MultiProofMessage::EmptyProof { sequence_number: _, state } => {
self.on_hashed_state_update(state);
}
MultiProofMessage::BlockAccessList(_) => todo!(),
MultiProofMessage::FinishedStateUpdates => {
finished_state_updates = true;
}
}
}
}
self.process_updates()?;
if finished_state_updates &&
self.account_updates.is_empty() &&
self.storage_updates.iter().all(|(_, updates)| updates.is_empty())
{
break;
}
}
// Process any remaining pending account updates.
if !self.pending_account_updates.is_empty() {
self.process_updates()?;
}
debug!(target: "engine::root", "All proofs processed, ending calculation");
let start = Instant::now();
let (state_root, trie_updates) =
self.trie.root_with_updates(&self.proof_worker_handle).map_err(|e| {
ParallelStateRootError::Other(format!("could not calculate state root: {e:?}"))
})?;
let end = Instant::now();
self.metrics.sparse_trie_final_update_duration_histogram.record(end.duration_since(start));
self.metrics.sparse_trie_total_duration_histogram.record(end.duration_since(now));
Ok(StateRootComputeOutcome { state_root, trie_updates })
}
fn on_prewarm_targets(&mut self, targets: VersionedMultiProofTargets) {
let VersionedMultiProofTargets::V2(targets) = targets else {
unreachable!("sparse trie as cache must only be used with V2 multiproof targets");
};
for target in targets.account_targets {
// Only touch accounts that are not yet present in the updates set.
self.account_updates.entry(target.key()).or_insert(LeafUpdate::Touched);
}
for (address, slots) in targets.storage_targets {
for slot in slots {
// Only touch storages that are not yet present in the updates set.
self.storage_updates
.entry(address)
.or_default()
.entry(slot.key())
.or_insert(LeafUpdate::Touched);
}
// Touch corresponding account leaf to make sure its revealed in accounts trie for
// storage root update.
self.account_updates.entry(address).or_insert(LeafUpdate::Touched);
}
}
/// Processes a state update and encodes all state changes as trie updates.
#[instrument(
level = "debug",
target = "engine::tree::payload_processor::sparse_trie",
skip_all,
fields(accounts = update.len())
)]
fn on_state_update(&mut self, update: EvmState) {
let hashed_state_update = evm_state_to_hashed_post_state(update);
self.on_hashed_state_update(hashed_state_update)
}
/// Processes a hashed state update and encodes all state changes as trie updates.
fn on_hashed_state_update(&mut self, hashed_state_update: HashedPostState) {
for (address, storage) in hashed_state_update.storages {
for (slot, value) in storage.storage {
let encoded = if value.is_zero() {
Vec::new()
} else {
alloy_rlp::encode_fixed_size(&value).to_vec()
};
self.storage_updates
.entry(address)
.or_default()
.insert(slot, LeafUpdate::Changed(encoded));
}
// Make sure account is tracked in `account_updates` so that it is revealed in accounts
// trie for storage root update.
self.account_updates.entry(address).or_insert(LeafUpdate::Touched);
// Make sure account is tracked in `pending_account_updates` so that once storage root
// is computed, it will be updated in the accounts trie.
self.pending_account_updates.entry(address).or_insert(None);
}
for (address, account) in hashed_state_update.accounts {
// Track account as touched.
//
// This might overwrite an existing update, which is fine, because storage root from it
// is already tracked in the trie and can be easily fetched again.
self.account_updates.insert(address, LeafUpdate::Touched);
// Track account in `pending_account_updates` so that once storage root is computed,
// it will be updated in the accounts trie.
self.pending_account_updates.insert(address, Some(account));
}
}
fn on_proof_result(
&mut self,
result: ProofResultMessage,
) -> Result<(), ParallelStateRootError> {
let ProofResult::V2(result) = result.result? else {
unreachable!("sparse trie as cache must only be used with multiproof v2");
};
self.trie.reveal_decoded_multiproof_v2(result).map_err(|e| {
ParallelStateRootError::Other(format!("could not reveal multiproof: {e:?}"))
})
}
/// Applies updates to the sparse trie and dispatches requested multiproof targets.
fn process_updates(&mut self) -> Result<(), ProviderError> {
let mut targets = MultiProofTargetsV2::default();
for (addr, updates) in &mut self.storage_updates {
let trie = self.trie.get_or_create_storage_trie_mut(*addr);
trie.update_leaves(updates, |path, min_len| {
targets
.storage_targets
.entry(*addr)
.or_default()
.push(Target::new(path).with_min_len(min_len));
})
.map_err(ProviderError::other)?;
// If all storage updates were processed, we can now compute the new storage root.
if updates.is_empty() {
let storage_root =
trie.root().expect("updates are drained, trie should be revealed by now");
// If there is a pending account update for this address with known info, we can
// encode it into proper update right away.
if let Entry::Occupied(entry) = self.pending_account_updates.entry(*addr) &&
entry.get().is_some()
{
let account = entry.remove().expect("just checked, should be Some");
let encoded = if account.is_none_or(|account| account.is_empty()) &&
storage_root == EMPTY_ROOT_HASH
{
Vec::new()
} else {
// TODO: optimize allocation
alloy_rlp::encode(
account.unwrap_or_default().into_trie_account(storage_root),
)
};
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
}
}
}
// Now handle pending account updates that can be upgraded to a proper update.
self.pending_account_updates.retain(|addr, account| {
// If account has pending storage updates, it is still pending.
if self.storage_updates.get(addr).is_some_and(|updates| !updates.is_empty()) {
return true;
}
// Get the current account state either from the trie or from latest account update.
let trie_account = if let Some(LeafUpdate::Changed(encoded)) = self.account_updates.get(addr) {
Some(encoded).filter(|encoded| !encoded.is_empty())
} else if !self.account_updates.contains_key(addr) {
self.trie.get_account_value(addr)
} else {
// Needs to be revealed first
return true;
};
let trie_account = trie_account.map(|value| TrieAccount::decode(&mut &value[..]).expect("invalid account RLP"));
let (account, storage_root) = if let Some(account) = account.take() {
// If account is Some(_) here it means it didn't have any storage updates
// and we can fetch the storage root directly from the account trie.
//
// If it did have storage updates, we would've had processed it above when iterating over storage tries.
let storage_root = trie_account.map(|account| account.storage_root).unwrap_or(EMPTY_ROOT_HASH);
(account, storage_root)
} else {
(trie_account.map(Into::into), self.trie.storage_root(addr).expect("account had storage updates that were applied to its trie, storage root must be revealed by now"))
};
let encoded = if account.is_none_or(|account| account.is_empty()) && storage_root == EMPTY_ROOT_HASH {
Vec::new()
} else {
let account = account.unwrap_or_default().into_trie_account(storage_root);
// TODO: optimize allocation
alloy_rlp::encode(account)
};
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
false
});
// Process account trie updates and fill the account targets.
self.trie
.trie_mut()
.update_leaves(&mut self.account_updates, |target, min_len| {
targets.account_targets.push(Target::new(target).with_min_len(min_len));
})
.map_err(ProviderError::other)?;
if !targets.is_empty() {
self.proof_worker_handle.dispatch_account_multiproof(AccountMultiproofInput::V2 {
targets,
proof_result_sender: ProofResultContext::new(
self.proof_result_tx.clone(),
0,
HashedPostState::default(),
Instant::now(),
),
})?;
}
Ok(())
}
}
/// Outcome of the state root computation, including the state root itself with
/// the trie updates.
#[derive(Debug)]

View File

@@ -792,11 +792,6 @@ where
// Execute transactions
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
let mut transactions = transactions.into_iter();
// Some executors may execute transactions that do not append receipts during the
// main loop (e.g., system transactions whose receipts are added during finalization).
// In that case, invoking the callback on every transaction would resend the previous
// receipt with the same index and can panic the ordered root builder.
let mut last_sent_len = 0usize;
loop {
// Measure time spent waiting for next transaction from iterator
// (e.g., parallel signature recovery)
@@ -823,14 +818,10 @@ where
let gas_used = executor.execute_transaction(tx)?;
self.metrics.record_transaction_execution(tx_start.elapsed());
let current_len = executor.receipts().len();
if current_len > last_sent_len {
last_sent_len = current_len;
// Send the latest receipt to the background task for incremental root computation.
if let Some(receipt) = executor.receipts().last() {
let tx_index = current_len - 1;
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
}
// Send the latest receipt to the background task for incremental root computation
if let Some(receipt) = executor.receipts().last() {
let tx_index = executor.receipts().len() - 1;
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
}
enter.record("gas_used", gas_used);
@@ -1112,13 +1103,10 @@ where
/// while the trie input computation is deferred until the overlay is actually needed.
///
/// If parent is on disk (no in-memory blocks), returns `None` for the lazy overlay.
///
/// Uses a cached overlay if available for the canonical head (the common case).
fn get_parent_lazy_overlay(
parent_hash: B256,
state: &EngineApiTreeState<N>,
) -> (Option<LazyOverlay>, B256) {
// Get blocks leading to the parent to determine the anchor
let (anchor_hash, blocks) =
state.tree_state.blocks_by_hash(parent_hash).unwrap_or_else(|| (parent_hash, vec![]));
@@ -1127,17 +1115,6 @@ where
return (None, anchor_hash);
}
// Try to use the cached overlay if it matches both parent hash and anchor
if let Some(cached) = state.tree_state.get_cached_overlay(parent_hash, anchor_hash) {
debug!(
target: "engine::tree::payload_validator",
%parent_hash,
%anchor_hash,
"Using cached canonical overlay"
);
return (Some(cached.overlay.clone()), cached.anchor_hash);
}
debug!(
target: "engine::tree::payload_validator",
%anchor_hash,

View File

@@ -6,7 +6,7 @@ use alloy_primitives::{
map::{HashMap, HashSet},
BlockNumber, B256,
};
use reth_chain_state::{DeferredTrieData, EthPrimitives, ExecutedBlock, LazyOverlay};
use reth_chain_state::{EthPrimitives, ExecutedBlock};
use reth_primitives_traits::{AlloyBlockHeader, NodePrimitives, SealedHeader};
use std::{
collections::{btree_map, hash_map, BTreeMap, VecDeque},
@@ -38,12 +38,6 @@ pub struct TreeState<N: NodePrimitives = EthPrimitives> {
pub(crate) current_canonical_head: BlockNumHash,
/// The engine API variant of this handler
pub(crate) engine_kind: EngineApiKind,
/// Pre-computed lazy overlay for the canonical head.
///
/// This is optimistically prepared after the canonical head changes, so that
/// the next payload building on the canonical head can use it immediately
/// without recomputing.
pub(crate) cached_canonical_overlay: Option<PreparedCanonicalOverlay>,
}
impl<N: NodePrimitives> TreeState<N> {
@@ -55,7 +49,6 @@ impl<N: NodePrimitives> TreeState<N> {
current_canonical_head,
parent_to_child: HashMap::default(),
engine_kind,
cached_canonical_overlay: None,
}
}
@@ -99,66 +92,6 @@ impl<N: NodePrimitives> TreeState<N> {
Some((parent_hash, blocks))
}
/// Prepares a cached lazy overlay for the current canonical head.
///
/// This should be called after the canonical head changes to optimistically
/// prepare the overlay for the next payload that will likely build on it.
///
/// Returns a clone of the [`LazyOverlay`] so the caller can spawn a background
/// task to trigger computation via [`LazyOverlay::get`]. This ensures the overlay
/// is actually computed before the next payload arrives.
pub(crate) fn prepare_canonical_overlay(&mut self) -> Option<LazyOverlay> {
let canonical_hash = self.current_canonical_head.hash;
// Get blocks leading to the canonical head
let Some((anchor_hash, blocks)) = self.blocks_by_hash(canonical_hash) else {
// Canonical head not in memory (persisted), no overlay needed
self.cached_canonical_overlay = None;
return None;
};
// Extract deferred trie data handles from blocks (newest to oldest)
let handles: Vec<DeferredTrieData> = blocks.iter().map(|b| b.trie_data_handle()).collect();
let overlay = LazyOverlay::new(anchor_hash, handles);
self.cached_canonical_overlay = Some(PreparedCanonicalOverlay {
parent_hash: canonical_hash,
overlay: overlay.clone(),
anchor_hash,
});
debug!(
target: "engine::tree",
%canonical_hash,
%anchor_hash,
num_blocks = blocks.len(),
"Prepared cached canonical overlay"
);
Some(overlay)
}
/// Returns the cached overlay if it matches the requested parent hash and anchor.
///
/// Both parent hash and anchor hash must match to ensure the overlay is valid.
/// This prevents using a stale overlay after persistence has advanced the anchor.
pub(crate) fn get_cached_overlay(
&self,
parent_hash: B256,
expected_anchor: B256,
) -> Option<&PreparedCanonicalOverlay> {
self.cached_canonical_overlay.as_ref().filter(|cached| {
cached.parent_hash == parent_hash && cached.anchor_hash == expected_anchor
})
}
/// Invalidates the cached overlay.
///
/// Should be called when the anchor changes (e.g., after persistence).
pub(crate) fn invalidate_cached_overlay(&mut self) {
self.cached_canonical_overlay = None;
}
/// Insert executed block into the state.
pub(crate) fn insert_executed(&mut self, executed: ExecutedBlock<N>) {
let hash = executed.recovered_block().hash();
@@ -355,9 +288,6 @@ impl<N: NodePrimitives> TreeState<N> {
if let Some(finalized_num_hash) = finalized_num_hash {
self.prune_finalized_sidechains(finalized_num_hash);
}
// Invalidate the cached overlay since blocks were removed and the anchor may have changed
self.invalidate_cached_overlay();
}
/// Updates the canonical head to the given block.
@@ -425,39 +355,6 @@ impl<N: NodePrimitives> TreeState<N> {
}
}
/// Pre-computed lazy overlay for the canonical head block.
///
/// This is prepared **optimistically** when the canonical head changes, allowing
/// the next payload (which typically builds on the canonical head) to reuse
/// the pre-computed overlay immediately without re-traversing in-memory blocks.
///
/// The overlay captures deferred trie data handles from all in-memory blocks
/// between the canonical head and the persisted anchor. When a new payload
/// arrives building on the canonical head, this cached overlay can be used
/// directly instead of calling `blocks_by_hash` and collecting handles again.
///
/// # Invalidation
///
/// The cached overlay is invalidated when:
/// - Persistence completes (anchor changes)
/// - The canonical head changes to a different block
#[derive(Debug, Clone)]
pub struct PreparedCanonicalOverlay {
/// The block hash for which this overlay is prepared as a parent.
///
/// When a payload arrives with this parent hash, the overlay can be reused.
pub parent_hash: B256,
/// The pre-computed lazy overlay containing deferred trie data handles.
///
/// This is computed optimistically after `set_canonical_head` so subsequent
/// payloads don't need to re-collect the handles.
pub overlay: LazyOverlay,
/// The anchor hash (persisted ancestor) this overlay is based on.
///
/// Used to verify the overlay is still valid (anchor hasn't changed due to persistence).
pub anchor_hash: B256,
}
#[cfg(test)]
mod tests {
use super::*;

View File

@@ -259,7 +259,6 @@ impl TestHarness {
current_canonical_head: blocks.last().unwrap().recovered_block().num_hash(),
parent_to_child,
engine_kind: EngineApiKind::Ethereum,
cached_canonical_overlay: None,
};
let last_executed_block = blocks.last().unwrap().clone();

View File

@@ -174,7 +174,7 @@ where
}
Commands::P2P(command) => runner.run_until_ctrl_c(command.execute::<N>()),
Commands::Config(command) => runner.run_until_ctrl_c(command.execute()),
Commands::Prune(command) => runner.run_command_until_exit(|ctx| command.execute::<N>(ctx)),
Commands::Prune(command) => runner.run_until_ctrl_c(command.execute::<N>()),
#[cfg(feature = "dev")]
Commands::TestVectors(command) => runner.run_until_ctrl_c(command.execute()),
Commands::ReExecute(command) => runner.run_until_ctrl_c(command.execute::<N>(components)),

View File

@@ -171,7 +171,7 @@ pub enum SparseTrieErrorKind {
/// Path to the node.
path: Nibbles,
/// Node that was at the path when revealing.
node: Box<dyn core::fmt::Debug + Send + Sync>,
node: Box<dyn core::fmt::Debug + Send>,
},
/// RLP error.
#[error(transparent)]
@@ -184,7 +184,7 @@ pub enum SparseTrieErrorKind {
},
/// Other.
#[error(transparent)]
Other(#[from] Box<dyn core::error::Error + Send + Sync>),
Other(#[from] Box<dyn core::error::Error + Send>),
}
/// Trie witness errors.

View File

@@ -251,7 +251,7 @@ pub async fn test_exex_context_with_chain_spec(
db,
chain_spec.clone(),
StaticFileProvider::read_write(static_dir.keep()).expect("static file provider"),
RocksDBProvider::builder(rocksdb_dir.keep()).with_default_tables().build().unwrap(),
RocksDBProvider::builder(rocksdb_dir.keep()).build().unwrap(),
)?;
let genesis_hash = init_genesis(&provider_factory)?;

View File

@@ -83,28 +83,6 @@ impl From<&'static str> for FileClientError {
}
impl<B: FullBlock> FileClient<B> {
/// Create a new file client from a slice of sealed blocks.
pub fn from_blocks(blocks: impl IntoIterator<Item = SealedBlock<B>>) -> Self {
let blocks: Vec<_> = blocks.into_iter().collect();
let capacity = blocks.len();
let mut headers = HashMap::with_capacity(capacity);
let mut hash_to_number = HashMap::with_capacity(capacity);
let mut bodies = HashMap::with_capacity(capacity);
for block in blocks {
let number = block.number();
let hash = block.hash();
let (header, body) = block.split_sealed_header_body();
headers.insert(number, header.into_header());
hash_to_number.insert(hash, number);
bodies.insert(hash, body);
}
Self { headers, hash_to_number, bodies }
}
/// Create a new file client from a file path.
pub async fn new<P: AsRef<Path>>(
path: P,

View File

@@ -20,7 +20,7 @@ use std::{
sync::Arc,
task::{Context, Poll},
};
use tracing::trace;
use tracing::{debug, trace};
#[cfg_attr(doc, aquamarine::aquamarine)]
/// Contains the connectivity related state of the network.
@@ -259,7 +259,7 @@ impl<N: NetworkPrimitives> Swarm<N> {
if self.sessions.is_valid_fork_id(fork_id) {
self.state_mut().peers_mut().set_discovered_fork_id(peer_id, fork_id);
} else {
trace!(target: "net", ?peer_id, remote_fork_id=?fork_id, our_fork_id=?self.sessions.fork_id(), "fork id mismatch, removing peer");
debug!(target: "net", ?peer_id, remote_fork_id=?fork_id, our_fork_id=?self.sessions.fork_id(), "fork id mismatch, removing peer");
self.state_mut().peers_mut().remove_peer(peer_id);
}
}

View File

@@ -35,9 +35,8 @@ pub struct DefaultEngineValues {
allow_unwind_canonical_header: bool,
storage_worker_count: Option<usize>,
account_worker_count: Option<usize>,
disable_proof_v2: bool,
enable_proof_v2: bool,
cache_metrics_disabled: bool,
enable_sparse_trie_as_cache: bool,
}
impl DefaultEngineValues {
@@ -162,9 +161,9 @@ impl DefaultEngineValues {
self
}
/// Set whether to disable proof V2 by default
pub const fn with_disable_proof_v2(mut self, v: bool) -> Self {
self.disable_proof_v2 = v;
/// Set whether to enable proof V2 by default
pub const fn with_enable_proof_v2(mut self, v: bool) -> Self {
self.enable_proof_v2 = v;
self
}
@@ -173,12 +172,6 @@ impl DefaultEngineValues {
self.cache_metrics_disabled = v;
self
}
/// Set whether to enable sparse trie as cache by default
pub const fn with_enable_sparse_trie_as_cache(mut self, v: bool) -> Self {
self.enable_sparse_trie_as_cache = v;
self
}
}
impl Default for DefaultEngineValues {
@@ -202,9 +195,8 @@ impl Default for DefaultEngineValues {
allow_unwind_canonical_header: false,
storage_worker_count: None,
account_worker_count: None,
disable_proof_v2: false,
enable_proof_v2: false,
cache_metrics_disabled: false,
enable_sparse_trie_as_cache: false,
}
}
}
@@ -325,17 +317,13 @@ pub struct EngineArgs {
#[arg(long = "engine.account-worker-count", default_value = Resettable::from(DefaultEngineValues::get_global().account_worker_count.map(|v| v.to_string().into())))]
pub account_worker_count: Option<usize>,
/// Disable V2 storage proofs for state root calculations
#[arg(long = "engine.disable-proof-v2", default_value_t = DefaultEngineValues::get_global().disable_proof_v2)]
pub disable_proof_v2: bool,
/// Enable V2 storage proofs for state root calculations
#[arg(long = "engine.enable-proof-v2", default_value_t = DefaultEngineValues::get_global().enable_proof_v2)]
pub enable_proof_v2: bool,
/// Disable cache metrics recording, which can take up to 50ms with large cached state.
#[arg(long = "engine.disable-cache-metrics", default_value_t = DefaultEngineValues::get_global().cache_metrics_disabled)]
pub cache_metrics_disabled: bool,
/// Enable sparse trie as cache.
#[arg(long = "engine.enable-sparse-trie-as-cache", default_value_t = DefaultEngineValues::get_global().enable_sparse_trie_as_cache, conflicts_with = "disable_proof_v2")]
pub enable_sparse_trie_as_cache: bool,
}
#[allow(deprecated)]
@@ -360,9 +348,8 @@ impl Default for EngineArgs {
allow_unwind_canonical_header,
storage_worker_count,
account_worker_count,
disable_proof_v2,
enable_proof_v2,
cache_metrics_disabled,
enable_sparse_trie_as_cache,
} = DefaultEngineValues::get_global().clone();
Self {
persistence_threshold,
@@ -387,9 +374,8 @@ impl Default for EngineArgs {
allow_unwind_canonical_header,
storage_worker_count,
account_worker_count,
disable_proof_v2,
enable_proof_v2,
cache_metrics_disabled,
enable_sparse_trie_as_cache,
}
}
}
@@ -424,9 +410,8 @@ impl EngineArgs {
config = config.with_account_worker_count(count);
}
config = config.with_disable_proof_v2(self.disable_proof_v2);
config = config.with_enable_proof_v2(self.enable_proof_v2);
config = config.without_cache_metrics(self.cache_metrics_disabled);
config = config.with_enable_sparse_trie_as_cache(self.enable_sparse_trie_as_cache);
config
}
@@ -477,9 +462,8 @@ mod tests {
allow_unwind_canonical_header: true,
storage_worker_count: Some(16),
account_worker_count: Some(8),
disable_proof_v2: false,
enable_proof_v2: false,
cache_metrics_disabled: true,
enable_sparse_trie_as_cache: false,
};
let parsed_args = CommandParser::<EngineArgs>::parse_from([

View File

@@ -5,9 +5,7 @@ use alloy_primitives::{Address, BlockNumber};
use clap::{builder::RangedU64ValueParser, Args};
use reth_chainspec::EthereumHardforks;
use reth_config::config::PruneConfig;
use reth_prune_types::{
PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_UNWIND_SAFE_DISTANCE,
};
use reth_prune_types::{PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_PRUNING_DISTANCE};
use std::{collections::BTreeMap, ops::Not, sync::OnceLock};
/// Global static pruning defaults
@@ -70,9 +68,9 @@ impl Default for DefaultPruningValues {
full_prune_modes: PruneModes {
sender_recovery: Some(PruneMode::Full),
transaction_lookup: None,
receipts: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
account_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
storage_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
receipts: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
account_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
storage_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
// This field is ignored when full_bodies_history_use_pre_merge is true
bodies_history: None,
receipts_log_filter: Default::default(),
@@ -82,9 +80,9 @@ impl Default for DefaultPruningValues {
sender_recovery: Some(PruneMode::Full),
transaction_lookup: Some(PruneMode::Full),
receipts: Some(PruneMode::Full),
account_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
storage_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
bodies_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
account_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
storage_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
bodies_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
receipts_log_filter: Default::default(),
},
}
@@ -95,8 +93,7 @@ impl Default for DefaultPruningValues {
#[derive(Debug, Clone, Args, PartialEq, Eq, Default)]
#[command(next_help_heading = "Pruning")]
pub struct PruningArgs {
/// Run full node. Only the most recent [`MINIMUM_UNWIND_SAFE_DISTANCE`] block states are
/// stored.
/// Run full node. Only the most recent [`MINIMUM_PRUNING_DISTANCE`] block states are stored.
#[arg(long, default_value_t = false, conflicts_with = "minimal")]
pub full: bool,

View File

@@ -1,27 +1,13 @@
//! clap [Args](clap::Args) for `RocksDB` table routing configuration
use clap::{ArgAction, Args};
use reth_storage_api::StorageSettings;
/// Default value for `tx_hash` routing flag.
/// Default value for `RocksDB` routing flags.
///
/// Derived from [`StorageSettings::base()`] to ensure CLI defaults match storage defaults.
const fn default_tx_hash_in_rocksdb() -> bool {
StorageSettings::base().transaction_hash_numbers_in_rocksdb
}
/// Default value for `storages_history` routing flag.
///
/// Derived from [`StorageSettings::base()`] to ensure CLI defaults match storage defaults.
const fn default_storages_history_in_rocksdb() -> bool {
StorageSettings::base().storages_history_in_rocksdb
}
/// Default value for `account_history` routing flag.
///
/// Derived from [`StorageSettings::base()`] to ensure CLI defaults match storage defaults.
const fn default_account_history_in_rocksdb() -> bool {
StorageSettings::base().account_history_in_rocksdb
/// When the `edge` feature is enabled, defaults to `true` to enable edge storage features.
/// Otherwise defaults to `false` for legacy behavior.
const fn default_rocksdb_flag() -> bool {
cfg!(feature = "edge")
}
/// Parameters for `RocksDB` table routing configuration.
@@ -42,21 +28,21 @@ pub struct RocksDbArgs {
///
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
/// Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
#[arg(long = "rocksdb.tx-hash", default_value_t = default_tx_hash_in_rocksdb(), action = ArgAction::Set)]
#[arg(long = "rocksdb.tx-hash", default_value_t = default_rocksdb_flag(), action = ArgAction::Set)]
pub tx_hash: bool,
/// Route storages history tables to `RocksDB` instead of MDBX.
///
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
/// Defaults to `false`.
#[arg(long = "rocksdb.storages-history", default_value_t = default_storages_history_in_rocksdb(), action = ArgAction::Set)]
/// Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
#[arg(long = "rocksdb.storages-history", default_value_t = default_rocksdb_flag(), action = ArgAction::Set)]
pub storages_history: bool,
/// Route account history tables to `RocksDB` instead of MDBX.
///
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
/// Defaults to `false`.
#[arg(long = "rocksdb.account-history", default_value_t = default_account_history_in_rocksdb(), action = ArgAction::Set)]
/// Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
#[arg(long = "rocksdb.account-history", default_value_t = default_rocksdb_flag(), action = ArgAction::Set)]
pub account_history: bool,
}
@@ -64,9 +50,9 @@ impl Default for RocksDbArgs {
fn default() -> Self {
Self {
all: false,
tx_hash: default_tx_hash_in_rocksdb(),
storages_history: default_storages_history_in_rocksdb(),
account_history: default_account_history_in_rocksdb(),
tx_hash: default_rocksdb_flag(),
storages_history: default_rocksdb_flag(),
account_history: default_rocksdb_flag(),
}
}
}
@@ -120,25 +106,7 @@ mod tests {
fn test_parse_all_flag() {
let args = CommandParser::<RocksDbArgs>::parse_from(["reth", "--rocksdb.all"]).args;
assert!(args.all);
assert_eq!(args.tx_hash, default_tx_hash_in_rocksdb());
}
#[test]
fn test_defaults_match_storage_settings() {
let args = RocksDbArgs::default();
let settings = StorageSettings::base();
assert_eq!(
args.tx_hash, settings.transaction_hash_numbers_in_rocksdb,
"tx_hash default should match StorageSettings::base()"
);
assert_eq!(
args.storages_history, settings.storages_history_in_rocksdb,
"storages_history default should match StorageSettings::base()"
);
assert_eq!(
args.account_history, settings.account_history_in_rocksdb,
"account_history default should match StorageSettings::base()"
);
assert_eq!(args.tx_hash, default_rocksdb_flag());
}
#[test]

View File

@@ -645,7 +645,7 @@ pub struct RpcServerArgs {
///
/// When enabled, transactions that fail execution will be skipped, and all subsequent
/// transactions from the same sender will also be skipped.
#[arg(long = "testing.skip-invalid-transactions", default_value_t = true)]
#[arg(long = "testing.skip-invalid-transactions", default_value_t = false)]
pub testing_skip_invalid_transactions: bool,
}
@@ -859,7 +859,7 @@ impl Default for RpcServerArgs {
rpc_state_cache,
gas_price_oracle,
rpc_send_raw_transaction_sync_timeout,
testing_skip_invalid_transactions: true,
testing_skip_invalid_transactions: false,
}
}
}

View File

@@ -90,7 +90,7 @@ impl StaticFilesArgs {
/// args.
///
/// If `minimal` is true, uses [`MINIMAL_BLOCKS_PER_FILE`] blocks per file as the default for
/// all segments.
/// headers, transactions, and receipts segments.
pub fn merge_with_config(&self, config: StaticFilesConfig, minimal: bool) -> StaticFilesConfig {
let minimal_blocks_per_file = minimal.then_some(MINIMAL_BLOCKS_PER_FILE);
StaticFilesConfig {
@@ -109,15 +109,12 @@ impl StaticFilesArgs {
.or(config.blocks_per_file.receipts),
transaction_senders: self
.blocks_per_file_transaction_senders
.or(minimal_blocks_per_file)
.or(config.blocks_per_file.transaction_senders),
account_change_sets: self
.blocks_per_file_account_change_sets
.or(minimal_blocks_per_file)
.or(config.blocks_per_file.account_change_sets),
storage_change_sets: self
.blocks_per_file_storage_change_sets
.or(minimal_blocks_per_file)
.or(config.blocks_per_file.storage_change_sets),
},
}

View File

@@ -507,7 +507,7 @@ impl RethTransactionPoolConfig for TxPoolArgs {
PoolConfig {
local_transactions_config: LocalTransactionConfig {
no_exemptions: self.no_locals,
local_addresses: self.locals.iter().copied().collect(),
local_addresses: self.locals.clone().into_iter().collect(),
propagate_local_transactions: !self.no_local_transactions_propagation,
},
pending_limit: SubPoolLimit {

View File

@@ -1,5 +1,7 @@
//! OP stack variation of chain spec constants.
use alloy_primitives::hex;
//------------------------------- BASE MAINNET -------------------------------//
/// Max gas limit on Base: <https://basescan.org/block/17208876>
@@ -9,3 +11,13 @@ pub const BASE_MAINNET_MAX_GAS_LIMIT: u64 = 105_000_000;
/// Max gas limit on Base Sepolia: <https://sepolia.basescan.org/block/12506483>
pub const BASE_SEPOLIA_MAX_GAS_LIMIT: u64 = 45_000_000;
//----------------------------------- DEV ------------------------------------//
/// Dummy system transaction for dev mode
/// OP Mainnet transaction at index 0 in block 124665056.
///
/// <https://optimistic.etherscan.io/tx/0x312e290cf36df704a2217b015d6455396830b0ce678b860ebfcc30f41403d7b1>
pub const TX_SET_L1_BLOCK_OP_MAINNET_BLOCK_124665056: [u8; 251] = hex!(
"7ef8f8a0683079df94aa5b9cf86687d739a60a9b4f0835e520ec4d664e2e415dca17a6df94deaddeaddeaddeaddeaddeaddeaddeaddead00019442000000000000000000000000000000000000158080830f424080b8a4440a5e200000146b000f79c500000000000000040000000066d052e700000000013ad8a3000000000000000000000000000000000000000000000000000000003ef1278700000000000000000000000000000000000000000000000000000000000000012fdf87b89884a61e74b322bbcf60386f543bfae7827725efaaf0ab1de2294a590000000000000000000000006887246668a3b87f54deb3b94ba47a6f63f32985"
);

View File

@@ -106,9 +106,7 @@ where
}
Commands::P2P(command) => runner.run_until_ctrl_c(command.execute::<OpNode>()),
Commands::Config(command) => runner.run_until_ctrl_c(command.execute()),
Commands::Prune(command) => {
runner.run_command_until_exit(|ctx| command.execute::<OpNode>(ctx))
}
Commands::Prune(command) => runner.run_until_ctrl_c(command.execute::<OpNode>()),
#[cfg(feature = "dev")]
Commands::TestVectors(command) => runner.run_until_ctrl_c(command.execute()),
Commands::ReExecute(command) => {

View File

@@ -4,17 +4,14 @@ use crate::{OpEthApi, OpEthApiError, SequencerClient};
use alloy_primitives::{Bytes, B256};
use alloy_rpc_types_eth::TransactionInfo;
use futures::StreamExt;
use op_alloy_consensus::{
transaction::{OpDepositInfo, OpTransactionInfo},
OpTransaction,
};
use op_alloy_consensus::{transaction::OpTransactionInfo, OpTransaction};
use reth_chain_state::CanonStateSubscriptions;
use reth_optimism_primitives::DepositReceipt;
use reth_primitives_traits::{Recovered, SignedTransaction, SignerRecoverable, WithEncoded};
use reth_rpc_eth_api::{
helpers::{spec::SignersForRpc, EthTransactions, LoadReceipt, LoadTransaction, SpawnBlocking},
EthApiTypes as _, FromEthApiError, FromEvmError, RpcConvert, RpcNodeCore, RpcReceipt,
TxInfoMapper,
try_into_op_tx_info, EthApiTypes as _, FromEthApiError, FromEvmError, RpcConvert, RpcNodeCore,
RpcReceipt, TxInfoMapper,
};
use reth_rpc_eth_types::{block::convert_transaction_receipt, EthApiError, TransactionSource};
use reth_storage_api::{errors::ProviderError, ProviderTx, ReceiptProvider, TransactionsProvider};
@@ -285,18 +282,6 @@ where
type Err = ProviderError;
fn try_map(&self, tx: &T, tx_info: TransactionInfo) -> Result<Self::Out, ProviderError> {
let deposit_meta = if tx.is_deposit() {
self.provider.receipt_by_hash(*tx.tx_hash())?.and_then(|receipt| {
receipt.as_deposit_receipt().map(|receipt| OpDepositInfo {
deposit_receipt_version: receipt.deposit_receipt_version,
deposit_nonce: receipt.deposit_nonce,
})
})
} else {
None
}
.unwrap_or_default();
Ok(OpTransactionInfo::new(tx_info, deposit_meta))
try_into_op_tx_info(&self.provider, tx, tx_info)
}
}

View File

@@ -89,12 +89,6 @@ impl From<revm_state::Account> for Account {
}
}
impl From<TrieAccount> for Account {
fn from(value: TrieAccount) -> Self {
Self { balance: value.balance, nonce: value.nonce, bytecode_hash: Some(value.code_hash) }
}
}
impl InMemorySize for Account {
fn size(&self) -> usize {
size_of::<Self>()

View File

@@ -149,7 +149,21 @@ where
let elapsed = start.elapsed();
self.metrics.duration_seconds.record(elapsed);
output.debug_log(tip_block_number, deleted_entries, elapsed);
let message = match output.progress {
PruneProgress::HasMoreData(_) => "Pruner interrupted and has more data to prune",
PruneProgress::Finished => "Pruner finished",
};
debug!(
target: "pruner",
%tip_block_number,
?elapsed,
?deleted_entries,
?limiter,
?output,
?stats,
"{message}",
);
self.event_sender.notify(PrunerEvent::Finished { tip_block_number, elapsed, stats });

View File

@@ -26,10 +26,6 @@ use tracing::{instrument, trace};
/// [`tables::AccountsHistory`]. We want to prune them to the same block number.
const ACCOUNT_HISTORY_TABLES_TO_PRUNE: usize = 2;
/// Maximum entries to process per internal batch for account history.
/// This bounds memory usage of the `highest_deleted_accounts` `HashMap`.
const MAX_ACCOUNT_HISTORY_ENTRIES_PER_RUN: usize = 200_000;
#[derive(Debug)]
pub struct AccountHistory {
mode: PruneMode,
@@ -82,10 +78,6 @@ where
impl AccountHistory {
/// Prunes account history when changesets are stored in static files.
///
/// When no limit is provided, uses internal batching to process ALL changesets in chunks,
/// preventing OOM by limiting the size of the `highest_deleted_accounts` `HashMap` per batch.
/// When a limit is provided, respects that limit and may return `HasMoreData`.
fn prune_static_files<Provider>(
&self,
provider: &Provider,
@@ -109,100 +101,52 @@ impl AccountHistory {
))
}
let mut total_pruned = 0;
let mut actual_last_pruned_block: Option<BlockNumber> = None;
let mut overall_done = false;
// The size of this map it's limited by `prune_delete_limit * blocks_since_last_run /
// ACCOUNT_HISTORY_TABLES_TO_PRUNE`, and with the current defaults it's usually `3500 * 5 /
// 2`, so 8750 entries. Each entry is `160 bit + 64 bit`, so the total size should be up to
// ~0.25MB + some hashmap overhead. `blocks_since_last_run` is additionally limited by the
// `max_reorg_depth`, so no OOM is expected here.
let mut highest_deleted_accounts = FxHashMap::default();
let mut last_changeset_pruned_block = None;
let mut pruned_changesets = 0;
let mut done = true;
let mut walker = StaticFileAccountChangesetWalker::new(provider, range).peekable();
loop {
let mut highest_deleted_accounts = FxHashMap::with_capacity_and_hasher(
MAX_ACCOUNT_HISTORY_ENTRIES_PER_RUN,
Default::default(),
);
let mut batch_last_block: Option<BlockNumber> = None;
let mut batch_count = 0;
while batch_count < MAX_ACCOUNT_HISTORY_ENTRIES_PER_RUN {
// Check limit before consuming next item
if limiter.is_limit_reached() {
break;
}
match walker.next() {
Some(Ok((block_number, changeset))) => {
highest_deleted_accounts.insert(changeset.address, block_number);
batch_last_block = Some(block_number);
batch_count += 1;
limiter.increment_deleted_entries_count();
}
Some(Err(e)) => return Err(e.into()),
None => break, // Iterator exhausted
}
}
// No more data in this batch
if batch_count == 0 {
break;
}
actual_last_pruned_block = batch_last_block;
// Check if there's more data after this batch
let has_more_data = walker.peek().is_some();
// Track whether we're done for the final output
overall_done = !has_more_data;
trace!(target: "pruner", pruned = %batch_count, done = %overall_done, "Pruned account history batch (changesets from static files)");
let result = HistoryPruneResult {
highest_deleted: highest_deleted_accounts,
last_pruned_block: batch_last_block,
pruned_count: batch_count,
done: overall_done,
};
let output = finalize_history_prune::<_, tables::AccountsHistory, _, _>(
provider,
result,
range_end,
&limiter,
ShardedKey::new,
|a, b| a.key == b.key,
)?;
total_pruned += output.pruned;
// If external limit reached and there's more data, stop
// Otherwise continue to next batch (or exit if no more data)
if limiter.is_limit_reached() && has_more_data {
let walker = StaticFileAccountChangesetWalker::new(provider, range);
for result in walker {
if limiter.is_limit_reached() {
done = false;
break;
}
let (block_number, changeset) = result?;
highest_deleted_accounts.insert(changeset.address, block_number);
last_changeset_pruned_block = Some(block_number);
pruned_changesets += 1;
limiter.increment_deleted_entries_count();
}
// Delete static files once at end
if let Some(last_block) = actual_last_pruned_block {
// Delete static file jars below the pruned block
if let Some(last_block) = last_changeset_pruned_block {
provider
.static_file_provider()
.delete_segment_below_block(StaticFileSegment::AccountChangeSets, last_block + 1)?;
}
trace!(target: "pruner", pruned = %pruned_changesets, %done, "Pruned account history (changesets from static files)");
trace!(target: "pruner", pruned = %total_pruned, %overall_done, "Pruned account history (changesets from static files)");
let progress = limiter.progress(overall_done);
let last_checkpoint_block = actual_last_pruned_block
.map(|block| if overall_done { block } else { block.saturating_sub(1) })
.unwrap_or(range_end);
Ok(SegmentOutput {
progress,
pruned: total_pruned,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: Some(last_checkpoint_block),
tx_number: None,
}),
})
let result = HistoryPruneResult {
highest_deleted: highest_deleted_accounts,
last_pruned_block: last_changeset_pruned_block,
pruned_count: pruned_changesets,
done,
};
finalize_history_prune::<_, tables::AccountsHistory, _, _>(
provider,
result,
range_end,
&limiter,
ShardedKey::new,
|a, b| a.key == b.key,
)
.map_err(Into::into)
}
fn prune_database<Provider>(

View File

@@ -11,7 +11,7 @@ use reth_provider::{
};
use reth_prune_types::{
PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment, ReceiptsLogPruneConfig, SegmentOutput,
MINIMUM_UNWIND_SAFE_DISTANCE,
MINIMUM_PRUNING_DISTANCE,
};
use tracing::{instrument, trace};
#[derive(Debug)]
@@ -49,8 +49,8 @@ where
fn prune(&self, provider: &Provider, input: PruneInput) -> Result<SegmentOutput, PrunerError> {
// Contract log filtering removes every receipt possible except the ones in the list. So,
// for the other receipts it's as if they had a `PruneMode::Distance()` of
// `MINIMUM_UNWIND_SAFE_DISTANCE`.
let to_block = PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)
// `MINIMUM_PRUNING_DISTANCE`.
let to_block = PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)
.prune_target_block(input.to_block, PruneSegment::ContractLogs, PrunePurpose::User)?
.map(|(bn, _)| bn)
.unwrap_or_default();

View File

@@ -1,18 +1,14 @@
use crate::{
db_ext::DbTxPruneExt,
segments::{self, PruneInput, Segment},
segments::{PruneInput, Segment},
PrunerError,
};
use reth_db_api::{tables, transaction::DbTxMut};
use reth_provider::{
BlockReader, DBProvider, EitherWriterDestination, StaticFileProviderFactory,
StorageSettingsCache, TransactionsProvider,
};
use reth_provider::{BlockReader, DBProvider, TransactionsProvider};
use reth_prune_types::{
PruneMode, PruneProgress, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint,
};
use reth_static_file_types::StaticFileSegment;
use tracing::{debug, instrument, trace};
use tracing::{instrument, trace};
#[derive(Debug)]
pub struct SenderRecovery {
@@ -27,11 +23,7 @@ impl SenderRecovery {
impl<Provider> Segment<Provider> for SenderRecovery
where
Provider: DBProvider<Tx: DbTxMut>
+ TransactionsProvider
+ BlockReader
+ StorageSettingsCache
+ StaticFileProviderFactory,
Provider: DBProvider<Tx: DbTxMut> + TransactionsProvider + BlockReader,
{
fn segment(&self) -> PruneSegment {
PruneSegment::SenderRecovery
@@ -47,16 +39,6 @@ where
#[instrument(target = "pruner", skip(self, provider), ret(level = "trace"))]
fn prune(&self, provider: &Provider, input: PruneInput) -> Result<SegmentOutput, PrunerError> {
if EitherWriterDestination::senders(provider).is_static_file() {
debug!(target: "pruner", "Pruning transaction senders from static files.");
return segments::prune_static_files(
provider,
input,
StaticFileSegment::TransactionSenders,
)
}
debug!(target: "pruner", "Pruning transaction senders from database.");
let tx_range = match input.get_next_tx_num_range(provider)? {
Some(range) => range,
None => {

View File

@@ -27,10 +27,6 @@ use tracing::{instrument, trace};
/// [`tables::StoragesHistory`]. We want to prune them to the same block number.
const STORAGE_HISTORY_TABLES_TO_PRUNE: usize = 2;
/// Maximum entries to process per internal batch for storage history.
/// This bounds memory usage of the `highest_deleted_storages` `HashMap`.
const MAX_STORAGE_HISTORY_ENTRIES_PER_RUN: usize = 200_000;
#[derive(Debug)]
pub struct StorageHistory {
mode: PruneMode,
@@ -82,11 +78,6 @@ where
impl StorageHistory {
/// Prunes storage history when changesets are stored in static files.
///
/// Uses internal batching to bound memory usage of the `highest_deleted_storages` `HashMap`.
/// Each batch processes up to [`MAX_STORAGE_HISTORY_ENTRIES_PER_RUN`] entries, then flushes
/// to the history index before continuing. This prevents OOM when pruning large ranges with
/// many unique (address, `storage_key`) pairs.
fn prune_static_files<Provider>(
&self,
provider: &Provider,
@@ -110,105 +101,56 @@ impl StorageHistory {
))
}
let mut total_pruned = 0;
let mut actual_last_pruned_block: Option<BlockNumber> = None;
let mut overall_done = false;
// The size of this map is limited by `prune_delete_limit * blocks_since_last_run /
// STORAGE_HISTORY_TABLES_TO_PRUNE`, and with current defaults it's usually `3500 * 5
// / 2`, so 8750 entries. Each entry is `160 bit + 256 bit + 64 bit`, so the total
// size should be up to ~0.5MB + some hashmap overhead. `blocks_since_last_run` is
// additionally limited by the `max_reorg_depth`, so no OOM is expected here.
let mut highest_deleted_storages = FxHashMap::default();
let mut last_changeset_pruned_block = None;
let mut pruned_changesets = 0;
let mut done = true;
let mut walker =
provider.static_file_provider().walk_storage_changeset_range(range).peekable();
loop {
let mut highest_deleted_storages = FxHashMap::with_capacity_and_hasher(
MAX_STORAGE_HISTORY_ENTRIES_PER_RUN,
Default::default(),
);
let mut batch_last_block: Option<BlockNumber> = None;
let mut batch_count = 0;
while batch_count < MAX_STORAGE_HISTORY_ENTRIES_PER_RUN {
// Check limit before consuming next item
if limiter.is_limit_reached() {
break;
}
match walker.next() {
Some(Ok((block_address, entry))) => {
let block_number = block_address.block_number();
let address = block_address.address();
highest_deleted_storages.insert((address, entry.key), block_number);
batch_last_block = Some(block_number);
batch_count += 1;
limiter.increment_deleted_entries_count();
}
Some(Err(e)) => return Err(e.into()),
None => break, // Iterator exhausted
}
}
// No more data in this batch
if batch_count == 0 {
break;
}
actual_last_pruned_block = batch_last_block;
// Check if there's more data after this batch
let has_more_data = walker.peek().is_some();
// Track whether we're done for the final output
overall_done = !has_more_data;
trace!(target: "pruner", pruned = %batch_count, done = %overall_done, "Pruned storage history batch (changesets from static files)");
let result = HistoryPruneResult {
highest_deleted: highest_deleted_storages,
last_pruned_block: batch_last_block,
pruned_count: batch_count,
done: overall_done,
};
let output = finalize_history_prune::<_, tables::StoragesHistory, (Address, B256), _>(
provider,
result,
range_end,
&limiter,
|(address, storage_key), block_number| {
StorageShardedKey::new(address, storage_key, block_number)
},
|a, b| a.address == b.address && a.sharded_key.key == b.sharded_key.key,
)?;
total_pruned += output.pruned;
// If external limit reached and there's more data, stop
// Otherwise continue to next batch (or exit if no more data)
if limiter.is_limit_reached() && has_more_data {
let walker = provider.static_file_provider().walk_storage_changeset_range(range);
for result in walker {
if limiter.is_limit_reached() {
done = false;
break;
}
let (block_address, entry) = result?;
let block_number = block_address.block_number();
let address = block_address.address();
highest_deleted_storages.insert((address, entry.key), block_number);
last_changeset_pruned_block = Some(block_number);
pruned_changesets += 1;
limiter.increment_deleted_entries_count();
}
// Delete static file jars below the pruned block (once at end)
if let Some(last_block) = actual_last_pruned_block {
// Delete static file jars below the pruned block
if let Some(last_block) = last_changeset_pruned_block {
provider
.static_file_provider()
.delete_segment_below_block(StaticFileSegment::StorageChangeSets, last_block + 1)?;
}
trace!(target: "pruner", pruned = %pruned_changesets, %done, "Pruned storage history (changesets from static files)");
trace!(target: "pruner", pruned = %total_pruned, %overall_done, "Pruned storage history (changesets from static files)");
let progress = limiter.progress(overall_done);
let last_checkpoint_block = actual_last_pruned_block
.map(|block| if overall_done { block } else { block.saturating_sub(1) })
.unwrap_or(range_end);
Ok(SegmentOutput {
progress,
pruned: total_pruned,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: Some(last_checkpoint_block),
tx_number: None,
}),
})
let result = HistoryPruneResult {
highest_deleted: highest_deleted_storages,
last_pruned_block: last_changeset_pruned_block,
pruned_count: pruned_changesets,
done,
};
finalize_history_prune::<_, tables::StoragesHistory, (Address, B256), _>(
provider,
result,
range_end,
&limiter,
|(address, storage_key), block_number| {
StorageShardedKey::new(address, storage_key, block_number)
},
|a, b| a.address == b.address && a.sharded_key.key == b.sharded_key.key,
)
.map_err(Into::into)
}
fn prune_database<Provider>(

View File

@@ -18,7 +18,6 @@ alloy-primitives.workspace = true
derive_more.workspace = true
strum = { workspace = true, features = ["derive"] }
thiserror.workspace = true
tracing.workspace = true
modular-bitfield = { workspace = true, optional = true }
serde = { workspace = true, features = ["derive"], optional = true }
@@ -43,9 +42,8 @@ std = [
"derive_more/std",
"serde?/std",
"serde_json/std",
"strum/std",
"thiserror/std",
"tracing/std",
"strum/std",
]
test-utils = [
"std",

View File

@@ -30,9 +30,7 @@ pub use pruner::{
SegmentOutputCheckpoint,
};
pub use segment::{PrunePurpose, PruneSegment, PruneSegmentError};
pub use target::{
PruneModes, UnwindTargetPrunedError, MINIMUM_DISTANCE, MINIMUM_UNWIND_SAFE_DISTANCE,
};
pub use target::{PruneModes, UnwindTargetPrunedError, MINIMUM_PRUNING_DISTANCE};
/// Configuration for pruning receipts not associated with logs emitted by the specified contracts.
#[derive(Debug, Clone, PartialEq, Eq, Default)]

View File

@@ -41,12 +41,8 @@ impl PruneMode {
segment: PruneSegment,
purpose: PrunePurpose,
) -> Result<Option<(BlockNumber, Self)>, PruneSegmentError> {
let min_blocks = segment.min_blocks();
let result = match self {
Self::Full if min_blocks == 0 => Some((tip, *self)),
// For segments with min_blocks > 0, Full mode behaves like Distance(min_blocks)
Self::Full if min_blocks <= tip => Some((tip - min_blocks, *self)),
Self::Full => None, // Nothing to prune yet
Self::Full if segment.min_blocks() == 0 => Some((tip, *self)),
Self::Distance(distance) if *distance > tip => None, // Nothing to prune yet
Self::Distance(distance) if *distance >= segment.min_blocks() => {
Some((tip - distance, *self))
@@ -88,7 +84,9 @@ impl PruneMode {
#[cfg(test)]
mod tests {
use crate::{PruneMode, PrunePurpose, PruneSegment, MINIMUM_UNWIND_SAFE_DISTANCE};
use crate::{
PruneMode, PrunePurpose, PruneSegment, PruneSegmentError, MINIMUM_PRUNING_DISTANCE,
};
use assert_matches::assert_matches;
use serde::Deserialize;
@@ -98,8 +96,8 @@ mod tests {
let segment = PruneSegment::AccountHistory;
let tests = vec![
// Full mode with min_blocks > 0 behaves like Distance(min_blocks)
(PruneMode::Full, Ok(Some(tip - segment.min_blocks()))),
// MINIMUM_PRUNING_DISTANCE makes this impossible
(PruneMode::Full, Err(PruneSegmentError::Configuration(segment))),
// Nothing to prune
(PruneMode::Distance(tip + 1), Ok(None)),
(
@@ -109,12 +107,12 @@ mod tests {
// Nothing to prune
(PruneMode::Before(tip + 1), Ok(None)),
(
PruneMode::Before(tip - MINIMUM_UNWIND_SAFE_DISTANCE),
Ok(Some(tip - MINIMUM_UNWIND_SAFE_DISTANCE - 1)),
PruneMode::Before(tip - MINIMUM_PRUNING_DISTANCE),
Ok(Some(tip - MINIMUM_PRUNING_DISTANCE - 1)),
),
(
PruneMode::Before(tip - MINIMUM_UNWIND_SAFE_DISTANCE - 1),
Ok(Some(tip - MINIMUM_UNWIND_SAFE_DISTANCE - 2)),
PruneMode::Before(tip - MINIMUM_PRUNING_DISTANCE - 1),
Ok(Some(tip - MINIMUM_PRUNING_DISTANCE - 2)),
),
// Nothing to prune
(PruneMode::Before(tip - 1), Ok(None)),
@@ -148,13 +146,13 @@ mod tests {
let tests = vec![
(PruneMode::Distance(tip + 1), 1, !should_prune),
(
PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE + 1),
tip - MINIMUM_UNWIND_SAFE_DISTANCE - 1,
PruneMode::Distance(MINIMUM_PRUNING_DISTANCE + 1),
tip - MINIMUM_PRUNING_DISTANCE - 1,
!should_prune,
),
(
PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE + 1),
tip - MINIMUM_UNWIND_SAFE_DISTANCE - 2,
PruneMode::Distance(MINIMUM_PRUNING_DISTANCE + 1),
tip - MINIMUM_PRUNING_DISTANCE - 2,
should_prune,
),
(PruneMode::Before(tip + 1), 1, should_prune),

View File

@@ -1,9 +1,7 @@
use crate::{PruneCheckpoint, PruneMode, PruneSegment};
use alloc::{format, string::ToString, vec::Vec};
use alloc::vec::Vec;
use alloy_primitives::{BlockNumber, TxNumber};
use core::time::Duration;
use derive_more::Display;
use tracing::debug;
/// Pruner run output.
#[derive(Debug)]
@@ -20,49 +18,6 @@ impl From<PruneProgress> for PrunerOutput {
}
}
impl PrunerOutput {
/// Logs a human-readable summary of the pruner run at DEBUG level.
///
/// Format: `"Pruner finished tip=24328929 deleted=10886 elapsed=148ms
/// segments=AccountHistory[24318865, done] ..."`
#[inline]
pub fn debug_log(
&self,
tip_block_number: BlockNumber,
deleted_entries: usize,
elapsed: Duration,
) {
let message = match self.progress {
PruneProgress::HasMoreData(_) => "Pruner interrupted, has more data",
PruneProgress::Finished => "Pruner finished",
};
let segments: Vec<_> = self
.segments
.iter()
.filter(|(_, seg)| seg.pruned > 0)
.map(|(segment, seg)| {
let block = seg
.checkpoint
.and_then(|c| c.block_number)
.map(|b| b.to_string())
.unwrap_or_else(|| "?".to_string());
let status = if seg.progress.is_finished() { "done" } else { "in_progress" };
format!("{segment}[{block}, {status}]")
})
.collect();
debug!(
target: "pruner",
%tip_block_number,
deleted_entries,
?elapsed,
segments = %segments.join(" "),
"{message}",
);
}
}
/// Represents information of a pruner run for a segment.
#[derive(Debug, Clone, PartialEq, Eq, Display)]
#[display("(table={segment}, pruned={pruned}, status={progress})")]

View File

@@ -1,6 +1,6 @@
#![allow(deprecated)] // necessary to all defining deprecated `PruneSegment` variants
use crate::{MINIMUM_DISTANCE, MINIMUM_UNWIND_SAFE_DISTANCE};
use crate::MINIMUM_PRUNING_DISTANCE;
use derive_more::Display;
use strum::{EnumIter, IntoEnumIterator};
use thiserror::Error;
@@ -65,10 +65,9 @@ impl PruneSegment {
/// Returns minimum number of blocks to keep in the database for this segment.
pub const fn min_blocks(&self) -> u64 {
match self {
Self::SenderRecovery | Self::TransactionLookup => 0,
Self::Receipts | Self::Bodies => MINIMUM_DISTANCE,
Self::SenderRecovery | Self::TransactionLookup | Self::Receipts | Self::Bodies => 0,
Self::ContractLogs | Self::AccountHistory | Self::StorageHistory => {
MINIMUM_UNWIND_SAFE_DISTANCE
MINIMUM_PRUNING_DISTANCE
}
#[expect(deprecated)]
#[expect(clippy::match_same_arms)]

View File

@@ -9,12 +9,7 @@ use crate::{PruneCheckpoint, PruneMode, PruneSegment, ReceiptsLogPruneConfig};
/// consensus protocol.
/// 2. Another 10k blocks to have a room for maneuver in case when things go wrong and a manual
/// unwind is required.
pub const MINIMUM_UNWIND_SAFE_DISTANCE: u64 = 32 * 2 + 10_000;
/// Minimum blocks to retain for receipts and bodies to ensure reorg safety.
/// This prevents pruning data that may be needed when handling chain reorganizations,
/// specifically when `canonical_block_by_hash` needs to reconstruct `ExecutedBlock` from disk.
pub const MINIMUM_DISTANCE: u64 = 64;
pub const MINIMUM_PRUNING_DISTANCE: u64 = 32 * 2 + 10_000;
/// Type of history that can be pruned
#[derive(Debug, Error, PartialEq, Eq, Clone)]
@@ -61,7 +56,7 @@ pub struct PruneModes {
any(test, feature = "serde"),
serde(
skip_serializing_if = "Option::is_none",
deserialize_with = "deserialize_opt_prune_mode_with_min_blocks::<MINIMUM_UNWIND_SAFE_DISTANCE, _>"
deserialize_with = "deserialize_opt_prune_mode_with_min_blocks::<MINIMUM_PRUNING_DISTANCE, _>"
)
)]
pub account_history: Option<PruneMode>,
@@ -70,7 +65,7 @@ pub struct PruneModes {
any(test, feature = "serde"),
serde(
skip_serializing_if = "Option::is_none",
deserialize_with = "deserialize_opt_prune_mode_with_min_blocks::<MINIMUM_UNWIND_SAFE_DISTANCE, _>"
deserialize_with = "deserialize_opt_prune_mode_with_min_blocks::<MINIMUM_PRUNING_DISTANCE, _>"
)
)]
pub storage_history: Option<PruneMode>,

View File

@@ -117,7 +117,10 @@ impl tokio_util::codec::Decoder for StreamCodec {
buf.advance(start_idx);
}
let bts = buf.split_to(idx + 1 - start_idx);
return Ok(String::from_utf8(bts.into()).ok())
return match String::from_utf8(bts.into()) {
Ok(val) => Ok(Some(val)),
Err(_) => Ok(None),
}
}
}
Ok(None)

View File

@@ -14,6 +14,7 @@ workspace = true
[dependencies]
# reth
reth-primitives-traits.workspace = true
reth-storage-api = { workspace = true, optional = true }
reth-evm.workspace = true
reth-ethereum-primitives.workspace = true
@@ -30,6 +31,7 @@ alloy-evm = { workspace = true, features = ["rpc"] }
op-alloy-consensus = { workspace = true, optional = true }
op-alloy-rpc-types = { workspace = true, optional = true }
op-alloy-network = { workspace = true, optional = true }
reth-optimism-primitives = { workspace = true, optional = true }
# io
jsonrpsee-types.workspace = true
@@ -49,6 +51,8 @@ op = [
"dep:op-alloy-consensus",
"dep:op-alloy-rpc-types",
"dep:op-alloy-network",
"dep:reth-optimism-primitives",
"dep:reth-storage-api",
"reth-evm/op",
"reth-primitives-traits/op",
"alloy-evm/op",

View File

@@ -24,3 +24,6 @@ pub use transaction::{
};
pub use alloy_evm::rpc::{CallFees, CallFeesError, EthTxEnvError, TryIntoTxEnv};
#[cfg(feature = "op")]
pub use transaction::op::*;

View File

@@ -29,7 +29,7 @@ impl TryFromReceiptResponse<alloy_network::Ethereum> for reth_ethereum_primitive
}
#[cfg(feature = "op")]
impl TryFromReceiptResponse<op_alloy_network::Optimism> for op_alloy_consensus::OpReceipt {
impl TryFromReceiptResponse<op_alloy_network::Optimism> for reth_optimism_primitives::OpReceipt {
type Error = Infallible;
fn from_receipt_response(

View File

@@ -872,8 +872,40 @@ pub mod op {
use super::*;
use alloy_consensus::SignableTransaction;
use alloy_signer::Signature;
use op_alloy_consensus::{transaction::OpTransactionInfo, OpTxEnvelope};
use op_alloy_consensus::{
transaction::{OpDepositInfo, OpTransactionInfo},
OpTxEnvelope,
};
use op_alloy_rpc_types::OpTransactionRequest;
use reth_optimism_primitives::DepositReceipt;
use reth_primitives_traits::SignedTransaction;
use reth_storage_api::{errors::ProviderError, ReceiptProvider};
/// Creates [`OpTransactionInfo`] by adding [`OpDepositInfo`] to [`TransactionInfo`] if `tx` is
/// a deposit.
pub fn try_into_op_tx_info<Tx, T>(
provider: &T,
tx: &Tx,
tx_info: TransactionInfo,
) -> Result<OpTransactionInfo, ProviderError>
where
Tx: op_alloy_consensus::OpTransaction + SignedTransaction,
T: ReceiptProvider<Receipt: DepositReceipt>,
{
let deposit_meta = if tx.is_deposit() {
provider.receipt_by_hash(*tx.tx_hash())?.and_then(|receipt| {
receipt.as_deposit_receipt().map(|receipt| OpDepositInfo {
deposit_receipt_version: receipt.deposit_receipt_version,
deposit_nonce: receipt.deposit_nonce,
})
})
} else {
None
}
.unwrap_or_default();
Ok(OpTransactionInfo::new(tx_info, deposit_meta))
}
impl<T: op_alloy_consensus::OpTransaction + alloy_consensus::Transaction> FromConsensusTx<T>
for op_alloy_rpc_types::Transaction<T>
@@ -932,7 +964,9 @@ impl TryFromTransactionResponse<alloy_network::Ethereum>
}
#[cfg(feature = "op")]
impl TryFromTransactionResponse<op_alloy_network::Optimism> for op_alloy_consensus::OpTxEnvelope {
impl TryFromTransactionResponse<op_alloy_network::Optimism>
for reth_optimism_primitives::OpTransactionSigned
{
type Error = Infallible;
fn from_transaction_response(
@@ -981,6 +1015,7 @@ mod transaction_response_tests {
fn test_optimism_transaction_conversion() {
use op_alloy_consensus::OpTxEnvelope;
use op_alloy_network::Optimism;
use reth_optimism_primitives::OpTransactionSigned;
let signed_tx = Signed::new_unchecked(
TxLegacy::default(),
@@ -1003,10 +1038,7 @@ mod transaction_response_tests {
deposit_receipt_version: None,
};
let result =
<OpTxEnvelope as TryFromTransactionResponse<Optimism>>::from_transaction_response(
tx_response,
);
let result = <OpTransactionSigned as TryFromTransactionResponse<Optimism>>::from_transaction_response(tx_response);
assert!(result.is_ok());
}

View File

@@ -318,9 +318,9 @@ where
.map(|inner| {
let full_log = alloy_rpc_types_eth::Log {
inner,
block_hash: Some(current_block.hash()),
block_number: Some(current_block.number()),
block_timestamp: Some(current_block.timestamp()),
block_hash: None,
block_number: None,
block_timestamp: None,
transaction_hash: Some(*item.tx.tx_hash()),
transaction_index: Some(tx_index as u64),
log_index: Some(log_index),

View File

@@ -73,7 +73,7 @@ reth-ethereum-consensus.workspace = true
reth-evm-ethereum.workspace = true
reth-consensus = { workspace = true, features = ["test-utils"] }
reth-network-p2p = { workspace = true, features = ["test-utils"] }
reth-downloaders = { workspace = true, features = ["file-client"] }
reth-downloaders.workspace = true
reth-static-file.workspace = true
reth-stages-api = { workspace = true, features = ["test-utils"] }
reth-storage-api.workspace = true
@@ -82,11 +82,8 @@ reth-trie = { workspace = true, features = ["test-utils"] }
reth-provider = { workspace = true, features = ["test-utils"] }
reth-network-peers.workspace = true
alloy-genesis.workspace = true
alloy-primitives = { workspace = true, features = ["getrandom", "rand"] }
alloy-rlp.workspace = true
reth-db-common.workspace = true
reth-tracing.workspace = true
tokio = { workspace = true, features = ["rt", "sync", "macros"] }
assert_matches.workspace = true
@@ -122,7 +119,6 @@ test-utils = [
"reth-evm-ethereum/test-utils",
]
rocksdb = ["reth-provider/rocksdb"]
edge = ["reth-provider/edge", "reth-db-common/edge", "rocksdb"]
[[bench]]
name = "criterion"

View File

@@ -3,7 +3,7 @@ use itertools::Itertools;
use reth_config::config::{EtlConfig, HashingConfig};
use reth_db_api::{
cursor::{DbCursorRO, DbDupCursorRW},
models::CompactU256,
models::{BlockNumberAddress, CompactU256},
table::Decompress,
tables,
transaction::{DbTx, DbTxMut},
@@ -179,7 +179,7 @@ where
let (range, unwind_progress, _) =
input.unwind_block_range_with_threshold(self.commit_threshold);
provider.unwind_storage_hashing_range(range)?;
provider.unwind_storage_hashing_range(BlockNumberAddress::range(range))?;
let mut stage_checkpoint =
input.checkpoint.storage_hashing_stage_checkpoint().unwrap_or_default();
@@ -227,7 +227,7 @@ mod tests {
use rand::Rng;
use reth_db_api::{
cursor::{DbCursorRW, DbDupCursorRO},
models::{BlockNumberAddress, StoredBlockBodyIndices},
models::StoredBlockBodyIndices,
};
use reth_ethereum_primitives::Block;
use reth_primitives_traits::SealedBlock;

View File

@@ -166,7 +166,7 @@ where
let (range, unwind_progress, _) =
input.unwind_block_range_with_threshold(self.commit_threshold);
provider.unwind_storage_history_indices_range(range)?;
provider.unwind_storage_history_indices_range(BlockNumberAddress::range(range))?;
Ok(UnwindOutput { checkpoint: StageCheckpoint::new(unwind_progress) })
}

View File

@@ -1,490 +0,0 @@
//! Pipeline forward sync and unwind tests.
use alloy_consensus::{constants::ETH_TO_WEI, Header, TxEip1559, TxReceipt};
use alloy_eips::eip1559::INITIAL_BASE_FEE;
use alloy_genesis::{Genesis, GenesisAccount};
use alloy_primitives::{bytes, Address, Bytes, TxKind, B256, U256};
use reth_chainspec::{ChainSpecBuilder, ChainSpecProvider, MAINNET};
use reth_config::config::StageConfig;
use reth_consensus::noop::NoopConsensus;
use reth_db_api::{cursor::DbCursorRO, models::BlockNumberAddress, transaction::DbTx};
use reth_db_common::init::init_genesis;
use reth_downloaders::{
bodies::bodies::BodiesDownloaderBuilder, file_client::FileClient,
headers::reverse_headers::ReverseHeadersDownloaderBuilder,
};
use reth_ethereum_primitives::{Block, BlockBody, Transaction};
use reth_evm::{execute::Executor, ConfigureEvm};
use reth_evm_ethereum::EthEvmConfig;
use reth_network_p2p::{
bodies::downloader::BodyDownloader,
headers::downloader::{HeaderDownloader, SyncTarget},
};
use reth_primitives_traits::{
crypto::secp256k1::public_key_to_address,
proofs::{calculate_receipt_root, calculate_transaction_root},
RecoveredBlock, SealedBlock,
};
use reth_provider::{
test_utils::create_test_provider_factory_with_chain_spec, BlockNumReader, DBProvider,
DatabaseProviderFactory, HeaderProvider, OriginalValuesKnown, StageCheckpointReader,
StateWriter, StaticFileProviderFactory,
};
use reth_prune_types::PruneModes;
use reth_revm::database::StateProviderDatabase;
use reth_stages::sets::DefaultStages;
use reth_stages_api::{Pipeline, StageId};
use reth_static_file::StaticFileProducer;
use reth_storage_api::{
ChangeSetReader, StateProvider, StorageChangeSetReader, StorageSettingsCache,
};
use reth_testing_utils::generators::{self, generate_key, sign_tx_with_key_pair};
use reth_trie::{HashedPostState, KeccakKeyHasher, StateRoot};
use reth_trie_db::DatabaseStateRoot;
use std::sync::Arc;
use tokio::sync::watch;
/// Counter contract deployed bytecode compiled with Solidity 0.8.31.
/// ```solidity
/// contract Counter {
/// uint256 public count;
/// function increment() public { count += 1; }
/// }
/// ```
const COUNTER_DEPLOYED_BYTECODE: Bytes = bytes!(
"6080604052348015600e575f5ffd5b50600436106030575f3560e01c806306661abd146034578063d09de08a14604e575b5f5ffd5b603a6056565b604051604591906089565b60405180910390f35b6054605b565b005b5f5481565b60015f5f828254606a919060cd565b92505081905550565b5f819050919050565b6083816073565b82525050565b5f602082019050609a5f830184607c565b92915050565b7f4e487b71000000000000000000000000000000000000000000000000000000005f52601160045260245ffd5b5f60d5826073565b915060de836073565b925082820190508082111560f35760f260a0565b5b9291505056fea2646970667358221220576016d010ec2f4f83b992fb97d16efd1bc54110c97aa5d5cb47d20d3b39a35264736f6c634300081f0033"
);
/// `increment()` function selector: `keccak256("increment()")`[:4]
const INCREMENT_SELECTOR: [u8; 4] = [0xd0, 0x9d, 0xe0, 0x8a];
/// Contract address (deterministic for test)
const CONTRACT_ADDRESS: Address = Address::new([0x42; 20]);
/// Creates a `FileClient` populated with the given blocks.
fn create_file_client_from_blocks(blocks: Vec<SealedBlock<Block>>) -> Arc<FileClient<Block>> {
Arc::new(FileClient::from_blocks(blocks))
}
/// Verifies that changesets are queryable from the correct source based on storage settings.
///
/// Queries static files when changesets are configured to be stored there, otherwise queries MDBX.
fn assert_changesets_queryable(
provider_factory: &reth_provider::ProviderFactory<
reth_provider::test_utils::MockNodeTypesWithDB,
>,
block_range: std::ops::RangeInclusive<u64>,
) -> eyre::Result<()> {
let provider = provider_factory.provider()?;
let settings = provider.cached_storage_settings();
// Verify storage changesets
if settings.storage_changesets_in_static_files {
let static_file_provider = provider_factory.static_file_provider();
static_file_provider.initialize_index()?;
let storage_changesets =
static_file_provider.storage_changesets_range(block_range.clone())?;
assert!(
!storage_changesets.is_empty(),
"storage changesets should be queryable from static files for blocks {:?}",
block_range
);
} else {
let storage_changesets: Vec<_> = provider
.tx_ref()
.cursor_dup_read::<reth_db::tables::StorageChangeSets>()?
.walk_range(BlockNumberAddress::range(block_range.clone()))?
.collect::<Result<Vec<_>, _>>()?;
assert!(
!storage_changesets.is_empty(),
"storage changesets should be queryable from MDBX for blocks {:?}",
block_range
);
}
// Verify account changesets
if settings.account_changesets_in_static_files {
let static_file_provider = provider_factory.static_file_provider();
static_file_provider.initialize_index()?;
let account_changesets =
static_file_provider.account_changesets_range(block_range.clone())?;
assert!(
!account_changesets.is_empty(),
"account changesets should be queryable from static files for blocks {:?}",
block_range
);
} else {
let account_changesets: Vec<_> = provider
.tx_ref()
.cursor_read::<reth_db::tables::AccountChangeSets>()?
.walk_range(block_range.clone())?
.collect::<Result<Vec<_>, _>>()?;
assert!(
!account_changesets.is_empty(),
"account changesets should be queryable from MDBX for blocks {:?}",
block_range
);
}
Ok(())
}
/// Builds downloaders from a `FileClient`.
fn build_downloaders_from_file_client(
file_client: Arc<FileClient<Block>>,
genesis: reth_primitives_traits::SealedHeader<Header>,
stages_config: StageConfig,
consensus: Arc<NoopConsensus>,
provider_factory: reth_provider::ProviderFactory<
reth_provider::test_utils::MockNodeTypesWithDB,
>,
) -> (impl HeaderDownloader<Header = Header>, impl BodyDownloader<Block = Block>) {
let tip = file_client.tip().expect("file client should have tip");
let min_block = file_client.min_block().expect("file client should have min block");
let max_block = file_client.max_block().expect("file client should have max block");
let mut header_downloader = ReverseHeadersDownloaderBuilder::new(stages_config.headers)
.build(file_client.clone(), consensus.clone())
.into_task();
header_downloader.update_local_head(genesis);
header_downloader.update_sync_target(SyncTarget::Tip(tip));
let mut body_downloader = BodiesDownloaderBuilder::new(stages_config.bodies)
.build(file_client, consensus, provider_factory)
.into_task();
body_downloader.set_download_range(min_block..=max_block).expect("set download range");
(header_downloader, body_downloader)
}
/// Builds a pipeline with `DefaultStages`.
fn build_pipeline<H, B>(
provider_factory: reth_provider::ProviderFactory<
reth_provider::test_utils::MockNodeTypesWithDB,
>,
header_downloader: H,
body_downloader: B,
max_block: u64,
tip: B256,
) -> Pipeline<reth_provider::test_utils::MockNodeTypesWithDB>
where
H: HeaderDownloader<Header = Header> + 'static,
B: BodyDownloader<Block = Block> + 'static,
{
let consensus = NoopConsensus::arc();
let stages_config = StageConfig::default();
let evm_config = EthEvmConfig::new(provider_factory.chain_spec());
let (tip_tx, tip_rx) = watch::channel(B256::ZERO);
let static_file_producer =
StaticFileProducer::new(provider_factory.clone(), PruneModes::default());
let stages = DefaultStages::new(
provider_factory.clone(),
tip_rx,
consensus,
header_downloader,
body_downloader,
evm_config,
stages_config,
PruneModes::default(),
None,
);
let pipeline = Pipeline::builder()
.with_tip_sender(tip_tx)
.with_max_block(max_block)
.with_fail_on_unwind(true)
.add_stages(stages)
.build(provider_factory, static_file_producer);
pipeline.set_tip(tip);
pipeline
}
/// Tests pipeline with ALL stages enabled using both ETH transfers and contract storage changes.
///
/// This test:
/// 1. Pre-funds a signer account and deploys a Counter contract in genesis
/// 2. Each block contains two transactions:
/// - ETH transfer to a recipient (account state changes)
/// - Counter `increment()` call (storage state changes)
/// 3. Runs the full pipeline with ALL stages enabled
/// 4. Forward syncs to block 5, unwinds to block 2
///
/// This exercises both account and storage hashing/history stages.
#[tokio::test(flavor = "multi_thread")]
async fn test_pipeline() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
// Generate a keypair for signing transactions
let mut rng = generators::rng();
let key_pair = generate_key(&mut rng);
let signer_address = public_key_to_address(key_pair.public_key());
// Recipient address for ETH transfers
let recipient_address = Address::new([0x11; 20]);
// Create a chain spec with:
// - Signer pre-funded with 1000 ETH
// - Counter contract pre-deployed at CONTRACT_ADDRESS
let initial_balance = U256::from(ETH_TO_WEI) * U256::from(1000);
let chain_spec = Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(Genesis {
alloc: [
(
signer_address,
GenesisAccount { balance: initial_balance, ..Default::default() },
),
(
CONTRACT_ADDRESS,
GenesisAccount {
code: Some(COUNTER_DEPLOYED_BYTECODE),
..Default::default()
},
),
]
.into(),
..MAINNET.genesis.clone()
})
.shanghai_activated()
.build(),
);
let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec.clone());
init_genesis(&provider_factory).expect("init genesis");
let genesis = provider_factory.sealed_header(0)?.expect("genesis should exist");
let evm_config = EthEvmConfig::new(chain_spec.clone());
// Build blocks by actually executing transactions to get correct state roots
let num_blocks = 5u64;
let mut blocks: Vec<SealedBlock<Block>> = Vec::new();
let mut parent_hash = genesis.hash();
let gas_price = INITIAL_BASE_FEE as u128;
let transfer_value = U256::from(ETH_TO_WEI); // 1 ETH per block
for block_num in 1..=num_blocks {
// Each block has 2 transactions: ETH transfer + Counter increment
let base_nonce = (block_num - 1) * 2;
// Transaction 1: ETH transfer
let eth_transfer_tx = sign_tx_with_key_pair(
key_pair,
Transaction::Eip1559(TxEip1559 {
chain_id: chain_spec.chain.id(),
nonce: base_nonce,
gas_limit: 21_000,
max_fee_per_gas: gas_price,
max_priority_fee_per_gas: 0,
to: TxKind::Call(recipient_address),
value: transfer_value,
input: Bytes::new(),
..Default::default()
}),
);
// Transaction 2: Counter increment
let counter_tx = sign_tx_with_key_pair(
key_pair,
Transaction::Eip1559(TxEip1559 {
chain_id: chain_spec.chain.id(),
nonce: base_nonce + 1,
gas_limit: 100_000, // Enough gas for SSTORE operations
max_fee_per_gas: gas_price,
max_priority_fee_per_gas: 0,
to: TxKind::Call(CONTRACT_ADDRESS),
value: U256::ZERO,
input: Bytes::from(INCREMENT_SELECTOR.to_vec()),
..Default::default()
}),
);
let transactions = vec![eth_transfer_tx, counter_tx];
let tx_root = calculate_transaction_root(&transactions);
// Build a temporary header for execution
let temp_header = Header {
parent_hash,
number: block_num,
gas_limit: 30_000_000,
base_fee_per_gas: Some(INITIAL_BASE_FEE),
timestamp: block_num * 12,
..Default::default()
};
// Execute the block to get the state changes
let provider = provider_factory.database_provider_rw()?;
let block_with_senders = RecoveredBlock::new_unhashed(
Block::new(
temp_header.clone(),
BlockBody {
transactions: transactions.clone(),
ommers: Vec::new(),
withdrawals: None,
},
),
vec![signer_address, signer_address], // Both txs from same sender
);
// Execute in a scope so state_provider is dropped before we use provider for writes
let output = {
let state_provider = provider.latest();
let db = StateProviderDatabase::new(&*state_provider);
let executor = evm_config.batch_executor(db);
executor.execute(&block_with_senders)?
};
let gas_used = output.gas_used;
// Convert bundle state to hashed post state and compute state root
let hashed_state =
HashedPostState::from_bundle_state::<KeccakKeyHasher>(output.state.state());
let (state_root, _trie_updates) = StateRoot::overlay_root_with_updates(
provider.tx_ref(),
&hashed_state.clone().into_sorted(),
)?;
// Create receipts for receipt root calculation (one per transaction)
let receipts: Vec<_> = output.receipts.iter().map(|r| r.with_bloom_ref()).collect();
let receipts_root = calculate_receipt_root(&receipts);
let header = Header {
parent_hash,
number: block_num,
state_root,
transactions_root: tx_root,
receipts_root,
gas_limit: 30_000_000,
gas_used,
base_fee_per_gas: Some(INITIAL_BASE_FEE),
timestamp: block_num * 12,
..Default::default()
};
let block: SealedBlock<Block> = SealedBlock::seal_parts(
header.clone(),
BlockBody { transactions, ommers: Vec::new(), withdrawals: None },
);
// Write the plain state to database so subsequent blocks build on it
let plain_state = output.state.to_plain_state(OriginalValuesKnown::Yes);
provider.write_state_changes(plain_state)?;
provider.write_hashed_state(&hashed_state.into_sorted())?;
provider.commit()?;
parent_hash = block.hash();
blocks.push(block);
}
// Create a fresh provider factory for the pipeline (clean state from genesis)
// This is needed because we wrote state during block generation for computing state roots
let pipeline_provider_factory =
create_test_provider_factory_with_chain_spec(chain_spec.clone());
init_genesis(&pipeline_provider_factory).expect("init genesis");
let pipeline_genesis =
pipeline_provider_factory.sealed_header(0)?.expect("genesis should exist");
let pipeline_consensus = NoopConsensus::arc();
let file_client = create_file_client_from_blocks(blocks);
let max_block = file_client.max_block().unwrap();
let tip = file_client.tip().expect("tip");
let stages_config = StageConfig::default();
let (header_downloader, body_downloader) = build_downloaders_from_file_client(
file_client,
pipeline_genesis,
stages_config,
pipeline_consensus,
pipeline_provider_factory.clone(),
);
let pipeline = build_pipeline(
pipeline_provider_factory.clone(),
header_downloader,
body_downloader,
max_block,
tip,
);
let (mut pipeline, result) = pipeline.run_as_fut(None).await;
result?;
// Verify forward sync
{
let provider = pipeline_provider_factory.provider()?;
let last_block = provider.last_block_number()?;
assert_eq!(last_block, 5, "should have synced 5 blocks");
for stage_id in [
StageId::Headers,
StageId::Bodies,
StageId::SenderRecovery,
StageId::Execution,
StageId::AccountHashing,
StageId::StorageHashing,
StageId::MerkleExecute,
StageId::TransactionLookup,
StageId::IndexAccountHistory,
StageId::IndexStorageHistory,
StageId::Finish,
] {
let checkpoint = provider.get_stage_checkpoint(stage_id)?;
assert_eq!(
checkpoint.map(|c| c.block_number),
Some(5),
"{stage_id} checkpoint should be at block 5"
);
}
// Verify the counter contract's storage was updated
// After 5 blocks with 1 increment each, slot 0 should be 5
let state = provider.latest();
let counter_storage = state.storage(CONTRACT_ADDRESS, B256::ZERO)?;
assert_eq!(
counter_storage,
Some(U256::from(5)),
"Counter storage slot 0 should be 5 after 5 increments"
);
}
// Verify changesets are queryable before unwind
// This validates that the #21561 fix works - unwind needs to read changesets from the correct
// source
assert_changesets_queryable(&pipeline_provider_factory, 1..=5)?;
// Unwind to block 2
let unwind_target = 2u64;
pipeline.unwind(unwind_target, None)?;
// Verify unwind
{
let provider = pipeline_provider_factory.provider()?;
for stage_id in [
StageId::Headers,
StageId::Bodies,
StageId::SenderRecovery,
StageId::Execution,
StageId::AccountHashing,
StageId::StorageHashing,
StageId::MerkleExecute,
StageId::TransactionLookup,
StageId::IndexAccountHistory,
StageId::IndexStorageHistory,
] {
let checkpoint = provider.get_stage_checkpoint(stage_id)?;
if let Some(cp) = checkpoint {
assert!(
cp.block_number <= unwind_target,
"{stage_id} checkpoint {} should be <= {unwind_target}",
cp.block_number
);
}
}
}
Ok(())
}

View File

@@ -1,4 +1,4 @@
use crate::{find_fixed_range, BlockNumber, Compression};
use crate::{BlockNumber, Compression};
use alloc::{format, string::String, vec::Vec};
use alloy_primitives::TxNumber;
use core::{
@@ -376,20 +376,6 @@ impl SegmentHeader {
self.expected_block_range.start()
}
/// Sets the expected block start of the segment, using the file boundary end
/// from `find_fixed_range`.
///
/// This is useful for non-zero genesis blocks where the actual starting block
/// differs from the file range start determined by `find_fixed_range`.
/// For example, if `blocks_per_file` is 500 and genesis is at 502, the range
/// becomes 502..=999 (start at genesis, end at file boundary).
pub const fn set_expected_block_start(&mut self, block: BlockNumber) {
let blocks_per_file =
self.expected_block_range.end() - self.expected_block_range.start() + 1;
let file_range = find_fixed_range(block, blocks_per_file);
self.expected_block_range = SegmentRangeInclusive::new(block, file_range.end());
}
/// The expected block end of the segment.
pub const fn expected_block_end(&self) -> BlockNumber {
self.expected_block_range.end()

View File

@@ -28,7 +28,7 @@ alloy-genesis.workspace = true
alloy-consensus.workspace = true
# optimism
op-alloy-consensus = { workspace = true, optional = true }
reth-optimism-primitives = { workspace = true, optional = true, features = ["serde", "reth-codec"] }
# codecs
modular-bitfield.workspace = true
@@ -85,11 +85,11 @@ arbitrary = [
"reth-prune-types/arbitrary",
"reth-stages-types/arbitrary",
"alloy-consensus/arbitrary",
"op-alloy-consensus?/arbitrary",
"reth-optimism-primitives?/arbitrary",
"reth-ethereum-primitives/arbitrary",
]
op = [
"dep:op-alloy-consensus",
"dep:reth-optimism-primitives",
"reth-codecs/op",
"reth-primitives-traits/op",
]

View File

@@ -63,9 +63,9 @@ impl StorageSettings {
transaction_senders_in_static_files: true,
account_changesets_in_static_files: true,
storage_changesets_in_static_files: true,
storages_history_in_rocksdb: true,
storages_history_in_rocksdb: false,
transaction_hash_numbers_in_rocksdb: true,
account_history_in_rocksdb: true,
account_history_in_rocksdb: false,
}
}

View File

@@ -240,9 +240,9 @@ impl_compression_for_compact!(
#[cfg(feature = "op")]
mod op {
use super::*;
use op_alloy_consensus::{OpReceipt, OpTxEnvelope};
use reth_optimism_primitives::{OpReceipt, OpTransactionSigned};
impl_compression_for_compact!(OpTxEnvelope, OpReceipt);
impl_compression_for_compact!(OpTransactionSigned, OpReceipt);
}
macro_rules! impl_compression_fixed_compact {

View File

@@ -20,9 +20,9 @@ pub type DupCursorMutTy<TX, T> = <TX as DbTxMut>::DupCursorMut<T>;
/// Read only transaction
pub trait DbTx: Debug + Send {
/// Cursor type for this read-only transaction
type Cursor<T: Table>: DbCursorRO<T> + Send;
type Cursor<T: Table>: DbCursorRO<T> + Send + Sync;
/// `DupCursor` type for this read-only transaction
type DupCursor<T: DupSort>: DbDupCursorRO<T> + DbCursorRO<T> + Send;
type DupCursor<T: DupSort>: DbDupCursorRO<T> + DbCursorRO<T> + Send + Sync;
/// Get value by an owned key
fn get<T: Table>(&self, key: T::Key) -> Result<Option<T::Value>, DatabaseError>;
@@ -51,13 +51,14 @@ pub trait DbTx: Debug + Send {
/// Read write transaction that allows writing to database
pub trait DbTxMut: Send {
/// Read-Write Cursor type
type CursorMut<T: Table>: DbCursorRW<T> + DbCursorRO<T> + Send;
type CursorMut<T: Table>: DbCursorRW<T> + DbCursorRO<T> + Send + Sync;
/// Read-Write `DupCursor` type
type DupCursorMut<T: DupSort>: DbDupCursorRW<T>
+ DbCursorRW<T>
+ DbDupCursorRO<T>
+ DbCursorRO<T>
+ Send;
+ Send
+ Sync;
/// Put value to database
fn put<T: Table>(&self, key: T::Key, value: T::Value) -> Result<(), DatabaseError>;

View File

@@ -6,12 +6,7 @@ use alloy_primitives::{keccak256, map::HashMap, Address, B256, U256};
use reth_chainspec::EthChainSpec;
use reth_codecs::Compact;
use reth_config::config::EtlConfig;
use reth_db_api::{
models::{storage_sharded_key::StorageShardedKey, ShardedKey},
tables,
transaction::DbTxMut,
BlockNumberList, DatabaseError,
};
use reth_db_api::{tables, transaction::DbTxMut, DatabaseError};
use reth_etl::Collector;
use reth_execution_errors::StateRootError;
use reth_primitives_traits::{
@@ -19,11 +14,11 @@ use reth_primitives_traits::{
};
use reth_provider::{
errors::provider::ProviderResult, providers::StaticFileWriter, BlockHashReader, BlockNumReader,
BundleStateInit, ChainSpecProvider, DBProvider, DatabaseProviderFactory, EitherWriter,
ExecutionOutcome, HashingWriter, HeaderProvider, HistoryWriter, MetadataProvider,
MetadataWriter, NodePrimitivesProvider, OriginalValuesKnown, ProviderError, RevertsInit,
RocksDBProviderFactory, StageCheckpointReader, StageCheckpointWriter, StateWriteConfig,
StateWriter, StaticFileProviderFactory, StorageSettings, StorageSettingsCache, TrieWriter,
BundleStateInit, ChainSpecProvider, DBProvider, DatabaseProviderFactory, ExecutionOutcome,
HashingWriter, HeaderProvider, HistoryWriter, MetadataProvider, MetadataWriter,
OriginalValuesKnown, ProviderError, RevertsInit, StageCheckpointReader, StageCheckpointWriter,
StateWriteConfig, StateWriter, StaticFileProviderFactory, StorageSettings,
StorageSettingsCache, TrieWriter,
};
use reth_stages_types::{StageCheckpoint, StageId};
use reth_static_file_types::StaticFileSegment;
@@ -108,9 +103,6 @@ where
+ TrieWriter
+ MetadataWriter
+ ChainSpecProvider
+ StorageSettingsCache
+ RocksDBProviderFactory
+ NodePrimitivesProvider
+ AsRef<PF::ProviderRW>,
PF::ChainSpec: EthChainSpec<Header = <PF::Primitives as NodePrimitives>::BlockHeader>,
{
@@ -146,9 +138,6 @@ where
+ TrieWriter
+ MetadataWriter
+ ChainSpecProvider
+ StorageSettingsCache
+ RocksDBProviderFactory
+ NodePrimitivesProvider
+ AsRef<PF::ProviderRW>,
PF::ChainSpec: EthChainSpec<Header = <PF::Primitives as NodePrimitives>::BlockHeader>,
{
@@ -211,26 +200,6 @@ where
// Behaviour reserved only for new nodes should be set in the storage settings.
provider_rw.write_storage_settings(genesis_storage_settings)?;
// For non-zero genesis blocks, set expected_block_start BEFORE insert_genesis_state.
// When block_range is None, next_block_number() uses expected_block_start. By default,
// expected_block_start comes from find_fixed_range which returns the file range start (0),
// not the genesis block number. This would cause increment_block(N) to fail.
let static_file_provider = provider_rw.static_file_provider();
if genesis_block_number > 0 {
if genesis_storage_settings.account_changesets_in_static_files {
static_file_provider
.get_writer(genesis_block_number, StaticFileSegment::AccountChangeSets)?
.user_header_mut()
.set_expected_block_start(genesis_block_number);
}
if genesis_storage_settings.storage_changesets_in_static_files {
static_file_provider
.get_writer(genesis_block_number, StaticFileSegment::StorageChangeSets)?
.user_header_mut()
.set_expected_block_start(genesis_block_number);
}
}
insert_genesis_hashes(&provider_rw, alloc.iter())?;
insert_genesis_history(&provider_rw, alloc.iter())?;
@@ -248,11 +217,16 @@ where
provider_rw.save_stage_checkpoint(stage, checkpoint)?;
}
// Static file segments start empty, so we need to initialize the block range.
// For genesis blocks with non-zero block numbers, we use get_writer() instead of
// latest_writer() and set_block_range() to ensure static files start at the correct block.
// Static file segments start empty, so we need to initialize the genesis block.
//
// We do not do this for changesets because they get initialized in `insert_state` /
// `write_state` / `write_state_reverts`. If the node is configured for writing changesets to
// static files they will be written there, otherwise they will be written to the DB.
let static_file_provider = provider_rw.static_file_provider();
// Static file segments start empty, so we need to initialize the genesis block.
// For genesis blocks with non-zero block numbers, we need to use get_writer() instead of
// latest_writer() to ensure the genesis block is stored in the correct static file range.
static_file_provider
.get_writer(genesis_block_number, StaticFileSegment::Receipts)?
.user_header_mut()
@@ -412,64 +386,37 @@ where
}
/// Inserts history indices for genesis accounts and storage.
///
/// Writes to either MDBX or `RocksDB` based on storage settings configuration,
/// using [`EitherWriter`] to abstract over the storage backend.
pub fn insert_genesis_history<'a, 'b, Provider>(
provider: &Provider,
alloc: impl Iterator<Item = (&'a Address, &'b GenesisAccount)> + Clone,
) -> ProviderResult<()>
where
Provider: DBProvider<Tx: DbTxMut>
+ HistoryWriter
+ ChainSpecProvider
+ StorageSettingsCache
+ RocksDBProviderFactory
+ NodePrimitivesProvider,
Provider: DBProvider<Tx: DbTxMut> + HistoryWriter + ChainSpecProvider,
{
let genesis_block_number = provider.chain_spec().genesis_header().number();
insert_history(provider, alloc, genesis_block_number)
}
/// Inserts history indices for genesis accounts and storage.
///
/// Writes to either MDBX or `RocksDB` based on storage settings configuration,
/// using [`EitherWriter`] to abstract over the storage backend.
pub fn insert_history<'a, 'b, Provider>(
provider: &Provider,
alloc: impl Iterator<Item = (&'a Address, &'b GenesisAccount)> + Clone,
block: u64,
) -> ProviderResult<()>
where
Provider: DBProvider<Tx: DbTxMut>
+ HistoryWriter
+ StorageSettingsCache
+ RocksDBProviderFactory
+ NodePrimitivesProvider,
Provider: DBProvider<Tx: DbTxMut> + HistoryWriter,
{
provider.with_rocksdb_batch(|batch| {
let mut writer = EitherWriter::new_accounts_history(provider, batch)?;
let list = BlockNumberList::new([block]).expect("single block always fits");
for (addr, _) in alloc.clone() {
writer.upsert_account_history(ShardedKey::last(*addr), &list)?;
}
trace!(target: "reth::cli", "Inserted account history");
Ok(((), writer.into_raw_rocksdb_batch()))
})?;
let account_transitions = alloc.clone().map(|(addr, _)| (*addr, [block]));
provider.insert_account_history_index(account_transitions)?;
provider.with_rocksdb_batch(|batch| {
let mut writer = EitherWriter::new_storages_history(provider, batch)?;
let list = BlockNumberList::new([block]).expect("single block always fits");
for (addr, account) in alloc {
if let Some(storage) = &account.storage {
for key in storage.keys() {
writer.upsert_storage_history(StorageShardedKey::last(*addr, *key), &list)?;
}
}
}
trace!(target: "reth::cli", "Inserted storage history");
Ok(((), writer.into_raw_rocksdb_batch()))
})?;
trace!(target: "reth::cli", "Inserted account history");
let storage_transitions = alloc
.filter_map(|(addr, account)| account.storage.as_ref().map(|storage| (addr, storage)))
.flat_map(|(addr, storage)| storage.keys().map(|key| ((*addr, *key), [block])));
provider.insert_storage_history_index(storage_transitions)?;
trace!(target: "reth::cli", "Inserted storage history");
Ok(())
}
@@ -545,9 +492,6 @@ where
+ HashingWriter
+ TrieWriter
+ StateWriter
+ StorageSettingsCache
+ RocksDBProviderFactory
+ NodePrimitivesProvider
+ AsRef<Provider>,
{
if etl_config.file_size == 0 {
@@ -684,9 +628,6 @@ where
+ HashingWriter
+ HistoryWriter
+ StateWriter
+ StorageSettingsCache
+ RocksDBProviderFactory
+ NodePrimitivesProvider
+ AsRef<Provider>,
{
let accounts_len = collector.len();
@@ -947,59 +888,27 @@ mod tests {
let factory = create_test_provider_factory_with_chain_spec(chain_spec);
init_genesis(&factory).unwrap();
let expected_accounts = vec![
(ShardedKey::new(address_with_balance, u64::MAX), IntegerList::new([0]).unwrap()),
(ShardedKey::new(address_with_storage, u64::MAX), IntegerList::new([0]).unwrap()),
];
let expected_storages = vec![(
StorageShardedKey::new(address_with_storage, storage_key, u64::MAX),
IntegerList::new([0]).unwrap(),
)];
let provider = factory.provider().unwrap();
let collect_from_mdbx = |factory: &ProviderFactory<MockNodeTypesWithDB>| {
let provider = factory.provider().unwrap();
let tx = provider.tx_ref();
(
collect_table_entries::<Arc<DatabaseEnv>, tables::AccountsHistory>(tx).unwrap(),
collect_table_entries::<Arc<DatabaseEnv>, tables::StoragesHistory>(tx).unwrap(),
)
};
let tx = provider.tx_ref();
#[cfg(feature = "edge")]
{
let settings = factory.cached_storage_settings();
let rocksdb = factory.rocksdb_provider();
assert_eq!(
collect_table_entries::<Arc<DatabaseEnv>, tables::AccountsHistory>(tx)
.expect("failed to collect"),
vec![
(ShardedKey::new(address_with_balance, u64::MAX), IntegerList::new([0]).unwrap()),
(ShardedKey::new(address_with_storage, u64::MAX), IntegerList::new([0]).unwrap())
],
);
let collect_rocksdb = |rocksdb: &reth_provider::providers::RocksDBProvider| {
(
rocksdb
.iter::<tables::AccountsHistory>()
.unwrap()
.collect::<Result<Vec<_>, _>>()
.unwrap(),
rocksdb
.iter::<tables::StoragesHistory>()
.unwrap()
.collect::<Result<Vec<_>, _>>()
.unwrap(),
)
};
let (accounts, storages) = if settings.account_history_in_rocksdb {
collect_rocksdb(&rocksdb)
} else {
collect_from_mdbx(&factory)
};
assert_eq!(accounts, expected_accounts);
assert_eq!(storages, expected_storages);
}
#[cfg(not(feature = "edge"))]
{
let (accounts, storages) = collect_from_mdbx(&factory);
assert_eq!(accounts, expected_accounts);
assert_eq!(storages, expected_storages);
}
assert_eq!(
collect_table_entries::<Arc<DatabaseEnv>, tables::StoragesHistory>(tx)
.expect("failed to collect"),
vec![(
StorageShardedKey::new(address_with_storage, storage_key, u64::MAX),
IntegerList::new([0]).unwrap()
)],
);
}
#[test]

View File

@@ -137,8 +137,7 @@ where
for (k, _, v, _) in input {
crsr.append(k, &v).expect("submit");
}
drop(crsr);
tx.commit().unwrap()
tx.inner.commit().unwrap()
},
)
});
@@ -158,8 +157,8 @@ where
let (k, _, v, _) = input.get(index).unwrap().clone();
crsr.insert(k, &v).expect("submit");
}
drop(crsr);
tx.commit().unwrap()
tx.inner.commit().unwrap()
},
)
});
@@ -220,8 +219,7 @@ where
for (k, _, v, _) in input {
crsr.append_dup(k, v).expect("submit");
}
drop(crsr);
tx.commit().unwrap()
tx.inner.commit().unwrap()
},
)
});
@@ -241,7 +239,7 @@ where
let (k, _, v, _) = input.get(index).unwrap().clone();
tx.put::<T>(k, v).unwrap();
}
tx.commit().unwrap()
tx.inner.commit().unwrap();
},
)
});

View File

@@ -16,9 +16,10 @@ use reth_db_api::{
cursor::DbCursorRW,
database::Database,
table::{Table, TableRow},
transaction::{DbTx, DbTxMut},
transaction::DbTxMut,
};
use reth_fs_util as fs;
use std::hint::black_box;
mod utils;
use utils::*;
@@ -177,13 +178,17 @@ fn append<T>(db: DatabaseEnv, input: Vec<(<T as Table>::Key, <T as Table>::Value
where
T: Table,
{
let tx = db.tx_mut().expect("tx");
let mut crsr = tx.cursor_write::<T>().expect("cursor");
for (k, v) in input {
crsr.append(k, &v).expect("submit");
{
let tx = db.tx_mut().expect("tx");
let mut crsr = tx.cursor_write::<T>().expect("cursor");
black_box({
for (k, v) in input {
crsr.append(k, &v).expect("submit");
}
tx.inner.commit().unwrap()
});
}
drop(crsr);
tx.commit().unwrap();
db
}
@@ -191,13 +196,17 @@ fn insert<T>(db: DatabaseEnv, input: Vec<(<T as Table>::Key, <T as Table>::Value
where
T: Table,
{
let tx = db.tx_mut().expect("tx");
let mut crsr = tx.cursor_write::<T>().expect("cursor");
for (k, v) in input {
crsr.insert(k, &v).expect("submit");
{
let tx = db.tx_mut().expect("tx");
let mut crsr = tx.cursor_write::<T>().expect("cursor");
black_box({
for (k, v) in input {
crsr.insert(k, &v).expect("submit");
}
tx.inner.commit().unwrap()
});
}
drop(crsr);
tx.commit().unwrap();
db
}
@@ -205,11 +214,16 @@ fn put<T>(db: DatabaseEnv, input: Vec<(<T as Table>::Key, <T as Table>::Value)>)
where
T: Table,
{
let tx = db.tx_mut().expect("tx");
for (k, v) in input {
tx.put::<T>(k, v).expect("submit");
{
let tx = db.tx_mut().expect("tx");
black_box({
for (k, v) in input {
tx.put::<T>(k, v).expect("submit");
}
tx.inner.commit().unwrap()
});
}
tx.commit().unwrap();
db
}
@@ -229,11 +243,11 @@ where
T: Table,
{
db.view(|tx| {
let table_db = tx.inner().open_db(Some(T::NAME)).map_err(|_| "Could not open db.").unwrap();
let table_db = tx.inner.open_db(Some(T::NAME)).map_err(|_| "Could not open db.").unwrap();
println!(
"{:?}\n",
tx.inner()
tx.inner
.db_stat(table_db.dbi())
.map_err(|_| format!("Could not find table: {}", T::NAME))
.map(|stats| {

View File

@@ -5,7 +5,7 @@ use alloy_primitives::Bytes;
use reth_db::{test_utils::create_test_rw_db_with_path, DatabaseEnv};
use reth_db_api::{
table::{Compress, Encode, Table, TableRow},
transaction::{DbTx, DbTxMut},
transaction::DbTxMut,
Database,
};
use reth_fs_util as fs;
@@ -68,7 +68,7 @@ where
for (k, _, v, _) in pair.clone() {
tx.put::<T>(k, v).expect("submit");
}
tx.commit().unwrap();
tx.inner.commit().unwrap();
}
db.into_inner_db()

View File

@@ -274,11 +274,10 @@ impl DatabaseMetrics for DatabaseEnv {
let _ = self
.view(|tx| {
for table in Tables::ALL.iter().map(Tables::name) {
let table_db =
tx.inner().open_db(Some(table)).wrap_err("Could not open db.")?;
let table_db = tx.inner.open_db(Some(table)).wrap_err("Could not open db.")?;
let stats = tx
.inner()
.inner
.db_stat(table_db.dbi())
.wrap_err(format!("Could not find table: {table}"))?;

View File

@@ -30,7 +30,7 @@ const LONG_TRANSACTION_DURATION: Duration = Duration::from_secs(60);
#[derive(Debug)]
pub struct Tx<K: TransactionKind> {
/// Libmdbx-sys transaction.
inner: Transaction<K>,
pub inner: Transaction<K>,
/// Cached MDBX DBIs for reuse.
dbis: Arc<HashMap<&'static str, MDBX_dbi>>,
@@ -62,11 +62,6 @@ impl<K: TransactionKind> Tx<K> {
Ok(Self { inner, dbis, metrics_handler })
}
/// Returns a reference to the inner libmdbx transaction.
pub const fn inner(&self) -> &Transaction<K> {
&self.inner
}
/// Gets this transaction ID.
pub fn id(&self) -> reth_libmdbx::Result<u64> {
self.metrics_handler.as_ref().map_or_else(|| self.inner.id(), |handler| Ok(handler.txn_id))

View File

@@ -104,14 +104,6 @@ pub enum ProviderError {
/// State is not available for the given block number because it is pruned.
#[error("state at block #{_0} is pruned")]
StateAtBlockPruned(BlockNumber),
/// State is not available because the block has not been executed yet.
#[error("state at block #{requested} is not available, block has not been executed yet (latest executed: #{executed})")]
BlockNotExecuted {
/// The block number that was requested.
requested: BlockNumber,
/// The latest executed block number.
executed: BlockNumber,
},
/// Block data is not available because history has expired.
///
/// The requested block number is below the earliest available block.

View File

@@ -728,7 +728,7 @@ impl<N: ProviderNodeTypes> StorageChangeSetReader for BlockchainProvider<N> {
fn storage_changesets_range(
&self,
range: impl RangeBounds<BlockNumber>,
range: RangeInclusive<BlockNumber>,
) -> ProviderResult<Vec<(BlockNumberAddress, StorageEntry)>> {
self.consistent_provider()?.storage_changesets_range(range)
}

View File

@@ -1397,7 +1397,7 @@ impl<N: ProviderNodeTypes> StorageChangeSetReader for ConsistentProvider<N> {
fn storage_changesets_range(
&self,
range: impl RangeBounds<BlockNumber>,
range: RangeInclusive<BlockNumber>,
) -> ProviderResult<Vec<(BlockNumberAddress, StorageEntry)>> {
let range = to_range(range);
let mut changesets = Vec::new();

Some files were not shown because too many files have changed in this diff Show More