mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
140 Commits
fix/persis
...
feat/bal-c
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6cbcfe01a0 | ||
|
|
830cd5e355 | ||
|
|
f77d7d5983 | ||
|
|
a2237c534e | ||
|
|
1bd8fab887 | ||
|
|
22a68756c7 | ||
|
|
d99c0ffd62 | ||
|
|
ad476e2b5c | ||
|
|
6df249c1f1 | ||
|
|
5a076df09a | ||
|
|
f07629eac0 | ||
|
|
f643e93c35 | ||
|
|
653362a436 | ||
|
|
a02508600c | ||
|
|
937a7f226d | ||
|
|
a0df561117 | ||
|
|
be5a4ac7a6 | ||
|
|
0c854b6f14 | ||
|
|
28a31cd579 | ||
|
|
da12451c9c | ||
|
|
247ce3c4e9 | ||
|
|
bf43ebaa29 | ||
|
|
a01ecce73f | ||
|
|
3e55c6ca6e | ||
|
|
2ac7d719f3 | ||
|
|
965705ff88 | ||
|
|
ebe2ca1366 | ||
|
|
cc242f83fd | ||
|
|
12cf3d685b | ||
|
|
ad5b533ad1 | ||
|
|
118f15f345 | ||
|
|
97481f69e5 | ||
|
|
f692ac7d1e | ||
|
|
4b1c341ced | ||
|
|
865f8f8951 | ||
|
|
492fc20fd1 | ||
|
|
ad9886abb8 | ||
|
|
5c3e45cd6b | ||
|
|
68fdba32d2 | ||
|
|
8f6a0a2992 | ||
|
|
ec9c7f8d3e | ||
|
|
dbdaf068f0 | ||
|
|
055bf63ee9 | ||
|
|
2305c3ebeb | ||
|
|
eb55c3c3da | ||
|
|
72e1467ba3 | ||
|
|
74edce0089 | ||
|
|
8c645d5762 | ||
|
|
b7d2ee2566 | ||
|
|
7609deddda | ||
|
|
ec50fd40b3 | ||
|
|
624ddc5779 | ||
|
|
dd72cfe23e | ||
|
|
ff8ac97e33 | ||
|
|
0974485863 | ||
|
|
274394e777 | ||
|
|
1954c91a60 | ||
|
|
9cf82c8403 | ||
|
|
f85fcba872 | ||
|
|
ebaa4bda3a | ||
|
|
04d4c9a02f | ||
|
|
3065a328f9 | ||
|
|
a0aac13f75 | ||
|
|
9f5cf847cc | ||
|
|
43a84f1231 | ||
|
|
df1413167a | ||
|
|
3f50a36191 | ||
|
|
d750b4976d | ||
|
|
7a65d2595d | ||
|
|
5a5c21cc1b | ||
|
|
8a8a9126d6 | ||
|
|
6f73c2447d | ||
|
|
2cae438642 | ||
|
|
37b5db0d47 | ||
|
|
238433e146 | ||
|
|
660964a0f5 | ||
|
|
22b465dd64 | ||
|
|
3ff575b877 | ||
|
|
d12752dc8a | ||
|
|
869b5d0851 | ||
|
|
78de3d8f61 | ||
|
|
bc79cc44c9 | ||
|
|
ff8f434dcd | ||
|
|
9662dc5271 | ||
|
|
3ba37082dc | ||
|
|
7934294988 | ||
|
|
7371bd3f29 | ||
|
|
80980b8e4d | ||
|
|
2e2cd67663 | ||
|
|
4f009728e2 | ||
|
|
39d5ae73e8 | ||
|
|
5ef200eaad | ||
|
|
d002dacc13 | ||
|
|
bb39cba504 | ||
|
|
bd144a4c42 | ||
|
|
a0845bab18 | ||
|
|
346cc0da71 | ||
|
|
ea3d4663ae | ||
|
|
3667d3b5aa | ||
|
|
7cfb19c98e | ||
|
|
5a38871489 | ||
|
|
c825c8c187 | ||
|
|
8f37cd08fc | ||
|
|
c9dad4765d | ||
|
|
1d55abeef3 | ||
|
|
f7460e219c | ||
|
|
0c66315f20 | ||
|
|
6a2010e595 | ||
|
|
c2435ff6f8 | ||
|
|
52ec8e9491 | ||
|
|
a901d80ee6 | ||
|
|
915164078f | ||
|
|
be3234d848 | ||
|
|
f624372334 | ||
|
|
40bc9d3860 | ||
|
|
1ea574417f | ||
|
|
27e055f790 | ||
|
|
d5dc0b27eb | ||
|
|
c11c13000f | ||
|
|
6bf43ab24a | ||
|
|
574bde0d6f | ||
|
|
79b8ffb828 | ||
|
|
c617d25c36 | ||
|
|
b96a30821f | ||
|
|
012fbf5110 | ||
|
|
d7a5d1f872 | ||
|
|
3a39251f79 | ||
|
|
f6dbf2d82d | ||
|
|
13707faf1a | ||
|
|
6e6415690c | ||
|
|
b81e373d78 | ||
|
|
a164654145 | ||
|
|
905bb95f8b | ||
|
|
13c32625bc | ||
|
|
1be9fab5bf | ||
|
|
80eb0d0fb6 | ||
|
|
5e178f6ac6 | ||
|
|
b4b64096c8 | ||
|
|
e313de818b | ||
|
|
86c414081a |
43
.github/CODEOWNERS
vendored
43
.github/CODEOWNERS
vendored
@@ -1,45 +1,52 @@
|
||||
* @gakonst
|
||||
crates/blockchain-tree-api/ @rakita @mattsse @Rjected
|
||||
crates/blockchain-tree/ @rakita @mattsse @Rjected
|
||||
crates/chain-state/ @fgimenez @mattsse
|
||||
crates/chainspec/ @Rjected @joshieDo @mattsse
|
||||
crates/cli/ @mattsse
|
||||
crates/config/ @shekhirin @mattsse @Rjected
|
||||
crates/consensus/ @mattsse @Rjected
|
||||
crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez
|
||||
crates/engine/ @mattsse @Rjected @fgimenez @mediocregopher @yongkangc
|
||||
crates/era/ @mattsse @RomanHodulak
|
||||
crates/engine/ @mattsse @Rjected @mediocregopher @yongkangc
|
||||
crates/era/ @mattsse
|
||||
crates/era-downloader/ @mattsse
|
||||
crates/era-utils/ @mattsse
|
||||
crates/errors/ @mattsse
|
||||
crates/ethereum-forks/ @mattsse @Rjected
|
||||
crates/ethereum/ @mattsse @Rjected
|
||||
crates/etl/ @joshieDo @shekhirin
|
||||
crates/evm/ @rakita @mattsse @Rjected
|
||||
crates/evm/ @mattsse @Rjected @klkvr
|
||||
crates/exex/ @shekhirin
|
||||
crates/fs-util/ @mattsse
|
||||
crates/metrics/ @mattsse @Rjected
|
||||
crates/net/ @mattsse @Rjected
|
||||
crates/net/downloaders/ @Rjected
|
||||
crates/node/ @mattsse @Rjected @klkvr
|
||||
crates/optimism/ @mattsse @Rjected @fgimenez
|
||||
crates/optimism/ @mattsse @Rjected
|
||||
crates/payload/ @mattsse @Rjected
|
||||
crates/primitives-traits/ @Rjected @RomanHodulak @mattsse @klkvr
|
||||
crates/primitives-traits/ @Rjected @mattsse @klkvr
|
||||
crates/primitives/ @Rjected @mattsse @klkvr
|
||||
crates/prune/ @shekhirin @joshieDo
|
||||
crates/ress @shekhirin @Rjected
|
||||
crates/revm/ @mattsse @rakita
|
||||
crates/rpc/ @mattsse @Rjected @RomanHodulak
|
||||
crates/ress/ @shekhirin @Rjected
|
||||
crates/revm/ @mattsse
|
||||
crates/rpc/ @mattsse @Rjected
|
||||
crates/stages/ @shekhirin @mediocregopher
|
||||
crates/static-file/ @joshieDo @shekhirin
|
||||
crates/stateless/ @mattsse
|
||||
crates/storage/codecs/ @joshieDo
|
||||
crates/storage/db-api/ @joshieDo @rakita
|
||||
crates/storage/db-api/ @joshieDo
|
||||
crates/storage/db-common/ @Rjected
|
||||
crates/storage/db/ @joshieDo @rakita
|
||||
crates/storage/errors/ @rakita
|
||||
crates/storage/libmdbx-rs/ @rakita @shekhirin
|
||||
crates/storage/db/ @joshieDo
|
||||
crates/storage/errors/ @joshieDo
|
||||
crates/storage/libmdbx-rs/ @shekhirin
|
||||
crates/storage/nippy-jar/ @joshieDo @shekhirin
|
||||
crates/storage/provider/ @rakita @joshieDo @shekhirin
|
||||
crates/storage/provider/ @joshieDo @shekhirin @yongkangc
|
||||
crates/storage/storage-api/ @joshieDo
|
||||
crates/tasks/ @mattsse
|
||||
crates/tokio-util/ @fgimenez
|
||||
crates/tokio-util/ @mattsse
|
||||
crates/tracing/ @mattsse @shekhirin
|
||||
crates/tracing-otlp/ @mattsse @Rjected
|
||||
crates/transaction-pool/ @mattsse @yongkangc
|
||||
crates/trie/ @Rjected @shekhirin @mediocregopher
|
||||
crates/trie/ @Rjected @shekhirin @mediocregopher @yongkangc
|
||||
bin/reth/ @mattsse @shekhirin @Rjected
|
||||
bin/reth-bench/ @mattsse @Rjected @shekhirin @yongkangc
|
||||
bin/reth-bench-compare/ @mediocregopher @shekhirin @yongkangc
|
||||
etc/ @Rjected @shekhirin
|
||||
.github/ @gakonst @DaniPopes
|
||||
|
||||
14
.github/scripts/codspeed-build.sh
vendored
14
.github/scripts/codspeed-build.sh
vendored
@@ -1,14 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eo pipefail
|
||||
|
||||
# TODO: Benchmarks run WAY too slow due to excessive amount of iterations.
|
||||
|
||||
cmd=(cargo codspeed build --profile profiling)
|
||||
crates=(
|
||||
-p reth-primitives
|
||||
-p reth-trie
|
||||
-p reth-trie-common
|
||||
-p reth-trie-sparse
|
||||
)
|
||||
|
||||
"${cmd[@]}" --features test-utils "${crates[@]}"
|
||||
14
.github/workflows/bench.yml
vendored
14
.github/workflows/bench.yml
vendored
@@ -17,6 +17,16 @@ name: bench
|
||||
jobs:
|
||||
codspeed:
|
||||
runs-on: depot-ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
partition: [1, 2]
|
||||
total_partitions: [2]
|
||||
include:
|
||||
- partition: 1
|
||||
crates: "-p reth-primitives -p reth-trie-common -p reth-trie-sparse"
|
||||
- partition: 2
|
||||
crates: "-p reth-trie"
|
||||
name: codspeed (${{ matrix.partition }}/${{ matrix.total_partitions }})
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
@@ -32,10 +42,10 @@ jobs:
|
||||
with:
|
||||
tool: cargo-codspeed
|
||||
- name: Build the benchmark target(s)
|
||||
run: ./.github/scripts/codspeed-build.sh
|
||||
run: cargo codspeed build --profile profiling --features test-utils ${{ matrix.crates }}
|
||||
- name: Run the benchmarks
|
||||
uses: CodSpeedHQ/action@v4
|
||||
with:
|
||||
run: cargo codspeed run --workspace
|
||||
run: cargo codspeed run ${{ matrix.crates }}
|
||||
mode: instrumentation
|
||||
token: ${{ secrets.CODSPEED_TOKEN }}
|
||||
|
||||
65
.github/workflows/check-alloy.yml
vendored
Normal file
65
.github/workflows/check-alloy.yml
vendored
Normal file
@@ -0,0 +1,65 @@
|
||||
# Checks reth compilation against alloy branches to detect breaking changes.
|
||||
# Run on-demand via workflow_dispatch.
|
||||
|
||||
name: Check Alloy Breaking Changes
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
alloy_branch:
|
||||
description: 'Branch/rev for alloy-rs/alloy (leave empty to skip)'
|
||||
required: false
|
||||
type: string
|
||||
alloy_evm_branch:
|
||||
description: 'Branch/rev for alloy-rs/evm (alloy-evm, alloy-op-evm) (leave empty to skip)'
|
||||
required: false
|
||||
type: string
|
||||
op_alloy_branch:
|
||||
description: 'Branch/rev for alloy-rs/op-alloy (leave empty to skip)'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Check compilation with patched alloy
|
||||
runs-on: depot-ubuntu-latest-16
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
|
||||
- name: Apply alloy patches
|
||||
run: |
|
||||
ARGS=""
|
||||
if [ -n "${{ inputs.alloy_branch }}" ]; then
|
||||
ARGS="$ARGS --alloy ${{ inputs.alloy_branch }}"
|
||||
fi
|
||||
if [ -n "${{ inputs.alloy_evm_branch }}" ]; then
|
||||
ARGS="$ARGS --evm ${{ inputs.alloy_evm_branch }}"
|
||||
fi
|
||||
if [ -n "${{ inputs.op_alloy_branch }}" ]; then
|
||||
ARGS="$ARGS --op ${{ inputs.op_alloy_branch }}"
|
||||
fi
|
||||
|
||||
if [ -z "$ARGS" ]; then
|
||||
echo "No branches specified, nothing to patch"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
./scripts/patch-alloy.sh $ARGS
|
||||
|
||||
echo "=== Final patch section ==="
|
||||
tail -50 Cargo.toml
|
||||
|
||||
- name: Check workspace
|
||||
run: cargo clippy --workspace --lib --examples --tests --benches --all-features --locked
|
||||
env:
|
||||
RUSTFLAGS: -D warnings
|
||||
2
.github/workflows/dependencies.yml
vendored
2
.github/workflows/dependencies.yml
vendored
@@ -15,6 +15,6 @@ permissions:
|
||||
|
||||
jobs:
|
||||
update:
|
||||
uses: ithacaxyz/ci/.github/workflows/cargo-update-pr.yml@main
|
||||
uses: tempoxyz/ci/.github/workflows/cargo-update-pr.yml@main
|
||||
secrets:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
2
.github/workflows/lint.yml
vendored
2
.github/workflows/lint.yml
vendored
@@ -285,7 +285,7 @@ jobs:
|
||||
- run: zepter run check
|
||||
|
||||
deny:
|
||||
uses: ithacaxyz/ci/.github/workflows/deny.yml@main
|
||||
uses: tempoxyz/ci/.github/workflows/deny.yml@main
|
||||
|
||||
lint-success:
|
||||
name: lint success
|
||||
|
||||
@@ -249,7 +249,7 @@ Write comments that remain valuable after the PR is merged. Future readers won't
|
||||
unsafe impl GlobalAlloc for LimitedAllocator { ... }
|
||||
|
||||
// Binary search requires sorted input. Panics on unsorted slices.
|
||||
fn find_index(items: &[Item], target: &Item) -> Option
|
||||
fn find_index(items: &[Item], target: &Item) -> Option<usize>
|
||||
|
||||
// Timeout set to 5s to match EVM block processing limits
|
||||
const TRACER_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
835
Cargo.lock
generated
835
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
97
Cargo.toml
97
Cargo.toml
@@ -1,5 +1,5 @@
|
||||
[workspace.package]
|
||||
version = "1.10.0"
|
||||
version = "1.10.2"
|
||||
edition = "2024"
|
||||
rust-version = "1.88"
|
||||
license = "MIT OR Apache-2.0"
|
||||
@@ -473,22 +473,22 @@ reth-ress-protocol = { path = "crates/ress/protocol" }
|
||||
reth-ress-provider = { path = "crates/ress/provider" }
|
||||
|
||||
# revm
|
||||
revm = { version = "33.1.0", default-features = false }
|
||||
revm-bytecode = { version = "7.1.1", default-features = false }
|
||||
revm-database = { version = "9.0.5", default-features = false }
|
||||
revm-state = { version = "8.1.1", default-features = false }
|
||||
revm-primitives = { version = "21.0.2", default-features = false }
|
||||
revm-interpreter = { version = "31.1.0", default-features = false }
|
||||
revm-database-interface = { version = "8.0.5", default-features = false }
|
||||
op-revm = { version = "14.1.0", default-features = false }
|
||||
revm-inspectors = "0.33.2"
|
||||
revm = { version = "34.0.0", default-features = false }
|
||||
revm-bytecode = { version = "8.0.0", default-features = false }
|
||||
revm-database = { version = "10.0.0", default-features = false }
|
||||
revm-state = { version = "9.0.0", default-features = false }
|
||||
revm-primitives = { version = "22.0.0", default-features = false }
|
||||
revm-interpreter = { version = "32.0.0", default-features = false }
|
||||
revm-database-interface = { version = "9.0.0", default-features = false }
|
||||
op-revm = { version = "15.0.0", default-features = false }
|
||||
revm-inspectors = "0.34.0"
|
||||
|
||||
# eth
|
||||
alloy-chains = { version = "0.2.5", default-features = false }
|
||||
alloy-dyn-abi = "1.4.3"
|
||||
alloy-dyn-abi = "1.5.2"
|
||||
alloy-eip2124 = { version = "0.2.0", default-features = false }
|
||||
alloy-eip7928 = { version = "0.1.0", default-features = false }
|
||||
alloy-evm = { version = "0.25.1", default-features = false }
|
||||
alloy-eip7928 = { version = "0.3.0", default-features = false }
|
||||
alloy-evm = { version = "0.27.0", default-features = false }
|
||||
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
|
||||
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
|
||||
alloy-sol-macro = "1.5.0"
|
||||
@@ -497,36 +497,36 @@ alloy-trie = { version = "0.9.1", default-features = false }
|
||||
|
||||
alloy-hardforks = "0.4.5"
|
||||
|
||||
alloy-consensus = { version = "1.4.3", default-features = false }
|
||||
alloy-contract = { version = "1.4.3", default-features = false }
|
||||
alloy-eips = { version = "1.4.3", default-features = false }
|
||||
alloy-genesis = { version = "1.4.3", default-features = false }
|
||||
alloy-json-rpc = { version = "1.4.3", default-features = false }
|
||||
alloy-network = { version = "1.4.3", default-features = false }
|
||||
alloy-network-primitives = { version = "1.4.3", default-features = false }
|
||||
alloy-provider = { version = "1.4.3", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-client = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types = { version = "1.4.3", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "1.4.3", default-features = false }
|
||||
alloy-serde = { version = "1.4.3", default-features = false }
|
||||
alloy-signer = { version = "1.4.3", default-features = false }
|
||||
alloy-signer-local = { version = "1.4.3", default-features = false }
|
||||
alloy-transport = { version = "1.4.3" }
|
||||
alloy-transport-http = { version = "1.4.3", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "1.4.3", default-features = false }
|
||||
alloy-transport-ws = { version = "1.4.3", default-features = false }
|
||||
alloy-consensus = { version = "1.5.2", default-features = false }
|
||||
alloy-contract = { version = "1.5.2", default-features = false }
|
||||
alloy-eips = { version = "1.5.2", default-features = false }
|
||||
alloy-genesis = { version = "1.5.2", default-features = false }
|
||||
alloy-json-rpc = { version = "1.5.2", default-features = false }
|
||||
alloy-network = { version = "1.5.2", default-features = false }
|
||||
alloy-network-primitives = { version = "1.5.2", default-features = false }
|
||||
alloy-provider = { version = "1.5.2", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-client = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types = { version = "1.5.2", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "1.5.2", default-features = false }
|
||||
alloy-serde = { version = "1.5.2", default-features = false }
|
||||
alloy-signer = { version = "1.5.2", default-features = false }
|
||||
alloy-signer-local = { version = "1.5.2", default-features = false }
|
||||
alloy-transport = { version = "1.5.2" }
|
||||
alloy-transport-http = { version = "1.5.2", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "1.5.2", default-features = false }
|
||||
alloy-transport-ws = { version = "1.5.2", default-features = false }
|
||||
|
||||
# op
|
||||
alloy-op-evm = { version = "0.25.0", default-features = false }
|
||||
alloy-op-evm = { version = "0.27.0", default-features = false }
|
||||
alloy-op-hardforks = "0.4.4"
|
||||
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
|
||||
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
|
||||
@@ -739,15 +739,15 @@ tracing-subscriber = { version = "0.3", default-features = false }
|
||||
tracing-tracy = "0.11"
|
||||
triehash = "0.8"
|
||||
typenum = "1.15.0"
|
||||
vergen = "9.0.4"
|
||||
vergen = "9.1.0"
|
||||
visibility = "0.1.1"
|
||||
walkdir = "2.3.3"
|
||||
vergen-git2 = "1.0.5"
|
||||
vergen-git2 = "9.1.0"
|
||||
|
||||
# networking
|
||||
ipnet = "2.11"
|
||||
|
||||
# [patch.crates-io]
|
||||
[patch.crates-io]
|
||||
# alloy-consensus = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-contract = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-eips = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
@@ -790,5 +790,10 @@ ipnet = "2.11"
|
||||
# jsonrpsee-http-client = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
|
||||
# jsonrpsee-types = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
|
||||
|
||||
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
|
||||
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
|
||||
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
|
||||
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
|
||||
|
||||
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "3020ea8" }
|
||||
|
||||
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
|
||||
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
|
||||
|
||||
@@ -163,6 +163,7 @@ impl NodeManager {
|
||||
"eth,reth".to_string(),
|
||||
"--disable-discovery".to_string(),
|
||||
"--trusted-only".to_string(),
|
||||
"--disable-tx-gossip".to_string(),
|
||||
]);
|
||||
|
||||
// Add tracing arguments if OTLP endpoint is configured
|
||||
|
||||
@@ -32,7 +32,7 @@ alloy-eips.workspace = true
|
||||
alloy-json-rpc.workspace = true
|
||||
alloy-consensus.workspace = true
|
||||
alloy-network.workspace = true
|
||||
alloy-primitives.workspace = true
|
||||
alloy-primitives = { workspace = true, features = ["rand"] }
|
||||
alloy-provider = { workspace = true, features = ["engine-api", "pubsub", "reqwest-rustls-tls"], default-features = false }
|
||||
alloy-pubsub.workspace = true
|
||||
alloy-rpc-client = { workspace = true, features = ["pubsub"] }
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use crate::{
|
||||
authenticated_transport::AuthenticatedTransportConnect,
|
||||
bench::{
|
||||
helpers::{build_payload, prepare_payload_request, rpc_block_to_header},
|
||||
helpers::{build_payload, parse_gas_limit, prepare_payload_request, rpc_block_to_header},
|
||||
output::GasRampPayloadFile,
|
||||
},
|
||||
valid_payload::{call_forkchoice_updated, call_new_payload, payload_to_new_payload},
|
||||
@@ -25,9 +25,16 @@ use tracing::info;
|
||||
/// `reth benchmark gas-limit-ramp` command.
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Command {
|
||||
/// Number of blocks to generate.
|
||||
#[arg(long, value_name = "BLOCKS")]
|
||||
blocks: u64,
|
||||
/// Number of blocks to generate. Mutually exclusive with --target-gas-limit.
|
||||
#[arg(long, value_name = "BLOCKS", conflicts_with = "target_gas_limit")]
|
||||
blocks: Option<u64>,
|
||||
|
||||
/// Target gas limit to ramp up to. The benchmark will generate blocks until the gas limit
|
||||
/// reaches or exceeds this value. Mutually exclusive with --blocks.
|
||||
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 2G = 2
|
||||
/// billion).
|
||||
#[arg(long, value_name = "TARGET_GAS_LIMIT", conflicts_with = "blocks", value_parser = parse_gas_limit)]
|
||||
target_gas_limit: Option<u64>,
|
||||
|
||||
/// The Engine API RPC URL.
|
||||
#[arg(long = "engine-rpc-url", value_name = "ENGINE_RPC_URL")]
|
||||
@@ -42,12 +49,37 @@ pub struct Command {
|
||||
output: PathBuf,
|
||||
}
|
||||
|
||||
/// Mode for determining when to stop ramping.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum RampMode {
|
||||
/// Ramp for a fixed number of blocks.
|
||||
Blocks(u64),
|
||||
/// Ramp until reaching or exceeding target gas limit.
|
||||
TargetGasLimit(u64),
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `benchmark gas-limit-ramp` command.
|
||||
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
|
||||
if self.blocks == 0 {
|
||||
return Err(eyre::eyre!("--blocks must be greater than 0"));
|
||||
}
|
||||
let mode = match (self.blocks, self.target_gas_limit) {
|
||||
(Some(blocks), None) => {
|
||||
if blocks == 0 {
|
||||
return Err(eyre::eyre!("--blocks must be greater than 0"));
|
||||
}
|
||||
RampMode::Blocks(blocks)
|
||||
}
|
||||
(None, Some(target)) => {
|
||||
if target == 0 {
|
||||
return Err(eyre::eyre!("--target-gas-limit must be greater than 0"));
|
||||
}
|
||||
RampMode::TargetGasLimit(target)
|
||||
}
|
||||
_ => {
|
||||
return Err(eyre::eyre!(
|
||||
"Exactly one of --blocks or --target-gas-limit must be specified"
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
// Ensure output directory exists
|
||||
if self.output.is_file() {
|
||||
@@ -84,14 +116,31 @@ impl Command {
|
||||
|
||||
let canonical_parent = parent_header.number;
|
||||
let start_block = canonical_parent + 1;
|
||||
let end_block = start_block + self.blocks - 1;
|
||||
|
||||
info!(canonical_parent, start_block, end_block, "Starting gas limit ramp benchmark");
|
||||
match mode {
|
||||
RampMode::Blocks(blocks) => {
|
||||
info!(
|
||||
canonical_parent,
|
||||
start_block,
|
||||
end_block = start_block + blocks - 1,
|
||||
"Starting gas limit ramp benchmark (block count mode)"
|
||||
);
|
||||
}
|
||||
RampMode::TargetGasLimit(target) => {
|
||||
info!(
|
||||
canonical_parent,
|
||||
start_block,
|
||||
current_gas_limit = parent_header.gas_limit,
|
||||
target_gas_limit = target,
|
||||
"Starting gas limit ramp benchmark (target gas limit mode)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut next_block_number = start_block;
|
||||
let mut blocks_processed = 0u64;
|
||||
let total_benchmark_duration = Instant::now();
|
||||
|
||||
while next_block_number <= end_block {
|
||||
while !should_stop(mode, blocks_processed, parent_header.gas_limit) {
|
||||
let timestamp = parent_header.timestamp.saturating_add(1);
|
||||
|
||||
let request = prepare_payload_request(&chain_spec, timestamp, parent_hash);
|
||||
@@ -140,13 +189,13 @@ impl Command {
|
||||
|
||||
parent_header = block.header;
|
||||
parent_hash = block_hash;
|
||||
next_block_number += 1;
|
||||
blocks_processed += 1;
|
||||
}
|
||||
|
||||
let final_gas_limit = parent_header.gas_limit;
|
||||
info!(
|
||||
total_duration=?total_benchmark_duration.elapsed(),
|
||||
blocks_processed = self.blocks,
|
||||
blocks_processed,
|
||||
final_gas_limit,
|
||||
"Benchmark complete"
|
||||
);
|
||||
@@ -158,3 +207,10 @@ impl Command {
|
||||
const fn max_gas_limit_increase(parent_gas_limit: u64) -> u64 {
|
||||
(parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR).saturating_sub(1)
|
||||
}
|
||||
|
||||
const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u64) -> bool {
|
||||
match mode {
|
||||
RampMode::Blocks(target_blocks) => blocks_processed >= target_blocks,
|
||||
RampMode::TargetGasLimit(target) => current_gas_limit >= target,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
//! This command fetches transactions from existing blocks and packs them into a single
|
||||
//! large block using the `testing_buildBlockV1` RPC endpoint.
|
||||
|
||||
use crate::authenticated_transport::AuthenticatedTransportConnect;
|
||||
use crate::{
|
||||
authenticated_transport::AuthenticatedTransportConnect, bench::helpers::parse_gas_limit,
|
||||
};
|
||||
use alloy_eips::{BlockNumberOrTag, Typed2718};
|
||||
use alloy_primitives::{Bytes, B256};
|
||||
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
|
||||
@@ -202,7 +204,9 @@ pub struct Command {
|
||||
jwt_secret: std::path::PathBuf,
|
||||
|
||||
/// Target gas to pack into the block.
|
||||
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000")]
|
||||
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 1G = 1
|
||||
/// billion).
|
||||
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000", value_parser = parse_gas_limit)]
|
||||
target_gas: u64,
|
||||
|
||||
/// Starting block number to fetch transactions from.
|
||||
|
||||
@@ -1,6 +1,29 @@
|
||||
//! Common helpers for reth-bench commands.
|
||||
|
||||
use crate::valid_payload::call_forkchoice_updated;
|
||||
|
||||
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
|
||||
///
|
||||
/// Examples: "30000000", "30M", "1G", "2G"
|
||||
pub(crate) fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
|
||||
let s = s.trim();
|
||||
if s.is_empty() {
|
||||
return Err(eyre::eyre!("empty value"));
|
||||
}
|
||||
|
||||
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
|
||||
(prefix, 1_000_000_000u64)
|
||||
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
|
||||
(prefix, 1_000_000u64)
|
||||
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
|
||||
(prefix, 1_000u64)
|
||||
} else {
|
||||
(s, 1u64)
|
||||
};
|
||||
|
||||
let base: u64 = num_str.trim().parse()?;
|
||||
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
|
||||
}
|
||||
use alloy_consensus::Header;
|
||||
use alloy_eips::eip4844::kzg_to_versioned_hash;
|
||||
use alloy_primitives::{Address, B256};
|
||||
@@ -194,3 +217,50 @@ pub(crate) async fn get_payload_with_sidecar(
|
||||
_ => panic!("This tool does not support getPayload versions past v5"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_plain_number() {
|
||||
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
|
||||
assert_eq!(parse_gas_limit("1").unwrap(), 1);
|
||||
assert_eq!(parse_gas_limit("0").unwrap(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_k_suffix() {
|
||||
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
|
||||
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
|
||||
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_m_suffix() {
|
||||
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
|
||||
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
|
||||
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_g_suffix() {
|
||||
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
|
||||
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
|
||||
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_with_whitespace() {
|
||||
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
|
||||
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_errors() {
|
||||
assert!(parse_gas_limit("").is_err());
|
||||
assert!(parse_gas_limit("abc").is_err());
|
||||
assert!(parse_gas_limit("G").is_err());
|
||||
assert!(parse_gas_limit("-1G").is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ mod new_payload_fcu;
|
||||
mod new_payload_only;
|
||||
mod output;
|
||||
mod replay_payloads;
|
||||
mod send_invalid_payload;
|
||||
mod send_payload;
|
||||
|
||||
/// `reth bench` command
|
||||
@@ -74,6 +75,18 @@ pub enum Subcommands {
|
||||
/// `reth-bench replay-payloads --payload-dir ./payloads --engine-rpc-url
|
||||
/// http://localhost:8551 --jwt-secret ~/.local/share/reth/mainnet/jwt.hex`
|
||||
ReplayPayloads(replay_payloads::Command),
|
||||
|
||||
/// Generate and send an invalid `engine_newPayload` request for testing.
|
||||
///
|
||||
/// Takes a valid block and modifies fields to make it invalid, allowing you to test
|
||||
/// Engine API rejection behavior. Block hash is recalculated after modifications
|
||||
/// unless `--invalid-block-hash` or `--skip-hash-recalc` is used.
|
||||
///
|
||||
/// Example:
|
||||
///
|
||||
/// `cast block latest --full --json | reth-bench send-invalid-payload --rpc-url localhost:5000
|
||||
/// --jwt-secret $(cat ~/.local/share/reth/mainnet/jwt.hex) --invalid-state-root`
|
||||
SendInvalidPayload(Box<send_invalid_payload::Command>),
|
||||
}
|
||||
|
||||
impl BenchmarkCommand {
|
||||
@@ -89,6 +102,7 @@ impl BenchmarkCommand {
|
||||
Subcommands::SendPayload(command) => command.execute(ctx).await,
|
||||
Subcommands::GenerateBigBlock(command) => command.execute(ctx).await,
|
||||
Subcommands::ReplayPayloads(command) => command.execute(ctx).await,
|
||||
Subcommands::SendInvalidPayload(command) => (*command).execute(ctx).await,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -180,7 +180,7 @@ impl Command {
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| {
|
||||
e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
|
||||
e.file_name().to_string_lossy().starts_with("payload_")
|
||||
e.file_name().to_string_lossy().starts_with("payload_block_")
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -191,7 +191,7 @@ impl Command {
|
||||
let name = e.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
// Extract index from "payload_NNN.json"
|
||||
let index_str = name_str.strip_prefix("payload_")?.strip_suffix(".json")?;
|
||||
let index_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
|
||||
let index: u64 = index_str.parse().ok()?;
|
||||
Some((index, e.path()))
|
||||
})
|
||||
|
||||
219
bin/reth-bench/src/bench/send_invalid_payload/invalidation.rs
Normal file
219
bin/reth-bench/src/bench/send_invalid_payload/invalidation.rs
Normal file
@@ -0,0 +1,219 @@
|
||||
use alloy_eips::eip4895::Withdrawal;
|
||||
use alloy_primitives::{Address, Bloom, Bytes, B256, U256};
|
||||
use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3};
|
||||
|
||||
/// Configuration for invalidating payload fields
|
||||
#[derive(Debug, Default)]
|
||||
pub(super) struct InvalidationConfig {
|
||||
// Explicit value overrides (Option<T>)
|
||||
pub(super) parent_hash: Option<B256>,
|
||||
pub(super) fee_recipient: Option<Address>,
|
||||
pub(super) state_root: Option<B256>,
|
||||
pub(super) receipts_root: Option<B256>,
|
||||
pub(super) logs_bloom: Option<Bloom>,
|
||||
pub(super) prev_randao: Option<B256>,
|
||||
pub(super) block_number: Option<u64>,
|
||||
pub(super) gas_limit: Option<u64>,
|
||||
pub(super) gas_used: Option<u64>,
|
||||
pub(super) timestamp: Option<u64>,
|
||||
pub(super) extra_data: Option<Bytes>,
|
||||
pub(super) base_fee_per_gas: Option<u64>,
|
||||
pub(super) block_hash: Option<B256>,
|
||||
pub(super) blob_gas_used: Option<u64>,
|
||||
pub(super) excess_blob_gas: Option<u64>,
|
||||
|
||||
// Auto-invalidation flags
|
||||
pub(super) invalidate_parent_hash: bool,
|
||||
pub(super) invalidate_state_root: bool,
|
||||
pub(super) invalidate_receipts_root: bool,
|
||||
pub(super) invalidate_gas_used: bool,
|
||||
pub(super) invalidate_block_number: bool,
|
||||
pub(super) invalidate_timestamp: bool,
|
||||
pub(super) invalidate_base_fee: bool,
|
||||
pub(super) invalidate_transactions: bool,
|
||||
pub(super) invalidate_block_hash: bool,
|
||||
pub(super) invalidate_withdrawals: bool,
|
||||
pub(super) invalidate_blob_gas_used: bool,
|
||||
pub(super) invalidate_excess_blob_gas: bool,
|
||||
}
|
||||
|
||||
impl InvalidationConfig {
|
||||
/// Returns true if `block_hash` is being explicitly set or auto-invalidated.
|
||||
/// When true, the caller should skip recalculating the block hash since it will be overwritten.
|
||||
pub(super) const fn should_skip_hash_recalc(&self) -> bool {
|
||||
self.block_hash.is_some() || self.invalidate_block_hash
|
||||
}
|
||||
|
||||
/// Applies invalidations to a V1 payload, returns list of what was changed.
|
||||
pub(super) fn apply_to_payload_v1(&self, payload: &mut ExecutionPayloadV1) -> Vec<String> {
|
||||
let mut changes = Vec::new();
|
||||
|
||||
// Explicit value overrides
|
||||
if let Some(parent_hash) = self.parent_hash {
|
||||
payload.parent_hash = parent_hash;
|
||||
changes.push(format!("parent_hash = {parent_hash}"));
|
||||
}
|
||||
|
||||
if let Some(fee_recipient) = self.fee_recipient {
|
||||
payload.fee_recipient = fee_recipient;
|
||||
changes.push(format!("fee_recipient = {fee_recipient}"));
|
||||
}
|
||||
|
||||
if let Some(state_root) = self.state_root {
|
||||
payload.state_root = state_root;
|
||||
changes.push(format!("state_root = {state_root}"));
|
||||
}
|
||||
|
||||
if let Some(receipts_root) = self.receipts_root {
|
||||
payload.receipts_root = receipts_root;
|
||||
changes.push(format!("receipts_root = {receipts_root}"));
|
||||
}
|
||||
|
||||
if let Some(logs_bloom) = self.logs_bloom {
|
||||
payload.logs_bloom = logs_bloom;
|
||||
changes.push("logs_bloom = <custom>".to_string());
|
||||
}
|
||||
|
||||
if let Some(prev_randao) = self.prev_randao {
|
||||
payload.prev_randao = prev_randao;
|
||||
changes.push(format!("prev_randao = {prev_randao}"));
|
||||
}
|
||||
|
||||
if let Some(block_number) = self.block_number {
|
||||
payload.block_number = block_number;
|
||||
changes.push(format!("block_number = {block_number}"));
|
||||
}
|
||||
|
||||
if let Some(gas_limit) = self.gas_limit {
|
||||
payload.gas_limit = gas_limit;
|
||||
changes.push(format!("gas_limit = {gas_limit}"));
|
||||
}
|
||||
|
||||
if let Some(gas_used) = self.gas_used {
|
||||
payload.gas_used = gas_used;
|
||||
changes.push(format!("gas_used = {gas_used}"));
|
||||
}
|
||||
|
||||
if let Some(timestamp) = self.timestamp {
|
||||
payload.timestamp = timestamp;
|
||||
changes.push(format!("timestamp = {timestamp}"));
|
||||
}
|
||||
|
||||
if let Some(ref extra_data) = self.extra_data {
|
||||
payload.extra_data = extra_data.clone();
|
||||
changes.push(format!("extra_data = {} bytes", extra_data.len()));
|
||||
}
|
||||
|
||||
if let Some(base_fee_per_gas) = self.base_fee_per_gas {
|
||||
payload.base_fee_per_gas = U256::from_limbs([base_fee_per_gas, 0, 0, 0]);
|
||||
changes.push(format!("base_fee_per_gas = {base_fee_per_gas}"));
|
||||
}
|
||||
|
||||
if let Some(block_hash) = self.block_hash {
|
||||
payload.block_hash = block_hash;
|
||||
changes.push(format!("block_hash = {block_hash}"));
|
||||
}
|
||||
|
||||
// Auto-invalidation flags
|
||||
if self.invalidate_parent_hash {
|
||||
let random_hash = B256::random();
|
||||
payload.parent_hash = random_hash;
|
||||
changes.push(format!("parent_hash = {random_hash} (auto-invalidated: random)"));
|
||||
}
|
||||
|
||||
if self.invalidate_state_root {
|
||||
payload.state_root = B256::ZERO;
|
||||
changes.push("state_root = ZERO (auto-invalidated: empty trie root)".to_string());
|
||||
}
|
||||
|
||||
if self.invalidate_receipts_root {
|
||||
payload.receipts_root = B256::ZERO;
|
||||
changes.push("receipts_root = ZERO (auto-invalidated)".to_string());
|
||||
}
|
||||
|
||||
if self.invalidate_gas_used {
|
||||
let invalid_gas = payload.gas_limit + 1;
|
||||
payload.gas_used = invalid_gas;
|
||||
changes.push(format!("gas_used = {invalid_gas} (auto-invalidated: exceeds gas_limit)"));
|
||||
}
|
||||
|
||||
if self.invalidate_block_number {
|
||||
let invalid_number = payload.block_number + 999;
|
||||
payload.block_number = invalid_number;
|
||||
changes.push(format!("block_number = {invalid_number} (auto-invalidated: huge gap)"));
|
||||
}
|
||||
|
||||
if self.invalidate_timestamp {
|
||||
payload.timestamp = 0;
|
||||
changes.push("timestamp = 0 (auto-invalidated: impossibly old)".to_string());
|
||||
}
|
||||
|
||||
if self.invalidate_base_fee {
|
||||
payload.base_fee_per_gas = U256::ZERO;
|
||||
changes
|
||||
.push("base_fee_per_gas = 0 (auto-invalidated: invalid post-London)".to_string());
|
||||
}
|
||||
|
||||
if self.invalidate_transactions {
|
||||
let invalid_tx = Bytes::from_static(&[0xff, 0xff, 0xff]);
|
||||
payload.transactions.insert(0, invalid_tx);
|
||||
changes.push("transactions = prepended invalid RLP (auto-invalidated)".to_string());
|
||||
}
|
||||
|
||||
if self.invalidate_block_hash {
|
||||
let random_hash = B256::random();
|
||||
payload.block_hash = random_hash;
|
||||
changes.push(format!("block_hash = {random_hash} (auto-invalidated: random)"));
|
||||
}
|
||||
|
||||
changes
|
||||
}
|
||||
|
||||
/// Applies invalidations to a V2 payload, returns list of what was changed.
|
||||
pub(super) fn apply_to_payload_v2(&self, payload: &mut ExecutionPayloadV2) -> Vec<String> {
|
||||
let mut changes = self.apply_to_payload_v1(&mut payload.payload_inner);
|
||||
|
||||
// Handle withdrawals invalidation (V2+)
|
||||
if self.invalidate_withdrawals {
|
||||
let fake_withdrawal = Withdrawal {
|
||||
index: u64::MAX,
|
||||
validator_index: u64::MAX,
|
||||
address: Address::ZERO,
|
||||
amount: u64::MAX,
|
||||
};
|
||||
payload.withdrawals.push(fake_withdrawal);
|
||||
changes.push("withdrawals = added fake withdrawal (auto-invalidated)".to_string());
|
||||
}
|
||||
|
||||
changes
|
||||
}
|
||||
|
||||
/// Applies invalidations to a V3 payload, returns list of what was changed.
|
||||
pub(super) fn apply_to_payload_v3(&self, payload: &mut ExecutionPayloadV3) -> Vec<String> {
|
||||
let mut changes = self.apply_to_payload_v2(&mut payload.payload_inner);
|
||||
|
||||
// Explicit overrides for V3 fields
|
||||
if let Some(blob_gas_used) = self.blob_gas_used {
|
||||
payload.blob_gas_used = blob_gas_used;
|
||||
changes.push(format!("blob_gas_used = {blob_gas_used}"));
|
||||
}
|
||||
|
||||
if let Some(excess_blob_gas) = self.excess_blob_gas {
|
||||
payload.excess_blob_gas = excess_blob_gas;
|
||||
changes.push(format!("excess_blob_gas = {excess_blob_gas}"));
|
||||
}
|
||||
|
||||
// Auto-invalidation for V3 fields
|
||||
if self.invalidate_blob_gas_used {
|
||||
payload.blob_gas_used = u64::MAX;
|
||||
changes.push("blob_gas_used = MAX (auto-invalidated)".to_string());
|
||||
}
|
||||
|
||||
if self.invalidate_excess_blob_gas {
|
||||
payload.excess_blob_gas = u64::MAX;
|
||||
changes.push("excess_blob_gas = MAX (auto-invalidated)".to_string());
|
||||
}
|
||||
|
||||
changes
|
||||
}
|
||||
}
|
||||
367
bin/reth-bench/src/bench/send_invalid_payload/mod.rs
Normal file
367
bin/reth-bench/src/bench/send_invalid_payload/mod.rs
Normal file
@@ -0,0 +1,367 @@
|
||||
//! Command for sending invalid payloads to test Engine API rejection.
|
||||
|
||||
mod invalidation;
|
||||
use invalidation::InvalidationConfig;
|
||||
|
||||
use alloy_primitives::{Address, B256};
|
||||
use alloy_provider::network::AnyRpcBlock;
|
||||
use alloy_rpc_types_engine::ExecutionPayload;
|
||||
use clap::Parser;
|
||||
use eyre::{OptionExt, Result};
|
||||
use op_alloy_consensus::OpTxEnvelope;
|
||||
use reth_cli_runner::CliContext;
|
||||
use std::io::{BufReader, Read, Write};
|
||||
|
||||
/// Command for generating and sending an invalid `engine_newPayload` request.
|
||||
///
|
||||
/// Takes a valid block and modifies fields to make it invalid for testing
|
||||
/// Engine API rejection behavior. Block hash is recalculated after modifications
|
||||
/// unless `--invalidate-block-hash` or `--skip-hash-recalc` is used.
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Command {
|
||||
// ==================== Input Options ====================
|
||||
/// Path to the JSON file containing the block. If not specified, stdin will be used.
|
||||
#[arg(short, long, help_heading = "Input Options")]
|
||||
path: Option<String>,
|
||||
|
||||
/// The engine RPC URL to use.
|
||||
#[arg(
|
||||
short,
|
||||
long,
|
||||
help_heading = "Input Options",
|
||||
required_if_eq_any([("mode", "execute"), ("mode", "cast")]),
|
||||
required_unless_present("mode")
|
||||
)]
|
||||
rpc_url: Option<String>,
|
||||
|
||||
/// The JWT secret to use. Can be either a path to a file containing the secret or the secret
|
||||
/// itself.
|
||||
#[arg(short, long, help_heading = "Input Options")]
|
||||
jwt_secret: Option<String>,
|
||||
|
||||
/// The newPayload version to use (3 or 4).
|
||||
#[arg(long, default_value_t = 3, help_heading = "Input Options")]
|
||||
new_payload_version: u8,
|
||||
|
||||
/// The output mode to use.
|
||||
#[arg(long, value_enum, default_value = "execute", help_heading = "Input Options")]
|
||||
mode: Mode,
|
||||
|
||||
// ==================== Explicit Value Overrides ====================
|
||||
/// Override the parent hash with a specific value.
|
||||
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
|
||||
parent_hash: Option<B256>,
|
||||
|
||||
/// Override the fee recipient (coinbase) with a specific address.
|
||||
#[arg(long, value_name = "ADDR", help_heading = "Explicit Value Overrides")]
|
||||
fee_recipient: Option<Address>,
|
||||
|
||||
/// Override the state root with a specific value.
|
||||
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
|
||||
state_root: Option<B256>,
|
||||
|
||||
/// Override the receipts root with a specific value.
|
||||
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
|
||||
receipts_root: Option<B256>,
|
||||
|
||||
/// Override the block number with a specific value.
|
||||
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
|
||||
block_number: Option<u64>,
|
||||
|
||||
/// Override the gas limit with a specific value.
|
||||
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
|
||||
gas_limit: Option<u64>,
|
||||
|
||||
/// Override the gas used with a specific value.
|
||||
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
|
||||
gas_used: Option<u64>,
|
||||
|
||||
/// Override the timestamp with a specific value.
|
||||
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
|
||||
timestamp: Option<u64>,
|
||||
|
||||
/// Override the base fee per gas with a specific value.
|
||||
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
|
||||
base_fee_per_gas: Option<u64>,
|
||||
|
||||
/// Override the block hash with a specific value (skips hash recalculation).
|
||||
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
|
||||
block_hash: Option<B256>,
|
||||
|
||||
/// Override the blob gas used with a specific value.
|
||||
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
|
||||
blob_gas_used: Option<u64>,
|
||||
|
||||
/// Override the excess blob gas with a specific value.
|
||||
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
|
||||
excess_blob_gas: Option<u64>,
|
||||
|
||||
/// Override the parent beacon block root with a specific value.
|
||||
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
|
||||
parent_beacon_block_root: Option<B256>,
|
||||
|
||||
/// Override the requests hash with a specific value (EIP-7685).
|
||||
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
|
||||
requests_hash: Option<B256>,
|
||||
|
||||
// ==================== Auto-Invalidation Flags ====================
|
||||
/// Invalidate the parent hash by setting it to a random value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_parent_hash: bool,
|
||||
|
||||
/// Invalidate the state root by setting it to a random value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_state_root: bool,
|
||||
|
||||
/// Invalidate the receipts root by setting it to a random value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_receipts_root: bool,
|
||||
|
||||
/// Invalidate the gas used by setting it to an incorrect value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_gas_used: bool,
|
||||
|
||||
/// Invalidate the block number by setting it to an incorrect value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_block_number: bool,
|
||||
|
||||
/// Invalidate the timestamp by setting it to an incorrect value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_timestamp: bool,
|
||||
|
||||
/// Invalidate the base fee by setting it to an incorrect value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_base_fee: bool,
|
||||
|
||||
/// Invalidate the transactions by modifying them.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_transactions: bool,
|
||||
|
||||
/// Invalidate the block hash by not recalculating it after modifications.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_block_hash: bool,
|
||||
|
||||
/// Invalidate the withdrawals by modifying them.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_withdrawals: bool,
|
||||
|
||||
/// Invalidate the blob gas used by setting it to an incorrect value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_blob_gas_used: bool,
|
||||
|
||||
/// Invalidate the excess blob gas by setting it to an incorrect value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_excess_blob_gas: bool,
|
||||
|
||||
/// Invalidate the requests hash by setting it to a random value (EIP-7685).
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_requests_hash: bool,
|
||||
|
||||
// ==================== Meta Flags ====================
|
||||
/// Skip block hash recalculation after modifications.
|
||||
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
|
||||
skip_hash_recalc: bool,
|
||||
|
||||
/// Print what would be done without actually sending the payload.
|
||||
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
|
||||
dry_run: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, clap::ValueEnum)]
|
||||
enum Mode {
|
||||
/// Execute the `cast` command. This works with blocks of any size, because it pipes the
|
||||
/// payload into the `cast` command.
|
||||
Execute,
|
||||
/// Print the `cast` command. Caution: this may not work with large blocks because of the
|
||||
/// command length limit.
|
||||
Cast,
|
||||
/// Print the JSON payload. Can be piped into `cast` command if the block is small enough.
|
||||
Json,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Read input from either a file or stdin
|
||||
fn read_input(&self) -> Result<String> {
|
||||
Ok(match &self.path {
|
||||
Some(path) => reth_fs_util::read_to_string(path)?,
|
||||
None => String::from_utf8(
|
||||
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Load JWT secret from either a file or use the provided string directly
|
||||
fn load_jwt_secret(&self) -> Result<Option<String>> {
|
||||
match &self.jwt_secret {
|
||||
Some(secret) => match std::fs::read_to_string(secret) {
|
||||
Ok(contents) => Ok(Some(contents.trim().to_string())),
|
||||
Err(_) => Ok(Some(secret.clone())),
|
||||
},
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Build `InvalidationConfig` from command flags
|
||||
const fn build_invalidation_config(&self) -> InvalidationConfig {
|
||||
InvalidationConfig {
|
||||
parent_hash: self.parent_hash,
|
||||
fee_recipient: self.fee_recipient,
|
||||
state_root: self.state_root,
|
||||
receipts_root: self.receipts_root,
|
||||
logs_bloom: None,
|
||||
prev_randao: None,
|
||||
block_number: self.block_number,
|
||||
gas_limit: self.gas_limit,
|
||||
gas_used: self.gas_used,
|
||||
timestamp: self.timestamp,
|
||||
extra_data: None,
|
||||
base_fee_per_gas: self.base_fee_per_gas,
|
||||
block_hash: self.block_hash,
|
||||
blob_gas_used: self.blob_gas_used,
|
||||
excess_blob_gas: self.excess_blob_gas,
|
||||
invalidate_parent_hash: self.invalidate_parent_hash,
|
||||
invalidate_state_root: self.invalidate_state_root,
|
||||
invalidate_receipts_root: self.invalidate_receipts_root,
|
||||
invalidate_gas_used: self.invalidate_gas_used,
|
||||
invalidate_block_number: self.invalidate_block_number,
|
||||
invalidate_timestamp: self.invalidate_timestamp,
|
||||
invalidate_base_fee: self.invalidate_base_fee,
|
||||
invalidate_transactions: self.invalidate_transactions,
|
||||
invalidate_block_hash: self.invalidate_block_hash,
|
||||
invalidate_withdrawals: self.invalidate_withdrawals,
|
||||
invalidate_blob_gas_used: self.invalidate_blob_gas_used,
|
||||
invalidate_excess_blob_gas: self.invalidate_excess_blob_gas,
|
||||
}
|
||||
}
|
||||
|
||||
/// Execute the command
|
||||
pub async fn execute(self, _ctx: CliContext) -> Result<()> {
|
||||
let block_json = self.read_input()?;
|
||||
let jwt_secret = self.load_jwt_secret()?;
|
||||
|
||||
let block = serde_json::from_str::<AnyRpcBlock>(&block_json)?
|
||||
.into_inner()
|
||||
.map_header(|header| header.map(|h| h.into_header_with_defaults()))
|
||||
.try_map_transactions(|tx| tx.try_into_either::<OpTxEnvelope>())?
|
||||
.into_consensus();
|
||||
|
||||
let config = self.build_invalidation_config();
|
||||
|
||||
let parent_beacon_block_root =
|
||||
self.parent_beacon_block_root.or(block.header.parent_beacon_block_root);
|
||||
let blob_versioned_hashes =
|
||||
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
|
||||
let use_v4 = block.header.requests_hash.is_some();
|
||||
let requests_hash = self.requests_hash.or(block.header.requests_hash);
|
||||
|
||||
let mut execution_payload = ExecutionPayload::from_block_slow(&block).0;
|
||||
|
||||
let changes = match &mut execution_payload {
|
||||
ExecutionPayload::V1(p) => config.apply_to_payload_v1(p),
|
||||
ExecutionPayload::V2(p) => config.apply_to_payload_v2(p),
|
||||
ExecutionPayload::V3(p) => config.apply_to_payload_v3(p),
|
||||
};
|
||||
|
||||
let skip_recalc = self.skip_hash_recalc || config.should_skip_hash_recalc();
|
||||
if !skip_recalc {
|
||||
let new_hash = match execution_payload.clone().into_block_raw() {
|
||||
Ok(block) => block.header.hash_slow(),
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"Warning: Could not recalculate block hash: {e}. Using original hash."
|
||||
);
|
||||
match &execution_payload {
|
||||
ExecutionPayload::V1(p) => p.block_hash,
|
||||
ExecutionPayload::V2(p) => p.payload_inner.block_hash,
|
||||
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash,
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
match &mut execution_payload {
|
||||
ExecutionPayload::V1(p) => p.block_hash = new_hash,
|
||||
ExecutionPayload::V2(p) => p.payload_inner.block_hash = new_hash,
|
||||
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash = new_hash,
|
||||
}
|
||||
}
|
||||
|
||||
if self.dry_run {
|
||||
println!("=== Dry Run ===");
|
||||
println!("Changes that would be applied:");
|
||||
for change in &changes {
|
||||
println!(" - {}", change);
|
||||
}
|
||||
if changes.is_empty() {
|
||||
println!(" (no changes)");
|
||||
}
|
||||
if skip_recalc {
|
||||
println!(" - Block hash recalculation: SKIPPED");
|
||||
} else {
|
||||
println!(" - Block hash recalculation: PERFORMED");
|
||||
}
|
||||
println!("\nResulting payload JSON:");
|
||||
let json = serde_json::to_string_pretty(&execution_payload)?;
|
||||
println!("{}", json);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let json_request = if use_v4 {
|
||||
serde_json::to_string(&(
|
||||
execution_payload,
|
||||
blob_versioned_hashes,
|
||||
parent_beacon_block_root,
|
||||
requests_hash.unwrap_or_default(),
|
||||
))?
|
||||
} else {
|
||||
serde_json::to_string(&(
|
||||
execution_payload,
|
||||
blob_versioned_hashes,
|
||||
parent_beacon_block_root,
|
||||
))?
|
||||
};
|
||||
|
||||
match self.mode {
|
||||
Mode::Execute => {
|
||||
let mut command = std::process::Command::new("cast");
|
||||
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
|
||||
command.arg("rpc").arg(method).arg("--raw");
|
||||
if let Some(rpc_url) = self.rpc_url {
|
||||
command.arg("--rpc-url").arg(rpc_url);
|
||||
}
|
||||
if let Some(secret) = &jwt_secret {
|
||||
command.arg("--jwt-secret").arg(secret);
|
||||
}
|
||||
|
||||
let mut process = command.stdin(std::process::Stdio::piped()).spawn()?;
|
||||
|
||||
process
|
||||
.stdin
|
||||
.take()
|
||||
.ok_or_eyre("stdin not available")?
|
||||
.write_all(json_request.as_bytes())?;
|
||||
|
||||
process.wait()?;
|
||||
}
|
||||
Mode::Cast => {
|
||||
let mut cmd = format!(
|
||||
"cast rpc engine_newPayloadV{} --raw '{}'",
|
||||
self.new_payload_version, json_request
|
||||
);
|
||||
|
||||
if let Some(rpc_url) = self.rpc_url {
|
||||
cmd += &format!(" --rpc-url {rpc_url}");
|
||||
}
|
||||
if let Some(secret) = &jwt_secret {
|
||||
cmd += &format!(" --jwt-secret {secret}");
|
||||
}
|
||||
|
||||
println!("{cmd}");
|
||||
}
|
||||
Mode::Json => {
|
||||
println!("{json_request}");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -25,8 +25,8 @@
|
||||
//! - `jemalloc-unprefixed`: Uses unprefixed jemalloc symbols.
|
||||
//! - `tracy-allocator`: Enables [Tracy](https://github.com/wolfpld/tracy) profiler allocator
|
||||
//! integration for memory profiling.
|
||||
//! - `snmalloc`: Uses [snmalloc](https://github.com/snmalloc/snmalloc) as the global allocator. Use
|
||||
//! `--no-default-features` when enabling this, as jemalloc takes precedence.
|
||||
//! - `snmalloc`: Uses [snmalloc](https://github.com/microsoft/snmalloc) as the global allocator.
|
||||
//! Use `--no-default-features` when enabling this, as jemalloc takes precedence.
|
||||
//! - `snmalloc-native`: Uses snmalloc with native CPU optimizations. Use `--no-default-features`
|
||||
//! when enabling this.
|
||||
//!
|
||||
|
||||
@@ -41,6 +41,7 @@ derive_more.workspace = true
|
||||
metrics.workspace = true
|
||||
parking_lot.workspace = true
|
||||
pin-project.workspace = true
|
||||
rayon = { workspace = true, optional = true }
|
||||
serde = { workspace = true, optional = true }
|
||||
|
||||
# optional deps for test-utils
|
||||
@@ -84,6 +85,7 @@ test-utils = [
|
||||
"reth-trie/test-utils",
|
||||
"reth-ethereum-primitives/test-utils",
|
||||
]
|
||||
rayon = ["dep:rayon"]
|
||||
|
||||
[[bench]]
|
||||
name = "canonical_hashes_range"
|
||||
|
||||
@@ -163,14 +163,29 @@ impl DeferredTrieData {
|
||||
anchor_hash: B256,
|
||||
ancestors: &[Self],
|
||||
) -> ComputedTrieData {
|
||||
let sorted_hashed_state = match Arc::try_unwrap(hashed_state) {
|
||||
Ok(state) => state.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
};
|
||||
let sorted_trie_updates = match Arc::try_unwrap(trie_updates) {
|
||||
Ok(updates) => updates.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
};
|
||||
#[cfg(feature = "rayon")]
|
||||
let (sorted_hashed_state, sorted_trie_updates) = rayon::join(
|
||||
|| match Arc::try_unwrap(hashed_state) {
|
||||
Ok(state) => state.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
},
|
||||
|| match Arc::try_unwrap(trie_updates) {
|
||||
Ok(updates) => updates.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
},
|
||||
);
|
||||
|
||||
#[cfg(not(feature = "rayon"))]
|
||||
let (sorted_hashed_state, sorted_trie_updates) = (
|
||||
match Arc::try_unwrap(hashed_state) {
|
||||
Ok(state) => state.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
},
|
||||
match Arc::try_unwrap(trie_updates) {
|
||||
Ok(updates) => updates.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
},
|
||||
);
|
||||
|
||||
// Reuse parent's overlay if available and anchors match.
|
||||
// We can only reuse the parent's overlay if it was built on top of the same
|
||||
@@ -192,10 +207,10 @@ impl DeferredTrieData {
|
||||
);
|
||||
// Only trigger COW clone if there's actually data to add.
|
||||
if !sorted_hashed_state.is_empty() {
|
||||
Arc::make_mut(&mut overlay.state).extend_ref(&sorted_hashed_state);
|
||||
Arc::make_mut(&mut overlay.state).extend_ref_and_sort(&sorted_hashed_state);
|
||||
}
|
||||
if !sorted_trie_updates.is_empty() {
|
||||
Arc::make_mut(&mut overlay.nodes).extend_ref(&sorted_trie_updates);
|
||||
Arc::make_mut(&mut overlay.nodes).extend_ref_and_sort(&sorted_trie_updates);
|
||||
}
|
||||
overlay
|
||||
}
|
||||
@@ -228,8 +243,53 @@ impl DeferredTrieData {
|
||||
/// In normal operation, the parent always has a cached overlay and this
|
||||
/// function is never called.
|
||||
///
|
||||
/// Iterates ancestors oldest -> newest, then extends with current block's data,
|
||||
/// so later state takes precedence.
|
||||
/// When the `rayon` feature is enabled, uses parallel collection and merge:
|
||||
/// 1. Collects ancestor data in parallel (each `wait_cloned()` may compute)
|
||||
/// 2. Merges hashed state and trie updates in parallel with each other
|
||||
/// 3. Uses tree reduction within each merge for O(log n) depth
|
||||
#[cfg(feature = "rayon")]
|
||||
fn merge_ancestors_into_overlay(
|
||||
ancestors: &[Self],
|
||||
sorted_hashed_state: &HashedPostStateSorted,
|
||||
sorted_trie_updates: &TrieUpdatesSorted,
|
||||
) -> TrieInputSorted {
|
||||
// Early exit: no ancestors means just wrap current block's data
|
||||
if ancestors.is_empty() {
|
||||
return TrieInputSorted::new(
|
||||
Arc::new(sorted_trie_updates.clone()),
|
||||
Arc::new(sorted_hashed_state.clone()),
|
||||
Default::default(),
|
||||
);
|
||||
}
|
||||
|
||||
// Collect ancestor data, unzipping states and updates into Arc slices
|
||||
let (states, updates): (Vec<_>, Vec<_>) = ancestors
|
||||
.iter()
|
||||
.map(|a| {
|
||||
let data = a.wait_cloned();
|
||||
(data.hashed_state, data.trie_updates)
|
||||
})
|
||||
.unzip();
|
||||
|
||||
// Merge state and nodes in parallel with each other using tree reduction
|
||||
let (state, nodes) = rayon::join(
|
||||
|| {
|
||||
let mut merged = HashedPostStateSorted::merge_parallel(&states);
|
||||
merged.extend_ref_and_sort(sorted_hashed_state);
|
||||
merged
|
||||
},
|
||||
|| {
|
||||
let mut merged = TrieUpdatesSorted::merge_parallel(&updates);
|
||||
merged.extend_ref_and_sort(sorted_trie_updates);
|
||||
merged
|
||||
},
|
||||
);
|
||||
|
||||
TrieInputSorted::new(Arc::new(nodes), Arc::new(state), Default::default())
|
||||
}
|
||||
|
||||
/// Merge all ancestors and current block's data into a single overlay (sequential fallback).
|
||||
#[cfg(not(feature = "rayon"))]
|
||||
fn merge_ancestors_into_overlay(
|
||||
ancestors: &[Self],
|
||||
sorted_hashed_state: &HashedPostStateSorted,
|
||||
@@ -242,13 +302,13 @@ impl DeferredTrieData {
|
||||
|
||||
for ancestor in ancestors {
|
||||
let ancestor_data = ancestor.wait_cloned();
|
||||
state_mut.extend_ref(ancestor_data.hashed_state.as_ref());
|
||||
nodes_mut.extend_ref(ancestor_data.trie_updates.as_ref());
|
||||
state_mut.extend_ref_and_sort(ancestor_data.hashed_state.as_ref());
|
||||
nodes_mut.extend_ref_and_sort(ancestor_data.trie_updates.as_ref());
|
||||
}
|
||||
|
||||
// Extend with current block's sorted data last (takes precedence)
|
||||
state_mut.extend_ref(sorted_hashed_state);
|
||||
nodes_mut.extend_ref(sorted_trie_updates);
|
||||
state_mut.extend_ref_and_sort(sorted_hashed_state);
|
||||
nodes_mut.extend_ref_and_sort(sorted_trie_updates);
|
||||
|
||||
overlay
|
||||
}
|
||||
@@ -287,6 +347,11 @@ impl DeferredTrieData {
|
||||
&inputs.ancestors,
|
||||
);
|
||||
*state = DeferredState::Ready(computed.clone());
|
||||
|
||||
// Release lock before inputs (and its ancestors) drop to avoid holding it
|
||||
// while their potential last Arc refs drop (which could trigger recursive locking)
|
||||
drop(state);
|
||||
|
||||
computed
|
||||
}
|
||||
}
|
||||
@@ -516,7 +581,7 @@ mod tests {
|
||||
let hashed_state = Arc::new(HashedPostStateSorted::new(accounts, B256Map::default()));
|
||||
let trie_updates = Arc::default();
|
||||
let mut overlay = TrieInputSorted::default();
|
||||
Arc::make_mut(&mut overlay.state).extend_ref(hashed_state.as_ref());
|
||||
Arc::make_mut(&mut overlay.state).extend_ref_and_sort(hashed_state.as_ref());
|
||||
|
||||
DeferredTrieData::ready(ComputedTrieData {
|
||||
hashed_state,
|
||||
|
||||
@@ -10,15 +10,15 @@ use alloy_primitives::{map::HashMap, BlockNumber, TxHash, B256};
|
||||
use parking_lot::RwLock;
|
||||
use reth_chainspec::ChainInfo;
|
||||
use reth_ethereum_primitives::EthPrimitives;
|
||||
use reth_execution_types::{Chain, ExecutionOutcome};
|
||||
use reth_execution_types::{BlockExecutionOutput, BlockExecutionResult, Chain, ExecutionOutcome};
|
||||
use reth_metrics::{metrics::Gauge, Metrics};
|
||||
use reth_primitives_traits::{
|
||||
BlockBody as _, IndexedTx, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader,
|
||||
SignedTransaction,
|
||||
};
|
||||
use reth_storage_api::StateProviderBox;
|
||||
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted};
|
||||
use std::{collections::BTreeMap, ops::Deref, sync::Arc, time::Instant};
|
||||
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, LazyTrieData, TrieInputSorted};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::Instant};
|
||||
use tokio::sync::{broadcast, watch};
|
||||
|
||||
/// Size of the broadcast channel used to notify canonical state events.
|
||||
@@ -648,7 +648,7 @@ impl<N: NodePrimitives> BlockState<N> {
|
||||
}
|
||||
|
||||
/// Returns the `Receipts` of executed block that determines the state.
|
||||
pub fn receipts(&self) -> &Vec<Vec<N::Receipt>> {
|
||||
pub fn receipts(&self) -> &Vec<N::Receipt> {
|
||||
&self.block.execution_outcome().receipts
|
||||
}
|
||||
|
||||
@@ -659,15 +659,7 @@ impl<N: NodePrimitives> BlockState<N> {
|
||||
///
|
||||
/// This clones the vector of receipts. To avoid it, use [`Self::executed_block_receipts_ref`].
|
||||
pub fn executed_block_receipts(&self) -> Vec<N::Receipt> {
|
||||
let receipts = self.receipts();
|
||||
|
||||
debug_assert!(
|
||||
receipts.len() <= 1,
|
||||
"Expected at most one block's worth of receipts, found {}",
|
||||
receipts.len()
|
||||
);
|
||||
|
||||
receipts.first().cloned().unwrap_or_default()
|
||||
self.receipts().clone()
|
||||
}
|
||||
|
||||
/// Returns a slice of `Receipt` of executed block that determines the state.
|
||||
@@ -675,15 +667,7 @@ impl<N: NodePrimitives> BlockState<N> {
|
||||
/// has only one element corresponding to the executed block associated to
|
||||
/// the state.
|
||||
pub fn executed_block_receipts_ref(&self) -> &[N::Receipt] {
|
||||
let receipts = self.receipts();
|
||||
|
||||
debug_assert!(
|
||||
receipts.len() <= 1,
|
||||
"Expected at most one block's worth of receipts, found {}",
|
||||
receipts.len()
|
||||
);
|
||||
|
||||
receipts.first().map(|receipts| receipts.deref()).unwrap_or_default()
|
||||
self.receipts()
|
||||
}
|
||||
|
||||
/// Returns an iterator over __parent__ `BlockStates`.
|
||||
@@ -767,7 +751,7 @@ pub struct ExecutedBlock<N: NodePrimitives = EthPrimitives> {
|
||||
/// Recovered Block
|
||||
pub recovered_block: Arc<RecoveredBlock<N::Block>>,
|
||||
/// Block's execution outcome.
|
||||
pub execution_output: Arc<ExecutionOutcome<N::Receipt>>,
|
||||
pub execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
|
||||
/// Deferred trie data produced by execution.
|
||||
///
|
||||
/// This allows deferring the computation of the trie data which can be expensive.
|
||||
@@ -779,7 +763,15 @@ impl<N: NodePrimitives> Default for ExecutedBlock<N> {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
recovered_block: Default::default(),
|
||||
execution_output: Default::default(),
|
||||
execution_output: Arc::new(BlockExecutionOutput {
|
||||
result: BlockExecutionResult {
|
||||
receipts: Default::default(),
|
||||
requests: Default::default(),
|
||||
gas_used: 0,
|
||||
blob_gas_used: 0,
|
||||
},
|
||||
state: Default::default(),
|
||||
}),
|
||||
trie_data: DeferredTrieData::ready(ComputedTrieData::default()),
|
||||
}
|
||||
}
|
||||
@@ -800,7 +792,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
|
||||
/// payload builders). This is the safe default path.
|
||||
pub fn new(
|
||||
recovered_block: Arc<RecoveredBlock<N::Block>>,
|
||||
execution_output: Arc<ExecutionOutcome<N::Receipt>>,
|
||||
execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
|
||||
trie_data: ComputedTrieData,
|
||||
) -> Self {
|
||||
Self { recovered_block, execution_output, trie_data: DeferredTrieData::ready(trie_data) }
|
||||
@@ -822,7 +814,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
|
||||
/// Use [`Self::new()`] instead when trie data is already computed and available immediately.
|
||||
pub const fn with_deferred_trie_data(
|
||||
recovered_block: Arc<RecoveredBlock<N::Block>>,
|
||||
execution_output: Arc<ExecutionOutcome<N::Receipt>>,
|
||||
execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
|
||||
trie_data: DeferredTrieData,
|
||||
) -> Self {
|
||||
Self { recovered_block, execution_output, trie_data }
|
||||
@@ -842,7 +834,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
|
||||
|
||||
/// Returns a reference to the block's execution outcome
|
||||
#[inline]
|
||||
pub fn execution_outcome(&self) -> &ExecutionOutcome<N::Receipt> {
|
||||
pub fn execution_outcome(&self) -> &BlockExecutionOutput<N::Receipt> {
|
||||
&self.execution_output
|
||||
}
|
||||
|
||||
@@ -958,16 +950,20 @@ impl<N: NodePrimitives<SignedTx: SignedTransaction>> NewCanonicalChain<N> {
|
||||
[first, rest @ ..] => {
|
||||
let mut chain = Chain::from_block(
|
||||
first.recovered_block().clone(),
|
||||
first.execution_outcome().clone(),
|
||||
first.trie_updates(),
|
||||
first.hashed_state(),
|
||||
ExecutionOutcome::from((
|
||||
first.execution_outcome().clone(),
|
||||
first.block_number(),
|
||||
)),
|
||||
LazyTrieData::ready(first.hashed_state(), first.trie_updates()),
|
||||
);
|
||||
for exec in rest {
|
||||
chain.append_block(
|
||||
exec.recovered_block().clone(),
|
||||
exec.execution_outcome().clone(),
|
||||
exec.trie_updates(),
|
||||
exec.hashed_state(),
|
||||
ExecutionOutcome::from((
|
||||
exec.execution_outcome().clone(),
|
||||
exec.block_number(),
|
||||
)),
|
||||
LazyTrieData::ready(exec.hashed_state(), exec.trie_updates()),
|
||||
);
|
||||
}
|
||||
chain
|
||||
@@ -1264,7 +1260,7 @@ mod tests {
|
||||
|
||||
let state = BlockState::new(block);
|
||||
|
||||
assert_eq!(state.receipts(), &receipts);
|
||||
assert_eq!(state.receipts(), receipts.first().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1544,15 +1540,12 @@ mod tests {
|
||||
// Test commit notification
|
||||
let chain_commit = NewCanonicalChain::Commit { new: vec![block0.clone(), block1.clone()] };
|
||||
|
||||
// Build expected trie updates map
|
||||
let mut expected_trie_updates = BTreeMap::new();
|
||||
expected_trie_updates.insert(0, block0.trie_updates());
|
||||
expected_trie_updates.insert(1, block1.trie_updates());
|
||||
|
||||
// Build expected hashed state map
|
||||
let mut expected_hashed_state = BTreeMap::new();
|
||||
expected_hashed_state.insert(0, block0.hashed_state());
|
||||
expected_hashed_state.insert(1, block1.hashed_state());
|
||||
// Build expected trie data map
|
||||
let mut expected_trie_data = BTreeMap::new();
|
||||
expected_trie_data
|
||||
.insert(0, LazyTrieData::ready(block0.hashed_state(), block0.trie_updates()));
|
||||
expected_trie_data
|
||||
.insert(1, LazyTrieData::ready(block1.hashed_state(), block1.trie_updates()));
|
||||
|
||||
// Build expected execution outcome (first_block matches first block number)
|
||||
let commit_execution_outcome = ExecutionOutcome {
|
||||
@@ -1568,8 +1561,7 @@ mod tests {
|
||||
new: Arc::new(Chain::new(
|
||||
vec![block0.recovered_block().clone(), block1.recovered_block().clone()],
|
||||
commit_execution_outcome,
|
||||
expected_trie_updates,
|
||||
expected_hashed_state
|
||||
expected_trie_data,
|
||||
))
|
||||
}
|
||||
);
|
||||
@@ -1580,25 +1572,17 @@ mod tests {
|
||||
old: vec![block1.clone(), block2.clone()],
|
||||
};
|
||||
|
||||
// Build expected trie updates for old chain
|
||||
let mut old_trie_updates = BTreeMap::new();
|
||||
old_trie_updates.insert(1, block1.trie_updates());
|
||||
old_trie_updates.insert(2, block2.trie_updates());
|
||||
// Build expected trie data for old chain
|
||||
let mut old_trie_data = BTreeMap::new();
|
||||
old_trie_data.insert(1, LazyTrieData::ready(block1.hashed_state(), block1.trie_updates()));
|
||||
old_trie_data.insert(2, LazyTrieData::ready(block2.hashed_state(), block2.trie_updates()));
|
||||
|
||||
// Build expected trie updates for new chain
|
||||
let mut new_trie_updates = BTreeMap::new();
|
||||
new_trie_updates.insert(1, block1a.trie_updates());
|
||||
new_trie_updates.insert(2, block2a.trie_updates());
|
||||
|
||||
// Build expected hashed state for old chain
|
||||
let mut old_hashed_state = BTreeMap::new();
|
||||
old_hashed_state.insert(1, block1.hashed_state());
|
||||
old_hashed_state.insert(2, block2.hashed_state());
|
||||
|
||||
// Build expected hashed state for new chain
|
||||
let mut new_hashed_state = BTreeMap::new();
|
||||
new_hashed_state.insert(1, block1a.hashed_state());
|
||||
new_hashed_state.insert(2, block2a.hashed_state());
|
||||
// Build expected trie data for new chain
|
||||
let mut new_trie_data = BTreeMap::new();
|
||||
new_trie_data
|
||||
.insert(1, LazyTrieData::ready(block1a.hashed_state(), block1a.trie_updates()));
|
||||
new_trie_data
|
||||
.insert(2, LazyTrieData::ready(block2a.hashed_state(), block2a.trie_updates()));
|
||||
|
||||
// Build expected execution outcome for reorg chains (first_block matches first block
|
||||
// number)
|
||||
@@ -1615,14 +1599,12 @@ mod tests {
|
||||
old: Arc::new(Chain::new(
|
||||
vec![block1.recovered_block().clone(), block2.recovered_block().clone()],
|
||||
reorg_execution_outcome.clone(),
|
||||
old_trie_updates,
|
||||
old_hashed_state
|
||||
old_trie_data,
|
||||
)),
|
||||
new: Arc::new(Chain::new(
|
||||
vec![block1a.recovered_block().clone(), block2a.recovered_block().clone()],
|
||||
reorg_execution_outcome,
|
||||
new_trie_updates,
|
||||
new_hashed_state
|
||||
new_trie_data,
|
||||
))
|
||||
}
|
||||
);
|
||||
|
||||
186
crates/chain-state/src/lazy_overlay.rs
Normal file
186
crates/chain-state/src/lazy_overlay.rs
Normal file
@@ -0,0 +1,186 @@
|
||||
//! Lazy overlay computation for trie input.
|
||||
//!
|
||||
//! This module provides [`LazyOverlay`], a type that computes the [`TrieInputSorted`]
|
||||
//! lazily on first access. This allows execution to start before the trie overlay
|
||||
//! is fully computed.
|
||||
|
||||
use crate::DeferredTrieData;
|
||||
use alloy_primitives::B256;
|
||||
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use tracing::{debug, trace};
|
||||
|
||||
/// Inputs captured for lazy overlay computation.
|
||||
#[derive(Clone)]
|
||||
struct LazyOverlayInputs {
|
||||
/// The persisted ancestor hash (anchor) this overlay should be built on.
|
||||
anchor_hash: B256,
|
||||
/// Deferred trie data handles for all in-memory blocks (newest to oldest).
|
||||
blocks: Vec<DeferredTrieData>,
|
||||
}
|
||||
|
||||
/// Lazily computed trie overlay.
|
||||
///
|
||||
/// Captures the inputs needed to compute a [`TrieInputSorted`] and defers the actual
|
||||
/// computation until first access. This is conceptually similar to [`DeferredTrieData`]
|
||||
/// but for overlay computation.
|
||||
///
|
||||
/// # Fast Path vs Slow Path
|
||||
///
|
||||
/// - **Fast path**: If the tip block's cached `anchored_trie_input` is ready and its `anchor_hash`
|
||||
/// matches our expected anchor, we can reuse it directly (O(1)).
|
||||
/// - **Slow path**: Otherwise, we merge all ancestor blocks' trie data into a new overlay.
|
||||
#[derive(Clone)]
|
||||
pub struct LazyOverlay {
|
||||
/// Computed result, cached after first access.
|
||||
inner: Arc<OnceLock<TrieInputSorted>>,
|
||||
/// Inputs for lazy computation.
|
||||
inputs: LazyOverlayInputs,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for LazyOverlay {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("LazyOverlay")
|
||||
.field("anchor_hash", &self.inputs.anchor_hash)
|
||||
.field("num_blocks", &self.inputs.blocks.len())
|
||||
.field("computed", &self.inner.get().is_some())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl LazyOverlay {
|
||||
/// Create a new lazy overlay with the given anchor hash and block handles.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `anchor_hash` - The persisted ancestor hash this overlay is built on top of
|
||||
/// * `blocks` - Deferred trie data handles for in-memory blocks (newest to oldest)
|
||||
pub fn new(anchor_hash: B256, blocks: Vec<DeferredTrieData>) -> Self {
|
||||
Self { inner: Arc::new(OnceLock::new()), inputs: LazyOverlayInputs { anchor_hash, blocks } }
|
||||
}
|
||||
|
||||
/// Returns the anchor hash this overlay is built on.
|
||||
pub const fn anchor_hash(&self) -> B256 {
|
||||
self.inputs.anchor_hash
|
||||
}
|
||||
|
||||
/// Returns the number of in-memory blocks this overlay covers.
|
||||
pub const fn num_blocks(&self) -> usize {
|
||||
self.inputs.blocks.len()
|
||||
}
|
||||
|
||||
/// Returns true if the overlay has already been computed.
|
||||
pub fn is_computed(&self) -> bool {
|
||||
self.inner.get().is_some()
|
||||
}
|
||||
|
||||
/// Returns the computed trie input, computing it if necessary.
|
||||
///
|
||||
/// The first call triggers computation (which may block waiting for deferred data).
|
||||
/// Subsequent calls return the cached result immediately.
|
||||
pub fn get(&self) -> &TrieInputSorted {
|
||||
self.inner.get_or_init(|| self.compute())
|
||||
}
|
||||
|
||||
/// Returns the overlay as (nodes, state) tuple for use with `OverlayStateProviderFactory`.
|
||||
pub fn as_overlay(&self) -> (Arc<TrieUpdatesSorted>, Arc<HashedPostStateSorted>) {
|
||||
let input = self.get();
|
||||
(Arc::clone(&input.nodes), Arc::clone(&input.state))
|
||||
}
|
||||
|
||||
/// Compute the trie input overlay.
|
||||
fn compute(&self) -> TrieInputSorted {
|
||||
let anchor_hash = self.inputs.anchor_hash;
|
||||
let blocks = &self.inputs.blocks;
|
||||
|
||||
if blocks.is_empty() {
|
||||
debug!(target: "chain_state::lazy_overlay", "No in-memory blocks, returning empty overlay");
|
||||
return TrieInputSorted::default();
|
||||
}
|
||||
|
||||
// Fast path: Check if tip block's overlay is ready and anchor matches.
|
||||
// The tip block (first in list) has the cumulative overlay from all ancestors.
|
||||
if let Some(tip) = blocks.first() {
|
||||
let data = tip.wait_cloned();
|
||||
if let Some(anchored) = &data.anchored_trie_input {
|
||||
if anchored.anchor_hash == anchor_hash {
|
||||
trace!(target: "chain_state::lazy_overlay", %anchor_hash, "Reusing tip block's cached overlay (fast path)");
|
||||
return (*anchored.trie_input).clone();
|
||||
}
|
||||
debug!(
|
||||
target: "chain_state::lazy_overlay",
|
||||
computed_anchor = %anchored.anchor_hash,
|
||||
%anchor_hash,
|
||||
"Anchor mismatch, falling back to merge"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Slow path: Merge all blocks' trie data into a new overlay.
|
||||
debug!(target: "chain_state::lazy_overlay", num_blocks = blocks.len(), "Merging blocks (slow path)");
|
||||
Self::merge_blocks(blocks)
|
||||
}
|
||||
|
||||
/// Merge all blocks' trie data into a single [`TrieInputSorted`].
|
||||
///
|
||||
/// Blocks are ordered newest to oldest.
|
||||
fn merge_blocks(blocks: &[DeferredTrieData]) -> TrieInputSorted {
|
||||
if blocks.is_empty() {
|
||||
return TrieInputSorted::default();
|
||||
}
|
||||
|
||||
let state =
|
||||
HashedPostStateSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().hashed_state));
|
||||
let nodes =
|
||||
TrieUpdatesSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().trie_updates));
|
||||
|
||||
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use reth_trie::{updates::TrieUpdates, HashedPostState};
|
||||
|
||||
fn empty_deferred(anchor: B256) -> DeferredTrieData {
|
||||
DeferredTrieData::pending(
|
||||
Arc::new(HashedPostState::default()),
|
||||
Arc::new(TrieUpdates::default()),
|
||||
anchor,
|
||||
Vec::new(),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_blocks_returns_default() {
|
||||
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
|
||||
let result = overlay.get();
|
||||
assert!(result.state.is_empty());
|
||||
assert!(result.nodes.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_block_uses_data_directly() {
|
||||
let anchor = B256::random();
|
||||
let deferred = empty_deferred(anchor);
|
||||
let overlay = LazyOverlay::new(anchor, vec![deferred]);
|
||||
|
||||
assert!(!overlay.is_computed());
|
||||
let _ = overlay.get();
|
||||
assert!(overlay.is_computed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_after_first_access() {
|
||||
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
|
||||
|
||||
// First access computes
|
||||
let _ = overlay.get();
|
||||
assert!(overlay.is_computed());
|
||||
|
||||
// Second access uses cache
|
||||
let _ = overlay.get();
|
||||
assert!(overlay.is_computed());
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,9 @@ pub use in_memory::*;
|
||||
mod deferred_trie;
|
||||
pub use deferred_trie::*;
|
||||
|
||||
mod lazy_overlay;
|
||||
pub use lazy_overlay::*;
|
||||
|
||||
mod noop;
|
||||
|
||||
mod chain_info;
|
||||
|
||||
@@ -280,7 +280,6 @@ mod tests {
|
||||
vec![block1.clone(), block2.clone()],
|
||||
ExecutionOutcome::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
));
|
||||
|
||||
// Create a commit notification
|
||||
@@ -319,13 +318,11 @@ mod tests {
|
||||
vec![block1.clone()],
|
||||
ExecutionOutcome::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
));
|
||||
let new_chain = Arc::new(Chain::new(
|
||||
vec![block2.clone(), block3.clone()],
|
||||
ExecutionOutcome::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
));
|
||||
|
||||
// Create a reorg notification
|
||||
@@ -391,7 +388,6 @@ mod tests {
|
||||
vec![block1.clone(), block2.clone()],
|
||||
execution_outcome,
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
));
|
||||
|
||||
// Create a commit notification containing the new chain segment.
|
||||
@@ -449,12 +445,8 @@ mod tests {
|
||||
ExecutionOutcome { receipts: old_receipts, ..Default::default() };
|
||||
|
||||
// Create an old chain segment to be reverted, containing `old_block1`.
|
||||
let old_chain: Arc<Chain> = Arc::new(Chain::new(
|
||||
vec![old_block1.clone()],
|
||||
old_execution_outcome,
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
));
|
||||
let old_chain: Arc<Chain> =
|
||||
Arc::new(Chain::new(vec![old_block1.clone()], old_execution_outcome, BTreeMap::new()));
|
||||
|
||||
// Define block2 for the new chain segment, which will be committed.
|
||||
let mut body = BlockBody::<TransactionSigned>::default();
|
||||
@@ -482,12 +474,8 @@ mod tests {
|
||||
ExecutionOutcome { receipts: new_receipts, ..Default::default() };
|
||||
|
||||
// Create a new chain segment to be committed, containing `new_block1`.
|
||||
let new_chain = Arc::new(Chain::new(
|
||||
vec![new_block1.clone()],
|
||||
new_execution_outcome,
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
));
|
||||
let new_chain =
|
||||
Arc::new(Chain::new(vec![new_block1.clone()], new_execution_outcome, BTreeMap::new()));
|
||||
|
||||
// Create a reorg notification with both reverted (old) and committed (new) chain segments.
|
||||
let notification = CanonStateNotification::Reorg { old: old_chain, new: new_chain };
|
||||
|
||||
@@ -3,10 +3,7 @@ use crate::{
|
||||
CanonStateSubscriptions, ComputedTrieData,
|
||||
};
|
||||
use alloy_consensus::{Header, SignableTransaction, TxEip1559, TxReceipt, EMPTY_ROOT_HASH};
|
||||
use alloy_eips::{
|
||||
eip1559::{ETHEREUM_BLOCK_GAS_LIMIT_30M, INITIAL_BASE_FEE},
|
||||
eip7685::Requests,
|
||||
};
|
||||
use alloy_eips::eip1559::{ETHEREUM_BLOCK_GAS_LIMIT_30M, INITIAL_BASE_FEE};
|
||||
use alloy_primitives::{Address, BlockNumber, B256, U256};
|
||||
use alloy_signer::SignerSync;
|
||||
use alloy_signer_local::PrivateKeySigner;
|
||||
@@ -16,7 +13,7 @@ use reth_chainspec::{ChainSpec, EthereumHardfork, MIN_TRANSACTION_GAS};
|
||||
use reth_ethereum_primitives::{
|
||||
Block, BlockBody, EthPrimitives, Receipt, Transaction, TransactionSigned,
|
||||
};
|
||||
use reth_execution_types::{Chain, ExecutionOutcome};
|
||||
use reth_execution_types::{BlockExecutionOutput, BlockExecutionResult, Chain, ExecutionOutcome};
|
||||
use reth_primitives_traits::{
|
||||
proofs::{calculate_receipt_root, calculate_transaction_root, calculate_withdrawals_root},
|
||||
Account, NodePrimitives, Recovered, RecoveredBlock, SealedBlock, SealedHeader,
|
||||
@@ -201,7 +198,7 @@ impl<N: NodePrimitives> TestBlockBuilder<N> {
|
||||
fn get_executed_block(
|
||||
&mut self,
|
||||
block_number: BlockNumber,
|
||||
receipts: Vec<Vec<Receipt>>,
|
||||
mut receipts: Vec<Vec<Receipt>>,
|
||||
parent_hash: B256,
|
||||
) -> ExecutedBlock {
|
||||
let block = self.generate_random_block(block_number, parent_hash);
|
||||
@@ -209,12 +206,15 @@ impl<N: NodePrimitives> TestBlockBuilder<N> {
|
||||
let trie_data = ComputedTrieData::default();
|
||||
ExecutedBlock::new(
|
||||
Arc::new(RecoveredBlock::new_sealed(block, senders)),
|
||||
Arc::new(ExecutionOutcome::new(
|
||||
BundleState::default(),
|
||||
receipts,
|
||||
block_number,
|
||||
vec![Requests::default()],
|
||||
)),
|
||||
Arc::new(BlockExecutionOutput {
|
||||
result: BlockExecutionResult {
|
||||
receipts: receipts.pop().unwrap_or_default(),
|
||||
requests: Default::default(),
|
||||
gas_used: 0,
|
||||
blob_gas_used: 0,
|
||||
},
|
||||
state: BundleState::default(),
|
||||
}),
|
||||
trie_data,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -278,6 +278,7 @@ pub fn create_chain_config(
|
||||
// Check if DAO fork is supported (it has an activation block)
|
||||
let dao_fork_support = hardforks.fork(EthereumHardfork::Dao) != ForkCondition::Never;
|
||||
|
||||
#[allow(clippy::needless_update)]
|
||||
ChainConfig {
|
||||
chain_id: chain.map(|c| c.id()).unwrap_or(0),
|
||||
homestead_block: block_num(EthereumHardfork::Homestead),
|
||||
@@ -313,6 +314,7 @@ pub fn create_chain_config(
|
||||
extra_fields: Default::default(),
|
||||
deposit_contract_address,
|
||||
blob_schedule,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -50,6 +50,7 @@ reth-stages-types = { workspace = true, optional = true }
|
||||
reth-static-file-types = { workspace = true, features = ["clap"] }
|
||||
reth-static-file.workspace = true
|
||||
reth-tasks.workspace = true
|
||||
reth-storage-api.workspace = true
|
||||
reth-trie = { workspace = true, features = ["metrics"] }
|
||||
reth-trie-db = { workspace = true, features = ["metrics"] }
|
||||
reth-trie-common.workspace = true
|
||||
@@ -131,4 +132,4 @@ arbitrary = [
|
||||
"reth-ethereum-primitives/arbitrary",
|
||||
]
|
||||
|
||||
edge = ["reth-db-common/edge", "reth-stages/rocksdb"]
|
||||
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb"]
|
||||
|
||||
@@ -19,7 +19,7 @@ use reth_node_builder::{
|
||||
Node, NodeComponents, NodeComponentsBuilder, NodeTypes, NodeTypesWithDBAdapter,
|
||||
};
|
||||
use reth_node_core::{
|
||||
args::{DatabaseArgs, DatadirArgs, StaticFilesArgs},
|
||||
args::{DatabaseArgs, DatadirArgs, RocksDbArgs, StaticFilesArgs},
|
||||
dirs::{ChainPath, DataDirPath},
|
||||
};
|
||||
use reth_provider::{
|
||||
@@ -27,7 +27,7 @@ use reth_provider::{
|
||||
BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider,
|
||||
StaticFileProviderBuilder,
|
||||
},
|
||||
ProviderFactory, StaticFileProviderFactory,
|
||||
ProviderFactory, StaticFileProviderFactory, StorageSettings,
|
||||
};
|
||||
use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget};
|
||||
use reth_static_file::StaticFileProducer;
|
||||
@@ -66,9 +66,24 @@ pub struct EnvironmentArgs<C: ChainSpecParser> {
|
||||
/// All static files related arguments
|
||||
#[command(flatten)]
|
||||
pub static_files: StaticFilesArgs,
|
||||
|
||||
/// All `RocksDB` related arguments
|
||||
#[command(flatten)]
|
||||
pub rocksdb: RocksDbArgs,
|
||||
}
|
||||
|
||||
impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
/// Returns the effective storage settings derived from static-file and `RocksDB` CLI args.
|
||||
pub fn storage_settings(&self) -> StorageSettings {
|
||||
StorageSettings::base()
|
||||
.with_receipts_in_static_files(self.static_files.receipts)
|
||||
.with_transaction_senders_in_static_files(self.static_files.transaction_senders)
|
||||
.with_account_changesets_in_static_files(self.static_files.account_changesets)
|
||||
.with_transaction_hash_numbers_in_rocksdb(self.rocksdb.all || self.rocksdb.tx_hash)
|
||||
.with_storages_history_in_rocksdb(self.rocksdb.all || self.rocksdb.storages_history)
|
||||
.with_account_history_in_rocksdb(self.rocksdb.all || self.rocksdb.account_history)
|
||||
}
|
||||
|
||||
/// Initializes environment according to [`AccessRights`] and returns an instance of
|
||||
/// [`Environment`].
|
||||
pub fn init<N: CliNodeTypes>(&self, access: AccessRights) -> eyre::Result<Environment<N>>
|
||||
@@ -121,17 +136,17 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
})
|
||||
}
|
||||
};
|
||||
// TransactionDB only support read-write mode
|
||||
let rocksdb_provider = RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level)
|
||||
.with_read_only(!access.is_read_write())
|
||||
.build()?;
|
||||
|
||||
let provider_factory =
|
||||
self.create_provider_factory(&config, db, sfp, rocksdb_provider, access)?;
|
||||
if access.is_read_write() {
|
||||
debug!(target: "reth::cli", chain=%self.chain.chain(), genesis=?self.chain.genesis_hash(), "Initializing genesis");
|
||||
init_genesis_with_settings(&provider_factory, self.static_files.to_settings())?;
|
||||
init_genesis_with_settings(&provider_factory, self.storage_settings())?;
|
||||
}
|
||||
|
||||
Ok(Environment { config, provider_factory, data_dir })
|
||||
|
||||
@@ -1,145 +0,0 @@
|
||||
use crate::{
|
||||
common::CliNodeTypes,
|
||||
db::get::{maybe_json_value_parser, table_key},
|
||||
};
|
||||
use alloy_primitives::map::foldhash::fast::FixedState;
|
||||
use clap::Parser;
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_db::DatabaseEnv;
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
|
||||
Tables,
|
||||
};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
|
||||
use reth_provider::{providers::ProviderNodeTypes, DBProvider};
|
||||
use std::{
|
||||
hash::{BuildHasher, Hasher},
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
/// The arguments for the `reth db checksum` command
|
||||
pub struct Command {
|
||||
/// The table name
|
||||
table: Tables,
|
||||
|
||||
/// The start of the range to checksum.
|
||||
#[arg(long, value_parser = maybe_json_value_parser)]
|
||||
start_key: Option<String>,
|
||||
|
||||
/// The end of the range to checksum.
|
||||
#[arg(long, value_parser = maybe_json_value_parser)]
|
||||
end_key: Option<String>,
|
||||
|
||||
/// The maximum number of records that are queried and used to compute the
|
||||
/// checksum.
|
||||
#[arg(long)]
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `db checksum` command
|
||||
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
self,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
) -> eyre::Result<()> {
|
||||
warn!("This command should be run without the node running!");
|
||||
self.table.view(&ChecksumViewer {
|
||||
tool,
|
||||
start_key: self.start_key,
|
||||
end_key: self.end_key,
|
||||
limit: self.limit,
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
|
||||
tool: &'a DbTool<N>,
|
||||
start_key: Option<String>,
|
||||
end_key: Option<String>,
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
|
||||
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
|
||||
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
|
||||
type Error = eyre::Report;
|
||||
|
||||
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
|
||||
let provider =
|
||||
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
|
||||
let tx = provider.tx_ref();
|
||||
info!(
|
||||
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
|
||||
self.start_key, self.end_key, self.limit
|
||||
);
|
||||
|
||||
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
|
||||
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
|
||||
(Some(start), Some(end)) => {
|
||||
let start_key = table_key::<T>(start).map(RawKey::new)?;
|
||||
let end_key = table_key::<T>(end).map(RawKey::new)?;
|
||||
cursor.walk_range(start_key..=end_key)?
|
||||
}
|
||||
(None, Some(end)) => {
|
||||
let end_key = table_key::<T>(end).map(RawKey::new)?;
|
||||
|
||||
cursor.walk_range(..=end_key)?
|
||||
}
|
||||
(Some(start), None) => {
|
||||
let start_key = table_key::<T>(start).map(RawKey::new)?;
|
||||
cursor.walk_range(start_key..)?
|
||||
}
|
||||
(None, None) => cursor.walk_range(..)?,
|
||||
};
|
||||
|
||||
let start_time = Instant::now();
|
||||
let mut hasher = FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher();
|
||||
let mut total = 0;
|
||||
|
||||
let limit = self.limit.unwrap_or(usize::MAX);
|
||||
let mut enumerate_start_key = None;
|
||||
let mut enumerate_end_key = None;
|
||||
for (index, entry) in walker.enumerate() {
|
||||
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
|
||||
|
||||
if index.is_multiple_of(100_000) {
|
||||
info!("Hashed {index} entries.");
|
||||
}
|
||||
|
||||
hasher.write(k.raw_key());
|
||||
hasher.write(v.raw_value());
|
||||
|
||||
if enumerate_start_key.is_none() {
|
||||
enumerate_start_key = Some(k.clone());
|
||||
}
|
||||
enumerate_end_key = Some(k);
|
||||
|
||||
total = index + 1;
|
||||
if total >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
info!("Hashed {total} entries.");
|
||||
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
|
||||
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
|
||||
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
|
||||
}
|
||||
|
||||
let checksum = hasher.finish();
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
|
||||
|
||||
Ok((checksum, elapsed))
|
||||
}
|
||||
}
|
||||
288
crates/cli/commands/src/db/checksum/mod.rs
Normal file
288
crates/cli/commands/src/db/checksum/mod.rs
Normal file
@@ -0,0 +1,288 @@
|
||||
use crate::{
|
||||
common::CliNodeTypes,
|
||||
db::get::{maybe_json_value_parser, table_key},
|
||||
};
|
||||
use alloy_primitives::map::foldhash::fast::FixedState;
|
||||
use clap::Parser;
|
||||
use itertools::Itertools;
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_db::{static_file::iter_static_files, DatabaseEnv};
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
|
||||
Tables,
|
||||
};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
|
||||
use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProviderFactory};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use std::{
|
||||
hash::{BuildHasher, Hasher},
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
#[cfg(all(unix, feature = "edge"))]
|
||||
mod rocksdb;
|
||||
|
||||
/// Interval for logging progress during checksum computation.
|
||||
const PROGRESS_LOG_INTERVAL: usize = 100_000;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
/// The arguments for the `reth db checksum` command
|
||||
pub struct Command {
|
||||
#[command(subcommand)]
|
||||
subcommand: Subcommand,
|
||||
}
|
||||
|
||||
#[derive(clap::Subcommand, Debug)]
|
||||
enum Subcommand {
|
||||
/// Calculates the checksum of a database table
|
||||
Mdbx {
|
||||
/// The table name
|
||||
table: Tables,
|
||||
|
||||
/// The start of the range to checksum.
|
||||
#[arg(long, value_parser = maybe_json_value_parser)]
|
||||
start_key: Option<String>,
|
||||
|
||||
/// The end of the range to checksum.
|
||||
#[arg(long, value_parser = maybe_json_value_parser)]
|
||||
end_key: Option<String>,
|
||||
|
||||
/// The maximum number of records that are queried and used to compute the
|
||||
/// checksum.
|
||||
#[arg(long)]
|
||||
limit: Option<usize>,
|
||||
},
|
||||
/// Calculates the checksum of a static file segment
|
||||
StaticFile {
|
||||
/// The static file segment
|
||||
#[arg(value_enum)]
|
||||
segment: StaticFileSegment,
|
||||
|
||||
/// The block number to start from (inclusive).
|
||||
#[arg(long)]
|
||||
start_block: Option<u64>,
|
||||
|
||||
/// The block number to end at (inclusive).
|
||||
#[arg(long)]
|
||||
end_block: Option<u64>,
|
||||
|
||||
/// The maximum number of rows to checksum.
|
||||
#[arg(long)]
|
||||
limit: Option<usize>,
|
||||
},
|
||||
/// Calculates the checksum of a RocksDB table
|
||||
#[cfg(all(unix, feature = "edge"))]
|
||||
Rocksdb {
|
||||
/// The RocksDB table
|
||||
#[arg(value_enum)]
|
||||
table: rocksdb::RocksDbTable,
|
||||
|
||||
/// The maximum number of records to checksum.
|
||||
#[arg(long)]
|
||||
limit: Option<usize>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `db checksum` command
|
||||
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
self,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
) -> eyre::Result<()> {
|
||||
warn!("This command should be run without the node running!");
|
||||
|
||||
match self.subcommand {
|
||||
Subcommand::Mdbx { table, start_key, end_key, limit } => {
|
||||
table.view(&ChecksumViewer { tool, start_key, end_key, limit })?;
|
||||
}
|
||||
Subcommand::StaticFile { segment, start_block, end_block, limit } => {
|
||||
checksum_static_file(tool, segment, start_block, end_block, limit)?;
|
||||
}
|
||||
#[cfg(all(unix, feature = "edge"))]
|
||||
Subcommand::Rocksdb { table, limit } => {
|
||||
rocksdb::checksum_rocksdb(tool, table, limit)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a new hasher with the standard seed used for checksum computation.
|
||||
fn checksum_hasher() -> impl Hasher {
|
||||
FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher()
|
||||
}
|
||||
|
||||
fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
segment: StaticFileSegment,
|
||||
start_block: Option<u64>,
|
||||
end_block: Option<u64>,
|
||||
limit: Option<usize>,
|
||||
) -> eyre::Result<()> {
|
||||
let static_file_provider = tool.provider_factory.static_file_provider();
|
||||
if let Err(err) = static_file_provider.check_consistency(&tool.provider_factory.provider()?) {
|
||||
warn!("Error checking consistency of static files: {err}");
|
||||
}
|
||||
|
||||
let static_files = iter_static_files(static_file_provider.directory())?;
|
||||
|
||||
let ranges = static_files
|
||||
.get(segment)
|
||||
.ok_or_else(|| eyre::eyre!("No static files found for segment: {}", segment))?;
|
||||
|
||||
let start_time = Instant::now();
|
||||
let mut hasher = checksum_hasher();
|
||||
let mut total = 0usize;
|
||||
let limit = limit.unwrap_or(usize::MAX);
|
||||
|
||||
let start_block = start_block.unwrap_or(0);
|
||||
let end_block = end_block.unwrap_or(u64::MAX);
|
||||
|
||||
info!(
|
||||
"Computing checksum for {} static files, start_block={}, end_block={}, limit={:?}",
|
||||
segment,
|
||||
start_block,
|
||||
end_block,
|
||||
if limit == usize::MAX { None } else { Some(limit) }
|
||||
);
|
||||
|
||||
'outer: for (block_range, _header) in ranges.iter().sorted_by_key(|(range, _)| range.start()) {
|
||||
if block_range.end() < start_block || block_range.start() > end_block {
|
||||
continue;
|
||||
}
|
||||
|
||||
let fixed_block_range = static_file_provider.find_fixed_range(segment, block_range.start());
|
||||
let jar_provider = static_file_provider
|
||||
.get_segment_provider_for_range(segment, || Some(fixed_block_range), None)?
|
||||
.ok_or_else(|| {
|
||||
eyre::eyre!(
|
||||
"Failed to get segment provider for segment {} at range {}",
|
||||
segment,
|
||||
block_range
|
||||
)
|
||||
})?;
|
||||
|
||||
let mut cursor = jar_provider.cursor()?;
|
||||
|
||||
while let Ok(Some(row)) = cursor.next_row() {
|
||||
for col_data in row.iter() {
|
||||
hasher.write(col_data);
|
||||
}
|
||||
|
||||
total += 1;
|
||||
|
||||
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
|
||||
info!("Hashed {total} entries.");
|
||||
}
|
||||
|
||||
if total >= limit {
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
|
||||
// Explicitly drop provider before removing from cache to avoid deadlock
|
||||
drop(jar_provider);
|
||||
static_file_provider.remove_cached_provider(segment, fixed_block_range.end());
|
||||
}
|
||||
|
||||
let checksum = hasher.finish();
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
info!(
|
||||
"Checksum for static file segment `{}`: {:#x} ({} entries, elapsed: {:?})",
|
||||
segment, checksum, total, elapsed
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
|
||||
tool: &'a DbTool<N>,
|
||||
start_key: Option<String>,
|
||||
end_key: Option<String>,
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
|
||||
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
|
||||
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
|
||||
type Error = eyre::Report;
|
||||
|
||||
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
|
||||
let provider =
|
||||
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
|
||||
let tx = provider.tx_ref();
|
||||
info!(
|
||||
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
|
||||
self.start_key, self.end_key, self.limit
|
||||
);
|
||||
|
||||
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
|
||||
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
|
||||
(Some(start), Some(end)) => {
|
||||
let start_key = table_key::<T>(start).map(RawKey::new)?;
|
||||
let end_key = table_key::<T>(end).map(RawKey::new)?;
|
||||
cursor.walk_range(start_key..=end_key)?
|
||||
}
|
||||
(None, Some(end)) => {
|
||||
let end_key = table_key::<T>(end).map(RawKey::new)?;
|
||||
|
||||
cursor.walk_range(..=end_key)?
|
||||
}
|
||||
(Some(start), None) => {
|
||||
let start_key = table_key::<T>(start).map(RawKey::new)?;
|
||||
cursor.walk_range(start_key..)?
|
||||
}
|
||||
(None, None) => cursor.walk_range(..)?,
|
||||
};
|
||||
|
||||
let start_time = Instant::now();
|
||||
let mut hasher = checksum_hasher();
|
||||
let mut total = 0;
|
||||
|
||||
let limit = self.limit.unwrap_or(usize::MAX);
|
||||
let mut enumerate_start_key = None;
|
||||
let mut enumerate_end_key = None;
|
||||
for (index, entry) in walker.enumerate() {
|
||||
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
|
||||
|
||||
if index.is_multiple_of(PROGRESS_LOG_INTERVAL) {
|
||||
info!("Hashed {index} entries.");
|
||||
}
|
||||
|
||||
hasher.write(k.raw_key());
|
||||
hasher.write(v.raw_value());
|
||||
|
||||
if enumerate_start_key.is_none() {
|
||||
enumerate_start_key = Some(k.clone());
|
||||
}
|
||||
enumerate_end_key = Some(k);
|
||||
|
||||
total = index + 1;
|
||||
if total >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
info!("Hashed {total} entries.");
|
||||
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
|
||||
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
|
||||
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
|
||||
}
|
||||
|
||||
let checksum = hasher.finish();
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
|
||||
|
||||
Ok((checksum, elapsed))
|
||||
}
|
||||
}
|
||||
106
crates/cli/commands/src/db/checksum/rocksdb.rs
Normal file
106
crates/cli/commands/src/db/checksum/rocksdb.rs
Normal file
@@ -0,0 +1,106 @@
|
||||
//! RocksDB checksum implementation.
|
||||
|
||||
use super::{checksum_hasher, PROGRESS_LOG_INTERVAL};
|
||||
use crate::common::CliNodeTypes;
|
||||
use clap::ValueEnum;
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_db::{tables, DatabaseEnv};
|
||||
use reth_db_api::table::Table;
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::NodeTypesWithDBAdapter;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use std::{hash::Hasher, sync::Arc, time::Instant};
|
||||
use tracing::info;
|
||||
|
||||
/// RocksDB tables that can be checksummed.
|
||||
#[derive(Debug, Clone, Copy, ValueEnum)]
|
||||
pub enum RocksDbTable {
|
||||
/// Transaction hash to transaction number mapping
|
||||
TransactionHashNumbers,
|
||||
/// Account history indices
|
||||
AccountsHistory,
|
||||
/// Storage history indices
|
||||
StoragesHistory,
|
||||
}
|
||||
|
||||
impl RocksDbTable {
|
||||
/// Returns the table name as a string
|
||||
const fn name(&self) -> &'static str {
|
||||
match self {
|
||||
Self::TransactionHashNumbers => tables::TransactionHashNumbers::NAME,
|
||||
Self::AccountsHistory => tables::AccountsHistory::NAME,
|
||||
Self::StoragesHistory => tables::StoragesHistory::NAME,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Computes a checksum for a RocksDB table.
|
||||
pub fn checksum_rocksdb<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
table: RocksDbTable,
|
||||
limit: Option<usize>,
|
||||
) -> eyre::Result<()> {
|
||||
let rocksdb = tool.provider_factory.rocksdb_provider();
|
||||
|
||||
let start_time = Instant::now();
|
||||
let limit = limit.unwrap_or(usize::MAX);
|
||||
|
||||
info!(
|
||||
"Computing checksum for RocksDB table `{}`, limit={:?}",
|
||||
table.name(),
|
||||
if limit == usize::MAX { None } else { Some(limit) }
|
||||
);
|
||||
|
||||
let (checksum, total) = match table {
|
||||
RocksDbTable::TransactionHashNumbers => {
|
||||
checksum_rocksdb_table::<tables::TransactionHashNumbers>(&rocksdb, limit)?
|
||||
}
|
||||
RocksDbTable::AccountsHistory => {
|
||||
checksum_rocksdb_table::<tables::AccountsHistory>(&rocksdb, limit)?
|
||||
}
|
||||
RocksDbTable::StoragesHistory => {
|
||||
checksum_rocksdb_table::<tables::StoragesHistory>(&rocksdb, limit)?
|
||||
}
|
||||
};
|
||||
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
info!(
|
||||
"Checksum for RocksDB table `{}`: {:#x} ({} entries, elapsed: {:?})",
|
||||
table.name(),
|
||||
checksum,
|
||||
total,
|
||||
elapsed
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Computes checksum for a specific RocksDB table by iterating over rows.
|
||||
fn checksum_rocksdb_table<T: Table>(
|
||||
rocksdb: &reth_provider::providers::RocksDBProvider,
|
||||
limit: usize,
|
||||
) -> eyre::Result<(u64, usize)> {
|
||||
let iter = rocksdb.raw_iter::<T>()?;
|
||||
let mut hasher = checksum_hasher();
|
||||
let mut total = 0usize;
|
||||
|
||||
for entry in iter {
|
||||
let (key_bytes, value_bytes) = entry?;
|
||||
|
||||
hasher.write(&key_bytes);
|
||||
hasher.write(&value_bytes);
|
||||
|
||||
total += 1;
|
||||
|
||||
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
|
||||
info!("Hashed {total} entries.");
|
||||
}
|
||||
|
||||
if total >= limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok((hasher.finish(), total))
|
||||
}
|
||||
@@ -21,6 +21,7 @@ use reth_node_builder::NodeTypesWithDB;
|
||||
use reth_primitives_traits::ValueWithSubKey;
|
||||
use reth_provider::{providers::ProviderNodeTypes, ChangeSetReader, StaticFileProviderFactory};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::StorageChangeSetReader;
|
||||
use tracing::error;
|
||||
|
||||
/// The arguments for the `reth db get` command
|
||||
@@ -82,6 +83,33 @@ impl Command {
|
||||
table.view(&GetValueViewer { tool, key, subkey, end_key, end_subkey, raw })?
|
||||
}
|
||||
Subcommand::StaticFile { segment, key, subkey, raw } => {
|
||||
if let StaticFileSegment::StorageChangeSets = segment {
|
||||
let storage_key =
|
||||
table_subkey::<tables::StorageChangeSets>(subkey.as_deref()).ok();
|
||||
let key = table_key::<tables::StorageChangeSets>(&key)?;
|
||||
|
||||
let provider = tool.provider_factory.static_file_provider();
|
||||
|
||||
if let Some(storage_key) = storage_key {
|
||||
let entry = provider.get_storage_before_block(
|
||||
key.block_number(),
|
||||
key.address(),
|
||||
storage_key,
|
||||
)?;
|
||||
|
||||
if let Some(entry) = entry {
|
||||
println!("{}", serde_json::to_string_pretty(&entry)?);
|
||||
} else {
|
||||
error!(target: "reth::cli", "No content for the given table key.");
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let changesets = provider.storage_changeset(key.block_number())?;
|
||||
println!("{}", serde_json::to_string_pretty(&changesets)?);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (key, subkey, mask): (u64, _, _) = match segment {
|
||||
StaticFileSegment::Headers => (
|
||||
table_key::<tables::Headers>(&key)?,
|
||||
@@ -112,6 +140,9 @@ impl Command {
|
||||
AccountChangesetMask::MASK,
|
||||
)
|
||||
}
|
||||
StaticFileSegment::StorageChangeSets => {
|
||||
unreachable!("storage changesets handled above");
|
||||
}
|
||||
};
|
||||
|
||||
// handle account changesets differently if a subkey is provided.
|
||||
@@ -190,6 +221,9 @@ impl Command {
|
||||
StaticFileSegment::AccountChangeSets => {
|
||||
unreachable!("account changeset static files are special cased before this match")
|
||||
}
|
||||
StaticFileSegment::StorageChangeSets => {
|
||||
unreachable!("storage changeset static files are special cased before this match")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,19 +61,21 @@ impl Command {
|
||||
}
|
||||
|
||||
/// Generate [`ListFilter`] from command.
|
||||
pub fn list_filter(&self) -> ListFilter {
|
||||
let search = self
|
||||
.search
|
||||
.as_ref()
|
||||
.map(|search| {
|
||||
pub fn list_filter(&self) -> eyre::Result<ListFilter> {
|
||||
let search = match self.search.as_deref() {
|
||||
Some(search) => {
|
||||
if let Some(search) = search.strip_prefix("0x") {
|
||||
return hex::decode(search).unwrap()
|
||||
hex::decode(search).wrap_err(
|
||||
"Invalid hex content after 0x prefix in --search (expected valid hex like 0xdeadbeef).",
|
||||
)?
|
||||
} else {
|
||||
search.as_bytes().to_vec()
|
||||
}
|
||||
search.as_bytes().to_vec()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
}
|
||||
None => Vec::new(),
|
||||
};
|
||||
|
||||
ListFilter {
|
||||
Ok(ListFilter {
|
||||
skip: self.skip,
|
||||
len: self.len,
|
||||
search,
|
||||
@@ -82,7 +84,7 @@ impl Command {
|
||||
min_value_size: self.min_value_size,
|
||||
reverse: self.reverse,
|
||||
only_count: self.count,
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,7 +117,7 @@ impl<N: NodeTypes> TableViewer<()> for ListTableViewer<'_, N> {
|
||||
}
|
||||
|
||||
|
||||
let list_filter = self.args.list_filter();
|
||||
let list_filter = self.args.list_filter()?;
|
||||
|
||||
if self.args.json || self.args.count {
|
||||
let (list, count) = self.tool.list::<T>(&list_filter)?;
|
||||
|
||||
@@ -39,7 +39,7 @@ pub enum Subcommands {
|
||||
Stats(stats::Command),
|
||||
/// Lists the contents of a table
|
||||
List(list::Command),
|
||||
/// Calculates the content checksum of a table
|
||||
/// Calculates the content checksum of a table or static file segment
|
||||
Checksum(checksum::Command),
|
||||
/// Create a diff between two database tables or two entire databases.
|
||||
Diff(diff::Command),
|
||||
@@ -162,7 +162,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
let access_rights =
|
||||
if command.dry_run { AccessRights::RO } else { AccessRights::RW };
|
||||
db_exec!(self.env, tool, N, access_rights, {
|
||||
command.execute(&tool, ctx.task_executor.clone(), &data_dir)?;
|
||||
command.execute(&tool, ctx.task_executor, &data_dir)?;
|
||||
});
|
||||
}
|
||||
Subcommands::StaticFileHeader(command) => {
|
||||
|
||||
@@ -69,6 +69,11 @@ pub enum SetCommand {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store storage changesets in static files instead of the database
|
||||
StorageChangesets {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
}
|
||||
|
||||
impl Command {
|
||||
@@ -115,6 +120,7 @@ impl Command {
|
||||
transaction_hash_numbers_in_rocksdb: _,
|
||||
account_history_in_rocksdb: _,
|
||||
account_changesets_in_static_files: _,
|
||||
storage_changesets_in_static_files: _,
|
||||
} = settings.unwrap_or_else(StorageSettings::legacy);
|
||||
|
||||
// Update the setting based on the key
|
||||
@@ -167,6 +173,14 @@ impl Command {
|
||||
settings.account_history_in_rocksdb = value;
|
||||
println!("Set account_history_in_rocksdb = {}", value);
|
||||
}
|
||||
SetCommand::StorageChangesets { value } => {
|
||||
if settings.storage_changesets_in_static_files == value {
|
||||
println!("storage_changesets_in_static_files is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.storage_changesets_in_static_files = value;
|
||||
println!("Set storage_changesets_in_static_files = {}", value);
|
||||
}
|
||||
}
|
||||
|
||||
// Write updated settings
|
||||
|
||||
@@ -11,7 +11,10 @@ use reth_db_common::DbTool;
|
||||
use reth_fs_util as fs;
|
||||
use reth_node_builder::{NodePrimitives, NodeTypesWithDB, NodeTypesWithDBAdapter};
|
||||
use reth_node_core::dirs::{ChainPath, DataDirPath};
|
||||
use reth_provider::providers::{ProviderNodeTypes, StaticFileProvider};
|
||||
use reth_provider::{
|
||||
providers::{ProviderNodeTypes, StaticFileProvider},
|
||||
RocksDBProviderFactory,
|
||||
};
|
||||
use reth_static_file_types::SegmentRangeInclusive;
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
@@ -61,6 +64,11 @@ impl Command {
|
||||
let db_stats_table = self.db_stats_table(tool)?;
|
||||
println!("{db_stats_table}");
|
||||
|
||||
println!("\n");
|
||||
|
||||
let rocksdb_stats_table = self.rocksdb_stats_table(tool);
|
||||
println!("{rocksdb_stats_table}");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -148,6 +156,60 @@ impl Command {
|
||||
Ok(table)
|
||||
}
|
||||
|
||||
fn rocksdb_stats_table<N: NodeTypesWithDB>(&self, tool: &DbTool<N>) -> ComfyTable {
|
||||
let mut table = ComfyTable::new();
|
||||
table.load_preset(comfy_table::presets::ASCII_MARKDOWN);
|
||||
table.set_header([
|
||||
"RocksDB Table Name",
|
||||
"# Entries",
|
||||
"SST Size",
|
||||
"Memtable Size",
|
||||
"Total Size",
|
||||
"Pending Compaction",
|
||||
]);
|
||||
|
||||
let stats = tool.provider_factory.rocksdb_provider().table_stats();
|
||||
let mut total_sst: u64 = 0;
|
||||
let mut total_memtable: u64 = 0;
|
||||
let mut total_size: u64 = 0;
|
||||
let mut total_pending: u64 = 0;
|
||||
|
||||
for stat in &stats {
|
||||
total_sst += stat.sst_size_bytes;
|
||||
total_memtable += stat.memtable_size_bytes;
|
||||
total_size += stat.estimated_size_bytes;
|
||||
total_pending += stat.pending_compaction_bytes;
|
||||
let mut row = Row::new();
|
||||
row.add_cell(Cell::new(&stat.name))
|
||||
.add_cell(Cell::new(stat.estimated_num_keys))
|
||||
.add_cell(Cell::new(human_bytes(stat.sst_size_bytes as f64)))
|
||||
.add_cell(Cell::new(human_bytes(stat.memtable_size_bytes as f64)))
|
||||
.add_cell(Cell::new(human_bytes(stat.estimated_size_bytes as f64)))
|
||||
.add_cell(Cell::new(human_bytes(stat.pending_compaction_bytes as f64)));
|
||||
table.add_row(row);
|
||||
}
|
||||
|
||||
if !stats.is_empty() {
|
||||
let max_widths = table.column_max_content_widths();
|
||||
let mut separator = Row::new();
|
||||
for width in max_widths {
|
||||
separator.add_cell(Cell::new("-".repeat(width as usize)));
|
||||
}
|
||||
table.add_row(separator);
|
||||
|
||||
let mut row = Row::new();
|
||||
row.add_cell(Cell::new("RocksDB Total"))
|
||||
.add_cell(Cell::new(""))
|
||||
.add_cell(Cell::new(human_bytes(total_sst as f64)))
|
||||
.add_cell(Cell::new(human_bytes(total_memtable as f64)))
|
||||
.add_cell(Cell::new(human_bytes(total_size as f64)))
|
||||
.add_cell(Cell::new(human_bytes(total_pending as f64)));
|
||||
table.add_row(row);
|
||||
}
|
||||
|
||||
table
|
||||
}
|
||||
|
||||
fn static_files_stats_table<N: NodePrimitives>(
|
||||
&self,
|
||||
data_dir: ChainPath<DataDirPath>,
|
||||
|
||||
@@ -2,14 +2,15 @@ use crate::common::EnvironmentArgs;
|
||||
use clap::Parser;
|
||||
use eyre::Result;
|
||||
use lz4::Decoder;
|
||||
use reqwest::Client;
|
||||
use reqwest::{blocking::Client as BlockingClient, header::RANGE, Client, StatusCode};
|
||||
use reth_chainspec::{EthChainSpec, EthereumHardforks};
|
||||
use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_fs_util as fs;
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
io::{self, Read, Write},
|
||||
path::Path,
|
||||
fs::OpenOptions,
|
||||
io::{self, BufWriter, Read, Write},
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, OnceLock},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
@@ -327,18 +328,158 @@ fn extract_from_file(path: &Path, format: CompressionFormat, target_dir: &Path)
|
||||
extract_archive(file, total_size, format, target_dir)
|
||||
}
|
||||
|
||||
/// Fetches the snapshot from a remote URL, uncompressing it in a streaming fashion.
|
||||
const MAX_DOWNLOAD_RETRIES: u32 = 10;
|
||||
const RETRY_BACKOFF_SECS: u64 = 5;
|
||||
|
||||
/// Wrapper that tracks download progress while writing data.
|
||||
/// Used with [`io::copy`] to display progress during downloads.
|
||||
struct ProgressWriter<W> {
|
||||
inner: W,
|
||||
progress: DownloadProgress,
|
||||
}
|
||||
|
||||
impl<W: Write> Write for ProgressWriter<W> {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
let n = self.inner.write(buf)?;
|
||||
let _ = self.progress.update(n as u64);
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()
|
||||
}
|
||||
}
|
||||
|
||||
/// Downloads a file with resume support using HTTP Range requests.
|
||||
/// Automatically retries on failure, resuming from where it left off.
|
||||
/// Returns the path to the downloaded file and its total size.
|
||||
fn resumable_download(url: &str, target_dir: &Path) -> Result<(PathBuf, u64)> {
|
||||
let file_name = Url::parse(url)
|
||||
.ok()
|
||||
.and_then(|u| u.path_segments()?.next_back().map(|s| s.to_string()))
|
||||
.unwrap_or_else(|| "snapshot.tar".to_string());
|
||||
|
||||
let final_path = target_dir.join(&file_name);
|
||||
let part_path = target_dir.join(format!("{file_name}.part"));
|
||||
|
||||
let client = BlockingClient::builder().timeout(Duration::from_secs(30)).build()?;
|
||||
|
||||
let mut total_size: Option<u64> = None;
|
||||
let mut last_error: Option<eyre::Error> = None;
|
||||
|
||||
for attempt in 1..=MAX_DOWNLOAD_RETRIES {
|
||||
let existing_size = fs::metadata(&part_path).map(|m| m.len()).unwrap_or(0);
|
||||
|
||||
if let Some(total) = total_size &&
|
||||
existing_size >= total
|
||||
{
|
||||
fs::rename(&part_path, &final_path)?;
|
||||
info!(target: "reth::cli", "Download complete: {}", final_path.display());
|
||||
return Ok((final_path, total));
|
||||
}
|
||||
|
||||
if attempt > 1 {
|
||||
info!(target: "reth::cli",
|
||||
"Retry attempt {}/{} - resuming from {} bytes",
|
||||
attempt, MAX_DOWNLOAD_RETRIES, existing_size
|
||||
);
|
||||
}
|
||||
|
||||
let mut request = client.get(url);
|
||||
if existing_size > 0 {
|
||||
request = request.header(RANGE, format!("bytes={existing_size}-"));
|
||||
if attempt == 1 {
|
||||
info!(target: "reth::cli", "Resuming download from {} bytes", existing_size);
|
||||
}
|
||||
}
|
||||
|
||||
let response = match request.send().and_then(|r| r.error_for_status()) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
last_error = Some(e.into());
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
info!(target: "reth::cli",
|
||||
"Download failed, retrying in {} seconds...", RETRY_BACKOFF_SECS
|
||||
);
|
||||
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let is_partial = response.status() == StatusCode::PARTIAL_CONTENT;
|
||||
|
||||
let size = if is_partial {
|
||||
response
|
||||
.headers()
|
||||
.get("Content-Range")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.split('/').next_back())
|
||||
.and_then(|v| v.parse().ok())
|
||||
} else {
|
||||
response.content_length()
|
||||
};
|
||||
|
||||
if total_size.is_none() {
|
||||
total_size = size;
|
||||
}
|
||||
|
||||
let current_total = total_size.ok_or_else(|| {
|
||||
eyre::eyre!("Server did not provide Content-Length or Content-Range header")
|
||||
})?;
|
||||
|
||||
let file = if is_partial && existing_size > 0 {
|
||||
OpenOptions::new()
|
||||
.append(true)
|
||||
.open(&part_path)
|
||||
.map_err(|e| fs::FsPathError::open(e, &part_path))?
|
||||
} else {
|
||||
fs::create_file(&part_path)?
|
||||
};
|
||||
|
||||
let start_offset = if is_partial { existing_size } else { 0 };
|
||||
let mut progress = DownloadProgress::new(current_total);
|
||||
progress.downloaded = start_offset;
|
||||
|
||||
let mut writer = ProgressWriter { inner: BufWriter::new(file), progress };
|
||||
let mut reader = response;
|
||||
|
||||
let copy_result = io::copy(&mut reader, &mut writer);
|
||||
let flush_result = writer.inner.flush();
|
||||
println!();
|
||||
|
||||
if let Err(e) = copy_result.and(flush_result) {
|
||||
last_error = Some(e.into());
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
info!(target: "reth::cli",
|
||||
"Download interrupted, retrying in {} seconds...", RETRY_BACKOFF_SECS
|
||||
);
|
||||
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
fs::rename(&part_path, &final_path)?;
|
||||
info!(target: "reth::cli", "Download complete: {}", final_path.display());
|
||||
return Ok((final_path, current_total));
|
||||
}
|
||||
|
||||
Err(last_error
|
||||
.unwrap_or_else(|| eyre::eyre!("Download failed after {} attempts", MAX_DOWNLOAD_RETRIES)))
|
||||
}
|
||||
|
||||
/// Fetches the snapshot from a remote URL with resume support, then extracts it.
|
||||
fn download_and_extract(url: &str, format: CompressionFormat, target_dir: &Path) -> Result<()> {
|
||||
let client = reqwest::blocking::Client::builder().build()?;
|
||||
let response = client.get(url).send()?.error_for_status()?;
|
||||
let (downloaded_path, total_size) = resumable_download(url, target_dir)?;
|
||||
|
||||
let total_size = response.content_length().ok_or_else(|| {
|
||||
eyre::eyre!(
|
||||
"Server did not provide Content-Length header. This is required for snapshot downloads"
|
||||
)
|
||||
})?;
|
||||
info!(target: "reth::cli", "Extracting snapshot...");
|
||||
let file = fs::open(&downloaded_path)?;
|
||||
extract_archive(file, total_size, format, target_dir)?;
|
||||
|
||||
extract_archive(response, total_size, format, target_dir)
|
||||
fs::remove_file(&downloaded_path)?;
|
||||
info!(target: "reth::cli", "Removed downloaded archive");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Downloads and extracts a snapshot, blocking until finished.
|
||||
|
||||
@@ -10,7 +10,8 @@ use reth_node_builder::NodeBuilder;
|
||||
use reth_node_core::{
|
||||
args::{
|
||||
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, EraArgs, MetricArgs,
|
||||
NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
|
||||
NetworkArgs, PayloadBuilderArgs, PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs,
|
||||
TxPoolArgs,
|
||||
},
|
||||
node_config::NodeConfig,
|
||||
version,
|
||||
@@ -102,6 +103,10 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
|
||||
#[command(flatten)]
|
||||
pub pruning: PruningArgs,
|
||||
|
||||
/// All `RocksDB` table routing arguments
|
||||
#[command(flatten)]
|
||||
pub rocksdb: RocksDbArgs,
|
||||
|
||||
/// Engine cli arguments
|
||||
#[command(flatten, next_help_heading = "Engine")]
|
||||
pub engine: EngineArgs,
|
||||
@@ -166,12 +171,16 @@ where
|
||||
db,
|
||||
dev,
|
||||
pruning,
|
||||
rocksdb,
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
ext,
|
||||
} = self;
|
||||
|
||||
// Validate RocksDB arguments
|
||||
rocksdb.validate()?;
|
||||
|
||||
// set up node config
|
||||
let mut node_config = NodeConfig {
|
||||
datadir,
|
||||
@@ -187,6 +196,7 @@ where
|
||||
db,
|
||||
dev,
|
||||
pruning,
|
||||
rocksdb,
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
|
||||
34
crates/cli/commands/src/p2p/enode.rs
Normal file
34
crates/cli/commands/src/p2p/enode.rs
Normal file
@@ -0,0 +1,34 @@
|
||||
//! Enode identifier command
|
||||
|
||||
use clap::Parser;
|
||||
use reth_cli_util::get_secret_key;
|
||||
use reth_network_peers::NodeRecord;
|
||||
use std::{
|
||||
net::{IpAddr, Ipv4Addr, SocketAddr},
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
/// Print the enode identifier for a given secret key.
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct Command {
|
||||
/// Path to the secret key file for discovery.
|
||||
pub discovery_secret: PathBuf,
|
||||
|
||||
/// Optional IP address to include in the enode URL.
|
||||
///
|
||||
/// If not provided, defaults to 0.0.0.0.
|
||||
#[arg(long)]
|
||||
pub ip: Option<IpAddr>,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute the enode command.
|
||||
pub fn execute(self) -> eyre::Result<()> {
|
||||
let sk = get_secret_key(&self.discovery_secret)?;
|
||||
let ip = self.ip.unwrap_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED));
|
||||
let addr = SocketAddr::new(ip, 30303);
|
||||
let enr = NodeRecord::from_secret_key(addr, &sk);
|
||||
println!("{enr}");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,7 @@ use reth_node_core::{
|
||||
};
|
||||
|
||||
pub mod bootnode;
|
||||
pub mod enode;
|
||||
pub mod rlpx;
|
||||
|
||||
/// `reth p2p` command
|
||||
@@ -85,6 +86,9 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
Subcommands::Bootnode(command) => {
|
||||
command.execute().await?;
|
||||
}
|
||||
Subcommands::Enode(command) => {
|
||||
command.execute()?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -99,6 +103,7 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
Subcommands::Body { args, .. } => Some(&args.chain),
|
||||
Subcommands::Rlpx(_) => None,
|
||||
Subcommands::Bootnode(_) => None,
|
||||
Subcommands::Enode(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -126,6 +131,8 @@ pub enum Subcommands<C: ChainSpecParser> {
|
||||
Rlpx(rlpx::Command),
|
||||
/// Bootnode command
|
||||
Bootnode(bootnode::Command),
|
||||
/// Print enode identifier
|
||||
Enode(enode::Command),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Parser)]
|
||||
@@ -225,4 +232,16 @@ mod tests {
|
||||
let _args: Command<EthereumChainSpecParser> =
|
||||
Command::parse_from(["reth", "body", "--chain", "mainnet", "1000"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_enode_cmd() {
|
||||
let _args: Command<EthereumChainSpecParser> =
|
||||
Command::parse_from(["reth", "enode", "/tmp/secret"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_enode_cmd_with_ip() {
|
||||
let _args: Command<EthereumChainSpecParser> =
|
||||
Command::parse_from(["reth", "enode", "/tmp/secret", "--ip", "192.168.1.1"]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,9 +42,9 @@ pub struct Command<C: ChainSpecParser> {
|
||||
#[arg(long)]
|
||||
to: Option<u64>,
|
||||
|
||||
/// Number of tasks to run in parallel
|
||||
#[arg(long, default_value = "10")]
|
||||
num_tasks: u64,
|
||||
/// Number of tasks to run in parallel. Defaults to the number of available CPUs.
|
||||
#[arg(long)]
|
||||
num_tasks: Option<u64>,
|
||||
|
||||
/// Continues with execution when an invalid block is encountered and collects these blocks.
|
||||
#[arg(long)]
|
||||
@@ -84,12 +84,16 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
}
|
||||
};
|
||||
|
||||
let num_tasks = self.num_tasks.unwrap_or_else(|| {
|
||||
std::thread::available_parallelism().map(|n| n.get() as u64).unwrap_or(10)
|
||||
});
|
||||
|
||||
let total_blocks = max_block - min_block;
|
||||
let total_gas = calculate_gas_used_from_headers(
|
||||
&provider_factory.static_file_provider(),
|
||||
min_block..=max_block,
|
||||
)?;
|
||||
let blocks_per_task = total_blocks / self.num_tasks;
|
||||
let blocks_per_task = total_blocks / num_tasks;
|
||||
|
||||
let db_at = {
|
||||
let provider_factory = provider_factory.clone();
|
||||
@@ -107,10 +111,10 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
let _guard = cancellation.drop_guard();
|
||||
|
||||
let mut tasks = JoinSet::new();
|
||||
for i in 0..self.num_tasks {
|
||||
for i in 0..num_tasks {
|
||||
let start_block = min_block + i * blocks_per_task;
|
||||
let end_block =
|
||||
if i == self.num_tasks - 1 { max_block } else { start_block + blocks_per_task };
|
||||
if i == num_tasks - 1 { max_block } else { start_block + blocks_per_task };
|
||||
|
||||
// Spawn thread executing blocks
|
||||
let provider_factory = provider_factory.clone();
|
||||
@@ -148,7 +152,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
};
|
||||
|
||||
if let Err(err) = consensus
|
||||
.validate_block_post_execution(&block, &result)
|
||||
.validate_block_post_execution(&block, &result, None)
|
||||
.wrap_err_with(|| {
|
||||
format!("Failed to validate block {} {}", block.number(), block.hash())
|
||||
})
|
||||
|
||||
@@ -15,7 +15,8 @@ use reth_db_common::{
|
||||
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
|
||||
use reth_node_core::args::StageEnum;
|
||||
use reth_provider::{
|
||||
DBProvider, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter,
|
||||
DBProvider, RocksDBProviderFactory, StaticFileProviderFactory, StaticFileWriter,
|
||||
StorageSettingsCache,
|
||||
};
|
||||
use reth_prune::PruneSegment;
|
||||
use reth_stages::StageId;
|
||||
@@ -90,11 +91,14 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
StaticFileSegment::AccountChangeSets => {
|
||||
writer.prune_account_changesets(highest_block)?;
|
||||
}
|
||||
StaticFileSegment::StorageChangeSets => {
|
||||
writer.prune_storage_changesets(highest_block)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let provider_rw = tool.provider_factory.database_provider_rw()?;
|
||||
let provider_rw = tool.provider_factory.unwind_provider_rw()?;
|
||||
let tx = provider_rw.tx_ref();
|
||||
|
||||
match self.stage {
|
||||
@@ -168,8 +172,20 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
)?;
|
||||
}
|
||||
StageEnum::AccountHistory | StageEnum::StorageHistory => {
|
||||
tx.clear::<tables::AccountsHistory>()?;
|
||||
tx.clear::<tables::StoragesHistory>()?;
|
||||
let settings = provider_rw.cached_storage_settings();
|
||||
let rocksdb = tool.provider_factory.rocksdb_provider();
|
||||
|
||||
if settings.account_history_in_rocksdb {
|
||||
rocksdb.clear::<tables::AccountsHistory>()?;
|
||||
} else {
|
||||
tx.clear::<tables::AccountsHistory>()?;
|
||||
}
|
||||
|
||||
if settings.storages_history_in_rocksdb {
|
||||
rocksdb.clear::<tables::StoragesHistory>()?;
|
||||
} else {
|
||||
tx.clear::<tables::StoragesHistory>()?;
|
||||
}
|
||||
|
||||
reset_stage_checkpoint(tx, StageId::IndexAccountHistory)?;
|
||||
reset_stage_checkpoint(tx, StageId::IndexStorageHistory)?;
|
||||
@@ -177,7 +193,14 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
insert_genesis_history(&provider_rw, self.env.chain.genesis().alloc.iter())?;
|
||||
}
|
||||
StageEnum::TxLookup => {
|
||||
tx.clear::<tables::TransactionHashNumbers>()?;
|
||||
if provider_rw.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
|
||||
tool.provider_factory
|
||||
.rocksdb_provider()
|
||||
.clear::<tables::TransactionHashNumbers>()?;
|
||||
} else {
|
||||
tx.clear::<tables::TransactionHashNumbers>()?;
|
||||
}
|
||||
|
||||
reset_prune_checkpoint(tx, PruneSegment::TransactionLookup)?;
|
||||
|
||||
reset_stage_checkpoint(tx, StageId::TransactionLookup)?;
|
||||
|
||||
@@ -121,7 +121,16 @@ pub fn install() {
|
||||
unsafe {
|
||||
let alt_stack_size: usize = min_sigstack_size() + 64 * 1024;
|
||||
let mut alt_stack: libc::stack_t = mem::zeroed();
|
||||
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, 1).unwrap()).cast();
|
||||
// Both SysV AMD64 ABI and aarch64 ABI require 16 bytes alignment. We are going to be
|
||||
// generous here and just use a size of a page.
|
||||
let raw_page_sz = libc::sysconf(libc::_SC_PAGESIZE);
|
||||
let page_sz = if raw_page_sz == -1 {
|
||||
// Fallback alignment in case sysconf fails.
|
||||
4096_usize
|
||||
} else {
|
||||
raw_page_sz as usize
|
||||
};
|
||||
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, page_sz).unwrap()).cast();
|
||||
alt_stack.ss_size = alt_stack_size;
|
||||
libc::sigaltstack(&raw const alt_stack, ptr::null_mut());
|
||||
|
||||
|
||||
@@ -438,6 +438,8 @@ pub struct BlocksPerFileConfig {
|
||||
pub transaction_senders: Option<u64>,
|
||||
/// Number of blocks per file for the account changesets segment.
|
||||
pub account_change_sets: Option<u64>,
|
||||
/// Number of blocks per file for the storage changesets segment.
|
||||
pub storage_change_sets: Option<u64>,
|
||||
}
|
||||
|
||||
impl StaticFilesConfig {
|
||||
@@ -451,6 +453,7 @@ impl StaticFilesConfig {
|
||||
receipts,
|
||||
transaction_senders,
|
||||
account_change_sets,
|
||||
storage_change_sets,
|
||||
} = self.blocks_per_file;
|
||||
eyre::ensure!(headers != Some(0), "Headers segment blocks per file must be greater than 0");
|
||||
eyre::ensure!(
|
||||
@@ -469,6 +472,10 @@ impl StaticFilesConfig {
|
||||
account_change_sets != Some(0),
|
||||
"Account changesets segment blocks per file must be greater than 0"
|
||||
);
|
||||
eyre::ensure!(
|
||||
storage_change_sets != Some(0),
|
||||
"Storage changesets segment blocks per file must be greater than 0"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -480,6 +487,7 @@ impl StaticFilesConfig {
|
||||
receipts,
|
||||
transaction_senders,
|
||||
account_change_sets,
|
||||
storage_change_sets,
|
||||
} = self.blocks_per_file;
|
||||
|
||||
let mut map = StaticFileMap::default();
|
||||
@@ -492,6 +500,7 @@ impl StaticFilesConfig {
|
||||
StaticFileSegment::Receipts => receipts,
|
||||
StaticFileSegment::TransactionSenders => transaction_senders,
|
||||
StaticFileSegment::AccountChangeSets => account_change_sets,
|
||||
StaticFileSegment::StorageChangeSets => storage_change_sets,
|
||||
};
|
||||
|
||||
if let Some(blocks_per_file) = blocks_per_file {
|
||||
|
||||
@@ -15,6 +15,12 @@ use alloc::{boxed::Box, fmt::Debug, string::String, sync::Arc, vec::Vec};
|
||||
use alloy_consensus::Header;
|
||||
use alloy_primitives::{BlockHash, BlockNumber, Bloom, B256};
|
||||
use core::error::Error;
|
||||
|
||||
/// Pre-computed receipt root and logs bloom.
|
||||
///
|
||||
/// When provided to [`FullConsensus::validate_block_post_execution`], this allows skipping
|
||||
/// the receipt root computation and using the pre-computed values instead.
|
||||
pub type ReceiptRootBloom = (B256, Bloom);
|
||||
use reth_execution_types::BlockExecutionResult;
|
||||
use reth_primitives_traits::{
|
||||
constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK, MINIMUM_GAS_LIMIT},
|
||||
@@ -39,11 +45,15 @@ pub trait FullConsensus<N: NodePrimitives>: Consensus<N::Block> {
|
||||
///
|
||||
/// See the Yellow Paper sections 4.3.2 "Holistic Validity".
|
||||
///
|
||||
/// If `receipt_root_bloom` is provided, the implementation should use the pre-computed
|
||||
/// receipt root and logs bloom instead of computing them from the receipts.
|
||||
///
|
||||
/// Note: validating blocks does not include other validations of the Consensus
|
||||
fn validate_block_post_execution(
|
||||
&self,
|
||||
block: &RecoveredBlock<N::Block>,
|
||||
result: &BlockExecutionResult<N::Receipt>,
|
||||
receipt_root_bloom: Option<ReceiptRootBloom>,
|
||||
) -> Result<(), ConsensusError>;
|
||||
}
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
//!
|
||||
//! **Not for production use** - provides no security guarantees or consensus validation.
|
||||
|
||||
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator};
|
||||
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator, ReceiptRootBloom};
|
||||
use alloc::sync::Arc;
|
||||
use reth_execution_types::BlockExecutionResult;
|
||||
use reth_primitives_traits::{Block, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
|
||||
@@ -76,6 +76,7 @@ impl<N: NodePrimitives> FullConsensus<N> for NoopConsensus {
|
||||
&self,
|
||||
_block: &RecoveredBlock<N::Block>,
|
||||
_result: &BlockExecutionResult<N::Receipt>,
|
||||
_receipt_root_bloom: Option<ReceiptRootBloom>,
|
||||
) -> Result<(), ConsensusError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator};
|
||||
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator, ReceiptRootBloom};
|
||||
use core::sync::atomic::{AtomicBool, Ordering};
|
||||
use reth_execution_types::BlockExecutionResult;
|
||||
use reth_primitives_traits::{Block, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
|
||||
@@ -51,6 +51,7 @@ impl<N: NodePrimitives> FullConsensus<N> for TestConsensus {
|
||||
&self,
|
||||
_block: &RecoveredBlock<N::Block>,
|
||||
_result: &BlockExecutionResult<N::Receipt>,
|
||||
_receipt_root_bloom: Option<ReceiptRootBloom>,
|
||||
) -> Result<(), ConsensusError> {
|
||||
if self.fail_validation() {
|
||||
Err(ConsensusError::BaseFeeMissing)
|
||||
|
||||
@@ -448,12 +448,14 @@ mod tests {
|
||||
nonce: account.nonce,
|
||||
code_hash: account.bytecode_hash.unwrap_or_default(),
|
||||
code: None,
|
||||
account_id: None,
|
||||
}),
|
||||
original_info: (i == 0).then(|| AccountInfo {
|
||||
balance: account.balance.checked_div(U256::from(2)).unwrap_or(U256::ZERO),
|
||||
nonce: 0,
|
||||
code_hash: account.bytecode_hash.unwrap_or_default(),
|
||||
code: None,
|
||||
account_id: None,
|
||||
}),
|
||||
storage,
|
||||
status: AccountStatus::default(),
|
||||
|
||||
@@ -34,6 +34,11 @@ fn default_account_worker_count() -> usize {
|
||||
/// The size of proof targets chunk to spawn in one multiproof calculation.
|
||||
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE: usize = 60;
|
||||
|
||||
/// The size of proof targets chunk to spawn in one multiproof calculation when V2 proofs are
|
||||
/// enabled. This is 4x the default chunk size to take advantage of more efficient V2 proof
|
||||
/// computation.
|
||||
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2: usize = DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE * 4;
|
||||
|
||||
/// Default number of reserved CPU cores for non-reth processes.
|
||||
///
|
||||
/// This will be deducted from the thread count of main reth global threadpool.
|
||||
@@ -137,6 +142,8 @@ pub struct TreeConfig {
|
||||
account_worker_count: usize,
|
||||
/// Whether to enable V2 storage proofs.
|
||||
enable_proof_v2: bool,
|
||||
/// Whether to disable cache metrics recording (can be expensive with large cached state).
|
||||
disable_cache_metrics: bool,
|
||||
}
|
||||
|
||||
impl Default for TreeConfig {
|
||||
@@ -166,6 +173,7 @@ impl Default for TreeConfig {
|
||||
storage_worker_count: default_storage_worker_count(),
|
||||
account_worker_count: default_account_worker_count(),
|
||||
enable_proof_v2: false,
|
||||
disable_cache_metrics: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -198,6 +206,7 @@ impl TreeConfig {
|
||||
storage_worker_count: usize,
|
||||
account_worker_count: usize,
|
||||
enable_proof_v2: bool,
|
||||
disable_cache_metrics: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
persistence_threshold,
|
||||
@@ -224,6 +233,7 @@ impl TreeConfig {
|
||||
storage_worker_count,
|
||||
account_worker_count,
|
||||
enable_proof_v2,
|
||||
disable_cache_metrics,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -262,6 +272,17 @@ impl TreeConfig {
|
||||
self.multiproof_chunk_size
|
||||
}
|
||||
|
||||
/// Return the multiproof task chunk size, using the V2 default if V2 proofs are enabled
|
||||
/// and the chunk size is at the default value.
|
||||
pub const fn effective_multiproof_chunk_size(&self) -> usize {
|
||||
if self.enable_proof_v2 && self.multiproof_chunk_size == DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE
|
||||
{
|
||||
DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2
|
||||
} else {
|
||||
self.multiproof_chunk_size
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the number of reserved CPU cores for non-reth processes
|
||||
pub const fn reserved_cpu_cores(&self) -> usize {
|
||||
self.reserved_cpu_cores
|
||||
@@ -516,4 +537,15 @@ impl TreeConfig {
|
||||
self.enable_proof_v2 = enable_proof_v2;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether cache metrics recording is disabled.
|
||||
pub const fn disable_cache_metrics(&self) -> bool {
|
||||
self.disable_cache_metrics
|
||||
}
|
||||
|
||||
/// Setter for whether to disable cache metrics recording.
|
||||
pub const fn without_cache_metrics(mut self, disable_cache_metrics: bool) -> Self {
|
||||
self.disable_cache_metrics = disable_cache_metrics;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
@@ -62,7 +62,8 @@ pub trait EngineTypes:
|
||||
+ TryInto<Self::ExecutionPayloadEnvelopeV2>
|
||||
+ TryInto<Self::ExecutionPayloadEnvelopeV3>
|
||||
+ TryInto<Self::ExecutionPayloadEnvelopeV4>
|
||||
+ TryInto<Self::ExecutionPayloadEnvelopeV5>,
|
||||
+ TryInto<Self::ExecutionPayloadEnvelopeV5>
|
||||
+ TryInto<Self::ExecutionPayloadEnvelopeV6>,
|
||||
> + DeserializeOwned
|
||||
+ Serialize
|
||||
{
|
||||
@@ -106,6 +107,14 @@ pub trait EngineTypes:
|
||||
+ Send
|
||||
+ Sync
|
||||
+ 'static;
|
||||
/// Execution Payload V6 envelope type.
|
||||
type ExecutionPayloadEnvelopeV6: DeserializeOwned
|
||||
+ Serialize
|
||||
+ Clone
|
||||
+ Unpin
|
||||
+ Send
|
||||
+ Sync
|
||||
+ 'static;
|
||||
}
|
||||
|
||||
/// Type that validates the payloads processed by the engine API.
|
||||
|
||||
@@ -12,7 +12,7 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
# reth
|
||||
reth-chain-state.workspace = true
|
||||
reth-chain-state = { workspace = true, features = ["rayon"] }
|
||||
reth-chainspec = { workspace = true, optional = true }
|
||||
reth-consensus.workspace = true
|
||||
reth-db.workspace = true
|
||||
@@ -34,6 +34,7 @@ reth-trie-parallel.workspace = true
|
||||
reth-trie-sparse = { workspace = true, features = ["std", "metrics"] }
|
||||
reth-trie-sparse-parallel = { workspace = true, features = ["std"] }
|
||||
reth-trie.workspace = true
|
||||
reth-trie-common.workspace = true
|
||||
reth-trie-db.workspace = true
|
||||
|
||||
# alloy
|
||||
@@ -95,7 +96,7 @@ reth-tracing.workspace = true
|
||||
reth-node-ethereum.workspace = true
|
||||
reth-e2e-test-utils.workspace = true
|
||||
|
||||
# alloy
|
||||
# revm
|
||||
revm-state.workspace = true
|
||||
|
||||
assert_matches.workspace = true
|
||||
@@ -134,6 +135,7 @@ test-utils = [
|
||||
"reth-static-file",
|
||||
"reth-tracing",
|
||||
"reth-trie/test-utils",
|
||||
"reth-trie-common/test-utils",
|
||||
"reth-trie-db/test-utils",
|
||||
"reth-trie-sparse/test-utils",
|
||||
"reth-prune-types?/test-utils",
|
||||
|
||||
@@ -26,7 +26,9 @@ fn create_bench_state(num_accounts: usize) -> EvmState {
|
||||
nonce: 10,
|
||||
code_hash: B256::from_slice(&rng.random::<[u8; 32]>()),
|
||||
code: Default::default(),
|
||||
account_id: None,
|
||||
},
|
||||
original_info: Box::new(AccountInfo::default()),
|
||||
storage,
|
||||
status: AccountStatus::empty(),
|
||||
transaction_id: 0,
|
||||
|
||||
@@ -62,6 +62,7 @@ fn create_bench_state_updates(params: &BenchParams) -> Vec<EvmState> {
|
||||
storage: HashMap::default(),
|
||||
status: AccountStatus::SelfDestructed,
|
||||
transaction_id: 0,
|
||||
original_info: Box::new(AccountInfo::default()),
|
||||
}
|
||||
} else {
|
||||
RevmAccount {
|
||||
@@ -70,6 +71,7 @@ fn create_bench_state_updates(params: &BenchParams) -> Vec<EvmState> {
|
||||
nonce: rng.random::<u64>(),
|
||||
code_hash: KECCAK_EMPTY,
|
||||
code: Some(Default::default()),
|
||||
account_id: None,
|
||||
},
|
||||
storage: (0..rng.random_range(0..=params.storage_slots_per_account))
|
||||
.map(|_| {
|
||||
@@ -84,6 +86,7 @@ fn create_bench_state_updates(params: &BenchParams) -> Vec<EvmState> {
|
||||
})
|
||||
.collect(),
|
||||
status: AccountStatus::Touched,
|
||||
original_info: Box::new(AccountInfo::default()),
|
||||
transaction_id: 0,
|
||||
}
|
||||
};
|
||||
|
||||
@@ -606,12 +606,21 @@ pub(crate) struct SavedCache {
|
||||
/// A guard to track in-flight usage of this cache.
|
||||
/// The cache is considered available if the strong count is 1.
|
||||
usage_guard: Arc<()>,
|
||||
|
||||
/// Whether to skip cache metrics recording (can be expensive with large cached state).
|
||||
disable_cache_metrics: bool,
|
||||
}
|
||||
|
||||
impl SavedCache {
|
||||
/// Creates a new instance with the internals
|
||||
pub(super) fn new(hash: B256, caches: ExecutionCache, metrics: CachedStateMetrics) -> Self {
|
||||
Self { hash, caches, metrics, usage_guard: Arc::new(()) }
|
||||
Self { hash, caches, metrics, usage_guard: Arc::new(()), disable_cache_metrics: false }
|
||||
}
|
||||
|
||||
/// Sets whether to disable cache metrics recording.
|
||||
pub(super) const fn with_disable_cache_metrics(mut self, disable: bool) -> Self {
|
||||
self.disable_cache_metrics = disable;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns the hash for this cache
|
||||
@@ -619,9 +628,9 @@ impl SavedCache {
|
||||
self.hash
|
||||
}
|
||||
|
||||
/// Splits the cache into its caches and metrics, consuming it.
|
||||
pub(crate) fn split(self) -> (ExecutionCache, CachedStateMetrics) {
|
||||
(self.caches, self.metrics)
|
||||
/// Splits the cache into its caches, metrics, and `disable_cache_metrics` flag, consuming it.
|
||||
pub(crate) fn split(self) -> (ExecutionCache, CachedStateMetrics, bool) {
|
||||
(self.caches, self.metrics, self.disable_cache_metrics)
|
||||
}
|
||||
|
||||
/// Returns true if the cache is available for use (no other tasks are currently using it).
|
||||
@@ -645,7 +654,13 @@ impl SavedCache {
|
||||
}
|
||||
|
||||
/// Updates the metrics for the [`ExecutionCache`].
|
||||
///
|
||||
/// Note: This can be expensive with large cached state as it iterates over
|
||||
/// all storage entries. Use `with_disable_cache_metrics(true)` to skip.
|
||||
pub(crate) fn update_metrics(&self) {
|
||||
if self.disable_cache_metrics {
|
||||
return;
|
||||
}
|
||||
self.metrics.storage_cache_size.set(self.caches.total_storage_slots() as f64);
|
||||
self.metrics.account_cache_size.set(self.caches.account_cache.entry_count() as f64);
|
||||
self.metrics.code_cache_size.set(self.caches.code_cache.entry_count() as f64);
|
||||
|
||||
@@ -1,25 +1,15 @@
|
||||
use crate::tree::{error::InsertBlockFatalError, MeteredStateHook, TreeOutcome};
|
||||
use alloy_consensus::transaction::TxHashRef;
|
||||
use alloy_evm::{
|
||||
block::{BlockExecutor, ExecutableTx},
|
||||
Evm,
|
||||
};
|
||||
use crate::tree::{error::InsertBlockFatalError, TreeOutcome};
|
||||
use alloy_rpc_types_engine::{PayloadStatus, PayloadStatusEnum};
|
||||
use core::borrow::BorrowMut;
|
||||
use reth_engine_primitives::{ForkchoiceStatus, OnForkChoiceUpdated};
|
||||
use reth_errors::{BlockExecutionError, ProviderError};
|
||||
use reth_evm::{metrics::ExecutorMetrics, OnStateHook};
|
||||
use reth_errors::ProviderError;
|
||||
use reth_evm::metrics::ExecutorMetrics;
|
||||
use reth_execution_types::BlockExecutionOutput;
|
||||
use reth_metrics::{
|
||||
metrics::{Counter, Gauge, Histogram},
|
||||
Metrics,
|
||||
};
|
||||
use reth_primitives_traits::SignedTransaction;
|
||||
use reth_trie::updates::TrieUpdates;
|
||||
use revm::database::{states::bundle_state::BundleRetention, State};
|
||||
use revm_primitives::Address;
|
||||
use std::time::Instant;
|
||||
use tracing::{debug_span, trace};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Metrics for the `EngineApi`.
|
||||
#[derive(Debug, Default)]
|
||||
@@ -35,101 +25,24 @@ pub(crate) struct EngineApiMetrics {
|
||||
}
|
||||
|
||||
impl EngineApiMetrics {
|
||||
/// Helper function for metered execution
|
||||
fn metered<F, R>(&self, f: F) -> R
|
||||
where
|
||||
F: FnOnce() -> (u64, R),
|
||||
{
|
||||
// Execute the block and record the elapsed time.
|
||||
let execute_start = Instant::now();
|
||||
let (gas_used, output) = f();
|
||||
let execution_duration = execute_start.elapsed().as_secs_f64();
|
||||
|
||||
// Update gas metrics.
|
||||
self.executor.gas_processed_total.increment(gas_used);
|
||||
self.executor.gas_per_second.set(gas_used as f64 / execution_duration);
|
||||
self.executor.gas_used_histogram.record(gas_used as f64);
|
||||
self.executor.execution_histogram.record(execution_duration);
|
||||
self.executor.execution_duration.set(execution_duration);
|
||||
|
||||
output
|
||||
}
|
||||
|
||||
/// Execute the given block using the provided [`BlockExecutor`] and update metrics for the
|
||||
/// execution.
|
||||
/// Records metrics for block execution.
|
||||
///
|
||||
/// This method updates metrics for execution time, gas usage, and the number
|
||||
/// of accounts, storage slots and bytecodes loaded and updated.
|
||||
pub(crate) fn execute_metered<E, DB>(
|
||||
/// of accounts, storage slots and bytecodes updated.
|
||||
pub(crate) fn record_block_execution<R>(
|
||||
&self,
|
||||
executor: E,
|
||||
mut transactions: impl Iterator<Item = Result<impl ExecutableTx<E>, BlockExecutionError>>,
|
||||
transaction_count: usize,
|
||||
state_hook: Box<dyn OnStateHook>,
|
||||
) -> Result<(BlockExecutionOutput<E::Receipt>, Vec<Address>), BlockExecutionError>
|
||||
where
|
||||
DB: alloy_evm::Database,
|
||||
E: BlockExecutor<Evm: Evm<DB: BorrowMut<State<DB>>>, Transaction: SignedTransaction>,
|
||||
{
|
||||
// clone here is cheap, all the metrics are Option<Arc<_>>. additionally
|
||||
// they are globally registered so that the data recorded in the hook will
|
||||
// be accessible.
|
||||
let wrapper = MeteredStateHook { metrics: self.executor.clone(), inner_hook: state_hook };
|
||||
output: &BlockExecutionOutput<R>,
|
||||
execution_duration: Duration,
|
||||
) {
|
||||
let execution_secs = execution_duration.as_secs_f64();
|
||||
let gas_used = output.result.gas_used;
|
||||
|
||||
let mut senders = Vec::with_capacity(transaction_count);
|
||||
let mut executor = executor.with_state_hook(Some(Box::new(wrapper)));
|
||||
|
||||
let f = || {
|
||||
let start = Instant::now();
|
||||
debug_span!(target: "engine::tree", "pre execution")
|
||||
.entered()
|
||||
.in_scope(|| executor.apply_pre_execution_changes())?;
|
||||
self.executor.pre_execution_histogram.record(start.elapsed());
|
||||
|
||||
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
|
||||
loop {
|
||||
let start = Instant::now();
|
||||
let Some(tx) = transactions.next() else { break };
|
||||
self.executor.transaction_wait_histogram.record(start.elapsed());
|
||||
|
||||
let tx = tx?;
|
||||
senders.push(*tx.signer());
|
||||
|
||||
let span =
|
||||
debug_span!(target: "engine::tree", "execute tx", tx_hash=?tx.tx().tx_hash());
|
||||
let enter = span.entered();
|
||||
trace!(target: "engine::tree", "Executing transaction");
|
||||
let start = Instant::now();
|
||||
let gas_used = executor.execute_transaction(tx)?;
|
||||
self.executor.transaction_execution_histogram.record(start.elapsed());
|
||||
|
||||
// record the tx gas used
|
||||
enter.record("gas_used", gas_used);
|
||||
}
|
||||
drop(exec_span);
|
||||
|
||||
let start = Instant::now();
|
||||
let result = debug_span!(target: "engine::tree", "finish")
|
||||
.entered()
|
||||
.in_scope(|| executor.finish())
|
||||
.map(|(evm, result)| (evm.into_db(), result));
|
||||
self.executor.post_execution_histogram.record(start.elapsed());
|
||||
|
||||
result
|
||||
};
|
||||
|
||||
// Use metered to execute and track timing/gas metrics
|
||||
let (mut db, result) = self.metered(|| {
|
||||
let res = f();
|
||||
let gas_used = res.as_ref().map(|r| r.1.gas_used).unwrap_or(0);
|
||||
(gas_used, res)
|
||||
})?;
|
||||
|
||||
// merge transitions into bundle state
|
||||
debug_span!(target: "engine::tree", "merge transitions")
|
||||
.entered()
|
||||
.in_scope(|| db.borrow_mut().merge_transitions(BundleRetention::Reverts));
|
||||
let output = BlockExecutionOutput { result, state: db.borrow_mut().take_bundle() };
|
||||
// Update gas metrics
|
||||
self.executor.gas_processed_total.increment(gas_used);
|
||||
self.executor.gas_per_second.set(gas_used as f64 / execution_secs);
|
||||
self.executor.gas_used_histogram.record(gas_used as f64);
|
||||
self.executor.execution_histogram.record(execution_secs);
|
||||
self.executor.execution_duration.set(execution_secs);
|
||||
|
||||
// Update the metrics for the number of accounts, storage slots and bytecodes updated
|
||||
let accounts = output.state.state.len();
|
||||
@@ -140,8 +53,31 @@ impl EngineApiMetrics {
|
||||
self.executor.accounts_updated_histogram.record(accounts as f64);
|
||||
self.executor.storage_slots_updated_histogram.record(storage_slots as f64);
|
||||
self.executor.bytecodes_updated_histogram.record(bytecodes as f64);
|
||||
}
|
||||
|
||||
Ok((output, senders))
|
||||
/// Returns a reference to the executor metrics for use in state hooks.
|
||||
pub(crate) const fn executor_metrics(&self) -> &ExecutorMetrics {
|
||||
&self.executor
|
||||
}
|
||||
|
||||
/// Records the duration of block pre-execution changes (e.g., beacon root update).
|
||||
pub(crate) fn record_pre_execution(&self, elapsed: Duration) {
|
||||
self.executor.pre_execution_histogram.record(elapsed);
|
||||
}
|
||||
|
||||
/// Records the duration of block post-execution changes (e.g., finalization).
|
||||
pub(crate) fn record_post_execution(&self, elapsed: Duration) {
|
||||
self.executor.post_execution_histogram.record(elapsed);
|
||||
}
|
||||
|
||||
/// Records the time spent waiting for the next transaction from the iterator.
|
||||
pub(crate) fn record_transaction_wait(&self, elapsed: Duration) {
|
||||
self.executor.transaction_wait_histogram.record(elapsed);
|
||||
}
|
||||
|
||||
/// Records the duration of a single transaction execution.
|
||||
pub(crate) fn record_transaction_execution(&self, elapsed: Duration) {
|
||||
self.executor.transaction_execution_histogram.record(elapsed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -197,6 +133,12 @@ pub(crate) struct EngineMetrics {
|
||||
#[derive(Metrics)]
|
||||
#[metrics(scope = "consensus.engine.beacon")]
|
||||
pub(crate) struct ForkchoiceUpdatedMetrics {
|
||||
/// Finish time of the latest forkchoice updated call.
|
||||
#[metric(skip)]
|
||||
pub(crate) latest_finish_at: Option<Instant>,
|
||||
/// Start time of the latest forkchoice updated call.
|
||||
#[metric(skip)]
|
||||
pub(crate) latest_start_at: Option<Instant>,
|
||||
/// The total count of forkchoice updated messages received.
|
||||
pub(crate) forkchoice_updated_messages: Counter,
|
||||
/// The total count of forkchoice updated messages with payload received.
|
||||
@@ -219,18 +161,35 @@ pub(crate) struct ForkchoiceUpdatedMetrics {
|
||||
pub(crate) forkchoice_updated_last: Gauge,
|
||||
/// Time diff between new payload call response and the next forkchoice updated call request.
|
||||
pub(crate) new_payload_forkchoice_updated_time_diff: Histogram,
|
||||
/// Time from previous forkchoice updated finish to current forkchoice updated start (idle
|
||||
/// time).
|
||||
pub(crate) time_between_forkchoice_updated: Histogram,
|
||||
/// Time from previous forkchoice updated start to current forkchoice updated start (total
|
||||
/// interval).
|
||||
pub(crate) forkchoice_updated_interval: Histogram,
|
||||
}
|
||||
|
||||
impl ForkchoiceUpdatedMetrics {
|
||||
/// Increment the forkchoiceUpdated counter based on the given result
|
||||
pub(crate) fn update_response_metrics(
|
||||
&self,
|
||||
&mut self,
|
||||
start: Instant,
|
||||
latest_new_payload_at: &mut Option<Instant>,
|
||||
has_attrs: bool,
|
||||
result: &Result<TreeOutcome<OnForkChoiceUpdated>, ProviderError>,
|
||||
) {
|
||||
let elapsed = start.elapsed();
|
||||
let finish = Instant::now();
|
||||
let elapsed = finish - start;
|
||||
|
||||
if let Some(prev_finish) = self.latest_finish_at {
|
||||
self.time_between_forkchoice_updated.record(start - prev_finish);
|
||||
}
|
||||
if let Some(prev_start) = self.latest_start_at {
|
||||
self.forkchoice_updated_interval.record(start - prev_start);
|
||||
}
|
||||
self.latest_finish_at = Some(finish);
|
||||
self.latest_start_at = Some(start);
|
||||
|
||||
match result {
|
||||
Ok(outcome) => match outcome.outcome.forkchoice_status() {
|
||||
ForkchoiceStatus::Valid => self.forkchoice_updated_valid.increment(1),
|
||||
@@ -257,7 +216,10 @@ impl ForkchoiceUpdatedMetrics {
|
||||
pub(crate) struct NewPayloadStatusMetrics {
|
||||
/// Finish time of the latest new payload call.
|
||||
#[metric(skip)]
|
||||
pub(crate) latest_at: Option<Instant>,
|
||||
pub(crate) latest_finish_at: Option<Instant>,
|
||||
/// Start time of the latest new payload call.
|
||||
#[metric(skip)]
|
||||
pub(crate) latest_start_at: Option<Instant>,
|
||||
/// The total count of new payload messages received.
|
||||
pub(crate) new_payload_messages: Counter,
|
||||
/// The total count of new payload messages that we responded to with
|
||||
@@ -285,6 +247,10 @@ pub(crate) struct NewPayloadStatusMetrics {
|
||||
pub(crate) new_payload_latency: Histogram,
|
||||
/// Latency for the last new payload call.
|
||||
pub(crate) new_payload_last: Gauge,
|
||||
/// Time from previous payload finish to current payload start (idle time).
|
||||
pub(crate) time_between_new_payloads: Histogram,
|
||||
/// Time from previous payload start to current payload start (total interval).
|
||||
pub(crate) new_payload_interval: Histogram,
|
||||
}
|
||||
|
||||
impl NewPayloadStatusMetrics {
|
||||
@@ -298,7 +264,14 @@ impl NewPayloadStatusMetrics {
|
||||
let finish = Instant::now();
|
||||
let elapsed = finish - start;
|
||||
|
||||
self.latest_at = Some(finish);
|
||||
if let Some(prev_finish) = self.latest_finish_at {
|
||||
self.time_between_new_payloads.record(start - prev_finish);
|
||||
}
|
||||
if let Some(prev_start) = self.latest_start_at {
|
||||
self.new_payload_interval.record(start - prev_start);
|
||||
}
|
||||
self.latest_finish_at = Some(finish);
|
||||
self.latest_start_at = Some(start);
|
||||
match result {
|
||||
Ok(outcome) => match outcome.outcome.status {
|
||||
PayloadStatusEnum::Valid => {
|
||||
@@ -334,10 +307,6 @@ pub(crate) struct BlockValidationMetrics {
|
||||
pub(crate) state_root_histogram: Histogram,
|
||||
/// Histogram of deferred trie computation duration.
|
||||
pub(crate) deferred_trie_compute_duration: Histogram,
|
||||
/// Histogram of time spent waiting for deferred trie data to become available.
|
||||
pub(crate) deferred_trie_wait_duration: Histogram,
|
||||
/// Trie input computation duration
|
||||
pub(crate) trie_input_duration: Histogram,
|
||||
/// Payload conversion and validation latency
|
||||
pub(crate) payload_validation_duration: Gauge,
|
||||
/// Histogram of payload validation latency
|
||||
@@ -387,133 +356,10 @@ pub(crate) struct BlockBufferMetrics {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_eips::eip7685::Requests;
|
||||
use alloy_evm::block::StateChangeSource;
|
||||
use alloy_primitives::{B256, U256};
|
||||
use metrics_util::debugging::{DebuggingRecorder, Snapshotter};
|
||||
use reth_ethereum_primitives::{Receipt, TransactionSigned};
|
||||
use reth_evm_ethereum::EthEvm;
|
||||
use reth_ethereum_primitives::Receipt;
|
||||
use reth_execution_types::BlockExecutionResult;
|
||||
use reth_primitives_traits::RecoveredBlock;
|
||||
use revm::{
|
||||
context::result::{ExecutionResult, Output, ResultAndState, SuccessReason},
|
||||
database::State,
|
||||
database_interface::EmptyDB,
|
||||
inspector::NoOpInspector,
|
||||
state::{Account, AccountInfo, AccountStatus, EvmState, EvmStorage, EvmStorageSlot},
|
||||
Context, MainBuilder, MainContext,
|
||||
};
|
||||
use revm_primitives::Bytes;
|
||||
use std::sync::mpsc;
|
||||
|
||||
/// A simple mock executor for testing that doesn't require complex EVM setup
|
||||
struct MockExecutor {
|
||||
state: EvmState,
|
||||
hook: Option<Box<dyn OnStateHook>>,
|
||||
}
|
||||
|
||||
impl MockExecutor {
|
||||
fn new(state: EvmState) -> Self {
|
||||
Self { state, hook: None }
|
||||
}
|
||||
}
|
||||
|
||||
// Mock Evm type for testing
|
||||
type MockEvm = EthEvm<State<EmptyDB>, NoOpInspector>;
|
||||
|
||||
impl BlockExecutor for MockExecutor {
|
||||
type Transaction = TransactionSigned;
|
||||
type Receipt = Receipt;
|
||||
type Evm = MockEvm;
|
||||
|
||||
fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn execute_transaction_without_commit(
|
||||
&mut self,
|
||||
_tx: impl ExecutableTx<Self>,
|
||||
) -> Result<ResultAndState<<Self::Evm as Evm>::HaltReason>, BlockExecutionError> {
|
||||
// Call hook with our mock state for each transaction
|
||||
if let Some(hook) = self.hook.as_mut() {
|
||||
hook.on_state(StateChangeSource::Transaction(0), &self.state);
|
||||
}
|
||||
|
||||
Ok(ResultAndState::new(
|
||||
ExecutionResult::Success {
|
||||
reason: SuccessReason::Return,
|
||||
gas_used: 1000, // Mock gas used
|
||||
gas_refunded: 0,
|
||||
logs: vec![],
|
||||
output: Output::Call(Bytes::from(vec![])),
|
||||
},
|
||||
Default::default(),
|
||||
))
|
||||
}
|
||||
|
||||
fn commit_transaction(
|
||||
&mut self,
|
||||
_output: ResultAndState<<Self::Evm as Evm>::HaltReason>,
|
||||
_tx: impl ExecutableTx<Self>,
|
||||
) -> Result<u64, BlockExecutionError> {
|
||||
Ok(1000)
|
||||
}
|
||||
|
||||
fn finish(
|
||||
self,
|
||||
) -> Result<(Self::Evm, BlockExecutionResult<Self::Receipt>), BlockExecutionError> {
|
||||
let Self { hook, state, .. } = self;
|
||||
|
||||
// Call hook with our mock state
|
||||
if let Some(mut hook) = hook {
|
||||
hook.on_state(StateChangeSource::Transaction(0), &state);
|
||||
}
|
||||
|
||||
// Create a mock EVM
|
||||
let db = State::builder()
|
||||
.with_database(EmptyDB::default())
|
||||
.with_bundle_update()
|
||||
.without_state_clear()
|
||||
.build();
|
||||
let evm = EthEvm::new(
|
||||
Context::mainnet().with_db(db).build_mainnet_with_inspector(NoOpInspector {}),
|
||||
false,
|
||||
);
|
||||
|
||||
// Return successful result like the original tests
|
||||
Ok((
|
||||
evm,
|
||||
BlockExecutionResult {
|
||||
receipts: vec![],
|
||||
requests: Requests::default(),
|
||||
gas_used: 1000,
|
||||
blob_gas_used: 0,
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
fn set_state_hook(&mut self, hook: Option<Box<dyn OnStateHook>>) {
|
||||
self.hook = hook;
|
||||
}
|
||||
|
||||
fn evm(&self) -> &Self::Evm {
|
||||
panic!("Mock executor evm() not implemented")
|
||||
}
|
||||
|
||||
fn evm_mut(&mut self) -> &mut Self::Evm {
|
||||
panic!("Mock executor evm_mut() not implemented")
|
||||
}
|
||||
}
|
||||
|
||||
struct ChannelStateHook {
|
||||
output: i32,
|
||||
sender: mpsc::Sender<i32>,
|
||||
}
|
||||
|
||||
impl OnStateHook for ChannelStateHook {
|
||||
fn on_state(&mut self, _source: StateChangeSource, _state: &EvmState) {
|
||||
let _ = self.sender.send(self.output);
|
||||
}
|
||||
}
|
||||
use reth_revm::db::BundleState;
|
||||
|
||||
fn setup_test_recorder() -> Snapshotter {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
@@ -523,37 +369,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_executor_metrics_hook_called() {
|
||||
let metrics = EngineApiMetrics::default();
|
||||
let input = RecoveredBlock::<reth_ethereum_primitives::Block>::default();
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let expected_output = 42;
|
||||
let state_hook = Box::new(ChannelStateHook { sender: tx, output: expected_output });
|
||||
|
||||
let state = EvmState::default();
|
||||
let executor = MockExecutor::new(state);
|
||||
|
||||
// This will fail to create the EVM but should still call the hook
|
||||
let _result = metrics.execute_metered::<_, EmptyDB>(
|
||||
executor,
|
||||
input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>),
|
||||
input.transaction_count(),
|
||||
state_hook,
|
||||
);
|
||||
|
||||
// Check if hook was called (it might not be if finish() fails early)
|
||||
match rx.try_recv() {
|
||||
Ok(actual_output) => assert_eq!(actual_output, expected_output),
|
||||
Err(_) => {
|
||||
// Hook wasn't called, which is expected if the mock fails early
|
||||
// The test still validates that the code compiles and runs
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_executor_metrics_hook_metrics_recorded() {
|
||||
fn test_record_block_execution_metrics() {
|
||||
let snapshotter = setup_test_recorder();
|
||||
let metrics = EngineApiMetrics::default();
|
||||
|
||||
@@ -562,42 +378,17 @@ mod tests {
|
||||
metrics.executor.gas_per_second.set(0.0);
|
||||
metrics.executor.gas_used_histogram.record(0.0);
|
||||
|
||||
let input = RecoveredBlock::<reth_ethereum_primitives::Block>::default();
|
||||
|
||||
let (tx, _rx) = mpsc::channel();
|
||||
let state_hook = Box::new(ChannelStateHook { sender: tx, output: 42 });
|
||||
|
||||
// Create a state with some data
|
||||
let state = {
|
||||
let mut state = EvmState::default();
|
||||
let storage =
|
||||
EvmStorage::from_iter([(U256::from(1), EvmStorageSlot::new(U256::from(2), 0))]);
|
||||
state.insert(
|
||||
Default::default(),
|
||||
Account {
|
||||
info: AccountInfo {
|
||||
balance: U256::from(100),
|
||||
nonce: 10,
|
||||
code_hash: B256::random(),
|
||||
code: Default::default(),
|
||||
},
|
||||
storage,
|
||||
status: AccountStatus::default(),
|
||||
transaction_id: 0,
|
||||
},
|
||||
);
|
||||
state
|
||||
let output = BlockExecutionOutput::<Receipt> {
|
||||
state: BundleState::default(),
|
||||
result: BlockExecutionResult {
|
||||
receipts: vec![],
|
||||
requests: Requests::default(),
|
||||
gas_used: 21000,
|
||||
blob_gas_used: 0,
|
||||
},
|
||||
};
|
||||
|
||||
let executor = MockExecutor::new(state);
|
||||
|
||||
// Execute (will fail but should still update some metrics)
|
||||
let _result = metrics.execute_metered::<_, EmptyDB>(
|
||||
executor,
|
||||
input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>),
|
||||
input.transaction_count(),
|
||||
state_hook,
|
||||
);
|
||||
metrics.record_block_execution(&output, Duration::from_millis(100));
|
||||
|
||||
let snapshot = snapshotter.snapshot().into_vec();
|
||||
|
||||
|
||||
@@ -30,8 +30,9 @@ use reth_payload_primitives::{
|
||||
};
|
||||
use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
|
||||
use reth_provider::{
|
||||
BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, HashedPostStateProvider,
|
||||
ProviderError, StageCheckpointReader, StateProviderBox, StateProviderFactory, StateReader,
|
||||
BlockExecutionOutput, BlockExecutionResult, BlockNumReader, BlockReader, ChangeSetReader,
|
||||
DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader,
|
||||
StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader,
|
||||
TransactionVariant,
|
||||
};
|
||||
use reth_revm::database::StateProviderDatabase;
|
||||
@@ -84,6 +85,12 @@ pub mod state;
|
||||
/// backfill this gap.
|
||||
pub(crate) const MIN_BLOCKS_FOR_PIPELINE_RUN: u64 = EPOCH_SLOTS;
|
||||
|
||||
/// The minimum number of blocks to retain in the changeset cache after eviction.
|
||||
///
|
||||
/// This ensures that recent trie changesets are kept in memory for potential reorgs,
|
||||
/// even when the finalized block is not set (e.g., on L2s like Optimism).
|
||||
const CHANGESET_CACHE_RETENTION_BLOCKS: u64 = 64;
|
||||
|
||||
/// A builder for creating state providers that can be used across threads.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct StateProviderBuilder<N: NodePrimitives, P> {
|
||||
@@ -317,6 +324,7 @@ where
|
||||
<P as DatabaseProviderFactory>::Provider: BlockReader<Block = N::Block, Header = N::BlockHeader>
|
||||
+ StageCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader,
|
||||
C: ConfigureEvm<Primitives = N> + 'static,
|
||||
T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>,
|
||||
@@ -1376,19 +1384,27 @@ where
|
||||
debug!(target: "engine::tree", ?last_persisted_block_hash, ?last_persisted_block_number, elapsed=?start_time.elapsed(), "Finished persisting, calling finish");
|
||||
self.persistence_state.finish(last_persisted_block_hash, last_persisted_block_number);
|
||||
|
||||
// Evict trie changesets for blocks below the finalized block, but keep at least 64 blocks
|
||||
if let Some(finalized) = self.canonical_in_memory_state.get_finalized_num_hash() {
|
||||
let min_threshold = last_persisted_block_number.saturating_sub(64);
|
||||
let eviction_threshold = finalized.number.min(min_threshold);
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
last_persisted = last_persisted_block_number,
|
||||
finalized_number = finalized.number,
|
||||
eviction_threshold,
|
||||
"Evicting changesets below threshold"
|
||||
);
|
||||
self.changeset_cache.evict(eviction_threshold);
|
||||
}
|
||||
// Evict trie changesets for blocks below the eviction threshold.
|
||||
// Keep at least CHANGESET_CACHE_RETENTION_BLOCKS from the persisted tip, and also respect
|
||||
// the finalized block if set.
|
||||
let min_threshold =
|
||||
last_persisted_block_number.saturating_sub(CHANGESET_CACHE_RETENTION_BLOCKS);
|
||||
let eviction_threshold =
|
||||
if let Some(finalized) = self.canonical_in_memory_state.get_finalized_num_hash() {
|
||||
// Use the minimum of finalized block and retention threshold to be conservative
|
||||
finalized.number.min(min_threshold)
|
||||
} else {
|
||||
// When finalized is not set (e.g., on L2s), use the retention threshold
|
||||
min_threshold
|
||||
};
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
last_persisted = last_persisted_block_number,
|
||||
finalized_number = ?self.canonical_in_memory_state.get_finalized_num_hash().map(|f| f.number),
|
||||
eviction_threshold,
|
||||
"Evicting changesets below threshold"
|
||||
);
|
||||
self.changeset_cache.evict(eviction_threshold);
|
||||
|
||||
self.on_new_persisted_block()?;
|
||||
Ok(())
|
||||
@@ -1478,7 +1494,7 @@ where
|
||||
|
||||
self.metrics.engine.forkchoice_updated.update_response_metrics(
|
||||
start,
|
||||
&mut self.metrics.engine.new_payload.latest_at,
|
||||
&mut self.metrics.engine.new_payload.latest_finish_at,
|
||||
has_attrs,
|
||||
&output,
|
||||
);
|
||||
@@ -1676,6 +1692,18 @@ where
|
||||
)));
|
||||
return Ok(());
|
||||
}
|
||||
} else {
|
||||
// We don't have the head block or any of its ancestors buffered. Request
|
||||
// a download for the head block which will then trigger further sync.
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
head_hash = %sync_target_state.head_block_hash,
|
||||
"Backfill complete but head block not buffered, requesting download"
|
||||
);
|
||||
self.emit_event(EngineApiEvent::Download(DownloadRequest::single_block(
|
||||
sync_target_state.head_block_hash,
|
||||
)));
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// try to close the gap by executing buffered blocks that are child blocks of the new head
|
||||
@@ -1761,7 +1789,7 @@ where
|
||||
}
|
||||
|
||||
let min_block = self.persistence_state.last_persisted_block.number;
|
||||
self.state.tree_state.canonical_block_number().saturating_sub(min_block) >=
|
||||
self.state.tree_state.canonical_block_number().saturating_sub(min_block) >
|
||||
self.config.persistence_threshold()
|
||||
}
|
||||
|
||||
@@ -1856,7 +1884,7 @@ where
|
||||
.sealed_block_with_senders(hash.into(), TransactionVariant::WithHash)?
|
||||
.ok_or_else(|| ProviderError::HeaderNotFound(hash.into()))?
|
||||
.split_sealed();
|
||||
let execution_output = self
|
||||
let mut execution_output = self
|
||||
.provider
|
||||
.get_state(block.header().number())?
|
||||
.ok_or_else(|| ProviderError::StateForNumberNotFound(block.header().number()))?;
|
||||
@@ -1880,9 +1908,19 @@ where
|
||||
let trie_data =
|
||||
ComputedTrieData::without_trie_input(sorted_hashed_state, sorted_trie_updates);
|
||||
|
||||
let execution_output = Arc::new(BlockExecutionOutput {
|
||||
state: execution_output.bundle,
|
||||
result: BlockExecutionResult {
|
||||
receipts: execution_output.receipts.pop().unwrap_or_default(),
|
||||
requests: execution_output.requests.pop().unwrap_or_default(),
|
||||
gas_used: block.gas_used(),
|
||||
blob_gas_used: block.blob_gas_used().unwrap_or_default(),
|
||||
},
|
||||
});
|
||||
|
||||
Ok(Some(ExecutedBlock::new(
|
||||
Arc::new(RecoveredBlock::new_sealed(block, senders)),
|
||||
Arc::new(execution_output),
|
||||
execution_output,
|
||||
trie_data,
|
||||
)))
|
||||
}
|
||||
|
||||
@@ -101,7 +101,7 @@ impl<'a> Iterator for BALSlotIter<'a> {
|
||||
return None;
|
||||
}
|
||||
|
||||
return Some((address, slot));
|
||||
return Some((address, StorageKey::from(slot)));
|
||||
}
|
||||
|
||||
// Move to next account
|
||||
@@ -177,13 +177,11 @@ where
|
||||
let mut storage_map = HashedStorage::new(false);
|
||||
|
||||
for slot_changes in &account_changes.storage_changes {
|
||||
let hashed_slot = keccak256(slot_changes.slot);
|
||||
let hashed_slot = keccak256(slot_changes.slot.to_be_bytes::<32>());
|
||||
|
||||
// Get the last change for this slot
|
||||
if let Some(last_change) = slot_changes.changes.last() {
|
||||
storage_map
|
||||
.storage
|
||||
.insert(hashed_slot, U256::from_be_bytes(last_change.new_value.0));
|
||||
storage_map.storage.insert(hashed_slot, last_change.new_value);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -237,8 +235,8 @@ mod tests {
|
||||
let provider = StateProviderTest::default();
|
||||
|
||||
let address = Address::random();
|
||||
let slot = StorageKey::random();
|
||||
let value = B256::random();
|
||||
let slot = U256::random();
|
||||
let value = U256::random();
|
||||
|
||||
let slot_changes = SlotChanges { slot, changes: vec![StorageChange::new(0, value)] };
|
||||
|
||||
@@ -258,10 +256,10 @@ mod tests {
|
||||
assert!(result.storages.contains_key(&hashed_address));
|
||||
|
||||
let storage = result.storages.get(&hashed_address).unwrap();
|
||||
let hashed_slot = keccak256(slot);
|
||||
let hashed_slot = keccak256(slot.to_be_bytes::<32>());
|
||||
|
||||
let stored_value = storage.storage.get(&hashed_slot).unwrap();
|
||||
assert_eq!(*stored_value, U256::from_be_bytes(value.0));
|
||||
assert_eq!(*stored_value, value);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -392,15 +390,15 @@ mod tests {
|
||||
let provider = StateProviderTest::default();
|
||||
|
||||
let address = Address::random();
|
||||
let slot = StorageKey::random();
|
||||
let slot = U256::random();
|
||||
|
||||
// Multiple changes to the same slot - should take the last one
|
||||
let slot_changes = SlotChanges {
|
||||
slot,
|
||||
changes: vec![
|
||||
StorageChange::new(0, B256::from(U256::from(100).to_be_bytes::<32>())),
|
||||
StorageChange::new(1, B256::from(U256::from(200).to_be_bytes::<32>())),
|
||||
StorageChange::new(2, B256::from(U256::from(300).to_be_bytes::<32>())),
|
||||
StorageChange::new(0, U256::from(100)),
|
||||
StorageChange::new(1, U256::from(200)),
|
||||
StorageChange::new(2, U256::from(300)),
|
||||
],
|
||||
};
|
||||
|
||||
@@ -418,7 +416,7 @@ mod tests {
|
||||
|
||||
let hashed_address = keccak256(address);
|
||||
let storage = result.storages.get(&hashed_address).unwrap();
|
||||
let hashed_slot = keccak256(slot);
|
||||
let hashed_slot = keccak256(slot.to_be_bytes::<32>());
|
||||
|
||||
let stored_value = storage.storage.get(&hashed_slot).unwrap();
|
||||
|
||||
@@ -438,15 +436,15 @@ mod tests {
|
||||
address: addr1,
|
||||
storage_changes: vec![
|
||||
SlotChanges {
|
||||
slot: StorageKey::from(U256::from(100)),
|
||||
changes: vec![StorageChange::new(0, B256::ZERO)],
|
||||
slot: U256::from(100),
|
||||
changes: vec![StorageChange::new(0, U256::ZERO)],
|
||||
},
|
||||
SlotChanges {
|
||||
slot: StorageKey::from(U256::from(101)),
|
||||
changes: vec![StorageChange::new(0, B256::ZERO)],
|
||||
slot: U256::from(101),
|
||||
changes: vec![StorageChange::new(0, U256::ZERO)],
|
||||
},
|
||||
],
|
||||
storage_reads: vec![StorageKey::from(U256::from(102))],
|
||||
storage_reads: vec![U256::from(102)],
|
||||
balance_changes: vec![],
|
||||
nonce_changes: vec![],
|
||||
code_changes: vec![],
|
||||
@@ -456,10 +454,10 @@ mod tests {
|
||||
let account2 = AccountChanges {
|
||||
address: addr2,
|
||||
storage_changes: vec![SlotChanges {
|
||||
slot: StorageKey::from(U256::from(200)),
|
||||
changes: vec![StorageChange::new(0, B256::ZERO)],
|
||||
slot: U256::from(200),
|
||||
changes: vec![StorageChange::new(0, U256::ZERO)],
|
||||
}],
|
||||
storage_reads: vec![StorageKey::from(U256::from(201))],
|
||||
storage_reads: vec![U256::from(201)],
|
||||
balance_changes: vec![],
|
||||
nonce_changes: vec![],
|
||||
code_changes: vec![],
|
||||
@@ -470,15 +468,15 @@ mod tests {
|
||||
address: addr3,
|
||||
storage_changes: vec![
|
||||
SlotChanges {
|
||||
slot: StorageKey::from(U256::from(300)),
|
||||
changes: vec![StorageChange::new(0, B256::ZERO)],
|
||||
slot: U256::from(300),
|
||||
changes: vec![StorageChange::new(0, U256::ZERO)],
|
||||
},
|
||||
SlotChanges {
|
||||
slot: StorageKey::from(U256::from(301)),
|
||||
changes: vec![StorageChange::new(0, B256::ZERO)],
|
||||
slot: U256::from(301),
|
||||
changes: vec![StorageChange::new(0, U256::ZERO)],
|
||||
},
|
||||
],
|
||||
storage_reads: vec![StorageKey::from(U256::from(302))],
|
||||
storage_reads: vec![U256::from(302)],
|
||||
balance_changes: vec![],
|
||||
nonce_changes: vec![],
|
||||
code_changes: vec![],
|
||||
|
||||
@@ -15,23 +15,26 @@ use crate::tree::{
|
||||
};
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::eip1898::BlockWithParent;
|
||||
use alloy_evm::{block::StateChangeSource, ToTxEnv};
|
||||
use alloy_evm::block::StateChangeSource;
|
||||
use alloy_primitives::B256;
|
||||
use crossbeam_channel::Sender as CrossbeamSender;
|
||||
use executor::WorkloadExecutor;
|
||||
use metrics::Counter;
|
||||
use multiproof::{SparseTrieUpdate, *};
|
||||
use parking_lot::RwLock;
|
||||
use prewarm::PrewarmMetrics;
|
||||
use rayon::prelude::*;
|
||||
use reth_evm::{
|
||||
block::ExecutableTxParts,
|
||||
execute::{ExecutableTxFor, WithTxEnv},
|
||||
ConfigureEvm, EvmEnvFor, ExecutableTxIterator, ExecutableTxTuple, OnStateHook, SpecFor,
|
||||
TxEnvFor,
|
||||
};
|
||||
use reth_execution_types::ExecutionOutcome;
|
||||
use reth_metrics::Metrics;
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_provider::{
|
||||
BlockReader, DatabaseProviderROFactory, StateProvider, StateProviderFactory, StateReader,
|
||||
BlockExecutionOutput, BlockReader, DatabaseProviderROFactory, StateProvider,
|
||||
StateProviderFactory, StateReader,
|
||||
};
|
||||
use reth_revm::{db::BundleState, state::EvmState};
|
||||
use reth_trie::{hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory};
|
||||
@@ -61,6 +64,7 @@ mod configured_sparse_trie;
|
||||
pub mod executor;
|
||||
pub mod multiproof;
|
||||
pub mod prewarm;
|
||||
pub mod receipt_root_task;
|
||||
pub mod sparse_trie;
|
||||
|
||||
use configured_sparse_trie::ConfiguredSparseTrie;
|
||||
@@ -98,7 +102,7 @@ pub const SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY: usize = 1_000_000;
|
||||
|
||||
/// Type alias for [`PayloadHandle`] returned by payload processor spawn methods.
|
||||
type IteratorPayloadHandle<Evm, I, N> = PayloadHandle<
|
||||
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxTuple>::Tx>,
|
||||
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxIterator<Evm>>::Recovered>,
|
||||
<I as ExecutableTxTuple>::Error,
|
||||
<N as NodePrimitives>::Receipt,
|
||||
>;
|
||||
@@ -138,6 +142,8 @@ where
|
||||
disable_parallel_sparse_trie: bool,
|
||||
/// Maximum concurrency for prewarm task.
|
||||
prewarm_max_concurrency: usize,
|
||||
/// Whether to disable cache metrics recording.
|
||||
disable_cache_metrics: bool,
|
||||
}
|
||||
|
||||
impl<N, Evm> PayloadProcessor<Evm>
|
||||
@@ -170,6 +176,7 @@ where
|
||||
sparse_state_trie: Arc::default(),
|
||||
disable_parallel_sparse_trie: config.disable_parallel_sparse_trie(),
|
||||
prewarm_max_concurrency: config.prewarm_max_concurrency(),
|
||||
disable_cache_metrics: config.disable_cache_metrics(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -241,6 +248,9 @@ where
|
||||
let (to_sparse_trie, sparse_trie_rx) = channel();
|
||||
let (to_multi_proof, from_multi_proof) = crossbeam_channel::unbounded();
|
||||
|
||||
// Extract V2 proofs flag early so we can pass it to prewarm
|
||||
let v2_proofs_enabled = config.enable_proof_v2();
|
||||
|
||||
// Handle BAL-based optimization if available
|
||||
let prewarm_handle = if let Some(bal) = bal {
|
||||
// When BAL is present, use BAL prewarming and send BAL to multiproof
|
||||
@@ -257,6 +267,7 @@ where
|
||||
provider_builder.clone(),
|
||||
None, // Don't send proof targets when BAL is present
|
||||
Some(bal),
|
||||
v2_proofs_enabled,
|
||||
)
|
||||
} else {
|
||||
// Normal path: spawn with transaction prewarming
|
||||
@@ -267,6 +278,7 @@ where
|
||||
provider_builder.clone(),
|
||||
Some(to_multi_proof.clone()),
|
||||
None,
|
||||
v2_proofs_enabled,
|
||||
)
|
||||
};
|
||||
|
||||
@@ -274,7 +286,6 @@ where
|
||||
let task_ctx = ProofTaskCtx::new(multiproof_provider_factory);
|
||||
let storage_worker_count = config.storage_worker_count();
|
||||
let account_worker_count = config.account_worker_count();
|
||||
let v2_proofs_enabled = config.enable_proof_v2();
|
||||
let proof_handle = ProofWorkerHandle::new(
|
||||
self.executor.handle().clone(),
|
||||
task_ctx,
|
||||
@@ -286,10 +297,13 @@ where
|
||||
let multi_proof_task = MultiProofTask::new(
|
||||
proof_handle.clone(),
|
||||
to_sparse_trie,
|
||||
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size()),
|
||||
config
|
||||
.multiproof_chunking_enabled()
|
||||
.then_some(config.effective_multiproof_chunk_size()),
|
||||
to_multi_proof.clone(),
|
||||
from_multi_proof,
|
||||
);
|
||||
)
|
||||
.with_v2_proofs_enabled(v2_proofs_enabled);
|
||||
|
||||
// spawn multi-proof task
|
||||
let parent_span = span.clone();
|
||||
@@ -299,7 +313,7 @@ where
|
||||
// Build a state provider for the multiproof task
|
||||
let provider = provider_builder.build().expect("failed to build provider");
|
||||
let provider = if let Some(saved_cache) = saved_cache {
|
||||
let (cache, metrics) = saved_cache.split();
|
||||
let (cache, metrics, _) = saved_cache.split();
|
||||
Box::new(CachedStateProvider::new(provider, cache, metrics))
|
||||
as Box<dyn StateProvider>
|
||||
} else {
|
||||
@@ -338,8 +352,9 @@ where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
{
|
||||
let (prewarm_rx, execution_rx, size_hint) = self.spawn_tx_iterator(transactions);
|
||||
// This path doesn't use multiproof, so V2 proofs flag doesn't matter
|
||||
let prewarm_handle =
|
||||
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal);
|
||||
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal, false);
|
||||
PayloadHandle {
|
||||
to_multi_proof: None,
|
||||
prewarm_handle,
|
||||
@@ -355,8 +370,8 @@ where
|
||||
&self,
|
||||
transactions: I,
|
||||
) -> (
|
||||
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Tx>>,
|
||||
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Tx>, I::Error>>,
|
||||
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Recovered>>,
|
||||
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Recovered>, I::Error>>,
|
||||
usize,
|
||||
) {
|
||||
let (transactions, convert) = transactions.into();
|
||||
@@ -371,7 +386,10 @@ where
|
||||
self.executor.spawn_blocking(move || {
|
||||
transactions.enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
|
||||
let tx = convert(tx);
|
||||
let tx = tx.map(|tx| WithTxEnv { tx_env: tx.to_tx_env(), tx: Arc::new(tx) });
|
||||
let tx = tx.map(|tx| {
|
||||
let (tx_env, tx) = tx.into_parts();
|
||||
WithTxEnv { tx_env, tx: Arc::new(tx) }
|
||||
});
|
||||
// Only send Ok(_) variants to prewarming task.
|
||||
if let Ok(tx) = &tx {
|
||||
let _ = prewarm_tx.send(tx.clone());
|
||||
@@ -406,6 +424,7 @@ where
|
||||
}
|
||||
|
||||
/// Spawn prewarming optionally wired to the multiproof task for target updates.
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
fn spawn_caching_with<P>(
|
||||
&self,
|
||||
env: ExecutionEnv<Evm>,
|
||||
@@ -414,6 +433,7 @@ where
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
bal: Option<Arc<BlockAccessList>>,
|
||||
v2_proofs_enabled: bool,
|
||||
) -> CacheTaskHandle<N::Receipt>
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
@@ -436,6 +456,7 @@ where
|
||||
terminate_execution: Arc::new(AtomicBool::new(false)),
|
||||
precompile_cache_disabled: self.precompile_cache_disabled,
|
||||
precompile_cache_map: self.precompile_cache_map.clone(),
|
||||
v2_proofs_enabled,
|
||||
};
|
||||
|
||||
let (prewarm_task, to_prewarm_task) = PrewarmCacheTask::new(
|
||||
@@ -476,6 +497,7 @@ where
|
||||
debug!("creating new execution cache on cache miss");
|
||||
let cache = ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size);
|
||||
SavedCache::new(parent_hash, cache, CachedStateMetrics::zeroed())
|
||||
.with_disable_cache_metrics(self.disable_cache_metrics)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -557,6 +579,7 @@ where
|
||||
block_with_parent: BlockWithParent,
|
||||
bundle_state: &BundleState,
|
||||
) {
|
||||
let disable_cache_metrics = self.disable_cache_metrics;
|
||||
self.execution_cache.update_with_guard(|cached| {
|
||||
if cached.as_ref().is_some_and(|c| c.executed_block_hash() != block_with_parent.parent) {
|
||||
debug!(
|
||||
@@ -570,7 +593,8 @@ where
|
||||
// Take existing cache (if any) or create fresh caches
|
||||
let (caches, cache_metrics) = match cached.take() {
|
||||
Some(existing) => {
|
||||
existing.split()
|
||||
let (c, m, _) = existing.split();
|
||||
(c, m)
|
||||
}
|
||||
None => (
|
||||
ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size),
|
||||
@@ -579,7 +603,8 @@ where
|
||||
};
|
||||
|
||||
// Insert the block's bundle state into cache
|
||||
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics);
|
||||
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics)
|
||||
.with_disable_cache_metrics(disable_cache_metrics);
|
||||
if new_cache.cache().insert_state(bundle_state).is_err() {
|
||||
*cached = None;
|
||||
debug!(target: "engine::caching", "cleared execution cache on update error");
|
||||
@@ -665,13 +690,15 @@ impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
|
||||
|
||||
/// Terminates the entire caching task.
|
||||
///
|
||||
/// If the [`ExecutionOutcome`] is provided it will update the shared cache using its
|
||||
/// If the [`BlockExecutionOutput`] is provided it will update the shared cache using its
|
||||
/// bundle state. Using `Arc<ExecutionOutcome>` allows sharing with the main execution
|
||||
/// path without cloning the expensive `BundleState`.
|
||||
///
|
||||
/// Returns a sender for the channel that should be notified on block validation success.
|
||||
pub(super) fn terminate_caching(
|
||||
&mut self,
|
||||
execution_outcome: Option<Arc<ExecutionOutcome<R>>>,
|
||||
) {
|
||||
execution_outcome: Option<Arc<BlockExecutionOutput<R>>>,
|
||||
) -> Option<mpsc::Sender<()>> {
|
||||
self.prewarm_handle.terminate_caching(execution_outcome)
|
||||
}
|
||||
|
||||
@@ -707,15 +734,21 @@ impl<R: Send + Sync + 'static> CacheTaskHandle<R> {
|
||||
|
||||
/// Terminates the entire pre-warming task.
|
||||
///
|
||||
/// If the [`ExecutionOutcome`] is provided it will update the shared cache using its
|
||||
/// If the [`BlockExecutionOutput`] is provided it will update the shared cache using its
|
||||
/// bundle state. Using `Arc<ExecutionOutcome>` avoids cloning the expensive `BundleState`.
|
||||
#[must_use = "sender must be used and notified on block validation success"]
|
||||
pub(super) fn terminate_caching(
|
||||
&mut self,
|
||||
execution_outcome: Option<Arc<ExecutionOutcome<R>>>,
|
||||
) {
|
||||
execution_outcome: Option<Arc<BlockExecutionOutput<R>>>,
|
||||
) -> Option<mpsc::Sender<()>> {
|
||||
if let Some(tx) = self.to_prewarm_task.take() {
|
||||
let event = PrewarmTaskEvent::Terminate { execution_outcome };
|
||||
let (valid_block_tx, valid_block_rx) = mpsc::channel();
|
||||
let event = PrewarmTaskEvent::Terminate { execution_outcome, valid_block_rx };
|
||||
let _ = tx.send(event);
|
||||
|
||||
Some(valid_block_tx)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -724,7 +757,10 @@ impl<R> Drop for CacheTaskHandle<R> {
|
||||
fn drop(&mut self) {
|
||||
// Ensure we always terminate on drop - send None without needing Send + Sync bounds
|
||||
if let Some(tx) = self.to_prewarm_task.take() {
|
||||
let _ = tx.send(PrewarmTaskEvent::Terminate { execution_outcome: None });
|
||||
let _ = tx.send(PrewarmTaskEvent::Terminate {
|
||||
execution_outcome: None,
|
||||
valid_block_rx: mpsc::channel().1,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -758,6 +794,8 @@ impl<R> Drop for CacheTaskHandle<R> {
|
||||
struct ExecutionCache {
|
||||
/// Guarded cloneable cache identified by a block hash.
|
||||
inner: Arc<RwLock<Option<SavedCache>>>,
|
||||
/// Metrics for cache operations.
|
||||
metrics: ExecutionCacheMetrics,
|
||||
}
|
||||
|
||||
impl ExecutionCache {
|
||||
@@ -799,6 +837,10 @@ impl ExecutionCache {
|
||||
if hash_matches && available {
|
||||
return Some(c.clone());
|
||||
}
|
||||
|
||||
if hash_matches && !available {
|
||||
self.metrics.execution_cache_in_use.increment(1);
|
||||
}
|
||||
} else {
|
||||
debug!(target: "engine::caching", %parent_hash, "No cache found");
|
||||
}
|
||||
@@ -834,6 +876,15 @@ impl ExecutionCache {
|
||||
}
|
||||
}
|
||||
|
||||
/// Metrics for execution cache operations.
|
||||
#[derive(Metrics, Clone)]
|
||||
#[metrics(scope = "consensus.engine.beacon")]
|
||||
pub(crate) struct ExecutionCacheMetrics {
|
||||
/// Counter for when the execution cache was unavailable because other threads
|
||||
/// (e.g., prewarming) are still using it.
|
||||
pub(crate) execution_cache_in_use: Counter,
|
||||
}
|
||||
|
||||
/// EVM context required to execute a block.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ExecutionEnv<Evm: ConfigureEvm> {
|
||||
@@ -1059,7 +1110,9 @@ mod tests {
|
||||
nonce: rng.random::<u64>(),
|
||||
code_hash: KECCAK_EMPTY,
|
||||
code: Some(Default::default()),
|
||||
account_id: None,
|
||||
},
|
||||
original_info: Box::new(AccountInfo::default()),
|
||||
storage,
|
||||
status: AccountStatus::Touched,
|
||||
transaction_id: 0,
|
||||
|
||||
@@ -11,14 +11,18 @@ use reth_metrics::Metrics;
|
||||
use reth_provider::AccountReader;
|
||||
use reth_revm::state::EvmState;
|
||||
use reth_trie::{
|
||||
added_removed_keys::MultiAddedRemovedKeys, DecodedMultiProof, HashedPostState, HashedStorage,
|
||||
added_removed_keys::MultiAddedRemovedKeys, proof_v2, HashedPostState, HashedStorage,
|
||||
MultiProofTargets,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use reth_trie_parallel::stats::ParallelTrieTracker;
|
||||
use reth_trie_parallel::{
|
||||
proof::ParallelProof,
|
||||
proof_task::{
|
||||
AccountMultiproofInput, ProofResultContext, ProofResultMessage, ProofWorkerHandle,
|
||||
AccountMultiproofInput, ProofResult, ProofResultContext, ProofResultMessage,
|
||||
ProofWorkerHandle,
|
||||
},
|
||||
targets_v2::{ChunkedMultiProofTargetsV2, MultiProofTargetsV2},
|
||||
};
|
||||
use revm_primitives::map::{hash_map, B256Map};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::Instant};
|
||||
@@ -63,12 +67,12 @@ const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300;
|
||||
|
||||
/// A trie update that can be applied to sparse trie alongside the proofs for touched parts of the
|
||||
/// state.
|
||||
#[derive(Default, Debug)]
|
||||
#[derive(Debug)]
|
||||
pub struct SparseTrieUpdate {
|
||||
/// The state update that was used to calculate the proof
|
||||
pub(crate) state: HashedPostState,
|
||||
/// The calculated multiproof
|
||||
pub(crate) multiproof: DecodedMultiProof,
|
||||
pub(crate) multiproof: ProofResult,
|
||||
}
|
||||
|
||||
impl SparseTrieUpdate {
|
||||
@@ -80,7 +84,11 @@ impl SparseTrieUpdate {
|
||||
/// Construct update from multiproof.
|
||||
#[cfg(test)]
|
||||
pub(super) fn from_multiproof(multiproof: reth_trie::MultiProof) -> alloy_rlp::Result<Self> {
|
||||
Ok(Self { multiproof: multiproof.try_into()?, ..Default::default() })
|
||||
let stats = ParallelTrieTracker::default().finish();
|
||||
Ok(Self {
|
||||
state: HashedPostState::default(),
|
||||
multiproof: ProofResult::Legacy(multiproof.try_into()?, stats),
|
||||
})
|
||||
}
|
||||
|
||||
/// Extend update with contents of the other.
|
||||
@@ -94,7 +102,7 @@ impl SparseTrieUpdate {
|
||||
#[derive(Debug)]
|
||||
pub(super) enum MultiProofMessage {
|
||||
/// Prefetch proof targets
|
||||
PrefetchProofs(MultiProofTargets),
|
||||
PrefetchProofs(VersionedMultiProofTargets),
|
||||
/// New state update from transaction execution with its source
|
||||
StateUpdate(Source, EvmState),
|
||||
/// State update that can be applied to the sparse trie without any new proofs.
|
||||
@@ -141,22 +149,27 @@ impl ProofSequencer {
|
||||
/// Adds a proof with the corresponding state update and returns all sequential proofs and state
|
||||
/// updates if we have a continuous sequence
|
||||
fn add_proof(&mut self, sequence: u64, update: SparseTrieUpdate) -> Vec<SparseTrieUpdate> {
|
||||
if sequence >= self.next_to_deliver {
|
||||
// Optimization: fast path for in-order delivery to avoid BTreeMap overhead.
|
||||
// If this is the expected sequence, return it immediately without buffering.
|
||||
if sequence == self.next_to_deliver {
|
||||
let mut consecutive_proofs = Vec::with_capacity(1);
|
||||
consecutive_proofs.push(update);
|
||||
self.next_to_deliver += 1;
|
||||
|
||||
// Check if we have subsequent proofs in the pending buffer
|
||||
while let Some(pending) = self.pending_proofs.remove(&self.next_to_deliver) {
|
||||
consecutive_proofs.push(pending);
|
||||
self.next_to_deliver += 1;
|
||||
}
|
||||
|
||||
return consecutive_proofs;
|
||||
}
|
||||
|
||||
if sequence > self.next_to_deliver {
|
||||
self.pending_proofs.insert(sequence, update);
|
||||
}
|
||||
|
||||
let mut consecutive_proofs = Vec::with_capacity(self.pending_proofs.len());
|
||||
let mut current_sequence = self.next_to_deliver;
|
||||
|
||||
// keep collecting proofs and state updates as long as we have consecutive sequence numbers
|
||||
while let Some(pending) = self.pending_proofs.remove(¤t_sequence) {
|
||||
consecutive_proofs.push(pending);
|
||||
current_sequence += 1;
|
||||
}
|
||||
|
||||
self.next_to_deliver += consecutive_proofs.len() as u64;
|
||||
|
||||
consecutive_proofs
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// Returns true if we still have pending proofs
|
||||
@@ -218,12 +231,155 @@ pub(crate) fn evm_state_to_hashed_post_state(update: EvmState) -> HashedPostStat
|
||||
hashed_state
|
||||
}
|
||||
|
||||
/// Extends a `MultiProofTargets` with the contents of a `VersionedMultiProofTargets`,
|
||||
/// regardless of which variant the latter is.
|
||||
fn extend_multiproof_targets(dest: &mut MultiProofTargets, src: &VersionedMultiProofTargets) {
|
||||
match src {
|
||||
VersionedMultiProofTargets::Legacy(targets) => {
|
||||
dest.extend_ref(targets);
|
||||
}
|
||||
VersionedMultiProofTargets::V2(targets) => {
|
||||
// Add all account targets
|
||||
for target in &targets.account_targets {
|
||||
dest.entry(target.key()).or_default();
|
||||
}
|
||||
|
||||
// Add all storage targets
|
||||
for (hashed_address, slots) in &targets.storage_targets {
|
||||
let slot_set = dest.entry(*hashed_address).or_default();
|
||||
for slot in slots {
|
||||
slot_set.insert(slot.key());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A set of multiproof targets which can be either in the legacy or V2 representations.
|
||||
#[derive(Debug)]
|
||||
pub(super) enum VersionedMultiProofTargets {
|
||||
/// Legacy targets
|
||||
Legacy(MultiProofTargets),
|
||||
/// V2 targets
|
||||
V2(MultiProofTargetsV2),
|
||||
}
|
||||
|
||||
impl VersionedMultiProofTargets {
|
||||
/// Returns true if there are no account or storage targets.
|
||||
fn is_empty(&self) -> bool {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.is_empty(),
|
||||
Self::V2(targets) => targets.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of account targets in the multiproof target
|
||||
fn account_targets_len(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.len(),
|
||||
Self::V2(targets) => targets.account_targets.len(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of storage targets in the multiproof target
|
||||
fn storage_targets_len(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.values().map(|slots| slots.len()).sum::<usize>(),
|
||||
Self::V2(targets) => {
|
||||
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of accounts in the multiproof targets.
|
||||
fn len(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.len(),
|
||||
Self::V2(targets) => targets.account_targets.len(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the total storage slot count across all accounts.
|
||||
fn storage_count(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.values().map(|slots| slots.len()).sum(),
|
||||
Self::V2(targets) => targets.storage_targets.values().map(|slots| slots.len()).sum(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of items that will be considered during chunking.
|
||||
fn chunking_length(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.chunking_length(),
|
||||
Self::V2(targets) => {
|
||||
// For V2, count accounts + storage slots
|
||||
targets.account_targets.len() +
|
||||
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Retains the targets representing the difference with another `MultiProofTargets`.
|
||||
/// Removes all targets that are already present in `other`.
|
||||
fn retain_difference(&mut self, other: &MultiProofTargets) {
|
||||
match self {
|
||||
Self::Legacy(targets) => {
|
||||
targets.retain_difference(other);
|
||||
}
|
||||
Self::V2(targets) => {
|
||||
// Remove account targets that exist in other
|
||||
targets.account_targets.retain(|target| !other.contains_key(&target.key()));
|
||||
|
||||
// For each account in storage_targets, remove slots that exist in other
|
||||
targets.storage_targets.retain(|hashed_address, slots| {
|
||||
if let Some(other_slots) = other.get(hashed_address) {
|
||||
slots.retain(|slot| !other_slots.contains(&slot.key()));
|
||||
!slots.is_empty()
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extends this `VersionedMultiProofTargets` with the contents of another.
|
||||
///
|
||||
/// Panics if the variants do not match.
|
||||
fn extend(&mut self, other: Self) {
|
||||
match (self, other) {
|
||||
(Self::Legacy(dest), Self::Legacy(src)) => {
|
||||
dest.extend(src);
|
||||
}
|
||||
(Self::V2(dest), Self::V2(src)) => {
|
||||
dest.account_targets.extend(src.account_targets);
|
||||
for (addr, slots) in src.storage_targets {
|
||||
dest.storage_targets.entry(addr).or_default().extend(slots);
|
||||
}
|
||||
}
|
||||
_ => panic!("Cannot extend VersionedMultiProofTargets with mismatched variants"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Chunks this `VersionedMultiProofTargets` into smaller chunks of the given size.
|
||||
fn chunks(self, chunk_size: usize) -> Box<dyn Iterator<Item = Self>> {
|
||||
match self {
|
||||
Self::Legacy(targets) => {
|
||||
Box::new(MultiProofTargets::chunks(targets, chunk_size).map(Self::Legacy))
|
||||
}
|
||||
Self::V2(targets) => {
|
||||
Box::new(ChunkedMultiProofTargetsV2::new(targets, chunk_size).map(Self::V2))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Input parameters for dispatching a multiproof calculation.
|
||||
#[derive(Debug)]
|
||||
struct MultiproofInput {
|
||||
source: Option<Source>,
|
||||
hashed_state_update: HashedPostState,
|
||||
proof_targets: MultiProofTargets,
|
||||
proof_targets: VersionedMultiProofTargets,
|
||||
proof_sequence_number: u64,
|
||||
state_root_message_sender: CrossbeamSender<MultiProofMessage>,
|
||||
multi_added_removed_keys: Option<Arc<MultiAddedRemovedKeys>>,
|
||||
@@ -258,8 +414,6 @@ pub struct MultiproofManager {
|
||||
proof_result_tx: CrossbeamSender<ProofResultMessage>,
|
||||
/// Metrics
|
||||
metrics: MultiProofTaskMetrics,
|
||||
/// Whether to use V2 storage proofs
|
||||
v2_proofs_enabled: bool,
|
||||
}
|
||||
|
||||
impl MultiproofManager {
|
||||
@@ -273,9 +427,7 @@ impl MultiproofManager {
|
||||
metrics.max_storage_workers.set(proof_worker_handle.total_storage_workers() as f64);
|
||||
metrics.max_account_workers.set(proof_worker_handle.total_account_workers() as f64);
|
||||
|
||||
let v2_proofs_enabled = proof_worker_handle.v2_proofs_enabled();
|
||||
|
||||
Self { metrics, proof_worker_handle, proof_result_tx, v2_proofs_enabled }
|
||||
Self { metrics, proof_worker_handle, proof_result_tx }
|
||||
}
|
||||
|
||||
/// Dispatches a new multiproof calculation to worker pools.
|
||||
@@ -320,41 +472,48 @@ impl MultiproofManager {
|
||||
multi_added_removed_keys,
|
||||
} = multiproof_input;
|
||||
|
||||
let account_targets = proof_targets.len();
|
||||
let storage_targets = proof_targets.values().map(|slots| slots.len()).sum::<usize>();
|
||||
|
||||
trace!(
|
||||
target: "engine::tree::payload_processor::multiproof",
|
||||
proof_sequence_number,
|
||||
?proof_targets,
|
||||
account_targets,
|
||||
storage_targets,
|
||||
account_targets = proof_targets.account_targets_len(),
|
||||
storage_targets = proof_targets.storage_targets_len(),
|
||||
?source,
|
||||
"Dispatching multiproof to workers"
|
||||
);
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
// Extend prefix sets with targets
|
||||
let frozen_prefix_sets =
|
||||
ParallelProof::extend_prefix_sets_with_targets(&Default::default(), &proof_targets);
|
||||
// Workers will send ProofResultMessage directly to proof_result_rx
|
||||
let proof_result_sender = ProofResultContext::new(
|
||||
self.proof_result_tx.clone(),
|
||||
proof_sequence_number,
|
||||
hashed_state_update,
|
||||
start,
|
||||
);
|
||||
|
||||
// Dispatch account multiproof to worker pool with result sender
|
||||
let input = AccountMultiproofInput {
|
||||
targets: proof_targets,
|
||||
prefix_sets: frozen_prefix_sets,
|
||||
collect_branch_node_masks: true,
|
||||
multi_added_removed_keys,
|
||||
// Workers will send ProofResultMessage directly to proof_result_rx
|
||||
proof_result_sender: ProofResultContext::new(
|
||||
self.proof_result_tx.clone(),
|
||||
proof_sequence_number,
|
||||
hashed_state_update,
|
||||
start,
|
||||
),
|
||||
v2_proofs_enabled: self.v2_proofs_enabled,
|
||||
let input = match proof_targets {
|
||||
VersionedMultiProofTargets::Legacy(proof_targets) => {
|
||||
// Extend prefix sets with targets
|
||||
let frozen_prefix_sets = ParallelProof::extend_prefix_sets_with_targets(
|
||||
&Default::default(),
|
||||
&proof_targets,
|
||||
);
|
||||
|
||||
AccountMultiproofInput::Legacy {
|
||||
targets: proof_targets,
|
||||
prefix_sets: frozen_prefix_sets,
|
||||
collect_branch_node_masks: true,
|
||||
multi_added_removed_keys,
|
||||
proof_result_sender,
|
||||
}
|
||||
}
|
||||
VersionedMultiProofTargets::V2(proof_targets) => {
|
||||
AccountMultiproofInput::V2 { targets: proof_targets, proof_result_sender }
|
||||
}
|
||||
};
|
||||
|
||||
// Dispatch account multiproof to worker pool with result sender
|
||||
if let Err(e) = self.proof_worker_handle.dispatch_account_multiproof(input) {
|
||||
error!(target: "engine::tree::payload_processor::multiproof", ?e, "Failed to dispatch account multiproof");
|
||||
return;
|
||||
@@ -556,6 +715,9 @@ pub(super) struct MultiProofTask {
|
||||
/// there are any active workers and force chunking across workers. This is to prevent tasks
|
||||
/// which are very long from hitting a single worker.
|
||||
max_targets_for_chunking: usize,
|
||||
/// Whether or not V2 proof calculation is enabled. If enabled then [`MultiProofTargetsV2`]
|
||||
/// will be produced by state updates.
|
||||
v2_proofs_enabled: bool,
|
||||
}
|
||||
|
||||
impl MultiProofTask {
|
||||
@@ -587,9 +749,16 @@ impl MultiProofTask {
|
||||
),
|
||||
metrics,
|
||||
max_targets_for_chunking: DEFAULT_MAX_TARGETS_FOR_CHUNKING,
|
||||
v2_proofs_enabled: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enables V2 proof target generation on state updates.
|
||||
pub(super) const fn with_v2_proofs_enabled(mut self, v2_proofs_enabled: bool) -> Self {
|
||||
self.v2_proofs_enabled = v2_proofs_enabled;
|
||||
self
|
||||
}
|
||||
|
||||
/// Handles request for proof prefetch.
|
||||
///
|
||||
/// Returns how many multiproof tasks were dispatched for the prefetch request.
|
||||
@@ -597,37 +766,29 @@ impl MultiProofTask {
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor::multiproof",
|
||||
skip_all,
|
||||
fields(accounts = targets.len(), chunks = 0)
|
||||
fields(accounts = targets.account_targets_len(), chunks = 0)
|
||||
)]
|
||||
fn on_prefetch_proof(&mut self, mut targets: MultiProofTargets) -> u64 {
|
||||
fn on_prefetch_proof(&mut self, mut targets: VersionedMultiProofTargets) -> u64 {
|
||||
// Remove already fetched proof targets to avoid redundant work.
|
||||
targets.retain_difference(&self.fetched_proof_targets);
|
||||
self.fetched_proof_targets.extend_ref(&targets);
|
||||
extend_multiproof_targets(&mut self.fetched_proof_targets, &targets);
|
||||
|
||||
// Make sure all target accounts have an `AddedRemovedKeySet` in the
|
||||
// For Legacy multiproofs, make sure all target accounts have an `AddedRemovedKeySet` in the
|
||||
// [`MultiAddedRemovedKeys`]. Even if there are not any known removed keys for the account,
|
||||
// we still want to optimistically fetch extension children for the leaf addition case.
|
||||
self.multi_added_removed_keys.touch_accounts(targets.keys().copied());
|
||||
|
||||
// Clone+Arc MultiAddedRemovedKeys for sharing with the dispatched multiproof tasks
|
||||
let multi_added_removed_keys = Arc::new(MultiAddedRemovedKeys {
|
||||
account: self.multi_added_removed_keys.account.clone(),
|
||||
storages: targets
|
||||
.keys()
|
||||
.filter_map(|account| {
|
||||
self.multi_added_removed_keys
|
||||
.storages
|
||||
.get(account)
|
||||
.cloned()
|
||||
.map(|keys| (*account, keys))
|
||||
})
|
||||
.collect(),
|
||||
});
|
||||
// V2 multiproofs don't need this.
|
||||
let multi_added_removed_keys =
|
||||
if let VersionedMultiProofTargets::Legacy(legacy_targets) = &targets {
|
||||
self.multi_added_removed_keys.touch_accounts(legacy_targets.keys().copied());
|
||||
Some(Arc::new(self.multi_added_removed_keys.clone()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
self.metrics.prefetch_proof_targets_accounts_histogram.record(targets.len() as f64);
|
||||
self.metrics
|
||||
.prefetch_proof_targets_storages_histogram
|
||||
.record(targets.values().map(|slots| slots.len()).sum::<usize>() as f64);
|
||||
.record(targets.storage_count() as f64);
|
||||
|
||||
let chunking_len = targets.chunking_length();
|
||||
let available_account_workers =
|
||||
@@ -641,7 +802,7 @@ impl MultiProofTask {
|
||||
self.max_targets_for_chunking,
|
||||
available_account_workers,
|
||||
available_storage_workers,
|
||||
MultiProofTargets::chunks,
|
||||
VersionedMultiProofTargets::chunks,
|
||||
|proof_targets| {
|
||||
self.multiproof_manager.dispatch(MultiproofInput {
|
||||
source: None,
|
||||
@@ -649,7 +810,7 @@ impl MultiProofTask {
|
||||
proof_targets,
|
||||
proof_sequence_number: self.proof_sequencer.next_sequence(),
|
||||
state_root_message_sender: self.tx.clone(),
|
||||
multi_added_removed_keys: Some(multi_added_removed_keys.clone()),
|
||||
multi_added_removed_keys: multi_added_removed_keys.clone(),
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -752,6 +913,7 @@ impl MultiProofTask {
|
||||
self.multiproof_manager.proof_worker_handle.available_account_workers();
|
||||
let available_storage_workers =
|
||||
self.multiproof_manager.proof_worker_handle.available_storage_workers();
|
||||
|
||||
let num_chunks = dispatch_with_chunking(
|
||||
not_fetched_state_update,
|
||||
chunking_len,
|
||||
@@ -765,8 +927,9 @@ impl MultiProofTask {
|
||||
&hashed_state_update,
|
||||
&self.fetched_proof_targets,
|
||||
&multi_added_removed_keys,
|
||||
self.v2_proofs_enabled,
|
||||
);
|
||||
spawned_proof_targets.extend_ref(&proof_targets);
|
||||
extend_multiproof_targets(&mut spawned_proof_targets, &proof_targets);
|
||||
|
||||
self.multiproof_manager.dispatch(MultiproofInput {
|
||||
source: Some(source),
|
||||
@@ -866,7 +1029,10 @@ impl MultiProofTask {
|
||||
batch_metrics.proofs_processed += 1;
|
||||
if let Some(combined_update) = self.on_proof(
|
||||
sequence_number,
|
||||
SparseTrieUpdate { state, multiproof: Default::default() },
|
||||
SparseTrieUpdate {
|
||||
state,
|
||||
multiproof: ProofResult::empty(self.v2_proofs_enabled),
|
||||
},
|
||||
) {
|
||||
let _ = self.to_sparse_trie.send(combined_update);
|
||||
}
|
||||
@@ -893,8 +1059,7 @@ impl MultiProofTask {
|
||||
}
|
||||
|
||||
let account_targets = merged_targets.len();
|
||||
let storage_targets =
|
||||
merged_targets.values().map(|slots| slots.len()).sum::<usize>();
|
||||
let storage_targets = merged_targets.storage_count();
|
||||
batch_metrics.prefetch_proofs_requested += self.on_prefetch_proof(merged_targets);
|
||||
trace!(
|
||||
target: "engine::tree::payload_processor::multiproof",
|
||||
@@ -998,7 +1163,10 @@ impl MultiProofTask {
|
||||
|
||||
if let Some(combined_update) = self.on_proof(
|
||||
sequence_number,
|
||||
SparseTrieUpdate { state, multiproof: Default::default() },
|
||||
SparseTrieUpdate {
|
||||
state,
|
||||
multiproof: ProofResult::empty(self.v2_proofs_enabled),
|
||||
},
|
||||
) {
|
||||
let _ = self.to_sparse_trie.send(combined_update);
|
||||
}
|
||||
@@ -1101,7 +1269,7 @@ impl MultiProofTask {
|
||||
|
||||
let update = SparseTrieUpdate {
|
||||
state: proof_result.state,
|
||||
multiproof: proof_result_data.proof,
|
||||
multiproof: proof_result_data,
|
||||
};
|
||||
|
||||
if let Some(combined_update) =
|
||||
@@ -1191,7 +1359,7 @@ struct MultiproofBatchCtx {
|
||||
/// received.
|
||||
updates_finished_time: Option<Instant>,
|
||||
/// Reusable buffer for accumulating prefetch targets during batching.
|
||||
accumulated_prefetch_targets: Vec<MultiProofTargets>,
|
||||
accumulated_prefetch_targets: Vec<VersionedMultiProofTargets>,
|
||||
}
|
||||
|
||||
impl MultiproofBatchCtx {
|
||||
@@ -1237,40 +1405,77 @@ fn get_proof_targets(
|
||||
state_update: &HashedPostState,
|
||||
fetched_proof_targets: &MultiProofTargets,
|
||||
multi_added_removed_keys: &MultiAddedRemovedKeys,
|
||||
) -> MultiProofTargets {
|
||||
let mut targets = MultiProofTargets::default();
|
||||
v2_enabled: bool,
|
||||
) -> VersionedMultiProofTargets {
|
||||
if v2_enabled {
|
||||
let mut targets = MultiProofTargetsV2::default();
|
||||
|
||||
// first collect all new accounts (not previously fetched)
|
||||
for hashed_address in state_update.accounts.keys() {
|
||||
if !fetched_proof_targets.contains_key(hashed_address) {
|
||||
targets.insert(*hashed_address, HashSet::default());
|
||||
// first collect all new accounts (not previously fetched)
|
||||
for &hashed_address in state_update.accounts.keys() {
|
||||
if !fetched_proof_targets.contains_key(&hashed_address) {
|
||||
targets.account_targets.push(hashed_address.into());
|
||||
}
|
||||
}
|
||||
|
||||
// then process storage slots for all accounts in the state update
|
||||
for (hashed_address, storage) in &state_update.storages {
|
||||
let fetched = fetched_proof_targets.get(hashed_address);
|
||||
|
||||
// If the storage is wiped, we still need to fetch the account proof.
|
||||
if storage.wiped && fetched.is_none() {
|
||||
targets.account_targets.push(Into::<proof_v2::Target>::into(*hashed_address));
|
||||
continue
|
||||
}
|
||||
|
||||
let changed_slots = storage
|
||||
.storage
|
||||
.keys()
|
||||
.filter(|slot| !fetched.is_some_and(|f| f.contains(*slot)))
|
||||
.map(|slot| Into::<proof_v2::Target>::into(*slot))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if !changed_slots.is_empty() {
|
||||
targets.account_targets.push((*hashed_address).into());
|
||||
targets.storage_targets.insert(*hashed_address, changed_slots);
|
||||
}
|
||||
}
|
||||
|
||||
VersionedMultiProofTargets::V2(targets)
|
||||
} else {
|
||||
let mut targets = MultiProofTargets::default();
|
||||
|
||||
// first collect all new accounts (not previously fetched)
|
||||
for hashed_address in state_update.accounts.keys() {
|
||||
if !fetched_proof_targets.contains_key(hashed_address) {
|
||||
targets.insert(*hashed_address, HashSet::default());
|
||||
}
|
||||
}
|
||||
|
||||
// then process storage slots for all accounts in the state update
|
||||
for (hashed_address, storage) in &state_update.storages {
|
||||
let fetched = fetched_proof_targets.get(hashed_address);
|
||||
let storage_added_removed_keys = multi_added_removed_keys.get_storage(hashed_address);
|
||||
let mut changed_slots = storage
|
||||
.storage
|
||||
.keys()
|
||||
.filter(|slot| {
|
||||
!fetched.is_some_and(|f| f.contains(*slot)) ||
|
||||
storage_added_removed_keys.is_some_and(|k| k.is_removed(slot))
|
||||
})
|
||||
.peekable();
|
||||
|
||||
// If the storage is wiped, we still need to fetch the account proof.
|
||||
if storage.wiped && fetched.is_none() {
|
||||
targets.entry(*hashed_address).or_default();
|
||||
}
|
||||
|
||||
if changed_slots.peek().is_some() {
|
||||
targets.entry(*hashed_address).or_default().extend(changed_slots);
|
||||
}
|
||||
}
|
||||
|
||||
VersionedMultiProofTargets::Legacy(targets)
|
||||
}
|
||||
|
||||
// then process storage slots for all accounts in the state update
|
||||
for (hashed_address, storage) in &state_update.storages {
|
||||
let fetched = fetched_proof_targets.get(hashed_address);
|
||||
let storage_added_removed_keys = multi_added_removed_keys.get_storage(hashed_address);
|
||||
let mut changed_slots = storage
|
||||
.storage
|
||||
.keys()
|
||||
.filter(|slot| {
|
||||
!fetched.is_some_and(|f| f.contains(*slot)) ||
|
||||
storage_added_removed_keys.is_some_and(|k| k.is_removed(slot))
|
||||
})
|
||||
.peekable();
|
||||
|
||||
// If the storage is wiped, we still need to fetch the account proof.
|
||||
if storage.wiped && fetched.is_none() {
|
||||
targets.entry(*hashed_address).or_default();
|
||||
}
|
||||
|
||||
if changed_slots.peek().is_some() {
|
||||
targets.entry(*hashed_address).or_default().extend(changed_slots);
|
||||
}
|
||||
}
|
||||
|
||||
targets
|
||||
}
|
||||
|
||||
/// Dispatches work items as a single unit or in chunks based on target size and worker
|
||||
@@ -1318,7 +1523,7 @@ mod tests {
|
||||
use reth_provider::{
|
||||
providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory,
|
||||
BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, LatestStateProvider,
|
||||
PruneCheckpointReader, StageCheckpointReader, StateProviderBox,
|
||||
PruneCheckpointReader, StageCheckpointReader, StateProviderBox, StorageChangeSetReader,
|
||||
};
|
||||
use reth_trie::MultiProof;
|
||||
use reth_trie_db::ChangesetCache;
|
||||
@@ -1345,6 +1550,7 @@ mod tests {
|
||||
+ StageCheckpointReader
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader,
|
||||
> + Clone
|
||||
+ Send
|
||||
@@ -1476,12 +1682,24 @@ mod tests {
|
||||
state
|
||||
}
|
||||
|
||||
fn unwrap_legacy_targets(targets: VersionedMultiProofTargets) -> MultiProofTargets {
|
||||
match targets {
|
||||
VersionedMultiProofTargets::Legacy(targets) => targets,
|
||||
VersionedMultiProofTargets::V2(_) => panic!("Expected Legacy targets"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_proof_targets_new_account_targets() {
|
||||
let state = create_get_proof_targets_state();
|
||||
let fetched = MultiProofTargets::default();
|
||||
|
||||
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&MultiAddedRemovedKeys::new(),
|
||||
false,
|
||||
));
|
||||
|
||||
// should return all accounts as targets since nothing was fetched before
|
||||
assert_eq!(targets.len(), state.accounts.len());
|
||||
@@ -1495,7 +1713,12 @@ mod tests {
|
||||
let state = create_get_proof_targets_state();
|
||||
let fetched = MultiProofTargets::default();
|
||||
|
||||
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&MultiAddedRemovedKeys::new(),
|
||||
false,
|
||||
));
|
||||
|
||||
// verify storage slots are included for accounts with storage
|
||||
for (addr, storage) in &state.storages {
|
||||
@@ -1523,7 +1746,12 @@ mod tests {
|
||||
// mark the account as already fetched
|
||||
fetched.insert(*fetched_addr, HashSet::default());
|
||||
|
||||
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&MultiAddedRemovedKeys::new(),
|
||||
false,
|
||||
));
|
||||
|
||||
// should not include the already fetched account since it has no storage updates
|
||||
assert!(!targets.contains_key(fetched_addr));
|
||||
@@ -1543,7 +1771,12 @@ mod tests {
|
||||
fetched_slots.insert(fetched_slot);
|
||||
fetched.insert(*addr, fetched_slots);
|
||||
|
||||
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&MultiAddedRemovedKeys::new(),
|
||||
false,
|
||||
));
|
||||
|
||||
// should not include the already fetched storage slot
|
||||
let target_slots = &targets[addr];
|
||||
@@ -1556,7 +1789,12 @@ mod tests {
|
||||
let state = HashedPostState::default();
|
||||
let fetched = MultiProofTargets::default();
|
||||
|
||||
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&MultiAddedRemovedKeys::new(),
|
||||
false,
|
||||
));
|
||||
|
||||
assert!(targets.is_empty());
|
||||
}
|
||||
@@ -1583,7 +1821,12 @@ mod tests {
|
||||
fetched_slots.insert(slot1);
|
||||
fetched.insert(addr1, fetched_slots);
|
||||
|
||||
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&MultiAddedRemovedKeys::new(),
|
||||
false,
|
||||
));
|
||||
|
||||
assert!(targets.contains_key(&addr2));
|
||||
assert!(!targets[&addr1].contains(&slot1));
|
||||
@@ -1609,7 +1852,12 @@ mod tests {
|
||||
assert!(!state.accounts.contains_key(&addr));
|
||||
assert!(!fetched.contains_key(&addr));
|
||||
|
||||
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&MultiAddedRemovedKeys::new(),
|
||||
false,
|
||||
));
|
||||
|
||||
// verify that we still get the storage slots for the unmodified account
|
||||
assert!(targets.contains_key(&addr));
|
||||
@@ -1651,7 +1899,12 @@ mod tests {
|
||||
removed_state.storages.insert(addr, removed_storage);
|
||||
multi_added_removed_keys.update_with_state(&removed_state);
|
||||
|
||||
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&multi_added_removed_keys,
|
||||
false,
|
||||
));
|
||||
|
||||
// slot1 should be included despite being fetched, because it's marked as removed
|
||||
assert!(targets.contains_key(&addr));
|
||||
@@ -1678,7 +1931,12 @@ mod tests {
|
||||
storage.storage.insert(slot1, U256::from(100));
|
||||
state.storages.insert(addr, storage);
|
||||
|
||||
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&multi_added_removed_keys,
|
||||
false,
|
||||
));
|
||||
|
||||
// account should be included because storage is wiped and account wasn't fetched
|
||||
assert!(targets.contains_key(&addr));
|
||||
@@ -1721,7 +1979,12 @@ mod tests {
|
||||
removed_state.storages.insert(addr, removed_storage);
|
||||
multi_added_removed_keys.update_with_state(&removed_state);
|
||||
|
||||
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
|
||||
let targets = unwrap_legacy_targets(get_proof_targets(
|
||||
&state,
|
||||
&fetched,
|
||||
&multi_added_removed_keys,
|
||||
false,
|
||||
));
|
||||
|
||||
// only slots in the state update can be included, so slot3 should not appear
|
||||
assert!(!targets.contains_key(&addr));
|
||||
@@ -1748,9 +2011,12 @@ mod tests {
|
||||
targets3.insert(addr3, HashSet::default());
|
||||
|
||||
let tx = task.tx.clone();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(targets1)).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(targets2)).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(targets3)).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets1)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets2)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets3)))
|
||||
.unwrap();
|
||||
|
||||
let proofs_requested =
|
||||
if let Ok(MultiProofMessage::PrefetchProofs(targets)) = task.rx.recv() {
|
||||
@@ -1764,11 +2030,12 @@ mod tests {
|
||||
|
||||
assert_eq!(num_batched, 3);
|
||||
assert_eq!(merged_targets.len(), 3);
|
||||
assert!(merged_targets.contains_key(&addr1));
|
||||
assert!(merged_targets.contains_key(&addr2));
|
||||
assert!(merged_targets.contains_key(&addr3));
|
||||
let legacy_targets = unwrap_legacy_targets(merged_targets);
|
||||
assert!(legacy_targets.contains_key(&addr1));
|
||||
assert!(legacy_targets.contains_key(&addr2));
|
||||
assert!(legacy_targets.contains_key(&addr3));
|
||||
|
||||
task.on_prefetch_proof(merged_targets)
|
||||
task.on_prefetch_proof(VersionedMultiProofTargets::Legacy(legacy_targets))
|
||||
} else {
|
||||
panic!("Expected PrefetchProofs message");
|
||||
};
|
||||
@@ -1811,7 +2078,9 @@ mod tests {
|
||||
nonce: 1,
|
||||
code_hash: Default::default(),
|
||||
code: Default::default(),
|
||||
account_id: None,
|
||||
},
|
||||
original_info: Box::new(revm_state::AccountInfo::default()),
|
||||
transaction_id: Default::default(),
|
||||
storage: Default::default(),
|
||||
status: revm_state::AccountStatus::Touched,
|
||||
@@ -1828,7 +2097,9 @@ mod tests {
|
||||
nonce: 2,
|
||||
code_hash: Default::default(),
|
||||
code: Default::default(),
|
||||
account_id: None,
|
||||
},
|
||||
original_info: Box::new(revm_state::AccountInfo::default()),
|
||||
transaction_id: Default::default(),
|
||||
storage: Default::default(),
|
||||
status: revm_state::AccountStatus::Touched,
|
||||
@@ -1839,11 +2110,16 @@ mod tests {
|
||||
|
||||
// Queue: [PrefetchProofs1, PrefetchProofs2, StateUpdate1, StateUpdate2, PrefetchProofs3]
|
||||
let tx = task.tx.clone();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(targets1)).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(targets2)).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets1)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets2)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update1)).unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update2)).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(targets3.clone())).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(
|
||||
targets3.clone(),
|
||||
)))
|
||||
.unwrap();
|
||||
|
||||
// Step 1: Receive and batch PrefetchProofs (should get targets1 + targets2)
|
||||
let mut pending_msg: Option<MultiProofMessage> = None;
|
||||
@@ -1869,9 +2145,10 @@ mod tests {
|
||||
// Should have batched exactly 2 PrefetchProofs (not 3!)
|
||||
assert_eq!(num_batched, 2, "Should batch only until different message type");
|
||||
assert_eq!(merged_targets.len(), 2);
|
||||
assert!(merged_targets.contains_key(&addr1));
|
||||
assert!(merged_targets.contains_key(&addr2));
|
||||
assert!(!merged_targets.contains_key(&addr3), "addr3 should NOT be in first batch");
|
||||
let legacy_targets = unwrap_legacy_targets(merged_targets);
|
||||
assert!(legacy_targets.contains_key(&addr1));
|
||||
assert!(legacy_targets.contains_key(&addr2));
|
||||
assert!(!legacy_targets.contains_key(&addr3), "addr3 should NOT be in first batch");
|
||||
} else {
|
||||
panic!("Expected PrefetchProofs message");
|
||||
}
|
||||
@@ -1896,7 +2173,8 @@ mod tests {
|
||||
match task.rx.try_recv() {
|
||||
Ok(MultiProofMessage::PrefetchProofs(targets)) => {
|
||||
assert_eq!(targets.len(), 1);
|
||||
assert!(targets.contains_key(&addr3));
|
||||
let legacy_targets = unwrap_legacy_targets(targets);
|
||||
assert!(legacy_targets.contains_key(&addr3));
|
||||
}
|
||||
_ => panic!("PrefetchProofs3 was lost!"),
|
||||
}
|
||||
@@ -1930,7 +2208,9 @@ mod tests {
|
||||
nonce: 1,
|
||||
code_hash: Default::default(),
|
||||
code: Default::default(),
|
||||
account_id: None,
|
||||
},
|
||||
original_info: Box::new(revm_state::AccountInfo::default()),
|
||||
transaction_id: Default::default(),
|
||||
storage: Default::default(),
|
||||
status: revm_state::AccountStatus::Touched,
|
||||
@@ -1940,9 +2220,13 @@ mod tests {
|
||||
let source = StateChangeSource::Transaction(99);
|
||||
|
||||
let tx = task.tx.clone();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(prefetch1)).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(prefetch1)))
|
||||
.unwrap();
|
||||
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update)).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(prefetch2.clone())).unwrap();
|
||||
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(
|
||||
prefetch2.clone(),
|
||||
)))
|
||||
.unwrap();
|
||||
|
||||
let mut ctx = MultiproofBatchCtx::new(Instant::now());
|
||||
let mut batch_metrics = MultiproofBatchMetrics::default();
|
||||
@@ -1975,7 +2259,8 @@ mod tests {
|
||||
match task.rx.try_recv() {
|
||||
Ok(MultiProofMessage::PrefetchProofs(targets)) => {
|
||||
assert_eq!(targets.len(), 1);
|
||||
assert!(targets.contains_key(&prefetch_addr2));
|
||||
let legacy_targets = unwrap_legacy_targets(targets);
|
||||
assert!(legacy_targets.contains_key(&prefetch_addr2));
|
||||
}
|
||||
other => panic!("Expected PrefetchProofs2 in channel, got {:?}", other),
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ use crate::tree::{
|
||||
payload_processor::{
|
||||
bal::{total_slots, BALSlotIter},
|
||||
executor::WorkloadExecutor,
|
||||
multiproof::MultiProofMessage,
|
||||
multiproof::{MultiProofMessage, VersionedMultiProofTargets},
|
||||
ExecutionCache as PayloadExecutionCache,
|
||||
},
|
||||
precompile_cache::{CachedPrecompile, PrecompileCacheMap},
|
||||
@@ -29,11 +29,13 @@ use alloy_evm::Database;
|
||||
use alloy_primitives::{keccak256, map::B256Set, B256};
|
||||
use crossbeam_channel::Sender as CrossbeamSender;
|
||||
use metrics::{Counter, Gauge, Histogram};
|
||||
use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, SpecFor};
|
||||
use reth_execution_types::ExecutionOutcome;
|
||||
use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, RecoveredTx, SpecFor};
|
||||
use reth_metrics::Metrics;
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_provider::{AccountReader, BlockReader, StateProvider, StateProviderFactory, StateReader};
|
||||
use reth_provider::{
|
||||
AccountReader, BlockExecutionOutput, BlockReader, StateProvider, StateProviderFactory,
|
||||
StateReader,
|
||||
};
|
||||
use reth_revm::{database::StateProviderDatabase, state::EvmState};
|
||||
use reth_trie::MultiProofTargets;
|
||||
use std::{
|
||||
@@ -235,7 +237,7 @@ where
|
||||
}
|
||||
|
||||
/// If configured and the tx returned proof targets, emit the targets the transaction produced
|
||||
fn send_multi_proof_targets(&self, targets: Option<MultiProofTargets>) {
|
||||
fn send_multi_proof_targets(&self, targets: Option<VersionedMultiProofTargets>) {
|
||||
if self.is_execution_terminated() {
|
||||
// if execution is already terminated then we dont need to send more proof fetch
|
||||
// messages
|
||||
@@ -259,7 +261,11 @@ where
|
||||
///
|
||||
/// This method is called from `run()` only after all execution tasks are complete.
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
|
||||
fn save_cache(self, execution_outcome: Arc<ExecutionOutcome<N::Receipt>>) {
|
||||
fn save_cache(
|
||||
self,
|
||||
execution_outcome: Arc<BlockExecutionOutput<N::Receipt>>,
|
||||
valid_block_rx: mpsc::Receiver<()>,
|
||||
) {
|
||||
let start = Instant::now();
|
||||
|
||||
let Self { execution_cache, ctx: PrewarmContext { env, metrics, saved_cache, .. }, .. } =
|
||||
@@ -272,12 +278,13 @@ where
|
||||
execution_cache.update_with_guard(|cached| {
|
||||
// consumes the `SavedCache` held by the prewarming task, which releases its usage
|
||||
// guard
|
||||
let (caches, cache_metrics) = saved_cache.split();
|
||||
let new_cache = SavedCache::new(hash, caches, cache_metrics);
|
||||
let (caches, cache_metrics, disable_cache_metrics) = saved_cache.split();
|
||||
let new_cache = SavedCache::new(hash, caches, cache_metrics)
|
||||
.with_disable_cache_metrics(disable_cache_metrics);
|
||||
|
||||
// Insert state into cache while holding the lock
|
||||
// Access the BundleState through the shared ExecutionOutcome
|
||||
if new_cache.cache().insert_state(execution_outcome.state()).is_err() {
|
||||
if new_cache.cache().insert_state(&execution_outcome.state).is_err() {
|
||||
// Clear the cache on error to prevent having a polluted cache
|
||||
*cached = None;
|
||||
debug!(target: "engine::caching", "cleared execution cache on update error");
|
||||
@@ -286,9 +293,16 @@ where
|
||||
|
||||
new_cache.update_metrics();
|
||||
|
||||
// Replace the shared cache with the new one; the previous cache (if any) is
|
||||
// dropped.
|
||||
*cached = Some(new_cache);
|
||||
if valid_block_rx.recv().is_ok() {
|
||||
// Replace the shared cache with the new one; the previous cache (if any) is
|
||||
// dropped.
|
||||
*cached = Some(new_cache);
|
||||
} else {
|
||||
// Block was invalid; caches were already mutated by insert_state above,
|
||||
// so we must clear to prevent using polluted state
|
||||
*cached = None;
|
||||
debug!(target: "engine::caching", "cleared execution cache on invalid block");
|
||||
}
|
||||
});
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
@@ -419,9 +433,10 @@ where
|
||||
// completed executing a set of transactions
|
||||
self.send_multi_proof_targets(proof_targets);
|
||||
}
|
||||
PrewarmTaskEvent::Terminate { execution_outcome } => {
|
||||
PrewarmTaskEvent::Terminate { execution_outcome, valid_block_rx } => {
|
||||
trace!(target: "engine::tree::payload_processor::prewarm", "Received termination signal");
|
||||
final_execution_outcome = Some(execution_outcome);
|
||||
final_execution_outcome =
|
||||
Some(execution_outcome.map(|outcome| (outcome, valid_block_rx)));
|
||||
|
||||
if finished_execution {
|
||||
// all tasks are done, we can exit, which will save caches and exit
|
||||
@@ -446,8 +461,8 @@ where
|
||||
debug!(target: "engine::tree::payload_processor::prewarm", "Completed prewarm execution");
|
||||
|
||||
// save caches and finish using the shared ExecutionOutcome
|
||||
if let Some(Some(execution_outcome)) = final_execution_outcome {
|
||||
self.save_cache(execution_outcome);
|
||||
if let Some(Some((execution_outcome, valid_block_rx))) = final_execution_outcome {
|
||||
self.save_cache(execution_outcome, valid_block_rx);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -469,6 +484,8 @@ where
|
||||
pub(super) terminate_execution: Arc<AtomicBool>,
|
||||
pub(super) precompile_cache_disabled: bool,
|
||||
pub(super) precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
|
||||
/// Whether V2 proof calculation is enabled.
|
||||
pub(super) v2_proofs_enabled: bool,
|
||||
}
|
||||
|
||||
impl<N, P, Evm> PrewarmContext<N, P, Evm>
|
||||
@@ -477,10 +494,12 @@ where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
Evm: ConfigureEvm<Primitives = N> + 'static,
|
||||
{
|
||||
/// Splits this context into an evm, an evm config, metrics, and the atomic bool for terminating
|
||||
/// execution.
|
||||
/// Splits this context into an evm, an evm config, metrics, the atomic bool for terminating
|
||||
/// execution, and whether V2 proofs are enabled.
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
|
||||
fn evm_for_ctx(self) -> Option<(EvmFor<Evm, impl Database>, PrewarmMetrics, Arc<AtomicBool>)> {
|
||||
fn evm_for_ctx(
|
||||
self,
|
||||
) -> Option<(EvmFor<Evm, impl Database>, PrewarmMetrics, Arc<AtomicBool>, bool)> {
|
||||
let Self {
|
||||
env,
|
||||
evm_config,
|
||||
@@ -490,6 +509,7 @@ where
|
||||
terminate_execution,
|
||||
precompile_cache_disabled,
|
||||
precompile_cache_map,
|
||||
v2_proofs_enabled,
|
||||
} = self;
|
||||
|
||||
let mut state_provider = match provider.build() {
|
||||
@@ -539,7 +559,7 @@ where
|
||||
});
|
||||
}
|
||||
|
||||
Some((evm, metrics, terminate_execution))
|
||||
Some((evm, metrics, terminate_execution, v2_proofs_enabled))
|
||||
}
|
||||
|
||||
/// Accepts an [`mpsc::Receiver`] of transactions and a handle to prewarm task. Executes
|
||||
@@ -560,16 +580,24 @@ where
|
||||
) where
|
||||
Tx: ExecutableTxFor<Evm>,
|
||||
{
|
||||
let Some((mut evm, metrics, terminate_execution)) = self.evm_for_ctx() else { return };
|
||||
let Some((mut evm, metrics, terminate_execution, v2_proofs_enabled)) = self.evm_for_ctx()
|
||||
else {
|
||||
return
|
||||
};
|
||||
|
||||
while let Ok(IndexedTransaction { index, tx }) = {
|
||||
let _enter = debug_span!(target: "engine::tree::payload_processor::prewarm", "recv tx")
|
||||
.entered();
|
||||
txs.recv()
|
||||
} {
|
||||
let enter =
|
||||
debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm tx", index, tx_hash=%tx.tx().tx_hash())
|
||||
.entered();
|
||||
let enter = debug_span!(
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
"prewarm tx",
|
||||
index,
|
||||
tx_hash = %tx.tx().tx_hash(),
|
||||
is_success = tracing::field::Empty,
|
||||
)
|
||||
.entered();
|
||||
|
||||
// create the tx env
|
||||
let start = Instant::now();
|
||||
@@ -581,7 +609,8 @@ where
|
||||
break
|
||||
}
|
||||
|
||||
let res = match evm.transact(&tx) {
|
||||
let (tx_env, tx) = tx.into_parts();
|
||||
let res = match evm.transact(tx_env) {
|
||||
Ok(res) => res,
|
||||
Err(err) => {
|
||||
trace!(
|
||||
@@ -618,7 +647,8 @@ where
|
||||
let _enter =
|
||||
debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm outcome", index, tx_hash=%tx.tx().tx_hash())
|
||||
.entered();
|
||||
let (targets, storage_targets) = multiproof_targets_from_state(res.state);
|
||||
let (targets, storage_targets) =
|
||||
multiproof_targets_from_state(res.state, v2_proofs_enabled);
|
||||
metrics.prefetch_storage_targets.record(storage_targets as f64);
|
||||
let _ = sender.send(PrewarmTaskEvent::Outcome { proof_targets: Some(targets) });
|
||||
drop(_enter);
|
||||
@@ -763,9 +793,22 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a set of [`MultiProofTargets`] and the total amount of storage targets, based on the
|
||||
/// Returns a set of [`VersionedMultiProofTargets`] and the total amount of storage targets, based
|
||||
/// on the given state.
|
||||
fn multiproof_targets_from_state(
|
||||
state: EvmState,
|
||||
v2_enabled: bool,
|
||||
) -> (VersionedMultiProofTargets, usize) {
|
||||
if v2_enabled {
|
||||
multiproof_targets_v2_from_state(state)
|
||||
} else {
|
||||
multiproof_targets_legacy_from_state(state)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns legacy [`MultiProofTargets`] and the total amount of storage targets, based on the
|
||||
/// given state.
|
||||
fn multiproof_targets_from_state(state: EvmState) -> (MultiProofTargets, usize) {
|
||||
fn multiproof_targets_legacy_from_state(state: EvmState) -> (VersionedMultiProofTargets, usize) {
|
||||
let mut targets = MultiProofTargets::with_capacity(state.len());
|
||||
let mut storage_targets = 0;
|
||||
for (addr, account) in state {
|
||||
@@ -795,7 +838,50 @@ fn multiproof_targets_from_state(state: EvmState) -> (MultiProofTargets, usize)
|
||||
targets.insert(keccak256(addr), storage_set);
|
||||
}
|
||||
|
||||
(targets, storage_targets)
|
||||
(VersionedMultiProofTargets::Legacy(targets), storage_targets)
|
||||
}
|
||||
|
||||
/// Returns V2 [`reth_trie_parallel::targets_v2::MultiProofTargetsV2`] and the total amount of
|
||||
/// storage targets, based on the given state.
|
||||
fn multiproof_targets_v2_from_state(state: EvmState) -> (VersionedMultiProofTargets, usize) {
|
||||
use reth_trie::proof_v2;
|
||||
use reth_trie_parallel::targets_v2::MultiProofTargetsV2;
|
||||
|
||||
let mut targets = MultiProofTargetsV2::default();
|
||||
let mut storage_target_count = 0;
|
||||
for (addr, account) in state {
|
||||
// if the account was not touched, or if the account was selfdestructed, do not
|
||||
// fetch proofs for it
|
||||
//
|
||||
// Since selfdestruct can only happen in the same transaction, we can skip
|
||||
// prefetching proofs for selfdestructed accounts
|
||||
//
|
||||
// See: https://eips.ethereum.org/EIPS/eip-6780
|
||||
if !account.is_touched() || account.is_selfdestructed() {
|
||||
continue
|
||||
}
|
||||
|
||||
let hashed_address = keccak256(addr);
|
||||
targets.account_targets.push(hashed_address.into());
|
||||
|
||||
let mut storage_slots = Vec::with_capacity(account.storage.len());
|
||||
for (key, slot) in account.storage {
|
||||
// do nothing if unchanged
|
||||
if !slot.is_changed() {
|
||||
continue
|
||||
}
|
||||
|
||||
let hashed_slot = keccak256(B256::new(key.to_be_bytes()));
|
||||
storage_slots.push(proof_v2::Target::from(hashed_slot));
|
||||
}
|
||||
|
||||
storage_target_count += storage_slots.len();
|
||||
if !storage_slots.is_empty() {
|
||||
targets.storage_targets.insert(hashed_address, storage_slots);
|
||||
}
|
||||
}
|
||||
|
||||
(VersionedMultiProofTargets::V2(targets), storage_target_count)
|
||||
}
|
||||
|
||||
/// The events the pre-warm task can handle.
|
||||
@@ -810,12 +896,17 @@ pub(super) enum PrewarmTaskEvent<R> {
|
||||
Terminate {
|
||||
/// The final execution outcome. Using `Arc` allows sharing with the main execution
|
||||
/// path without cloning the expensive `BundleState`.
|
||||
execution_outcome: Option<Arc<ExecutionOutcome<R>>>,
|
||||
execution_outcome: Option<Arc<BlockExecutionOutput<R>>>,
|
||||
/// Receiver for the block validation result.
|
||||
///
|
||||
/// Cache saving is racing the state root validation. We optimistically construct the
|
||||
/// updated cache but only save it once we know the block is valid.
|
||||
valid_block_rx: mpsc::Receiver<()>,
|
||||
},
|
||||
/// The outcome of a pre-warm task
|
||||
Outcome {
|
||||
/// The prepared proof targets based on the evm state outcome
|
||||
proof_targets: Option<MultiProofTargets>,
|
||||
proof_targets: Option<VersionedMultiProofTargets>,
|
||||
},
|
||||
/// Finished executing all transactions
|
||||
FinishedTxExecution {
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
//! Receipt root computation in a background task.
|
||||
//!
|
||||
//! This module provides a streaming receipt root builder that computes the receipt trie root
|
||||
//! in a background thread. Receipts are sent via a channel with their index, and for each
|
||||
//! receipt received, the builder incrementally flushes leaves to the underlying
|
||||
//! [`OrderedTrieRootEncodedBuilder`] when possible. When the channel closes, the task returns the
|
||||
//! computed root.
|
||||
|
||||
use alloy_eips::Encodable2718;
|
||||
use alloy_primitives::{Bloom, B256};
|
||||
use crossbeam_channel::Receiver;
|
||||
use reth_primitives_traits::Receipt;
|
||||
use reth_trie_common::ordered_root::OrderedTrieRootEncodedBuilder;
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
/// Receipt with index, ready to be sent to the background task for encoding and trie building.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IndexedReceipt<R> {
|
||||
/// The transaction index within the block.
|
||||
pub index: usize,
|
||||
/// The receipt.
|
||||
pub receipt: R,
|
||||
}
|
||||
|
||||
impl<R> IndexedReceipt<R> {
|
||||
/// Creates a new indexed receipt.
|
||||
#[inline]
|
||||
pub const fn new(index: usize, receipt: R) -> Self {
|
||||
Self { index, receipt }
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle for running the receipt root computation in a background task.
|
||||
///
|
||||
/// This struct holds the channels needed to receive receipts and send the result.
|
||||
/// Use [`Self::run`] to execute the computation (typically in a spawned blocking task).
|
||||
#[derive(Debug)]
|
||||
pub struct ReceiptRootTaskHandle<R> {
|
||||
/// Receiver for indexed receipts.
|
||||
receipt_rx: Receiver<IndexedReceipt<R>>,
|
||||
/// Sender for the computed result.
|
||||
result_tx: oneshot::Sender<(B256, Bloom)>,
|
||||
}
|
||||
|
||||
impl<R: Receipt> ReceiptRootTaskHandle<R> {
|
||||
/// Creates a new handle from the receipt receiver and result sender channels.
|
||||
pub const fn new(
|
||||
receipt_rx: Receiver<IndexedReceipt<R>>,
|
||||
result_tx: oneshot::Sender<(B256, Bloom)>,
|
||||
) -> Self {
|
||||
Self { receipt_rx, result_tx }
|
||||
}
|
||||
|
||||
/// Runs the receipt root computation, consuming the handle.
|
||||
///
|
||||
/// This method receives indexed receipts from the channel, encodes them,
|
||||
/// and builds the trie incrementally. When all receipts have been received
|
||||
/// (channel closed), it sends the result through the oneshot channel.
|
||||
///
|
||||
/// This is designed to be called inside a blocking task (e.g., via
|
||||
/// `executor.spawn_blocking(move || handle.run(receipts_len))`).
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `receipts_len` - The total number of receipts expected. This is needed to correctly order
|
||||
/// the trie keys according to RLP encoding rules.
|
||||
pub fn run(self, receipts_len: usize) {
|
||||
let mut builder = OrderedTrieRootEncodedBuilder::new(receipts_len);
|
||||
let mut aggregated_bloom = Bloom::ZERO;
|
||||
let mut encode_buf = Vec::new();
|
||||
let mut received_count = 0usize;
|
||||
|
||||
for indexed_receipt in self.receipt_rx {
|
||||
let receipt_with_bloom = indexed_receipt.receipt.with_bloom_ref();
|
||||
|
||||
encode_buf.clear();
|
||||
receipt_with_bloom.encode_2718(&mut encode_buf);
|
||||
|
||||
aggregated_bloom |= *receipt_with_bloom.bloom_ref();
|
||||
builder.push_unchecked(indexed_receipt.index, &encode_buf);
|
||||
received_count += 1;
|
||||
}
|
||||
|
||||
let Ok(root) = builder.finalize() else {
|
||||
// Finalize fails if we didn't receive exactly `receipts_len` receipts. This can
|
||||
// happen if execution was aborted early (e.g., invalid transaction encountered).
|
||||
// We return without sending a result, allowing the caller to handle the abort.
|
||||
tracing::error!(
|
||||
target: "engine::tree::payload_processor",
|
||||
expected = receipts_len,
|
||||
received = received_count,
|
||||
"Receipt root task received incomplete receipts, execution likely aborted"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let _ = self.result_tx.send((root, aggregated_bloom));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_consensus::{proofs::calculate_receipt_root, TxReceipt};
|
||||
use alloy_primitives::{b256, hex, Address, Bytes, Log};
|
||||
use crossbeam_channel::bounded;
|
||||
use reth_ethereum_primitives::{Receipt, TxType};
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_receipt_root_task_empty() {
|
||||
let (_tx, rx) = bounded::<IndexedReceipt<Receipt>>(1);
|
||||
let (result_tx, result_rx) = oneshot::channel();
|
||||
drop(_tx);
|
||||
|
||||
let handle = ReceiptRootTaskHandle::new(rx, result_tx);
|
||||
tokio::task::spawn_blocking(move || handle.run(0)).await.unwrap();
|
||||
|
||||
let (root, bloom) = result_rx.await.unwrap();
|
||||
|
||||
// Empty trie root
|
||||
assert_eq!(root, reth_trie_common::EMPTY_ROOT_HASH);
|
||||
assert_eq!(bloom, Bloom::ZERO);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_receipt_root_task_single_receipt() {
|
||||
let receipts: Vec<Receipt> = vec![Receipt::default()];
|
||||
|
||||
let (tx, rx) = bounded(1);
|
||||
let (result_tx, result_rx) = oneshot::channel();
|
||||
let receipts_len = receipts.len();
|
||||
|
||||
let handle = ReceiptRootTaskHandle::new(rx, result_tx);
|
||||
let join_handle = tokio::task::spawn_blocking(move || handle.run(receipts_len));
|
||||
|
||||
for (i, receipt) in receipts.clone().into_iter().enumerate() {
|
||||
tx.send(IndexedReceipt::new(i, receipt)).unwrap();
|
||||
}
|
||||
drop(tx);
|
||||
|
||||
join_handle.await.unwrap();
|
||||
let (root, _bloom) = result_rx.await.unwrap();
|
||||
|
||||
// Verify against the standard calculation
|
||||
let receipts_with_bloom: Vec<_> = receipts.iter().map(|r| r.with_bloom_ref()).collect();
|
||||
let expected_root = calculate_receipt_root(&receipts_with_bloom);
|
||||
|
||||
assert_eq!(root, expected_root);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_receipt_root_task_multiple_receipts() {
|
||||
let receipts: Vec<Receipt> = vec![Receipt::default(); 5];
|
||||
|
||||
let (tx, rx) = bounded(4);
|
||||
let (result_tx, result_rx) = oneshot::channel();
|
||||
let receipts_len = receipts.len();
|
||||
|
||||
let handle = ReceiptRootTaskHandle::new(rx, result_tx);
|
||||
let join_handle = tokio::task::spawn_blocking(move || handle.run(receipts_len));
|
||||
|
||||
for (i, receipt) in receipts.into_iter().enumerate() {
|
||||
tx.send(IndexedReceipt::new(i, receipt)).unwrap();
|
||||
}
|
||||
drop(tx);
|
||||
|
||||
join_handle.await.unwrap();
|
||||
let (root, bloom) = result_rx.await.unwrap();
|
||||
|
||||
// Verify against expected values from existing test
|
||||
assert_eq!(
|
||||
root,
|
||||
b256!("0x61353b4fb714dc1fccacbf7eafc4273e62f3d1eed716fe41b2a0cd2e12c63ebc")
|
||||
);
|
||||
assert_eq!(
|
||||
bloom,
|
||||
Bloom::from(hex!("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_receipt_root_matches_standard_calculation() {
|
||||
// Create some receipts with actual data
|
||||
let receipts = vec![
|
||||
Receipt {
|
||||
tx_type: TxType::Legacy,
|
||||
cumulative_gas_used: 21000,
|
||||
success: true,
|
||||
logs: vec![],
|
||||
},
|
||||
Receipt {
|
||||
tx_type: TxType::Eip1559,
|
||||
cumulative_gas_used: 42000,
|
||||
success: true,
|
||||
logs: vec![Log {
|
||||
address: Address::ZERO,
|
||||
data: alloy_primitives::LogData::new_unchecked(vec![B256::ZERO], Bytes::new()),
|
||||
}],
|
||||
},
|
||||
Receipt {
|
||||
tx_type: TxType::Eip2930,
|
||||
cumulative_gas_used: 63000,
|
||||
success: false,
|
||||
logs: vec![],
|
||||
},
|
||||
];
|
||||
|
||||
// Calculate expected values first (before we move receipts)
|
||||
let receipts_with_bloom: Vec<_> = receipts.iter().map(|r| r.with_bloom_ref()).collect();
|
||||
let expected_root = calculate_receipt_root(&receipts_with_bloom);
|
||||
let expected_bloom =
|
||||
receipts_with_bloom.iter().fold(Bloom::ZERO, |bloom, r| bloom | r.bloom_ref());
|
||||
|
||||
// Calculate using the task
|
||||
let (tx, rx) = bounded(4);
|
||||
let (result_tx, result_rx) = oneshot::channel();
|
||||
let receipts_len = receipts.len();
|
||||
|
||||
let handle = ReceiptRootTaskHandle::new(rx, result_tx);
|
||||
let join_handle = tokio::task::spawn_blocking(move || handle.run(receipts_len));
|
||||
|
||||
for (i, receipt) in receipts.into_iter().enumerate() {
|
||||
tx.send(IndexedReceipt::new(i, receipt)).unwrap();
|
||||
}
|
||||
drop(tx);
|
||||
|
||||
join_handle.await.unwrap();
|
||||
let (task_root, task_bloom) = result_rx.await.unwrap();
|
||||
|
||||
assert_eq!(task_root, expected_root);
|
||||
assert_eq!(task_bloom, expected_bloom);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_receipt_root_task_out_of_order() {
|
||||
let receipts: Vec<Receipt> = vec![Receipt::default(); 5];
|
||||
|
||||
// Calculate expected values first (before we move receipts)
|
||||
let receipts_with_bloom: Vec<_> = receipts.iter().map(|r| r.with_bloom_ref()).collect();
|
||||
let expected_root = calculate_receipt_root(&receipts_with_bloom);
|
||||
|
||||
let (tx, rx) = bounded(4);
|
||||
let (result_tx, result_rx) = oneshot::channel();
|
||||
let receipts_len = receipts.len();
|
||||
|
||||
let handle = ReceiptRootTaskHandle::new(rx, result_tx);
|
||||
let join_handle = tokio::task::spawn_blocking(move || handle.run(receipts_len));
|
||||
|
||||
// Send in reverse order to test out-of-order handling
|
||||
for (i, receipt) in receipts.into_iter().enumerate().rev() {
|
||||
tx.send(IndexedReceipt::new(i, receipt)).unwrap();
|
||||
}
|
||||
drop(tx);
|
||||
|
||||
join_handle.await.unwrap();
|
||||
let (root, _bloom) = result_rx.await.unwrap();
|
||||
|
||||
assert_eq!(root, expected_root);
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,7 @@ use crate::tree::payload_processor::multiproof::{MultiProofTaskMetrics, SparseTr
|
||||
use alloy_primitives::B256;
|
||||
use rayon::iter::{ParallelBridge, ParallelIterator};
|
||||
use reth_trie::{updates::TrieUpdates, Nibbles};
|
||||
use reth_trie_parallel::root::ParallelStateRootError;
|
||||
use reth_trie_parallel::{proof_task::ProofResult, root::ParallelStateRootError};
|
||||
use reth_trie_sparse::{
|
||||
errors::{SparseStateTrieResult, SparseTrieErrorKind},
|
||||
provider::{TrieNodeProvider, TrieNodeProviderFactory},
|
||||
@@ -97,8 +97,8 @@ where
|
||||
debug!(
|
||||
target: "engine::root",
|
||||
num_updates,
|
||||
account_proofs = update.multiproof.account_subtree.len(),
|
||||
storage_proofs = update.multiproof.storages.len(),
|
||||
account_proofs = update.multiproof.account_proofs_len(),
|
||||
storage_proofs = update.multiproof.storage_proofs_len(),
|
||||
"Updating sparse trie"
|
||||
);
|
||||
|
||||
@@ -157,7 +157,14 @@ where
|
||||
let started_at = Instant::now();
|
||||
|
||||
// Reveal new accounts and storage slots.
|
||||
trie.reveal_decoded_multiproof(multiproof)?;
|
||||
match multiproof {
|
||||
ProofResult::Legacy(decoded, _) => {
|
||||
trie.reveal_decoded_multiproof(decoded)?;
|
||||
}
|
||||
ProofResult::V2(decoded_v2) => {
|
||||
trie.reveal_decoded_multiproof_v2(decoded_v2)?;
|
||||
}
|
||||
}
|
||||
let reveal_multiproof_elapsed = started_at.elapsed();
|
||||
trace!(
|
||||
target: "engine::root::sparse",
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
//! Types and traits for validating blocks and payloads.
|
||||
|
||||
/// Threshold for switching from `extend_ref` loop to `merge_batch` in `merge_overlay_trie_input`.
|
||||
///
|
||||
/// Benchmarked crossover: `extend_ref` wins up to ~64 blocks, `merge_batch` wins beyond.
|
||||
/// Using 64 as threshold since they're roughly equal there.
|
||||
const MERGE_BATCH_THRESHOLD: usize = 64;
|
||||
|
||||
use crate::tree::{
|
||||
cached_state::CachedStateProvider,
|
||||
error::{InsertBlockError, InsertBlockErrorKind, InsertPayloadError},
|
||||
@@ -13,17 +7,19 @@ use crate::tree::{
|
||||
payload_processor::{executor::WorkloadExecutor, PayloadProcessor},
|
||||
precompile_cache::{CachedPrecompile, CachedPrecompileMetrics, PrecompileCacheMap},
|
||||
sparse_trie::StateRootComputeOutcome,
|
||||
EngineApiMetrics, EngineApiTreeState, ExecutionEnv, PayloadHandle, StateProviderBuilder,
|
||||
StateProviderDatabase, TreeConfig,
|
||||
EngineApiMetrics, EngineApiTreeState, ExecutionEnv, MeteredStateHook, PayloadHandle,
|
||||
StateProviderBuilder, StateProviderDatabase, TreeConfig,
|
||||
};
|
||||
use alloy_consensus::transaction::Either;
|
||||
use alloy_consensus::transaction::{Either, TxHashRef};
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::{eip1898::BlockWithParent, NumHash};
|
||||
use alloy_evm::Evm;
|
||||
use alloy_primitives::B256;
|
||||
|
||||
use crate::tree::payload_processor::receipt_root_task::{IndexedReceipt, ReceiptRootTaskHandle};
|
||||
use rayon::prelude::*;
|
||||
use reth_chain_state::{CanonicalInMemoryState, DeferredTrieData, ExecutedBlock};
|
||||
use reth_consensus::{ConsensusError, FullConsensus};
|
||||
use reth_chain_state::{CanonicalInMemoryState, DeferredTrieData, ExecutedBlock, LazyOverlay};
|
||||
use reth_consensus::{ConsensusError, FullConsensus, ReceiptRootBloom};
|
||||
use reth_engine_primitives::{
|
||||
ConfigureEngineEvm, ExecutableTxIterator, ExecutionPayload, InvalidBlockHook, PayloadValidator,
|
||||
};
|
||||
@@ -41,15 +37,12 @@ use reth_primitives_traits::{
|
||||
};
|
||||
use reth_provider::{
|
||||
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader,
|
||||
ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, ExecutionOutcome,
|
||||
HashedPostStateProvider, ProviderError, PruneCheckpointReader, StageCheckpointReader,
|
||||
StateProvider, StateProviderFactory, StateReader,
|
||||
};
|
||||
use reth_revm::db::State;
|
||||
use reth_trie::{
|
||||
updates::{TrieUpdates, TrieUpdatesSorted},
|
||||
HashedPostState, HashedPostStateSorted, StateRoot, TrieInputSorted,
|
||||
ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider,
|
||||
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
|
||||
StateProviderFactory, StateReader, StorageChangeSetReader,
|
||||
};
|
||||
use reth_revm::db::{states::bundle_state::BundleRetention, State};
|
||||
use reth_trie::{updates::TrieUpdates, HashedPostState, StateRoot};
|
||||
use reth_trie_db::ChangesetCache;
|
||||
use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError};
|
||||
use revm_primitives::Address;
|
||||
@@ -151,6 +144,7 @@ where
|
||||
+ StageCheckpointReader
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader,
|
||||
> + BlockReader<Header = N::BlockHeader>
|
||||
+ ChangeSetReader
|
||||
@@ -376,7 +370,6 @@ where
|
||||
}
|
||||
|
||||
let parent_hash = input.parent_hash();
|
||||
let block_num_hash = input.num_hash();
|
||||
|
||||
trace!(target: "engine::tree::payload_validator", "Fetching block state provider");
|
||||
let _enter =
|
||||
@@ -431,26 +424,16 @@ where
|
||||
.map_err(Box::<dyn std::error::Error + Send + Sync>::from))
|
||||
.map(Arc::new);
|
||||
|
||||
// Compute trie input from ancestors once, before spawning payload processor.
|
||||
// This will be extended with the current block's hashed state after execution.
|
||||
let trie_input_start = Instant::now();
|
||||
let (trie_input, block_hash_for_overlay) =
|
||||
ensure_ok!(self.compute_trie_input(parent_hash, ctx.state()));
|
||||
|
||||
self.metrics
|
||||
.block_validation
|
||||
.trie_input_duration
|
||||
.record(trie_input_start.elapsed().as_secs_f64());
|
||||
// Create lazy overlay from ancestors - this doesn't block, allowing execution to start
|
||||
// before the trie data is ready. The overlay will be computed on first access.
|
||||
let (lazy_overlay, anchor_hash) = Self::get_parent_lazy_overlay(parent_hash, ctx.state());
|
||||
|
||||
// Create overlay factory for payload processor (StateRootTask path needs it for
|
||||
// multiproofs)
|
||||
let overlay_factory = {
|
||||
let TrieInputSorted { nodes, state, .. } = &trie_input;
|
||||
let overlay_factory =
|
||||
OverlayStateProviderFactory::new(self.provider.clone(), self.changeset_cache.clone())
|
||||
.with_block_hash(Some(block_hash_for_overlay))
|
||||
.with_trie_overlay(Some(Arc::clone(nodes)))
|
||||
.with_hashed_state_overlay(Some(Arc::clone(state)))
|
||||
};
|
||||
.with_block_hash(Some(anchor_hash))
|
||||
.with_lazy_overlay(lazy_overlay);
|
||||
|
||||
// Spawn the appropriate processor based on strategy
|
||||
let mut handle = ensure_ok!(self.spawn_payload_processor(
|
||||
@@ -473,19 +456,48 @@ where
|
||||
state_provider = Box::new(InstrumentedStateProvider::new(state_provider, "engine"));
|
||||
}
|
||||
|
||||
// Execute the block and handle any execution errors
|
||||
let (output, senders) = match self.execute_block(state_provider, env, &input, &mut handle) {
|
||||
Ok(output) => output,
|
||||
Err(err) => return self.handle_execution_error(input, err, &parent_block),
|
||||
};
|
||||
// Execute the block and handle any execution errors.
|
||||
// The receipt root task is spawned before execution and receives receipts incrementally
|
||||
// as transactions complete, allowing parallel computation during execution.
|
||||
let (output, senders, receipt_root_rx) =
|
||||
match self.execute_block(state_provider, env, &input, &mut handle) {
|
||||
Ok(output) => output,
|
||||
Err(err) => return self.handle_execution_error(input, err, &parent_block),
|
||||
};
|
||||
|
||||
// After executing the block we can stop prewarming transactions
|
||||
handle.stop_prewarming_execution();
|
||||
|
||||
// Create ExecutionOutcome early so we can terminate caching before validation and state
|
||||
// root computation. Using Arc allows sharing with both the caching task and the deferred
|
||||
// trie task without cloning the expensive BundleState.
|
||||
let output = Arc::new(output);
|
||||
|
||||
// Terminate caching task early since execution is complete and caching is no longer
|
||||
// needed. This frees up resources while state root computation continues.
|
||||
let valid_block_tx = handle.terminate_caching(Some(output.clone()));
|
||||
|
||||
let block = self.convert_to_block(input)?.with_senders(senders);
|
||||
|
||||
// Wait for the receipt root computation to complete.
|
||||
let receipt_root_bloom = receipt_root_rx
|
||||
.blocking_recv()
|
||||
.inspect_err(|_| {
|
||||
tracing::error!(
|
||||
target: "engine::tree::payload_validator",
|
||||
"Receipt root task dropped sender without result, receipt root calculation likely aborted"
|
||||
);
|
||||
})
|
||||
.ok();
|
||||
|
||||
let hashed_state = ensure_ok_post_block!(
|
||||
self.validate_post_execution(&block, &parent_block, &output, &mut ctx),
|
||||
self.validate_post_execution(
|
||||
&block,
|
||||
&parent_block,
|
||||
&output,
|
||||
&mut ctx,
|
||||
receipt_root_bloom
|
||||
),
|
||||
block
|
||||
);
|
||||
|
||||
@@ -584,16 +596,13 @@ where
|
||||
.into())
|
||||
}
|
||||
|
||||
// Create ExecutionOutcome and wrap in Arc for sharing with both the caching task
|
||||
// and the deferred trie task. This avoids cloning the expensive BundleState.
|
||||
let execution_outcome = Arc::new(ExecutionOutcome::from((output, block_num_hash.number)));
|
||||
|
||||
// Terminate prewarming task with the shared execution outcome
|
||||
handle.terminate_caching(Some(Arc::clone(&execution_outcome)));
|
||||
if let Some(valid_block_tx) = valid_block_tx {
|
||||
let _ = valid_block_tx.send(());
|
||||
}
|
||||
|
||||
Ok(self.spawn_deferred_trie_task(
|
||||
block,
|
||||
execution_outcome,
|
||||
output,
|
||||
&ctx,
|
||||
hashed_state,
|
||||
trie_output,
|
||||
@@ -634,15 +643,29 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Executes a block with the given state provider
|
||||
/// Executes a block with the given state provider.
|
||||
///
|
||||
/// This method orchestrates block execution:
|
||||
/// 1. Sets up the EVM with state database and precompile caching
|
||||
/// 2. Spawns a background task for incremental receipt root computation
|
||||
/// 3. Executes transactions with metrics collection via state hooks
|
||||
/// 4. Merges state transitions and records execution metrics
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)]
|
||||
#[expect(clippy::type_complexity)]
|
||||
fn execute_block<S, Err, T>(
|
||||
&mut self,
|
||||
state_provider: S,
|
||||
env: ExecutionEnv<Evm>,
|
||||
input: &BlockOrPayload<T>,
|
||||
handle: &mut PayloadHandle<impl ExecutableTxFor<Evm>, Err, N::Receipt>,
|
||||
) -> Result<(BlockExecutionOutput<N::Receipt>, Vec<Address>), InsertBlockErrorKind>
|
||||
) -> Result<
|
||||
(
|
||||
BlockExecutionOutput<N::Receipt>,
|
||||
Vec<Address>,
|
||||
tokio::sync::oneshot::Receiver<(B256, alloy_primitives::Bloom)>,
|
||||
),
|
||||
InsertBlockErrorKind,
|
||||
>
|
||||
where
|
||||
S: StateProvider + Send,
|
||||
Err: core::error::Error + Send + Sync + 'static,
|
||||
@@ -681,18 +704,123 @@ where
|
||||
});
|
||||
}
|
||||
|
||||
// Spawn background task to compute receipt root and logs bloom incrementally.
|
||||
// Unbounded channel is used since tx count bounds capacity anyway (max ~30k txs per block).
|
||||
let receipts_len = input.transaction_count();
|
||||
let (receipt_tx, receipt_rx) = crossbeam_channel::unbounded();
|
||||
let (result_tx, result_rx) = tokio::sync::oneshot::channel();
|
||||
let task_handle = ReceiptRootTaskHandle::new(receipt_rx, result_tx);
|
||||
self.payload_processor.executor().spawn_blocking(move || task_handle.run(receipts_len));
|
||||
|
||||
// Wrap the state hook with metrics collection
|
||||
let inner_hook = Box::new(handle.state_hook());
|
||||
let state_hook =
|
||||
MeteredStateHook { metrics: self.metrics.executor_metrics().clone(), inner_hook };
|
||||
|
||||
let transaction_count = input.transaction_count();
|
||||
let executor = executor.with_state_hook(Some(Box::new(state_hook)));
|
||||
|
||||
let execution_start = Instant::now();
|
||||
let state_hook = Box::new(handle.state_hook());
|
||||
let (output, senders) = self.metrics.execute_metered(
|
||||
|
||||
// Execute all transactions and finalize
|
||||
let (executor, senders) = self.execute_transactions(
|
||||
executor,
|
||||
handle.iter_transactions().map(|res| res.map_err(BlockExecutionError::other)),
|
||||
input.transaction_count(),
|
||||
state_hook,
|
||||
transaction_count,
|
||||
handle.iter_transactions(),
|
||||
&receipt_tx,
|
||||
)?;
|
||||
let execution_finish = Instant::now();
|
||||
let execution_time = execution_finish.duration_since(execution_start);
|
||||
debug!(target: "engine::tree::payload_validator", elapsed = ?execution_time, "Executed block");
|
||||
Ok((output, senders))
|
||||
drop(receipt_tx);
|
||||
|
||||
// Finish execution and get the result
|
||||
let post_exec_start = Instant::now();
|
||||
let (_evm, result) = debug_span!(target: "engine::tree", "finish")
|
||||
.in_scope(|| executor.finish())
|
||||
.map(|(evm, result)| (evm.into_db(), result))?;
|
||||
self.metrics.record_post_execution(post_exec_start.elapsed());
|
||||
|
||||
// Merge transitions into bundle state
|
||||
debug_span!(target: "engine::tree", "merge transitions")
|
||||
.in_scope(|| db.merge_transitions(BundleRetention::Reverts));
|
||||
|
||||
let output = BlockExecutionOutput { result, state: db.take_bundle() };
|
||||
|
||||
let execution_duration = execution_start.elapsed();
|
||||
self.metrics.record_block_execution(&output, execution_duration);
|
||||
|
||||
debug!(target: "engine::tree::payload_validator", elapsed = ?execution_duration, "Executed block");
|
||||
Ok((output, senders, result_rx))
|
||||
}
|
||||
|
||||
/// Executes transactions and collects senders, streaming receipts to a background task.
|
||||
///
|
||||
/// This method handles:
|
||||
/// - Applying pre-execution changes (e.g., beacon root updates)
|
||||
/// - Executing each transaction with timing metrics
|
||||
/// - Streaming receipts to the receipt root computation task
|
||||
/// - Collecting transaction senders for later use
|
||||
///
|
||||
/// Returns the executor (for finalization) and the collected senders.
|
||||
fn execute_transactions<E, Tx, InnerTx, Err>(
|
||||
&self,
|
||||
mut executor: E,
|
||||
transaction_count: usize,
|
||||
transactions: impl Iterator<Item = Result<Tx, Err>>,
|
||||
receipt_tx: &crossbeam_channel::Sender<IndexedReceipt<N::Receipt>>,
|
||||
) -> Result<(E, Vec<Address>), BlockExecutionError>
|
||||
where
|
||||
E: BlockExecutor<Receipt = N::Receipt>,
|
||||
Tx: alloy_evm::block::ExecutableTx<E> + alloy_evm::RecoveredTx<InnerTx>,
|
||||
InnerTx: TxHashRef,
|
||||
Err: core::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
let mut senders = Vec::with_capacity(transaction_count);
|
||||
|
||||
// Apply pre-execution changes (e.g., beacon root update)
|
||||
let pre_exec_start = Instant::now();
|
||||
debug_span!(target: "engine::tree", "pre execution")
|
||||
.in_scope(|| executor.apply_pre_execution_changes())?;
|
||||
self.metrics.record_pre_execution(pre_exec_start.elapsed());
|
||||
|
||||
// Execute transactions
|
||||
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
|
||||
let mut transactions = transactions.into_iter();
|
||||
loop {
|
||||
// Measure time spent waiting for next transaction from iterator
|
||||
// (e.g., parallel signature recovery)
|
||||
let wait_start = Instant::now();
|
||||
let Some(tx_result) = transactions.next() else { break };
|
||||
self.metrics.record_transaction_wait(wait_start.elapsed());
|
||||
|
||||
let tx = tx_result.map_err(BlockExecutionError::other)?;
|
||||
let tx_signer = *<Tx as alloy_evm::RecoveredTx<InnerTx>>::signer(&tx);
|
||||
let tx_hash = <Tx as alloy_evm::RecoveredTx<InnerTx>>::tx(&tx).tx_hash();
|
||||
|
||||
senders.push(tx_signer);
|
||||
|
||||
let span = debug_span!(
|
||||
target: "engine::tree",
|
||||
"execute tx",
|
||||
?tx_hash,
|
||||
gas_used = tracing::field::Empty,
|
||||
);
|
||||
let enter = span.entered();
|
||||
trace!(target: "engine::tree", "Executing transaction");
|
||||
|
||||
let tx_start = Instant::now();
|
||||
let gas_used = executor.execute_transaction(tx)?;
|
||||
self.metrics.record_transaction_execution(tx_start.elapsed());
|
||||
|
||||
// Send the latest receipt to the background task for incremental root computation
|
||||
if let Some(receipt) = executor.receipts().last() {
|
||||
let tx_index = executor.receipts().len() - 1;
|
||||
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
|
||||
}
|
||||
|
||||
enter.record("gas_used", gas_used);
|
||||
}
|
||||
drop(exec_span);
|
||||
|
||||
Ok((executor, senders))
|
||||
}
|
||||
|
||||
/// Compute state root for the given hashed post state in parallel.
|
||||
@@ -750,6 +878,9 @@ where
|
||||
/// - parent header validation
|
||||
/// - post-execution consensus validation
|
||||
/// - state-root based post-execution validation
|
||||
///
|
||||
/// If `receipt_root_bloom` is provided, it will be used instead of computing the receipt root
|
||||
/// and logs bloom from the receipts.
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)]
|
||||
fn validate_post_execution<T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>>(
|
||||
&self,
|
||||
@@ -757,6 +888,7 @@ where
|
||||
parent_block: &SealedHeader<N::BlockHeader>,
|
||||
output: &BlockExecutionOutput<N::Receipt>,
|
||||
ctx: &mut TreeCtx<'_, N>,
|
||||
receipt_root_bloom: Option<ReceiptRootBloom>,
|
||||
) -> Result<HashedPostState, InsertBlockErrorKind>
|
||||
where
|
||||
V: PayloadValidator<T, Block = N::Block>,
|
||||
@@ -783,7 +915,9 @@ where
|
||||
let _enter =
|
||||
debug_span!(target: "engine::tree::payload_validator", "validate_block_post_execution")
|
||||
.entered();
|
||||
if let Err(err) = self.consensus.validate_block_post_execution(block, output) {
|
||||
if let Err(err) =
|
||||
self.consensus.validate_block_post_execution(block, output, receipt_root_bloom)
|
||||
{
|
||||
// call post-block hook
|
||||
self.on_invalid_block(parent_block, block, output, None, ctx.state_mut());
|
||||
return Err(err.into())
|
||||
@@ -954,128 +1088,36 @@ where
|
||||
self.invalid_block_hook.on_invalid_block(parent_header, block, output, trie_updates);
|
||||
}
|
||||
|
||||
/// Computes [`TrieInputSorted`] for the provided parent hash by combining database state
|
||||
/// with in-memory overlays.
|
||||
/// Creates a [`LazyOverlay`] for the parent block without blocking.
|
||||
///
|
||||
/// The goal of this function is to take in-memory blocks and generate a [`TrieInputSorted`]
|
||||
/// that extends from the highest persisted ancestor up through the parent. This enables state
|
||||
/// root computation and proof generation without requiring all blocks to be persisted
|
||||
/// first.
|
||||
/// Returns a lazy overlay that will compute the trie input on first access, and the anchor
|
||||
/// block hash (the highest persisted ancestor). This allows execution to start immediately
|
||||
/// while the trie input computation is deferred until the overlay is actually needed.
|
||||
///
|
||||
/// It works as follows:
|
||||
/// 1. Collect in-memory overlay blocks using [`crate::tree::TreeState::blocks_by_hash`]. This
|
||||
/// returns the highest persisted ancestor hash (`block_hash`) and the list of in-memory
|
||||
/// blocks building on top of it.
|
||||
/// 2. Fast path: If the tip in-memory block's trie input is already anchored to `block_hash`
|
||||
/// (its `anchor_hash` matches `block_hash`), reuse it directly.
|
||||
/// 3. Slow path: Build a new [`TrieInputSorted`] by aggregating the overlay blocks (from oldest
|
||||
/// to newest) on top of the database state at `block_hash`.
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_validator",
|
||||
skip_all,
|
||||
fields(parent_hash)
|
||||
)]
|
||||
fn compute_trie_input(
|
||||
&self,
|
||||
/// If parent is on disk (no in-memory blocks), returns `None` for the lazy overlay.
|
||||
fn get_parent_lazy_overlay(
|
||||
parent_hash: B256,
|
||||
state: &EngineApiTreeState<N>,
|
||||
) -> ProviderResult<(TrieInputSorted, B256)> {
|
||||
let wait_start = Instant::now();
|
||||
let (block_hash, blocks) =
|
||||
) -> (Option<LazyOverlay>, B256) {
|
||||
let (anchor_hash, blocks) =
|
||||
state.tree_state.blocks_by_hash(parent_hash).unwrap_or_else(|| (parent_hash, vec![]));
|
||||
|
||||
// Fast path: if the tip block's anchor matches the persisted ancestor hash, reuse its
|
||||
// TrieInput. This means the TrieInputSorted already aggregates all in-memory overlays
|
||||
// from that ancestor, so we can avoid re-aggregation.
|
||||
if let Some(tip_block) = blocks.first() {
|
||||
let data = tip_block.trie_data();
|
||||
if let (Some(anchor_hash), Some(trie_input)) =
|
||||
(data.anchor_hash(), data.trie_input().cloned()) &&
|
||||
anchor_hash == block_hash
|
||||
{
|
||||
trace!(target: "engine::tree::payload_validator", %block_hash,"Reusing trie input with matching anchor hash");
|
||||
self.metrics
|
||||
.block_validation
|
||||
.deferred_trie_wait_duration
|
||||
.record(wait_start.elapsed().as_secs_f64());
|
||||
return Ok(((*trie_input).clone(), block_hash));
|
||||
}
|
||||
}
|
||||
|
||||
if blocks.is_empty() {
|
||||
debug!(target: "engine::tree::payload_validator", "Parent found on disk");
|
||||
} else {
|
||||
debug!(target: "engine::tree::payload_validator", historical = ?block_hash, blocks = blocks.len(), "Parent found in memory");
|
||||
debug!(target: "engine::tree::payload_validator", "Parent found on disk, no lazy overlay needed");
|
||||
return (None, anchor_hash);
|
||||
}
|
||||
|
||||
// Extend with contents of parent in-memory blocks directly in sorted form.
|
||||
let input = Self::merge_overlay_trie_input(&blocks);
|
||||
debug!(
|
||||
target: "engine::tree::payload_validator",
|
||||
%anchor_hash,
|
||||
num_blocks = blocks.len(),
|
||||
"Creating lazy overlay for in-memory blocks"
|
||||
);
|
||||
|
||||
self.metrics
|
||||
.block_validation
|
||||
.deferred_trie_wait_duration
|
||||
.record(wait_start.elapsed().as_secs_f64());
|
||||
Ok((input, block_hash))
|
||||
}
|
||||
// Extract deferred trie data handles (non-blocking)
|
||||
let handles: Vec<DeferredTrieData> = blocks.iter().map(|b| b.trie_data_handle()).collect();
|
||||
|
||||
/// Aggregates in-memory blocks into a single [`TrieInputSorted`] by combining their
|
||||
/// state changes.
|
||||
///
|
||||
/// The input `blocks` vector is ordered newest -> oldest (see `TreeState::blocks_by_hash`).
|
||||
///
|
||||
/// Uses `extend_ref` loop for small k, k-way `merge_batch` for large k.
|
||||
/// See [`MERGE_BATCH_THRESHOLD`] for crossover point.
|
||||
fn merge_overlay_trie_input(blocks: &[ExecutedBlock<N>]) -> TrieInputSorted {
|
||||
if blocks.is_empty() {
|
||||
return TrieInputSorted::default();
|
||||
}
|
||||
|
||||
// Single block: return Arc directly without cloning
|
||||
if blocks.len() == 1 {
|
||||
let data = blocks[0].trie_data();
|
||||
return TrieInputSorted {
|
||||
state: Arc::clone(&data.hashed_state),
|
||||
nodes: Arc::clone(&data.trie_updates),
|
||||
prefix_sets: Default::default(),
|
||||
};
|
||||
}
|
||||
|
||||
if blocks.len() < MERGE_BATCH_THRESHOLD {
|
||||
// Small k: extend_ref loop is faster
|
||||
// Iterate oldest->newest so newer values override older ones
|
||||
let mut blocks_iter = blocks.iter().rev();
|
||||
let first = blocks_iter.next().expect("blocks is non-empty");
|
||||
let data = first.trie_data();
|
||||
|
||||
let mut state = Arc::clone(&data.hashed_state);
|
||||
let mut nodes = Arc::clone(&data.trie_updates);
|
||||
let state_mut = Arc::make_mut(&mut state);
|
||||
let nodes_mut = Arc::make_mut(&mut nodes);
|
||||
|
||||
for block in blocks_iter {
|
||||
let data = block.trie_data();
|
||||
state_mut.extend_ref(data.hashed_state.as_ref());
|
||||
nodes_mut.extend_ref(data.trie_updates.as_ref());
|
||||
}
|
||||
|
||||
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
|
||||
} else {
|
||||
// Large k: merge_batch is faster (O(n log k) via k-way merge)
|
||||
let trie_data: Vec<_> = blocks.iter().map(|b| b.trie_data()).collect();
|
||||
|
||||
let merged_state = HashedPostStateSorted::merge_batch(
|
||||
trie_data.iter().map(|d| d.hashed_state.as_ref()),
|
||||
);
|
||||
let merged_nodes =
|
||||
TrieUpdatesSorted::merge_batch(trie_data.iter().map(|d| d.trie_updates.as_ref()));
|
||||
|
||||
TrieInputSorted {
|
||||
state: Arc::new(merged_state),
|
||||
nodes: Arc::new(merged_nodes),
|
||||
prefix_sets: Default::default(),
|
||||
}
|
||||
}
|
||||
(Some(LazyOverlay::new(anchor_hash, handles)), anchor_hash)
|
||||
}
|
||||
|
||||
/// Spawns a background task to compute and sort trie data for the executed block.
|
||||
@@ -1097,7 +1139,7 @@ where
|
||||
fn spawn_deferred_trie_task(
|
||||
&self,
|
||||
block: RecoveredBlock<N::Block>,
|
||||
execution_outcome: Arc<ExecutionOutcome<N::Receipt>>,
|
||||
execution_outcome: Arc<BlockExecutionOutput<N::Receipt>>,
|
||||
ctx: &TreeCtx<'_, N>,
|
||||
hashed_state: HashedPostState,
|
||||
trie_output: TrieUpdates,
|
||||
@@ -1295,6 +1337,7 @@ where
|
||||
+ StageCheckpointReader
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader,
|
||||
> + BlockReader<Header = N::BlockHeader>
|
||||
+ StateProviderFactory
|
||||
@@ -1344,7 +1387,7 @@ where
|
||||
fn on_inserted_executed_block(&self, block: ExecutedBlock<N>) {
|
||||
self.payload_processor.on_inserted_executed_block(
|
||||
block.recovered_block.block_with_parent(),
|
||||
block.execution_output.state(),
|
||||
&block.execution_output.state,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ use reth_ethereum_engine_primitives::EthEngineTypes;
|
||||
use reth_ethereum_primitives::{Block, EthPrimitives};
|
||||
use reth_evm_ethereum::MockEvmConfig;
|
||||
use reth_primitives_traits::Block as _;
|
||||
use reth_provider::{test_utils::MockEthProvider, ExecutionOutcome};
|
||||
use reth_provider::test_utils::MockEthProvider;
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
str::FromStr,
|
||||
@@ -491,7 +491,7 @@ fn test_tree_persist_block_batch() {
|
||||
let chain_spec = MAINNET.clone();
|
||||
let mut test_block_builder = TestBlockBuilder::eth().with_chain_spec((*chain_spec).clone());
|
||||
|
||||
// we need at least tree_config.persistence_threshold() + 1 blocks to
|
||||
// we need more than tree_config.persistence_threshold() +1 blocks to
|
||||
// trigger the persistence task.
|
||||
let blocks: Vec<_> = test_block_builder
|
||||
.get_executed_blocks(1..tree_config.persistence_threshold() + 2)
|
||||
@@ -531,7 +531,7 @@ async fn test_tree_persist_blocks() {
|
||||
let chain_spec = MAINNET.clone();
|
||||
let mut test_block_builder = TestBlockBuilder::eth().with_chain_spec((*chain_spec).clone());
|
||||
|
||||
// we need at least tree_config.persistence_threshold() + 1 blocks to
|
||||
// we need more than tree_config.persistence_threshold() +1 blocks to
|
||||
// trigger the persistence task.
|
||||
let blocks: Vec<_> = test_block_builder
|
||||
.get_executed_blocks(1..tree_config.persistence_threshold() + 2)
|
||||
@@ -838,7 +838,7 @@ fn test_tree_state_on_new_head_deep_fork() {
|
||||
for block in &chain_a {
|
||||
test_harness.tree.state.tree_state.insert_executed(ExecutedBlock::new(
|
||||
Arc::new(block.clone()),
|
||||
Arc::new(ExecutionOutcome::default()),
|
||||
Arc::new(BlockExecutionOutput::default()),
|
||||
empty_trie_data(),
|
||||
));
|
||||
}
|
||||
@@ -847,7 +847,7 @@ fn test_tree_state_on_new_head_deep_fork() {
|
||||
for block in &chain_b {
|
||||
test_harness.tree.state.tree_state.insert_executed(ExecutedBlock::new(
|
||||
Arc::new(block.clone()),
|
||||
Arc::new(ExecutionOutcome::default()),
|
||||
Arc::new(BlockExecutionOutput::default()),
|
||||
empty_trie_data(),
|
||||
));
|
||||
}
|
||||
@@ -1008,6 +1008,15 @@ async fn test_engine_tree_live_sync_transition_required_blocks_requested() {
|
||||
_ => panic!("Unexpected event: {event:#?}"),
|
||||
}
|
||||
|
||||
// After backfill completes with head not buffered, we also request head download
|
||||
let event = test_harness.from_tree_rx.recv().await.unwrap();
|
||||
match event {
|
||||
EngineApiEvent::Download(DownloadRequest::BlockSet(hash_set)) => {
|
||||
assert_eq!(hash_set, HashSet::from_iter([main_chain_last_hash]));
|
||||
}
|
||||
_ => panic!("Unexpected event: {event:#?}"),
|
||||
}
|
||||
|
||||
let _ = test_harness
|
||||
.tree
|
||||
.on_engine_message(FromEngine::DownloadedBlocks(vec![main_chain
|
||||
|
||||
@@ -15,7 +15,7 @@ use alloc::{fmt::Debug, sync::Arc};
|
||||
use alloy_consensus::{constants::MAXIMUM_EXTRA_DATA_SIZE, EMPTY_OMMER_ROOT_HASH};
|
||||
use alloy_eips::eip7840::BlobParams;
|
||||
use reth_chainspec::{EthChainSpec, EthereumHardforks};
|
||||
use reth_consensus::{Consensus, ConsensusError, FullConsensus, HeaderValidator};
|
||||
use reth_consensus::{Consensus, ConsensusError, FullConsensus, HeaderValidator, ReceiptRootBloom};
|
||||
use reth_consensus_common::validation::{
|
||||
validate_4844_header_standalone, validate_against_parent_4844,
|
||||
validate_against_parent_eip1559_base_fee, validate_against_parent_gas_limit,
|
||||
@@ -74,8 +74,15 @@ where
|
||||
&self,
|
||||
block: &RecoveredBlock<N::Block>,
|
||||
result: &BlockExecutionResult<N::Receipt>,
|
||||
receipt_root_bloom: Option<ReceiptRootBloom>,
|
||||
) -> Result<(), ConsensusError> {
|
||||
validate_block_post_execution(block, &self.chain_spec, &result.receipts, &result.requests)
|
||||
validate_block_post_execution(
|
||||
block,
|
||||
&self.chain_spec,
|
||||
&result.receipts,
|
||||
&result.requests,
|
||||
receipt_root_bloom,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,11 +12,15 @@ use reth_primitives_traits::{
|
||||
///
|
||||
/// - Compares the receipts root in the block header to the block body
|
||||
/// - Compares the gas used in the block header to the actual gas usage after execution
|
||||
///
|
||||
/// If `receipt_root_bloom` is provided, the pre-computed receipt root and logs bloom are used
|
||||
/// instead of computing them from the receipts.
|
||||
pub fn validate_block_post_execution<B, R, ChainSpec>(
|
||||
block: &RecoveredBlock<B>,
|
||||
chain_spec: &ChainSpec,
|
||||
receipts: &[R],
|
||||
requests: &Requests,
|
||||
receipt_root_bloom: Option<(B256, Bloom)>,
|
||||
) -> Result<(), ConsensusError>
|
||||
where
|
||||
B: Block,
|
||||
@@ -37,19 +41,26 @@ where
|
||||
// operation as hashing that is required for state root got calculated in every
|
||||
// transaction This was replaced with is_success flag.
|
||||
// See more about EIP here: https://eips.ethereum.org/EIPS/eip-658
|
||||
if chain_spec.is_byzantium_active_at_block(block.header().number()) &&
|
||||
let Err(error) = verify_receipts(
|
||||
block.header().receipts_root(),
|
||||
block.header().logs_bloom(),
|
||||
receipts,
|
||||
)
|
||||
{
|
||||
let receipts = receipts
|
||||
.iter()
|
||||
.map(|r| Bytes::from(r.with_bloom_ref().encoded_2718()))
|
||||
.collect::<Vec<_>>();
|
||||
tracing::debug!(%error, ?receipts, "receipts verification failed");
|
||||
return Err(error)
|
||||
if chain_spec.is_byzantium_active_at_block(block.header().number()) {
|
||||
let result = if let Some((receipts_root, logs_bloom)) = receipt_root_bloom {
|
||||
compare_receipts_root_and_logs_bloom(
|
||||
receipts_root,
|
||||
logs_bloom,
|
||||
block.header().receipts_root(),
|
||||
block.header().logs_bloom(),
|
||||
)
|
||||
} else {
|
||||
verify_receipts(block.header().receipts_root(), block.header().logs_bloom(), receipts)
|
||||
};
|
||||
|
||||
if let Err(error) = result {
|
||||
let receipts = receipts
|
||||
.iter()
|
||||
.map(|r| Bytes::from(r.with_bloom_ref().encoded_2718()))
|
||||
.collect::<Vec<_>>();
|
||||
tracing::debug!(%error, ?receipts, "receipts verification failed");
|
||||
return Err(error)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate that the header requests hash matches the calculated requests hash
|
||||
|
||||
@@ -17,10 +17,11 @@ pub use payload::{payload_id, BlobSidecars, EthBuiltPayload, EthPayloadBuilderAt
|
||||
mod error;
|
||||
pub use error::*;
|
||||
|
||||
use alloy_rpc_types_engine::{ExecutionData, ExecutionPayload, ExecutionPayloadEnvelopeV5};
|
||||
use alloy_rpc_types_engine::{ExecutionData, ExecutionPayload};
|
||||
pub use alloy_rpc_types_engine::{
|
||||
ExecutionPayloadEnvelopeV2, ExecutionPayloadEnvelopeV3, ExecutionPayloadEnvelopeV4,
|
||||
ExecutionPayloadV1, PayloadAttributes as EthPayloadAttributes,
|
||||
ExecutionPayloadEnvelopeV5, ExecutionPayloadEnvelopeV6, ExecutionPayloadV1,
|
||||
PayloadAttributes as EthPayloadAttributes,
|
||||
};
|
||||
use reth_engine_primitives::EngineTypes;
|
||||
use reth_payload_primitives::{BuiltPayload, PayloadTypes};
|
||||
@@ -66,13 +67,15 @@ where
|
||||
+ TryInto<ExecutionPayloadEnvelopeV2>
|
||||
+ TryInto<ExecutionPayloadEnvelopeV3>
|
||||
+ TryInto<ExecutionPayloadEnvelopeV4>
|
||||
+ TryInto<ExecutionPayloadEnvelopeV5>,
|
||||
+ TryInto<ExecutionPayloadEnvelopeV5>
|
||||
+ TryInto<ExecutionPayloadEnvelopeV6>,
|
||||
{
|
||||
type ExecutionPayloadEnvelopeV1 = ExecutionPayloadV1;
|
||||
type ExecutionPayloadEnvelopeV2 = ExecutionPayloadEnvelopeV2;
|
||||
type ExecutionPayloadEnvelopeV3 = ExecutionPayloadEnvelopeV3;
|
||||
type ExecutionPayloadEnvelopeV4 = ExecutionPayloadEnvelopeV4;
|
||||
type ExecutionPayloadEnvelopeV5 = ExecutionPayloadEnvelopeV5;
|
||||
type ExecutionPayloadEnvelopeV6 = ExecutionPayloadEnvelopeV6;
|
||||
}
|
||||
|
||||
/// A default payload type for [`EthEngineTypes`]
|
||||
|
||||
@@ -11,8 +11,8 @@ use alloy_primitives::{Address, B256, U256};
|
||||
use alloy_rlp::Encodable;
|
||||
use alloy_rpc_types_engine::{
|
||||
BlobsBundleV1, BlobsBundleV2, ExecutionPayloadEnvelopeV2, ExecutionPayloadEnvelopeV3,
|
||||
ExecutionPayloadEnvelopeV4, ExecutionPayloadEnvelopeV5, ExecutionPayloadFieldV2,
|
||||
ExecutionPayloadV1, ExecutionPayloadV3, PayloadAttributes, PayloadId,
|
||||
ExecutionPayloadEnvelopeV4, ExecutionPayloadEnvelopeV5, ExecutionPayloadEnvelopeV6,
|
||||
ExecutionPayloadFieldV2, ExecutionPayloadV1, ExecutionPayloadV3, PayloadAttributes, PayloadId,
|
||||
};
|
||||
use core::convert::Infallible;
|
||||
use reth_ethereum_primitives::EthPrimitives;
|
||||
@@ -160,6 +160,13 @@ impl EthBuiltPayload {
|
||||
execution_requests: requests.unwrap_or_default(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Try converting built payload into [`ExecutionPayloadEnvelopeV6`].
|
||||
///
|
||||
/// Note: Amsterdam fork is not yet implemented, so this conversion is not yet supported.
|
||||
pub fn try_into_v6(self) -> Result<ExecutionPayloadEnvelopeV6, BuiltPayloadConversionError> {
|
||||
unimplemented!("ExecutionPayloadEnvelopeV6 not yet supported")
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives> BuiltPayload for EthBuiltPayload<N> {
|
||||
@@ -227,6 +234,14 @@ impl TryFrom<EthBuiltPayload> for ExecutionPayloadEnvelopeV5 {
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<EthBuiltPayload> for ExecutionPayloadEnvelopeV6 {
|
||||
type Error = BuiltPayloadConversionError;
|
||||
|
||||
fn try_from(value: EthBuiltPayload) -> Result<Self, Self::Error> {
|
||||
value.try_into_v6()
|
||||
}
|
||||
}
|
||||
|
||||
/// An enum representing blob transaction sidecars belonging to [`EthBuiltPayload`].
|
||||
#[derive(Clone, Default, Debug)]
|
||||
pub enum BlobSidecars {
|
||||
|
||||
@@ -188,6 +188,7 @@ where
|
||||
block: &'a SealedBlock<Block>,
|
||||
) -> Result<EthBlockExecutionCtx<'a>, Self::Error> {
|
||||
Ok(EthBlockExecutionCtx {
|
||||
tx_count_hint: Some(block.transaction_count()),
|
||||
parent_hash: block.header().parent_hash,
|
||||
parent_beacon_block_root: block.header().parent_beacon_block_root,
|
||||
ommers: &block.body().ommers,
|
||||
@@ -202,6 +203,7 @@ where
|
||||
attributes: Self::NextBlockEnvCtx,
|
||||
) -> Result<EthBlockExecutionCtx<'_>, Self::Error> {
|
||||
Ok(EthBlockExecutionCtx {
|
||||
tx_count_hint: None,
|
||||
parent_hash: parent.hash(),
|
||||
parent_beacon_block_root: attributes.parent_beacon_block_root,
|
||||
ommers: &[],
|
||||
@@ -238,8 +240,9 @@ where
|
||||
revm_spec_by_timestamp_and_block_number(self.chain_spec(), timestamp, block_number);
|
||||
|
||||
// configure evm env based on parent block
|
||||
let mut cfg_env =
|
||||
CfgEnv::new().with_chain_id(self.chain_spec().chain().id()).with_spec(spec);
|
||||
let mut cfg_env = CfgEnv::new()
|
||||
.with_chain_id(self.chain_spec().chain().id())
|
||||
.with_spec_and_mainnet_gas_params(spec);
|
||||
|
||||
if let Some(blob_params) = &blob_params {
|
||||
cfg_env.set_max_blobs_per_tx(blob_params.max_blobs_per_tx);
|
||||
@@ -280,6 +283,7 @@ where
|
||||
payload: &'a ExecutionData,
|
||||
) -> Result<ExecutionCtxFor<'a, Self>, Self::Error> {
|
||||
Ok(EthBlockExecutionCtx {
|
||||
tx_count_hint: Some(payload.payload.transactions().len()),
|
||||
parent_hash: payload.parent_hash(),
|
||||
parent_beacon_block_root: payload.sidecar.parent_beacon_block_root(),
|
||||
ommers: &[],
|
||||
@@ -407,7 +411,7 @@ mod tests {
|
||||
let db = CacheDB::<EmptyDBTyped<ProviderError>>::default();
|
||||
|
||||
let evm_env = EvmEnv {
|
||||
cfg_env: CfgEnv::new().with_spec(SpecId::CONSTANTINOPLE),
|
||||
cfg_env: CfgEnv::new().with_spec_and_mainnet_gas_params(SpecId::CONSTANTINOPLE),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -474,7 +478,7 @@ mod tests {
|
||||
let db = CacheDB::<EmptyDBTyped<ProviderError>>::default();
|
||||
|
||||
let evm_env = EvmEnv {
|
||||
cfg_env: CfgEnv::new().with_spec(SpecId::CONSTANTINOPLE),
|
||||
cfg_env: CfgEnv::new().with_spec_and_mainnet_gas_params(SpecId::CONSTANTINOPLE),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use alloy_consensus::TxType;
|
||||
use alloy_evm::eth::receipt_builder::{ReceiptBuilder, ReceiptBuilderCtx};
|
||||
use reth_ethereum_primitives::{Receipt, TransactionSigned};
|
||||
use reth_evm::Evm;
|
||||
@@ -12,13 +13,10 @@ impl ReceiptBuilder for RethReceiptBuilder {
|
||||
type Transaction = TransactionSigned;
|
||||
type Receipt = Receipt;
|
||||
|
||||
fn build_receipt<E: Evm>(
|
||||
&self,
|
||||
ctx: ReceiptBuilderCtx<'_, Self::Transaction, E>,
|
||||
) -> Self::Receipt {
|
||||
let ReceiptBuilderCtx { tx, result, cumulative_gas_used, .. } = ctx;
|
||||
fn build_receipt<E: Evm>(&self, ctx: ReceiptBuilderCtx<'_, TxType, E>) -> Self::Receipt {
|
||||
let ReceiptBuilderCtx { tx_type, result, cumulative_gas_used, .. } = ctx;
|
||||
Receipt {
|
||||
tx_type: tx.tx_type(),
|
||||
tx_type,
|
||||
// Success flag was added in `EIP-658: Embedding transaction status code in
|
||||
// receipts`.
|
||||
success: result.is_success(),
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use crate::EthEvmConfig;
|
||||
use alloc::{boxed::Box, sync::Arc, vec, vec::Vec};
|
||||
use alloy_consensus::Header;
|
||||
use alloy_consensus::{Header, TxType};
|
||||
use alloy_eips::eip7685::Requests;
|
||||
use alloy_evm::precompiles::PrecompilesMap;
|
||||
use alloy_primitives::Bytes;
|
||||
@@ -11,14 +11,14 @@ use reth_evm::{
|
||||
block::{
|
||||
BlockExecutionError, BlockExecutor, BlockExecutorFactory, BlockExecutorFor, ExecutableTx,
|
||||
},
|
||||
eth::{EthBlockExecutionCtx, EthEvmContext},
|
||||
eth::{EthBlockExecutionCtx, EthEvmContext, EthTxResult},
|
||||
ConfigureEngineEvm, ConfigureEvm, Database, EthEvm, EthEvmFactory, Evm, EvmEnvFor, EvmFactory,
|
||||
ExecutableTxIterator, ExecutionCtxFor,
|
||||
ExecutableTxIterator, ExecutionCtxFor, RecoveredTx,
|
||||
};
|
||||
use reth_execution_types::{BlockExecutionResult, ExecutionOutcome};
|
||||
use reth_primitives_traits::{BlockTy, SealedBlock, SealedHeader};
|
||||
use revm::{
|
||||
context::result::{ExecutionResult, Output, ResultAndState, SuccessReason},
|
||||
context::result::{ExecutionResult, HaltReason, Output, ResultAndState, SuccessReason},
|
||||
database::State,
|
||||
Inspector,
|
||||
};
|
||||
@@ -65,7 +65,12 @@ impl BlockExecutorFactory for MockEvmConfig {
|
||||
DB: Database + 'a,
|
||||
I: Inspector<<Self::EvmFactory as EvmFactory>::Context<&'a mut State<DB>>> + 'a,
|
||||
{
|
||||
MockExecutor { result: self.exec_results.lock().pop().unwrap(), evm, hook: None }
|
||||
MockExecutor {
|
||||
result: self.exec_results.lock().pop().unwrap(),
|
||||
evm,
|
||||
hook: None,
|
||||
receipts: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,6 +81,7 @@ pub struct MockExecutor<'a, DB: Database, I> {
|
||||
evm: EthEvm<&'a mut State<DB>, I, PrecompilesMap>,
|
||||
#[debug(skip)]
|
||||
hook: Option<Box<dyn reth_evm::OnStateHook>>,
|
||||
receipts: Vec<Receipt>,
|
||||
}
|
||||
|
||||
impl<'a, DB: Database, I: Inspector<EthEvmContext<&'a mut State<DB>>>> BlockExecutor
|
||||
@@ -84,32 +90,37 @@ impl<'a, DB: Database, I: Inspector<EthEvmContext<&'a mut State<DB>>>> BlockExec
|
||||
type Evm = EthEvm<&'a mut State<DB>, I, PrecompilesMap>;
|
||||
type Transaction = TransactionSigned;
|
||||
type Receipt = Receipt;
|
||||
type Result = EthTxResult<HaltReason, TxType>;
|
||||
|
||||
fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn execute_transaction_without_commit(
|
||||
&mut self,
|
||||
_tx: impl ExecutableTx<Self>,
|
||||
) -> Result<ResultAndState<<Self::Evm as Evm>::HaltReason>, BlockExecutionError> {
|
||||
Ok(ResultAndState::new(
|
||||
ExecutionResult::Success {
|
||||
reason: SuccessReason::Return,
|
||||
gas_used: 0,
|
||||
gas_refunded: 0,
|
||||
logs: vec![],
|
||||
output: Output::Call(Bytes::from(vec![])),
|
||||
},
|
||||
Default::default(),
|
||||
))
|
||||
fn receipts(&self) -> &[Self::Receipt] {
|
||||
&self.receipts
|
||||
}
|
||||
|
||||
fn commit_transaction(
|
||||
fn execute_transaction_without_commit(
|
||||
&mut self,
|
||||
_output: ResultAndState<<Self::Evm as Evm>::HaltReason>,
|
||||
_tx: impl ExecutableTx<Self>,
|
||||
) -> Result<u64, BlockExecutionError> {
|
||||
tx: impl ExecutableTx<Self>,
|
||||
) -> Result<Self::Result, BlockExecutionError> {
|
||||
Ok(EthTxResult {
|
||||
result: ResultAndState::new(
|
||||
ExecutionResult::Success {
|
||||
reason: SuccessReason::Return,
|
||||
gas_used: 0,
|
||||
gas_refunded: 0,
|
||||
logs: vec![],
|
||||
output: Output::Call(Bytes::from(vec![])),
|
||||
},
|
||||
Default::default(),
|
||||
),
|
||||
tx_type: tx.into_parts().1.tx().tx_type(),
|
||||
blob_gas_used: 0,
|
||||
})
|
||||
}
|
||||
|
||||
fn commit_transaction(&mut self, _output: Self::Result) -> Result<u64, BlockExecutionError> {
|
||||
Ok(0)
|
||||
}
|
||||
|
||||
|
||||
@@ -38,6 +38,7 @@ fn create_database_with_beacon_root_contract() -> CacheDB<EmptyDB> {
|
||||
code_hash: keccak256(BEACON_ROOTS_CODE.clone()),
|
||||
nonce: 1,
|
||||
code: Some(Bytecode::new_raw(BEACON_ROOTS_CODE.clone())),
|
||||
account_id: None,
|
||||
};
|
||||
|
||||
db.insert_account_info(BEACON_ROOTS_ADDRESS, beacon_root_contract_account);
|
||||
@@ -53,6 +54,7 @@ fn create_database_with_withdrawal_requests_contract() -> CacheDB<EmptyDB> {
|
||||
balance: U256::ZERO,
|
||||
code_hash: keccak256(WITHDRAWAL_REQUEST_PREDEPLOY_CODE.clone()),
|
||||
code: Some(Bytecode::new_raw(WITHDRAWAL_REQUEST_PREDEPLOY_CODE.clone())),
|
||||
account_id: None,
|
||||
};
|
||||
|
||||
db.insert_account_info(
|
||||
@@ -339,6 +341,7 @@ fn create_database_with_block_hashes(latest_block: u64) -> CacheDB<EmptyDB> {
|
||||
code_hash: keccak256(HISTORY_STORAGE_CODE.clone()),
|
||||
code: Some(Bytecode::new_raw(HISTORY_STORAGE_CODE.clone())),
|
||||
nonce: 1,
|
||||
account_id: None,
|
||||
};
|
||||
|
||||
db.insert_account_info(HISTORY_STORAGE_ADDRESS, blockhashes_contract_account);
|
||||
|
||||
@@ -18,7 +18,7 @@ use reth_evm::{
|
||||
};
|
||||
use reth_network::{primitives::BasicNetworkPrimitives, NetworkHandle, PeersInfo};
|
||||
use reth_node_api::{
|
||||
AddOnsContext, FullNodeComponents, HeaderTy, NodeAddOns, NodePrimitives,
|
||||
AddOnsContext, BlockTy, FullNodeComponents, HeaderTy, NodeAddOns, NodePrimitives,
|
||||
PayloadAttributesBuilder, PrimitivesTy, TxTy,
|
||||
};
|
||||
use reth_node_builder::{
|
||||
@@ -53,8 +53,8 @@ use reth_rpc_eth_types::{error::FromEvmError, EthApiError};
|
||||
use reth_rpc_server_types::RethRpcModule;
|
||||
use reth_tracing::tracing::{debug, info};
|
||||
use reth_transaction_pool::{
|
||||
blobstore::DiskFileBlobStore, EthTransactionPool, PoolPooledTx, PoolTransaction,
|
||||
TransactionPool, TransactionValidationTaskExecutor,
|
||||
blobstore::DiskFileBlobStore, EthPooledTransaction, EthTransactionPool, PoolPooledTx,
|
||||
PoolTransaction, TransactionPool, TransactionValidationTaskExecutor,
|
||||
};
|
||||
use revm::context::TxEnv;
|
||||
use std::{marker::PhantomData, sync::Arc, time::SystemTime};
|
||||
@@ -464,7 +464,8 @@ where
|
||||
>,
|
||||
Node: FullNodeTypes<Types = Types>,
|
||||
{
|
||||
type Pool = EthTransactionPool<Node::Provider, DiskFileBlobStore>;
|
||||
type Pool =
|
||||
EthTransactionPool<Node::Provider, DiskFileBlobStore, EthPooledTransaction, BlockTy<Types>>;
|
||||
|
||||
async fn build_pool(self, ctx: &BuilderContext<Node>) -> eyre::Result<Self::Pool> {
|
||||
let pool_config = ctx.pool_config();
|
||||
|
||||
357
crates/ethereum/node/tests/e2e/invalid_payload.rs
Normal file
357
crates/ethereum/node/tests/e2e/invalid_payload.rs
Normal file
@@ -0,0 +1,357 @@
|
||||
//! Tests for handling invalid payloads via Engine API.
|
||||
//!
|
||||
//! This module tests the scenario where a node receives invalid payloads (e.g., with modified
|
||||
//! state roots) before receiving valid ones, ensuring the node can recover and continue.
|
||||
|
||||
use crate::utils::eth_payload_attributes;
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rpc_types_engine::{ExecutionPayloadV3, PayloadStatusEnum};
|
||||
use rand::{rngs::StdRng, Rng, SeedableRng};
|
||||
use reth_chainspec::{ChainSpecBuilder, MAINNET};
|
||||
use reth_e2e_test_utils::{setup_engine, transaction::TransactionTestContext};
|
||||
use reth_node_ethereum::EthereumNode;
|
||||
|
||||
use reth_rpc_api::EngineApiClient;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Tests that a node can handle receiving an invalid payload (with wrong state root)
|
||||
/// followed by the correct payload, and continue operating normally.
|
||||
///
|
||||
/// Setup:
|
||||
/// - Node 1: Produces valid payloads and advances the chain
|
||||
/// - Node 2: Receives payloads from node 1, but we also inject modified payloads with invalid state
|
||||
/// roots in between to verify error handling
|
||||
#[tokio::test]
|
||||
async fn can_handle_invalid_payload_then_valid() -> eyre::Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let seed: [u8; 32] = rand::rng().random();
|
||||
let mut rng = StdRng::from_seed(seed);
|
||||
println!("Seed: {seed:?}");
|
||||
|
||||
let chain_spec = Arc::new(
|
||||
ChainSpecBuilder::default()
|
||||
.chain(MAINNET.chain)
|
||||
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
|
||||
.cancun_activated()
|
||||
.build(),
|
||||
);
|
||||
|
||||
let (mut nodes, _tasks, wallet) = setup_engine::<EthereumNode>(
|
||||
2,
|
||||
chain_spec.clone(),
|
||||
false,
|
||||
Default::default(),
|
||||
eth_payload_attributes,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut producer = nodes.pop().unwrap();
|
||||
let receiver = nodes.pop().unwrap();
|
||||
|
||||
// Get engine API client for the receiver node
|
||||
let receiver_engine = receiver.auth_server_handle().http_client();
|
||||
|
||||
// Inject a transaction to allow block building (advance_block waits for transactions)
|
||||
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
|
||||
producer.rpc.inject_tx(raw_tx).await?;
|
||||
|
||||
// Build a valid payload on the producer
|
||||
let payload = producer.advance_block().await?;
|
||||
let valid_block = payload.block().clone();
|
||||
|
||||
// Create valid payload first, then corrupt the state root
|
||||
let mut invalid_payload = ExecutionPayloadV3::from_block_unchecked(
|
||||
valid_block.hash(),
|
||||
&valid_block.clone().into_block(),
|
||||
);
|
||||
let original_state_root = invalid_payload.payload_inner.payload_inner.state_root;
|
||||
invalid_payload.payload_inner.payload_inner.state_root = B256::random_with(&mut rng);
|
||||
|
||||
// Send the invalid payload to the receiver - should be rejected
|
||||
let invalid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
|
||||
&receiver_engine,
|
||||
invalid_payload.clone(),
|
||||
vec![],
|
||||
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!(
|
||||
"Invalid payload response: {:?} (state_root changed from {original_state_root} to {})",
|
||||
invalid_result.status, invalid_payload.payload_inner.payload_inner.state_root
|
||||
);
|
||||
|
||||
// The invalid payload should be rejected
|
||||
assert!(
|
||||
matches!(
|
||||
invalid_result.status,
|
||||
PayloadStatusEnum::Invalid { .. } | PayloadStatusEnum::Syncing
|
||||
),
|
||||
"Expected INVALID or SYNCING status for invalid payload, got {:?}",
|
||||
invalid_result.status
|
||||
);
|
||||
|
||||
// Now send the valid payload - should be accepted
|
||||
let valid_payload = ExecutionPayloadV3::from_block_unchecked(
|
||||
valid_block.hash(),
|
||||
&valid_block.clone().into_block(),
|
||||
);
|
||||
|
||||
let valid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
|
||||
&receiver_engine,
|
||||
valid_payload,
|
||||
vec![],
|
||||
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!("Valid payload response: {:?}", valid_result.status);
|
||||
|
||||
// The valid payload should be accepted
|
||||
assert!(
|
||||
matches!(
|
||||
valid_result.status,
|
||||
PayloadStatusEnum::Valid | PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted
|
||||
),
|
||||
"Expected VALID/SYNCING/ACCEPTED status for valid payload, got {:?}",
|
||||
valid_result.status
|
||||
);
|
||||
|
||||
// Update forkchoice on receiver to the valid block
|
||||
receiver.update_forkchoice(valid_block.hash(), valid_block.hash()).await?;
|
||||
|
||||
// Verify the receiver node is at the expected block
|
||||
let receiver_head = receiver.block_hash(1);
|
||||
let producer_head = producer.block_hash(1);
|
||||
assert_eq!(
|
||||
receiver_head, producer_head,
|
||||
"Receiver should have synced to the same chain as producer"
|
||||
);
|
||||
|
||||
println!(
|
||||
"Test passed: Receiver successfully handled invalid payloads and synced to valid chain"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Tests that a node can handle multiple consecutive invalid payloads
|
||||
/// before receiving a valid one.
|
||||
#[tokio::test]
|
||||
async fn can_handle_multiple_invalid_payloads() -> eyre::Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let seed: [u8; 32] = rand::rng().random();
|
||||
let mut rng = StdRng::from_seed(seed);
|
||||
println!("Seed: {seed:?}");
|
||||
|
||||
let chain_spec = Arc::new(
|
||||
ChainSpecBuilder::default()
|
||||
.chain(MAINNET.chain)
|
||||
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
|
||||
.cancun_activated()
|
||||
.build(),
|
||||
);
|
||||
|
||||
let (mut nodes, _tasks, wallet) = setup_engine::<EthereumNode>(
|
||||
2,
|
||||
chain_spec.clone(),
|
||||
false,
|
||||
Default::default(),
|
||||
eth_payload_attributes,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut producer = nodes.pop().unwrap();
|
||||
let receiver = nodes.pop().unwrap();
|
||||
|
||||
let receiver_engine = receiver.auth_server_handle().http_client();
|
||||
|
||||
// Inject a transaction to allow block building
|
||||
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
|
||||
producer.rpc.inject_tx(raw_tx).await?;
|
||||
|
||||
// Produce a valid block
|
||||
let payload = producer.advance_block().await?;
|
||||
let valid_block = payload.block().clone();
|
||||
|
||||
// Send multiple invalid payloads with different corruptions
|
||||
for i in 0..3 {
|
||||
// Create valid payload first, then corrupt the state root
|
||||
let mut invalid_payload = ExecutionPayloadV3::from_block_unchecked(
|
||||
valid_block.hash(),
|
||||
&valid_block.clone().into_block(),
|
||||
);
|
||||
invalid_payload.payload_inner.payload_inner.state_root = B256::random_with(&mut rng);
|
||||
|
||||
let result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
|
||||
&receiver_engine,
|
||||
invalid_payload,
|
||||
vec![],
|
||||
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!("Invalid payload {i}: status = {:?}", result.status);
|
||||
|
||||
assert!(
|
||||
matches!(result.status, PayloadStatusEnum::Invalid { .. } | PayloadStatusEnum::Syncing),
|
||||
"Expected INVALID or SYNCING for invalid payload {i}, got {:?}",
|
||||
result.status
|
||||
);
|
||||
}
|
||||
|
||||
// Now send the valid payload
|
||||
let valid_payload = ExecutionPayloadV3::from_block_unchecked(
|
||||
valid_block.hash(),
|
||||
&valid_block.clone().into_block(),
|
||||
);
|
||||
|
||||
let valid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
|
||||
&receiver_engine,
|
||||
valid_payload,
|
||||
vec![],
|
||||
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!("Valid payload: status = {:?}", valid_result.status);
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
valid_result.status,
|
||||
PayloadStatusEnum::Valid | PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted
|
||||
),
|
||||
"Expected valid status for correct payload, got {:?}",
|
||||
valid_result.status
|
||||
);
|
||||
|
||||
// Finalize the valid block
|
||||
receiver.update_forkchoice(valid_block.hash(), valid_block.hash()).await?;
|
||||
|
||||
println!("Test passed: Receiver handled multiple invalid payloads and accepted valid one");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Tests invalid payload handling with blocks that contain transactions.
|
||||
///
|
||||
/// This test sends real transactions to node 1, produces blocks with those transactions,
|
||||
/// then sends invalid (corrupted state root) and valid payloads to node 2.
|
||||
#[tokio::test]
|
||||
async fn can_handle_invalid_payload_with_transactions() -> eyre::Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let seed: [u8; 32] = rand::rng().random();
|
||||
let mut rng = StdRng::from_seed(seed);
|
||||
println!("Seed: {seed:?}");
|
||||
|
||||
let chain_spec = Arc::new(
|
||||
ChainSpecBuilder::default()
|
||||
.chain(MAINNET.chain)
|
||||
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
|
||||
.cancun_activated()
|
||||
.build(),
|
||||
);
|
||||
|
||||
let (mut nodes, _tasks, wallet) = setup_engine::<EthereumNode>(
|
||||
2,
|
||||
chain_spec.clone(),
|
||||
false,
|
||||
Default::default(),
|
||||
eth_payload_attributes,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut producer = nodes.pop().unwrap();
|
||||
let receiver = nodes.pop().unwrap();
|
||||
|
||||
let receiver_engine = receiver.auth_server_handle().http_client();
|
||||
|
||||
// Create and send a transaction to the producer node
|
||||
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
|
||||
let tx_hash = producer.rpc.inject_tx(raw_tx).await?;
|
||||
println!("Injected transaction {tx_hash}");
|
||||
|
||||
// Build a block containing the transaction
|
||||
let payload = producer.advance_block().await?;
|
||||
let valid_block = payload.block().clone();
|
||||
|
||||
// Verify the block contains a transaction
|
||||
let tx_count = valid_block.body().transactions().count();
|
||||
println!("Block contains {tx_count} transaction(s)");
|
||||
assert!(tx_count > 0, "Block should contain at least one transaction");
|
||||
|
||||
// Create invalid payload by corrupting the state root
|
||||
let mut invalid_payload = ExecutionPayloadV3::from_block_unchecked(
|
||||
valid_block.hash(),
|
||||
&valid_block.clone().into_block(),
|
||||
);
|
||||
let original_state_root = invalid_payload.payload_inner.payload_inner.state_root;
|
||||
invalid_payload.payload_inner.payload_inner.state_root = B256::random_with(&mut rng);
|
||||
|
||||
// Send invalid payload - should be rejected
|
||||
let invalid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
|
||||
&receiver_engine,
|
||||
invalid_payload.clone(),
|
||||
vec![],
|
||||
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!(
|
||||
"Invalid payload (with tx) response: {:?} (state_root changed from {original_state_root} to {})",
|
||||
invalid_result.status,
|
||||
invalid_payload.payload_inner.payload_inner.state_root
|
||||
);
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
invalid_result.status,
|
||||
PayloadStatusEnum::Invalid { .. } | PayloadStatusEnum::Syncing
|
||||
),
|
||||
"Expected INVALID or SYNCING for invalid payload with transactions, got {:?}",
|
||||
invalid_result.status
|
||||
);
|
||||
|
||||
// Send valid payload - should be accepted
|
||||
let valid_payload = ExecutionPayloadV3::from_block_unchecked(
|
||||
valid_block.hash(),
|
||||
&valid_block.clone().into_block(),
|
||||
);
|
||||
|
||||
let valid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
|
||||
&receiver_engine,
|
||||
valid_payload,
|
||||
vec![],
|
||||
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!("Valid payload (with tx) response: {:?}", valid_result.status);
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
valid_result.status,
|
||||
PayloadStatusEnum::Valid | PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted
|
||||
),
|
||||
"Expected valid status for correct payload with transactions, got {:?}",
|
||||
valid_result.status
|
||||
);
|
||||
|
||||
// Update forkchoice
|
||||
receiver.update_forkchoice(valid_block.hash(), valid_block.hash()).await?;
|
||||
|
||||
// Verify both nodes are at the same head
|
||||
let receiver_head = receiver.block_hash(1);
|
||||
let producer_head = producer.block_hash(1);
|
||||
assert_eq!(
|
||||
receiver_head, producer_head,
|
||||
"Receiver should have synced to the same chain as producer"
|
||||
);
|
||||
|
||||
println!("Test passed: Receiver handled invalid payloads with transactions correctly");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -4,6 +4,7 @@ mod blobs;
|
||||
mod custom_genesis;
|
||||
mod dev;
|
||||
mod eth;
|
||||
mod invalid_payload;
|
||||
mod p2p;
|
||||
mod pool;
|
||||
mod prestate;
|
||||
|
||||
@@ -153,7 +153,7 @@ async fn maintain_txpool_reorg() -> eyre::Result<()> {
|
||||
w1.address(),
|
||||
);
|
||||
let pooled_tx1 = EthPooledTransaction::new(tx1.clone(), 200);
|
||||
let tx_hash1 = *pooled_tx1.clone().hash();
|
||||
let tx_hash1 = *pooled_tx1.hash();
|
||||
|
||||
// build tx2 from wallet2
|
||||
let envelop2 = TransactionTestContext::transfer_tx(1, w2.clone()).await;
|
||||
@@ -162,7 +162,7 @@ async fn maintain_txpool_reorg() -> eyre::Result<()> {
|
||||
w2.address(),
|
||||
);
|
||||
let pooled_tx2 = EthPooledTransaction::new(tx2.clone(), 200);
|
||||
let tx_hash2 = *pooled_tx2.clone().hash();
|
||||
let tx_hash2 = *pooled_tx2.hash();
|
||||
|
||||
let block_info = BlockInfo {
|
||||
block_gas_limit: ETHEREUM_BLOCK_GAS_LIMIT_30M,
|
||||
|
||||
@@ -155,7 +155,7 @@ where
|
||||
let state_provider = client.state_by_block_hash(parent_header.hash())?;
|
||||
let state = StateProviderDatabase::new(state_provider.as_ref());
|
||||
let mut db =
|
||||
State::builder().with_database_ref(cached_reads.as_db(state)).with_bundle_update().build();
|
||||
State::builder().with_database(cached_reads.as_db_mut(state)).with_bundle_update().build();
|
||||
|
||||
let mut builder = evm_config
|
||||
.builder_for_next_block(
|
||||
@@ -247,7 +247,7 @@ where
|
||||
limit: MAX_RLP_BLOCK_SIZE,
|
||||
},
|
||||
);
|
||||
continue;
|
||||
continue
|
||||
}
|
||||
|
||||
// There's only limited amount of blob space available per block, so we need to check if
|
||||
|
||||
@@ -236,7 +236,7 @@ impl reth_codecs::Compact for Transaction {
|
||||
// # Panics
|
||||
//
|
||||
// A panic will be triggered if an identifier larger than 3 is passed from the database. For
|
||||
// optimism a identifier with value [`DEPOSIT_TX_TYPE_ID`] is allowed.
|
||||
// optimism an identifier with value [`DEPOSIT_TX_TYPE_ID`] is allowed.
|
||||
fn from_compact(buf: &[u8], identifier: usize) -> (Self, &[u8]) {
|
||||
let (tx_type, buf) = TxType::from_compact(buf, identifier);
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
use crate::{execute::ExecutableTxFor, ConfigureEvm, EvmEnvFor, ExecutionCtxFor};
|
||||
use crate::{execute::ExecutableTxFor, ConfigureEvm, EvmEnvFor, ExecutionCtxFor, TxEnvFor};
|
||||
use alloy_evm::{block::ExecutableTxParts, RecoveredTx};
|
||||
use rayon::prelude::*;
|
||||
use reth_primitives_traits::TxTy;
|
||||
|
||||
/// [`ConfigureEvm`] extension providing methods for executing payloads.
|
||||
pub trait ConfigureEngineEvm<ExecutionData>: ConfigureEvm {
|
||||
@@ -61,11 +63,16 @@ where
|
||||
|
||||
/// Iterator over executable transactions.
|
||||
pub trait ExecutableTxIterator<Evm: ConfigureEvm>:
|
||||
ExecutableTxTuple<Tx: ExecutableTxFor<Evm>>
|
||||
ExecutableTxTuple<Tx: ExecutableTxFor<Evm, Recovered = Self::Recovered>>
|
||||
{
|
||||
/// HACK: for some reason, this duplicated AT is the only way to enforce the inner Recovered:
|
||||
/// Send + Sync bound. Effectively alias for `Self::Tx::Recovered`.
|
||||
type Recovered: RecoveredTx<TxTy<Evm::Primitives>> + Send + Sync;
|
||||
}
|
||||
|
||||
impl<T, Evm: ConfigureEvm> ExecutableTxIterator<Evm> for T where
|
||||
T: ExecutableTxTuple<Tx: ExecutableTxFor<Evm>>
|
||||
impl<T, Evm: ConfigureEvm> ExecutableTxIterator<Evm> for T
|
||||
where
|
||||
T: ExecutableTxTuple<Tx: ExecutableTxFor<Evm, Recovered: Send + Sync>>,
|
||||
{
|
||||
type Recovered = <T::Tx as ExecutableTxParts<TxEnvFor<Evm>, TxTy<Evm::Primitives>>>::Recovered;
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ use alloy_consensus::{BlockHeader, Header};
|
||||
use alloy_eips::eip2718::WithEncoded;
|
||||
pub use alloy_evm::block::{BlockExecutor, BlockExecutorFactory};
|
||||
use alloy_evm::{
|
||||
block::{CommitChanges, ExecutableTx},
|
||||
block::{CommitChanges, ExecutableTxParts},
|
||||
Evm, EvmEnv, EvmFactory, RecoveredTx, ToTxEnv,
|
||||
};
|
||||
use alloy_primitives::{Address, B256};
|
||||
@@ -148,20 +148,6 @@ pub trait Executor<DB: Database>: Sized {
|
||||
fn size_hint(&self) -> usize;
|
||||
}
|
||||
|
||||
/// Helper type for the output of executing a block.
|
||||
///
|
||||
/// Deprecated: this type is unused within reth and will be removed in the next
|
||||
/// major release. Use `reth_execution_types::BlockExecutionResult` or
|
||||
/// `reth_execution_types::BlockExecutionOutput`.
|
||||
#[deprecated(note = "Use reth_execution_types::BlockExecutionResult or BlockExecutionOutput")]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ExecuteOutput<R> {
|
||||
/// Receipts obtained after executing a block.
|
||||
pub receipts: Vec<R>,
|
||||
/// Cumulative gas used in the block execution.
|
||||
pub gas_used: u64,
|
||||
}
|
||||
|
||||
/// Input for block building. Consumed by [`BlockAssembler`].
|
||||
///
|
||||
/// This struct contains all the data needed by the [`BlockAssembler`] to create
|
||||
@@ -415,49 +401,31 @@ where
|
||||
|
||||
/// Conversions for executable transactions.
|
||||
pub trait ExecutorTx<Executor: BlockExecutor> {
|
||||
/// Converts the transaction into [`ExecutableTx`].
|
||||
fn as_executable(&self) -> impl ExecutableTx<Executor>;
|
||||
|
||||
/// Converts the transaction into [`Recovered`].
|
||||
fn into_recovered(self) -> Recovered<Executor::Transaction>;
|
||||
/// Converts the transaction into a tuple of [`TxEnvFor`] and [`Recovered`].
|
||||
fn into_parts(self) -> (<Executor::Evm as Evm>::Tx, Recovered<Executor::Transaction>);
|
||||
}
|
||||
|
||||
impl<Executor: BlockExecutor> ExecutorTx<Executor>
|
||||
for WithEncoded<Recovered<Executor::Transaction>>
|
||||
{
|
||||
fn as_executable(&self) -> impl ExecutableTx<Executor> {
|
||||
self
|
||||
}
|
||||
|
||||
fn into_recovered(self) -> Recovered<Executor::Transaction> {
|
||||
self.1
|
||||
fn into_parts(self) -> (<Executor::Evm as Evm>::Tx, Recovered<Executor::Transaction>) {
|
||||
(self.to_tx_env(), self.1)
|
||||
}
|
||||
}
|
||||
|
||||
impl<Executor: BlockExecutor> ExecutorTx<Executor> for Recovered<Executor::Transaction> {
|
||||
fn as_executable(&self) -> impl ExecutableTx<Executor> {
|
||||
self
|
||||
}
|
||||
|
||||
fn into_recovered(self) -> Self {
|
||||
self
|
||||
fn into_parts(self) -> (<Executor::Evm as Evm>::Tx, Self) {
|
||||
(self.to_tx_env(), self)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T, Executor> ExecutorTx<Executor>
|
||||
for WithTxEnv<<<Executor as BlockExecutor>::Evm as Evm>::Tx, T>
|
||||
impl<Executor> ExecutorTx<Executor>
|
||||
for WithTxEnv<<Executor::Evm as Evm>::Tx, Recovered<Executor::Transaction>>
|
||||
where
|
||||
T: ExecutorTx<Executor> + Clone,
|
||||
Executor: BlockExecutor,
|
||||
<<Executor as BlockExecutor>::Evm as Evm>::Tx: Clone,
|
||||
Self: RecoveredTx<Executor::Transaction>,
|
||||
Executor: BlockExecutor<Transaction: Clone>,
|
||||
{
|
||||
fn as_executable(&self) -> impl ExecutableTx<Executor> {
|
||||
self
|
||||
}
|
||||
|
||||
fn into_recovered(self) -> Recovered<Executor::Transaction> {
|
||||
Arc::unwrap_or_clone(self.tx).into_recovered()
|
||||
fn into_parts(self) -> (<Executor::Evm as Evm>::Tx, Recovered<Executor::Transaction>) {
|
||||
(self.tx_env, Arc::unwrap_or_clone(self.tx))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -493,10 +461,11 @@ where
|
||||
&ExecutionResult<<<Self::Executor as BlockExecutor>::Evm as Evm>::HaltReason>,
|
||||
) -> CommitChanges,
|
||||
) -> Result<Option<u64>, BlockExecutionError> {
|
||||
let (tx_env, tx) = tx.into_parts();
|
||||
if let Some(gas_used) =
|
||||
self.executor.execute_transaction_with_commit_condition(tx.as_executable(), f)?
|
||||
self.executor.execute_transaction_with_commit_condition((tx_env, &tx), f)?
|
||||
{
|
||||
self.transactions.push(tx.into_recovered());
|
||||
self.transactions.push(tx);
|
||||
Ok(Some(gas_used))
|
||||
} else {
|
||||
Ok(None)
|
||||
@@ -623,20 +592,20 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// A helper trait marking a 'static type that can be converted into an [`ExecutableTx`] for block
|
||||
/// executor.
|
||||
/// A helper trait marking a 'static type that can be converted into an [`ExecutableTxParts`] for
|
||||
/// block executor.
|
||||
pub trait ExecutableTxFor<Evm: ConfigureEvm>:
|
||||
ToTxEnv<TxEnvFor<Evm>> + RecoveredTx<TxTy<Evm::Primitives>>
|
||||
ExecutableTxParts<TxEnvFor<Evm>, TxTy<Evm::Primitives>> + RecoveredTx<TxTy<Evm::Primitives>>
|
||||
{
|
||||
}
|
||||
|
||||
impl<T, Evm: ConfigureEvm> ExecutableTxFor<Evm> for T where
|
||||
T: ToTxEnv<TxEnvFor<Evm>> + RecoveredTx<TxTy<Evm::Primitives>>
|
||||
T: ExecutableTxParts<TxEnvFor<Evm>, TxTy<Evm::Primitives>> + RecoveredTx<TxTy<Evm::Primitives>>
|
||||
{
|
||||
}
|
||||
|
||||
/// A container for a transaction and a transaction environment.
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug)]
|
||||
pub struct WithTxEnv<TxEnv, T> {
|
||||
/// The transaction environment for EVM.
|
||||
pub tx_env: TxEnv,
|
||||
@@ -644,6 +613,12 @@ pub struct WithTxEnv<TxEnv, T> {
|
||||
pub tx: Arc<T>,
|
||||
}
|
||||
|
||||
impl<TxEnv: Clone, T> Clone for WithTxEnv<TxEnv, T> {
|
||||
fn clone(&self) -> Self {
|
||||
Self { tx_env: self.tx_env.clone(), tx: self.tx.clone() }
|
||||
}
|
||||
}
|
||||
|
||||
impl<TxEnv, Tx, T: RecoveredTx<Tx>> RecoveredTx<Tx> for WithTxEnv<TxEnv, T> {
|
||||
fn tx(&self) -> &Tx {
|
||||
self.tx.tx()
|
||||
@@ -654,9 +629,11 @@ impl<TxEnv, Tx, T: RecoveredTx<Tx>> RecoveredTx<Tx> for WithTxEnv<TxEnv, T> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<TxEnv: Clone, T> ToTxEnv<TxEnv> for WithTxEnv<TxEnv, T> {
|
||||
fn to_tx_env(&self) -> TxEnv {
|
||||
self.tx_env.clone()
|
||||
impl<TxEnv, T: RecoveredTx<Tx>, Tx> ExecutableTxParts<TxEnv, Tx> for WithTxEnv<TxEnv, T> {
|
||||
type Recovered = Arc<T>;
|
||||
|
||||
fn into_parts(self) -> (TxEnv, Self::Recovered) {
|
||||
(self.tx_env, self.tx)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -741,6 +718,7 @@ mod tests {
|
||||
nonce,
|
||||
code_hash: KECCAK_EMPTY,
|
||||
code: None,
|
||||
account_id: None,
|
||||
};
|
||||
state.insert_account(addr, account_info);
|
||||
state
|
||||
@@ -777,8 +755,13 @@ mod tests {
|
||||
|
||||
let mut state = setup_state_with_account(addr1, 100, 1);
|
||||
|
||||
let account2 =
|
||||
AccountInfo { balance: U256::from(200), nonce: 1, code_hash: KECCAK_EMPTY, code: None };
|
||||
let account2 = AccountInfo {
|
||||
balance: U256::from(200),
|
||||
nonce: 1,
|
||||
code_hash: KECCAK_EMPTY,
|
||||
code: None,
|
||||
account_id: None,
|
||||
};
|
||||
state.insert_account(addr2, account2);
|
||||
|
||||
let mut increments = HashMap::default();
|
||||
@@ -799,8 +782,13 @@ mod tests {
|
||||
|
||||
let mut state = setup_state_with_account(addr1, 100, 1);
|
||||
|
||||
let account2 =
|
||||
AccountInfo { balance: U256::from(200), nonce: 1, code_hash: KECCAK_EMPTY, code: None };
|
||||
let account2 = AccountInfo {
|
||||
balance: U256::from(200),
|
||||
nonce: 1,
|
||||
code_hash: KECCAK_EMPTY,
|
||||
code: None,
|
||||
account_id: None,
|
||||
};
|
||||
state.insert_account(addr2, account2);
|
||||
|
||||
let mut increments = HashMap::default();
|
||||
|
||||
@@ -399,7 +399,7 @@ pub trait ConfigureEvm: Clone + Debug + Send + Sync + Unpin {
|
||||
/// // Complete block building
|
||||
/// let outcome = builder.finish(state_provider)?;
|
||||
/// ```
|
||||
fn builder_for_next_block<'a, DB: Database>(
|
||||
fn builder_for_next_block<'a, DB: Database + 'a>(
|
||||
&'a self,
|
||||
db: &'a mut State<DB>,
|
||||
parent: &'a SealedHeader<<Self::Primitives as NodePrimitives>::BlockHeader>,
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
//! Contains [Chain], a chain of blocks and their final state.
|
||||
|
||||
use crate::ExecutionOutcome;
|
||||
use alloc::{borrow::Cow, collections::BTreeMap, sync::Arc, vec::Vec};
|
||||
use alloy_consensus::{transaction::Recovered, BlockHeader};
|
||||
use alloc::{borrow::Cow, collections::BTreeMap, vec::Vec};
|
||||
use alloy_consensus::{transaction::Recovered, BlockHeader, TxReceipt};
|
||||
use alloy_eips::{eip1898::ForkBlock, eip2718::Encodable2718, BlockNumHash};
|
||||
use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash};
|
||||
use alloy_primitives::{Address, BlockHash, BlockNumber, Log, TxHash};
|
||||
use core::{fmt, ops::RangeInclusive};
|
||||
use reth_primitives_traits::{
|
||||
transaction::signed::SignedTransaction, Block, BlockBody, IndexedTx, NodePrimitives,
|
||||
RecoveredBlock, SealedHeader,
|
||||
};
|
||||
use reth_trie_common::{updates::TrieUpdatesSorted, HashedPostStateSorted};
|
||||
use reth_trie_common::LazyTrieData;
|
||||
|
||||
/// A chain of blocks and their final state.
|
||||
///
|
||||
@@ -34,10 +34,10 @@ pub struct Chain<N: NodePrimitives = reth_ethereum_primitives::EthPrimitives> {
|
||||
///
|
||||
/// Additionally, it includes the individual state changes that led to the current state.
|
||||
execution_outcome: ExecutionOutcome<N::Receipt>,
|
||||
/// State trie updates for each block in the chain, keyed by block number.
|
||||
trie_updates: BTreeMap<BlockNumber, Arc<TrieUpdatesSorted>>,
|
||||
/// Hashed post state for each block in the chain, keyed by block number.
|
||||
hashed_state: BTreeMap<BlockNumber, Arc<HashedPostStateSorted>>,
|
||||
/// Lazy trie data for each block in the chain, keyed by block number.
|
||||
///
|
||||
/// Contains handles to lazily-initialized sorted trie updates and hashed state.
|
||||
trie_data: BTreeMap<BlockNumber, LazyTrieData>,
|
||||
}
|
||||
|
||||
type ChainTxReceiptMeta<'a, N> = (
|
||||
@@ -52,8 +52,7 @@ impl<N: NodePrimitives> Default for Chain<N> {
|
||||
Self {
|
||||
blocks: Default::default(),
|
||||
execution_outcome: Default::default(),
|
||||
trie_updates: Default::default(),
|
||||
hashed_state: Default::default(),
|
||||
trie_data: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -67,27 +66,23 @@ impl<N: NodePrimitives> Chain<N> {
|
||||
pub fn new(
|
||||
blocks: impl IntoIterator<Item = RecoveredBlock<N::Block>>,
|
||||
execution_outcome: ExecutionOutcome<N::Receipt>,
|
||||
trie_updates: BTreeMap<BlockNumber, Arc<TrieUpdatesSorted>>,
|
||||
hashed_state: BTreeMap<BlockNumber, Arc<HashedPostStateSorted>>,
|
||||
trie_data: BTreeMap<BlockNumber, LazyTrieData>,
|
||||
) -> Self {
|
||||
let blocks =
|
||||
blocks.into_iter().map(|b| (b.header().number(), b)).collect::<BTreeMap<_, _>>();
|
||||
debug_assert!(!blocks.is_empty(), "Chain should have at least one block");
|
||||
|
||||
Self { blocks, execution_outcome, trie_updates, hashed_state }
|
||||
Self { blocks, execution_outcome, trie_data }
|
||||
}
|
||||
|
||||
/// Create new Chain from a single block and its state.
|
||||
pub fn from_block(
|
||||
block: RecoveredBlock<N::Block>,
|
||||
execution_outcome: ExecutionOutcome<N::Receipt>,
|
||||
trie_updates: Arc<TrieUpdatesSorted>,
|
||||
hashed_state: Arc<HashedPostStateSorted>,
|
||||
trie_data: LazyTrieData,
|
||||
) -> Self {
|
||||
let block_number = block.header().number();
|
||||
let trie_updates_map = BTreeMap::from([(block_number, trie_updates)]);
|
||||
let hashed_state_map = BTreeMap::from([(block_number, hashed_state)]);
|
||||
Self::new([block], execution_outcome, trie_updates_map, hashed_state_map)
|
||||
Self::new([block], execution_outcome, BTreeMap::from([(block_number, trie_data)]))
|
||||
}
|
||||
|
||||
/// Get the blocks in this chain.
|
||||
@@ -105,37 +100,19 @@ impl<N: NodePrimitives> Chain<N> {
|
||||
self.blocks.values().map(|block| block.clone_sealed_header())
|
||||
}
|
||||
|
||||
/// Get all trie updates for this chain.
|
||||
pub const fn trie_updates(&self) -> &BTreeMap<BlockNumber, Arc<TrieUpdatesSorted>> {
|
||||
&self.trie_updates
|
||||
/// Get all trie data for this chain.
|
||||
pub const fn trie_data(&self) -> &BTreeMap<BlockNumber, LazyTrieData> {
|
||||
&self.trie_data
|
||||
}
|
||||
|
||||
/// Get trie updates for a specific block number.
|
||||
pub fn trie_updates_at(&self, block_number: BlockNumber) -> Option<&Arc<TrieUpdatesSorted>> {
|
||||
self.trie_updates.get(&block_number)
|
||||
/// Get trie data for a specific block number.
|
||||
pub fn trie_data_at(&self, block_number: BlockNumber) -> Option<&LazyTrieData> {
|
||||
self.trie_data.get(&block_number)
|
||||
}
|
||||
|
||||
/// Remove all trie updates for this chain.
|
||||
pub fn clear_trie_updates(&mut self) {
|
||||
self.trie_updates.clear();
|
||||
}
|
||||
|
||||
/// Get all hashed states for this chain.
|
||||
pub const fn hashed_state(&self) -> &BTreeMap<BlockNumber, Arc<HashedPostStateSorted>> {
|
||||
&self.hashed_state
|
||||
}
|
||||
|
||||
/// Get hashed state for a specific block number.
|
||||
pub fn hashed_state_at(
|
||||
&self,
|
||||
block_number: BlockNumber,
|
||||
) -> Option<&Arc<HashedPostStateSorted>> {
|
||||
self.hashed_state.get(&block_number)
|
||||
}
|
||||
|
||||
/// Remove all hashed states for this chain.
|
||||
pub fn clear_hashed_state(&mut self) {
|
||||
self.hashed_state.clear();
|
||||
/// Remove all trie data for this chain.
|
||||
pub fn clear_trie_data(&mut self) {
|
||||
self.trie_data.clear();
|
||||
}
|
||||
|
||||
/// Get execution outcome of this chain
|
||||
@@ -183,23 +160,16 @@ impl<N: NodePrimitives> Chain<N> {
|
||||
/// Destructure the chain into its inner components:
|
||||
/// 1. The blocks contained in the chain.
|
||||
/// 2. The execution outcome representing the final state.
|
||||
/// 3. The trie updates map.
|
||||
/// 4. The hashed state map.
|
||||
/// 3. The trie data map.
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub fn into_inner(
|
||||
self,
|
||||
) -> (
|
||||
ChainBlocks<'static, N::Block>,
|
||||
ExecutionOutcome<N::Receipt>,
|
||||
BTreeMap<BlockNumber, Arc<TrieUpdatesSorted>>,
|
||||
BTreeMap<BlockNumber, Arc<HashedPostStateSorted>>,
|
||||
BTreeMap<BlockNumber, LazyTrieData>,
|
||||
) {
|
||||
(
|
||||
ChainBlocks { blocks: Cow::Owned(self.blocks) },
|
||||
self.execution_outcome,
|
||||
self.trie_updates,
|
||||
self.hashed_state,
|
||||
)
|
||||
(ChainBlocks { blocks: Cow::Owned(self.blocks) }, self.execution_outcome, self.trie_data)
|
||||
}
|
||||
|
||||
/// Destructure the chain into its inner components:
|
||||
@@ -214,6 +184,19 @@ impl<N: NodePrimitives> Chain<N> {
|
||||
self.execution_outcome.receipts().iter()
|
||||
}
|
||||
|
||||
/// Returns an iterator over all receipts in the chain.
|
||||
pub fn receipts_iter(&self) -> impl Iterator<Item = &N::Receipt> + '_ {
|
||||
self.block_receipts_iter().flatten()
|
||||
}
|
||||
|
||||
/// Returns an iterator over all logs in the chain.
|
||||
pub fn logs_iter(&self) -> impl Iterator<Item = &Log> + '_
|
||||
where
|
||||
N::Receipt: TxReceipt<Log = Log>,
|
||||
{
|
||||
self.receipts_iter().flat_map(|receipt| receipt.logs())
|
||||
}
|
||||
|
||||
/// Returns an iterator over all blocks in the chain with increasing block number.
|
||||
pub fn blocks_iter(&self) -> impl Iterator<Item = &RecoveredBlock<N::Block>> + '_ {
|
||||
self.blocks().iter().map(|block| block.1)
|
||||
@@ -329,14 +312,12 @@ impl<N: NodePrimitives> Chain<N> {
|
||||
&mut self,
|
||||
block: RecoveredBlock<N::Block>,
|
||||
execution_outcome: ExecutionOutcome<N::Receipt>,
|
||||
trie_updates: Arc<TrieUpdatesSorted>,
|
||||
hashed_state: Arc<HashedPostStateSorted>,
|
||||
trie_data: LazyTrieData,
|
||||
) {
|
||||
let block_number = block.header().number();
|
||||
self.blocks.insert(block_number, block);
|
||||
self.execution_outcome.extend(execution_outcome);
|
||||
self.trie_updates.insert(block_number, trie_updates);
|
||||
self.hashed_state.insert(block_number, hashed_state);
|
||||
self.trie_data.insert(block_number, trie_data);
|
||||
}
|
||||
|
||||
/// Merge two chains by appending the given chain into the current one.
|
||||
@@ -355,8 +336,7 @@ impl<N: NodePrimitives> Chain<N> {
|
||||
// Insert blocks from other chain
|
||||
self.blocks.extend(other.blocks);
|
||||
self.execution_outcome.extend(other.execution_outcome);
|
||||
self.trie_updates.extend(other.trie_updates);
|
||||
self.hashed_state.extend(other.hashed_state);
|
||||
self.trie_data.extend(other.trie_data);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -583,14 +563,14 @@ pub(super) mod serde_bincode_compat {
|
||||
execution_outcome: value.execution_outcome.as_repr(),
|
||||
_trie_updates_legacy: None,
|
||||
trie_updates: value
|
||||
.trie_updates
|
||||
.trie_data
|
||||
.iter()
|
||||
.map(|(k, v)| (*k, v.as_ref().into()))
|
||||
.map(|(k, v)| (*k, v.get().trie_updates.as_ref().into()))
|
||||
.collect(),
|
||||
hashed_state: value
|
||||
.hashed_state
|
||||
.trie_data
|
||||
.iter()
|
||||
.map(|(k, v)| (*k, v.as_ref().into()))
|
||||
.map(|(k, v)| (*k, v.get().hashed_state.as_ref().into()))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
@@ -603,19 +583,24 @@ pub(super) mod serde_bincode_compat {
|
||||
>,
|
||||
{
|
||||
fn from(value: Chain<'a, N>) -> Self {
|
||||
use reth_trie_common::LazyTrieData;
|
||||
|
||||
let hashed_state_map: BTreeMap<_, _> =
|
||||
value.hashed_state.into_iter().map(|(k, v)| (k, Arc::new(v.into()))).collect();
|
||||
|
||||
let trie_data: BTreeMap<BlockNumber, LazyTrieData> = value
|
||||
.trie_updates
|
||||
.into_iter()
|
||||
.map(|(k, v)| {
|
||||
let hashed_state = hashed_state_map.get(&k).cloned().unwrap_or_default();
|
||||
(k, LazyTrieData::ready(hashed_state, Arc::new(v.into())))
|
||||
})
|
||||
.collect();
|
||||
|
||||
Self {
|
||||
blocks: value.blocks.0.into_owned(),
|
||||
execution_outcome: ExecutionOutcome::from_repr(value.execution_outcome),
|
||||
trie_updates: value
|
||||
.trie_updates
|
||||
.into_iter()
|
||||
.map(|(k, v)| (k, Arc::new(v.into())))
|
||||
.collect(),
|
||||
hashed_state: value
|
||||
.hashed_state
|
||||
.into_iter()
|
||||
.map(|(k, v)| (k, Arc::new(v.into())))
|
||||
.collect(),
|
||||
trie_data,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -676,7 +661,6 @@ pub(super) mod serde_bincode_compat {
|
||||
.unwrap()],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
),
|
||||
};
|
||||
|
||||
@@ -776,12 +760,8 @@ mod tests {
|
||||
let mut block_state_extended = execution_outcome1;
|
||||
block_state_extended.extend(execution_outcome2);
|
||||
|
||||
let chain: Chain = Chain::new(
|
||||
vec![block1.clone(), block2.clone()],
|
||||
block_state_extended,
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
);
|
||||
let chain: Chain =
|
||||
Chain::new(vec![block1.clone(), block2.clone()], block_state_extended, BTreeMap::new());
|
||||
|
||||
// return tip state
|
||||
assert_eq!(
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
use alloy_primitives::{Address, B256, U256};
|
||||
use reth_primitives_traits::{Account, Bytecode};
|
||||
use revm::database::BundleState;
|
||||
|
||||
pub use alloy_evm::block::BlockExecutionResult;
|
||||
@@ -23,3 +25,36 @@ pub struct BlockExecutionOutput<T> {
|
||||
/// The changed state of the block after execution.
|
||||
pub state: BundleState,
|
||||
}
|
||||
|
||||
impl<T> BlockExecutionOutput<T> {
|
||||
/// Return bytecode if known.
|
||||
pub fn bytecode(&self, code_hash: &B256) -> Option<Bytecode> {
|
||||
self.state.bytecode(code_hash).map(Bytecode)
|
||||
}
|
||||
|
||||
/// Get account if account is known.
|
||||
pub fn account(&self, address: &Address) -> Option<Option<Account>> {
|
||||
self.state.account(address).map(|a| a.info.as_ref().map(Into::into))
|
||||
}
|
||||
|
||||
/// Get storage if value is known.
|
||||
///
|
||||
/// This means that depending on status we can potentially return `U256::ZERO`.
|
||||
pub fn storage(&self, address: &Address, storage_key: U256) -> Option<U256> {
|
||||
self.state.account(address).and_then(|a| a.storage_slot(storage_key))
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Default for BlockExecutionOutput<T> {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
result: BlockExecutionResult {
|
||||
receipts: Default::default(),
|
||||
requests: Default::default(),
|
||||
gas_used: 0,
|
||||
blob_gas_used: 0,
|
||||
},
|
||||
state: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -249,6 +249,14 @@ impl<T> ExecutionOutcome<T> {
|
||||
&self.receipts[index]
|
||||
}
|
||||
|
||||
/// Returns an iterator over receipt slices, one per block.
|
||||
///
|
||||
/// This is a more ergonomic alternative to `receipts()` that yields slices
|
||||
/// instead of requiring indexing into a nested `Vec<Vec<T>>`.
|
||||
pub fn receipts_iter(&self) -> impl Iterator<Item = &[T]> + '_ {
|
||||
self.receipts.iter().map(|v| v.as_slice())
|
||||
}
|
||||
|
||||
/// Is execution outcome empty.
|
||||
pub const fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
@@ -934,10 +942,20 @@ mod tests {
|
||||
let address3 = Address::random();
|
||||
|
||||
// Set up account info with some changes
|
||||
let account_info1 =
|
||||
AccountInfo { nonce: 1, balance: U256::from(100), code_hash: B256::ZERO, code: None };
|
||||
let account_info2 =
|
||||
AccountInfo { nonce: 2, balance: U256::from(200), code_hash: B256::ZERO, code: None };
|
||||
let account_info1 = AccountInfo {
|
||||
nonce: 1,
|
||||
balance: U256::from(100),
|
||||
code_hash: B256::ZERO,
|
||||
code: None,
|
||||
account_id: None,
|
||||
};
|
||||
let account_info2 = AccountInfo {
|
||||
nonce: 2,
|
||||
balance: U256::from(200),
|
||||
code_hash: B256::ZERO,
|
||||
code: None,
|
||||
account_id: None,
|
||||
};
|
||||
|
||||
// Set up the bundle state with these accounts
|
||||
let mut bundle_state = BundleState::default();
|
||||
|
||||
@@ -149,7 +149,7 @@ where
|
||||
executor.into_state().take_bundle(),
|
||||
results,
|
||||
);
|
||||
let chain = Chain::new(blocks, outcome, BTreeMap::new(), BTreeMap::new());
|
||||
let chain = Chain::new(blocks, outcome, BTreeMap::new());
|
||||
Ok(chain)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -503,6 +503,7 @@ where
|
||||
}
|
||||
break
|
||||
}
|
||||
let buffer_full = this.buffer.len() >= this.max_capacity;
|
||||
|
||||
// Update capacity
|
||||
this.update_capacity();
|
||||
@@ -536,6 +537,12 @@ where
|
||||
// Update capacity
|
||||
this.update_capacity();
|
||||
|
||||
// If the buffer was full and we made space, we need to wake up to accept new notifications
|
||||
if buffer_full && this.buffer.len() < this.max_capacity {
|
||||
debug!(target: "exex::manager", "Buffer has space again, waking up senders");
|
||||
cx.waker().wake_by_ref();
|
||||
}
|
||||
|
||||
// Update watch channel block number
|
||||
let finished_height = this.exex_handles.iter_mut().try_fold(u64::MAX, |curr, exex| {
|
||||
exex.finished_height.map_or(Err(()), |height| Ok(height.number.min(curr)))
|
||||
@@ -687,7 +694,6 @@ mod tests {
|
||||
BlockWriter, Chain, DBProvider, DatabaseProviderFactory, TransactionVariant,
|
||||
};
|
||||
use reth_testing_utils::generators::{self, random_block, BlockParams};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
fn empty_finalized_header_stream() -> ForkChoiceStream<SealedHeader> {
|
||||
let (tx, rx) = watch::channel(None);
|
||||
@@ -789,12 +795,7 @@ mod tests {
|
||||
block1.set_block_number(10);
|
||||
|
||||
let notification1 = ExExNotification::ChainCommitted {
|
||||
new: Arc::new(Chain::new(
|
||||
vec![block1.clone()],
|
||||
Default::default(),
|
||||
Default::default(),
|
||||
Default::default(),
|
||||
)),
|
||||
new: Arc::new(Chain::new(vec![block1.clone()], Default::default(), Default::default())),
|
||||
};
|
||||
|
||||
// Push the first notification
|
||||
@@ -812,12 +813,7 @@ mod tests {
|
||||
block2.set_block_number(20);
|
||||
|
||||
let notification2 = ExExNotification::ChainCommitted {
|
||||
new: Arc::new(Chain::new(
|
||||
vec![block2.clone()],
|
||||
Default::default(),
|
||||
Default::default(),
|
||||
Default::default(),
|
||||
)),
|
||||
new: Arc::new(Chain::new(vec![block2.clone()], Default::default(), Default::default())),
|
||||
};
|
||||
|
||||
exex_manager.push_notification(notification2.clone());
|
||||
@@ -860,12 +856,7 @@ mod tests {
|
||||
block1.set_block_number(10);
|
||||
|
||||
let notification1 = ExExNotification::ChainCommitted {
|
||||
new: Arc::new(Chain::new(
|
||||
vec![block1.clone()],
|
||||
Default::default(),
|
||||
Default::default(),
|
||||
Default::default(),
|
||||
)),
|
||||
new: Arc::new(Chain::new(vec![block1.clone()], Default::default(), Default::default())),
|
||||
};
|
||||
|
||||
exex_manager.push_notification(notification1.clone());
|
||||
@@ -1093,7 +1084,6 @@ mod tests {
|
||||
vec![Default::default()],
|
||||
Default::default(),
|
||||
Default::default(),
|
||||
Default::default(),
|
||||
)),
|
||||
};
|
||||
|
||||
@@ -1164,7 +1154,6 @@ mod tests {
|
||||
vec![Default::default()],
|
||||
Default::default(),
|
||||
Default::default(),
|
||||
Default::default(),
|
||||
)),
|
||||
};
|
||||
|
||||
@@ -1209,12 +1198,7 @@ mod tests {
|
||||
block1.set_block_number(10);
|
||||
|
||||
let notification = ExExNotification::ChainCommitted {
|
||||
new: Arc::new(Chain::new(
|
||||
vec![block1.clone()],
|
||||
Default::default(),
|
||||
Default::default(),
|
||||
Default::default(),
|
||||
)),
|
||||
new: Arc::new(Chain::new(vec![block1.clone()], Default::default(), Default::default())),
|
||||
};
|
||||
|
||||
let mut cx = Context::from_waker(futures::task::noop_waker_ref());
|
||||
@@ -1363,17 +1347,11 @@ mod tests {
|
||||
new: Arc::new(Chain::new(
|
||||
vec![genesis_block.clone()],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
Default::default(),
|
||||
)),
|
||||
};
|
||||
let notification = ExExNotification::ChainCommitted {
|
||||
new: Arc::new(Chain::new(
|
||||
vec![block.clone()],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
new: Arc::new(Chain::new(vec![block.clone()], Default::default(), Default::default())),
|
||||
};
|
||||
|
||||
let (finalized_headers_tx, rx) = watch::channel(None);
|
||||
@@ -1443,4 +1421,78 @@ mod tests {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_deadlock_manager_wakes_after_buffer_clears() {
|
||||
// This test simulates the scenario where the buffer fills up, ingestion pauses,
|
||||
// and then space clears. We verify the manager wakes up to process pending items.
|
||||
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let wal = Wal::new(temp_dir.path()).unwrap();
|
||||
let provider_factory = create_test_provider_factory();
|
||||
init_genesis(&provider_factory).unwrap();
|
||||
let provider = BlockchainProvider::new(provider_factory.clone()).unwrap();
|
||||
|
||||
// 1. Setup Manager with Capacity = 1
|
||||
let (exex_handle, _, mut notifications) = ExExHandle::new(
|
||||
"test_exex".to_string(),
|
||||
Default::default(),
|
||||
provider,
|
||||
EthEvmConfig::mainnet(),
|
||||
wal.handle(),
|
||||
);
|
||||
|
||||
let max_capacity = 2;
|
||||
let exex_manager = ExExManager::new(
|
||||
provider_factory,
|
||||
vec![exex_handle],
|
||||
max_capacity,
|
||||
wal,
|
||||
empty_finalized_header_stream(),
|
||||
);
|
||||
|
||||
let manager_handle = exex_manager.handle();
|
||||
|
||||
// Spawn manager in background so it runs continuously
|
||||
tokio::spawn(async move {
|
||||
exex_manager.await.ok();
|
||||
});
|
||||
|
||||
// Helper to create notifications
|
||||
let mut rng = generators::rng();
|
||||
let mut make_notif = |id: u64| {
|
||||
let block = random_block(&mut rng, id, BlockParams::default()).try_recover().unwrap();
|
||||
ExExNotification::ChainCommitted {
|
||||
new: Arc::new(Chain::new(vec![block], Default::default(), Default::default())),
|
||||
}
|
||||
};
|
||||
|
||||
manager_handle.send(ExExNotificationSource::Pipeline, make_notif(1)).unwrap();
|
||||
|
||||
// Send the "Stuck" Item (Notification #100).
|
||||
// At this point, the Manager loop has skipped the ingestion logic because buffer is full
|
||||
// (buffer_full=true). This item sits in the unbounded 'handle_rx' channel waiting.
|
||||
manager_handle.send(ExExNotificationSource::Pipeline, make_notif(100)).unwrap();
|
||||
|
||||
// 3. Relieve Pressure
|
||||
// We consume items from the ExEx.
|
||||
// As we pull items out, the ExEx frees space -> Manager sends buffered item -> Manager
|
||||
// frees space. Once Manager frees space, the FIX (wake_by_ref) should trigger,
|
||||
// causing it to read Notif #100.
|
||||
|
||||
// Consume the jam
|
||||
let _ = notifications.next().await.unwrap();
|
||||
|
||||
// 4. Assert No Deadlock
|
||||
// We expect Notification #100 next.
|
||||
// If the wake_by_ref fix is missing, this will Time Out because the manager is sleeping
|
||||
// despite having empty buffer.
|
||||
let result =
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), notifications.next()).await;
|
||||
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"Deadlock detected! Manager failed to wake up and process Pending Item #100."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -501,7 +501,6 @@ mod tests {
|
||||
.try_recover()?],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
};
|
||||
|
||||
@@ -570,7 +569,6 @@ mod tests {
|
||||
.try_recover()?],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
};
|
||||
|
||||
@@ -638,7 +636,6 @@ mod tests {
|
||||
vec![exex_head_block.clone().try_recover()?],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
};
|
||||
wal.commit(&exex_head_notification)?;
|
||||
@@ -653,7 +650,6 @@ mod tests {
|
||||
.try_recover()?],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
};
|
||||
|
||||
@@ -711,7 +707,6 @@ mod tests {
|
||||
vec![exex_head_block.clone().try_recover()?],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
};
|
||||
wal.commit(&exex_head_notification)?;
|
||||
@@ -731,7 +726,6 @@ mod tests {
|
||||
.try_recover()?],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
};
|
||||
|
||||
|
||||
@@ -304,37 +304,24 @@ mod tests {
|
||||
vec![blocks[0].clone(), blocks[1].clone()],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
};
|
||||
let reverted_notification = ExExNotification::ChainReverted {
|
||||
old: Arc::new(Chain::new(
|
||||
vec![blocks[1].clone()],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
old: Arc::new(Chain::new(vec![blocks[1].clone()], Default::default(), BTreeMap::new())),
|
||||
};
|
||||
let committed_notification_2 = ExExNotification::ChainCommitted {
|
||||
new: Arc::new(Chain::new(
|
||||
vec![block_1_reorged.clone(), blocks[2].clone()],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
};
|
||||
let reorged_notification = ExExNotification::ChainReorged {
|
||||
old: Arc::new(Chain::new(
|
||||
vec![blocks[2].clone()],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
old: Arc::new(Chain::new(vec![blocks[2].clone()], Default::default(), BTreeMap::new())),
|
||||
new: Arc::new(Chain::new(
|
||||
vec![block_2_reorged.clone(), blocks[3].clone()],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
};
|
||||
|
||||
|
||||
@@ -189,7 +189,7 @@ mod tests {
|
||||
use reth_testing_utils::generators::{self, random_block};
|
||||
use reth_trie_common::{
|
||||
updates::{StorageTrieUpdates, TrieUpdates},
|
||||
BranchNodeCompact, HashedPostState, HashedStorage, Nibbles,
|
||||
BranchNodeCompact, HashedPostState, HashedStorage, LazyTrieData, Nibbles,
|
||||
};
|
||||
use std::{collections::BTreeMap, fs::File, sync::Arc};
|
||||
|
||||
@@ -241,18 +241,8 @@ mod tests {
|
||||
let new_block = random_block(&mut rng, 0, Default::default()).try_recover()?;
|
||||
|
||||
let notification = ExExNotification::ChainReorged {
|
||||
new: Arc::new(Chain::new(
|
||||
vec![new_block],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
old: Arc::new(Chain::new(
|
||||
vec![old_block],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
new: Arc::new(Chain::new(vec![new_block], Default::default(), BTreeMap::new())),
|
||||
old: Arc::new(Chain::new(vec![old_block], Default::default(), BTreeMap::new())),
|
||||
};
|
||||
|
||||
// Do a round trip serialization and deserialization
|
||||
@@ -346,13 +336,17 @@ mod tests {
|
||||
)]),
|
||||
};
|
||||
|
||||
let trie_data = LazyTrieData::ready(
|
||||
Arc::new(hashed_state.into_sorted()),
|
||||
Arc::new(trie_updates.into_sorted()),
|
||||
);
|
||||
|
||||
let notification: ExExNotification<reth_ethereum_primitives::EthPrimitives> =
|
||||
ExExNotification::ChainCommitted {
|
||||
new: Arc::new(Chain::new(
|
||||
vec![block],
|
||||
Default::default(),
|
||||
BTreeMap::from([(block_number, Arc::new(trie_updates.into_sorted()))]),
|
||||
BTreeMap::from([(block_number, Arc::new(hashed_state.into_sorted()))]),
|
||||
BTreeMap::from([(block_number, trie_data)]),
|
||||
)),
|
||||
};
|
||||
Ok(notification)
|
||||
|
||||
@@ -223,14 +223,12 @@ pub(super) mod serde_bincode_compat {
|
||||
.unwrap()],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
new: Arc::new(Chain::new(
|
||||
vec![RecoveredBlock::arbitrary(&mut arbitrary::Unstructured::new(&bytes))
|
||||
.unwrap()],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
},
|
||||
};
|
||||
|
||||
@@ -106,7 +106,7 @@ impl BanList {
|
||||
self.banned_ips.contains_key(ip)
|
||||
}
|
||||
|
||||
/// checks the ban list to see if it contains the given ip
|
||||
/// checks the ban list to see if it contains the given peer
|
||||
#[inline]
|
||||
pub fn is_banned_peer(&self, peer_id: &PeerId) -> bool {
|
||||
self.banned_peers.contains_key(peer_id)
|
||||
@@ -117,7 +117,7 @@ impl BanList {
|
||||
self.banned_ips.remove(ip);
|
||||
}
|
||||
|
||||
/// Unbans the ip address
|
||||
/// Unbans the peer
|
||||
pub fn unban_peer(&mut self, peer_id: &PeerId) {
|
||||
self.banned_peers.remove(peer_id);
|
||||
}
|
||||
|
||||
@@ -131,6 +131,8 @@ pub struct TransactionsManagerMetrics {
|
||||
/// capacity. Note, this is not a limit to the number of inflight requests, but a health
|
||||
/// measure.
|
||||
pub(crate) capacity_pending_pool_imports: Counter,
|
||||
/// Total number of transactions ignored because pending pool imports are at capacity.
|
||||
pub(crate) skipped_transactions_pending_pool_imports_at_capacity: Counter,
|
||||
/// The time it took to prepare transactions for import. This is mostly sender recovery.
|
||||
pub(crate) pool_import_prepare_duration: Histogram,
|
||||
|
||||
|
||||
@@ -429,11 +429,22 @@ impl<Pool: TransactionPool, N: NetworkPrimitives> TransactionsManager<Pool, N> {
|
||||
/// Returns `true` if [`TransactionsManager`] has capacity to request pending hashes. Returns
|
||||
/// `false` if [`TransactionsManager`] is operating close to full capacity.
|
||||
fn has_capacity_for_fetching_pending_hashes(&self) -> bool {
|
||||
self.pending_pool_imports_info
|
||||
.has_capacity(self.pending_pool_imports_info.max_pending_pool_imports) &&
|
||||
self.has_capacity_for_pending_pool_imports() &&
|
||||
self.transaction_fetcher.has_capacity_for_fetching_pending_hashes()
|
||||
}
|
||||
|
||||
/// Returns `true` if [`TransactionsManager`] has capacity for more pending pool imports.
|
||||
fn has_capacity_for_pending_pool_imports(&self) -> bool {
|
||||
self.remaining_pool_import_capacity() > 0
|
||||
}
|
||||
|
||||
/// Returns the remaining capacity for pending pool imports.
|
||||
fn remaining_pool_import_capacity(&self) -> usize {
|
||||
self.pending_pool_imports_info.max_pending_pool_imports.saturating_sub(
|
||||
self.pending_pool_imports_info.pending_pool_imports.load(Ordering::Relaxed),
|
||||
)
|
||||
}
|
||||
|
||||
fn report_peer_bad_transactions(&self, peer_id: PeerId) {
|
||||
self.report_peer(peer_id, ReputationChangeKind::BadTransactions);
|
||||
self.metrics.reported_bad_transactions.increment(1);
|
||||
@@ -1285,6 +1296,7 @@ where
|
||||
trace!(target: "net::tx", peer_id=format!("{peer_id:#}"), policy=?self.config.ingress_policy, "Ignoring full transactions from peer blocked by ingress policy");
|
||||
return;
|
||||
}
|
||||
|
||||
// ensure we didn't receive any blob transactions as these are disallowed to be
|
||||
// broadcasted in full
|
||||
|
||||
@@ -1335,7 +1347,13 @@ where
|
||||
return
|
||||
}
|
||||
|
||||
// Early return if we don't have capacity for any imports
|
||||
if !self.has_capacity_for_pending_pool_imports() {
|
||||
return
|
||||
}
|
||||
|
||||
let Some(peer) = self.peers.get_mut(&peer_id) else { return };
|
||||
let client_version = peer.client_version.clone();
|
||||
let mut transactions = transactions.0;
|
||||
|
||||
let start = Instant::now();
|
||||
@@ -1378,7 +1396,7 @@ where
|
||||
trace!(target: "net::tx",
|
||||
peer_id=format!("{peer_id:#}"),
|
||||
hash=%tx.tx_hash(),
|
||||
client_version=%peer.client_version,
|
||||
%client_version,
|
||||
"received a known bad transaction from peer"
|
||||
);
|
||||
has_bad_transactions = true;
|
||||
@@ -1387,6 +1405,18 @@ where
|
||||
true
|
||||
});
|
||||
|
||||
// Truncate to remaining capacity before recovery to avoid wasting CPU on transactions
|
||||
// that won't be imported anyway.
|
||||
let capacity = self.remaining_pool_import_capacity();
|
||||
if transactions.len() > capacity {
|
||||
let skipped = transactions.len() - capacity;
|
||||
transactions.truncate(capacity);
|
||||
self.metrics
|
||||
.skipped_transactions_pending_pool_imports_at_capacity
|
||||
.increment(skipped as u64);
|
||||
trace!(target: "net::tx", skipped, capacity, "Truncated transactions batch to capacity");
|
||||
}
|
||||
|
||||
let txs_len = transactions.len();
|
||||
|
||||
let new_txs = transactions
|
||||
@@ -1397,7 +1427,7 @@ where
|
||||
trace!(target: "net::tx",
|
||||
peer_id=format!("{peer_id:#}"),
|
||||
hash=%badtx.tx_hash(),
|
||||
client_version=%peer.client_version,
|
||||
client_version=%client_version,
|
||||
"failed ecrecovery for transaction"
|
||||
);
|
||||
None
|
||||
@@ -1448,7 +1478,7 @@ where
|
||||
self.metrics
|
||||
.occurrences_of_transaction_already_seen_by_peer
|
||||
.increment(num_already_seen_by_peer);
|
||||
trace!(target: "net::tx", num_txs=%num_already_seen_by_peer, ?peer_id, client=?peer.client_version, "Peer sent already seen transactions");
|
||||
trace!(target: "net::tx", num_txs=%num_already_seen_by_peer, ?peer_id, client=%client_version, "Peer sent already seen transactions");
|
||||
}
|
||||
|
||||
if has_bad_transactions {
|
||||
|
||||
@@ -4,7 +4,7 @@ use crate::{BuilderContext, FullNodeTypes};
|
||||
use alloy_primitives::Address;
|
||||
use reth_chain_state::CanonStateSubscriptions;
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_node_api::{NodeTypes, TxTy};
|
||||
use reth_node_api::{BlockTy, NodeTypes, TxTy};
|
||||
use reth_transaction_pool::{
|
||||
blobstore::DiskFileBlobStore, BlobStore, CoinbaseTipOrdering, PoolConfig, PoolTransaction,
|
||||
SubPoolLimit, TransactionPool, TransactionValidationTaskExecutor, TransactionValidator,
|
||||
@@ -129,7 +129,7 @@ impl<'a, Node: FullNodeTypes, V> TxPoolBuilder<'a, Node, V> {
|
||||
impl<'a, Node, V> TxPoolBuilder<'a, Node, TransactionValidationTaskExecutor<V>>
|
||||
where
|
||||
Node: FullNodeTypes<Types: NodeTypes<ChainSpec: EthereumHardforks>>,
|
||||
V: TransactionValidator + 'static,
|
||||
V: TransactionValidator<Block = BlockTy<Node::Types>> + 'static,
|
||||
V::Transaction:
|
||||
PoolTransaction<Consensus = TxTy<Node::Types>> + reth_transaction_pool::EthPoolTransaction,
|
||||
{
|
||||
@@ -248,7 +248,7 @@ fn spawn_pool_maintenance_task<Node, Pool>(
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
Node: FullNodeTypes<Types: NodeTypes<ChainSpec: EthereumHardforks>>,
|
||||
Pool: reth_transaction_pool::TransactionPoolExt + Clone + 'static,
|
||||
Pool: reth_transaction_pool::TransactionPoolExt<Block = BlockTy<Node::Types>> + Clone + 'static,
|
||||
Pool::Transaction: PoolTransaction<Consensus = TxTy<Node::Types>>,
|
||||
{
|
||||
let chain_events = ctx.provider().canonical_state_stream();
|
||||
@@ -280,7 +280,7 @@ pub fn spawn_maintenance_tasks<Node, Pool>(
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
Node: FullNodeTypes<Types: NodeTypes<ChainSpec: EthereumHardforks>>,
|
||||
Pool: reth_transaction_pool::TransactionPoolExt + Clone + 'static,
|
||||
Pool: reth_transaction_pool::TransactionPoolExt<Block = BlockTy<Node::Types>> + Clone + 'static,
|
||||
Pool::Transaction: PoolTransaction<Consensus = TxTy<Node::Types>>,
|
||||
{
|
||||
spawn_local_backup_task(ctx, pool.clone())?;
|
||||
|
||||
@@ -676,19 +676,13 @@ where
|
||||
|
||||
/// Convenience function to [`Self::init_genesis`]
|
||||
pub fn with_genesis(self) -> Result<Self, InitStorageError> {
|
||||
init_genesis_with_settings(
|
||||
self.provider_factory(),
|
||||
self.node_config().static_files.to_settings(),
|
||||
)?;
|
||||
init_genesis_with_settings(self.provider_factory(), self.node_config().storage_settings())?;
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
/// Write the genesis block and state if it has not already been written
|
||||
pub fn init_genesis(&self) -> Result<B256, InitStorageError> {
|
||||
init_genesis_with_settings(
|
||||
self.provider_factory(),
|
||||
self.node_config().static_files.to_settings(),
|
||||
)
|
||||
init_genesis_with_settings(self.provider_factory(), self.node_config().storage_settings())
|
||||
}
|
||||
|
||||
/// Creates a new `WithMeteredProvider` container and attaches it to the
|
||||
@@ -1283,6 +1277,10 @@ pub fn metrics_hooks<N: NodeTypesWithDB>(provider_factory: &ProviderFactory<N>)
|
||||
})
|
||||
}
|
||||
})
|
||||
.with_hook({
|
||||
let rocksdb = provider_factory.rocksdb_provider();
|
||||
move || throttle!(Duration::from_secs(5 * 60), || rocksdb.report_metrics())
|
||||
})
|
||||
.build()
|
||||
}
|
||||
|
||||
|
||||
@@ -4,10 +4,13 @@ use alloy_consensus::transaction::Either;
|
||||
use alloy_provider::network::AnyNetwork;
|
||||
use jsonrpsee::core::{DeserializeOwned, Serialize};
|
||||
use reth_chainspec::EthChainSpec;
|
||||
use reth_consensus_debug_client::{DebugConsensusClient, EtherscanBlockProvider, RpcBlockProvider};
|
||||
use reth_consensus_debug_client::{
|
||||
BlockProvider, DebugConsensusClient, EtherscanBlockProvider, RpcBlockProvider,
|
||||
};
|
||||
use reth_engine_local::LocalMiner;
|
||||
use reth_node_api::{
|
||||
BlockTy, FullNodeComponents, HeaderTy, PayloadAttrTy, PayloadAttributesBuilder, PayloadTypes,
|
||||
BlockTy, FullNodeComponents, FullNodeTypes, HeaderTy, PayloadAttrTy, PayloadAttributesBuilder,
|
||||
PayloadTypes,
|
||||
};
|
||||
use std::{
|
||||
future::{Future, IntoFuture},
|
||||
@@ -109,9 +112,16 @@ impl<L> DebugNodeLauncher<L> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Type alias for the default debug block provider. We use etherscan provider to satisfy the
|
||||
/// bounds.
|
||||
pub type DefaultDebugBlockProvider<N> = EtherscanBlockProvider<
|
||||
<<N as FullNodeTypes>::Types as DebugNode<N>>::RpcBlock,
|
||||
BlockTy<<N as FullNodeTypes>::Types>,
|
||||
>;
|
||||
|
||||
/// Future for the [`DebugNodeLauncher`].
|
||||
#[expect(missing_debug_implementations, clippy::type_complexity)]
|
||||
pub struct DebugNodeLauncherFuture<L, Target, N>
|
||||
pub struct DebugNodeLauncherFuture<L, Target, N, B = DefaultDebugBlockProvider<N>>
|
||||
where
|
||||
N: FullNodeComponents<Types: DebugNode<N>>,
|
||||
{
|
||||
@@ -121,14 +131,17 @@ where
|
||||
Option<Box<dyn PayloadAttributesBuilder<PayloadAttrTy<N::Types>, HeaderTy<N::Types>>>>,
|
||||
map_attributes:
|
||||
Option<Box<dyn Fn(PayloadAttrTy<N::Types>) -> PayloadAttrTy<N::Types> + Send + Sync>>,
|
||||
debug_block_provider: Option<B>,
|
||||
}
|
||||
|
||||
impl<L, Target, N, AddOns> DebugNodeLauncherFuture<L, Target, N>
|
||||
impl<L, Target, N, AddOns, B> DebugNodeLauncherFuture<L, Target, N, B>
|
||||
where
|
||||
N: FullNodeComponents<Types: DebugNode<N>>,
|
||||
AddOns: RethRpcAddOns<N>,
|
||||
L: LaunchNode<Target, Node = NodeHandle<N, AddOns>>,
|
||||
B: BlockProvider<Block = BlockTy<N::Types>> + Clone,
|
||||
{
|
||||
/// Sets a custom payload attributes builder for local mining in dev mode.
|
||||
pub fn with_payload_attributes_builder(
|
||||
self,
|
||||
builder: impl PayloadAttributesBuilder<PayloadAttrTy<N::Types>, HeaderTy<N::Types>>,
|
||||
@@ -138,9 +151,11 @@ where
|
||||
target: self.target,
|
||||
local_payload_attributes_builder: Some(Box::new(builder)),
|
||||
map_attributes: None,
|
||||
debug_block_provider: self.debug_block_provider,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets a function to map payload attributes before building.
|
||||
pub fn map_debug_payload_attributes(
|
||||
self,
|
||||
f: impl Fn(PayloadAttrTy<N::Types>) -> PayloadAttrTy<N::Types> + Send + Sync + 'static,
|
||||
@@ -150,16 +165,58 @@ where
|
||||
target: self.target,
|
||||
local_payload_attributes_builder: None,
|
||||
map_attributes: Some(Box::new(f)),
|
||||
debug_block_provider: self.debug_block_provider,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets a custom block provider for the debug consensus client.
|
||||
///
|
||||
/// When set, this provider will be used instead of creating an `EtherscanBlockProvider`
|
||||
/// or `RpcBlockProvider` from CLI arguments.
|
||||
pub fn with_debug_block_provider<B2>(
|
||||
self,
|
||||
provider: B2,
|
||||
) -> DebugNodeLauncherFuture<L, Target, N, B2>
|
||||
where
|
||||
B2: BlockProvider<Block = BlockTy<N::Types>> + Clone,
|
||||
{
|
||||
DebugNodeLauncherFuture {
|
||||
inner: self.inner,
|
||||
target: self.target,
|
||||
local_payload_attributes_builder: self.local_payload_attributes_builder,
|
||||
map_attributes: self.map_attributes,
|
||||
debug_block_provider: Some(provider),
|
||||
}
|
||||
}
|
||||
|
||||
async fn launch_node(self) -> eyre::Result<NodeHandle<N, AddOns>> {
|
||||
let Self { inner, target, local_payload_attributes_builder, map_attributes } = self;
|
||||
let Self {
|
||||
inner,
|
||||
target,
|
||||
local_payload_attributes_builder,
|
||||
map_attributes,
|
||||
debug_block_provider,
|
||||
} = self;
|
||||
|
||||
let handle = inner.launch_node(target).await?;
|
||||
|
||||
let config = &handle.node.config;
|
||||
if let Some(url) = config.debug.rpc_consensus_url.clone() {
|
||||
|
||||
if let Some(provider) = debug_block_provider {
|
||||
info!(target: "reth::cli", "Using custom debug block provider");
|
||||
|
||||
let rpc_consensus_client = DebugConsensusClient::new(
|
||||
handle.node.add_ons_handle.beacon_engine_handle.clone(),
|
||||
Arc::new(provider),
|
||||
);
|
||||
|
||||
handle
|
||||
.node
|
||||
.task_executor
|
||||
.spawn_critical("custom debug block provider consensus client", async move {
|
||||
rpc_consensus_client.run().await
|
||||
});
|
||||
} else if let Some(url) = config.debug.rpc_consensus_url.clone() {
|
||||
info!(target: "reth::cli", "Using RPC consensus client: {}", url);
|
||||
|
||||
let block_provider =
|
||||
@@ -180,14 +237,11 @@ where
|
||||
handle.node.task_executor.spawn_critical("rpc-ws consensus client", async move {
|
||||
rpc_consensus_client.run().await
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(maybe_custom_etherscan_url) = config.debug.etherscan.clone() {
|
||||
} else if let Some(maybe_custom_etherscan_url) = config.debug.etherscan.clone() {
|
||||
info!(target: "reth::cli", "Using etherscan as consensus client");
|
||||
|
||||
let chain = config.chain.chain();
|
||||
let etherscan_url = maybe_custom_etherscan_url.map(Ok).unwrap_or_else(|| {
|
||||
// If URL isn't provided, use default Etherscan URL for the chain if it is known
|
||||
chain
|
||||
.etherscan_urls()
|
||||
.map(|urls| urls.0.to_string())
|
||||
@@ -252,12 +306,13 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<L, Target, N, AddOns> IntoFuture for DebugNodeLauncherFuture<L, Target, N>
|
||||
impl<L, Target, N, AddOns, B> IntoFuture for DebugNodeLauncherFuture<L, Target, N, B>
|
||||
where
|
||||
Target: Send + 'static,
|
||||
N: FullNodeComponents<Types: DebugNode<N>>,
|
||||
AddOns: RethRpcAddOns<N> + 'static,
|
||||
L: LaunchNode<Target, Node = NodeHandle<N, AddOns>> + 'static,
|
||||
B: BlockProvider<Block = BlockTy<N::Types>> + Clone + 'static,
|
||||
{
|
||||
type Output = eyre::Result<NodeHandle<N, AddOns>>;
|
||||
type IntoFuture = Pin<Box<dyn Future<Output = eyre::Result<NodeHandle<N, AddOns>>> + Send>>;
|
||||
@@ -273,6 +328,7 @@ where
|
||||
N: FullNodeComponents<Types: DebugNode<N>>,
|
||||
AddOns: RethRpcAddOns<N> + 'static,
|
||||
L: LaunchNode<Target, Node = NodeHandle<N, AddOns>> + 'static,
|
||||
DefaultDebugBlockProvider<N>: BlockProvider<Block = BlockTy<N::Types>> + Clone,
|
||||
{
|
||||
type Node = NodeHandle<N, AddOns>;
|
||||
type Future = DebugNodeLauncherFuture<L, Target, N>;
|
||||
@@ -283,6 +339,7 @@ where
|
||||
target,
|
||||
local_payload_attributes_builder: None,
|
||||
map_attributes: None,
|
||||
debug_block_provider: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ use reth_node_core::{
|
||||
use reth_node_events::node;
|
||||
use reth_provider::{
|
||||
providers::{BlockchainProvider, NodeTypesForProvider},
|
||||
BlockNumReader, MetadataProvider,
|
||||
BlockNumReader, StorageSettingsCache,
|
||||
};
|
||||
use reth_tasks::TaskExecutor;
|
||||
use reth_tokio_util::EventSender;
|
||||
@@ -41,7 +41,6 @@ use reth_trie_db::ChangesetCache;
|
||||
use std::{future::Future, pin::Pin, sync::Arc};
|
||||
use tokio::sync::{mpsc::unbounded_channel, oneshot};
|
||||
use tokio_stream::wrappers::UnboundedReceiverStream;
|
||||
use tracing::warn;
|
||||
|
||||
/// The engine node launcher.
|
||||
#[derive(Debug)]
|
||||
@@ -104,24 +103,8 @@ impl EngineNodeLauncher {
|
||||
.with_adjusted_configs()
|
||||
// Create the provider factory with changeset cache
|
||||
.with_provider_factory::<_, <CB::Components as NodeComponents<T>>::Evm>(changeset_cache.clone()).await?
|
||||
.inspect(|ctx| {
|
||||
.inspect(|_| {
|
||||
info!(target: "reth::cli", "Database opened");
|
||||
match ctx.provider_factory().storage_settings() {
|
||||
Ok(settings) => {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
?settings,
|
||||
"Storage settings"
|
||||
);
|
||||
},
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "reth::cli",
|
||||
?err,
|
||||
"Failed to get storage settings"
|
||||
);
|
||||
},
|
||||
}
|
||||
})
|
||||
.with_prometheus_server().await?
|
||||
.inspect(|this| {
|
||||
@@ -130,6 +113,8 @@ impl EngineNodeLauncher {
|
||||
.with_genesis()?
|
||||
.inspect(|this: &LaunchContextWith<Attached<WithConfigs<<T::Types as NodeTypes>::ChainSpec>, _>>| {
|
||||
info!(target: "reth::cli", "\n{}", this.chain_spec().display_hardforks());
|
||||
let settings = this.provider_factory().cached_storage_settings();
|
||||
info!(target: "reth::cli", ?settings, "Loaded storage settings");
|
||||
})
|
||||
.with_metrics_task()
|
||||
// passing FullNodeTypes as type parameter here so that we can build
|
||||
|
||||
@@ -31,7 +31,7 @@ pub use builder::{add_ons::AddOns, *};
|
||||
|
||||
mod launch;
|
||||
pub use launch::{
|
||||
debug::{DebugNode, DebugNodeLauncher},
|
||||
debug::{DebugNode, DebugNodeLauncher, DebugNodeLauncherFuture, DefaultDebugBlockProvider},
|
||||
engine::EngineNodeLauncher,
|
||||
*,
|
||||
};
|
||||
|
||||
@@ -19,7 +19,6 @@ reth-cli-util.workspace = true
|
||||
reth-db = { workspace = true, features = ["mdbx"] }
|
||||
reth-storage-errors.workspace = true
|
||||
reth-storage-api = { workspace = true, features = ["std", "db-api"] }
|
||||
reth-provider.workspace = true
|
||||
reth-network = { workspace = true, features = ["serde"] }
|
||||
reth-network-p2p.workspace = true
|
||||
reth-rpc-eth-types.workspace = true
|
||||
@@ -92,7 +91,7 @@ min-debug-logs = ["tracing/release_max_level_debug"]
|
||||
min-trace-logs = ["tracing/release_max_level_trace"]
|
||||
|
||||
# Marker feature for edge/unstable builds - captured by vergen in build.rs
|
||||
edge = []
|
||||
edge = ["reth-storage-api/edge"]
|
||||
|
||||
[build-dependencies]
|
||||
vergen = { workspace = true, features = ["build", "cargo", "emit_and_set"] }
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user