mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
333 Commits
fix/mdbx-c
...
pr-21596
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3c0b81dc71 | ||
|
|
04cf37ad4a | ||
|
|
9edc72e278 | ||
|
|
20a182abc3 | ||
|
|
1f951685cb | ||
|
|
33e695349e | ||
|
|
5e2b06ba0c | ||
|
|
f0a9ac0ff6 | ||
|
|
3ad2425bd2 | ||
|
|
c263eac79a | ||
|
|
8ffcde8d1a | ||
|
|
262149069a | ||
|
|
4a5d43dc46 | ||
|
|
17d3e80953 | ||
|
|
5282231574 | ||
|
|
ba9bc425d9 | ||
|
|
4e2fa3db44 | ||
|
|
34811b908c | ||
|
|
781128eece | ||
|
|
435d915422 | ||
|
|
3ec065295e | ||
|
|
e1bc6d0f08 | ||
|
|
29072639d6 | ||
|
|
f90b5c8a7f | ||
|
|
d4fa6806b7 | ||
|
|
63742ab4ae | ||
|
|
08122bc1ea | ||
|
|
83afaf1aa7 | ||
|
|
d72300c685 | ||
|
|
faf64c712e | ||
|
|
b3d532ce9d | ||
|
|
9d064be77e | ||
|
|
e3c256340e | ||
|
|
d0df549ddb | ||
|
|
7ccb43ea13 | ||
|
|
20f48b1e50 | ||
|
|
0470c65e6c | ||
|
|
9de1f0905e | ||
|
|
327a1a6681 | ||
|
|
b8f27b73ad | ||
|
|
7ec5ff6483 | ||
|
|
f98af4ad9f | ||
|
|
d8e912f66b | ||
|
|
0572c4e0ca | ||
|
|
67a7a1c2d1 | ||
|
|
2b1833576b | ||
|
|
5592c362d4 | ||
|
|
6beec25f43 | ||
|
|
19bf580f93 | ||
|
|
796ba6d5dc | ||
|
|
5307dfc22b | ||
|
|
f380ed1581 | ||
|
|
f7313c755c | ||
|
|
3bc2191590 | ||
|
|
320f2a6015 | ||
|
|
70bfdafd26 | ||
|
|
e9fe0283a9 | ||
|
|
92b8857625 | ||
|
|
2d71243cf6 | ||
|
|
732bf712aa | ||
|
|
0901c2ca8b | ||
|
|
2352158b3d | ||
|
|
1a98605ce6 | ||
|
|
8d37f76d23 | ||
|
|
2d9cf4c989 | ||
|
|
f5ca71d2fb | ||
|
|
8d58c98034 | ||
|
|
50e0591540 | ||
|
|
013dfdf8c8 | ||
|
|
effa0ab4c7 | ||
|
|
543a85e9f3 | ||
|
|
88eb0beeb2 | ||
|
|
747c0169a7 | ||
|
|
497985ca86 | ||
|
|
48a999a81b | ||
|
|
d53858b3e2 | ||
|
|
6aa91b0020 | ||
|
|
e0a0a0d5fb | ||
|
|
231292b58e | ||
|
|
42765890b5 | ||
|
|
8417ddc0e8 | ||
|
|
1ca62d0696 | ||
|
|
928bf37297 | ||
|
|
aa5b12af44 | ||
|
|
f12acf17e6 | ||
|
|
2e05cec84b | ||
|
|
9eaa5a6303 | ||
|
|
ba8c8354e5 | ||
|
|
af3601c65d | ||
|
|
bff11ab663 | ||
|
|
08cd1cbda6 | ||
|
|
e4e05e9ef9 | ||
|
|
c8245594bc | ||
|
|
ed40ce8c4c | ||
|
|
1e734936d8 | ||
|
|
11d9f38077 | ||
|
|
226ce14ca1 | ||
|
|
a6e1dea2d7 | ||
|
|
71ed68e944 | ||
|
|
adecbd7814 | ||
|
|
26a37f3c00 | ||
|
|
0bfa7fa5fa | ||
|
|
18bec10a0b | ||
|
|
1e33821e19 | ||
|
|
da92733be8 | ||
|
|
c41c8e6cae | ||
|
|
1ccc174e7b | ||
|
|
f1459fcf91 | ||
|
|
94235d64a8 | ||
|
|
7fe60017cf | ||
|
|
f9ec2fafa0 | ||
|
|
768a687189 | ||
|
|
b87cde5479 | ||
|
|
ab685579f0 | ||
|
|
c7faafd183 | ||
|
|
935a2cc056 | ||
|
|
507cf58db0 | ||
|
|
6cfd369d17 | ||
|
|
934f462d01 | ||
|
|
d4f28b02ff | ||
|
|
963bfeeeed | ||
|
|
adbe6d9da0 | ||
|
|
6d19c0ed8e | ||
|
|
4baf2baec4 | ||
|
|
0b5f79e8c9 | ||
|
|
afe164baca | ||
|
|
31fdbe914c | ||
|
|
6870747246 | ||
|
|
0ad8c772e1 | ||
|
|
5440d0d89a | ||
|
|
0eea4d76e9 | ||
|
|
8a1702cd74 | ||
|
|
7feb56d5f6 | ||
|
|
0aa922c4e8 | ||
|
|
ccff9a08f0 | ||
|
|
eb788cc7cf | ||
|
|
fb05a0654f | ||
|
|
d5a36dcc00 | ||
|
|
ffbef9e3cd | ||
|
|
820c112e8e | ||
|
|
9285f7eafc | ||
|
|
9a4c6d8a11 | ||
|
|
963c26550a | ||
|
|
3648483512 | ||
|
|
ab418642b4 | ||
|
|
decb56fae1 | ||
|
|
ee1ec8f9f0 | ||
|
|
d7bf87da52 | ||
|
|
dd0c6d279f | ||
|
|
c137ed836f | ||
|
|
a543752f7d | ||
|
|
b814893221 | ||
|
|
fcef82261d | ||
|
|
d3846d98a9 | ||
|
|
1f536cce65 | ||
|
|
0ddaf1b26c | ||
|
|
830cd5e355 | ||
|
|
f77d7d5983 | ||
|
|
a2237c534e | ||
|
|
1bd8fab887 | ||
|
|
22a68756c7 | ||
|
|
d99c0ffd62 | ||
|
|
ad476e2b5c | ||
|
|
6df249c1f1 | ||
|
|
5a076df09a | ||
|
|
f07629eac0 | ||
|
|
f643e93c35 | ||
|
|
653362a436 | ||
|
|
a02508600c | ||
|
|
937a7f226d | ||
|
|
a0df561117 | ||
|
|
be5a4ac7a6 | ||
|
|
0c854b6f14 | ||
|
|
28a31cd579 | ||
|
|
da12451c9c | ||
|
|
247ce3c4e9 | ||
|
|
bf43ebaa29 | ||
|
|
a01ecce73f | ||
|
|
3e55c6ca6e | ||
|
|
2ac7d719f3 | ||
|
|
965705ff88 | ||
|
|
ebe2ca1366 | ||
|
|
cc242f83fd | ||
|
|
12cf3d685b | ||
|
|
ad5b533ad1 | ||
|
|
118f15f345 | ||
|
|
97481f69e5 | ||
|
|
f692ac7d1e | ||
|
|
4b1c341ced | ||
|
|
865f8f8951 | ||
|
|
492fc20fd1 | ||
|
|
ad9886abb8 | ||
|
|
5c3e45cd6b | ||
|
|
68fdba32d2 | ||
|
|
8f6a0a2992 | ||
|
|
ec9c7f8d3e | ||
|
|
dbdaf068f0 | ||
|
|
055bf63ee9 | ||
|
|
2305c3ebeb | ||
|
|
eb55c3c3da | ||
|
|
72e1467ba3 | ||
|
|
74edce0089 | ||
|
|
8c645d5762 | ||
|
|
b7d2ee2566 | ||
|
|
7609deddda | ||
|
|
ec50fd40b3 | ||
|
|
624ddc5779 | ||
|
|
dd72cfe23e | ||
|
|
ff8ac97e33 | ||
|
|
0974485863 | ||
|
|
274394e777 | ||
|
|
1954c91a60 | ||
|
|
9cf82c8403 | ||
|
|
f85fcba872 | ||
|
|
ebaa4bda3a | ||
|
|
04d4c9a02f | ||
|
|
3065a328f9 | ||
|
|
43a84f1231 | ||
|
|
5a5c21cc1b | ||
|
|
8a8a9126d6 | ||
|
|
6f73c2447d | ||
|
|
2cae438642 | ||
|
|
37b5db0d47 | ||
|
|
238433e146 | ||
|
|
660964a0f5 | ||
|
|
22b465dd64 | ||
|
|
3ff575b877 | ||
|
|
d12752dc8a | ||
|
|
869b5d0851 | ||
|
|
78de3d8f61 | ||
|
|
bc79cc44c9 | ||
|
|
ff8f434dcd | ||
|
|
9662dc5271 | ||
|
|
3ba37082dc | ||
|
|
7934294988 | ||
|
|
7371bd3f29 | ||
|
|
80980b8e4d | ||
|
|
2e2cd67663 | ||
|
|
4f009728e2 | ||
|
|
39d5ae73e8 | ||
|
|
5ef200eaad | ||
|
|
d002dacc13 | ||
|
|
bb39cba504 | ||
|
|
bd144a4c42 | ||
|
|
a0845bab18 | ||
|
|
346cc0da71 | ||
|
|
ea3d4663ae | ||
|
|
3667d3b5aa | ||
|
|
7cfb19c98e | ||
|
|
5a38871489 | ||
|
|
c825c8c187 | ||
|
|
8f37cd08fc | ||
|
|
c9dad4765d | ||
|
|
1d55abeef3 | ||
|
|
f7460e219c | ||
|
|
0c66315f20 | ||
|
|
6a2010e595 | ||
|
|
c2435ff6f8 | ||
|
|
52ec8e9491 | ||
|
|
a901d80ee6 | ||
|
|
915164078f | ||
|
|
be3234d848 | ||
|
|
f624372334 | ||
|
|
40bc9d3860 | ||
|
|
1ea574417f | ||
|
|
27e055f790 | ||
|
|
d5dc0b27eb | ||
|
|
c11c13000f | ||
|
|
6bf43ab24a | ||
|
|
574bde0d6f | ||
|
|
79b8ffb828 | ||
|
|
c617d25c36 | ||
|
|
b96a30821f | ||
|
|
012fbf5110 | ||
|
|
d7a5d1f872 | ||
|
|
3a39251f79 | ||
|
|
f6dbf2d82d | ||
|
|
13707faf1a | ||
|
|
6e6415690c | ||
|
|
b81e373d78 | ||
|
|
a164654145 | ||
|
|
905bb95f8b | ||
|
|
13c32625bc | ||
|
|
1be9fab5bf | ||
|
|
80eb0d0fb6 | ||
|
|
5e178f6ac6 | ||
|
|
b4b64096c8 | ||
|
|
e313de818b | ||
|
|
86c414081a | ||
|
|
a74cb9cbc3 | ||
|
|
e25411c32b | ||
|
|
ec3323bba0 | ||
|
|
26cd132631 | ||
|
|
079f59c2be | ||
|
|
e9b079ad62 | ||
|
|
b1f107b171 | ||
|
|
7d0e7e72de | ||
|
|
f012b3391e | ||
|
|
d225fc1d7f | ||
|
|
d469b7f1d0 | ||
|
|
9bcd3712c8 | ||
|
|
b25f32a977 | ||
|
|
905de96944 | ||
|
|
27fbd9a7de | ||
|
|
26a99ac5a3 | ||
|
|
1265a89c21 | ||
|
|
b9ff5941eb | ||
|
|
a75a0a5db7 | ||
|
|
0a4bac77d0 | ||
|
|
1fbd5a95f8 | ||
|
|
1bc07fad8e | ||
|
|
8cb506c4d3 | ||
|
|
15f16a5a2e | ||
|
|
5cf1d2a0b0 | ||
|
|
59fb25d892 | ||
|
|
665a0a8553 | ||
|
|
54735ce0f4 | ||
|
|
a73e73adef | ||
|
|
4f3bd3eac1 | ||
|
|
ae41823be6 | ||
|
|
1fa71f893c | ||
|
|
c6b17848dd | ||
|
|
a5dd7d0106 | ||
|
|
61354e6c21 | ||
|
|
2444533a04 | ||
|
|
8fa01eb62e | ||
|
|
c5e00e4aeb | ||
|
|
98a35cc870 | ||
|
|
46d670eca5 | ||
|
|
25906b7b3e | ||
|
|
1b3d815cb8 | ||
|
|
23f3f8e820 | ||
|
|
2663942b50 |
@@ -12,7 +12,7 @@ workflows:
|
||||
# Check that `A` activates the features of `B`.
|
||||
"propagate-feature",
|
||||
# These are the features to check:
|
||||
"--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,js-tracer,portable,keccak-cache-global",
|
||||
"--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,tracy,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,otlp-logs,js-tracer,portable,keccak-cache-global",
|
||||
# Do not try to add a new section to `[features]` of `A` only because `B` exposes that feature. There are edge-cases where this is still needed, but we can add them manually.
|
||||
"--left-side-feature-missing=ignore",
|
||||
# Ignore the case that `A` it outside of the workspace. Otherwise it will report errors in external dependencies that we have no influence on.
|
||||
|
||||
43
.github/CODEOWNERS
vendored
43
.github/CODEOWNERS
vendored
@@ -1,45 +1,52 @@
|
||||
* @gakonst
|
||||
crates/blockchain-tree-api/ @rakita @mattsse @Rjected
|
||||
crates/blockchain-tree/ @rakita @mattsse @Rjected
|
||||
crates/chain-state/ @fgimenez @mattsse
|
||||
crates/chainspec/ @Rjected @joshieDo @mattsse
|
||||
crates/cli/ @mattsse
|
||||
crates/config/ @shekhirin @mattsse @Rjected
|
||||
crates/consensus/ @mattsse @Rjected
|
||||
crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez
|
||||
crates/engine/ @mattsse @Rjected @fgimenez @mediocregopher @yongkangc
|
||||
crates/era/ @mattsse @RomanHodulak
|
||||
crates/engine/ @mattsse @Rjected @mediocregopher @yongkangc
|
||||
crates/era/ @mattsse
|
||||
crates/era-downloader/ @mattsse
|
||||
crates/era-utils/ @mattsse
|
||||
crates/errors/ @mattsse
|
||||
crates/ethereum-forks/ @mattsse @Rjected
|
||||
crates/ethereum/ @mattsse @Rjected
|
||||
crates/etl/ @joshieDo @shekhirin
|
||||
crates/evm/ @rakita @mattsse @Rjected
|
||||
crates/evm/ @mattsse @Rjected @klkvr
|
||||
crates/exex/ @shekhirin
|
||||
crates/fs-util/ @mattsse
|
||||
crates/metrics/ @mattsse @Rjected
|
||||
crates/net/ @mattsse @Rjected
|
||||
crates/net/downloaders/ @Rjected
|
||||
crates/node/ @mattsse @Rjected @klkvr
|
||||
crates/optimism/ @mattsse @Rjected @fgimenez
|
||||
crates/optimism/ @mattsse @Rjected
|
||||
crates/payload/ @mattsse @Rjected
|
||||
crates/primitives-traits/ @Rjected @RomanHodulak @mattsse @klkvr
|
||||
crates/primitives-traits/ @Rjected @mattsse @klkvr
|
||||
crates/primitives/ @Rjected @mattsse @klkvr
|
||||
crates/prune/ @shekhirin @joshieDo
|
||||
crates/ress @shekhirin @Rjected
|
||||
crates/revm/ @mattsse @rakita
|
||||
crates/rpc/ @mattsse @Rjected @RomanHodulak
|
||||
crates/ress/ @shekhirin @Rjected
|
||||
crates/revm/ @mattsse
|
||||
crates/rpc/ @mattsse @Rjected
|
||||
crates/stages/ @shekhirin @mediocregopher
|
||||
crates/static-file/ @joshieDo @shekhirin
|
||||
crates/stateless/ @mattsse
|
||||
crates/storage/codecs/ @joshieDo
|
||||
crates/storage/db-api/ @joshieDo @rakita
|
||||
crates/storage/db-api/ @joshieDo
|
||||
crates/storage/db-common/ @Rjected
|
||||
crates/storage/db/ @joshieDo @rakita
|
||||
crates/storage/errors/ @rakita
|
||||
crates/storage/libmdbx-rs/ @rakita @shekhirin
|
||||
crates/storage/db/ @joshieDo
|
||||
crates/storage/errors/ @joshieDo
|
||||
crates/storage/libmdbx-rs/ @shekhirin
|
||||
crates/storage/nippy-jar/ @joshieDo @shekhirin
|
||||
crates/storage/provider/ @rakita @joshieDo @shekhirin
|
||||
crates/storage/provider/ @joshieDo @shekhirin @yongkangc
|
||||
crates/storage/storage-api/ @joshieDo
|
||||
crates/tasks/ @mattsse
|
||||
crates/tokio-util/ @fgimenez
|
||||
crates/tokio-util/ @mattsse
|
||||
crates/tracing/ @mattsse @shekhirin
|
||||
crates/tracing-otlp/ @mattsse @Rjected
|
||||
crates/transaction-pool/ @mattsse @yongkangc
|
||||
crates/trie/ @Rjected @shekhirin @mediocregopher
|
||||
crates/trie/ @Rjected @shekhirin @mediocregopher @yongkangc
|
||||
bin/reth/ @mattsse @shekhirin @Rjected
|
||||
bin/reth-bench/ @mattsse @Rjected @shekhirin @yongkangc
|
||||
bin/reth-bench-compare/ @mediocregopher @shekhirin @yongkangc
|
||||
etc/ @Rjected @shekhirin
|
||||
.github/ @gakonst @DaniPopes
|
||||
|
||||
14
.github/scripts/codspeed-build.sh
vendored
14
.github/scripts/codspeed-build.sh
vendored
@@ -1,14 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eo pipefail
|
||||
|
||||
# TODO: Benchmarks run WAY too slow due to excessive amount of iterations.
|
||||
|
||||
cmd=(cargo codspeed build --profile profiling)
|
||||
crates=(
|
||||
-p reth-primitives
|
||||
-p reth-trie
|
||||
-p reth-trie-common
|
||||
-p reth-trie-sparse
|
||||
)
|
||||
|
||||
"${cmd[@]}" --features test-utils "${crates[@]}"
|
||||
@@ -38,6 +38,6 @@ for pid in "${saving_pids[@]}"; do
|
||||
done
|
||||
|
||||
# Make sure we don't rebuild images on the CI jobs
|
||||
git apply ../.github/assets/hive/no_sim_build.diff
|
||||
git apply ../.github/scripts/hive/no_sim_build.diff
|
||||
go build .
|
||||
mv ./hive ../hive_assets/
|
||||
14
.github/workflows/bench.yml
vendored
14
.github/workflows/bench.yml
vendored
@@ -17,6 +17,16 @@ name: bench
|
||||
jobs:
|
||||
codspeed:
|
||||
runs-on: depot-ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
partition: [1, 2]
|
||||
total_partitions: [2]
|
||||
include:
|
||||
- partition: 1
|
||||
crates: "-p reth-primitives -p reth-trie-common -p reth-trie-sparse"
|
||||
- partition: 2
|
||||
crates: "-p reth-trie"
|
||||
name: codspeed (${{ matrix.partition }}/${{ matrix.total_partitions }})
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
@@ -32,10 +42,10 @@ jobs:
|
||||
with:
|
||||
tool: cargo-codspeed
|
||||
- name: Build the benchmark target(s)
|
||||
run: ./.github/scripts/codspeed-build.sh
|
||||
run: cargo codspeed build --profile profiling --features test-utils ${{ matrix.crates }}
|
||||
- name: Run the benchmarks
|
||||
uses: CodSpeedHQ/action@v4
|
||||
with:
|
||||
run: cargo codspeed run --workspace
|
||||
run: cargo codspeed run ${{ matrix.crates }}
|
||||
mode: instrumentation
|
||||
token: ${{ secrets.CODSPEED_TOKEN }}
|
||||
|
||||
65
.github/workflows/check-alloy.yml
vendored
Normal file
65
.github/workflows/check-alloy.yml
vendored
Normal file
@@ -0,0 +1,65 @@
|
||||
# Checks reth compilation against alloy branches to detect breaking changes.
|
||||
# Run on-demand via workflow_dispatch.
|
||||
|
||||
name: Check Alloy Breaking Changes
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
alloy_branch:
|
||||
description: 'Branch/rev for alloy-rs/alloy (leave empty to skip)'
|
||||
required: false
|
||||
type: string
|
||||
alloy_evm_branch:
|
||||
description: 'Branch/rev for alloy-rs/evm (alloy-evm, alloy-op-evm) (leave empty to skip)'
|
||||
required: false
|
||||
type: string
|
||||
op_alloy_branch:
|
||||
description: 'Branch/rev for alloy-rs/op-alloy (leave empty to skip)'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Check compilation with patched alloy
|
||||
runs-on: depot-ubuntu-latest-16
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
|
||||
- name: Apply alloy patches
|
||||
run: |
|
||||
ARGS=""
|
||||
if [ -n "${{ inputs.alloy_branch }}" ]; then
|
||||
ARGS="$ARGS --alloy ${{ inputs.alloy_branch }}"
|
||||
fi
|
||||
if [ -n "${{ inputs.alloy_evm_branch }}" ]; then
|
||||
ARGS="$ARGS --evm ${{ inputs.alloy_evm_branch }}"
|
||||
fi
|
||||
if [ -n "${{ inputs.op_alloy_branch }}" ]; then
|
||||
ARGS="$ARGS --op ${{ inputs.op_alloy_branch }}"
|
||||
fi
|
||||
|
||||
if [ -z "$ARGS" ]; then
|
||||
echo "No branches specified, nothing to patch"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
./scripts/patch-alloy.sh $ARGS
|
||||
|
||||
echo "=== Final patch section ==="
|
||||
tail -50 Cargo.toml
|
||||
|
||||
- name: Check workspace
|
||||
run: cargo clippy --workspace --lib --examples --tests --benches --all-features --locked
|
||||
env:
|
||||
RUSTFLAGS: -D warnings
|
||||
2
.github/workflows/dependencies.yml
vendored
2
.github/workflows/dependencies.yml
vendored
@@ -15,6 +15,6 @@ permissions:
|
||||
|
||||
jobs:
|
||||
update:
|
||||
uses: ithacaxyz/ci/.github/workflows/cargo-update-pr.yml@main
|
||||
uses: tempoxyz/ci/.github/workflows/cargo-update-pr.yml@main
|
||||
secrets:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
54
.github/workflows/docker-git.yml
vendored
54
.github/workflows/docker-git.yml
vendored
@@ -1,54 +0,0 @@
|
||||
# Publishes the Docker image, only to be used with `workflow_dispatch`. The
|
||||
# images from this workflow will be tagged with the git sha of the branch used
|
||||
# and will NOT tag it as `latest`.
|
||||
|
||||
name: docker-git
|
||||
|
||||
on:
|
||||
workflow_dispatch: {}
|
||||
|
||||
env:
|
||||
REPO_NAME: ${{ github.repository_owner }}/reth
|
||||
IMAGE_NAME: ${{ github.repository_owner }}/reth
|
||||
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
|
||||
CARGO_TERM_COLOR: always
|
||||
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
|
||||
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
|
||||
DOCKER_USERNAME: ${{ github.actor }}
|
||||
GIT_SHA: ${{ github.sha }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: build and push
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
build:
|
||||
- name: 'Build and push the git-sha-tagged reth image'
|
||||
command: 'make PROFILE=maxperf GIT_SHA=$GIT_SHA docker-build-push-git-sha'
|
||||
- name: 'Build and push the git-sha-tagged op-reth image'
|
||||
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME GIT_SHA=$GIT_SHA PROFILE=maxperf op-docker-build-push-git-sha'
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Install cross main
|
||||
id: cross_main
|
||||
run: |
|
||||
cargo install cross --git https://github.com/cross-rs/cross
|
||||
- name: Log in to Docker
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
|
||||
- name: Set up Docker builder
|
||||
run: |
|
||||
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
|
||||
docker buildx create --use --name cross-builder
|
||||
- name: Build and push ${{ matrix.build.name }}
|
||||
run: ${{ matrix.build.command }}
|
||||
65
.github/workflows/docker-nightly.yml
vendored
65
.github/workflows/docker-nightly.yml
vendored
@@ -1,65 +0,0 @@
|
||||
# Publishes the nightly Docker image.
|
||||
|
||||
name: docker-nightly
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 1 * * *"
|
||||
env:
|
||||
REPO_NAME: ${{ github.repository_owner }}/reth
|
||||
IMAGE_NAME: ${{ github.repository_owner }}/reth
|
||||
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
|
||||
CARGO_TERM_COLOR: always
|
||||
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
|
||||
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
|
||||
DOCKER_USERNAME: ${{ github.actor }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: build and push
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
build:
|
||||
- name: 'Build and push the nightly reth image'
|
||||
command: 'make PROFILE=maxperf docker-build-push-nightly'
|
||||
- name: 'Build and push the nightly edge profiling reth image'
|
||||
command: 'make PROFILE=profiling docker-build-push-nightly-edge-profiling'
|
||||
- name: 'Build and push the nightly profiling reth image'
|
||||
command: 'make PROFILE=profiling docker-build-push-nightly-profiling'
|
||||
- name: 'Build and push the nightly op-reth image'
|
||||
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push-nightly'
|
||||
- name: 'Build and push the nightly edge profiling op-reth image'
|
||||
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=profiling op-docker-build-push-nightly-edge-profiling'
|
||||
- name: 'Build and push the nightly profiling op-reth image'
|
||||
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=profiling op-docker-build-push-nightly-profiling'
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Remove bloatware
|
||||
uses: laverdet/remove-bloatware@v1.0.0
|
||||
with:
|
||||
docker: true
|
||||
lang: rust
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Install cross main
|
||||
id: cross_main
|
||||
run: |
|
||||
cargo install cross --git https://github.com/cross-rs/cross
|
||||
- name: Log in to Docker
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
|
||||
- name: Set up Docker builder
|
||||
run: |
|
||||
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
|
||||
docker buildx create --use --name cross-builder
|
||||
- name: Build and push ${{ matrix.build.name }}
|
||||
run: ${{ matrix.build.command }}
|
||||
155
.github/workflows/docker.yml
vendored
155
.github/workflows/docker.yml
vendored
@@ -1,4 +1,9 @@
|
||||
# Publishes the Docker image.
|
||||
# Publishes Docker images.
|
||||
#
|
||||
# Triggers:
|
||||
# - Push tag v*: builds release (RC or latest)
|
||||
# - Schedule: builds nightly + profiling
|
||||
# - Manual: builds git-sha or nightly
|
||||
|
||||
name: docker
|
||||
|
||||
@@ -6,84 +11,94 @@ on:
|
||||
push:
|
||||
tags:
|
||||
- v*
|
||||
|
||||
env:
|
||||
IMAGE_NAME: ${{ github.repository_owner }}/reth
|
||||
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
|
||||
CARGO_TERM_COLOR: always
|
||||
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
|
||||
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
|
||||
DOCKER_USERNAME: ${{ github.actor }}
|
||||
schedule:
|
||||
- cron: "0 1 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
build_type:
|
||||
description: "Build type"
|
||||
required: true
|
||||
type: choice
|
||||
options:
|
||||
- git-sha
|
||||
- nightly
|
||||
default: git-sha
|
||||
dry_run:
|
||||
description: "Skip pushing images (dry run)"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
build-rc:
|
||||
if: contains(github.ref, '-rc')
|
||||
name: build and push as release candidate
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
build:
|
||||
- name: "Build and push reth image"
|
||||
command: "make IMAGE_NAME=$IMAGE_NAME DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME PROFILE=maxperf docker-build-push"
|
||||
- name: "Build and push op-reth image"
|
||||
command: "make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push"
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Install cross main
|
||||
id: cross_main
|
||||
run: |
|
||||
cargo install cross --git https://github.com/cross-rs/cross
|
||||
- name: Log in to Docker
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
|
||||
- name: Set up Docker builder
|
||||
run: |
|
||||
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
|
||||
docker buildx create --use --name cross-builder
|
||||
- name: Build and push ${{ matrix.build.name }}
|
||||
run: ${{ matrix.build.command }}
|
||||
|
||||
build:
|
||||
if: ${{ !contains(github.ref, '-rc') }}
|
||||
name: build and push as latest
|
||||
name: Build Docker images
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
build:
|
||||
- name: "Build and push reth image"
|
||||
command: "make IMAGE_NAME=$IMAGE_NAME DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME PROFILE=maxperf docker-build-push-latest"
|
||||
- name: "Build and push op-reth image"
|
||||
command: "make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push-latest"
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
|
||||
- name: Set up Depot CLI
|
||||
uses: depot/setup-action@v1
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Install cross main
|
||||
id: cross_main
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Get git info for vergen
|
||||
id: git
|
||||
run: |
|
||||
cargo install cross --git https://github.com/cross-rs/cross
|
||||
- name: Log in to Docker
|
||||
echo "sha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
|
||||
echo "describe=$(git describe --always --tags)" >> "$GITHUB_OUTPUT"
|
||||
echo "dirty=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Determine build parameters
|
||||
id: params
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
|
||||
- name: Set up Docker builder
|
||||
run: |
|
||||
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
|
||||
docker buildx create --use --name cross-builder
|
||||
- name: Build and push ${{ matrix.build.name }}
|
||||
run: ${{ matrix.build.command }}
|
||||
REGISTRY="ghcr.io/${{ github.repository_owner }}"
|
||||
|
||||
if [[ "${{ github.event_name }}" == "push" ]]; then
|
||||
VERSION="${GITHUB_REF#refs/tags/}"
|
||||
echo "targets=ethereum optimism" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Add 'latest' tag for non-RC releases
|
||||
if [[ ! "$VERSION" =~ -rc ]]; then
|
||||
echo "ethereum_tags=${REGISTRY}/reth:${VERSION},${REGISTRY}/reth:latest" >> "$GITHUB_OUTPUT"
|
||||
echo "optimism_tags=${REGISTRY}/op-reth:${VERSION},${REGISTRY}/op-reth:latest" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "ethereum_tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
echo "optimism_tags=${REGISTRY}/op-reth:${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
elif [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ inputs.build_type }}" == "nightly" ]]; then
|
||||
echo "targets=nightly" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
|
||||
echo "optimism_tags=${REGISTRY}/op-reth:nightly" >> "$GITHUB_OUTPUT"
|
||||
|
||||
else
|
||||
# git-sha build
|
||||
echo "targets=ethereum optimism" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
|
||||
echo "optimism_tags=${REGISTRY}/op-reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build and push images
|
||||
uses: depot/bake-action@v1
|
||||
env:
|
||||
VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }}
|
||||
VERGEN_GIT_DESCRIBE: ${{ steps.git.outputs.describe }}
|
||||
VERGEN_GIT_DIRTY: ${{ steps.git.outputs.dirty }}
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
files: docker-bake.hcl
|
||||
targets: ${{ steps.params.outputs.targets }}
|
||||
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
|
||||
set: |
|
||||
ethereum.tags=${{ steps.params.outputs.ethereum_tags }}
|
||||
optimism.tags=${{ steps.params.outputs.optimism_tags }}
|
||||
|
||||
21
.github/workflows/e2e.yml
vendored
21
.github/workflows/e2e.yml
vendored
@@ -44,3 +44,24 @@ jobs:
|
||||
--exclude 'op-reth' \
|
||||
--exclude 'reth' \
|
||||
-E 'binary(e2e_testsuite)'
|
||||
|
||||
rocksdb:
|
||||
name: e2e-rocksdb
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Run RocksDB e2e tests
|
||||
run: |
|
||||
cargo nextest run \
|
||||
--locked --features "edge" \
|
||||
-p reth-e2e-test-utils \
|
||||
-E 'binary(rocksdb)'
|
||||
|
||||
10
.github/workflows/hive.yml
vendored
10
.github/workflows/hive.yml
vendored
@@ -58,11 +58,11 @@ jobs:
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ./hive_assets
|
||||
key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/assets/hive/build_simulators.sh') }}
|
||||
key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/scripts/hive/build_simulators.sh') }}
|
||||
|
||||
- name: Build hive assets
|
||||
if: steps.cache-hive.outputs.cache-hit != 'true'
|
||||
run: .github/assets/hive/build_simulators.sh
|
||||
run: .github/scripts/hive/build_simulators.sh
|
||||
|
||||
- name: Load cached Docker images
|
||||
if: steps.cache-hive.outputs.cache-hit == 'true'
|
||||
@@ -213,7 +213,7 @@ jobs:
|
||||
path: /tmp
|
||||
|
||||
- name: Load Docker images
|
||||
run: .github/assets/hive/load_images.sh
|
||||
run: .github/scripts/hive/load_images.sh
|
||||
|
||||
- name: Move hive binary
|
||||
run: |
|
||||
@@ -241,11 +241,11 @@ jobs:
|
||||
FILTER="/"
|
||||
fi
|
||||
echo "filter: $FILTER"
|
||||
.github/assets/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
|
||||
.github/scripts/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
|
||||
|
||||
- name: Parse hive output
|
||||
run: |
|
||||
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.yaml --ignored .github/assets/hive/ignored_tests.yaml
|
||||
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/scripts/hive/parse.py {} --exclusion .github/scripts/hive/expected_failures.yaml --ignored .github/scripts/hive/ignored_tests.yaml
|
||||
|
||||
- name: Print simulator output
|
||||
if: ${{ failure() }}
|
||||
|
||||
10
.github/workflows/integration.yml
vendored
10
.github/workflows/integration.yml
vendored
@@ -22,7 +22,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: test / ${{ matrix.network }}
|
||||
name: test / ${{ matrix.network }} / ${{ matrix.storage }}
|
||||
if: github.event_name != 'schedule'
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
env:
|
||||
@@ -30,13 +30,17 @@ jobs:
|
||||
strategy:
|
||||
matrix:
|
||||
network: ["ethereum", "optimism"]
|
||||
storage: ["stable", "edge"]
|
||||
exclude:
|
||||
- network: optimism
|
||||
storage: edge
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- name: Install Geth
|
||||
run: .github/assets/install_geth.sh
|
||||
run: .github/scripts/install_geth.sh
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
@@ -46,7 +50,7 @@ jobs:
|
||||
name: Run tests
|
||||
run: |
|
||||
cargo nextest run \
|
||||
--locked --features "asm-keccak ${{ matrix.network }}" \
|
||||
--locked --features "asm-keccak ${{ matrix.network }} ${{ matrix.storage == 'edge' && 'edge' || '' }}" \
|
||||
--workspace --exclude ef-tests \
|
||||
-E "kind(test) and not binary(e2e_testsuite)"
|
||||
- if: matrix.network == 'optimism'
|
||||
|
||||
2
.github/workflows/label-pr.yml
vendored
2
.github/workflows/label-pr.yml
vendored
@@ -19,5 +19,5 @@ jobs:
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const label_pr = require('./.github/assets/label_pr.js')
|
||||
const label_pr = require('./.github/scripts/label_pr.js')
|
||||
await label_pr({github, context})
|
||||
|
||||
6
.github/workflows/lint.yml
vendored
6
.github/workflows/lint.yml
vendored
@@ -76,7 +76,7 @@ jobs:
|
||||
- name: Run Wasm checks
|
||||
run: |
|
||||
sudo apt update && sudo apt install gcc-multilib
|
||||
.github/assets/check_wasm.sh
|
||||
.github/scripts/check_wasm.sh
|
||||
|
||||
riscv:
|
||||
runs-on: depot-ubuntu-latest
|
||||
@@ -94,7 +94,7 @@ jobs:
|
||||
cache-on-failure: true
|
||||
- uses: dcarbone/install-jq-action@v3
|
||||
- name: Run RISC-V checks
|
||||
run: .github/assets/check_rv32imac.sh
|
||||
run: .github/scripts/check_rv32imac.sh
|
||||
|
||||
crate-checks:
|
||||
name: crate-checks (${{ matrix.partition }}/${{ matrix.total_partitions }})
|
||||
@@ -285,7 +285,7 @@ jobs:
|
||||
- run: zepter run check
|
||||
|
||||
deny:
|
||||
uses: ithacaxyz/ci/.github/workflows/deny.yml@main
|
||||
uses: tempoxyz/ci/.github/workflows/deny.yml@main
|
||||
|
||||
lint-success:
|
||||
name: lint success
|
||||
|
||||
2
.github/workflows/prepare-reth.yml
vendored
2
.github/workflows/prepare-reth.yml
vendored
@@ -43,7 +43,7 @@ jobs:
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: .github/assets/hive/Dockerfile
|
||||
file: .github/scripts/hive/Dockerfile
|
||||
tags: ${{ inputs.image_tag }}
|
||||
outputs: type=docker,dest=./artifacts/reth_image.tar
|
||||
build-args: |
|
||||
|
||||
@@ -249,7 +249,7 @@ Write comments that remain valuable after the PR is merged. Future readers won't
|
||||
unsafe impl GlobalAlloc for LimitedAllocator { ... }
|
||||
|
||||
// Binary search requires sorted input. Panics on unsorted slices.
|
||||
fn find_index(items: &[Item], target: &Item) -> Option
|
||||
fn find_index(items: &[Item], target: &Item) -> Option<usize>
|
||||
|
||||
// Timeout set to 5s to match EVM block processing limits
|
||||
const TRACER_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
1632
Cargo.lock
generated
1632
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
125
Cargo.toml
125
Cargo.toml
@@ -1,5 +1,5 @@
|
||||
[workspace.package]
|
||||
version = "1.9.3"
|
||||
version = "1.10.2"
|
||||
edition = "2024"
|
||||
rust-version = "1.88"
|
||||
license = "MIT OR Apache-2.0"
|
||||
@@ -77,6 +77,10 @@ members = [
|
||||
"crates/optimism/cli",
|
||||
"crates/optimism/consensus",
|
||||
"crates/optimism/evm/",
|
||||
"crates/optimism/examples/custom-node",
|
||||
"crates/optimism/examples/engine-api-access",
|
||||
"crates/optimism/examples/exex-hello-world",
|
||||
"crates/optimism/examples/op-db-access",
|
||||
"crates/optimism/flashblocks/",
|
||||
"crates/optimism/hardforks/",
|
||||
"crates/optimism/node/",
|
||||
@@ -145,7 +149,6 @@ members = [
|
||||
"examples/beacon-api-sse/",
|
||||
"examples/bsc-p2p",
|
||||
"examples/custom-dev-node/",
|
||||
"examples/custom-node/",
|
||||
"examples/custom-engine-types/",
|
||||
"examples/custom-evm/",
|
||||
"examples/custom-hardforks/",
|
||||
@@ -154,10 +157,7 @@ members = [
|
||||
"examples/custom-payload-builder/",
|
||||
"examples/custom-rlpx-subprotocol",
|
||||
"examples/custom-rpc-middleware",
|
||||
"examples/custom-node",
|
||||
"examples/db-access",
|
||||
"examples/engine-api-access",
|
||||
"examples/exex-hello-world",
|
||||
"examples/exex-subscription",
|
||||
"examples/exex-test",
|
||||
"examples/full-contract-state",
|
||||
@@ -168,7 +168,6 @@ members = [
|
||||
"examples/node-builder-api/",
|
||||
"examples/node-custom-rpc/",
|
||||
"examples/node-event-hooks/",
|
||||
"examples/op-db-access/",
|
||||
"examples/polygon-p2p/",
|
||||
"examples/rpc-db/",
|
||||
"examples/precompile-cache/",
|
||||
@@ -473,60 +472,60 @@ reth-ress-protocol = { path = "crates/ress/protocol" }
|
||||
reth-ress-provider = { path = "crates/ress/provider" }
|
||||
|
||||
# revm
|
||||
revm = { version = "33.1.0", default-features = false }
|
||||
revm-bytecode = { version = "7.1.1", default-features = false }
|
||||
revm-database = { version = "9.0.5", default-features = false }
|
||||
revm-state = { version = "8.1.1", default-features = false }
|
||||
revm-primitives = { version = "21.0.2", default-features = false }
|
||||
revm-interpreter = { version = "31.1.0", default-features = false }
|
||||
revm-database-interface = { version = "8.0.5", default-features = false }
|
||||
op-revm = { version = "14.1.0", default-features = false }
|
||||
revm-inspectors = "0.33.2"
|
||||
revm = { version = "34.0.0", default-features = false }
|
||||
revm-bytecode = { version = "8.0.0", default-features = false }
|
||||
revm-database = { version = "10.0.0", default-features = false }
|
||||
revm-state = { version = "9.0.0", default-features = false }
|
||||
revm-primitives = { version = "22.0.0", default-features = false }
|
||||
revm-interpreter = { version = "32.0.0", default-features = false }
|
||||
revm-database-interface = { version = "9.0.0", default-features = false }
|
||||
op-revm = { version = "15.0.0", default-features = false }
|
||||
revm-inspectors = "0.34.1"
|
||||
|
||||
# eth
|
||||
alloy-dyn-abi = "1.5.4"
|
||||
alloy-primitives = { version = "1.5.4", default-features = false, features = ["map-foldhash"] }
|
||||
alloy-sol-types = { version = "1.5.4", default-features = false }
|
||||
|
||||
alloy-chains = { version = "0.2.5", default-features = false }
|
||||
alloy-dyn-abi = "1.4.1"
|
||||
alloy-eip2124 = { version = "0.2.0", default-features = false }
|
||||
alloy-eip7928 = { version = "0.1.0", default-features = false }
|
||||
alloy-evm = { version = "0.25.1", default-features = false }
|
||||
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
|
||||
alloy-eip7928 = { version = "0.3.0", default-features = false }
|
||||
alloy-evm = { version = "0.27.0", default-features = false }
|
||||
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
|
||||
alloy-sol-macro = "1.5.0"
|
||||
alloy-sol-types = { version = "1.5.0", default-features = false }
|
||||
alloy-trie = { version = "0.9.1", default-features = false }
|
||||
|
||||
alloy-hardforks = "0.4.5"
|
||||
|
||||
alloy-consensus = { version = "1.2.1", default-features = false }
|
||||
alloy-contract = { version = "1.2.1", default-features = false }
|
||||
alloy-eips = { version = "1.2.1", default-features = false }
|
||||
alloy-genesis = { version = "1.2.1", default-features = false }
|
||||
alloy-json-rpc = { version = "1.2.1", default-features = false }
|
||||
alloy-network = { version = "1.2.1", default-features = false }
|
||||
alloy-network-primitives = { version = "1.2.1", default-features = false }
|
||||
alloy-provider = { version = "1.2.1", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-client = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types = { version = "1.2.1", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "1.2.1", default-features = false }
|
||||
alloy-serde = { version = "1.2.1", default-features = false }
|
||||
alloy-signer = { version = "1.2.1", default-features = false }
|
||||
alloy-signer-local = { version = "1.2.1", default-features = false }
|
||||
alloy-transport = { version = "1.2.1" }
|
||||
alloy-transport-http = { version = "1.2.1", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "1.2.1", default-features = false }
|
||||
alloy-transport-ws = { version = "1.2.1", default-features = false }
|
||||
alloy-consensus = { version = "1.5.2", default-features = false }
|
||||
alloy-contract = { version = "1.5.2", default-features = false }
|
||||
alloy-eips = { version = "1.5.2", default-features = false }
|
||||
alloy-genesis = { version = "1.5.2", default-features = false }
|
||||
alloy-json-rpc = { version = "1.5.2", default-features = false }
|
||||
alloy-network = { version = "1.5.2", default-features = false }
|
||||
alloy-network-primitives = { version = "1.5.2", default-features = false }
|
||||
alloy-provider = { version = "1.5.2", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-client = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types = { version = "1.5.2", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "1.5.2", default-features = false }
|
||||
alloy-serde = { version = "1.5.2", default-features = false }
|
||||
alloy-signer = { version = "1.5.2", default-features = false }
|
||||
alloy-signer-local = { version = "1.5.2", default-features = false }
|
||||
alloy-transport = { version = "1.5.2" }
|
||||
alloy-transport-http = { version = "1.5.2", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "1.5.2", default-features = false }
|
||||
alloy-transport-ws = { version = "1.5.2", default-features = false }
|
||||
|
||||
# op
|
||||
alloy-op-evm = { version = "0.25.0", default-features = false }
|
||||
alloy-op-evm = { version = "0.27.0", default-features = false }
|
||||
alloy-op-hardforks = "0.4.4"
|
||||
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
|
||||
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
|
||||
@@ -555,6 +554,7 @@ dirs-next = "2.0.0"
|
||||
dyn-clone = "1.0.17"
|
||||
eyre = "0.6"
|
||||
fdlimit = "0.3.0"
|
||||
fixed-map = { version = "0.9", default-features = false }
|
||||
humantime = "2.1"
|
||||
humantime-serde = "1.1"
|
||||
itertools = { version = "0.14", default-features = false }
|
||||
@@ -587,7 +587,7 @@ tracing-appender = "0.2"
|
||||
url = { version = "2.3", default-features = false }
|
||||
zstd = "0.13"
|
||||
byteorder = "1"
|
||||
mini-moka = "0.10"
|
||||
fixed-cache = { version = "0.1.7", features = ["stats"] }
|
||||
moka = "0.12"
|
||||
tar-no-std = { version = "0.3.2", default-features = false }
|
||||
miniz_oxide = { version = "0.8.4", default-features = false }
|
||||
@@ -596,9 +596,9 @@ chrono = "0.4.41"
|
||||
# metrics
|
||||
metrics = "0.24.0"
|
||||
metrics-derive = "0.1"
|
||||
metrics-exporter-prometheus = { version = "0.16.0", default-features = false }
|
||||
metrics-exporter-prometheus = { version = "0.18.0", default-features = false }
|
||||
metrics-process = "2.1.0"
|
||||
metrics-util = { default-features = false, version = "0.19.0" }
|
||||
metrics-util = { default-features = false, version = "0.20.0" }
|
||||
|
||||
# proc-macros
|
||||
proc-macro2 = "1.0"
|
||||
@@ -664,12 +664,13 @@ opentelemetry_sdk = "0.31"
|
||||
opentelemetry = "0.31"
|
||||
opentelemetry-otlp = "0.31"
|
||||
opentelemetry-semantic-conventions = "0.31"
|
||||
opentelemetry-appender-tracing = "0.31"
|
||||
tracing-opentelemetry = "0.32"
|
||||
|
||||
# misc-testing
|
||||
arbitrary = "1.3"
|
||||
assert_matches = "1.5.0"
|
||||
criterion = { package = "codspeed-criterion-compat", version = "2.7" }
|
||||
criterion = { package = "codspeed-criterion-compat", version = "4.3" }
|
||||
insta = "1.41"
|
||||
proptest = "1.7"
|
||||
proptest-derive = "0.5"
|
||||
@@ -731,20 +732,21 @@ snap = "1.1.1"
|
||||
socket2 = { version = "0.5", default-features = false }
|
||||
sysinfo = { version = "0.33", default-features = false }
|
||||
tracing-journald = "0.3"
|
||||
tracing-logfmt = "0.3.3"
|
||||
tracing-logfmt = "=0.3.5"
|
||||
tracing-samply = "0.1"
|
||||
tracing-subscriber = { version = "0.3", default-features = false }
|
||||
tracing-tracy = "0.11"
|
||||
triehash = "0.8"
|
||||
typenum = "1.15.0"
|
||||
vergen = "9.0.4"
|
||||
vergen = "9.1.0"
|
||||
visibility = "0.1.1"
|
||||
walkdir = "2.3.3"
|
||||
vergen-git2 = "1.0.5"
|
||||
vergen-git2 = "9.1.0"
|
||||
|
||||
# networking
|
||||
ipnet = "2.11"
|
||||
|
||||
# [patch.crates-io]
|
||||
[patch.crates-io]
|
||||
# alloy-consensus = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-contract = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-eips = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
@@ -787,5 +789,10 @@ ipnet = "2.11"
|
||||
# jsonrpsee-http-client = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
|
||||
# jsonrpsee-types = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
|
||||
|
||||
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
|
||||
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
|
||||
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
|
||||
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
|
||||
|
||||
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "3020ea8" }
|
||||
|
||||
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
|
||||
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
# This image is meant to enable cross-architecture builds.
|
||||
# It assumes the reth binary has already been compiled for `$TARGETPLATFORM` and is
|
||||
# locatable in `./dist/bin/$TARGETARCH`
|
||||
FROM --platform=$TARGETPLATFORM ubuntu:22.04
|
||||
|
||||
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
|
||||
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
|
||||
|
||||
# Filled by docker buildx
|
||||
ARG TARGETARCH
|
||||
|
||||
COPY ./dist/bin/$TARGETARCH/reth /usr/local/bin/reth
|
||||
|
||||
EXPOSE 30303 30303/udp 9001 8545 8546
|
||||
ENTRYPOINT ["/usr/local/bin/reth"]
|
||||
84
Dockerfile.depot
Normal file
84
Dockerfile.depot
Normal file
@@ -0,0 +1,84 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
|
||||
# Unified Dockerfile for reth and op-reth, optimized for Depot builds
|
||||
# Usage:
|
||||
# reth: --build-arg BINARY=reth
|
||||
# op-reth: --build-arg BINARY=op-reth --build-arg MANIFEST_PATH=crates/optimism/bin
|
||||
|
||||
FROM rust:1 AS builder
|
||||
WORKDIR /app
|
||||
|
||||
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
|
||||
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
|
||||
|
||||
RUN apt-get update && apt-get install -y libclang-dev pkg-config
|
||||
|
||||
# Install sccache for compilation caching
|
||||
RUN cargo install sccache --locked
|
||||
ENV RUSTC_WRAPPER=sccache
|
||||
ENV SCCACHE_DIR=/sccache
|
||||
ENV SCCACHE_WEBDAV_ENDPOINT=https://cache.depot.dev
|
||||
|
||||
# Binary to build (reth or op-reth)
|
||||
ARG BINARY=reth
|
||||
|
||||
# Manifest path for the binary
|
||||
ARG MANIFEST_PATH=bin/reth
|
||||
|
||||
# Build profile, release by default
|
||||
ARG BUILD_PROFILE=release
|
||||
ENV BUILD_PROFILE=$BUILD_PROFILE
|
||||
|
||||
# Extra Cargo flags
|
||||
ARG RUSTFLAGS=""
|
||||
ENV RUSTFLAGS="$RUSTFLAGS"
|
||||
|
||||
# Extra Cargo features
|
||||
ARG FEATURES=""
|
||||
ENV FEATURES=$FEATURES
|
||||
|
||||
# Git info for vergen (since .git is excluded from Docker context)
|
||||
ARG VERGEN_GIT_SHA=""
|
||||
ARG VERGEN_GIT_DESCRIBE=""
|
||||
ARG VERGEN_GIT_DIRTY="false"
|
||||
ENV VERGEN_GIT_SHA=$VERGEN_GIT_SHA
|
||||
ENV VERGEN_GIT_DESCRIBE=$VERGEN_GIT_DESCRIBE
|
||||
ENV VERGEN_GIT_DIRTY=$VERGEN_GIT_DIRTY
|
||||
|
||||
# Build application
|
||||
COPY --exclude=.git . .
|
||||
RUN --mount=type=secret,id=DEPOT_TOKEN,env=SCCACHE_WEBDAV_TOKEN \
|
||||
--mount=type=cache,target=/usr/local/cargo/registry,sharing=shared \
|
||||
--mount=type=cache,target=/usr/local/cargo/git,sharing=shared \
|
||||
--mount=type=cache,target=$SCCACHE_DIR,sharing=shared \
|
||||
cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin $BINARY --manifest-path $MANIFEST_PATH/Cargo.toml
|
||||
|
||||
RUN sccache --show-stats || true
|
||||
|
||||
# Copy binary to a known location (ARG not resolved in COPY)
|
||||
# Note: Custom profiles like maxperf/profiling output to target/<profile>/, not target/release/
|
||||
RUN cp /app/target/$BUILD_PROFILE/$BINARY /app/binary || \
|
||||
cp /app/target/release/$BINARY /app/binary
|
||||
|
||||
FROM ubuntu:24.04 AS runtime
|
||||
WORKDIR /app
|
||||
|
||||
# Binary name for entrypoint
|
||||
ARG BINARY=reth
|
||||
|
||||
# Install runtime dependencies
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends ca-certificates && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy binary from build stage and create canonical symlink for entrypoint
|
||||
COPY --from=builder /app/binary /usr/local/bin/
|
||||
RUN mv /usr/local/bin/binary /usr/local/bin/$BINARY && \
|
||||
ln -s /usr/local/bin/$BINARY /usr/local/bin/reth-binary && \
|
||||
chmod +x /usr/local/bin/$BINARY
|
||||
|
||||
# Copy licenses
|
||||
COPY LICENSE-* ./
|
||||
|
||||
EXPOSE 30303 30303/udp 9001 8545 8546
|
||||
ENTRYPOINT ["/usr/local/bin/reth-binary"]
|
||||
@@ -1,15 +0,0 @@
|
||||
# This image is meant to enable cross-architecture builds.
|
||||
# It assumes the reth binary has already been compiled for `$TARGETPLATFORM` and is
|
||||
# locatable in `./dist/bin/$TARGETARCH`
|
||||
FROM --platform=$TARGETPLATFORM ubuntu:22.04
|
||||
|
||||
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
|
||||
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
|
||||
|
||||
# Filled by docker buildx
|
||||
ARG TARGETARCH
|
||||
|
||||
COPY ./dist/bin/$TARGETARCH/op-reth /usr/local/bin/op-reth
|
||||
|
||||
EXPOSE 30303 30303/udp 9001 8545 8546
|
||||
ENTRYPOINT ["/usr/local/bin/op-reth"]
|
||||
134
Makefile
134
Makefile
@@ -35,9 +35,6 @@ EEST_TESTS_TAG := v4.5.0
|
||||
EEST_TESTS_URL := https://github.com/ethereum/execution-spec-tests/releases/download/$(EEST_TESTS_TAG)/fixtures_stable.tar.gz
|
||||
EEST_TESTS_DIR := ./testing/ef-tests/execution-spec-tests
|
||||
|
||||
# The docker image name
|
||||
DOCKER_IMAGE_NAME ?= ghcr.io/paradigmxyz/reth
|
||||
|
||||
##@ Help
|
||||
|
||||
.PHONY: help
|
||||
@@ -242,137 +239,6 @@ install-reth-bench: ## Build and install the reth binary under `$(CARGO_HOME)/bi
|
||||
--features "$(FEATURES)" \
|
||||
--profile "$(PROFILE)"
|
||||
|
||||
##@ Docker
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --driver docker-container --name cross-builder`
|
||||
.PHONY: docker-build-push
|
||||
docker-build-push: ## Build and push a cross-arch Docker image tagged with the latest git tag.
|
||||
$(call docker_build_push,$(GIT_TAG),$(GIT_TAG))
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --driver docker-container --name cross-builder`
|
||||
.PHONY: docker-build-push-git-sha
|
||||
docker-build-push-git-sha: ## Build and push a cross-arch Docker image tagged with the latest git sha.
|
||||
$(call docker_build_push,$(GIT_SHA),$(GIT_SHA))
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --driver docker-container --name cross-builder`
|
||||
.PHONY: docker-build-push-latest
|
||||
docker-build-push-latest: ## Build and push a cross-arch Docker image tagged with the latest git tag and `latest`.
|
||||
$(call docker_build_push,$(GIT_TAG),latest)
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --name cross-builder`
|
||||
.PHONY: docker-build-push-nightly
|
||||
docker-build-push-nightly: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
|
||||
$(call docker_build_push,nightly,nightly)
|
||||
|
||||
.PHONY: docker-build-push-nightly-edge-profiling
|
||||
docker-build-push-nightly-edge-profiling: FEATURES := $(FEATURES) edge
|
||||
docker-build-push-nightly-edge-profiling: ## Build and push cross-arch Docker image with edge features tagged with `nightly-edge-profiling`.
|
||||
$(call docker_build_push,nightly-edge-profiling,nightly-edge-profiling)
|
||||
|
||||
# Create a cross-arch Docker image with the given tags and push it
|
||||
define docker_build_push
|
||||
$(MAKE) build-x86_64-unknown-linux-gnu
|
||||
mkdir -p $(BIN_DIR)/amd64
|
||||
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/amd64/reth
|
||||
|
||||
$(MAKE) build-aarch64-unknown-linux-gnu
|
||||
mkdir -p $(BIN_DIR)/arm64
|
||||
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/arm64/reth
|
||||
|
||||
docker buildx build --file ./Dockerfile.cross . \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--tag $(DOCKER_IMAGE_NAME):$(1) \
|
||||
--tag $(DOCKER_IMAGE_NAME):$(2) \
|
||||
--provenance=false \
|
||||
--push
|
||||
endef
|
||||
|
||||
##@ Optimism docker
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --driver docker-container --name cross-builder`
|
||||
.PHONY: op-docker-build-push
|
||||
op-docker-build-push: ## Build and push a cross-arch Docker image tagged with the latest git tag.
|
||||
$(call op_docker_build_push,$(GIT_TAG),$(GIT_TAG))
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --driver docker-container --name cross-builder`
|
||||
.PHONY: op-docker-build-push-git-sha
|
||||
op-docker-build-push-git-sha: ## Build and push a cross-arch Docker image tagged with the latest git sha.
|
||||
$(call op_docker_build_push,$(GIT_SHA),$(GIT_SHA))
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --driver docker-container --name cross-builder`
|
||||
.PHONY: op-docker-build-push-latest
|
||||
op-docker-build-push-latest: ## Build and push a cross-arch Docker image tagged with the latest git tag and `latest`.
|
||||
$(call op_docker_build_push,$(GIT_TAG),latest)
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --name cross-builder`
|
||||
.PHONY: op-docker-build-push-nightly
|
||||
op-docker-build-push-nightly: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
|
||||
$(call op_docker_build_push,nightly,nightly)
|
||||
|
||||
.PHONY: op-docker-build-push-nightly-edge-profiling
|
||||
op-docker-build-push-nightly-edge-profiling: FEATURES := $(FEATURES) edge
|
||||
op-docker-build-push-nightly-edge-profiling: ## Build and push cross-arch Docker image with edge features tagged with `nightly-edge-profiling`.
|
||||
$(call op_docker_build_push,nightly-edge-profiling,nightly-edge-profiling)
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --name cross-builder`
|
||||
.PHONY: docker-build-push-nightly-profiling
|
||||
docker-build-push-nightly-profiling: ## Build and push cross-arch Docker image with profiling profile tagged with nightly-profiling.
|
||||
$(call docker_build_push,nightly-profiling,nightly-profiling)
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --name cross-builder`
|
||||
.PHONY: op-docker-build-push-nightly-profiling
|
||||
op-docker-build-push-nightly-profiling: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
|
||||
$(call op_docker_build_push,nightly-profiling,nightly-profiling)
|
||||
|
||||
|
||||
# Create a cross-arch Docker image with the given tags and push it
|
||||
define op_docker_build_push
|
||||
$(MAKE) op-build-x86_64-unknown-linux-gnu
|
||||
mkdir -p $(BIN_DIR)/amd64
|
||||
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/amd64/op-reth
|
||||
|
||||
$(MAKE) op-build-aarch64-unknown-linux-gnu
|
||||
mkdir -p $(BIN_DIR)/arm64
|
||||
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/arm64/op-reth
|
||||
|
||||
docker buildx build --file ./DockerfileOp.cross . \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--tag $(DOCKER_IMAGE_NAME):$(1) \
|
||||
--tag $(DOCKER_IMAGE_NAME):$(2) \
|
||||
--provenance=false \
|
||||
--push
|
||||
endef
|
||||
|
||||
##@ Other
|
||||
|
||||
.PHONY: clean
|
||||
|
||||
@@ -71,7 +71,11 @@ jemalloc = [
|
||||
"reth-node-core/jemalloc",
|
||||
]
|
||||
jemalloc-prof = ["reth-cli-util/jemalloc-prof"]
|
||||
tracy-allocator = ["reth-cli-util/tracy-allocator"]
|
||||
tracy-allocator = ["reth-cli-util/tracy-allocator", "tracy"]
|
||||
tracy = [
|
||||
"reth-node-core/tracy",
|
||||
"reth-tracing/tracy",
|
||||
]
|
||||
|
||||
min-error-logs = [
|
||||
"tracing/release_max_level_error",
|
||||
|
||||
@@ -147,6 +147,11 @@ pub(crate) struct Args {
|
||||
#[arg(long)]
|
||||
pub no_clear_cache: bool,
|
||||
|
||||
/// Skip waiting for the node to sync before starting benchmarks.
|
||||
/// When enabled, assumes the node is already synced and skips the initial tip check.
|
||||
#[arg(long)]
|
||||
pub skip_wait_syncing: bool,
|
||||
|
||||
#[command(flatten)]
|
||||
pub logs: LogArgs,
|
||||
|
||||
@@ -269,10 +274,10 @@ impl Args {
|
||||
/// Get the default RPC URL for a given chain
|
||||
const fn get_default_rpc_url(chain: &Chain) -> &'static str {
|
||||
match chain.id() {
|
||||
8453 => "https://base-mainnet.rpc.ithaca.xyz", // base
|
||||
8453 => "https://base.reth.rs/rpc", // base
|
||||
84532 => "https://base-sepolia.rpc.ithaca.xyz", // base-sepolia
|
||||
27082 => "https://rpc.hoodi.ethpandaops.io", // hoodi
|
||||
_ => "https://reth-ethereum.ithaca.xyz/rpc", // mainnet and fallback
|
||||
_ => "https://ethereum.reth.rs/rpc", // mainnet and fallback
|
||||
}
|
||||
}
|
||||
|
||||
@@ -578,7 +583,11 @@ async fn run_warmup_phase(
|
||||
node_manager.start_node(&binary_path, warmup_ref, "warmup", &additional_args).await?;
|
||||
|
||||
// Wait for node to be ready and get its current tip
|
||||
let current_tip = node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?;
|
||||
let current_tip = if args.skip_wait_syncing {
|
||||
node_manager.wait_for_rpc_and_get_tip(&mut node_process).await?
|
||||
} else {
|
||||
node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?
|
||||
};
|
||||
info!("Warmup node is ready at tip: {}", current_tip);
|
||||
|
||||
// Clear filesystem caches before warmup run only (unless disabled)
|
||||
@@ -632,7 +641,11 @@ async fn run_benchmark_workflow(
|
||||
let (mut node_process, _) = node_manager
|
||||
.start_node(&binary_path, &args.baseline_ref, "baseline", &additional_args)
|
||||
.await?;
|
||||
let starting_tip = node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?;
|
||||
let starting_tip = if args.skip_wait_syncing {
|
||||
node_manager.wait_for_rpc_and_get_tip(&mut node_process).await?
|
||||
} else {
|
||||
node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?
|
||||
};
|
||||
info!("Node starting tip: {}", starting_tip);
|
||||
node_manager.stop_node(&mut node_process).await?;
|
||||
|
||||
@@ -699,7 +712,11 @@ async fn run_benchmark_workflow(
|
||||
node_manager.start_node(&binary_path, git_ref, ref_type, &additional_args).await?;
|
||||
|
||||
// Wait for node to be ready and get its current tip (wherever it is)
|
||||
let current_tip = node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?;
|
||||
let current_tip = if args.skip_wait_syncing {
|
||||
node_manager.wait_for_rpc_and_get_tip(&mut node_process).await?
|
||||
} else {
|
||||
node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?
|
||||
};
|
||||
info!("Node is ready at tip: {}", current_tip);
|
||||
|
||||
// Calculate benchmark range
|
||||
|
||||
@@ -163,6 +163,7 @@ impl NodeManager {
|
||||
"eth,reth".to_string(),
|
||||
"--disable-discovery".to_string(),
|
||||
"--trusted-only".to_string(),
|
||||
"--disable-tx-gossip".to_string(),
|
||||
]);
|
||||
|
||||
// Add tracing arguments if OTLP endpoint is configured
|
||||
@@ -458,6 +459,76 @@ impl NodeManager {
|
||||
.wrap_err("Timed out waiting for node to be ready and synced")?
|
||||
}
|
||||
|
||||
/// Wait for the node RPC to be ready and return its current tip, without waiting for sync.
|
||||
///
|
||||
/// This is faster than `wait_for_node_ready_and_get_tip` but may return a tip while
|
||||
/// the node is still syncing.
|
||||
pub(crate) async fn wait_for_rpc_and_get_tip(
|
||||
&self,
|
||||
child: &mut tokio::process::Child,
|
||||
) -> Result<u64> {
|
||||
info!("Waiting for node RPC to be ready (skipping sync wait)...");
|
||||
|
||||
let max_wait = Duration::from_secs(60);
|
||||
let check_interval = Duration::from_secs(2);
|
||||
let rpc_url = "http://localhost:8545";
|
||||
|
||||
let url = rpc_url.parse().map_err(|e| eyre!("Invalid RPC URL '{}': {}", rpc_url, e))?;
|
||||
let provider = ProviderBuilder::new().connect_http(url);
|
||||
|
||||
let start_time = tokio::time::Instant::now();
|
||||
let mut iteration = 0;
|
||||
|
||||
timeout(max_wait, async {
|
||||
loop {
|
||||
iteration += 1;
|
||||
debug!(
|
||||
"RPC readiness check iteration {} (elapsed: {:?})",
|
||||
iteration,
|
||||
start_time.elapsed()
|
||||
);
|
||||
|
||||
if let Some(status) = child.try_wait()? {
|
||||
return Err(eyre!("Node process exited unexpectedly with {status}"));
|
||||
}
|
||||
|
||||
match provider.get_block_number().await {
|
||||
Ok(tip) => {
|
||||
debug!("HTTP RPC ready at block: {}, checking WebSocket...", tip);
|
||||
|
||||
let ws_url = format!("ws://localhost:{}", DEFAULT_WS_RPC_PORT);
|
||||
let ws_connect = WsConnect::new(&ws_url);
|
||||
|
||||
match RpcClient::connect_pubsub(ws_connect).await {
|
||||
Ok(_) => {
|
||||
info!(
|
||||
"Node RPC is ready at block: {} (took {:?}, {} iterations)",
|
||||
tip,
|
||||
start_time.elapsed(),
|
||||
iteration
|
||||
);
|
||||
return Ok(tip);
|
||||
}
|
||||
Err(e) => {
|
||||
debug!(
|
||||
"HTTP RPC ready but WebSocket not ready yet (iteration {}): {:?}",
|
||||
iteration, e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("RPC not ready yet (iteration {}): {:?}", iteration, e);
|
||||
}
|
||||
}
|
||||
|
||||
sleep(check_interval).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.wrap_err("Timed out waiting for node RPC to be ready")?
|
||||
}
|
||||
|
||||
/// Stop the reth node gracefully
|
||||
pub(crate) async fn stop_node(&self, child: &mut tokio::process::Child) -> Result<()> {
|
||||
let pid = child.id().ok_or_eyre("Child process ID should be available")?;
|
||||
|
||||
@@ -17,21 +17,26 @@ workspace = true
|
||||
reth-cli-runner.workspace = true
|
||||
reth-cli-util.workspace = true
|
||||
reth-engine-primitives.workspace = true
|
||||
reth-ethereum-primitives.workspace = true
|
||||
reth-fs-util.workspace = true
|
||||
reth-node-api.workspace = true
|
||||
reth-node-core.workspace = true
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-rpc-api.workspace = true
|
||||
|
||||
reth-tracing.workspace = true
|
||||
reth-chainspec.workspace = true
|
||||
|
||||
# alloy
|
||||
alloy-eips.workspace = true
|
||||
alloy-json-rpc.workspace = true
|
||||
alloy-consensus.workspace = true
|
||||
alloy-network.workspace = true
|
||||
alloy-primitives.workspace = true
|
||||
alloy-primitives = { workspace = true, features = ["rand"] }
|
||||
alloy-provider = { workspace = true, features = ["engine-api", "pubsub", "reqwest-rustls-tls"], default-features = false }
|
||||
alloy-pubsub.workspace = true
|
||||
alloy-rpc-client = { workspace = true, features = ["pubsub"] }
|
||||
alloy-rpc-types-engine.workspace = true
|
||||
alloy-rpc-types-engine = { workspace = true, features = ["kzg"] }
|
||||
alloy-transport-http.workspace = true
|
||||
alloy-transport-ipc.workspace = true
|
||||
alloy-transport-ws.workspace = true
|
||||
@@ -85,7 +90,11 @@ jemalloc = [
|
||||
"reth-node-core/jemalloc",
|
||||
]
|
||||
jemalloc-prof = ["reth-cli-util/jemalloc-prof"]
|
||||
tracy-allocator = ["reth-cli-util/tracy-allocator"]
|
||||
tracy-allocator = ["reth-cli-util/tracy-allocator", "tracy"]
|
||||
tracy = [
|
||||
"reth-node-core/tracy",
|
||||
"reth-tracing/tracy",
|
||||
]
|
||||
|
||||
min-error-logs = [
|
||||
"tracing/release_max_level_error",
|
||||
|
||||
216
bin/reth-bench/src/bench/gas_limit_ramp.rs
Normal file
216
bin/reth-bench/src/bench/gas_limit_ramp.rs
Normal file
@@ -0,0 +1,216 @@
|
||||
//! Benchmarks empty block processing by ramping the block gas limit.
|
||||
|
||||
use crate::{
|
||||
authenticated_transport::AuthenticatedTransportConnect,
|
||||
bench::{
|
||||
helpers::{build_payload, parse_gas_limit, prepare_payload_request, rpc_block_to_header},
|
||||
output::GasRampPayloadFile,
|
||||
},
|
||||
valid_payload::{call_forkchoice_updated, call_new_payload, payload_to_new_payload},
|
||||
};
|
||||
use alloy_eips::BlockNumberOrTag;
|
||||
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
|
||||
use alloy_rpc_client::ClientBuilder;
|
||||
use alloy_rpc_types_engine::{ExecutionPayload, ForkchoiceState, JwtSecret};
|
||||
|
||||
use clap::Parser;
|
||||
use reqwest::Url;
|
||||
use reth_chainspec::ChainSpec;
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_ethereum_primitives::TransactionSigned;
|
||||
use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK};
|
||||
use std::{path::PathBuf, time::Instant};
|
||||
use tracing::info;
|
||||
|
||||
/// `reth benchmark gas-limit-ramp` command.
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Command {
|
||||
/// Number of blocks to generate. Mutually exclusive with --target-gas-limit.
|
||||
#[arg(long, value_name = "BLOCKS", conflicts_with = "target_gas_limit")]
|
||||
blocks: Option<u64>,
|
||||
|
||||
/// Target gas limit to ramp up to. The benchmark will generate blocks until the gas limit
|
||||
/// reaches or exceeds this value. Mutually exclusive with --blocks.
|
||||
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 2G = 2
|
||||
/// billion).
|
||||
#[arg(long, value_name = "TARGET_GAS_LIMIT", conflicts_with = "blocks", value_parser = parse_gas_limit)]
|
||||
target_gas_limit: Option<u64>,
|
||||
|
||||
/// The Engine API RPC URL.
|
||||
#[arg(long = "engine-rpc-url", value_name = "ENGINE_RPC_URL")]
|
||||
engine_rpc_url: String,
|
||||
|
||||
/// Path to the JWT secret for Engine API authentication.
|
||||
#[arg(long = "jwt-secret", value_name = "JWT_SECRET")]
|
||||
jwt_secret: PathBuf,
|
||||
|
||||
/// Output directory for benchmark results and generated payloads.
|
||||
#[arg(long, value_name = "OUTPUT")]
|
||||
output: PathBuf,
|
||||
}
|
||||
|
||||
/// Mode for determining when to stop ramping.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum RampMode {
|
||||
/// Ramp for a fixed number of blocks.
|
||||
Blocks(u64),
|
||||
/// Ramp until reaching or exceeding target gas limit.
|
||||
TargetGasLimit(u64),
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `benchmark gas-limit-ramp` command.
|
||||
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
|
||||
let mode = match (self.blocks, self.target_gas_limit) {
|
||||
(Some(blocks), None) => {
|
||||
if blocks == 0 {
|
||||
return Err(eyre::eyre!("--blocks must be greater than 0"));
|
||||
}
|
||||
RampMode::Blocks(blocks)
|
||||
}
|
||||
(None, Some(target)) => {
|
||||
if target == 0 {
|
||||
return Err(eyre::eyre!("--target-gas-limit must be greater than 0"));
|
||||
}
|
||||
RampMode::TargetGasLimit(target)
|
||||
}
|
||||
_ => {
|
||||
return Err(eyre::eyre!(
|
||||
"Exactly one of --blocks or --target-gas-limit must be specified"
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
// Ensure output directory exists
|
||||
if self.output.is_file() {
|
||||
return Err(eyre::eyre!("Output path must be a directory"));
|
||||
}
|
||||
if !self.output.exists() {
|
||||
std::fs::create_dir_all(&self.output)?;
|
||||
info!("Created output directory: {:?}", self.output);
|
||||
}
|
||||
|
||||
// Set up authenticated provider (used for both Engine API and eth_ methods)
|
||||
let jwt = std::fs::read_to_string(&self.jwt_secret)?;
|
||||
let jwt = JwtSecret::from_hex(jwt)?;
|
||||
let auth_url = Url::parse(&self.engine_rpc_url)?;
|
||||
|
||||
info!("Connecting to Engine RPC at {}", auth_url);
|
||||
let auth_transport = AuthenticatedTransportConnect::new(auth_url, jwt);
|
||||
let client = ClientBuilder::default().connect_with(auth_transport).await?;
|
||||
let provider = RootProvider::<AnyNetwork>::new(client);
|
||||
|
||||
// Get chain spec - required for fork detection
|
||||
let chain_id = provider.get_chain_id().await?;
|
||||
let chain_spec = ChainSpec::from_chain_id(chain_id)
|
||||
.ok_or_else(|| eyre::eyre!("Unsupported chain id: {chain_id}"))?;
|
||||
|
||||
// Fetch the current head block as parent
|
||||
let parent_block = provider
|
||||
.get_block_by_number(BlockNumberOrTag::Latest)
|
||||
.full()
|
||||
.await?
|
||||
.ok_or_else(|| eyre::eyre!("Failed to fetch latest block"))?;
|
||||
|
||||
let (mut parent_header, mut parent_hash) = rpc_block_to_header(parent_block);
|
||||
|
||||
let canonical_parent = parent_header.number;
|
||||
let start_block = canonical_parent + 1;
|
||||
|
||||
match mode {
|
||||
RampMode::Blocks(blocks) => {
|
||||
info!(
|
||||
canonical_parent,
|
||||
start_block,
|
||||
end_block = start_block + blocks - 1,
|
||||
"Starting gas limit ramp benchmark (block count mode)"
|
||||
);
|
||||
}
|
||||
RampMode::TargetGasLimit(target) => {
|
||||
info!(
|
||||
canonical_parent,
|
||||
start_block,
|
||||
current_gas_limit = parent_header.gas_limit,
|
||||
target_gas_limit = target,
|
||||
"Starting gas limit ramp benchmark (target gas limit mode)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut blocks_processed = 0u64;
|
||||
let total_benchmark_duration = Instant::now();
|
||||
|
||||
while !should_stop(mode, blocks_processed, parent_header.gas_limit) {
|
||||
let timestamp = parent_header.timestamp.saturating_add(1);
|
||||
|
||||
let request = prepare_payload_request(&chain_spec, timestamp, parent_hash);
|
||||
let new_payload_version = request.new_payload_version;
|
||||
|
||||
let (payload, sidecar) = build_payload(&provider, request).await?;
|
||||
|
||||
let mut block =
|
||||
payload.clone().try_into_block_with_sidecar::<TransactionSigned>(&sidecar)?;
|
||||
|
||||
let max_increase = max_gas_limit_increase(parent_header.gas_limit);
|
||||
let gas_limit =
|
||||
parent_header.gas_limit.saturating_add(max_increase).min(MAXIMUM_GAS_LIMIT_BLOCK);
|
||||
|
||||
block.header.gas_limit = gas_limit;
|
||||
|
||||
let block_hash = block.header.hash_slow();
|
||||
// Regenerate the payload from the modified block, but keep the original sidecar
|
||||
// which contains the actual execution requests data (not just the hash)
|
||||
let (payload, _) = ExecutionPayload::from_block_unchecked(block_hash, &block);
|
||||
let (version, params) = payload_to_new_payload(
|
||||
payload,
|
||||
sidecar,
|
||||
false,
|
||||
block.header.withdrawals_root,
|
||||
Some(new_payload_version),
|
||||
)?;
|
||||
|
||||
// Save payload to file with version info for replay
|
||||
let payload_path =
|
||||
self.output.join(format!("payload_block_{}.json", block.header.number));
|
||||
let file =
|
||||
GasRampPayloadFile { version: version as u8, block_hash, params: params.clone() };
|
||||
let payload_json = serde_json::to_string_pretty(&file)?;
|
||||
std::fs::write(&payload_path, &payload_json)?;
|
||||
info!(block_number = block.header.number, path = %payload_path.display(), "Saved payload");
|
||||
|
||||
call_new_payload(&provider, version, params).await?;
|
||||
|
||||
let forkchoice_state = ForkchoiceState {
|
||||
head_block_hash: block_hash,
|
||||
safe_block_hash: block_hash,
|
||||
finalized_block_hash: block_hash,
|
||||
};
|
||||
call_forkchoice_updated(&provider, version, forkchoice_state, None).await?;
|
||||
|
||||
parent_header = block.header;
|
||||
parent_hash = block_hash;
|
||||
blocks_processed += 1;
|
||||
}
|
||||
|
||||
let final_gas_limit = parent_header.gas_limit;
|
||||
info!(
|
||||
total_duration=?total_benchmark_duration.elapsed(),
|
||||
blocks_processed,
|
||||
final_gas_limit,
|
||||
"Benchmark complete"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
const fn max_gas_limit_increase(parent_gas_limit: u64) -> u64 {
|
||||
(parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR).saturating_sub(1)
|
||||
}
|
||||
|
||||
const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u64) -> bool {
|
||||
match mode {
|
||||
RampMode::Blocks(target_blocks) => blocks_processed >= target_blocks,
|
||||
RampMode::TargetGasLimit(target) => current_gas_limit >= target,
|
||||
}
|
||||
}
|
||||
829
bin/reth-bench/src/bench/generate_big_block.rs
Normal file
829
bin/reth-bench/src/bench/generate_big_block.rs
Normal file
@@ -0,0 +1,829 @@
|
||||
//! Command for generating large blocks by packing transactions from real blocks.
|
||||
//!
|
||||
//! This command fetches transactions from existing blocks and packs them into a single
|
||||
//! large block using the `testing_buildBlockV1` RPC endpoint.
|
||||
|
||||
use crate::{
|
||||
authenticated_transport::AuthenticatedTransportConnect, bench::helpers::parse_gas_limit,
|
||||
};
|
||||
use alloy_eips::{BlockNumberOrTag, Typed2718};
|
||||
use alloy_primitives::{Bytes, B256};
|
||||
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
|
||||
use alloy_rpc_client::ClientBuilder;
|
||||
use alloy_rpc_types_engine::{
|
||||
ExecutionPayloadEnvelopeV4, ExecutionPayloadEnvelopeV5, ForkchoiceState, JwtSecret,
|
||||
PayloadAttributes,
|
||||
};
|
||||
use alloy_transport::layers::RetryBackoffLayer;
|
||||
use clap::Parser;
|
||||
use eyre::Context;
|
||||
use reqwest::Url;
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_rpc_api::TestingBuildBlockRequestV1;
|
||||
use std::future::Future;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{info, warn};
|
||||
|
||||
/// A single transaction with its gas used and raw encoded bytes.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RawTransaction {
|
||||
/// The actual gas used by the transaction (from receipt).
|
||||
pub gas_used: u64,
|
||||
/// The transaction type (e.g., 3 for EIP-4844 blob txs).
|
||||
pub tx_type: u8,
|
||||
/// The raw RLP-encoded transaction bytes.
|
||||
pub raw: Bytes,
|
||||
}
|
||||
|
||||
/// Abstraction over sources of transactions for big block generation.
|
||||
///
|
||||
/// Implementors provide transactions from different sources (RPC, database, files, etc.)
|
||||
pub trait TransactionSource {
|
||||
/// Fetch transactions from a specific block number.
|
||||
///
|
||||
/// Returns `Ok(None)` if the block doesn't exist.
|
||||
/// Returns `Ok(Some((transactions, gas_used)))` with the block's transactions and total gas.
|
||||
fn fetch_block_transactions(
|
||||
&self,
|
||||
block_number: u64,
|
||||
) -> impl Future<Output = eyre::Result<Option<(Vec<RawTransaction>, u64)>>> + Send;
|
||||
}
|
||||
|
||||
/// RPC-based transaction source that fetches from a remote node.
|
||||
#[derive(Debug)]
|
||||
pub struct RpcTransactionSource {
|
||||
provider: RootProvider<AnyNetwork>,
|
||||
}
|
||||
|
||||
impl RpcTransactionSource {
|
||||
/// Create a new RPC transaction source.
|
||||
pub const fn new(provider: RootProvider<AnyNetwork>) -> Self {
|
||||
Self { provider }
|
||||
}
|
||||
|
||||
/// Create from an RPC URL with retry backoff.
|
||||
pub fn from_url(rpc_url: &str) -> eyre::Result<Self> {
|
||||
let client = ClientBuilder::default()
|
||||
.layer(RetryBackoffLayer::new(10, 800, u64::MAX))
|
||||
.http(rpc_url.parse()?);
|
||||
let provider = RootProvider::<AnyNetwork>::new(client);
|
||||
Ok(Self { provider })
|
||||
}
|
||||
}
|
||||
|
||||
impl TransactionSource for RpcTransactionSource {
|
||||
async fn fetch_block_transactions(
|
||||
&self,
|
||||
block_number: u64,
|
||||
) -> eyre::Result<Option<(Vec<RawTransaction>, u64)>> {
|
||||
// Fetch block and receipts in parallel
|
||||
let (block, receipts) = tokio::try_join!(
|
||||
self.provider.get_block_by_number(block_number.into()).full(),
|
||||
self.provider.get_block_receipts(block_number.into())
|
||||
)?;
|
||||
|
||||
let Some(block) = block else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let Some(receipts) = receipts else {
|
||||
return Err(eyre::eyre!("Receipts not found for block {}", block_number));
|
||||
};
|
||||
|
||||
let block_gas_used = block.header.gas_used;
|
||||
|
||||
// Convert cumulative gas from receipts to per-tx gas_used
|
||||
let mut prev_cumulative = 0u64;
|
||||
let transactions: Vec<RawTransaction> = block
|
||||
.transactions
|
||||
.txns()
|
||||
.zip(receipts.iter())
|
||||
.map(|(tx, receipt)| {
|
||||
let cumulative = receipt.inner.inner.inner.receipt.cumulative_gas_used;
|
||||
let gas_used = cumulative - prev_cumulative;
|
||||
prev_cumulative = cumulative;
|
||||
|
||||
let with_encoded = tx.inner.inner.clone().into_encoded();
|
||||
RawTransaction {
|
||||
gas_used,
|
||||
tx_type: tx.inner.ty(),
|
||||
raw: with_encoded.encoded_bytes().clone(),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(Some((transactions, block_gas_used)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Collects transactions from a source up to a target gas usage.
|
||||
#[derive(Debug)]
|
||||
pub struct TransactionCollector<S> {
|
||||
source: S,
|
||||
target_gas: u64,
|
||||
}
|
||||
|
||||
impl<S: TransactionSource> TransactionCollector<S> {
|
||||
/// Create a new transaction collector.
|
||||
pub const fn new(source: S, target_gas: u64) -> Self {
|
||||
Self { source, target_gas }
|
||||
}
|
||||
|
||||
/// Collect transactions starting from the given block number.
|
||||
///
|
||||
/// Skips blob transactions (type 3) and collects until target gas is reached.
|
||||
/// Returns a `CollectionResult` with transactions, gas info, and next block.
|
||||
pub async fn collect(&self, start_block: u64) -> eyre::Result<CollectionResult> {
|
||||
self.collect_gas(start_block, self.target_gas).await
|
||||
}
|
||||
|
||||
/// Collect transactions up to a specific gas target.
|
||||
///
|
||||
/// This is used both for initial collection and for retry top-ups.
|
||||
pub async fn collect_gas(
|
||||
&self,
|
||||
start_block: u64,
|
||||
gas_target: u64,
|
||||
) -> eyre::Result<CollectionResult> {
|
||||
let mut transactions: Vec<RawTransaction> = Vec::new();
|
||||
let mut total_gas: u64 = 0;
|
||||
let mut current_block = start_block;
|
||||
|
||||
while total_gas < gas_target {
|
||||
let Some((block_txs, _)) = self.source.fetch_block_transactions(current_block).await?
|
||||
else {
|
||||
warn!(block = current_block, "Block not found, stopping");
|
||||
break;
|
||||
};
|
||||
|
||||
for tx in block_txs {
|
||||
// Skip blob transactions (EIP-4844, type 3)
|
||||
if tx.tx_type == 3 {
|
||||
continue;
|
||||
}
|
||||
|
||||
if total_gas + tx.gas_used <= gas_target {
|
||||
total_gas += tx.gas_used;
|
||||
transactions.push(tx);
|
||||
}
|
||||
|
||||
if total_gas >= gas_target {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
current_block += 1;
|
||||
|
||||
// Stop early if remaining gas is under 1M (close enough to target)
|
||||
let remaining_gas = gas_target.saturating_sub(total_gas);
|
||||
if remaining_gas < 1_000_000 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
total_txs = transactions.len(),
|
||||
gas_sent = total_gas,
|
||||
next_block = current_block,
|
||||
"Finished collecting transactions"
|
||||
);
|
||||
|
||||
Ok(CollectionResult { transactions, gas_sent: total_gas, next_block: current_block })
|
||||
}
|
||||
}
|
||||
|
||||
/// `reth bench generate-big-block` command
|
||||
///
|
||||
/// Generates a large block by fetching transactions from existing blocks and packing them
|
||||
/// into a single block using the `testing_buildBlockV1` RPC endpoint.
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Command {
|
||||
/// The RPC URL to use for fetching blocks (can be an external archive node).
|
||||
#[arg(long, value_name = "RPC_URL")]
|
||||
rpc_url: String,
|
||||
|
||||
/// The engine RPC URL (with JWT authentication).
|
||||
#[arg(long, value_name = "ENGINE_RPC_URL", default_value = "http://localhost:8551")]
|
||||
engine_rpc_url: String,
|
||||
|
||||
/// The RPC URL for `testing_buildBlockV1` calls (same node as engine, regular RPC port).
|
||||
#[arg(long, value_name = "TESTING_RPC_URL", default_value = "http://localhost:8545")]
|
||||
testing_rpc_url: String,
|
||||
|
||||
/// Path to the JWT secret file for engine API authentication.
|
||||
#[arg(long, value_name = "JWT_SECRET")]
|
||||
jwt_secret: std::path::PathBuf,
|
||||
|
||||
/// Target gas to pack into the block.
|
||||
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 1G = 1
|
||||
/// billion).
|
||||
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000", value_parser = parse_gas_limit)]
|
||||
target_gas: u64,
|
||||
|
||||
/// Block number to start fetching transactions from (required).
|
||||
///
|
||||
/// This must be the last canonical block BEFORE any gas limit ramping was performed.
|
||||
/// The command collects transactions from historical blocks starting at this number
|
||||
/// to pack into large blocks.
|
||||
///
|
||||
/// How to determine this value:
|
||||
/// - If starting from a fresh node (no gas limit ramp yet): use the current chain tip
|
||||
/// - If gas limit ramping has already been performed: use the block number that was the chain
|
||||
/// tip BEFORE ramping began (you must track this yourself)
|
||||
///
|
||||
/// Using a block after ramping started will cause transaction collection to fail
|
||||
/// because those blocks contain synthetic transactions that cannot be replayed.
|
||||
#[arg(long, value_name = "FROM_BLOCK")]
|
||||
from_block: u64,
|
||||
|
||||
/// Execute the payload (call newPayload + forkchoiceUpdated).
|
||||
/// If false, only builds the payload and prints it.
|
||||
#[arg(long, default_value = "false")]
|
||||
execute: bool,
|
||||
|
||||
/// Number of payloads to generate. Each payload uses the previous as parent.
|
||||
/// When count == 1, the payload is only generated and saved, not executed.
|
||||
/// When count > 1, each payload is executed before building the next.
|
||||
#[arg(long, default_value = "1")]
|
||||
count: u64,
|
||||
|
||||
/// Number of transaction batches to prefetch in background when count > 1.
|
||||
/// Higher values reduce latency but use more memory.
|
||||
#[arg(long, default_value = "4")]
|
||||
prefetch_buffer: usize,
|
||||
|
||||
/// Output directory for generated payloads. Each payload is saved as `payload_block_N.json`.
|
||||
#[arg(long, value_name = "OUTPUT_DIR")]
|
||||
output_dir: std::path::PathBuf,
|
||||
}
|
||||
|
||||
/// A built payload ready for execution.
|
||||
struct BuiltPayload {
|
||||
block_number: u64,
|
||||
envelope: ExecutionPayloadEnvelopeV4,
|
||||
block_hash: B256,
|
||||
timestamp: u64,
|
||||
/// The actual gas used in the built block.
|
||||
gas_used: u64,
|
||||
}
|
||||
|
||||
/// Result of collecting transactions from blocks.
|
||||
#[derive(Debug)]
|
||||
pub struct CollectionResult {
|
||||
/// Collected transactions with their gas info.
|
||||
pub transactions: Vec<RawTransaction>,
|
||||
/// Total gas sent (sum of historical `gas_used` for all collected txs).
|
||||
pub gas_sent: u64,
|
||||
/// Next block number to continue collecting from.
|
||||
pub next_block: u64,
|
||||
}
|
||||
|
||||
/// Constants for retry logic.
|
||||
const MAX_BUILD_RETRIES: u32 = 5;
|
||||
/// Maximum retries for fetching a transaction batch.
|
||||
const MAX_FETCH_RETRIES: u32 = 5;
|
||||
/// Tolerance: if `gas_used` is within 1M of target, don't retry.
|
||||
const MIN_TARGET_SLACK: u64 = 1_000_000;
|
||||
/// Maximum gas to request in retries (10x target as safety cap).
|
||||
const MAX_ADDITIONAL_GAS_MULTIPLIER: u64 = 10;
|
||||
|
||||
/// Fetches a batch of transactions with retry logic.
|
||||
///
|
||||
/// Returns `None` if all retries are exhausted.
|
||||
async fn fetch_batch_with_retry<S: TransactionSource>(
|
||||
collector: &TransactionCollector<S>,
|
||||
block: u64,
|
||||
) -> Option<CollectionResult> {
|
||||
for attempt in 1..=MAX_FETCH_RETRIES {
|
||||
match collector.collect(block).await {
|
||||
Ok(result) => return Some(result),
|
||||
Err(e) => {
|
||||
if attempt == MAX_FETCH_RETRIES {
|
||||
warn!(attempt, error = %e, "Failed to fetch transactions after max retries");
|
||||
return None;
|
||||
}
|
||||
warn!(attempt, error = %e, "Failed to fetch transactions, retrying...");
|
||||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Outcome of a build attempt check.
|
||||
enum RetryOutcome {
|
||||
/// Payload is close enough to target gas.
|
||||
Success,
|
||||
/// Max retries reached, accept what we have.
|
||||
MaxRetries,
|
||||
/// Need more transactions with the specified gas amount.
|
||||
NeedMore(u64),
|
||||
}
|
||||
|
||||
/// Buffer for receiving transaction batches from the fetcher.
|
||||
///
|
||||
/// This abstracts over the channel to allow the main loop to request
|
||||
/// batches on demand, including for retries.
|
||||
struct TxBuffer {
|
||||
receiver: mpsc::Receiver<CollectionResult>,
|
||||
}
|
||||
|
||||
impl TxBuffer {
|
||||
const fn new(receiver: mpsc::Receiver<CollectionResult>) -> Self {
|
||||
Self { receiver }
|
||||
}
|
||||
|
||||
/// Take the next available batch from the fetcher.
|
||||
async fn take_batch(&mut self) -> Option<CollectionResult> {
|
||||
self.receiver.recv().await
|
||||
}
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute the `generate-big-block` command
|
||||
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
|
||||
info!(target_gas = self.target_gas, count = self.count, "Generating big block(s)");
|
||||
|
||||
// Set up authenticated engine provider
|
||||
let jwt =
|
||||
std::fs::read_to_string(&self.jwt_secret).wrap_err("Failed to read JWT secret file")?;
|
||||
let jwt = JwtSecret::from_hex(jwt.trim())?;
|
||||
let auth_url = Url::parse(&self.engine_rpc_url)?;
|
||||
|
||||
info!("Connecting to Engine RPC at {}", auth_url);
|
||||
let auth_transport = AuthenticatedTransportConnect::new(auth_url.clone(), jwt);
|
||||
let auth_client = ClientBuilder::default().connect_with(auth_transport).await?;
|
||||
let auth_provider = RootProvider::<AnyNetwork>::new(auth_client);
|
||||
|
||||
// Set up testing RPC provider (for testing_buildBlockV1)
|
||||
info!("Connecting to Testing RPC at {}", self.testing_rpc_url);
|
||||
let testing_client = ClientBuilder::default()
|
||||
.layer(RetryBackoffLayer::new(10, 800, u64::MAX))
|
||||
.http(self.testing_rpc_url.parse()?);
|
||||
let testing_provider = RootProvider::<AnyNetwork>::new(testing_client);
|
||||
|
||||
// Get the parent block (latest canonical block)
|
||||
info!(endpoint = "engine", method = "eth_getBlockByNumber", block = "latest", "RPC call");
|
||||
let parent_block = auth_provider
|
||||
.get_block_by_number(BlockNumberOrTag::Latest)
|
||||
.await?
|
||||
.ok_or_else(|| eyre::eyre!("Failed to fetch latest block"))?;
|
||||
|
||||
let parent_hash = parent_block.header.hash;
|
||||
let parent_number = parent_block.header.number;
|
||||
let parent_timestamp = parent_block.header.timestamp;
|
||||
|
||||
info!(
|
||||
parent_hash = %parent_hash,
|
||||
parent_number = parent_number,
|
||||
"Using initial parent block"
|
||||
);
|
||||
|
||||
// Create output directory
|
||||
std::fs::create_dir_all(&self.output_dir).wrap_err_with(|| {
|
||||
format!("Failed to create output directory: {:?}", self.output_dir)
|
||||
})?;
|
||||
|
||||
let start_block = self.from_block;
|
||||
|
||||
// Use pipelined execution when generating multiple payloads
|
||||
if self.count > 1 {
|
||||
self.execute_pipelined(
|
||||
&auth_provider,
|
||||
&testing_provider,
|
||||
start_block,
|
||||
parent_hash,
|
||||
parent_timestamp,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
// Single payload - collect transactions and build with retry
|
||||
let tx_source = RpcTransactionSource::from_url(&self.rpc_url)?;
|
||||
let collector = TransactionCollector::new(tx_source, self.target_gas);
|
||||
let result = collector.collect(start_block).await?;
|
||||
|
||||
if result.transactions.is_empty() {
|
||||
return Err(eyre::eyre!("No transactions collected"));
|
||||
}
|
||||
|
||||
self.execute_sequential_with_retry(
|
||||
&auth_provider,
|
||||
&testing_provider,
|
||||
&collector,
|
||||
result,
|
||||
parent_hash,
|
||||
parent_timestamp,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
info!(count = self.count, output_dir = %self.output_dir.display(), "All payloads generated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sequential execution path with retry logic for underfilled payloads.
|
||||
async fn execute_sequential_with_retry<S: TransactionSource>(
|
||||
&self,
|
||||
auth_provider: &RootProvider<AnyNetwork>,
|
||||
testing_provider: &RootProvider<AnyNetwork>,
|
||||
collector: &TransactionCollector<S>,
|
||||
initial_result: CollectionResult,
|
||||
mut parent_hash: B256,
|
||||
mut parent_timestamp: u64,
|
||||
) -> eyre::Result<()> {
|
||||
let mut current_result = initial_result;
|
||||
|
||||
for i in 0..self.count {
|
||||
let built = self
|
||||
.build_with_retry(
|
||||
testing_provider,
|
||||
collector,
|
||||
&mut current_result,
|
||||
i,
|
||||
parent_hash,
|
||||
parent_timestamp,
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.save_payload(&built)?;
|
||||
|
||||
if self.execute || self.count > 1 {
|
||||
info!(payload = i + 1, block_hash = %built.block_hash, gas_used = built.gas_used, "Executing payload (newPayload + FCU)");
|
||||
self.execute_payload_v4(auth_provider, built.envelope, parent_hash).await?;
|
||||
info!(payload = i + 1, "Payload executed successfully");
|
||||
}
|
||||
|
||||
parent_hash = built.block_hash;
|
||||
parent_timestamp = built.timestamp;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Build a payload with retry logic when `gas_used` is below target.
|
||||
///
|
||||
/// Uses the ratio of `gas_used/gas_sent` to estimate how many more transactions
|
||||
/// are needed to hit the target gas.
|
||||
async fn build_with_retry<S: TransactionSource>(
|
||||
&self,
|
||||
testing_provider: &RootProvider<AnyNetwork>,
|
||||
collector: &TransactionCollector<S>,
|
||||
result: &mut CollectionResult,
|
||||
index: u64,
|
||||
parent_hash: B256,
|
||||
parent_timestamp: u64,
|
||||
) -> eyre::Result<BuiltPayload> {
|
||||
for attempt in 1..=MAX_BUILD_RETRIES {
|
||||
let tx_bytes: Vec<Bytes> = result.transactions.iter().map(|t| t.raw.clone()).collect();
|
||||
let gas_sent = result.gas_sent;
|
||||
|
||||
info!(
|
||||
payload = index + 1,
|
||||
attempt,
|
||||
tx_count = tx_bytes.len(),
|
||||
gas_sent,
|
||||
parent_hash = %parent_hash,
|
||||
"Building payload via testing_buildBlockV1"
|
||||
);
|
||||
|
||||
let built = Self::build_payload_static(
|
||||
testing_provider,
|
||||
&tx_bytes,
|
||||
index,
|
||||
parent_hash,
|
||||
parent_timestamp,
|
||||
)
|
||||
.await?;
|
||||
|
||||
match self.check_retry_outcome(&built, index, attempt, gas_sent) {
|
||||
RetryOutcome::Success | RetryOutcome::MaxRetries => return Ok(built),
|
||||
RetryOutcome::NeedMore(additional_gas) => {
|
||||
let additional =
|
||||
collector.collect_gas(result.next_block, additional_gas).await?;
|
||||
result.transactions.extend(additional.transactions);
|
||||
result.gas_sent = result.gas_sent.saturating_add(additional.gas_sent);
|
||||
result.next_block = additional.next_block;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
warn!(payload = index + 1, "Retry loop exited without returning a payload");
|
||||
Err(eyre::eyre!("build_with_retry exhausted retries without result"))
|
||||
}
|
||||
|
||||
/// Pipelined execution - fetches transactions in background, builds with retry.
|
||||
///
|
||||
/// The fetcher continuously produces transaction batches. The main loop consumes them,
|
||||
/// builds payloads with retry logic (requesting more transactions if underfilled),
|
||||
/// and executes each payload before moving to the next.
|
||||
async fn execute_pipelined(
|
||||
&self,
|
||||
auth_provider: &RootProvider<AnyNetwork>,
|
||||
testing_provider: &RootProvider<AnyNetwork>,
|
||||
start_block: u64,
|
||||
initial_parent_hash: B256,
|
||||
initial_parent_timestamp: u64,
|
||||
) -> eyre::Result<()> {
|
||||
// Create channel for transaction batches - fetcher sends CollectionResult
|
||||
let (tx_sender, tx_receiver) = mpsc::channel::<CollectionResult>(self.prefetch_buffer);
|
||||
|
||||
// Spawn background task to continuously fetch transaction batches
|
||||
let rpc_url = self.rpc_url.clone();
|
||||
let target_gas = self.target_gas;
|
||||
|
||||
let fetcher_handle = tokio::spawn(async move {
|
||||
let tx_source = match RpcTransactionSource::from_url(&rpc_url) {
|
||||
Ok(source) => source,
|
||||
Err(e) => {
|
||||
warn!(error = %e, "Failed to create transaction source");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
let collector = TransactionCollector::new(tx_source, target_gas);
|
||||
let mut current_block = start_block;
|
||||
|
||||
while let Some(batch) = fetch_batch_with_retry(&collector, current_block).await {
|
||||
if batch.transactions.is_empty() {
|
||||
info!(block = current_block, "Reached chain tip, stopping fetcher");
|
||||
break;
|
||||
}
|
||||
|
||||
info!(
|
||||
tx_count = batch.transactions.len(),
|
||||
gas_sent = batch.gas_sent,
|
||||
blocks = format!("{}..{}", current_block, batch.next_block),
|
||||
"Fetched transaction batch"
|
||||
);
|
||||
current_block = batch.next_block;
|
||||
|
||||
if tx_sender.send(batch).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Some(current_block)
|
||||
});
|
||||
|
||||
// Transaction buffer: holds transactions from batches + any extras from retries
|
||||
let mut tx_buffer = TxBuffer::new(tx_receiver);
|
||||
|
||||
let mut parent_hash = initial_parent_hash;
|
||||
let mut parent_timestamp = initial_parent_timestamp;
|
||||
|
||||
for i in 0..self.count {
|
||||
// Get initial batch of transactions for this payload
|
||||
let mut result = tx_buffer
|
||||
.take_batch()
|
||||
.await
|
||||
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
|
||||
|
||||
if result.transactions.is_empty() {
|
||||
return Err(eyre::eyre!("No transactions collected for payload {}", i + 1));
|
||||
}
|
||||
|
||||
// Build with retry - may need to request more transactions
|
||||
let built = self
|
||||
.build_with_retry_buffered(
|
||||
testing_provider,
|
||||
&mut tx_buffer,
|
||||
&mut result,
|
||||
i,
|
||||
parent_hash,
|
||||
parent_timestamp,
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.save_payload(&built)?;
|
||||
|
||||
let current_block_hash = built.block_hash;
|
||||
let current_timestamp = built.timestamp;
|
||||
|
||||
// Execute payload
|
||||
info!(payload = i + 1, block_hash = %current_block_hash, gas_used = built.gas_used, "Executing payload (newPayload + FCU)");
|
||||
self.execute_payload_v4(auth_provider, built.envelope, parent_hash).await?;
|
||||
info!(payload = i + 1, "Payload executed successfully");
|
||||
|
||||
parent_hash = current_block_hash;
|
||||
parent_timestamp = current_timestamp;
|
||||
}
|
||||
|
||||
// Clean up the fetcher task
|
||||
drop(tx_buffer);
|
||||
let _ = fetcher_handle.await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Build a payload with retry logic, using the buffered transaction source.
|
||||
async fn build_with_retry_buffered(
|
||||
&self,
|
||||
testing_provider: &RootProvider<AnyNetwork>,
|
||||
tx_buffer: &mut TxBuffer,
|
||||
result: &mut CollectionResult,
|
||||
index: u64,
|
||||
parent_hash: B256,
|
||||
parent_timestamp: u64,
|
||||
) -> eyre::Result<BuiltPayload> {
|
||||
for attempt in 1..=MAX_BUILD_RETRIES {
|
||||
let tx_bytes: Vec<Bytes> = result.transactions.iter().map(|t| t.raw.clone()).collect();
|
||||
let gas_sent = result.gas_sent;
|
||||
|
||||
info!(
|
||||
payload = index + 1,
|
||||
attempt,
|
||||
tx_count = tx_bytes.len(),
|
||||
gas_sent,
|
||||
parent_hash = %parent_hash,
|
||||
"Building payload via testing_buildBlockV1"
|
||||
);
|
||||
|
||||
let built = Self::build_payload_static(
|
||||
testing_provider,
|
||||
&tx_bytes,
|
||||
index,
|
||||
parent_hash,
|
||||
parent_timestamp,
|
||||
)
|
||||
.await?;
|
||||
|
||||
match self.check_retry_outcome(&built, index, attempt, gas_sent) {
|
||||
RetryOutcome::Success | RetryOutcome::MaxRetries => return Ok(built),
|
||||
RetryOutcome::NeedMore(additional_gas) => {
|
||||
let mut collected_gas = 0u64;
|
||||
while collected_gas < additional_gas {
|
||||
if let Some(batch) = tx_buffer.take_batch().await {
|
||||
collected_gas += batch.gas_sent;
|
||||
result.transactions.extend(batch.transactions);
|
||||
result.gas_sent = result.gas_sent.saturating_add(batch.gas_sent);
|
||||
result.next_block = batch.next_block;
|
||||
} else {
|
||||
warn!("Transaction fetcher exhausted, proceeding with available transactions");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
warn!(payload = index + 1, "Retry loop exited without returning a payload");
|
||||
Err(eyre::eyre!("build_with_retry_buffered exhausted retries without result"))
|
||||
}
|
||||
|
||||
/// Determines the outcome of a build attempt.
|
||||
fn check_retry_outcome(
|
||||
&self,
|
||||
built: &BuiltPayload,
|
||||
index: u64,
|
||||
attempt: u32,
|
||||
gas_sent: u64,
|
||||
) -> RetryOutcome {
|
||||
let gas_used = built.gas_used;
|
||||
|
||||
if gas_used + MIN_TARGET_SLACK >= self.target_gas {
|
||||
info!(
|
||||
payload = index + 1,
|
||||
gas_used,
|
||||
target_gas = self.target_gas,
|
||||
attempts = attempt,
|
||||
"Payload built successfully"
|
||||
);
|
||||
return RetryOutcome::Success;
|
||||
}
|
||||
|
||||
if attempt == MAX_BUILD_RETRIES {
|
||||
warn!(
|
||||
payload = index + 1,
|
||||
gas_used,
|
||||
target_gas = self.target_gas,
|
||||
gas_sent,
|
||||
"Underfilled after max retries, accepting payload"
|
||||
);
|
||||
return RetryOutcome::MaxRetries;
|
||||
}
|
||||
|
||||
if gas_used == 0 {
|
||||
warn!(
|
||||
payload = index + 1,
|
||||
"Zero gas used in payload, requesting fixed chunk of additional transactions"
|
||||
);
|
||||
return RetryOutcome::NeedMore(self.target_gas);
|
||||
}
|
||||
|
||||
let gas_sent_needed_total =
|
||||
(self.target_gas as u128 * gas_sent as u128).div_ceil(gas_used as u128) as u64;
|
||||
let additional = gas_sent_needed_total.saturating_sub(gas_sent);
|
||||
let additional = additional.min(self.target_gas * MAX_ADDITIONAL_GAS_MULTIPLIER);
|
||||
|
||||
if additional == 0 {
|
||||
info!(
|
||||
payload = index + 1,
|
||||
gas_used,
|
||||
target_gas = self.target_gas,
|
||||
"No additional transactions needed based on ratio"
|
||||
);
|
||||
return RetryOutcome::Success;
|
||||
}
|
||||
|
||||
let ratio = gas_used as f64 / gas_sent as f64;
|
||||
info!(
|
||||
payload = index + 1,
|
||||
gas_used,
|
||||
gas_sent,
|
||||
ratio = format!("{:.4}", ratio),
|
||||
additional_gas = additional,
|
||||
"Underfilled, collecting more transactions for retry"
|
||||
);
|
||||
RetryOutcome::NeedMore(additional)
|
||||
}
|
||||
|
||||
/// Build a single payload via `testing_buildBlockV1`.
|
||||
async fn build_payload_static(
|
||||
testing_provider: &RootProvider<AnyNetwork>,
|
||||
transactions: &[Bytes],
|
||||
index: u64,
|
||||
parent_hash: B256,
|
||||
parent_timestamp: u64,
|
||||
) -> eyre::Result<BuiltPayload> {
|
||||
let request = TestingBuildBlockRequestV1 {
|
||||
parent_block_hash: parent_hash,
|
||||
payload_attributes: PayloadAttributes {
|
||||
timestamp: parent_timestamp + 12,
|
||||
prev_randao: B256::ZERO,
|
||||
suggested_fee_recipient: alloy_primitives::Address::ZERO,
|
||||
withdrawals: Some(vec![]),
|
||||
parent_beacon_block_root: Some(B256::ZERO),
|
||||
},
|
||||
transactions: transactions.to_vec(),
|
||||
extra_data: None,
|
||||
};
|
||||
|
||||
let total_tx_bytes: usize = transactions.iter().map(|tx| tx.len()).sum();
|
||||
info!(
|
||||
payload = index + 1,
|
||||
tx_count = transactions.len(),
|
||||
total_tx_bytes = total_tx_bytes,
|
||||
parent_hash = %parent_hash,
|
||||
"Sending to testing_buildBlockV1"
|
||||
);
|
||||
let envelope: ExecutionPayloadEnvelopeV5 =
|
||||
testing_provider.client().request("testing_buildBlockV1", [request]).await?;
|
||||
|
||||
let v4_envelope = envelope.try_into_v4()?;
|
||||
|
||||
let inner = &v4_envelope.envelope_inner.execution_payload.payload_inner.payload_inner;
|
||||
let block_hash = inner.block_hash;
|
||||
let block_number = inner.block_number;
|
||||
let timestamp = inner.timestamp;
|
||||
let gas_used = inner.gas_used;
|
||||
|
||||
Ok(BuiltPayload { block_number, envelope: v4_envelope, block_hash, timestamp, gas_used })
|
||||
}
|
||||
|
||||
/// Save a payload to disk.
|
||||
fn save_payload(&self, payload: &BuiltPayload) -> eyre::Result<()> {
|
||||
let filename = format!("payload_block_{}.json", payload.block_number);
|
||||
let filepath = self.output_dir.join(&filename);
|
||||
let json = serde_json::to_string_pretty(&payload.envelope)?;
|
||||
std::fs::write(&filepath, &json)
|
||||
.wrap_err_with(|| format!("Failed to write payload to {:?}", filepath))?;
|
||||
info!(block_number = payload.block_number, block_hash = %payload.block_hash, path = %filepath.display(), "Payload saved");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn execute_payload_v4(
|
||||
&self,
|
||||
provider: &RootProvider<AnyNetwork>,
|
||||
envelope: ExecutionPayloadEnvelopeV4,
|
||||
parent_hash: B256,
|
||||
) -> eyre::Result<()> {
|
||||
let block_hash =
|
||||
envelope.envelope_inner.execution_payload.payload_inner.payload_inner.block_hash;
|
||||
|
||||
let status = provider
|
||||
.new_payload_v4(
|
||||
envelope.envelope_inner.execution_payload,
|
||||
vec![],
|
||||
B256::ZERO,
|
||||
envelope.execution_requests.to_vec(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
if !status.is_valid() {
|
||||
return Err(eyre::eyre!("Payload rejected: {:?}", status));
|
||||
}
|
||||
|
||||
let fcu_state = ForkchoiceState {
|
||||
head_block_hash: block_hash,
|
||||
safe_block_hash: parent_hash,
|
||||
finalized_block_hash: parent_hash,
|
||||
};
|
||||
|
||||
let fcu_result = provider.fork_choice_updated_v3(fcu_state, None).await?;
|
||||
|
||||
if !fcu_result.is_valid() {
|
||||
return Err(eyre::eyre!("FCU rejected: {:?}", fcu_result));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
293
bin/reth-bench/src/bench/helpers.rs
Normal file
293
bin/reth-bench/src/bench/helpers.rs
Normal file
@@ -0,0 +1,293 @@
|
||||
//! Common helpers for reth-bench commands.
|
||||
|
||||
use crate::valid_payload::call_forkchoice_updated;
|
||||
use eyre::Result;
|
||||
use std::io::{BufReader, Read};
|
||||
|
||||
/// Read input from either a file path or stdin.
|
||||
pub(crate) fn read_input(path: Option<&str>) -> Result<String> {
|
||||
Ok(match path {
|
||||
Some(path) => reth_fs_util::read_to_string(path)?,
|
||||
None => String::from_utf8(
|
||||
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Load JWT secret from either a file or use the provided string directly.
|
||||
pub(crate) fn load_jwt_secret(jwt_secret: Option<&str>) -> Result<Option<String>> {
|
||||
match jwt_secret {
|
||||
Some(secret) => {
|
||||
// Try to read as file first
|
||||
match std::fs::read_to_string(secret) {
|
||||
Ok(contents) => Ok(Some(contents.trim().to_string())),
|
||||
// If file read fails, use the string directly
|
||||
Err(_) => Ok(Some(secret.to_string())),
|
||||
}
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
|
||||
///
|
||||
/// Examples: "30000000", "30M", "1G", "2G"
|
||||
pub(crate) fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
|
||||
let s = s.trim();
|
||||
if s.is_empty() {
|
||||
return Err(eyre::eyre!("empty value"));
|
||||
}
|
||||
|
||||
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
|
||||
(prefix, 1_000_000_000u64)
|
||||
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
|
||||
(prefix, 1_000_000u64)
|
||||
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
|
||||
(prefix, 1_000u64)
|
||||
} else {
|
||||
(s, 1u64)
|
||||
};
|
||||
|
||||
let base: u64 = num_str.trim().parse()?;
|
||||
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
|
||||
}
|
||||
use alloy_consensus::Header;
|
||||
use alloy_eips::eip4844::kzg_to_versioned_hash;
|
||||
use alloy_primitives::{Address, B256};
|
||||
use alloy_provider::{ext::EngineApi, network::AnyNetwork, RootProvider};
|
||||
use alloy_rpc_types_engine::{
|
||||
CancunPayloadFields, ExecutionPayload, ExecutionPayloadSidecar, ForkchoiceState,
|
||||
PayloadAttributes, PayloadId, PraguePayloadFields,
|
||||
};
|
||||
use eyre::OptionExt;
|
||||
use reth_chainspec::{ChainSpec, EthereumHardforks};
|
||||
use reth_node_api::EngineApiMessageVersion;
|
||||
use tracing::debug;
|
||||
|
||||
/// Prepared payload request data for triggering block building.
|
||||
pub(crate) struct PayloadRequest {
|
||||
/// The payload attributes for the new block.
|
||||
pub(crate) attributes: PayloadAttributes,
|
||||
/// The forkchoice state pointing to the parent block.
|
||||
pub(crate) forkchoice_state: ForkchoiceState,
|
||||
/// The engine API version for FCU calls.
|
||||
pub(crate) fcu_version: EngineApiMessageVersion,
|
||||
/// The getPayload version to use (1-5).
|
||||
pub(crate) get_payload_version: u8,
|
||||
/// The newPayload version to use.
|
||||
pub(crate) new_payload_version: EngineApiMessageVersion,
|
||||
}
|
||||
|
||||
/// Prepare payload attributes and forkchoice state for a new block.
|
||||
pub(crate) fn prepare_payload_request(
|
||||
chain_spec: &ChainSpec,
|
||||
timestamp: u64,
|
||||
parent_hash: B256,
|
||||
) -> PayloadRequest {
|
||||
let shanghai_active = chain_spec.is_shanghai_active_at_timestamp(timestamp);
|
||||
let cancun_active = chain_spec.is_cancun_active_at_timestamp(timestamp);
|
||||
let prague_active = chain_spec.is_prague_active_at_timestamp(timestamp);
|
||||
let osaka_active = chain_spec.is_osaka_active_at_timestamp(timestamp);
|
||||
|
||||
// FCU version: V3 for Cancun+Prague+Osaka, V2 for Shanghai, V1 otherwise
|
||||
let fcu_version = if cancun_active {
|
||||
EngineApiMessageVersion::V3
|
||||
} else if shanghai_active {
|
||||
EngineApiMessageVersion::V2
|
||||
} else {
|
||||
EngineApiMessageVersion::V1
|
||||
};
|
||||
|
||||
// getPayload version: 5 for Osaka, 4 for Prague, 3 for Cancun, 2 for Shanghai, 1 otherwise
|
||||
// newPayload version: 4 for Prague+Osaka (no V5), 3 for Cancun, 2 for Shanghai, 1 otherwise
|
||||
let (get_payload_version, new_payload_version) = if osaka_active {
|
||||
(5, EngineApiMessageVersion::V4) // Osaka uses getPayloadV5 but newPayloadV4
|
||||
} else if prague_active {
|
||||
(4, EngineApiMessageVersion::V4)
|
||||
} else if cancun_active {
|
||||
(3, EngineApiMessageVersion::V3)
|
||||
} else if shanghai_active {
|
||||
(2, EngineApiMessageVersion::V2)
|
||||
} else {
|
||||
(1, EngineApiMessageVersion::V1)
|
||||
};
|
||||
|
||||
PayloadRequest {
|
||||
attributes: PayloadAttributes {
|
||||
timestamp,
|
||||
prev_randao: B256::ZERO,
|
||||
suggested_fee_recipient: Address::ZERO,
|
||||
withdrawals: shanghai_active.then(Vec::new),
|
||||
parent_beacon_block_root: cancun_active.then_some(B256::ZERO),
|
||||
},
|
||||
forkchoice_state: ForkchoiceState {
|
||||
head_block_hash: parent_hash,
|
||||
safe_block_hash: parent_hash,
|
||||
finalized_block_hash: parent_hash,
|
||||
},
|
||||
fcu_version,
|
||||
get_payload_version,
|
||||
new_payload_version,
|
||||
}
|
||||
}
|
||||
|
||||
/// Trigger payload building via FCU and retrieve the built payload.
|
||||
///
|
||||
/// This sends a forkchoiceUpdated with payload attributes to start building,
|
||||
/// then calls getPayload to retrieve the result.
|
||||
pub(crate) async fn build_payload(
|
||||
provider: &RootProvider<AnyNetwork>,
|
||||
request: PayloadRequest,
|
||||
) -> eyre::Result<(ExecutionPayload, ExecutionPayloadSidecar)> {
|
||||
let fcu_result = call_forkchoice_updated(
|
||||
provider,
|
||||
request.fcu_version,
|
||||
request.forkchoice_state,
|
||||
Some(request.attributes.clone()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let payload_id =
|
||||
fcu_result.payload_id.ok_or_eyre("Payload builder did not return a payload id")?;
|
||||
|
||||
get_payload_with_sidecar(
|
||||
provider,
|
||||
request.get_payload_version,
|
||||
payload_id,
|
||||
request.attributes.parent_beacon_block_root,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Convert an RPC block to a consensus header and block hash.
|
||||
pub(crate) fn rpc_block_to_header(block: alloy_provider::network::AnyRpcBlock) -> (Header, B256) {
|
||||
let block_hash = block.header.hash;
|
||||
let header = block.header.inner.clone().into_header_with_defaults();
|
||||
(header, block_hash)
|
||||
}
|
||||
|
||||
/// Compute versioned hashes from KZG commitments.
|
||||
fn versioned_hashes_from_commitments(
|
||||
commitments: &[alloy_primitives::FixedBytes<48>],
|
||||
) -> Vec<B256> {
|
||||
commitments.iter().map(|c| kzg_to_versioned_hash(c.as_ref())).collect()
|
||||
}
|
||||
|
||||
/// Fetch an execution payload using the appropriate engine API version.
|
||||
pub(crate) async fn get_payload_with_sidecar(
|
||||
provider: &RootProvider<AnyNetwork>,
|
||||
version: u8,
|
||||
payload_id: PayloadId,
|
||||
parent_beacon_block_root: Option<B256>,
|
||||
) -> eyre::Result<(ExecutionPayload, ExecutionPayloadSidecar)> {
|
||||
debug!(get_payload_version = ?version, ?payload_id, "Sending getPayload");
|
||||
|
||||
match version {
|
||||
1 => {
|
||||
let payload = provider.get_payload_v1(payload_id).await?;
|
||||
Ok((ExecutionPayload::V1(payload), ExecutionPayloadSidecar::none()))
|
||||
}
|
||||
2 => {
|
||||
let envelope = provider.get_payload_v2(payload_id).await?;
|
||||
let payload = match envelope.execution_payload {
|
||||
alloy_rpc_types_engine::ExecutionPayloadFieldV2::V1(p) => ExecutionPayload::V1(p),
|
||||
alloy_rpc_types_engine::ExecutionPayloadFieldV2::V2(p) => ExecutionPayload::V2(p),
|
||||
};
|
||||
Ok((payload, ExecutionPayloadSidecar::none()))
|
||||
}
|
||||
3 => {
|
||||
let envelope = provider.get_payload_v3(payload_id).await?;
|
||||
let versioned_hashes =
|
||||
versioned_hashes_from_commitments(&envelope.blobs_bundle.commitments);
|
||||
let cancun_fields = CancunPayloadFields {
|
||||
parent_beacon_block_root: parent_beacon_block_root
|
||||
.ok_or_eyre("parent_beacon_block_root required for V3")?,
|
||||
versioned_hashes,
|
||||
};
|
||||
Ok((
|
||||
ExecutionPayload::V3(envelope.execution_payload),
|
||||
ExecutionPayloadSidecar::v3(cancun_fields),
|
||||
))
|
||||
}
|
||||
4 => {
|
||||
let envelope = provider.get_payload_v4(payload_id).await?;
|
||||
let versioned_hashes = versioned_hashes_from_commitments(
|
||||
&envelope.envelope_inner.blobs_bundle.commitments,
|
||||
);
|
||||
let cancun_fields = CancunPayloadFields {
|
||||
parent_beacon_block_root: parent_beacon_block_root
|
||||
.ok_or_eyre("parent_beacon_block_root required for V4")?,
|
||||
versioned_hashes,
|
||||
};
|
||||
let prague_fields = PraguePayloadFields::new(envelope.execution_requests);
|
||||
Ok((
|
||||
ExecutionPayload::V3(envelope.envelope_inner.execution_payload),
|
||||
ExecutionPayloadSidecar::v4(cancun_fields, prague_fields),
|
||||
))
|
||||
}
|
||||
5 => {
|
||||
// V5 (Osaka) - use raw request since alloy doesn't have get_payload_v5 yet
|
||||
let envelope = provider.get_payload_v5(payload_id).await?;
|
||||
let versioned_hashes =
|
||||
versioned_hashes_from_commitments(&envelope.blobs_bundle.commitments);
|
||||
let cancun_fields = CancunPayloadFields {
|
||||
parent_beacon_block_root: parent_beacon_block_root
|
||||
.ok_or_eyre("parent_beacon_block_root required for V5")?,
|
||||
versioned_hashes,
|
||||
};
|
||||
let prague_fields = PraguePayloadFields::new(envelope.execution_requests);
|
||||
Ok((
|
||||
ExecutionPayload::V3(envelope.execution_payload),
|
||||
ExecutionPayloadSidecar::v4(cancun_fields, prague_fields),
|
||||
))
|
||||
}
|
||||
_ => panic!("This tool does not support getPayload versions past v5"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_plain_number() {
|
||||
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
|
||||
assert_eq!(parse_gas_limit("1").unwrap(), 1);
|
||||
assert_eq!(parse_gas_limit("0").unwrap(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_k_suffix() {
|
||||
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
|
||||
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
|
||||
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_m_suffix() {
|
||||
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
|
||||
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
|
||||
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_g_suffix() {
|
||||
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
|
||||
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
|
||||
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_with_whitespace() {
|
||||
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
|
||||
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_errors() {
|
||||
assert!(parse_gas_limit("").is_err());
|
||||
assert!(parse_gas_limit("abc").is_err());
|
||||
assert!(parse_gas_limit("G").is_err());
|
||||
assert!(parse_gas_limit("-1G").is_err());
|
||||
}
|
||||
}
|
||||
@@ -6,9 +6,18 @@ use reth_node_core::args::LogArgs;
|
||||
use reth_tracing::FileWorkerGuard;
|
||||
|
||||
mod context;
|
||||
mod gas_limit_ramp;
|
||||
mod generate_big_block;
|
||||
pub(crate) mod helpers;
|
||||
pub use generate_big_block::{
|
||||
RawTransaction, RpcTransactionSource, TransactionCollector, TransactionSource,
|
||||
};
|
||||
mod new_payload_fcu;
|
||||
mod new_payload_only;
|
||||
mod output;
|
||||
mod persistence_waiter;
|
||||
mod replay_payloads;
|
||||
mod send_invalid_payload;
|
||||
mod send_payload;
|
||||
|
||||
/// `reth bench` command
|
||||
@@ -27,6 +36,9 @@ pub enum Subcommands {
|
||||
/// Benchmark which calls `newPayload`, then `forkchoiceUpdated`.
|
||||
NewPayloadFcu(new_payload_fcu::Command),
|
||||
|
||||
/// Benchmark which builds empty blocks with a ramped gas limit.
|
||||
GasLimitRamp(gas_limit_ramp::Command),
|
||||
|
||||
/// Benchmark which only calls subsequent `newPayload` calls.
|
||||
NewPayloadOnly(new_payload_only::Command),
|
||||
|
||||
@@ -41,6 +53,41 @@ pub enum Subcommands {
|
||||
/// `cast block latest --full --json | reth-bench send-payload --rpc-url localhost:5000
|
||||
/// --jwt-secret $(cat ~/.local/share/reth/mainnet/jwt.hex)`
|
||||
SendPayload(send_payload::Command),
|
||||
|
||||
/// Generate a large block by packing transactions from existing blocks.
|
||||
///
|
||||
/// This command fetches transactions from real blocks and packs them into a single
|
||||
/// block using the `testing_buildBlockV1` RPC endpoint.
|
||||
///
|
||||
/// Example:
|
||||
///
|
||||
/// `reth-bench generate-big-block --rpc-url http://localhost:8545 --engine-rpc-url
|
||||
/// http://localhost:8551 --jwt-secret ~/.local/share/reth/mainnet/jwt.hex --target-gas
|
||||
/// 30000000`
|
||||
GenerateBigBlock(generate_big_block::Command),
|
||||
|
||||
/// Replay pre-generated payloads from a directory.
|
||||
///
|
||||
/// This command reads payload files from a previous `generate-big-block` run and replays
|
||||
/// them in sequence using `newPayload` followed by `forkchoiceUpdated`.
|
||||
///
|
||||
/// Example:
|
||||
///
|
||||
/// `reth-bench replay-payloads --payload-dir ./payloads --engine-rpc-url
|
||||
/// http://localhost:8551 --jwt-secret ~/.local/share/reth/mainnet/jwt.hex`
|
||||
ReplayPayloads(replay_payloads::Command),
|
||||
|
||||
/// Generate and send an invalid `engine_newPayload` request for testing.
|
||||
///
|
||||
/// Takes a valid block and modifies fields to make it invalid, allowing you to test
|
||||
/// Engine API rejection behavior. Block hash is recalculated after modifications
|
||||
/// unless `--invalid-block-hash` or `--skip-hash-recalc` is used.
|
||||
///
|
||||
/// Example:
|
||||
///
|
||||
/// `cast block latest --full --json | reth-bench send-invalid-payload --rpc-url localhost:5000
|
||||
/// --jwt-secret $(cat ~/.local/share/reth/mainnet/jwt.hex) --invalid-state-root`
|
||||
SendInvalidPayload(Box<send_invalid_payload::Command>),
|
||||
}
|
||||
|
||||
impl BenchmarkCommand {
|
||||
@@ -51,8 +98,12 @@ impl BenchmarkCommand {
|
||||
|
||||
match self.command {
|
||||
Subcommands::NewPayloadFcu(command) => command.execute(ctx).await,
|
||||
Subcommands::GasLimitRamp(command) => command.execute(ctx).await,
|
||||
Subcommands::NewPayloadOnly(command) => command.execute(ctx).await,
|
||||
Subcommands::SendPayload(command) => command.execute(ctx).await,
|
||||
Subcommands::GenerateBigBlock(command) => command.execute(ctx).await,
|
||||
Subcommands::ReplayPayloads(command) => command.execute(ctx).await,
|
||||
Subcommands::SendInvalidPayload(command) => (*command).execute(ctx).await,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -13,32 +13,25 @@ use crate::{
|
||||
bench::{
|
||||
context::BenchContext,
|
||||
output::{
|
||||
CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow, COMBINED_OUTPUT_SUFFIX,
|
||||
GAS_OUTPUT_SUFFIX,
|
||||
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
|
||||
},
|
||||
persistence_waiter::{
|
||||
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
|
||||
PERSISTENCE_CHECKPOINT_TIMEOUT,
|
||||
},
|
||||
},
|
||||
valid_payload::{block_to_new_payload, call_forkchoice_updated, call_new_payload},
|
||||
};
|
||||
use alloy_eips::BlockNumHash;
|
||||
use alloy_network::Ethereum;
|
||||
use alloy_provider::{Provider, RootProvider};
|
||||
use alloy_pubsub::SubscriptionStream;
|
||||
use alloy_rpc_client::RpcClient;
|
||||
use alloy_provider::Provider;
|
||||
use alloy_rpc_types_engine::ForkchoiceState;
|
||||
use alloy_transport_ws::WsConnect;
|
||||
use clap::Parser;
|
||||
use csv::Writer;
|
||||
use eyre::{Context, OptionExt};
|
||||
use futures::StreamExt;
|
||||
use humantime::parse_duration;
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
|
||||
use reth_node_core::args::BenchmarkArgs;
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::{debug, info};
|
||||
use url::Url;
|
||||
|
||||
const PERSISTENCE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
/// `reth benchmark new-payload-fcu` command
|
||||
#[derive(Debug, Parser)]
|
||||
@@ -107,7 +100,11 @@ impl Command {
|
||||
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
|
||||
(Some(duration), _) => Some(PersistenceWaiter::with_duration(duration)),
|
||||
(None, true) => {
|
||||
let sub = self.setup_persistence_subscription().await?;
|
||||
let ws_url = derive_ws_rpc_url(
|
||||
self.benchmark.ws_rpc_url.as_deref(),
|
||||
&self.benchmark.engine_rpc_url,
|
||||
)?;
|
||||
let sub = setup_persistence_subscription(ws_url).await?;
|
||||
Some(PersistenceWaiter::with_subscription(
|
||||
sub,
|
||||
self.persistence_threshold,
|
||||
@@ -123,6 +120,7 @@ impl Command {
|
||||
auth_provider,
|
||||
mut next_block,
|
||||
is_optimism,
|
||||
..
|
||||
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
|
||||
|
||||
let buffer_size = self.rpc_block_buffer_size;
|
||||
@@ -188,6 +186,7 @@ impl Command {
|
||||
result
|
||||
} {
|
||||
let gas_used = block.header.gas_used;
|
||||
let gas_limit = block.header.gas_limit;
|
||||
let block_number = block.header.number;
|
||||
let transaction_count = block.transactions.len() as u64;
|
||||
|
||||
@@ -211,6 +210,7 @@ impl Command {
|
||||
let fcu_latency = total_latency - new_payload_result.latency;
|
||||
let combined_result = CombinedResult {
|
||||
block_number,
|
||||
gas_limit,
|
||||
transaction_count,
|
||||
new_payload_result,
|
||||
fcu_latency,
|
||||
@@ -240,314 +240,26 @@ impl Command {
|
||||
// since the benchmark goal is measuring Ggas/s of newPayload/FCU, not persistence.
|
||||
drop(waiter);
|
||||
|
||||
let (gas_output_results, combined_results): (_, Vec<CombinedResult>) =
|
||||
let (gas_output_results, combined_results): (Vec<TotalGasRow>, Vec<CombinedResult>) =
|
||||
results.into_iter().unzip();
|
||||
|
||||
// Write CSV output files
|
||||
if let Some(ref path) = self.benchmark.output {
|
||||
let output_path = path.join(COMBINED_OUTPUT_SUFFIX);
|
||||
info!("Writing engine api call latency output to file: {:?}", output_path);
|
||||
let mut writer = Writer::from_path(&output_path)?;
|
||||
for result in combined_results {
|
||||
writer.serialize(result)?;
|
||||
}
|
||||
writer.flush()?;
|
||||
|
||||
let output_path = path.join(GAS_OUTPUT_SUFFIX);
|
||||
info!("Writing total gas output to file: {:?}", output_path);
|
||||
let mut writer = Writer::from_path(&output_path)?;
|
||||
for row in &gas_output_results {
|
||||
writer.serialize(row)?;
|
||||
}
|
||||
writer.flush()?;
|
||||
|
||||
info!("Finished writing benchmark output files to {:?}.", path);
|
||||
write_benchmark_results(path, &gas_output_results, &combined_results)?;
|
||||
}
|
||||
|
||||
let gas_output = TotalGasOutput::new(gas_output_results)?;
|
||||
let gas_output =
|
||||
TotalGasOutput::with_combined_results(gas_output_results, &combined_results)?;
|
||||
|
||||
info!(
|
||||
total_duration=?gas_output.total_duration,
|
||||
total_gas_used=?gas_output.total_gas_used,
|
||||
blocks_processed=?gas_output.blocks_processed,
|
||||
"Total Ggas/s: {:.4}",
|
||||
gas_output.total_gigagas_per_second()
|
||||
total_gas_used = gas_output.total_gas_used,
|
||||
total_duration = ?gas_output.total_duration,
|
||||
execution_duration = ?gas_output.execution_duration,
|
||||
blocks_processed = gas_output.blocks_processed,
|
||||
wall_clock_ggas_per_second = format_args!("{:.4}", gas_output.total_gigagas_per_second()),
|
||||
execution_ggas_per_second = format_args!("{:.4}", gas_output.execution_gigagas_per_second()),
|
||||
"Benchmark complete"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns the websocket RPC URL used for the persistence subscription.
|
||||
///
|
||||
/// Preference:
|
||||
/// - If `--ws-rpc-url` is provided, use it directly.
|
||||
/// - Otherwise, derive a WS RPC URL from `--engine-rpc-url`.
|
||||
///
|
||||
/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
|
||||
/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
|
||||
/// Since `BenchmarkArgs` only has the engine URL by default, we convert the scheme
|
||||
/// (http→ws, https→wss) and force the port to 8546.
|
||||
fn derive_ws_rpc_url(&self) -> eyre::Result<Url> {
|
||||
if let Some(ref ws_url) = self.benchmark.ws_rpc_url {
|
||||
let parsed: Url = ws_url
|
||||
.parse()
|
||||
.wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
|
||||
info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
|
||||
Ok(parsed)
|
||||
} else {
|
||||
let derived = engine_url_to_ws_url(&self.benchmark.engine_rpc_url)?;
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
engine_url = %self.benchmark.engine_rpc_url,
|
||||
%derived,
|
||||
"Derived WebSocket RPC URL from engine RPC URL"
|
||||
);
|
||||
Ok(derived)
|
||||
}
|
||||
}
|
||||
|
||||
/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
|
||||
async fn setup_persistence_subscription(&self) -> eyre::Result<PersistenceSubscription> {
|
||||
let ws_url = self.derive_ws_rpc_url()?;
|
||||
|
||||
info!("Connecting to WebSocket at {} for persistence subscription", ws_url);
|
||||
|
||||
let ws_connect = WsConnect::new(ws_url.to_string());
|
||||
let client = RpcClient::connect_pubsub(ws_connect)
|
||||
.await
|
||||
.wrap_err("Failed to connect to WebSocket RPC endpoint")?;
|
||||
let provider: RootProvider<Ethereum> = RootProvider::new(client);
|
||||
|
||||
let subscription = provider
|
||||
.subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
|
||||
.await
|
||||
.wrap_err("Failed to subscribe to persistence notifications")?;
|
||||
|
||||
info!("Subscribed to persistence notifications");
|
||||
|
||||
Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts an engine API URL to the default RPC websocket URL.
|
||||
///
|
||||
/// Transformations:
|
||||
/// - `http` → `ws`
|
||||
/// - `https` → `wss`
|
||||
/// - `ws` / `wss` keep their scheme
|
||||
/// - Port is always set to `8546`, reth's default RPC websocket port.
|
||||
///
|
||||
/// This is used when we only know the engine API URL (typically `:8551`) but
|
||||
/// need to connect to the node's WS RPC endpoint for persistence events.
|
||||
fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
|
||||
let url: Url = engine_url
|
||||
.parse()
|
||||
.wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
|
||||
|
||||
let mut ws_url = url.clone();
|
||||
|
||||
match ws_url.scheme() {
|
||||
"http" => ws_url
|
||||
.set_scheme("ws")
|
||||
.map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
|
||||
"https" => ws_url
|
||||
.set_scheme("wss")
|
||||
.map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
|
||||
"ws" | "wss" => {}
|
||||
scheme => {
|
||||
return Err(eyre::eyre!(
|
||||
"Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
ws_url.set_port(Some(8546)).map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
|
||||
|
||||
Ok(ws_url)
|
||||
}
|
||||
|
||||
/// Waits until the persistence subscription reports that `target` has been persisted.
|
||||
///
|
||||
/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
|
||||
/// - the subscription stream ends unexpectedly, or
|
||||
/// - `timeout` elapses before `target` is observed.
|
||||
async fn wait_for_persistence(
|
||||
stream: &mut SubscriptionStream<BlockNumHash>,
|
||||
target: u64,
|
||||
last_persisted: &mut u64,
|
||||
timeout: Duration,
|
||||
) -> eyre::Result<()> {
|
||||
tokio::time::timeout(timeout, async {
|
||||
while *last_persisted < target {
|
||||
match stream.next().await {
|
||||
Some(persisted) => {
|
||||
*last_persisted = persisted.number;
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
persisted_block = ?last_persisted,
|
||||
"Received persistence notification"
|
||||
);
|
||||
}
|
||||
None => {
|
||||
return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| {
|
||||
eyre::eyre!(
|
||||
"Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
|
||||
target,
|
||||
timeout,
|
||||
last_persisted
|
||||
)
|
||||
})?
|
||||
}
|
||||
|
||||
/// Wrapper that keeps both the subscription stream and the underlying provider alive.
|
||||
/// The provider must be kept alive for the subscription to continue receiving events.
|
||||
struct PersistenceSubscription {
|
||||
_provider: RootProvider<Ethereum>,
|
||||
stream: SubscriptionStream<BlockNumHash>,
|
||||
}
|
||||
|
||||
impl PersistenceSubscription {
|
||||
const fn new(
|
||||
provider: RootProvider<Ethereum>,
|
||||
stream: SubscriptionStream<BlockNumHash>,
|
||||
) -> Self {
|
||||
Self { _provider: provider, stream }
|
||||
}
|
||||
|
||||
const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
|
||||
&mut self.stream
|
||||
}
|
||||
}
|
||||
|
||||
/// Encapsulates the block waiting logic.
|
||||
///
|
||||
/// Provides a simple `on_block()` interface that handles both:
|
||||
/// - Fixed duration waits (when `wait_time` is set)
|
||||
/// - Persistence-based waits (when `subscription` is set)
|
||||
///
|
||||
/// For persistence mode, waits after every `(threshold + 1)` blocks.
|
||||
struct PersistenceWaiter {
|
||||
wait_time: Option<Duration>,
|
||||
subscription: Option<PersistenceSubscription>,
|
||||
blocks_sent: u64,
|
||||
last_persisted: u64,
|
||||
threshold: u64,
|
||||
timeout: Duration,
|
||||
}
|
||||
|
||||
impl PersistenceWaiter {
|
||||
const fn with_duration(wait_time: Duration) -> Self {
|
||||
Self {
|
||||
wait_time: Some(wait_time),
|
||||
subscription: None,
|
||||
blocks_sent: 0,
|
||||
last_persisted: 0,
|
||||
threshold: 0,
|
||||
timeout: Duration::ZERO,
|
||||
}
|
||||
}
|
||||
|
||||
const fn with_subscription(
|
||||
subscription: PersistenceSubscription,
|
||||
threshold: u64,
|
||||
timeout: Duration,
|
||||
) -> Self {
|
||||
Self {
|
||||
wait_time: None,
|
||||
subscription: Some(subscription),
|
||||
blocks_sent: 0,
|
||||
last_persisted: 0,
|
||||
threshold,
|
||||
timeout,
|
||||
}
|
||||
}
|
||||
|
||||
/// Called once per block. Waits based on the configured mode.
|
||||
#[allow(clippy::manual_is_multiple_of)]
|
||||
async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
|
||||
if let Some(wait_time) = self.wait_time {
|
||||
tokio::time::sleep(wait_time).await;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let Some(ref mut subscription) = self.subscription else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
self.blocks_sent += 1;
|
||||
|
||||
if self.blocks_sent % (self.threshold + 1) == 0 {
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
target_block = ?block_number,
|
||||
last_persisted = self.last_persisted,
|
||||
blocks_sent = self.blocks_sent,
|
||||
"Waiting for persistence"
|
||||
);
|
||||
|
||||
wait_for_persistence(
|
||||
subscription.stream_mut(),
|
||||
block_number,
|
||||
&mut self.last_persisted,
|
||||
self.timeout,
|
||||
)
|
||||
.await?;
|
||||
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
persisted = self.last_persisted,
|
||||
"Persistence caught up"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_engine_url_to_ws_url() {
|
||||
// http -> ws, always uses port 8546
|
||||
let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
|
||||
assert_eq!(result.as_str(), "ws://localhost:8546/");
|
||||
|
||||
// https -> wss
|
||||
let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
|
||||
assert_eq!(result.as_str(), "wss://localhost:8546/");
|
||||
|
||||
// Custom engine port still maps to 8546
|
||||
let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
|
||||
assert_eq!(result.port(), Some(8546));
|
||||
|
||||
// Already ws passthrough
|
||||
let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
|
||||
assert_eq!(result.scheme(), "ws");
|
||||
|
||||
// Invalid inputs
|
||||
assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
|
||||
assert!(engine_url_to_ws_url("not a valid url").is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_waiter_with_duration() {
|
||||
let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
|
||||
|
||||
let start = Instant::now();
|
||||
waiter.on_block(1).await.unwrap();
|
||||
waiter.on_block(2).await.unwrap();
|
||||
waiter.on_block(3).await.unwrap();
|
||||
|
||||
// Should have waited ~3ms total
|
||||
assert!(start.elapsed() >= Duration::from_millis(3));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,6 +49,7 @@ impl Command {
|
||||
auth_provider,
|
||||
mut next_block,
|
||||
is_optimism,
|
||||
..
|
||||
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
|
||||
|
||||
let buffer_size = self.rpc_block_buffer_size;
|
||||
@@ -96,11 +97,7 @@ impl Command {
|
||||
let transaction_count = block.transactions.len() as u64;
|
||||
let gas_used = block.header.gas_used;
|
||||
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
number=?block.header.number,
|
||||
"Sending payload to engine",
|
||||
);
|
||||
debug!(number=?block.header.number, "Sending payload to engine");
|
||||
|
||||
let (version, params) = block_to_new_payload(block, is_optimism)?;
|
||||
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
//! Contains various benchmark output formats, either for logging or for
|
||||
//! serialization to / from files.
|
||||
|
||||
use alloy_primitives::B256;
|
||||
use csv::Writer;
|
||||
use eyre::OptionExt;
|
||||
use reth_primitives_traits::constants::GIGAGAS;
|
||||
use serde::{ser::SerializeStruct, Serialize};
|
||||
use std::time::Duration;
|
||||
use serde::{ser::SerializeStruct, Deserialize, Serialize};
|
||||
use std::{fs, path::Path, time::Duration};
|
||||
use tracing::info;
|
||||
|
||||
/// This is the suffix for gas output csv files.
|
||||
pub(crate) const GAS_OUTPUT_SUFFIX: &str = "total_gas.csv";
|
||||
@@ -15,6 +18,17 @@ pub(crate) const COMBINED_OUTPUT_SUFFIX: &str = "combined_latency.csv";
|
||||
/// This is the suffix for new payload output csv files.
|
||||
pub(crate) const NEW_PAYLOAD_OUTPUT_SUFFIX: &str = "new_payload_latency.csv";
|
||||
|
||||
/// Serialized format for gas ramp payloads on disk.
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub(crate) struct GasRampPayloadFile {
|
||||
/// Engine API version (1-5).
|
||||
pub(crate) version: u8,
|
||||
/// The block hash for FCU.
|
||||
pub(crate) block_hash: B256,
|
||||
/// The params to pass to newPayload.
|
||||
pub(crate) params: serde_json::Value,
|
||||
}
|
||||
|
||||
/// This represents the results of a single `newPayload` call in the benchmark, containing the gas
|
||||
/// used and the `newPayload` latency.
|
||||
#[derive(Debug)]
|
||||
@@ -67,6 +81,8 @@ impl Serialize for NewPayloadResult {
|
||||
pub(crate) struct CombinedResult {
|
||||
/// The block number of the block being processed.
|
||||
pub(crate) block_number: u64,
|
||||
/// The gas limit of the block.
|
||||
pub(crate) gas_limit: u64,
|
||||
/// The number of transactions in the block.
|
||||
pub(crate) transaction_count: u64,
|
||||
/// The `newPayload` result.
|
||||
@@ -88,7 +104,7 @@ impl std::fmt::Display for CombinedResult {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"Payload {} processed at {:.4} Ggas/s, used {} total gas. Combined gas per second: {:.4} Ggas/s. fcu latency: {:?}, newPayload latency: {:?}",
|
||||
"Block {} processed at {:.4} Ggas/s, used {} total gas. Combined: {:.4} Ggas/s. fcu: {:?}, newPayload: {:?}",
|
||||
self.block_number,
|
||||
self.new_payload_result.gas_per_second() / GIGAGAS as f64,
|
||||
self.new_payload_result.gas_used,
|
||||
@@ -110,10 +126,11 @@ impl Serialize for CombinedResult {
|
||||
let fcu_latency = self.fcu_latency.as_micros();
|
||||
let new_payload_latency = self.new_payload_result.latency.as_micros();
|
||||
let total_latency = self.total_latency.as_micros();
|
||||
let mut state = serializer.serialize_struct("CombinedResult", 6)?;
|
||||
let mut state = serializer.serialize_struct("CombinedResult", 7)?;
|
||||
|
||||
// flatten the new payload result because this is meant for CSV writing
|
||||
state.serialize_field("block_number", &self.block_number)?;
|
||||
state.serialize_field("gas_limit", &self.gas_limit)?;
|
||||
state.serialize_field("transaction_count", &self.transaction_count)?;
|
||||
state.serialize_field("gas_used", &self.new_payload_result.gas_used)?;
|
||||
state.serialize_field("new_payload_latency", &new_payload_latency)?;
|
||||
@@ -141,30 +158,91 @@ pub(crate) struct TotalGasRow {
|
||||
pub(crate) struct TotalGasOutput {
|
||||
/// The total gas used in the benchmark.
|
||||
pub(crate) total_gas_used: u64,
|
||||
/// The total duration of the benchmark.
|
||||
/// The total wall-clock duration of the benchmark (includes wait times).
|
||||
pub(crate) total_duration: Duration,
|
||||
/// The total gas used per second.
|
||||
pub(crate) total_gas_per_second: f64,
|
||||
/// The total execution-only duration (excludes wait times).
|
||||
pub(crate) execution_duration: Duration,
|
||||
/// The number of blocks processed.
|
||||
pub(crate) blocks_processed: u64,
|
||||
}
|
||||
|
||||
impl TotalGasOutput {
|
||||
/// Create a new [`TotalGasOutput`] from a list of [`TotalGasRow`].
|
||||
/// Create a new [`TotalGasOutput`] from gas rows only.
|
||||
///
|
||||
/// Use this when execution-only timing is not available (e.g., `new_payload_only`).
|
||||
/// `execution_duration` will equal `total_duration`.
|
||||
pub(crate) fn new(rows: Vec<TotalGasRow>) -> eyre::Result<Self> {
|
||||
// the duration is obtained from the last row
|
||||
let total_duration = rows.last().map(|row| row.time).ok_or_eyre("empty results")?;
|
||||
let blocks_processed = rows.len() as u64;
|
||||
let total_gas_used: u64 = rows.into_iter().map(|row| row.gas_used).sum();
|
||||
let total_gas_per_second = total_gas_used as f64 / total_duration.as_secs_f64();
|
||||
|
||||
Ok(Self { total_gas_used, total_duration, total_gas_per_second, blocks_processed })
|
||||
Ok(Self {
|
||||
total_gas_used,
|
||||
total_duration,
|
||||
execution_duration: total_duration,
|
||||
blocks_processed,
|
||||
})
|
||||
}
|
||||
|
||||
/// Return the total gigagas per second.
|
||||
/// Create a new [`TotalGasOutput`] from gas rows and combined results.
|
||||
///
|
||||
/// - `rows`: Used for total gas and wall-clock duration
|
||||
/// - `combined_results`: Used for execution-only duration (sum of `total_latency`)
|
||||
pub(crate) fn with_combined_results(
|
||||
rows: Vec<TotalGasRow>,
|
||||
combined_results: &[CombinedResult],
|
||||
) -> eyre::Result<Self> {
|
||||
let total_duration = rows.last().map(|row| row.time).ok_or_eyre("empty results")?;
|
||||
let blocks_processed = rows.len() as u64;
|
||||
let total_gas_used: u64 = rows.into_iter().map(|row| row.gas_used).sum();
|
||||
|
||||
// Sum execution-only time from combined results
|
||||
let execution_duration: Duration = combined_results.iter().map(|r| r.total_latency).sum();
|
||||
|
||||
Ok(Self { total_gas_used, total_duration, execution_duration, blocks_processed })
|
||||
}
|
||||
|
||||
/// Return the total gigagas per second based on wall-clock time.
|
||||
pub(crate) fn total_gigagas_per_second(&self) -> f64 {
|
||||
self.total_gas_per_second / GIGAGAS as f64
|
||||
self.total_gas_used as f64 / self.total_duration.as_secs_f64() / GIGAGAS as f64
|
||||
}
|
||||
|
||||
/// Return the execution-only gigagas per second (excludes wait times).
|
||||
pub(crate) fn execution_gigagas_per_second(&self) -> f64 {
|
||||
self.total_gas_used as f64 / self.execution_duration.as_secs_f64() / GIGAGAS as f64
|
||||
}
|
||||
}
|
||||
|
||||
/// Write benchmark results to CSV files.
|
||||
///
|
||||
/// Writes two files to the output directory:
|
||||
/// - `combined_latency.csv`: Per-block latency results
|
||||
/// - `total_gas.csv`: Per-block gas usage over time
|
||||
pub(crate) fn write_benchmark_results(
|
||||
output_dir: &Path,
|
||||
gas_results: &[TotalGasRow],
|
||||
combined_results: &[CombinedResult],
|
||||
) -> eyre::Result<()> {
|
||||
fs::create_dir_all(output_dir)?;
|
||||
|
||||
let output_path = output_dir.join(COMBINED_OUTPUT_SUFFIX);
|
||||
info!("Writing engine api call latency output to file: {:?}", output_path);
|
||||
let mut writer = Writer::from_path(&output_path)?;
|
||||
for result in combined_results {
|
||||
writer.serialize(result)?;
|
||||
}
|
||||
writer.flush()?;
|
||||
|
||||
let output_path = output_dir.join(GAS_OUTPUT_SUFFIX);
|
||||
info!("Writing total gas output to file: {:?}", output_path);
|
||||
let mut writer = Writer::from_path(&output_path)?;
|
||||
for row in gas_results {
|
||||
writer.serialize(row)?;
|
||||
}
|
||||
writer.flush()?;
|
||||
|
||||
info!("Finished writing benchmark output files to {:?}.", output_dir);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// This serializes the `time` field of the [`TotalGasRow`] to microseconds.
|
||||
|
||||
304
bin/reth-bench/src/bench/persistence_waiter.rs
Normal file
304
bin/reth-bench/src/bench/persistence_waiter.rs
Normal file
@@ -0,0 +1,304 @@
|
||||
//! Persistence waiting utilities for benchmarks.
|
||||
//!
|
||||
//! Provides waiting behavior to control benchmark pacing:
|
||||
//! - **Fixed duration waits**: Sleep for a fixed time between blocks
|
||||
//! - **Persistence-based waits**: Wait for blocks to be persisted using
|
||||
//! `reth_subscribePersistedBlock` subscription
|
||||
|
||||
use alloy_eips::BlockNumHash;
|
||||
use alloy_network::Ethereum;
|
||||
use alloy_provider::{Provider, RootProvider};
|
||||
use alloy_pubsub::SubscriptionStream;
|
||||
use alloy_rpc_client::RpcClient;
|
||||
use alloy_transport_ws::WsConnect;
|
||||
use eyre::Context;
|
||||
use futures::StreamExt;
|
||||
use std::time::Duration;
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// Default `WebSocket` RPC port for reth.
|
||||
const DEFAULT_WS_RPC_PORT: u16 = 8546;
|
||||
use url::Url;
|
||||
|
||||
/// Default timeout for waiting on persistence.
|
||||
pub(crate) const PERSISTENCE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
/// Returns the websocket RPC URL used for the persistence subscription.
|
||||
///
|
||||
/// Preference:
|
||||
/// - If `ws_rpc_url` is provided, use it directly.
|
||||
/// - Otherwise, derive a WS RPC URL from `engine_rpc_url`.
|
||||
///
|
||||
/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
|
||||
/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
|
||||
/// Since we may only have the engine URL by default, we convert the scheme
|
||||
/// (http→ws, https→wss) and force the port to 8546.
|
||||
pub(crate) fn derive_ws_rpc_url(
|
||||
ws_rpc_url: Option<&str>,
|
||||
engine_rpc_url: &str,
|
||||
) -> eyre::Result<Url> {
|
||||
if let Some(ws_url) = ws_rpc_url {
|
||||
let parsed: Url = ws_url
|
||||
.parse()
|
||||
.wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
|
||||
info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
|
||||
Ok(parsed)
|
||||
} else {
|
||||
let derived = engine_url_to_ws_url(engine_rpc_url)?;
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
engine_url = %engine_rpc_url,
|
||||
%derived,
|
||||
"Derived WebSocket RPC URL from engine RPC URL"
|
||||
);
|
||||
Ok(derived)
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts an engine API URL to the default RPC websocket URL.
|
||||
///
|
||||
/// Transformations:
|
||||
/// - `http` → `ws`
|
||||
/// - `https` → `wss`
|
||||
/// - `ws` / `wss` keep their scheme
|
||||
/// - Port is always set to `8546`, reth's default RPC websocket port.
|
||||
///
|
||||
/// This is used when we only know the engine API URL (typically `:8551`) but
|
||||
/// need to connect to the node's WS RPC endpoint for persistence events.
|
||||
fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
|
||||
let url: Url = engine_url
|
||||
.parse()
|
||||
.wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
|
||||
|
||||
let mut ws_url = url.clone();
|
||||
|
||||
match ws_url.scheme() {
|
||||
"http" => ws_url
|
||||
.set_scheme("ws")
|
||||
.map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
|
||||
"https" => ws_url
|
||||
.set_scheme("wss")
|
||||
.map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
|
||||
"ws" | "wss" => {}
|
||||
scheme => {
|
||||
return Err(eyre::eyre!(
|
||||
"Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
ws_url
|
||||
.set_port(Some(DEFAULT_WS_RPC_PORT))
|
||||
.map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
|
||||
|
||||
Ok(ws_url)
|
||||
}
|
||||
|
||||
/// Waits until the persistence subscription reports that `target` has been persisted.
|
||||
///
|
||||
/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
|
||||
/// - the subscription stream ends unexpectedly, or
|
||||
/// - `timeout` elapses before `target` is observed.
|
||||
async fn wait_for_persistence(
|
||||
stream: &mut SubscriptionStream<BlockNumHash>,
|
||||
target: u64,
|
||||
last_persisted: &mut u64,
|
||||
timeout: Duration,
|
||||
) -> eyre::Result<()> {
|
||||
tokio::time::timeout(timeout, async {
|
||||
while *last_persisted < target {
|
||||
match stream.next().await {
|
||||
Some(persisted) => {
|
||||
*last_persisted = persisted.number;
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
persisted_block = ?last_persisted,
|
||||
"Received persistence notification"
|
||||
);
|
||||
}
|
||||
None => {
|
||||
return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| {
|
||||
eyre::eyre!(
|
||||
"Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
|
||||
target,
|
||||
timeout,
|
||||
last_persisted
|
||||
)
|
||||
})?
|
||||
}
|
||||
|
||||
/// Wrapper that keeps both the subscription stream and the underlying provider alive.
|
||||
/// The provider must be kept alive for the subscription to continue receiving events.
|
||||
pub(crate) struct PersistenceSubscription {
|
||||
_provider: RootProvider<Ethereum>,
|
||||
stream: SubscriptionStream<BlockNumHash>,
|
||||
}
|
||||
|
||||
impl PersistenceSubscription {
|
||||
const fn new(
|
||||
provider: RootProvider<Ethereum>,
|
||||
stream: SubscriptionStream<BlockNumHash>,
|
||||
) -> Self {
|
||||
Self { _provider: provider, stream }
|
||||
}
|
||||
|
||||
const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
|
||||
&mut self.stream
|
||||
}
|
||||
}
|
||||
|
||||
/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
|
||||
pub(crate) async fn setup_persistence_subscription(
|
||||
ws_url: Url,
|
||||
) -> eyre::Result<PersistenceSubscription> {
|
||||
info!("Connecting to WebSocket at {} for persistence subscription", ws_url);
|
||||
|
||||
let ws_connect = WsConnect::new(ws_url.to_string());
|
||||
let client = RpcClient::connect_pubsub(ws_connect)
|
||||
.await
|
||||
.wrap_err("Failed to connect to WebSocket RPC endpoint")?;
|
||||
let provider: RootProvider<Ethereum> = RootProvider::new(client);
|
||||
|
||||
let subscription = provider
|
||||
.subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
|
||||
.await
|
||||
.wrap_err("Failed to subscribe to persistence notifications")?;
|
||||
|
||||
info!("Subscribed to persistence notifications");
|
||||
|
||||
Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
|
||||
}
|
||||
|
||||
/// Encapsulates the block waiting logic.
|
||||
///
|
||||
/// Provides a simple `on_block()` interface that handles both:
|
||||
/// - Fixed duration waits (when `wait_time` is set)
|
||||
/// - Persistence-based waits (when `subscription` is set)
|
||||
///
|
||||
/// For persistence mode, waits after every `(threshold + 1)` blocks.
|
||||
pub(crate) struct PersistenceWaiter {
|
||||
wait_time: Option<Duration>,
|
||||
subscription: Option<PersistenceSubscription>,
|
||||
blocks_sent: u64,
|
||||
last_persisted: u64,
|
||||
threshold: u64,
|
||||
timeout: Duration,
|
||||
}
|
||||
|
||||
impl PersistenceWaiter {
|
||||
pub(crate) const fn with_duration(wait_time: Duration) -> Self {
|
||||
Self {
|
||||
wait_time: Some(wait_time),
|
||||
subscription: None,
|
||||
blocks_sent: 0,
|
||||
last_persisted: 0,
|
||||
threshold: 0,
|
||||
timeout: Duration::ZERO,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const fn with_subscription(
|
||||
subscription: PersistenceSubscription,
|
||||
threshold: u64,
|
||||
timeout: Duration,
|
||||
) -> Self {
|
||||
Self {
|
||||
wait_time: None,
|
||||
subscription: Some(subscription),
|
||||
blocks_sent: 0,
|
||||
last_persisted: 0,
|
||||
threshold,
|
||||
timeout,
|
||||
}
|
||||
}
|
||||
|
||||
/// Called once per block. Waits based on the configured mode.
|
||||
#[allow(clippy::manual_is_multiple_of)]
|
||||
pub(crate) async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
|
||||
if let Some(wait_time) = self.wait_time {
|
||||
tokio::time::sleep(wait_time).await;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let Some(ref mut subscription) = self.subscription else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
self.blocks_sent += 1;
|
||||
|
||||
if self.blocks_sent % (self.threshold + 1) == 0 {
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
target_block = ?block_number,
|
||||
last_persisted = self.last_persisted,
|
||||
blocks_sent = self.blocks_sent,
|
||||
"Waiting for persistence"
|
||||
);
|
||||
|
||||
wait_for_persistence(
|
||||
subscription.stream_mut(),
|
||||
block_number,
|
||||
&mut self.last_persisted,
|
||||
self.timeout,
|
||||
)
|
||||
.await?;
|
||||
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
persisted = self.last_persisted,
|
||||
"Persistence caught up"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::Instant;
|
||||
|
||||
#[test]
|
||||
fn test_engine_url_to_ws_url() {
|
||||
// http -> ws, always uses port 8546
|
||||
let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
|
||||
assert_eq!(result.as_str(), "ws://localhost:8546/");
|
||||
|
||||
// https -> wss
|
||||
let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
|
||||
assert_eq!(result.as_str(), "wss://localhost:8546/");
|
||||
|
||||
// Custom engine port still maps to 8546
|
||||
let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
|
||||
assert_eq!(result.port(), Some(8546));
|
||||
|
||||
// Already ws passthrough
|
||||
let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
|
||||
assert_eq!(result.scheme(), "ws");
|
||||
|
||||
// Invalid inputs
|
||||
assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
|
||||
assert!(engine_url_to_ws_url("not a valid url").is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_waiter_with_duration() {
|
||||
let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
|
||||
|
||||
let start = Instant::now();
|
||||
waiter.on_block(1).await.unwrap();
|
||||
waiter.on_block(2).await.unwrap();
|
||||
waiter.on_block(3).await.unwrap();
|
||||
|
||||
// Should have waited ~3ms total
|
||||
assert!(start.elapsed() >= Duration::from_millis(3));
|
||||
}
|
||||
}
|
||||
467
bin/reth-bench/src/bench/replay_payloads.rs
Normal file
467
bin/reth-bench/src/bench/replay_payloads.rs
Normal file
@@ -0,0 +1,467 @@
|
||||
//! Command for replaying pre-generated payloads from disk.
|
||||
//!
|
||||
//! This command reads `ExecutionPayloadEnvelopeV4` files from a directory and replays them
|
||||
//! in sequence using `newPayload` followed by `forkchoiceUpdated`.
|
||||
//!
|
||||
//! Supports configurable waiting behavior:
|
||||
//! - **`--wait-time`**: Fixed sleep interval between blocks.
|
||||
//! - **`--wait-for-persistence`**: Waits for every Nth block to be persisted using the
|
||||
//! `reth_subscribePersistedBlock` subscription, where N matches the engine's persistence
|
||||
//! threshold. This ensures the benchmark doesn't outpace persistence.
|
||||
//!
|
||||
//! Both options can be used together or independently.
|
||||
|
||||
use crate::{
|
||||
authenticated_transport::AuthenticatedTransportConnect,
|
||||
bench::{
|
||||
output::{
|
||||
write_benchmark_results, CombinedResult, GasRampPayloadFile, NewPayloadResult,
|
||||
TotalGasOutput, TotalGasRow,
|
||||
},
|
||||
persistence_waiter::{
|
||||
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
|
||||
PERSISTENCE_CHECKPOINT_TIMEOUT,
|
||||
},
|
||||
},
|
||||
valid_payload::{call_forkchoice_updated, call_new_payload},
|
||||
};
|
||||
use alloy_primitives::B256;
|
||||
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
|
||||
use alloy_rpc_client::ClientBuilder;
|
||||
use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV4, ForkchoiceState, JwtSecret};
|
||||
use clap::Parser;
|
||||
use eyre::Context;
|
||||
use humantime::parse_duration;
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
|
||||
use reth_node_api::EngineApiMessageVersion;
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{debug, info};
|
||||
use url::Url;
|
||||
|
||||
/// `reth bench replay-payloads` command
|
||||
///
|
||||
/// Replays pre-generated payloads from a directory by calling `newPayload` followed by
|
||||
/// `forkchoiceUpdated` for each payload in sequence.
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Command {
|
||||
/// The engine RPC URL (with JWT authentication).
|
||||
#[arg(long, value_name = "ENGINE_RPC_URL", default_value = "http://localhost:8551")]
|
||||
engine_rpc_url: String,
|
||||
|
||||
/// Path to the JWT secret file for engine API authentication.
|
||||
#[arg(long, value_name = "JWT_SECRET")]
|
||||
jwt_secret: PathBuf,
|
||||
|
||||
/// Directory containing payload files (`payload_block_N.json`).
|
||||
#[arg(long, value_name = "PAYLOAD_DIR")]
|
||||
payload_dir: PathBuf,
|
||||
|
||||
/// Optional limit on the number of payloads to replay.
|
||||
/// If not specified, replays all payloads in the directory.
|
||||
#[arg(long, value_name = "COUNT")]
|
||||
count: Option<usize>,
|
||||
|
||||
/// Skip the first N payloads.
|
||||
#[arg(long, value_name = "SKIP", default_value = "0")]
|
||||
skip: usize,
|
||||
|
||||
/// Optional directory containing gas ramp payloads to replay first.
|
||||
/// These are replayed before the main payloads to warm up the gas limit.
|
||||
#[arg(long, value_name = "GAS_RAMP_DIR")]
|
||||
gas_ramp_dir: Option<PathBuf>,
|
||||
|
||||
/// Optional output directory for benchmark results (CSV files).
|
||||
#[arg(long, value_name = "OUTPUT")]
|
||||
output: Option<PathBuf>,
|
||||
|
||||
/// How long to wait after a forkchoice update before sending the next payload.
|
||||
#[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)]
|
||||
wait_time: Option<Duration>,
|
||||
|
||||
/// Wait for blocks to be persisted before sending the next batch.
|
||||
///
|
||||
/// When enabled, waits for every Nth block to be persisted using the
|
||||
/// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
|
||||
/// doesn't outpace persistence.
|
||||
///
|
||||
/// The subscription uses the regular RPC websocket endpoint (no JWT required).
|
||||
#[arg(long, default_value = "false", verbatim_doc_comment)]
|
||||
wait_for_persistence: bool,
|
||||
|
||||
/// Engine persistence threshold used for deciding when to wait for persistence.
|
||||
///
|
||||
/// The benchmark waits after every `(threshold + 1)` blocks. By default this
|
||||
/// matches the engine's `DEFAULT_PERSISTENCE_THRESHOLD` (2), so waits occur
|
||||
/// at blocks 3, 6, 9, etc.
|
||||
#[arg(
|
||||
long = "persistence-threshold",
|
||||
value_name = "PERSISTENCE_THRESHOLD",
|
||||
default_value_t = DEFAULT_PERSISTENCE_THRESHOLD,
|
||||
verbatim_doc_comment
|
||||
)]
|
||||
persistence_threshold: u64,
|
||||
|
||||
/// Optional `WebSocket` RPC URL for persistence subscription.
|
||||
/// If not provided, derives from engine RPC URL by changing scheme to ws and port to 8546.
|
||||
#[arg(long, value_name = "WS_RPC_URL", verbatim_doc_comment)]
|
||||
ws_rpc_url: Option<String>,
|
||||
}
|
||||
|
||||
/// A loaded payload ready for execution.
|
||||
struct LoadedPayload {
|
||||
/// The index (from filename).
|
||||
index: u64,
|
||||
/// The payload envelope.
|
||||
envelope: ExecutionPayloadEnvelopeV4,
|
||||
/// The block hash.
|
||||
block_hash: B256,
|
||||
}
|
||||
|
||||
/// A gas ramp payload loaded from disk.
|
||||
struct GasRampPayload {
|
||||
/// Block number from filename.
|
||||
block_number: u64,
|
||||
/// Engine API version for newPayload.
|
||||
version: EngineApiMessageVersion,
|
||||
/// The file contents.
|
||||
file: GasRampPayloadFile,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute the `replay-payloads` command.
|
||||
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
|
||||
info!(payload_dir = %self.payload_dir.display(), "Replaying payloads");
|
||||
|
||||
// Log mode configuration
|
||||
if let Some(duration) = self.wait_time {
|
||||
info!("Using wait-time mode with {}ms delay between blocks", duration.as_millis());
|
||||
}
|
||||
if self.wait_for_persistence {
|
||||
info!(
|
||||
"Persistence waiting enabled (waits after every {} blocks to match engine gap > {} behavior)",
|
||||
self.persistence_threshold + 1,
|
||||
self.persistence_threshold
|
||||
);
|
||||
}
|
||||
|
||||
// Set up waiter based on configured options (duration takes precedence)
|
||||
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
|
||||
(Some(duration), _) => Some(PersistenceWaiter::with_duration(duration)),
|
||||
(None, true) => {
|
||||
let ws_url = derive_ws_rpc_url(self.ws_rpc_url.as_deref(), &self.engine_rpc_url)?;
|
||||
let sub = setup_persistence_subscription(ws_url).await?;
|
||||
Some(PersistenceWaiter::with_subscription(
|
||||
sub,
|
||||
self.persistence_threshold,
|
||||
PERSISTENCE_CHECKPOINT_TIMEOUT,
|
||||
))
|
||||
}
|
||||
(None, false) => None,
|
||||
};
|
||||
|
||||
// Set up authenticated engine provider
|
||||
let jwt =
|
||||
std::fs::read_to_string(&self.jwt_secret).wrap_err("Failed to read JWT secret file")?;
|
||||
let jwt = JwtSecret::from_hex(jwt.trim())?;
|
||||
let auth_url = Url::parse(&self.engine_rpc_url)?;
|
||||
|
||||
info!("Connecting to Engine RPC at {}", auth_url);
|
||||
let auth_transport = AuthenticatedTransportConnect::new(auth_url.clone(), jwt);
|
||||
let auth_client = ClientBuilder::default().connect_with(auth_transport).await?;
|
||||
let auth_provider = RootProvider::<AnyNetwork>::new(auth_client);
|
||||
|
||||
// Get parent block (latest canonical block) - we need this for the first FCU
|
||||
let parent_block = auth_provider
|
||||
.get_block_by_number(alloy_eips::BlockNumberOrTag::Latest)
|
||||
.await?
|
||||
.ok_or_else(|| eyre::eyre!("Failed to fetch latest block"))?;
|
||||
|
||||
let initial_parent_hash = parent_block.header.hash;
|
||||
let initial_parent_number = parent_block.header.number;
|
||||
|
||||
info!(
|
||||
parent_hash = %initial_parent_hash,
|
||||
parent_number = initial_parent_number,
|
||||
"Using initial parent block"
|
||||
);
|
||||
|
||||
// Load all payloads upfront to avoid I/O delays between phases
|
||||
let gas_ramp_payloads = if let Some(ref gas_ramp_dir) = self.gas_ramp_dir {
|
||||
let payloads = self.load_gas_ramp_payloads(gas_ramp_dir)?;
|
||||
if payloads.is_empty() {
|
||||
return Err(eyre::eyre!("No gas ramp payload files found in {:?}", gas_ramp_dir));
|
||||
}
|
||||
info!(count = payloads.len(), "Loaded gas ramp payloads from disk");
|
||||
payloads
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
let payloads = self.load_payloads()?;
|
||||
if payloads.is_empty() {
|
||||
return Err(eyre::eyre!("No payload files found in {:?}", self.payload_dir));
|
||||
}
|
||||
info!(count = payloads.len(), "Loaded main payloads from disk");
|
||||
|
||||
let mut parent_hash = initial_parent_hash;
|
||||
|
||||
// Replay gas ramp payloads first
|
||||
for (i, payload) in gas_ramp_payloads.iter().enumerate() {
|
||||
info!(
|
||||
gas_ramp_payload = i + 1,
|
||||
total = gas_ramp_payloads.len(),
|
||||
block_number = payload.block_number,
|
||||
block_hash = %payload.file.block_hash,
|
||||
"Executing gas ramp payload (newPayload + FCU)"
|
||||
);
|
||||
|
||||
call_new_payload(&auth_provider, payload.version, payload.file.params.clone()).await?;
|
||||
|
||||
let fcu_state = ForkchoiceState {
|
||||
head_block_hash: payload.file.block_hash,
|
||||
safe_block_hash: parent_hash,
|
||||
finalized_block_hash: parent_hash,
|
||||
};
|
||||
call_forkchoice_updated(&auth_provider, payload.version, fcu_state, None).await?;
|
||||
|
||||
info!(gas_ramp_payload = i + 1, "Gas ramp payload executed successfully");
|
||||
|
||||
if let Some(w) = &mut waiter {
|
||||
w.on_block(payload.block_number).await?;
|
||||
}
|
||||
|
||||
parent_hash = payload.file.block_hash;
|
||||
}
|
||||
|
||||
if !gas_ramp_payloads.is_empty() {
|
||||
info!(count = gas_ramp_payloads.len(), "All gas ramp payloads replayed");
|
||||
}
|
||||
|
||||
let mut results = Vec::new();
|
||||
let total_benchmark_duration = Instant::now();
|
||||
|
||||
for (i, payload) in payloads.iter().enumerate() {
|
||||
let envelope = &payload.envelope;
|
||||
let block_hash = payload.block_hash;
|
||||
let execution_payload = &envelope.envelope_inner.execution_payload;
|
||||
let inner_payload = &execution_payload.payload_inner.payload_inner;
|
||||
|
||||
let gas_used = inner_payload.gas_used;
|
||||
let gas_limit = inner_payload.gas_limit;
|
||||
let block_number = inner_payload.block_number;
|
||||
let transaction_count =
|
||||
execution_payload.payload_inner.payload_inner.transactions.len() as u64;
|
||||
|
||||
debug!(
|
||||
payload = i + 1,
|
||||
total = payloads.len(),
|
||||
index = payload.index,
|
||||
block_hash = %block_hash,
|
||||
"Executing payload (newPayload + FCU)"
|
||||
);
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
debug!(
|
||||
method = "engine_newPayloadV4",
|
||||
block_hash = %block_hash,
|
||||
"Sending newPayload"
|
||||
);
|
||||
|
||||
let status = auth_provider
|
||||
.new_payload_v4(
|
||||
execution_payload.clone(),
|
||||
vec![],
|
||||
B256::ZERO,
|
||||
envelope.execution_requests.to_vec(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
|
||||
|
||||
if !status.is_valid() {
|
||||
return Err(eyre::eyre!("Payload rejected: {:?}", status));
|
||||
}
|
||||
|
||||
let fcu_state = ForkchoiceState {
|
||||
head_block_hash: block_hash,
|
||||
safe_block_hash: parent_hash,
|
||||
finalized_block_hash: parent_hash,
|
||||
};
|
||||
|
||||
debug!(method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
|
||||
|
||||
let fcu_result = auth_provider.fork_choice_updated_v3(fcu_state, None).await?;
|
||||
|
||||
let total_latency = start.elapsed();
|
||||
let fcu_latency = total_latency - new_payload_result.latency;
|
||||
|
||||
let combined_result = CombinedResult {
|
||||
block_number,
|
||||
gas_limit,
|
||||
transaction_count,
|
||||
new_payload_result,
|
||||
fcu_latency,
|
||||
total_latency,
|
||||
};
|
||||
|
||||
let current_duration = total_benchmark_duration.elapsed();
|
||||
info!(%combined_result);
|
||||
|
||||
if let Some(w) = &mut waiter {
|
||||
w.on_block(block_number).await?;
|
||||
}
|
||||
|
||||
let gas_row =
|
||||
TotalGasRow { block_number, transaction_count, gas_used, time: current_duration };
|
||||
results.push((gas_row, combined_result));
|
||||
|
||||
debug!(?status, ?fcu_result, "Payload executed successfully");
|
||||
parent_hash = block_hash;
|
||||
}
|
||||
|
||||
// Drop waiter - we don't need to wait for final blocks to persist
|
||||
// since the benchmark goal is measuring Ggas/s of newPayload/FCU, not persistence.
|
||||
drop(waiter);
|
||||
|
||||
let (gas_output_results, combined_results): (Vec<TotalGasRow>, Vec<CombinedResult>) =
|
||||
results.into_iter().unzip();
|
||||
|
||||
if let Some(ref path) = self.output {
|
||||
write_benchmark_results(path, &gas_output_results, &combined_results)?;
|
||||
}
|
||||
|
||||
let gas_output =
|
||||
TotalGasOutput::with_combined_results(gas_output_results, &combined_results)?;
|
||||
info!(
|
||||
total_gas_used = gas_output.total_gas_used,
|
||||
total_duration = ?gas_output.total_duration,
|
||||
execution_duration = ?gas_output.execution_duration,
|
||||
blocks_processed = gas_output.blocks_processed,
|
||||
wall_clock_ggas_per_second = format_args!("{:.4}", gas_output.total_gigagas_per_second()),
|
||||
execution_ggas_per_second = format_args!("{:.4}", gas_output.execution_gigagas_per_second()),
|
||||
"Benchmark complete"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Load and parse all payload files from the directory.
|
||||
fn load_payloads(&self) -> eyre::Result<Vec<LoadedPayload>> {
|
||||
let mut payloads = Vec::new();
|
||||
|
||||
// Read directory entries
|
||||
let entries: Vec<_> = std::fs::read_dir(&self.payload_dir)
|
||||
.wrap_err_with(|| format!("Failed to read directory {:?}", self.payload_dir))?
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| {
|
||||
e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
|
||||
e.file_name().to_string_lossy().starts_with("payload_block_")
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Parse filenames to get indices and sort
|
||||
let mut indexed_paths: Vec<(u64, PathBuf)> = entries
|
||||
.into_iter()
|
||||
.filter_map(|e| {
|
||||
let name = e.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
// Extract index from "payload_NNN.json"
|
||||
let index_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
|
||||
let index: u64 = index_str.parse().ok()?;
|
||||
Some((index, e.path()))
|
||||
})
|
||||
.collect();
|
||||
|
||||
indexed_paths.sort_by_key(|(idx, _)| *idx);
|
||||
|
||||
// Apply skip and count
|
||||
let indexed_paths: Vec<_> = indexed_paths.into_iter().skip(self.skip).collect();
|
||||
let indexed_paths: Vec<_> = match self.count {
|
||||
Some(count) => indexed_paths.into_iter().take(count).collect(),
|
||||
None => indexed_paths,
|
||||
};
|
||||
|
||||
// Load each payload
|
||||
for (index, path) in indexed_paths {
|
||||
let content = std::fs::read_to_string(&path)
|
||||
.wrap_err_with(|| format!("Failed to read {:?}", path))?;
|
||||
let envelope: ExecutionPayloadEnvelopeV4 = serde_json::from_str(&content)
|
||||
.wrap_err_with(|| format!("Failed to parse {:?}", path))?;
|
||||
|
||||
let block_hash =
|
||||
envelope.envelope_inner.execution_payload.payload_inner.payload_inner.block_hash;
|
||||
|
||||
info!(
|
||||
index = index,
|
||||
block_hash = %block_hash,
|
||||
path = %path.display(),
|
||||
"Loaded payload"
|
||||
);
|
||||
|
||||
payloads.push(LoadedPayload { index, envelope, block_hash });
|
||||
}
|
||||
|
||||
Ok(payloads)
|
||||
}
|
||||
|
||||
/// Load and parse gas ramp payload files from a directory.
|
||||
fn load_gas_ramp_payloads(&self, dir: &PathBuf) -> eyre::Result<Vec<GasRampPayload>> {
|
||||
let mut payloads = Vec::new();
|
||||
|
||||
let entries: Vec<_> = std::fs::read_dir(dir)
|
||||
.wrap_err_with(|| format!("Failed to read directory {:?}", dir))?
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| {
|
||||
e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
|
||||
e.file_name().to_string_lossy().starts_with("payload_block_")
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Parse filenames to get block numbers and sort
|
||||
let mut indexed_paths: Vec<(u64, PathBuf)> = entries
|
||||
.into_iter()
|
||||
.filter_map(|e| {
|
||||
let name = e.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
// Extract block number from "payload_block_NNN.json"
|
||||
let block_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
|
||||
let block_number: u64 = block_str.parse().ok()?;
|
||||
Some((block_number, e.path()))
|
||||
})
|
||||
.collect();
|
||||
|
||||
indexed_paths.sort_by_key(|(num, _)| *num);
|
||||
|
||||
for (block_number, path) in indexed_paths {
|
||||
let content = std::fs::read_to_string(&path)
|
||||
.wrap_err_with(|| format!("Failed to read {:?}", path))?;
|
||||
let file: GasRampPayloadFile = serde_json::from_str(&content)
|
||||
.wrap_err_with(|| format!("Failed to parse {:?}", path))?;
|
||||
|
||||
let version = match file.version {
|
||||
1 => EngineApiMessageVersion::V1,
|
||||
2 => EngineApiMessageVersion::V2,
|
||||
3 => EngineApiMessageVersion::V3,
|
||||
4 => EngineApiMessageVersion::V4,
|
||||
5 => EngineApiMessageVersion::V5,
|
||||
v => return Err(eyre::eyre!("Invalid version {} in {:?}", v, path)),
|
||||
};
|
||||
|
||||
info!(
|
||||
block_number,
|
||||
block_hash = %file.block_hash,
|
||||
path = %path.display(),
|
||||
"Loaded gas ramp payload"
|
||||
);
|
||||
|
||||
payloads.push(GasRampPayload { block_number, version, file });
|
||||
}
|
||||
|
||||
Ok(payloads)
|
||||
}
|
||||
}
|
||||
219
bin/reth-bench/src/bench/send_invalid_payload/invalidation.rs
Normal file
219
bin/reth-bench/src/bench/send_invalid_payload/invalidation.rs
Normal file
@@ -0,0 +1,219 @@
|
||||
use alloy_eips::eip4895::Withdrawal;
|
||||
use alloy_primitives::{Address, Bloom, Bytes, B256, U256};
|
||||
use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3};
|
||||
|
||||
/// Configuration for invalidating payload fields
|
||||
#[derive(Debug, Default)]
|
||||
pub(super) struct InvalidationConfig {
|
||||
// Explicit value overrides (Option<T>)
|
||||
pub(super) parent_hash: Option<B256>,
|
||||
pub(super) fee_recipient: Option<Address>,
|
||||
pub(super) state_root: Option<B256>,
|
||||
pub(super) receipts_root: Option<B256>,
|
||||
pub(super) logs_bloom: Option<Bloom>,
|
||||
pub(super) prev_randao: Option<B256>,
|
||||
pub(super) block_number: Option<u64>,
|
||||
pub(super) gas_limit: Option<u64>,
|
||||
pub(super) gas_used: Option<u64>,
|
||||
pub(super) timestamp: Option<u64>,
|
||||
pub(super) extra_data: Option<Bytes>,
|
||||
pub(super) base_fee_per_gas: Option<u64>,
|
||||
pub(super) block_hash: Option<B256>,
|
||||
pub(super) blob_gas_used: Option<u64>,
|
||||
pub(super) excess_blob_gas: Option<u64>,
|
||||
|
||||
// Auto-invalidation flags
|
||||
pub(super) invalidate_parent_hash: bool,
|
||||
pub(super) invalidate_state_root: bool,
|
||||
pub(super) invalidate_receipts_root: bool,
|
||||
pub(super) invalidate_gas_used: bool,
|
||||
pub(super) invalidate_block_number: bool,
|
||||
pub(super) invalidate_timestamp: bool,
|
||||
pub(super) invalidate_base_fee: bool,
|
||||
pub(super) invalidate_transactions: bool,
|
||||
pub(super) invalidate_block_hash: bool,
|
||||
pub(super) invalidate_withdrawals: bool,
|
||||
pub(super) invalidate_blob_gas_used: bool,
|
||||
pub(super) invalidate_excess_blob_gas: bool,
|
||||
}
|
||||
|
||||
impl InvalidationConfig {
|
||||
/// Returns true if `block_hash` is being explicitly set or auto-invalidated.
|
||||
/// When true, the caller should skip recalculating the block hash since it will be overwritten.
|
||||
pub(super) const fn should_skip_hash_recalc(&self) -> bool {
|
||||
self.block_hash.is_some() || self.invalidate_block_hash
|
||||
}
|
||||
|
||||
/// Applies invalidations to a V1 payload, returns list of what was changed.
|
||||
pub(super) fn apply_to_payload_v1(&self, payload: &mut ExecutionPayloadV1) -> Vec<String> {
|
||||
let mut changes = Vec::new();
|
||||
|
||||
// Explicit value overrides
|
||||
if let Some(parent_hash) = self.parent_hash {
|
||||
payload.parent_hash = parent_hash;
|
||||
changes.push(format!("parent_hash = {parent_hash}"));
|
||||
}
|
||||
|
||||
if let Some(fee_recipient) = self.fee_recipient {
|
||||
payload.fee_recipient = fee_recipient;
|
||||
changes.push(format!("fee_recipient = {fee_recipient}"));
|
||||
}
|
||||
|
||||
if let Some(state_root) = self.state_root {
|
||||
payload.state_root = state_root;
|
||||
changes.push(format!("state_root = {state_root}"));
|
||||
}
|
||||
|
||||
if let Some(receipts_root) = self.receipts_root {
|
||||
payload.receipts_root = receipts_root;
|
||||
changes.push(format!("receipts_root = {receipts_root}"));
|
||||
}
|
||||
|
||||
if let Some(logs_bloom) = self.logs_bloom {
|
||||
payload.logs_bloom = logs_bloom;
|
||||
changes.push("logs_bloom = <custom>".to_string());
|
||||
}
|
||||
|
||||
if let Some(prev_randao) = self.prev_randao {
|
||||
payload.prev_randao = prev_randao;
|
||||
changes.push(format!("prev_randao = {prev_randao}"));
|
||||
}
|
||||
|
||||
if let Some(block_number) = self.block_number {
|
||||
payload.block_number = block_number;
|
||||
changes.push(format!("block_number = {block_number}"));
|
||||
}
|
||||
|
||||
if let Some(gas_limit) = self.gas_limit {
|
||||
payload.gas_limit = gas_limit;
|
||||
changes.push(format!("gas_limit = {gas_limit}"));
|
||||
}
|
||||
|
||||
if let Some(gas_used) = self.gas_used {
|
||||
payload.gas_used = gas_used;
|
||||
changes.push(format!("gas_used = {gas_used}"));
|
||||
}
|
||||
|
||||
if let Some(timestamp) = self.timestamp {
|
||||
payload.timestamp = timestamp;
|
||||
changes.push(format!("timestamp = {timestamp}"));
|
||||
}
|
||||
|
||||
if let Some(ref extra_data) = self.extra_data {
|
||||
payload.extra_data = extra_data.clone();
|
||||
changes.push(format!("extra_data = {} bytes", extra_data.len()));
|
||||
}
|
||||
|
||||
if let Some(base_fee_per_gas) = self.base_fee_per_gas {
|
||||
payload.base_fee_per_gas = U256::from_limbs([base_fee_per_gas, 0, 0, 0]);
|
||||
changes.push(format!("base_fee_per_gas = {base_fee_per_gas}"));
|
||||
}
|
||||
|
||||
if let Some(block_hash) = self.block_hash {
|
||||
payload.block_hash = block_hash;
|
||||
changes.push(format!("block_hash = {block_hash}"));
|
||||
}
|
||||
|
||||
// Auto-invalidation flags
|
||||
if self.invalidate_parent_hash {
|
||||
let random_hash = B256::random();
|
||||
payload.parent_hash = random_hash;
|
||||
changes.push(format!("parent_hash = {random_hash} (auto-invalidated: random)"));
|
||||
}
|
||||
|
||||
if self.invalidate_state_root {
|
||||
payload.state_root = B256::ZERO;
|
||||
changes.push("state_root = ZERO (auto-invalidated: empty trie root)".to_string());
|
||||
}
|
||||
|
||||
if self.invalidate_receipts_root {
|
||||
payload.receipts_root = B256::ZERO;
|
||||
changes.push("receipts_root = ZERO (auto-invalidated)".to_string());
|
||||
}
|
||||
|
||||
if self.invalidate_gas_used {
|
||||
let invalid_gas = payload.gas_limit + 1;
|
||||
payload.gas_used = invalid_gas;
|
||||
changes.push(format!("gas_used = {invalid_gas} (auto-invalidated: exceeds gas_limit)"));
|
||||
}
|
||||
|
||||
if self.invalidate_block_number {
|
||||
let invalid_number = payload.block_number + 999;
|
||||
payload.block_number = invalid_number;
|
||||
changes.push(format!("block_number = {invalid_number} (auto-invalidated: huge gap)"));
|
||||
}
|
||||
|
||||
if self.invalidate_timestamp {
|
||||
payload.timestamp = 0;
|
||||
changes.push("timestamp = 0 (auto-invalidated: impossibly old)".to_string());
|
||||
}
|
||||
|
||||
if self.invalidate_base_fee {
|
||||
payload.base_fee_per_gas = U256::ZERO;
|
||||
changes
|
||||
.push("base_fee_per_gas = 0 (auto-invalidated: invalid post-London)".to_string());
|
||||
}
|
||||
|
||||
if self.invalidate_transactions {
|
||||
let invalid_tx = Bytes::from_static(&[0xff, 0xff, 0xff]);
|
||||
payload.transactions.insert(0, invalid_tx);
|
||||
changes.push("transactions = prepended invalid RLP (auto-invalidated)".to_string());
|
||||
}
|
||||
|
||||
if self.invalidate_block_hash {
|
||||
let random_hash = B256::random();
|
||||
payload.block_hash = random_hash;
|
||||
changes.push(format!("block_hash = {random_hash} (auto-invalidated: random)"));
|
||||
}
|
||||
|
||||
changes
|
||||
}
|
||||
|
||||
/// Applies invalidations to a V2 payload, returns list of what was changed.
|
||||
pub(super) fn apply_to_payload_v2(&self, payload: &mut ExecutionPayloadV2) -> Vec<String> {
|
||||
let mut changes = self.apply_to_payload_v1(&mut payload.payload_inner);
|
||||
|
||||
// Handle withdrawals invalidation (V2+)
|
||||
if self.invalidate_withdrawals {
|
||||
let fake_withdrawal = Withdrawal {
|
||||
index: u64::MAX,
|
||||
validator_index: u64::MAX,
|
||||
address: Address::ZERO,
|
||||
amount: u64::MAX,
|
||||
};
|
||||
payload.withdrawals.push(fake_withdrawal);
|
||||
changes.push("withdrawals = added fake withdrawal (auto-invalidated)".to_string());
|
||||
}
|
||||
|
||||
changes
|
||||
}
|
||||
|
||||
/// Applies invalidations to a V3 payload, returns list of what was changed.
|
||||
pub(super) fn apply_to_payload_v3(&self, payload: &mut ExecutionPayloadV3) -> Vec<String> {
|
||||
let mut changes = self.apply_to_payload_v2(&mut payload.payload_inner);
|
||||
|
||||
// Explicit overrides for V3 fields
|
||||
if let Some(blob_gas_used) = self.blob_gas_used {
|
||||
payload.blob_gas_used = blob_gas_used;
|
||||
changes.push(format!("blob_gas_used = {blob_gas_used}"));
|
||||
}
|
||||
|
||||
if let Some(excess_blob_gas) = self.excess_blob_gas {
|
||||
payload.excess_blob_gas = excess_blob_gas;
|
||||
changes.push(format!("excess_blob_gas = {excess_blob_gas}"));
|
||||
}
|
||||
|
||||
// Auto-invalidation for V3 fields
|
||||
if self.invalidate_blob_gas_used {
|
||||
payload.blob_gas_used = u64::MAX;
|
||||
changes.push("blob_gas_used = MAX (auto-invalidated)".to_string());
|
||||
}
|
||||
|
||||
if self.invalidate_excess_blob_gas {
|
||||
payload.excess_blob_gas = u64::MAX;
|
||||
changes.push("excess_blob_gas = MAX (auto-invalidated)".to_string());
|
||||
}
|
||||
|
||||
changes
|
||||
}
|
||||
}
|
||||
347
bin/reth-bench/src/bench/send_invalid_payload/mod.rs
Normal file
347
bin/reth-bench/src/bench/send_invalid_payload/mod.rs
Normal file
@@ -0,0 +1,347 @@
|
||||
//! Command for sending invalid payloads to test Engine API rejection.
|
||||
|
||||
mod invalidation;
|
||||
use invalidation::InvalidationConfig;
|
||||
|
||||
use super::helpers::{load_jwt_secret, read_input};
|
||||
use alloy_primitives::{Address, B256};
|
||||
use alloy_provider::network::AnyRpcBlock;
|
||||
use alloy_rpc_types_engine::ExecutionPayload;
|
||||
use clap::Parser;
|
||||
use eyre::{OptionExt, Result};
|
||||
use op_alloy_consensus::OpTxEnvelope;
|
||||
use reth_cli_runner::CliContext;
|
||||
use std::io::Write;
|
||||
|
||||
/// Command for generating and sending an invalid `engine_newPayload` request.
|
||||
///
|
||||
/// Takes a valid block and modifies fields to make it invalid for testing
|
||||
/// Engine API rejection behavior. Block hash is recalculated after modifications
|
||||
/// unless `--invalidate-block-hash` or `--skip-hash-recalc` is used.
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Command {
|
||||
// ==================== Input Options ====================
|
||||
/// Path to the JSON file containing the block. If not specified, stdin will be used.
|
||||
#[arg(short, long, help_heading = "Input Options")]
|
||||
path: Option<String>,
|
||||
|
||||
/// The engine RPC URL to use.
|
||||
#[arg(
|
||||
short,
|
||||
long,
|
||||
help_heading = "Input Options",
|
||||
required_if_eq_any([("mode", "execute"), ("mode", "cast")]),
|
||||
required_unless_present("mode")
|
||||
)]
|
||||
rpc_url: Option<String>,
|
||||
|
||||
/// The JWT secret to use. Can be either a path to a file containing the secret or the secret
|
||||
/// itself.
|
||||
#[arg(short, long, help_heading = "Input Options")]
|
||||
jwt_secret: Option<String>,
|
||||
|
||||
/// The newPayload version to use (3 or 4).
|
||||
#[arg(long, default_value_t = 3, help_heading = "Input Options")]
|
||||
new_payload_version: u8,
|
||||
|
||||
/// The output mode to use.
|
||||
#[arg(long, value_enum, default_value = "execute", help_heading = "Input Options")]
|
||||
mode: Mode,
|
||||
|
||||
// ==================== Explicit Value Overrides ====================
|
||||
/// Override the parent hash with a specific value.
|
||||
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
|
||||
parent_hash: Option<B256>,
|
||||
|
||||
/// Override the fee recipient (coinbase) with a specific address.
|
||||
#[arg(long, value_name = "ADDR", help_heading = "Explicit Value Overrides")]
|
||||
fee_recipient: Option<Address>,
|
||||
|
||||
/// Override the state root with a specific value.
|
||||
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
|
||||
state_root: Option<B256>,
|
||||
|
||||
/// Override the receipts root with a specific value.
|
||||
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
|
||||
receipts_root: Option<B256>,
|
||||
|
||||
/// Override the block number with a specific value.
|
||||
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
|
||||
block_number: Option<u64>,
|
||||
|
||||
/// Override the gas limit with a specific value.
|
||||
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
|
||||
gas_limit: Option<u64>,
|
||||
|
||||
/// Override the gas used with a specific value.
|
||||
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
|
||||
gas_used: Option<u64>,
|
||||
|
||||
/// Override the timestamp with a specific value.
|
||||
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
|
||||
timestamp: Option<u64>,
|
||||
|
||||
/// Override the base fee per gas with a specific value.
|
||||
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
|
||||
base_fee_per_gas: Option<u64>,
|
||||
|
||||
/// Override the block hash with a specific value (skips hash recalculation).
|
||||
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
|
||||
block_hash: Option<B256>,
|
||||
|
||||
/// Override the blob gas used with a specific value.
|
||||
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
|
||||
blob_gas_used: Option<u64>,
|
||||
|
||||
/// Override the excess blob gas with a specific value.
|
||||
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
|
||||
excess_blob_gas: Option<u64>,
|
||||
|
||||
/// Override the parent beacon block root with a specific value.
|
||||
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
|
||||
parent_beacon_block_root: Option<B256>,
|
||||
|
||||
/// Override the requests hash with a specific value (EIP-7685).
|
||||
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
|
||||
requests_hash: Option<B256>,
|
||||
|
||||
// ==================== Auto-Invalidation Flags ====================
|
||||
/// Invalidate the parent hash by setting it to a random value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_parent_hash: bool,
|
||||
|
||||
/// Invalidate the state root by setting it to a random value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_state_root: bool,
|
||||
|
||||
/// Invalidate the receipts root by setting it to a random value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_receipts_root: bool,
|
||||
|
||||
/// Invalidate the gas used by setting it to an incorrect value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_gas_used: bool,
|
||||
|
||||
/// Invalidate the block number by setting it to an incorrect value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_block_number: bool,
|
||||
|
||||
/// Invalidate the timestamp by setting it to an incorrect value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_timestamp: bool,
|
||||
|
||||
/// Invalidate the base fee by setting it to an incorrect value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_base_fee: bool,
|
||||
|
||||
/// Invalidate the transactions by modifying them.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_transactions: bool,
|
||||
|
||||
/// Invalidate the block hash by not recalculating it after modifications.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_block_hash: bool,
|
||||
|
||||
/// Invalidate the withdrawals by modifying them.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_withdrawals: bool,
|
||||
|
||||
/// Invalidate the blob gas used by setting it to an incorrect value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_blob_gas_used: bool,
|
||||
|
||||
/// Invalidate the excess blob gas by setting it to an incorrect value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_excess_blob_gas: bool,
|
||||
|
||||
/// Invalidate the requests hash by setting it to a random value (EIP-7685).
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_requests_hash: bool,
|
||||
|
||||
// ==================== Meta Flags ====================
|
||||
/// Skip block hash recalculation after modifications.
|
||||
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
|
||||
skip_hash_recalc: bool,
|
||||
|
||||
/// Print what would be done without actually sending the payload.
|
||||
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
|
||||
dry_run: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, clap::ValueEnum)]
|
||||
enum Mode {
|
||||
/// Execute the `cast` command. This works with blocks of any size, because it pipes the
|
||||
/// payload into the `cast` command.
|
||||
Execute,
|
||||
/// Print the `cast` command. Caution: this may not work with large blocks because of the
|
||||
/// command length limit.
|
||||
Cast,
|
||||
/// Print the JSON payload. Can be piped into `cast` command if the block is small enough.
|
||||
Json,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Build `InvalidationConfig` from command flags
|
||||
const fn build_invalidation_config(&self) -> InvalidationConfig {
|
||||
InvalidationConfig {
|
||||
parent_hash: self.parent_hash,
|
||||
fee_recipient: self.fee_recipient,
|
||||
state_root: self.state_root,
|
||||
receipts_root: self.receipts_root,
|
||||
logs_bloom: None,
|
||||
prev_randao: None,
|
||||
block_number: self.block_number,
|
||||
gas_limit: self.gas_limit,
|
||||
gas_used: self.gas_used,
|
||||
timestamp: self.timestamp,
|
||||
extra_data: None,
|
||||
base_fee_per_gas: self.base_fee_per_gas,
|
||||
block_hash: self.block_hash,
|
||||
blob_gas_used: self.blob_gas_used,
|
||||
excess_blob_gas: self.excess_blob_gas,
|
||||
invalidate_parent_hash: self.invalidate_parent_hash,
|
||||
invalidate_state_root: self.invalidate_state_root,
|
||||
invalidate_receipts_root: self.invalidate_receipts_root,
|
||||
invalidate_gas_used: self.invalidate_gas_used,
|
||||
invalidate_block_number: self.invalidate_block_number,
|
||||
invalidate_timestamp: self.invalidate_timestamp,
|
||||
invalidate_base_fee: self.invalidate_base_fee,
|
||||
invalidate_transactions: self.invalidate_transactions,
|
||||
invalidate_block_hash: self.invalidate_block_hash,
|
||||
invalidate_withdrawals: self.invalidate_withdrawals,
|
||||
invalidate_blob_gas_used: self.invalidate_blob_gas_used,
|
||||
invalidate_excess_blob_gas: self.invalidate_excess_blob_gas,
|
||||
}
|
||||
}
|
||||
|
||||
/// Execute the command
|
||||
pub async fn execute(self, _ctx: CliContext) -> Result<()> {
|
||||
let block_json = read_input(self.path.as_deref())?;
|
||||
let jwt_secret = load_jwt_secret(self.jwt_secret.as_deref())?;
|
||||
|
||||
let block = serde_json::from_str::<AnyRpcBlock>(&block_json)?
|
||||
.into_inner()
|
||||
.map_header(|header| header.map(|h| h.into_header_with_defaults()))
|
||||
.try_map_transactions(|tx| tx.try_into_either::<OpTxEnvelope>())?
|
||||
.into_consensus();
|
||||
|
||||
let config = self.build_invalidation_config();
|
||||
|
||||
let parent_beacon_block_root =
|
||||
self.parent_beacon_block_root.or(block.header.parent_beacon_block_root);
|
||||
let blob_versioned_hashes =
|
||||
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
|
||||
let use_v4 = block.header.requests_hash.is_some();
|
||||
let requests_hash = self.requests_hash.or(block.header.requests_hash);
|
||||
|
||||
let mut execution_payload = ExecutionPayload::from_block_slow(&block).0;
|
||||
|
||||
let changes = match &mut execution_payload {
|
||||
ExecutionPayload::V1(p) => config.apply_to_payload_v1(p),
|
||||
ExecutionPayload::V2(p) => config.apply_to_payload_v2(p),
|
||||
ExecutionPayload::V3(p) => config.apply_to_payload_v3(p),
|
||||
};
|
||||
|
||||
let skip_recalc = self.skip_hash_recalc || config.should_skip_hash_recalc();
|
||||
if !skip_recalc {
|
||||
let new_hash = match execution_payload.clone().into_block_raw() {
|
||||
Ok(block) => block.header.hash_slow(),
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"Warning: Could not recalculate block hash: {e}. Using original hash."
|
||||
);
|
||||
match &execution_payload {
|
||||
ExecutionPayload::V1(p) => p.block_hash,
|
||||
ExecutionPayload::V2(p) => p.payload_inner.block_hash,
|
||||
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash,
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
match &mut execution_payload {
|
||||
ExecutionPayload::V1(p) => p.block_hash = new_hash,
|
||||
ExecutionPayload::V2(p) => p.payload_inner.block_hash = new_hash,
|
||||
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash = new_hash,
|
||||
}
|
||||
}
|
||||
|
||||
if self.dry_run {
|
||||
println!("=== Dry Run ===");
|
||||
println!("Changes that would be applied:");
|
||||
for change in &changes {
|
||||
println!(" - {}", change);
|
||||
}
|
||||
if changes.is_empty() {
|
||||
println!(" (no changes)");
|
||||
}
|
||||
if skip_recalc {
|
||||
println!(" - Block hash recalculation: SKIPPED");
|
||||
} else {
|
||||
println!(" - Block hash recalculation: PERFORMED");
|
||||
}
|
||||
println!("\nResulting payload JSON:");
|
||||
let json = serde_json::to_string_pretty(&execution_payload)?;
|
||||
println!("{}", json);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let json_request = if use_v4 {
|
||||
serde_json::to_string(&(
|
||||
execution_payload,
|
||||
blob_versioned_hashes,
|
||||
parent_beacon_block_root,
|
||||
requests_hash.unwrap_or_default(),
|
||||
))?
|
||||
} else {
|
||||
serde_json::to_string(&(
|
||||
execution_payload,
|
||||
blob_versioned_hashes,
|
||||
parent_beacon_block_root,
|
||||
))?
|
||||
};
|
||||
|
||||
match self.mode {
|
||||
Mode::Execute => {
|
||||
let mut command = std::process::Command::new("cast");
|
||||
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
|
||||
command.arg("rpc").arg(method).arg("--raw");
|
||||
if let Some(rpc_url) = self.rpc_url {
|
||||
command.arg("--rpc-url").arg(rpc_url);
|
||||
}
|
||||
if let Some(secret) = &jwt_secret {
|
||||
command.arg("--jwt-secret").arg(secret);
|
||||
}
|
||||
|
||||
let mut process = command.stdin(std::process::Stdio::piped()).spawn()?;
|
||||
|
||||
process
|
||||
.stdin
|
||||
.take()
|
||||
.ok_or_eyre("stdin not available")?
|
||||
.write_all(json_request.as_bytes())?;
|
||||
|
||||
process.wait()?;
|
||||
}
|
||||
Mode::Cast => {
|
||||
let mut cmd = format!(
|
||||
"cast rpc engine_newPayloadV{} --raw '{}'",
|
||||
self.new_payload_version, json_request
|
||||
);
|
||||
|
||||
if let Some(rpc_url) = self.rpc_url {
|
||||
cmd += &format!(" --rpc-url {rpc_url}");
|
||||
}
|
||||
if let Some(secret) = &jwt_secret {
|
||||
cmd += &format!(" --jwt-secret {secret}");
|
||||
}
|
||||
|
||||
println!("{cmd}");
|
||||
}
|
||||
Mode::Json => {
|
||||
println!("{json_request}");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,11 @@
|
||||
use super::helpers::{load_jwt_secret, read_input};
|
||||
use alloy_provider::network::AnyRpcBlock;
|
||||
use alloy_rpc_types_engine::ExecutionPayload;
|
||||
use clap::Parser;
|
||||
use eyre::{OptionExt, Result};
|
||||
use op_alloy_consensus::OpTxEnvelope;
|
||||
use reth_cli_runner::CliContext;
|
||||
use std::io::{BufReader, Read, Write};
|
||||
use std::io::Write;
|
||||
|
||||
/// Command for generating and sending an `engine_newPayload` request constructed from an RPC
|
||||
/// block.
|
||||
@@ -51,38 +52,13 @@ enum Mode {
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Read input from either a file or stdin
|
||||
fn read_input(&self) -> Result<String> {
|
||||
Ok(match &self.path {
|
||||
Some(path) => reth_fs_util::read_to_string(path)?,
|
||||
None => String::from_utf8(
|
||||
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Load JWT secret from either a file or use the provided string directly
|
||||
fn load_jwt_secret(&self) -> Result<Option<String>> {
|
||||
match &self.jwt_secret {
|
||||
Some(secret) => {
|
||||
// Try to read as file first
|
||||
match std::fs::read_to_string(secret) {
|
||||
Ok(contents) => Ok(Some(contents.trim().to_string())),
|
||||
// If file read fails, use the string directly
|
||||
Err(_) => Ok(Some(secret.clone())),
|
||||
}
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Execute the generate payload command
|
||||
pub async fn execute(self, _ctx: CliContext) -> Result<()> {
|
||||
// Load block
|
||||
let block_json = self.read_input()?;
|
||||
let block_json = read_input(self.path.as_deref())?;
|
||||
|
||||
// Load JWT secret
|
||||
let jwt_secret = self.load_jwt_secret()?;
|
||||
let jwt_secret = load_jwt_secret(self.jwt_secret.as_deref())?;
|
||||
|
||||
// Parse the block
|
||||
let block = serde_json::from_str::<AnyRpcBlock>(&block_json)?
|
||||
|
||||
@@ -3,15 +3,16 @@
|
||||
//! before sending additional calls.
|
||||
|
||||
use alloy_eips::eip7685::Requests;
|
||||
use alloy_primitives::B256;
|
||||
use alloy_provider::{ext::EngineApi, network::AnyRpcBlock, Network, Provider};
|
||||
use alloy_rpc_types_engine::{
|
||||
ExecutionPayload, ExecutionPayloadInputV2, ForkchoiceState, ForkchoiceUpdated,
|
||||
PayloadAttributes, PayloadStatus,
|
||||
ExecutionPayload, ExecutionPayloadInputV2, ExecutionPayloadSidecar, ForkchoiceState,
|
||||
ForkchoiceUpdated, PayloadAttributes, PayloadStatus,
|
||||
};
|
||||
use alloy_transport::TransportResult;
|
||||
use op_alloy_rpc_types_engine::OpExecutionPayloadV4;
|
||||
use reth_node_api::EngineApiMessageVersion;
|
||||
use tracing::error;
|
||||
use tracing::{debug, error};
|
||||
|
||||
/// An extension trait for providers that implement the engine API, to wait for a VALID response.
|
||||
#[async_trait::async_trait]
|
||||
@@ -52,6 +53,13 @@ where
|
||||
fork_choice_state: ForkchoiceState,
|
||||
payload_attributes: Option<PayloadAttributes>,
|
||||
) -> TransportResult<ForkchoiceUpdated> {
|
||||
debug!(
|
||||
method = "engine_forkchoiceUpdatedV1",
|
||||
?fork_choice_state,
|
||||
?payload_attributes,
|
||||
"Sending forkchoiceUpdated"
|
||||
);
|
||||
|
||||
let mut status =
|
||||
self.fork_choice_updated_v1(fork_choice_state, payload_attributes.clone()).await?;
|
||||
|
||||
@@ -82,6 +90,13 @@ where
|
||||
fork_choice_state: ForkchoiceState,
|
||||
payload_attributes: Option<PayloadAttributes>,
|
||||
) -> TransportResult<ForkchoiceUpdated> {
|
||||
debug!(
|
||||
method = "engine_forkchoiceUpdatedV2",
|
||||
?fork_choice_state,
|
||||
?payload_attributes,
|
||||
"Sending forkchoiceUpdated"
|
||||
);
|
||||
|
||||
let mut status =
|
||||
self.fork_choice_updated_v2(fork_choice_state, payload_attributes.clone()).await?;
|
||||
|
||||
@@ -112,6 +127,13 @@ where
|
||||
fork_choice_state: ForkchoiceState,
|
||||
payload_attributes: Option<PayloadAttributes>,
|
||||
) -> TransportResult<ForkchoiceUpdated> {
|
||||
debug!(
|
||||
method = "engine_forkchoiceUpdatedV3",
|
||||
?fork_choice_state,
|
||||
?payload_attributes,
|
||||
"Sending forkchoiceUpdated"
|
||||
);
|
||||
|
||||
let mut status =
|
||||
self.fork_choice_updated_v3(fork_choice_state, payload_attributes.clone()).await?;
|
||||
|
||||
@@ -148,33 +170,47 @@ pub(crate) fn block_to_new_payload(
|
||||
|
||||
// Convert to execution payload
|
||||
let (payload, sidecar) = ExecutionPayload::from_block_slow(&block);
|
||||
payload_to_new_payload(payload, sidecar, is_optimism, block.withdrawals_root, None)
|
||||
}
|
||||
|
||||
pub(crate) fn payload_to_new_payload(
|
||||
payload: ExecutionPayload,
|
||||
sidecar: ExecutionPayloadSidecar,
|
||||
is_optimism: bool,
|
||||
withdrawals_root: Option<B256>,
|
||||
target_version: Option<EngineApiMessageVersion>,
|
||||
) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value)> {
|
||||
let (version, params) = match payload {
|
||||
ExecutionPayload::V3(payload) => {
|
||||
let cancun = sidecar.cancun().unwrap();
|
||||
|
||||
if let Some(prague) = sidecar.prague() {
|
||||
// Use target version if provided (for Osaka), otherwise default to V4
|
||||
let version = target_version.unwrap_or(EngineApiMessageVersion::V4);
|
||||
|
||||
if is_optimism {
|
||||
let withdrawals_root = withdrawals_root.ok_or_else(|| {
|
||||
eyre::eyre!("Missing withdrawals root for Optimism payload")
|
||||
})?;
|
||||
(
|
||||
EngineApiMessageVersion::V4,
|
||||
version,
|
||||
serde_json::to_value((
|
||||
OpExecutionPayloadV4 {
|
||||
payload_inner: payload,
|
||||
withdrawals_root: block.withdrawals_root.unwrap(),
|
||||
},
|
||||
OpExecutionPayloadV4 { payload_inner: payload, withdrawals_root },
|
||||
cancun.versioned_hashes.clone(),
|
||||
cancun.parent_beacon_block_root,
|
||||
Requests::default(),
|
||||
))?,
|
||||
)
|
||||
} else {
|
||||
// Extract actual Requests from RequestsOrHash
|
||||
let requests = prague.requests.requests_hash();
|
||||
(
|
||||
EngineApiMessageVersion::V4,
|
||||
version,
|
||||
serde_json::to_value((
|
||||
payload,
|
||||
cancun.versioned_hashes.clone(),
|
||||
cancun.parent_beacon_block_root,
|
||||
prague.requests.requests_hash(),
|
||||
requests,
|
||||
))?,
|
||||
)
|
||||
}
|
||||
@@ -217,12 +253,16 @@ pub(crate) async fn call_new_payload<N: Network, P: Provider<N>>(
|
||||
) -> TransportResult<()> {
|
||||
let method = version.method_name();
|
||||
|
||||
debug!(method, "Sending newPayload");
|
||||
|
||||
let mut status: PayloadStatus = provider.client().request(method, ¶ms).await?;
|
||||
|
||||
while !status.is_valid() {
|
||||
if status.is_invalid() {
|
||||
error!(?status, ?params, "Invalid {method}",);
|
||||
panic!("Invalid {method}: {status:?}");
|
||||
return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(std::io::Error::other(
|
||||
format!("Invalid {method}: {status:?}"),
|
||||
))))
|
||||
}
|
||||
if status.is_syncing() {
|
||||
return Err(alloy_json_rpc::RpcError::UnsupportedFeature(
|
||||
@@ -237,12 +277,15 @@ pub(crate) async fn call_new_payload<N: Network, P: Provider<N>>(
|
||||
/// Calls the correct `engine_forkchoiceUpdated` method depending on the given
|
||||
/// `EngineApiMessageVersion`, using the provided forkchoice state and payload attributes for the
|
||||
/// actual engine api message call.
|
||||
///
|
||||
/// Note: For Prague (V4), we still use forkchoiceUpdatedV3 as there is no V4.
|
||||
pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
|
||||
provider: P,
|
||||
message_version: EngineApiMessageVersion,
|
||||
forkchoice_state: ForkchoiceState,
|
||||
payload_attributes: Option<PayloadAttributes>,
|
||||
) -> TransportResult<ForkchoiceUpdated> {
|
||||
// FCU V3 is used for both Cancun and Prague (there is no FCU V4)
|
||||
match message_version {
|
||||
EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 | EngineApiMessageVersion::V5 => {
|
||||
provider.fork_choice_updated_v3_wait(forkchoice_state, payload_attributes).await
|
||||
|
||||
@@ -81,12 +81,16 @@ backon.workspace = true
|
||||
tempfile.workspace = true
|
||||
|
||||
[features]
|
||||
default = ["jemalloc", "otlp", "reth-revm/portable", "js-tracer", "keccak-cache-global", "asm-keccak"]
|
||||
default = ["jemalloc", "otlp", "otlp-logs", "reth-revm/portable", "js-tracer", "keccak-cache-global", "asm-keccak"]
|
||||
|
||||
otlp = [
|
||||
"reth-ethereum-cli/otlp",
|
||||
"reth-node-core/otlp",
|
||||
]
|
||||
otlp-logs = [
|
||||
"reth-ethereum-cli/otlp-logs",
|
||||
"reth-node-core/otlp-logs",
|
||||
]
|
||||
js-tracer = [
|
||||
"reth-node-builder/js-tracer",
|
||||
"reth-node-ethereum/js-tracer",
|
||||
@@ -131,6 +135,11 @@ jemalloc-unprefixed = [
|
||||
tracy-allocator = [
|
||||
"reth-cli-util/tracy-allocator",
|
||||
"reth-ethereum-cli/tracy-allocator",
|
||||
"tracy",
|
||||
]
|
||||
tracy = [
|
||||
"reth-ethereum-cli/tracy",
|
||||
"reth-node-core/tracy",
|
||||
]
|
||||
|
||||
# Because jemalloc is default and preferred over snmalloc when both features are
|
||||
@@ -171,7 +180,7 @@ min-trace-logs = [
|
||||
"reth-node-core/min-trace-logs",
|
||||
]
|
||||
|
||||
edge = ["reth-ethereum-cli/edge"]
|
||||
edge = ["reth-ethereum-cli/edge", "reth-node-core/edge"]
|
||||
|
||||
[[bin]]
|
||||
name = "reth"
|
||||
|
||||
@@ -25,8 +25,8 @@
|
||||
//! - `jemalloc-unprefixed`: Uses unprefixed jemalloc symbols.
|
||||
//! - `tracy-allocator`: Enables [Tracy](https://github.com/wolfpld/tracy) profiler allocator
|
||||
//! integration for memory profiling.
|
||||
//! - `snmalloc`: Uses [snmalloc](https://github.com/snmalloc/snmalloc) as the global allocator. Use
|
||||
//! `--no-default-features` when enabling this, as jemalloc takes precedence.
|
||||
//! - `snmalloc`: Uses [snmalloc](https://github.com/microsoft/snmalloc) as the global allocator.
|
||||
//! Use `--no-default-features` when enabling this, as jemalloc takes precedence.
|
||||
//! - `snmalloc-native`: Uses snmalloc with native CPU optimizations. Use `--no-default-features`
|
||||
//! when enabling this.
|
||||
//!
|
||||
|
||||
@@ -41,6 +41,7 @@ derive_more.workspace = true
|
||||
metrics.workspace = true
|
||||
parking_lot.workspace = true
|
||||
pin-project.workspace = true
|
||||
rayon = { workspace = true, optional = true }
|
||||
serde = { workspace = true, optional = true }
|
||||
|
||||
# optional deps for test-utils
|
||||
@@ -84,6 +85,7 @@ test-utils = [
|
||||
"reth-trie/test-utils",
|
||||
"reth-ethereum-primitives/test-utils",
|
||||
]
|
||||
rayon = ["dep:rayon"]
|
||||
|
||||
[[bench]]
|
||||
name = "canonical_hashes_range"
|
||||
|
||||
@@ -163,14 +163,29 @@ impl DeferredTrieData {
|
||||
anchor_hash: B256,
|
||||
ancestors: &[Self],
|
||||
) -> ComputedTrieData {
|
||||
let sorted_hashed_state = match Arc::try_unwrap(hashed_state) {
|
||||
Ok(state) => state.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
};
|
||||
let sorted_trie_updates = match Arc::try_unwrap(trie_updates) {
|
||||
Ok(updates) => updates.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
};
|
||||
#[cfg(feature = "rayon")]
|
||||
let (sorted_hashed_state, sorted_trie_updates) = rayon::join(
|
||||
|| match Arc::try_unwrap(hashed_state) {
|
||||
Ok(state) => state.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
},
|
||||
|| match Arc::try_unwrap(trie_updates) {
|
||||
Ok(updates) => updates.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
},
|
||||
);
|
||||
|
||||
#[cfg(not(feature = "rayon"))]
|
||||
let (sorted_hashed_state, sorted_trie_updates) = (
|
||||
match Arc::try_unwrap(hashed_state) {
|
||||
Ok(state) => state.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
},
|
||||
match Arc::try_unwrap(trie_updates) {
|
||||
Ok(updates) => updates.into_sorted(),
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
},
|
||||
);
|
||||
|
||||
// Reuse parent's overlay if available and anchors match.
|
||||
// We can only reuse the parent's overlay if it was built on top of the same
|
||||
@@ -191,11 +206,33 @@ impl DeferredTrieData {
|
||||
Default::default(), // prefix_sets are per-block, not cumulative
|
||||
);
|
||||
// Only trigger COW clone if there's actually data to add.
|
||||
if !sorted_hashed_state.is_empty() {
|
||||
Arc::make_mut(&mut overlay.state).extend_ref(&sorted_hashed_state);
|
||||
#[cfg(feature = "rayon")]
|
||||
{
|
||||
rayon::join(
|
||||
|| {
|
||||
if !sorted_hashed_state.is_empty() {
|
||||
Arc::make_mut(&mut overlay.state)
|
||||
.extend_ref_and_sort(&sorted_hashed_state);
|
||||
}
|
||||
},
|
||||
|| {
|
||||
if !sorted_trie_updates.is_empty() {
|
||||
Arc::make_mut(&mut overlay.nodes)
|
||||
.extend_ref_and_sort(&sorted_trie_updates);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
if !sorted_trie_updates.is_empty() {
|
||||
Arc::make_mut(&mut overlay.nodes).extend_ref(&sorted_trie_updates);
|
||||
#[cfg(not(feature = "rayon"))]
|
||||
{
|
||||
if !sorted_hashed_state.is_empty() {
|
||||
Arc::make_mut(&mut overlay.state)
|
||||
.extend_ref_and_sort(&sorted_hashed_state);
|
||||
}
|
||||
if !sorted_trie_updates.is_empty() {
|
||||
Arc::make_mut(&mut overlay.nodes)
|
||||
.extend_ref_and_sort(&sorted_trie_updates);
|
||||
}
|
||||
}
|
||||
overlay
|
||||
}
|
||||
@@ -242,13 +279,22 @@ impl DeferredTrieData {
|
||||
|
||||
for ancestor in ancestors {
|
||||
let ancestor_data = ancestor.wait_cloned();
|
||||
state_mut.extend_ref(ancestor_data.hashed_state.as_ref());
|
||||
nodes_mut.extend_ref(ancestor_data.trie_updates.as_ref());
|
||||
state_mut.extend_ref_and_sort(ancestor_data.hashed_state.as_ref());
|
||||
nodes_mut.extend_ref_and_sort(ancestor_data.trie_updates.as_ref());
|
||||
}
|
||||
|
||||
// Extend with current block's sorted data last (takes precedence)
|
||||
state_mut.extend_ref(sorted_hashed_state);
|
||||
nodes_mut.extend_ref(sorted_trie_updates);
|
||||
#[cfg(feature = "rayon")]
|
||||
rayon::join(
|
||||
|| state_mut.extend_ref_and_sort(sorted_hashed_state),
|
||||
|| nodes_mut.extend_ref_and_sort(sorted_trie_updates),
|
||||
);
|
||||
|
||||
#[cfg(not(feature = "rayon"))]
|
||||
{
|
||||
state_mut.extend_ref_and_sort(sorted_hashed_state);
|
||||
nodes_mut.extend_ref_and_sort(sorted_trie_updates);
|
||||
}
|
||||
|
||||
overlay
|
||||
}
|
||||
@@ -287,6 +333,11 @@ impl DeferredTrieData {
|
||||
&inputs.ancestors,
|
||||
);
|
||||
*state = DeferredState::Ready(computed.clone());
|
||||
|
||||
// Release lock before inputs (and its ancestors) drop to avoid holding it
|
||||
// while their potential last Arc refs drop (which could trigger recursive locking)
|
||||
drop(state);
|
||||
|
||||
computed
|
||||
}
|
||||
}
|
||||
@@ -516,7 +567,7 @@ mod tests {
|
||||
let hashed_state = Arc::new(HashedPostStateSorted::new(accounts, B256Map::default()));
|
||||
let trie_updates = Arc::default();
|
||||
let mut overlay = TrieInputSorted::default();
|
||||
Arc::make_mut(&mut overlay.state).extend_ref(hashed_state.as_ref());
|
||||
Arc::make_mut(&mut overlay.state).extend_ref_and_sort(hashed_state.as_ref());
|
||||
|
||||
DeferredTrieData::ready(ComputedTrieData {
|
||||
hashed_state,
|
||||
|
||||
@@ -10,15 +10,18 @@ use alloy_primitives::{map::HashMap, BlockNumber, TxHash, B256};
|
||||
use parking_lot::RwLock;
|
||||
use reth_chainspec::ChainInfo;
|
||||
use reth_ethereum_primitives::EthPrimitives;
|
||||
use reth_execution_types::{Chain, ExecutionOutcome};
|
||||
use reth_execution_types::{BlockExecutionOutput, BlockExecutionResult, Chain, ExecutionOutcome};
|
||||
use reth_metrics::{metrics::Gauge, Metrics};
|
||||
use reth_primitives_traits::{
|
||||
BlockBody as _, IndexedTx, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader,
|
||||
SignedTransaction,
|
||||
};
|
||||
use reth_storage_api::StateProviderBox;
|
||||
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted};
|
||||
use std::{collections::BTreeMap, ops::Deref, sync::Arc, time::Instant};
|
||||
use reth_trie::{
|
||||
updates::TrieUpdatesSorted, HashedPostStateSorted, LazyTrieData, SortedTrieData,
|
||||
TrieInputSorted,
|
||||
};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::Instant};
|
||||
use tokio::sync::{broadcast, watch};
|
||||
|
||||
/// Size of the broadcast channel used to notify canonical state events.
|
||||
@@ -648,7 +651,7 @@ impl<N: NodePrimitives> BlockState<N> {
|
||||
}
|
||||
|
||||
/// Returns the `Receipts` of executed block that determines the state.
|
||||
pub fn receipts(&self) -> &Vec<Vec<N::Receipt>> {
|
||||
pub fn receipts(&self) -> &Vec<N::Receipt> {
|
||||
&self.block.execution_outcome().receipts
|
||||
}
|
||||
|
||||
@@ -659,15 +662,7 @@ impl<N: NodePrimitives> BlockState<N> {
|
||||
///
|
||||
/// This clones the vector of receipts. To avoid it, use [`Self::executed_block_receipts_ref`].
|
||||
pub fn executed_block_receipts(&self) -> Vec<N::Receipt> {
|
||||
let receipts = self.receipts();
|
||||
|
||||
debug_assert!(
|
||||
receipts.len() <= 1,
|
||||
"Expected at most one block's worth of receipts, found {}",
|
||||
receipts.len()
|
||||
);
|
||||
|
||||
receipts.first().cloned().unwrap_or_default()
|
||||
self.receipts().clone()
|
||||
}
|
||||
|
||||
/// Returns a slice of `Receipt` of executed block that determines the state.
|
||||
@@ -675,15 +670,7 @@ impl<N: NodePrimitives> BlockState<N> {
|
||||
/// has only one element corresponding to the executed block associated to
|
||||
/// the state.
|
||||
pub fn executed_block_receipts_ref(&self) -> &[N::Receipt] {
|
||||
let receipts = self.receipts();
|
||||
|
||||
debug_assert!(
|
||||
receipts.len() <= 1,
|
||||
"Expected at most one block's worth of receipts, found {}",
|
||||
receipts.len()
|
||||
);
|
||||
|
||||
receipts.first().map(|receipts| receipts.deref()).unwrap_or_default()
|
||||
self.receipts()
|
||||
}
|
||||
|
||||
/// Returns an iterator over __parent__ `BlockStates`.
|
||||
@@ -767,7 +754,7 @@ pub struct ExecutedBlock<N: NodePrimitives = EthPrimitives> {
|
||||
/// Recovered Block
|
||||
pub recovered_block: Arc<RecoveredBlock<N::Block>>,
|
||||
/// Block's execution outcome.
|
||||
pub execution_output: Arc<ExecutionOutcome<N::Receipt>>,
|
||||
pub execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
|
||||
/// Deferred trie data produced by execution.
|
||||
///
|
||||
/// This allows deferring the computation of the trie data which can be expensive.
|
||||
@@ -779,7 +766,15 @@ impl<N: NodePrimitives> Default for ExecutedBlock<N> {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
recovered_block: Default::default(),
|
||||
execution_output: Default::default(),
|
||||
execution_output: Arc::new(BlockExecutionOutput {
|
||||
result: BlockExecutionResult {
|
||||
receipts: Default::default(),
|
||||
requests: Default::default(),
|
||||
gas_used: 0,
|
||||
blob_gas_used: 0,
|
||||
},
|
||||
state: Default::default(),
|
||||
}),
|
||||
trie_data: DeferredTrieData::ready(ComputedTrieData::default()),
|
||||
}
|
||||
}
|
||||
@@ -800,7 +795,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
|
||||
/// payload builders). This is the safe default path.
|
||||
pub fn new(
|
||||
recovered_block: Arc<RecoveredBlock<N::Block>>,
|
||||
execution_output: Arc<ExecutionOutcome<N::Receipt>>,
|
||||
execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
|
||||
trie_data: ComputedTrieData,
|
||||
) -> Self {
|
||||
Self { recovered_block, execution_output, trie_data: DeferredTrieData::ready(trie_data) }
|
||||
@@ -822,7 +817,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
|
||||
/// Use [`Self::new()`] instead when trie data is already computed and available immediately.
|
||||
pub const fn with_deferred_trie_data(
|
||||
recovered_block: Arc<RecoveredBlock<N::Block>>,
|
||||
execution_output: Arc<ExecutionOutcome<N::Receipt>>,
|
||||
execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
|
||||
trie_data: DeferredTrieData,
|
||||
) -> Self {
|
||||
Self { recovered_block, execution_output, trie_data }
|
||||
@@ -842,7 +837,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
|
||||
|
||||
/// Returns a reference to the block's execution outcome
|
||||
#[inline]
|
||||
pub fn execution_outcome(&self) -> &ExecutionOutcome<N::Receipt> {
|
||||
pub fn execution_outcome(&self) -> &BlockExecutionOutput<N::Receipt> {
|
||||
&self.execution_output
|
||||
}
|
||||
|
||||
@@ -942,37 +937,53 @@ impl<N: NodePrimitives<SignedTx: SignedTransaction>> NewCanonicalChain<N> {
|
||||
pub fn to_chain_notification(&self) -> CanonStateNotification<N> {
|
||||
match self {
|
||||
Self::Commit { new } => {
|
||||
let new = Arc::new(new.iter().fold(Chain::default(), |mut chain, exec| {
|
||||
chain.append_block(
|
||||
exec.recovered_block().clone(),
|
||||
exec.execution_outcome().clone(),
|
||||
exec.trie_updates(),
|
||||
exec.hashed_state(),
|
||||
);
|
||||
chain
|
||||
}));
|
||||
CanonStateNotification::Commit { new }
|
||||
CanonStateNotification::Commit { new: Arc::new(Self::blocks_to_chain(new)) }
|
||||
}
|
||||
Self::Reorg { new, old } => {
|
||||
let new = Arc::new(new.iter().fold(Chain::default(), |mut chain, exec| {
|
||||
Self::Reorg { new, old } => CanonStateNotification::Reorg {
|
||||
new: Arc::new(Self::blocks_to_chain(new)),
|
||||
old: Arc::new(Self::blocks_to_chain(old)),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts a slice of executed blocks into a [`Chain`].
|
||||
fn blocks_to_chain(blocks: &[ExecutedBlock<N>]) -> Chain<N> {
|
||||
match blocks {
|
||||
[] => Chain::default(),
|
||||
[first, rest @ ..] => {
|
||||
let trie_data_handle = first.trie_data_handle();
|
||||
let mut chain = Chain::from_block(
|
||||
first.recovered_block().clone(),
|
||||
ExecutionOutcome::from((
|
||||
first.execution_outcome().clone(),
|
||||
first.block_number(),
|
||||
)),
|
||||
LazyTrieData::deferred(move || {
|
||||
let trie_data = trie_data_handle.wait_cloned();
|
||||
SortedTrieData {
|
||||
hashed_state: trie_data.hashed_state,
|
||||
trie_updates: trie_data.trie_updates,
|
||||
}
|
||||
}),
|
||||
);
|
||||
for exec in rest {
|
||||
let trie_data_handle = exec.trie_data_handle();
|
||||
chain.append_block(
|
||||
exec.recovered_block().clone(),
|
||||
exec.execution_outcome().clone(),
|
||||
exec.trie_updates(),
|
||||
exec.hashed_state(),
|
||||
ExecutionOutcome::from((
|
||||
exec.execution_outcome().clone(),
|
||||
exec.block_number(),
|
||||
)),
|
||||
LazyTrieData::deferred(move || {
|
||||
let trie_data = trie_data_handle.wait_cloned();
|
||||
SortedTrieData {
|
||||
hashed_state: trie_data.hashed_state,
|
||||
trie_updates: trie_data.trie_updates,
|
||||
}
|
||||
}),
|
||||
);
|
||||
chain
|
||||
}));
|
||||
let old = Arc::new(old.iter().fold(Chain::default(), |mut chain, exec| {
|
||||
chain.append_block(
|
||||
exec.recovered_block().clone(),
|
||||
exec.execution_outcome().clone(),
|
||||
exec.trie_updates(),
|
||||
exec.hashed_state(),
|
||||
);
|
||||
chain
|
||||
}));
|
||||
CanonStateNotification::Reorg { new, old }
|
||||
}
|
||||
chain
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1266,7 +1277,7 @@ mod tests {
|
||||
|
||||
let state = BlockState::new(block);
|
||||
|
||||
assert_eq!(state.receipts(), &receipts);
|
||||
assert_eq!(state.receipts(), receipts.first().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1543,33 +1554,31 @@ mod tests {
|
||||
let block2a =
|
||||
test_block_builder.get_executed_block_with_number(2, block1.recovered_block.hash());
|
||||
|
||||
let sample_execution_outcome = ExecutionOutcome {
|
||||
receipts: vec![vec![], vec![]],
|
||||
requests: vec![Requests::default(), Requests::default()],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Test commit notification
|
||||
let chain_commit = NewCanonicalChain::Commit { new: vec![block0.clone(), block1.clone()] };
|
||||
|
||||
// Build expected trie updates map
|
||||
let mut expected_trie_updates = BTreeMap::new();
|
||||
expected_trie_updates.insert(0, block0.trie_updates());
|
||||
expected_trie_updates.insert(1, block1.trie_updates());
|
||||
// Build expected trie data map
|
||||
let mut expected_trie_data = BTreeMap::new();
|
||||
expected_trie_data
|
||||
.insert(0, LazyTrieData::ready(block0.hashed_state(), block0.trie_updates()));
|
||||
expected_trie_data
|
||||
.insert(1, LazyTrieData::ready(block1.hashed_state(), block1.trie_updates()));
|
||||
|
||||
// Build expected hashed state map
|
||||
let mut expected_hashed_state = BTreeMap::new();
|
||||
expected_hashed_state.insert(0, block0.hashed_state());
|
||||
expected_hashed_state.insert(1, block1.hashed_state());
|
||||
// Build expected execution outcome (first_block matches first block number)
|
||||
let commit_execution_outcome = ExecutionOutcome {
|
||||
receipts: vec![vec![], vec![]],
|
||||
requests: vec![Requests::default(), Requests::default()],
|
||||
first_block: 0,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
chain_commit.to_chain_notification(),
|
||||
CanonStateNotification::Commit {
|
||||
new: Arc::new(Chain::new(
|
||||
vec![block0.recovered_block().clone(), block1.recovered_block().clone()],
|
||||
sample_execution_outcome.clone(),
|
||||
expected_trie_updates,
|
||||
expected_hashed_state
|
||||
commit_execution_outcome,
|
||||
expected_trie_data,
|
||||
))
|
||||
}
|
||||
);
|
||||
@@ -1580,40 +1589,39 @@ mod tests {
|
||||
old: vec![block1.clone(), block2.clone()],
|
||||
};
|
||||
|
||||
// Build expected trie updates for old chain
|
||||
let mut old_trie_updates = BTreeMap::new();
|
||||
old_trie_updates.insert(1, block1.trie_updates());
|
||||
old_trie_updates.insert(2, block2.trie_updates());
|
||||
// Build expected trie data for old chain
|
||||
let mut old_trie_data = BTreeMap::new();
|
||||
old_trie_data.insert(1, LazyTrieData::ready(block1.hashed_state(), block1.trie_updates()));
|
||||
old_trie_data.insert(2, LazyTrieData::ready(block2.hashed_state(), block2.trie_updates()));
|
||||
|
||||
// Build expected trie updates for new chain
|
||||
let mut new_trie_updates = BTreeMap::new();
|
||||
new_trie_updates.insert(1, block1a.trie_updates());
|
||||
new_trie_updates.insert(2, block2a.trie_updates());
|
||||
// Build expected trie data for new chain
|
||||
let mut new_trie_data = BTreeMap::new();
|
||||
new_trie_data
|
||||
.insert(1, LazyTrieData::ready(block1a.hashed_state(), block1a.trie_updates()));
|
||||
new_trie_data
|
||||
.insert(2, LazyTrieData::ready(block2a.hashed_state(), block2a.trie_updates()));
|
||||
|
||||
// Build expected hashed state for old chain
|
||||
let mut old_hashed_state = BTreeMap::new();
|
||||
old_hashed_state.insert(1, block1.hashed_state());
|
||||
old_hashed_state.insert(2, block2.hashed_state());
|
||||
|
||||
// Build expected hashed state for new chain
|
||||
let mut new_hashed_state = BTreeMap::new();
|
||||
new_hashed_state.insert(1, block1a.hashed_state());
|
||||
new_hashed_state.insert(2, block2a.hashed_state());
|
||||
// Build expected execution outcome for reorg chains (first_block matches first block
|
||||
// number)
|
||||
let reorg_execution_outcome = ExecutionOutcome {
|
||||
receipts: vec![vec![], vec![]],
|
||||
requests: vec![Requests::default(), Requests::default()],
|
||||
first_block: 1,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
chain_reorg.to_chain_notification(),
|
||||
CanonStateNotification::Reorg {
|
||||
old: Arc::new(Chain::new(
|
||||
vec![block1.recovered_block().clone(), block2.recovered_block().clone()],
|
||||
sample_execution_outcome.clone(),
|
||||
old_trie_updates,
|
||||
old_hashed_state
|
||||
reorg_execution_outcome.clone(),
|
||||
old_trie_data,
|
||||
)),
|
||||
new: Arc::new(Chain::new(
|
||||
vec![block1a.recovered_block().clone(), block2a.recovered_block().clone()],
|
||||
sample_execution_outcome,
|
||||
new_trie_updates,
|
||||
new_hashed_state
|
||||
reorg_execution_outcome,
|
||||
new_trie_data,
|
||||
))
|
||||
}
|
||||
);
|
||||
|
||||
186
crates/chain-state/src/lazy_overlay.rs
Normal file
186
crates/chain-state/src/lazy_overlay.rs
Normal file
@@ -0,0 +1,186 @@
|
||||
//! Lazy overlay computation for trie input.
|
||||
//!
|
||||
//! This module provides [`LazyOverlay`], a type that computes the [`TrieInputSorted`]
|
||||
//! lazily on first access. This allows execution to start before the trie overlay
|
||||
//! is fully computed.
|
||||
|
||||
use crate::DeferredTrieData;
|
||||
use alloy_primitives::B256;
|
||||
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use tracing::{debug, trace};
|
||||
|
||||
/// Inputs captured for lazy overlay computation.
|
||||
#[derive(Clone)]
|
||||
struct LazyOverlayInputs {
|
||||
/// The persisted ancestor hash (anchor) this overlay should be built on.
|
||||
anchor_hash: B256,
|
||||
/// Deferred trie data handles for all in-memory blocks (newest to oldest).
|
||||
blocks: Vec<DeferredTrieData>,
|
||||
}
|
||||
|
||||
/// Lazily computed trie overlay.
|
||||
///
|
||||
/// Captures the inputs needed to compute a [`TrieInputSorted`] and defers the actual
|
||||
/// computation until first access. This is conceptually similar to [`DeferredTrieData`]
|
||||
/// but for overlay computation.
|
||||
///
|
||||
/// # Fast Path vs Slow Path
|
||||
///
|
||||
/// - **Fast path**: If the tip block's cached `anchored_trie_input` is ready and its `anchor_hash`
|
||||
/// matches our expected anchor, we can reuse it directly (O(1)).
|
||||
/// - **Slow path**: Otherwise, we merge all ancestor blocks' trie data into a new overlay.
|
||||
#[derive(Clone)]
|
||||
pub struct LazyOverlay {
|
||||
/// Computed result, cached after first access.
|
||||
inner: Arc<OnceLock<TrieInputSorted>>,
|
||||
/// Inputs for lazy computation.
|
||||
inputs: LazyOverlayInputs,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for LazyOverlay {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("LazyOverlay")
|
||||
.field("anchor_hash", &self.inputs.anchor_hash)
|
||||
.field("num_blocks", &self.inputs.blocks.len())
|
||||
.field("computed", &self.inner.get().is_some())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl LazyOverlay {
|
||||
/// Create a new lazy overlay with the given anchor hash and block handles.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `anchor_hash` - The persisted ancestor hash this overlay is built on top of
|
||||
/// * `blocks` - Deferred trie data handles for in-memory blocks (newest to oldest)
|
||||
pub fn new(anchor_hash: B256, blocks: Vec<DeferredTrieData>) -> Self {
|
||||
Self { inner: Arc::new(OnceLock::new()), inputs: LazyOverlayInputs { anchor_hash, blocks } }
|
||||
}
|
||||
|
||||
/// Returns the anchor hash this overlay is built on.
|
||||
pub const fn anchor_hash(&self) -> B256 {
|
||||
self.inputs.anchor_hash
|
||||
}
|
||||
|
||||
/// Returns the number of in-memory blocks this overlay covers.
|
||||
pub const fn num_blocks(&self) -> usize {
|
||||
self.inputs.blocks.len()
|
||||
}
|
||||
|
||||
/// Returns true if the overlay has already been computed.
|
||||
pub fn is_computed(&self) -> bool {
|
||||
self.inner.get().is_some()
|
||||
}
|
||||
|
||||
/// Returns the computed trie input, computing it if necessary.
|
||||
///
|
||||
/// The first call triggers computation (which may block waiting for deferred data).
|
||||
/// Subsequent calls return the cached result immediately.
|
||||
pub fn get(&self) -> &TrieInputSorted {
|
||||
self.inner.get_or_init(|| self.compute())
|
||||
}
|
||||
|
||||
/// Returns the overlay as (nodes, state) tuple for use with `OverlayStateProviderFactory`.
|
||||
pub fn as_overlay(&self) -> (Arc<TrieUpdatesSorted>, Arc<HashedPostStateSorted>) {
|
||||
let input = self.get();
|
||||
(Arc::clone(&input.nodes), Arc::clone(&input.state))
|
||||
}
|
||||
|
||||
/// Compute the trie input overlay.
|
||||
fn compute(&self) -> TrieInputSorted {
|
||||
let anchor_hash = self.inputs.anchor_hash;
|
||||
let blocks = &self.inputs.blocks;
|
||||
|
||||
if blocks.is_empty() {
|
||||
debug!(target: "chain_state::lazy_overlay", "No in-memory blocks, returning empty overlay");
|
||||
return TrieInputSorted::default();
|
||||
}
|
||||
|
||||
// Fast path: Check if tip block's overlay is ready and anchor matches.
|
||||
// The tip block (first in list) has the cumulative overlay from all ancestors.
|
||||
if let Some(tip) = blocks.first() {
|
||||
let data = tip.wait_cloned();
|
||||
if let Some(anchored) = &data.anchored_trie_input {
|
||||
if anchored.anchor_hash == anchor_hash {
|
||||
trace!(target: "chain_state::lazy_overlay", %anchor_hash, "Reusing tip block's cached overlay (fast path)");
|
||||
return (*anchored.trie_input).clone();
|
||||
}
|
||||
debug!(
|
||||
target: "chain_state::lazy_overlay",
|
||||
computed_anchor = %anchored.anchor_hash,
|
||||
%anchor_hash,
|
||||
"Anchor mismatch, falling back to merge"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Slow path: Merge all blocks' trie data into a new overlay.
|
||||
debug!(target: "chain_state::lazy_overlay", num_blocks = blocks.len(), "Merging blocks (slow path)");
|
||||
Self::merge_blocks(blocks)
|
||||
}
|
||||
|
||||
/// Merge all blocks' trie data into a single [`TrieInputSorted`].
|
||||
///
|
||||
/// Blocks are ordered newest to oldest.
|
||||
fn merge_blocks(blocks: &[DeferredTrieData]) -> TrieInputSorted {
|
||||
if blocks.is_empty() {
|
||||
return TrieInputSorted::default();
|
||||
}
|
||||
|
||||
let state =
|
||||
HashedPostStateSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().hashed_state));
|
||||
let nodes =
|
||||
TrieUpdatesSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().trie_updates));
|
||||
|
||||
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use reth_trie::{updates::TrieUpdates, HashedPostState};
|
||||
|
||||
fn empty_deferred(anchor: B256) -> DeferredTrieData {
|
||||
DeferredTrieData::pending(
|
||||
Arc::new(HashedPostState::default()),
|
||||
Arc::new(TrieUpdates::default()),
|
||||
anchor,
|
||||
Vec::new(),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_blocks_returns_default() {
|
||||
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
|
||||
let result = overlay.get();
|
||||
assert!(result.state.is_empty());
|
||||
assert!(result.nodes.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_block_uses_data_directly() {
|
||||
let anchor = B256::random();
|
||||
let deferred = empty_deferred(anchor);
|
||||
let overlay = LazyOverlay::new(anchor, vec![deferred]);
|
||||
|
||||
assert!(!overlay.is_computed());
|
||||
let _ = overlay.get();
|
||||
assert!(overlay.is_computed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_after_first_access() {
|
||||
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
|
||||
|
||||
// First access computes
|
||||
let _ = overlay.get();
|
||||
assert!(overlay.is_computed());
|
||||
|
||||
// Second access uses cache
|
||||
let _ = overlay.get();
|
||||
assert!(overlay.is_computed());
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,9 @@ pub use in_memory::*;
|
||||
mod deferred_trie;
|
||||
pub use deferred_trie::*;
|
||||
|
||||
mod lazy_overlay;
|
||||
pub use lazy_overlay::*;
|
||||
|
||||
mod noop;
|
||||
|
||||
mod chain_info;
|
||||
|
||||
@@ -280,7 +280,6 @@ mod tests {
|
||||
vec![block1.clone(), block2.clone()],
|
||||
ExecutionOutcome::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
));
|
||||
|
||||
// Create a commit notification
|
||||
@@ -319,13 +318,11 @@ mod tests {
|
||||
vec![block1.clone()],
|
||||
ExecutionOutcome::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
));
|
||||
let new_chain = Arc::new(Chain::new(
|
||||
vec![block2.clone(), block3.clone()],
|
||||
ExecutionOutcome::default(),
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
));
|
||||
|
||||
// Create a reorg notification
|
||||
@@ -391,7 +388,6 @@ mod tests {
|
||||
vec![block1.clone(), block2.clone()],
|
||||
execution_outcome,
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
));
|
||||
|
||||
// Create a commit notification containing the new chain segment.
|
||||
@@ -449,12 +445,8 @@ mod tests {
|
||||
ExecutionOutcome { receipts: old_receipts, ..Default::default() };
|
||||
|
||||
// Create an old chain segment to be reverted, containing `old_block1`.
|
||||
let old_chain: Arc<Chain> = Arc::new(Chain::new(
|
||||
vec![old_block1.clone()],
|
||||
old_execution_outcome,
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
));
|
||||
let old_chain: Arc<Chain> =
|
||||
Arc::new(Chain::new(vec![old_block1.clone()], old_execution_outcome, BTreeMap::new()));
|
||||
|
||||
// Define block2 for the new chain segment, which will be committed.
|
||||
let mut body = BlockBody::<TransactionSigned>::default();
|
||||
@@ -482,12 +474,8 @@ mod tests {
|
||||
ExecutionOutcome { receipts: new_receipts, ..Default::default() };
|
||||
|
||||
// Create a new chain segment to be committed, containing `new_block1`.
|
||||
let new_chain = Arc::new(Chain::new(
|
||||
vec![new_block1.clone()],
|
||||
new_execution_outcome,
|
||||
BTreeMap::new(),
|
||||
BTreeMap::new(),
|
||||
));
|
||||
let new_chain =
|
||||
Arc::new(Chain::new(vec![new_block1.clone()], new_execution_outcome, BTreeMap::new()));
|
||||
|
||||
// Create a reorg notification with both reverted (old) and committed (new) chain segments.
|
||||
let notification = CanonStateNotification::Reorg { old: old_chain, new: new_chain };
|
||||
|
||||
@@ -3,10 +3,7 @@ use crate::{
|
||||
CanonStateSubscriptions, ComputedTrieData,
|
||||
};
|
||||
use alloy_consensus::{Header, SignableTransaction, TxEip1559, TxReceipt, EMPTY_ROOT_HASH};
|
||||
use alloy_eips::{
|
||||
eip1559::{ETHEREUM_BLOCK_GAS_LIMIT_30M, INITIAL_BASE_FEE},
|
||||
eip7685::Requests,
|
||||
};
|
||||
use alloy_eips::eip1559::{ETHEREUM_BLOCK_GAS_LIMIT_30M, INITIAL_BASE_FEE};
|
||||
use alloy_primitives::{Address, BlockNumber, B256, U256};
|
||||
use alloy_signer::SignerSync;
|
||||
use alloy_signer_local::PrivateKeySigner;
|
||||
@@ -16,7 +13,7 @@ use reth_chainspec::{ChainSpec, EthereumHardfork, MIN_TRANSACTION_GAS};
|
||||
use reth_ethereum_primitives::{
|
||||
Block, BlockBody, EthPrimitives, Receipt, Transaction, TransactionSigned,
|
||||
};
|
||||
use reth_execution_types::{Chain, ExecutionOutcome};
|
||||
use reth_execution_types::{BlockExecutionOutput, BlockExecutionResult, Chain, ExecutionOutcome};
|
||||
use reth_primitives_traits::{
|
||||
proofs::{calculate_receipt_root, calculate_transaction_root, calculate_withdrawals_root},
|
||||
Account, NodePrimitives, Recovered, RecoveredBlock, SealedBlock, SealedHeader,
|
||||
@@ -201,7 +198,7 @@ impl<N: NodePrimitives> TestBlockBuilder<N> {
|
||||
fn get_executed_block(
|
||||
&mut self,
|
||||
block_number: BlockNumber,
|
||||
receipts: Vec<Vec<Receipt>>,
|
||||
mut receipts: Vec<Vec<Receipt>>,
|
||||
parent_hash: B256,
|
||||
) -> ExecutedBlock {
|
||||
let block = self.generate_random_block(block_number, parent_hash);
|
||||
@@ -209,12 +206,15 @@ impl<N: NodePrimitives> TestBlockBuilder<N> {
|
||||
let trie_data = ComputedTrieData::default();
|
||||
ExecutedBlock::new(
|
||||
Arc::new(RecoveredBlock::new_sealed(block, senders)),
|
||||
Arc::new(ExecutionOutcome::new(
|
||||
BundleState::default(),
|
||||
receipts,
|
||||
block_number,
|
||||
vec![Requests::default()],
|
||||
)),
|
||||
Arc::new(BlockExecutionOutput {
|
||||
result: BlockExecutionResult {
|
||||
receipts: receipts.pop().unwrap_or_default(),
|
||||
requests: Default::default(),
|
||||
gas_used: 0,
|
||||
blob_gas_used: 0,
|
||||
},
|
||||
state: BundleState::default(),
|
||||
}),
|
||||
trie_data,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ pub use alloy_chains::{Chain, ChainKind, NamedChain};
|
||||
/// Re-export for convenience
|
||||
pub use reth_ethereum_forks::*;
|
||||
|
||||
pub use alloy_evm::EvmLimitParams;
|
||||
pub use api::EthChainSpec;
|
||||
pub use info::ChainInfo;
|
||||
#[cfg(any(test, feature = "test-utils"))]
|
||||
|
||||
@@ -278,6 +278,7 @@ pub fn create_chain_config(
|
||||
// Check if DAO fork is supported (it has an activation block)
|
||||
let dao_fork_support = hardforks.fork(EthereumHardfork::Dao) != ForkCondition::Never;
|
||||
|
||||
#[allow(clippy::needless_update)]
|
||||
ChainConfig {
|
||||
chain_id: chain.map(|c| c.id()).unwrap_or(0),
|
||||
homestead_block: block_num(EthereumHardfork::Homestead),
|
||||
@@ -313,6 +314,7 @@ pub fn create_chain_config(
|
||||
extra_fields: Default::default(),
|
||||
deposit_contract_address,
|
||||
blob_schedule,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -460,6 +462,18 @@ impl ChainSpec {
|
||||
pub fn builder() -> ChainSpecBuilder {
|
||||
ChainSpecBuilder::default()
|
||||
}
|
||||
|
||||
/// Map a chain ID to a known chain spec, if available.
|
||||
pub fn from_chain_id(chain_id: u64) -> Option<Arc<Self>> {
|
||||
match NamedChain::try_from(chain_id).ok()? {
|
||||
NamedChain::Mainnet => Some(MAINNET.clone()),
|
||||
NamedChain::Sepolia => Some(SEPOLIA.clone()),
|
||||
NamedChain::Holesky => Some(HOLESKY.clone()),
|
||||
NamedChain::Hoodi => Some(HOODI.clone()),
|
||||
NamedChain::Dev => Some(DEV.clone()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<H: BlockHeader> ChainSpec<H> {
|
||||
|
||||
@@ -50,6 +50,7 @@ reth-stages-types = { workspace = true, optional = true }
|
||||
reth-static-file-types = { workspace = true, features = ["clap"] }
|
||||
reth-static-file.workspace = true
|
||||
reth-tasks.workspace = true
|
||||
reth-storage-api.workspace = true
|
||||
reth-trie = { workspace = true, features = ["metrics"] }
|
||||
reth-trie-db = { workspace = true, features = ["metrics"] }
|
||||
reth-trie-common.workspace = true
|
||||
@@ -77,12 +78,14 @@ lz4.workspace = true
|
||||
zstd.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
parking_lot.workspace = true
|
||||
tar.workspace = true
|
||||
tracing.workspace = true
|
||||
backon.workspace = true
|
||||
secp256k1 = { workspace = true, features = ["global-context", "std", "recovery"] }
|
||||
tokio-stream.workspace = true
|
||||
reqwest.workspace = true
|
||||
url.workspace = true
|
||||
metrics.workspace = true
|
||||
|
||||
# io
|
||||
@@ -130,4 +133,4 @@ arbitrary = [
|
||||
"reth-ethereum-primitives/arbitrary",
|
||||
]
|
||||
|
||||
edge = ["reth-db-common/edge", "reth-stages/rocksdb"]
|
||||
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb"]
|
||||
|
||||
@@ -19,7 +19,7 @@ use reth_node_builder::{
|
||||
Node, NodeComponents, NodeComponentsBuilder, NodeTypes, NodeTypesWithDBAdapter,
|
||||
};
|
||||
use reth_node_core::{
|
||||
args::{DatabaseArgs, DatadirArgs, StaticFilesArgs},
|
||||
args::{DatabaseArgs, DatadirArgs, RocksDbArgs, StaticFilesArgs},
|
||||
dirs::{ChainPath, DataDirPath},
|
||||
};
|
||||
use reth_provider::{
|
||||
@@ -27,7 +27,7 @@ use reth_provider::{
|
||||
BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider,
|
||||
StaticFileProviderBuilder,
|
||||
},
|
||||
ProviderFactory, StaticFileProviderFactory,
|
||||
ProviderFactory, StaticFileProviderFactory, StorageSettings,
|
||||
};
|
||||
use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget};
|
||||
use reth_static_file::StaticFileProducer;
|
||||
@@ -66,9 +66,24 @@ pub struct EnvironmentArgs<C: ChainSpecParser> {
|
||||
/// All static files related arguments
|
||||
#[command(flatten)]
|
||||
pub static_files: StaticFilesArgs,
|
||||
|
||||
/// All `RocksDB` related arguments
|
||||
#[command(flatten)]
|
||||
pub rocksdb: RocksDbArgs,
|
||||
}
|
||||
|
||||
impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
/// Returns the effective storage settings derived from static-file and `RocksDB` CLI args.
|
||||
pub fn storage_settings(&self) -> StorageSettings {
|
||||
StorageSettings::base()
|
||||
.with_receipts_in_static_files(self.static_files.receipts)
|
||||
.with_transaction_senders_in_static_files(self.static_files.transaction_senders)
|
||||
.with_account_changesets_in_static_files(self.static_files.account_changesets)
|
||||
.with_transaction_hash_numbers_in_rocksdb(self.rocksdb.all || self.rocksdb.tx_hash)
|
||||
.with_storages_history_in_rocksdb(self.rocksdb.all || self.rocksdb.storages_history)
|
||||
.with_account_history_in_rocksdb(self.rocksdb.all || self.rocksdb.account_history)
|
||||
}
|
||||
|
||||
/// Initializes environment according to [`AccessRights`] and returns an instance of
|
||||
/// [`Environment`].
|
||||
pub fn init<N: CliNodeTypes>(&self, access: AccessRights) -> eyre::Result<Environment<N>>
|
||||
@@ -121,17 +136,17 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
})
|
||||
}
|
||||
};
|
||||
// TransactionDB only support read-write mode
|
||||
let rocksdb_provider = RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level)
|
||||
.with_read_only(!access.is_read_write())
|
||||
.build()?;
|
||||
|
||||
let provider_factory =
|
||||
self.create_provider_factory(&config, db, sfp, rocksdb_provider, access)?;
|
||||
if access.is_read_write() {
|
||||
debug!(target: "reth::cli", chain=%self.chain.chain(), genesis=?self.chain.genesis_hash(), "Initializing genesis");
|
||||
init_genesis_with_settings(&provider_factory, self.static_files.to_settings())?;
|
||||
init_genesis_with_settings(&provider_factory, self.storage_settings())?;
|
||||
}
|
||||
|
||||
Ok(Environment { config, provider_factory, data_dir })
|
||||
|
||||
@@ -1,145 +0,0 @@
|
||||
use crate::{
|
||||
common::CliNodeTypes,
|
||||
db::get::{maybe_json_value_parser, table_key},
|
||||
};
|
||||
use alloy_primitives::map::foldhash::fast::FixedState;
|
||||
use clap::Parser;
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_db::DatabaseEnv;
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
|
||||
Tables,
|
||||
};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
|
||||
use reth_provider::{providers::ProviderNodeTypes, DBProvider};
|
||||
use std::{
|
||||
hash::{BuildHasher, Hasher},
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
/// The arguments for the `reth db checksum` command
|
||||
pub struct Command {
|
||||
/// The table name
|
||||
table: Tables,
|
||||
|
||||
/// The start of the range to checksum.
|
||||
#[arg(long, value_parser = maybe_json_value_parser)]
|
||||
start_key: Option<String>,
|
||||
|
||||
/// The end of the range to checksum.
|
||||
#[arg(long, value_parser = maybe_json_value_parser)]
|
||||
end_key: Option<String>,
|
||||
|
||||
/// The maximum number of records that are queried and used to compute the
|
||||
/// checksum.
|
||||
#[arg(long)]
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `db checksum` command
|
||||
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
self,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
) -> eyre::Result<()> {
|
||||
warn!("This command should be run without the node running!");
|
||||
self.table.view(&ChecksumViewer {
|
||||
tool,
|
||||
start_key: self.start_key,
|
||||
end_key: self.end_key,
|
||||
limit: self.limit,
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
|
||||
tool: &'a DbTool<N>,
|
||||
start_key: Option<String>,
|
||||
end_key: Option<String>,
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
|
||||
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
|
||||
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
|
||||
type Error = eyre::Report;
|
||||
|
||||
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
|
||||
let provider =
|
||||
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
|
||||
let tx = provider.tx_ref();
|
||||
info!(
|
||||
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
|
||||
self.start_key, self.end_key, self.limit
|
||||
);
|
||||
|
||||
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
|
||||
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
|
||||
(Some(start), Some(end)) => {
|
||||
let start_key = table_key::<T>(start).map(RawKey::new)?;
|
||||
let end_key = table_key::<T>(end).map(RawKey::new)?;
|
||||
cursor.walk_range(start_key..=end_key)?
|
||||
}
|
||||
(None, Some(end)) => {
|
||||
let end_key = table_key::<T>(end).map(RawKey::new)?;
|
||||
|
||||
cursor.walk_range(..=end_key)?
|
||||
}
|
||||
(Some(start), None) => {
|
||||
let start_key = table_key::<T>(start).map(RawKey::new)?;
|
||||
cursor.walk_range(start_key..)?
|
||||
}
|
||||
(None, None) => cursor.walk_range(..)?,
|
||||
};
|
||||
|
||||
let start_time = Instant::now();
|
||||
let mut hasher = FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher();
|
||||
let mut total = 0;
|
||||
|
||||
let limit = self.limit.unwrap_or(usize::MAX);
|
||||
let mut enumerate_start_key = None;
|
||||
let mut enumerate_end_key = None;
|
||||
for (index, entry) in walker.enumerate() {
|
||||
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
|
||||
|
||||
if index.is_multiple_of(100_000) {
|
||||
info!("Hashed {index} entries.");
|
||||
}
|
||||
|
||||
hasher.write(k.raw_key());
|
||||
hasher.write(v.raw_value());
|
||||
|
||||
if enumerate_start_key.is_none() {
|
||||
enumerate_start_key = Some(k.clone());
|
||||
}
|
||||
enumerate_end_key = Some(k);
|
||||
|
||||
total = index + 1;
|
||||
if total >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
info!("Hashed {total} entries.");
|
||||
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
|
||||
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
|
||||
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
|
||||
}
|
||||
|
||||
let checksum = hasher.finish();
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
|
||||
|
||||
Ok((checksum, elapsed))
|
||||
}
|
||||
}
|
||||
288
crates/cli/commands/src/db/checksum/mod.rs
Normal file
288
crates/cli/commands/src/db/checksum/mod.rs
Normal file
@@ -0,0 +1,288 @@
|
||||
use crate::{
|
||||
common::CliNodeTypes,
|
||||
db::get::{maybe_json_value_parser, table_key},
|
||||
};
|
||||
use alloy_primitives::map::foldhash::fast::FixedState;
|
||||
use clap::Parser;
|
||||
use itertools::Itertools;
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_db::{static_file::iter_static_files, DatabaseEnv};
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
|
||||
Tables,
|
||||
};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
|
||||
use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProviderFactory};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use std::{
|
||||
hash::{BuildHasher, Hasher},
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
#[cfg(all(unix, feature = "edge"))]
|
||||
mod rocksdb;
|
||||
|
||||
/// Interval for logging progress during checksum computation.
|
||||
const PROGRESS_LOG_INTERVAL: usize = 100_000;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
/// The arguments for the `reth db checksum` command
|
||||
pub struct Command {
|
||||
#[command(subcommand)]
|
||||
subcommand: Subcommand,
|
||||
}
|
||||
|
||||
#[derive(clap::Subcommand, Debug)]
|
||||
enum Subcommand {
|
||||
/// Calculates the checksum of a database table
|
||||
Mdbx {
|
||||
/// The table name
|
||||
table: Tables,
|
||||
|
||||
/// The start of the range to checksum.
|
||||
#[arg(long, value_parser = maybe_json_value_parser)]
|
||||
start_key: Option<String>,
|
||||
|
||||
/// The end of the range to checksum.
|
||||
#[arg(long, value_parser = maybe_json_value_parser)]
|
||||
end_key: Option<String>,
|
||||
|
||||
/// The maximum number of records that are queried and used to compute the
|
||||
/// checksum.
|
||||
#[arg(long)]
|
||||
limit: Option<usize>,
|
||||
},
|
||||
/// Calculates the checksum of a static file segment
|
||||
StaticFile {
|
||||
/// The static file segment
|
||||
#[arg(value_enum)]
|
||||
segment: StaticFileSegment,
|
||||
|
||||
/// The block number to start from (inclusive).
|
||||
#[arg(long)]
|
||||
start_block: Option<u64>,
|
||||
|
||||
/// The block number to end at (inclusive).
|
||||
#[arg(long)]
|
||||
end_block: Option<u64>,
|
||||
|
||||
/// The maximum number of rows to checksum.
|
||||
#[arg(long)]
|
||||
limit: Option<usize>,
|
||||
},
|
||||
/// Calculates the checksum of a RocksDB table
|
||||
#[cfg(all(unix, feature = "edge"))]
|
||||
Rocksdb {
|
||||
/// The RocksDB table
|
||||
#[arg(value_enum)]
|
||||
table: rocksdb::RocksDbTable,
|
||||
|
||||
/// The maximum number of records to checksum.
|
||||
#[arg(long)]
|
||||
limit: Option<usize>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `db checksum` command
|
||||
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
self,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
) -> eyre::Result<()> {
|
||||
warn!("This command should be run without the node running!");
|
||||
|
||||
match self.subcommand {
|
||||
Subcommand::Mdbx { table, start_key, end_key, limit } => {
|
||||
table.view(&ChecksumViewer { tool, start_key, end_key, limit })?;
|
||||
}
|
||||
Subcommand::StaticFile { segment, start_block, end_block, limit } => {
|
||||
checksum_static_file(tool, segment, start_block, end_block, limit)?;
|
||||
}
|
||||
#[cfg(all(unix, feature = "edge"))]
|
||||
Subcommand::Rocksdb { table, limit } => {
|
||||
rocksdb::checksum_rocksdb(tool, table, limit)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a new hasher with the standard seed used for checksum computation.
|
||||
fn checksum_hasher() -> impl Hasher {
|
||||
FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher()
|
||||
}
|
||||
|
||||
fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
segment: StaticFileSegment,
|
||||
start_block: Option<u64>,
|
||||
end_block: Option<u64>,
|
||||
limit: Option<usize>,
|
||||
) -> eyre::Result<()> {
|
||||
let static_file_provider = tool.provider_factory.static_file_provider();
|
||||
if let Err(err) = static_file_provider.check_consistency(&tool.provider_factory.provider()?) {
|
||||
warn!("Error checking consistency of static files: {err}");
|
||||
}
|
||||
|
||||
let static_files = iter_static_files(static_file_provider.directory())?;
|
||||
|
||||
let ranges = static_files
|
||||
.get(segment)
|
||||
.ok_or_else(|| eyre::eyre!("No static files found for segment: {}", segment))?;
|
||||
|
||||
let start_time = Instant::now();
|
||||
let mut hasher = checksum_hasher();
|
||||
let mut total = 0usize;
|
||||
let limit = limit.unwrap_or(usize::MAX);
|
||||
|
||||
let start_block = start_block.unwrap_or(0);
|
||||
let end_block = end_block.unwrap_or(u64::MAX);
|
||||
|
||||
info!(
|
||||
"Computing checksum for {} static files, start_block={}, end_block={}, limit={:?}",
|
||||
segment,
|
||||
start_block,
|
||||
end_block,
|
||||
if limit == usize::MAX { None } else { Some(limit) }
|
||||
);
|
||||
|
||||
'outer: for (block_range, _header) in ranges.iter().sorted_by_key(|(range, _)| range.start()) {
|
||||
if block_range.end() < start_block || block_range.start() > end_block {
|
||||
continue;
|
||||
}
|
||||
|
||||
let fixed_block_range = static_file_provider.find_fixed_range(segment, block_range.start());
|
||||
let jar_provider = static_file_provider
|
||||
.get_segment_provider_for_range(segment, || Some(fixed_block_range), None)?
|
||||
.ok_or_else(|| {
|
||||
eyre::eyre!(
|
||||
"Failed to get segment provider for segment {} at range {}",
|
||||
segment,
|
||||
block_range
|
||||
)
|
||||
})?;
|
||||
|
||||
let mut cursor = jar_provider.cursor()?;
|
||||
|
||||
while let Ok(Some(row)) = cursor.next_row() {
|
||||
for col_data in row.iter() {
|
||||
hasher.write(col_data);
|
||||
}
|
||||
|
||||
total += 1;
|
||||
|
||||
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
|
||||
info!("Hashed {total} entries.");
|
||||
}
|
||||
|
||||
if total >= limit {
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
|
||||
// Explicitly drop provider before removing from cache to avoid deadlock
|
||||
drop(jar_provider);
|
||||
static_file_provider.remove_cached_provider(segment, fixed_block_range.end());
|
||||
}
|
||||
|
||||
let checksum = hasher.finish();
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
info!(
|
||||
"Checksum for static file segment `{}`: {:#x} ({} entries, elapsed: {:?})",
|
||||
segment, checksum, total, elapsed
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
|
||||
tool: &'a DbTool<N>,
|
||||
start_key: Option<String>,
|
||||
end_key: Option<String>,
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
|
||||
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
|
||||
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
|
||||
type Error = eyre::Report;
|
||||
|
||||
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
|
||||
let provider =
|
||||
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
|
||||
let tx = provider.tx_ref();
|
||||
info!(
|
||||
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
|
||||
self.start_key, self.end_key, self.limit
|
||||
);
|
||||
|
||||
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
|
||||
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
|
||||
(Some(start), Some(end)) => {
|
||||
let start_key = table_key::<T>(start).map(RawKey::new)?;
|
||||
let end_key = table_key::<T>(end).map(RawKey::new)?;
|
||||
cursor.walk_range(start_key..=end_key)?
|
||||
}
|
||||
(None, Some(end)) => {
|
||||
let end_key = table_key::<T>(end).map(RawKey::new)?;
|
||||
|
||||
cursor.walk_range(..=end_key)?
|
||||
}
|
||||
(Some(start), None) => {
|
||||
let start_key = table_key::<T>(start).map(RawKey::new)?;
|
||||
cursor.walk_range(start_key..)?
|
||||
}
|
||||
(None, None) => cursor.walk_range(..)?,
|
||||
};
|
||||
|
||||
let start_time = Instant::now();
|
||||
let mut hasher = checksum_hasher();
|
||||
let mut total = 0;
|
||||
|
||||
let limit = self.limit.unwrap_or(usize::MAX);
|
||||
let mut enumerate_start_key = None;
|
||||
let mut enumerate_end_key = None;
|
||||
for (index, entry) in walker.enumerate() {
|
||||
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
|
||||
|
||||
if index.is_multiple_of(PROGRESS_LOG_INTERVAL) {
|
||||
info!("Hashed {index} entries.");
|
||||
}
|
||||
|
||||
hasher.write(k.raw_key());
|
||||
hasher.write(v.raw_value());
|
||||
|
||||
if enumerate_start_key.is_none() {
|
||||
enumerate_start_key = Some(k.clone());
|
||||
}
|
||||
enumerate_end_key = Some(k);
|
||||
|
||||
total = index + 1;
|
||||
if total >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
info!("Hashed {total} entries.");
|
||||
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
|
||||
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
|
||||
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
|
||||
}
|
||||
|
||||
let checksum = hasher.finish();
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
|
||||
|
||||
Ok((checksum, elapsed))
|
||||
}
|
||||
}
|
||||
106
crates/cli/commands/src/db/checksum/rocksdb.rs
Normal file
106
crates/cli/commands/src/db/checksum/rocksdb.rs
Normal file
@@ -0,0 +1,106 @@
|
||||
//! RocksDB checksum implementation.
|
||||
|
||||
use super::{checksum_hasher, PROGRESS_LOG_INTERVAL};
|
||||
use crate::common::CliNodeTypes;
|
||||
use clap::ValueEnum;
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_db::{tables, DatabaseEnv};
|
||||
use reth_db_api::table::Table;
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::NodeTypesWithDBAdapter;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use std::{hash::Hasher, sync::Arc, time::Instant};
|
||||
use tracing::info;
|
||||
|
||||
/// RocksDB tables that can be checksummed.
|
||||
#[derive(Debug, Clone, Copy, ValueEnum)]
|
||||
pub enum RocksDbTable {
|
||||
/// Transaction hash to transaction number mapping
|
||||
TransactionHashNumbers,
|
||||
/// Account history indices
|
||||
AccountsHistory,
|
||||
/// Storage history indices
|
||||
StoragesHistory,
|
||||
}
|
||||
|
||||
impl RocksDbTable {
|
||||
/// Returns the table name as a string
|
||||
const fn name(&self) -> &'static str {
|
||||
match self {
|
||||
Self::TransactionHashNumbers => tables::TransactionHashNumbers::NAME,
|
||||
Self::AccountsHistory => tables::AccountsHistory::NAME,
|
||||
Self::StoragesHistory => tables::StoragesHistory::NAME,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Computes a checksum for a RocksDB table.
|
||||
pub fn checksum_rocksdb<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
table: RocksDbTable,
|
||||
limit: Option<usize>,
|
||||
) -> eyre::Result<()> {
|
||||
let rocksdb = tool.provider_factory.rocksdb_provider();
|
||||
|
||||
let start_time = Instant::now();
|
||||
let limit = limit.unwrap_or(usize::MAX);
|
||||
|
||||
info!(
|
||||
"Computing checksum for RocksDB table `{}`, limit={:?}",
|
||||
table.name(),
|
||||
if limit == usize::MAX { None } else { Some(limit) }
|
||||
);
|
||||
|
||||
let (checksum, total) = match table {
|
||||
RocksDbTable::TransactionHashNumbers => {
|
||||
checksum_rocksdb_table::<tables::TransactionHashNumbers>(&rocksdb, limit)?
|
||||
}
|
||||
RocksDbTable::AccountsHistory => {
|
||||
checksum_rocksdb_table::<tables::AccountsHistory>(&rocksdb, limit)?
|
||||
}
|
||||
RocksDbTable::StoragesHistory => {
|
||||
checksum_rocksdb_table::<tables::StoragesHistory>(&rocksdb, limit)?
|
||||
}
|
||||
};
|
||||
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
info!(
|
||||
"Checksum for RocksDB table `{}`: {:#x} ({} entries, elapsed: {:?})",
|
||||
table.name(),
|
||||
checksum,
|
||||
total,
|
||||
elapsed
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Computes checksum for a specific RocksDB table by iterating over rows.
|
||||
fn checksum_rocksdb_table<T: Table>(
|
||||
rocksdb: &reth_provider::providers::RocksDBProvider,
|
||||
limit: usize,
|
||||
) -> eyre::Result<(u64, usize)> {
|
||||
let iter = rocksdb.raw_iter::<T>()?;
|
||||
let mut hasher = checksum_hasher();
|
||||
let mut total = 0usize;
|
||||
|
||||
for entry in iter {
|
||||
let (key_bytes, value_bytes) = entry?;
|
||||
|
||||
hasher.write(&key_bytes);
|
||||
hasher.write(&value_bytes);
|
||||
|
||||
total += 1;
|
||||
|
||||
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
|
||||
info!("Hashed {total} entries.");
|
||||
}
|
||||
|
||||
if total >= limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok((hasher.finish(), total))
|
||||
}
|
||||
@@ -29,7 +29,7 @@ impl Command {
|
||||
let static_file_provider = tool.provider_factory.static_file_provider();
|
||||
let static_files = iter_static_files(static_file_provider.directory())?;
|
||||
|
||||
if let Some(segment_static_files) = static_files.get(&segment) {
|
||||
if let Some(segment_static_files) = static_files.get(segment) {
|
||||
for (block_range, _) in segment_static_files {
|
||||
static_file_provider.delete_jar(segment, block_range.start())?;
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ use reth_node_builder::NodeTypesWithDB;
|
||||
use reth_primitives_traits::ValueWithSubKey;
|
||||
use reth_provider::{providers::ProviderNodeTypes, ChangeSetReader, StaticFileProviderFactory};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::StorageChangeSetReader;
|
||||
use tracing::error;
|
||||
|
||||
/// The arguments for the `reth db get` command
|
||||
@@ -82,6 +83,33 @@ impl Command {
|
||||
table.view(&GetValueViewer { tool, key, subkey, end_key, end_subkey, raw })?
|
||||
}
|
||||
Subcommand::StaticFile { segment, key, subkey, raw } => {
|
||||
if let StaticFileSegment::StorageChangeSets = segment {
|
||||
let storage_key =
|
||||
table_subkey::<tables::StorageChangeSets>(subkey.as_deref()).ok();
|
||||
let key = table_key::<tables::StorageChangeSets>(&key)?;
|
||||
|
||||
let provider = tool.provider_factory.static_file_provider();
|
||||
|
||||
if let Some(storage_key) = storage_key {
|
||||
let entry = provider.get_storage_before_block(
|
||||
key.block_number(),
|
||||
key.address(),
|
||||
storage_key,
|
||||
)?;
|
||||
|
||||
if let Some(entry) = entry {
|
||||
println!("{}", serde_json::to_string_pretty(&entry)?);
|
||||
} else {
|
||||
error!(target: "reth::cli", "No content for the given table key.");
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let changesets = provider.storage_changeset(key.block_number())?;
|
||||
println!("{}", serde_json::to_string_pretty(&changesets)?);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (key, subkey, mask): (u64, _, _) = match segment {
|
||||
StaticFileSegment::Headers => (
|
||||
table_key::<tables::Headers>(&key)?,
|
||||
@@ -112,6 +140,9 @@ impl Command {
|
||||
AccountChangesetMask::MASK,
|
||||
)
|
||||
}
|
||||
StaticFileSegment::StorageChangeSets => {
|
||||
unreachable!("storage changesets handled above");
|
||||
}
|
||||
};
|
||||
|
||||
// handle account changesets differently if a subkey is provided.
|
||||
@@ -190,6 +221,9 @@ impl Command {
|
||||
StaticFileSegment::AccountChangeSets => {
|
||||
unreachable!("account changeset static files are special cased before this match")
|
||||
}
|
||||
StaticFileSegment::StorageChangeSets => {
|
||||
unreachable!("storage changeset static files are special cased before this match")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,19 +61,21 @@ impl Command {
|
||||
}
|
||||
|
||||
/// Generate [`ListFilter`] from command.
|
||||
pub fn list_filter(&self) -> ListFilter {
|
||||
let search = self
|
||||
.search
|
||||
.as_ref()
|
||||
.map(|search| {
|
||||
pub fn list_filter(&self) -> eyre::Result<ListFilter> {
|
||||
let search = match self.search.as_deref() {
|
||||
Some(search) => {
|
||||
if let Some(search) = search.strip_prefix("0x") {
|
||||
return hex::decode(search).unwrap()
|
||||
hex::decode(search).wrap_err(
|
||||
"Invalid hex content after 0x prefix in --search (expected valid hex like 0xdeadbeef).",
|
||||
)?
|
||||
} else {
|
||||
search.as_bytes().to_vec()
|
||||
}
|
||||
search.as_bytes().to_vec()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
}
|
||||
None => Vec::new(),
|
||||
};
|
||||
|
||||
ListFilter {
|
||||
Ok(ListFilter {
|
||||
skip: self.skip,
|
||||
len: self.len,
|
||||
search,
|
||||
@@ -82,7 +84,7 @@ impl Command {
|
||||
min_value_size: self.min_value_size,
|
||||
reverse: self.reverse,
|
||||
only_count: self.count,
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,8 +101,8 @@ impl<N: NodeTypes> TableViewer<()> for ListTableViewer<'_, N> {
|
||||
// We may be using the tui for a long time
|
||||
tx.disable_long_read_transaction_safety();
|
||||
|
||||
let table_db = tx.inner.open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?;
|
||||
let stats = tx.inner.db_stat(&table_db).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
|
||||
let table_db = tx.inner().open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?;
|
||||
let stats = tx.inner().db_stat(table_db.dbi()).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
|
||||
let total_entries = stats.entries();
|
||||
let final_entry_idx = total_entries.saturating_sub(1);
|
||||
if self.args.skip > final_entry_idx {
|
||||
@@ -115,7 +117,7 @@ impl<N: NodeTypes> TableViewer<()> for ListTableViewer<'_, N> {
|
||||
}
|
||||
|
||||
|
||||
let list_filter = self.args.list_filter();
|
||||
let list_filter = self.args.list_filter()?;
|
||||
|
||||
if self.args.json || self.args.count {
|
||||
let (list, count) = self.tool.list::<T>(&list_filter)?;
|
||||
|
||||
@@ -17,6 +17,7 @@ mod get;
|
||||
mod list;
|
||||
mod repair_trie;
|
||||
mod settings;
|
||||
mod state;
|
||||
mod static_file_header;
|
||||
mod stats;
|
||||
/// DB List TUI
|
||||
@@ -39,7 +40,7 @@ pub enum Subcommands {
|
||||
Stats(stats::Command),
|
||||
/// Lists the contents of a table
|
||||
List(list::Command),
|
||||
/// Calculates the content checksum of a table
|
||||
/// Calculates the content checksum of a table or static file segment
|
||||
Checksum(checksum::Command),
|
||||
/// Create a diff between two database tables or two entire databases.
|
||||
Diff(diff::Command),
|
||||
@@ -65,6 +66,8 @@ pub enum Subcommands {
|
||||
Settings(settings::Command),
|
||||
/// Gets storage size information for an account
|
||||
AccountStorage(account_storage::Command),
|
||||
/// Gets account state and storage at a specific block
|
||||
State(state::Command),
|
||||
}
|
||||
|
||||
/// Initializes a provider factory with specified access rights, and then execute with the provided
|
||||
@@ -162,7 +165,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
let access_rights =
|
||||
if command.dry_run { AccessRights::RO } else { AccessRights::RW };
|
||||
db_exec!(self.env, tool, N, access_rights, {
|
||||
command.execute(&tool, ctx.task_executor.clone(), &data_dir)?;
|
||||
command.execute(&tool, ctx.task_executor, &data_dir)?;
|
||||
});
|
||||
}
|
||||
Subcommands::StaticFileHeader(command) => {
|
||||
@@ -198,6 +201,11 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
command.execute(&tool)?;
|
||||
});
|
||||
}
|
||||
Subcommands::State(command) => {
|
||||
db_exec!(self.env, tool, N, AccessRights::RO, {
|
||||
command.execute(&tool)?;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -54,6 +54,26 @@ pub enum SetCommand {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store storage history in rocksdb instead of MDBX
|
||||
StoragesHistory {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store transaction hash to number mapping in rocksdb instead of MDBX
|
||||
TransactionHashNumbers {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store account history in rocksdb instead of MDBX
|
||||
AccountHistory {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store storage changesets in static files instead of the database
|
||||
StorageChangesets {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
}
|
||||
|
||||
impl Command {
|
||||
@@ -100,6 +120,7 @@ impl Command {
|
||||
transaction_hash_numbers_in_rocksdb: _,
|
||||
account_history_in_rocksdb: _,
|
||||
account_changesets_in_static_files: _,
|
||||
storage_changesets_in_static_files: _,
|
||||
} = settings.unwrap_or_else(StorageSettings::legacy);
|
||||
|
||||
// Update the setting based on the key
|
||||
@@ -128,6 +149,38 @@ impl Command {
|
||||
settings.account_changesets_in_static_files = value;
|
||||
println!("Set account_changesets_in_static_files = {}", value);
|
||||
}
|
||||
SetCommand::StoragesHistory { value } => {
|
||||
if settings.storages_history_in_rocksdb == value {
|
||||
println!("storages_history_in_rocksdb is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.storages_history_in_rocksdb = value;
|
||||
println!("Set storages_history_in_rocksdb = {}", value);
|
||||
}
|
||||
SetCommand::TransactionHashNumbers { value } => {
|
||||
if settings.transaction_hash_numbers_in_rocksdb == value {
|
||||
println!("transaction_hash_numbers_in_rocksdb is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.transaction_hash_numbers_in_rocksdb = value;
|
||||
println!("Set transaction_hash_numbers_in_rocksdb = {}", value);
|
||||
}
|
||||
SetCommand::AccountHistory { value } => {
|
||||
if settings.account_history_in_rocksdb == value {
|
||||
println!("account_history_in_rocksdb is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.account_history_in_rocksdb = value;
|
||||
println!("Set account_history_in_rocksdb = {}", value);
|
||||
}
|
||||
SetCommand::StorageChangesets { value } => {
|
||||
if settings.storage_changesets_in_static_files == value {
|
||||
println!("storage_changesets_in_static_files is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.storage_changesets_in_static_files = value;
|
||||
println!("Set storage_changesets_in_static_files = {}", value);
|
||||
}
|
||||
}
|
||||
|
||||
// Write updated settings
|
||||
|
||||
412
crates/cli/commands/src/db/state.rs
Normal file
412
crates/cli/commands/src/db/state.rs
Normal file
@@ -0,0 +1,412 @@
|
||||
use alloy_primitives::{Address, BlockNumber, B256, U256};
|
||||
use clap::Parser;
|
||||
use parking_lot::Mutex;
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbDupCursorRO},
|
||||
database::Database,
|
||||
tables,
|
||||
transaction::DbTx,
|
||||
};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::NodeTypesWithDB;
|
||||
use reth_provider::providers::ProviderNodeTypes;
|
||||
use reth_storage_api::{BlockNumReader, StateProvider, StorageSettingsCache};
|
||||
use std::{
|
||||
collections::BTreeSet,
|
||||
thread,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{error, info};
|
||||
|
||||
/// Log progress every 5 seconds
|
||||
const LOG_INTERVAL: Duration = Duration::from_secs(30);
|
||||
|
||||
/// The arguments for the `reth db state` command
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct Command {
|
||||
/// The account address to get state for
|
||||
address: Address,
|
||||
|
||||
/// Block number to query state at (uses current state if not provided)
|
||||
#[arg(long, short)]
|
||||
block: Option<BlockNumber>,
|
||||
|
||||
/// Maximum number of storage slots to display
|
||||
#[arg(long, short, default_value = "100")]
|
||||
limit: usize,
|
||||
|
||||
/// Output format (table, json, csv)
|
||||
#[arg(long, short, default_value = "table")]
|
||||
format: OutputFormat,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `db state` command
|
||||
pub fn execute<N: NodeTypesWithDB + ProviderNodeTypes>(
|
||||
self,
|
||||
tool: &DbTool<N>,
|
||||
) -> eyre::Result<()> {
|
||||
let address = self.address;
|
||||
let limit = self.limit;
|
||||
|
||||
if let Some(block) = self.block {
|
||||
self.execute_historical(tool, address, block, limit)
|
||||
} else {
|
||||
self.execute_current(tool, address, limit)
|
||||
}
|
||||
}
|
||||
|
||||
fn execute_current<N: NodeTypesWithDB + ProviderNodeTypes>(
|
||||
&self,
|
||||
tool: &DbTool<N>,
|
||||
address: Address,
|
||||
limit: usize,
|
||||
) -> eyre::Result<()> {
|
||||
let entries = tool.provider_factory.db_ref().view(|tx| {
|
||||
// Get account info
|
||||
let account = tx.get::<tables::PlainAccountState>(address)?;
|
||||
|
||||
// Get storage entries
|
||||
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
|
||||
let mut entries = Vec::new();
|
||||
let mut last_log = Instant::now();
|
||||
|
||||
let walker = cursor.walk_dup(Some(address), None)?;
|
||||
for (idx, entry) in walker.enumerate() {
|
||||
let (_, storage_entry) = entry?;
|
||||
|
||||
if storage_entry.value != U256::ZERO {
|
||||
entries.push((storage_entry.key, storage_entry.value));
|
||||
}
|
||||
|
||||
if entries.len() >= limit {
|
||||
break;
|
||||
}
|
||||
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots_scanned = idx,
|
||||
"Scanning storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
Ok::<_, eyre::Report>((account, entries))
|
||||
})??;
|
||||
|
||||
let (account, storage_entries) = entries;
|
||||
|
||||
self.print_results(address, None, account, &storage_entries);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn execute_historical<N: NodeTypesWithDB + ProviderNodeTypes>(
|
||||
&self,
|
||||
tool: &DbTool<N>,
|
||||
address: Address,
|
||||
block: BlockNumber,
|
||||
limit: usize,
|
||||
) -> eyre::Result<()> {
|
||||
let provider = tool.provider_factory.history_by_block_number(block)?;
|
||||
|
||||
// Get account info at that block
|
||||
let account = provider.basic_account(&address)?;
|
||||
|
||||
// Check storage settings to determine where history is stored
|
||||
let storage_settings = tool.provider_factory.cached_storage_settings();
|
||||
let history_in_rocksdb = storage_settings.storages_history_in_rocksdb;
|
||||
|
||||
// For historical queries, enumerate keys from history indices only
|
||||
// (not PlainStorageState, which reflects current state)
|
||||
let mut storage_keys = BTreeSet::new();
|
||||
|
||||
if history_in_rocksdb {
|
||||
error!(
|
||||
target: "reth::cli",
|
||||
"Historical storage queries with RocksDB backend are not yet supported. \
|
||||
Use MDBX for storage history or query current state without --block."
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Collect keys from MDBX StorageChangeSets using parallel scanning
|
||||
self.collect_mdbx_storage_keys_parallel(tool, address, &mut storage_keys)?;
|
||||
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
block = block,
|
||||
total_keys = storage_keys.len(),
|
||||
"Found storage keys to query"
|
||||
);
|
||||
|
||||
// Now query each key at the historical block using the StateProvider
|
||||
// This handles both MDBX and RocksDB backends transparently
|
||||
let mut entries = Vec::new();
|
||||
let mut last_log = Instant::now();
|
||||
|
||||
for (idx, key) in storage_keys.iter().enumerate() {
|
||||
match provider.storage(address, *key) {
|
||||
Ok(Some(value)) if value != U256::ZERO => {
|
||||
entries.push((*key, value));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
if entries.len() >= limit {
|
||||
break;
|
||||
}
|
||||
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
block = block,
|
||||
keys_total = storage_keys.len(),
|
||||
slots_scanned = idx,
|
||||
slots_found = entries.len(),
|
||||
"Scanning historical storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
self.print_results(address, Some(block), account, &entries);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Collects storage keys from MDBX StorageChangeSets using parallel block range scanning.
|
||||
fn collect_mdbx_storage_keys_parallel<N: NodeTypesWithDB + ProviderNodeTypes>(
|
||||
&self,
|
||||
tool: &DbTool<N>,
|
||||
address: Address,
|
||||
keys: &mut BTreeSet<B256>,
|
||||
) -> eyre::Result<()> {
|
||||
const CHUNK_SIZE: u64 = 500_000; // 500k blocks per thread
|
||||
let num_threads = std::thread::available_parallelism()
|
||||
.map(|p| p.get().saturating_sub(1).max(1))
|
||||
.unwrap_or(4);
|
||||
|
||||
// Get the current tip block
|
||||
let tip = tool.provider_factory.provider()?.best_block_number()?;
|
||||
|
||||
if tip == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
tip,
|
||||
chunk_size = CHUNK_SIZE,
|
||||
num_threads,
|
||||
"Starting parallel MDBX changeset scan"
|
||||
);
|
||||
|
||||
// Shared state for collecting keys
|
||||
let collected_keys: Mutex<BTreeSet<B256>> = Mutex::new(BTreeSet::new());
|
||||
let total_entries_scanned = Mutex::new(0usize);
|
||||
|
||||
// Create chunk ranges
|
||||
let mut chunks: Vec<(u64, u64)> = Vec::new();
|
||||
let mut start = 0u64;
|
||||
while start <= tip {
|
||||
let end = (start + CHUNK_SIZE - 1).min(tip);
|
||||
chunks.push((start, end));
|
||||
start = end + 1;
|
||||
}
|
||||
|
||||
let chunks_ref = &chunks;
|
||||
let next_chunk = Mutex::new(0usize);
|
||||
let next_chunk_ref = &next_chunk;
|
||||
let collected_keys_ref = &collected_keys;
|
||||
let total_entries_ref = &total_entries_scanned;
|
||||
|
||||
thread::scope(|s| {
|
||||
let handles: Vec<_> = (0..num_threads)
|
||||
.map(|thread_id| {
|
||||
s.spawn(move || {
|
||||
loop {
|
||||
// Get next chunk to process
|
||||
let chunk_idx = {
|
||||
let mut idx = next_chunk_ref.lock();
|
||||
if *idx >= chunks_ref.len() {
|
||||
return Ok::<_, eyre::Report>(());
|
||||
}
|
||||
let current = *idx;
|
||||
*idx += 1;
|
||||
current
|
||||
};
|
||||
|
||||
let (chunk_start, chunk_end) = chunks_ref[chunk_idx];
|
||||
|
||||
// Open a new read transaction for this chunk
|
||||
tool.provider_factory.db_ref().view(|tx| {
|
||||
tx.disable_long_read_transaction_safety();
|
||||
|
||||
let mut changeset_cursor =
|
||||
tx.cursor_read::<tables::StorageChangeSets>()?;
|
||||
let start_key =
|
||||
reth_db_api::models::BlockNumberAddress((chunk_start, address));
|
||||
let end_key =
|
||||
reth_db_api::models::BlockNumberAddress((chunk_end, address));
|
||||
|
||||
let mut local_keys = BTreeSet::new();
|
||||
let mut entries_in_chunk = 0usize;
|
||||
|
||||
if let Ok(walker) = changeset_cursor.walk_range(start_key..=end_key)
|
||||
{
|
||||
for (block_addr, storage_entry) in walker.flatten() {
|
||||
if block_addr.address() == address {
|
||||
local_keys.insert(storage_entry.key);
|
||||
}
|
||||
entries_in_chunk += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Merge into global state
|
||||
collected_keys_ref.lock().extend(local_keys);
|
||||
*total_entries_ref.lock() += entries_in_chunk;
|
||||
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
thread_id,
|
||||
chunk_start,
|
||||
chunk_end,
|
||||
entries_in_chunk,
|
||||
"Thread completed chunk"
|
||||
);
|
||||
|
||||
Ok::<_, eyre::Report>(())
|
||||
})??;
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
for handle in handles {
|
||||
handle.join().map_err(|_| eyre::eyre!("Thread panicked"))??;
|
||||
}
|
||||
|
||||
Ok::<_, eyre::Report>(())
|
||||
})?;
|
||||
|
||||
let final_keys = collected_keys.into_inner();
|
||||
let total = *total_entries_scanned.lock();
|
||||
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
total_entries = total,
|
||||
unique_keys = final_keys.len(),
|
||||
"Finished parallel MDBX changeset scan"
|
||||
);
|
||||
|
||||
keys.extend(final_keys);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn print_results(
|
||||
&self,
|
||||
address: Address,
|
||||
block: Option<BlockNumber>,
|
||||
account: Option<reth_primitives_traits::Account>,
|
||||
storage: &[(alloy_primitives::B256, U256)],
|
||||
) {
|
||||
match self.format {
|
||||
OutputFormat::Table => {
|
||||
println!("Account: {address}");
|
||||
if let Some(b) = block {
|
||||
println!("Block: {b}");
|
||||
} else {
|
||||
println!("Block: latest");
|
||||
}
|
||||
println!();
|
||||
|
||||
if let Some(acc) = account {
|
||||
println!("Nonce: {}", acc.nonce);
|
||||
println!("Balance: {} wei", acc.balance);
|
||||
if let Some(code_hash) = acc.bytecode_hash {
|
||||
println!("Code hash: {code_hash}");
|
||||
}
|
||||
} else {
|
||||
println!("Account not found");
|
||||
}
|
||||
|
||||
println!();
|
||||
println!("Storage ({} slots):", storage.len());
|
||||
println!("{:-<130}", "");
|
||||
println!("{:<66} | {:<64}", "Slot", "Value");
|
||||
println!("{:-<130}", "");
|
||||
for (key, value) in storage {
|
||||
println!("{key} | {value:#066x}");
|
||||
}
|
||||
}
|
||||
OutputFormat::Json => {
|
||||
let output = serde_json::json!({
|
||||
"address": address.to_string(),
|
||||
"block": block,
|
||||
"account": account.map(|a| serde_json::json!({
|
||||
"nonce": a.nonce,
|
||||
"balance": a.balance.to_string(),
|
||||
"code_hash": a.bytecode_hash.map(|h| h.to_string()),
|
||||
})),
|
||||
"storage": storage.iter().map(|(k, v)| {
|
||||
serde_json::json!({
|
||||
"key": k.to_string(),
|
||||
"value": format!("{v:#066x}"),
|
||||
})
|
||||
}).collect::<Vec<_>>(),
|
||||
});
|
||||
println!("{}", serde_json::to_string_pretty(&output).unwrap());
|
||||
}
|
||||
OutputFormat::Csv => {
|
||||
println!("slot,value");
|
||||
for (key, value) in storage {
|
||||
println!("{key},{value:#066x}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, clap::ValueEnum)]
|
||||
pub enum OutputFormat {
|
||||
#[default]
|
||||
Table,
|
||||
Json,
|
||||
Csv,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_state_args() {
|
||||
let cmd = Command::try_parse_from([
|
||||
"state",
|
||||
"0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045",
|
||||
"--block",
|
||||
"1000000",
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
cmd.address,
|
||||
"0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045".parse::<Address>().unwrap()
|
||||
);
|
||||
assert_eq!(cmd.block, Some(1000000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_state_args_no_block() {
|
||||
let cmd = Command::try_parse_from(["state", "0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045"])
|
||||
.unwrap();
|
||||
assert_eq!(cmd.block, None);
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,10 @@ use reth_db_common::DbTool;
|
||||
use reth_fs_util as fs;
|
||||
use reth_node_builder::{NodePrimitives, NodeTypesWithDB, NodeTypesWithDBAdapter};
|
||||
use reth_node_core::dirs::{ChainPath, DataDirPath};
|
||||
use reth_provider::providers::{ProviderNodeTypes, StaticFileProvider};
|
||||
use reth_provider::{
|
||||
providers::{ProviderNodeTypes, StaticFileProvider},
|
||||
RocksDBProviderFactory,
|
||||
};
|
||||
use reth_static_file_types::SegmentRangeInclusive;
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
@@ -61,6 +64,11 @@ impl Command {
|
||||
let db_stats_table = self.db_stats_table(tool)?;
|
||||
println!("{db_stats_table}");
|
||||
|
||||
println!("\n");
|
||||
|
||||
let rocksdb_stats_table = self.rocksdb_stats_table(tool);
|
||||
println!("{rocksdb_stats_table}");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -84,11 +92,11 @@ impl Command {
|
||||
db_tables.sort();
|
||||
let mut total_size = 0;
|
||||
for db_table in db_tables {
|
||||
let table_db = tx.inner.open_db(Some(db_table)).wrap_err("Could not open db.")?;
|
||||
let table_db = tx.inner().open_db(Some(db_table)).wrap_err("Could not open db.")?;
|
||||
|
||||
let stats = tx
|
||||
.inner
|
||||
.db_stat(&table_db)
|
||||
.inner()
|
||||
.db_stat(table_db.dbi())
|
||||
.wrap_err(format!("Could not find table: {db_table}"))?;
|
||||
|
||||
// Defaults to 16KB right now but we should
|
||||
@@ -128,8 +136,9 @@ impl Command {
|
||||
.add_cell(Cell::new(human_bytes(total_size as f64)));
|
||||
table.add_row(row);
|
||||
|
||||
let freelist = tx.inner.env().freelist()?;
|
||||
let pagesize = tx.inner.db_stat(&mdbx::Database::freelist_db())?.page_size() as usize;
|
||||
let freelist = tx.inner().env().freelist()?;
|
||||
let pagesize =
|
||||
tx.inner().db_stat(mdbx::Database::freelist_db().dbi())?.page_size() as usize;
|
||||
let freelist_size = freelist * pagesize;
|
||||
|
||||
let mut row = Row::new();
|
||||
@@ -147,6 +156,70 @@ impl Command {
|
||||
Ok(table)
|
||||
}
|
||||
|
||||
fn rocksdb_stats_table<N: NodeTypesWithDB>(&self, tool: &DbTool<N>) -> ComfyTable {
|
||||
let mut table = ComfyTable::new();
|
||||
table.load_preset(comfy_table::presets::ASCII_MARKDOWN);
|
||||
table.set_header([
|
||||
"RocksDB Table Name",
|
||||
"# Entries",
|
||||
"SST Size",
|
||||
"Memtable Size",
|
||||
"Total Size",
|
||||
"Pending Compaction",
|
||||
]);
|
||||
|
||||
let stats = tool.provider_factory.rocksdb_provider().table_stats();
|
||||
let mut total_sst: u64 = 0;
|
||||
let mut total_memtable: u64 = 0;
|
||||
let mut total_size: u64 = 0;
|
||||
let mut total_pending: u64 = 0;
|
||||
|
||||
for stat in &stats {
|
||||
total_sst += stat.sst_size_bytes;
|
||||
total_memtable += stat.memtable_size_bytes;
|
||||
total_size += stat.estimated_size_bytes;
|
||||
total_pending += stat.pending_compaction_bytes;
|
||||
let mut row = Row::new();
|
||||
row.add_cell(Cell::new(&stat.name))
|
||||
.add_cell(Cell::new(stat.estimated_num_keys))
|
||||
.add_cell(Cell::new(human_bytes(stat.sst_size_bytes as f64)))
|
||||
.add_cell(Cell::new(human_bytes(stat.memtable_size_bytes as f64)))
|
||||
.add_cell(Cell::new(human_bytes(stat.estimated_size_bytes as f64)))
|
||||
.add_cell(Cell::new(human_bytes(stat.pending_compaction_bytes as f64)));
|
||||
table.add_row(row);
|
||||
}
|
||||
|
||||
if !stats.is_empty() {
|
||||
let max_widths = table.column_max_content_widths();
|
||||
let mut separator = Row::new();
|
||||
for width in max_widths {
|
||||
separator.add_cell(Cell::new("-".repeat(width as usize)));
|
||||
}
|
||||
table.add_row(separator);
|
||||
|
||||
let mut row = Row::new();
|
||||
row.add_cell(Cell::new("RocksDB Total"))
|
||||
.add_cell(Cell::new(""))
|
||||
.add_cell(Cell::new(human_bytes(total_sst as f64)))
|
||||
.add_cell(Cell::new(human_bytes(total_memtable as f64)))
|
||||
.add_cell(Cell::new(human_bytes(total_size as f64)))
|
||||
.add_cell(Cell::new(human_bytes(total_pending as f64)));
|
||||
table.add_row(row);
|
||||
|
||||
let wal_size = tool.provider_factory.rocksdb_provider().wal_size_bytes();
|
||||
let mut row = Row::new();
|
||||
row.add_cell(Cell::new("WAL"))
|
||||
.add_cell(Cell::new(""))
|
||||
.add_cell(Cell::new(""))
|
||||
.add_cell(Cell::new(""))
|
||||
.add_cell(Cell::new(human_bytes(wal_size as f64)))
|
||||
.add_cell(Cell::new(""));
|
||||
table.add_row(row);
|
||||
}
|
||||
|
||||
table
|
||||
}
|
||||
|
||||
fn static_files_stats_table<N: NodePrimitives>(
|
||||
&self,
|
||||
data_dir: ChainPath<DataDirPath>,
|
||||
|
||||
@@ -2,20 +2,22 @@ use crate::common::EnvironmentArgs;
|
||||
use clap::Parser;
|
||||
use eyre::Result;
|
||||
use lz4::Decoder;
|
||||
use reqwest::Client;
|
||||
use reqwest::{blocking::Client as BlockingClient, header::RANGE, Client, StatusCode};
|
||||
use reth_chainspec::{EthChainSpec, EthereumHardforks};
|
||||
use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_fs_util as fs;
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
io::{self, Read, Write},
|
||||
path::Path,
|
||||
fs::OpenOptions,
|
||||
io::{self, BufWriter, Read, Write},
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, OnceLock},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tar::Archive;
|
||||
use tokio::task;
|
||||
use tracing::info;
|
||||
use url::Url;
|
||||
use zstd::stream::read::Decoder as ZstdDecoder;
|
||||
|
||||
const BYTE_UNITS: [&str; 4] = ["B", "KB", "MB", "GB"];
|
||||
@@ -85,6 +87,9 @@ impl DownloadDefaults {
|
||||
"\nIf no URL is provided, the latest mainnet archive snapshot\nwill be proposed for download from ",
|
||||
);
|
||||
help.push_str(self.default_base_url.as_ref());
|
||||
help.push_str(
|
||||
".\n\nLocal file:// URLs are also supported for extracting snapshots from disk.",
|
||||
);
|
||||
help
|
||||
}
|
||||
|
||||
@@ -170,12 +175,14 @@ struct DownloadProgress {
|
||||
downloaded: u64,
|
||||
total_size: u64,
|
||||
last_displayed: Instant,
|
||||
started_at: Instant,
|
||||
}
|
||||
|
||||
impl DownloadProgress {
|
||||
/// Creates new progress tracker with given total size
|
||||
fn new(total_size: u64) -> Self {
|
||||
Self { downloaded: 0, total_size, last_displayed: Instant::now() }
|
||||
let now = Instant::now();
|
||||
Self { downloaded: 0, total_size, last_displayed: now, started_at: now }
|
||||
}
|
||||
|
||||
/// Converts bytes to human readable format (B, KB, MB, GB)
|
||||
@@ -191,6 +198,18 @@ impl DownloadProgress {
|
||||
format!("{:.2} {}", size, BYTE_UNITS[unit_index])
|
||||
}
|
||||
|
||||
/// Format duration as human readable string
|
||||
fn format_duration(duration: Duration) -> String {
|
||||
let secs = duration.as_secs();
|
||||
if secs < 60 {
|
||||
format!("{secs}s")
|
||||
} else if secs < 3600 {
|
||||
format!("{}m {}s", secs / 60, secs % 60)
|
||||
} else {
|
||||
format!("{}h {}m", secs / 3600, (secs % 3600) / 60)
|
||||
}
|
||||
}
|
||||
|
||||
/// Updates progress bar
|
||||
fn update(&mut self, chunk_size: u64) -> Result<()> {
|
||||
self.downloaded += chunk_size;
|
||||
@@ -201,8 +220,24 @@ impl DownloadProgress {
|
||||
let formatted_total = Self::format_size(self.total_size);
|
||||
let progress = (self.downloaded as f64 / self.total_size as f64) * 100.0;
|
||||
|
||||
// Calculate ETA based on current speed
|
||||
let elapsed = self.started_at.elapsed();
|
||||
let eta = if self.downloaded > 0 {
|
||||
let remaining = self.total_size.saturating_sub(self.downloaded);
|
||||
let speed = self.downloaded as f64 / elapsed.as_secs_f64();
|
||||
if speed > 0.0 {
|
||||
Duration::from_secs_f64(remaining as f64 / speed)
|
||||
} else {
|
||||
Duration::ZERO
|
||||
}
|
||||
} else {
|
||||
Duration::ZERO
|
||||
};
|
||||
let eta_str = Self::format_duration(eta);
|
||||
|
||||
// Pad with spaces to clear any previous longer line
|
||||
print!(
|
||||
"\rDownloading and extracting... {progress:.2}% ({formatted_downloaded} / {formatted_total})",
|
||||
"\rDownloading and extracting... {progress:.2}% ({formatted_downloaded} / {formatted_total}) ETA: {eta_str} ",
|
||||
);
|
||||
io::stdout().flush()?;
|
||||
self.last_displayed = Instant::now();
|
||||
@@ -246,29 +281,30 @@ enum CompressionFormat {
|
||||
impl CompressionFormat {
|
||||
/// Detect compression format from file extension
|
||||
fn from_url(url: &str) -> Result<Self> {
|
||||
if url.ends_with(EXTENSION_TAR_LZ4) {
|
||||
let path =
|
||||
Url::parse(url).map(|u| u.path().to_string()).unwrap_or_else(|_| url.to_string());
|
||||
|
||||
if path.ends_with(EXTENSION_TAR_LZ4) {
|
||||
Ok(Self::Lz4)
|
||||
} else if url.ends_with(EXTENSION_TAR_ZSTD) {
|
||||
} else if path.ends_with(EXTENSION_TAR_ZSTD) {
|
||||
Ok(Self::Zstd)
|
||||
} else {
|
||||
Err(eyre::eyre!("Unsupported file format. Expected .tar.lz4 or .tar.zst, got: {}", url))
|
||||
Err(eyre::eyre!(
|
||||
"Unsupported file format. Expected .tar.lz4 or .tar.zst, got: {}",
|
||||
path
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Downloads and extracts a snapshot, blocking until finished.
|
||||
fn blocking_download_and_extract(url: &str, target_dir: &Path) -> Result<()> {
|
||||
let client = reqwest::blocking::Client::builder().build()?;
|
||||
let response = client.get(url).send()?.error_for_status()?;
|
||||
|
||||
let total_size = response.content_length().ok_or_else(|| {
|
||||
eyre::eyre!(
|
||||
"Server did not provide Content-Length header. This is required for snapshot downloads"
|
||||
)
|
||||
})?;
|
||||
|
||||
let progress_reader = ProgressReader::new(response, total_size);
|
||||
let format = CompressionFormat::from_url(url)?;
|
||||
/// Extracts a compressed tar archive to the target directory with progress tracking.
|
||||
fn extract_archive<R: Read>(
|
||||
reader: R,
|
||||
total_size: u64,
|
||||
format: CompressionFormat,
|
||||
target_dir: &Path,
|
||||
) -> Result<()> {
|
||||
let progress_reader = ProgressReader::new(reader, total_size);
|
||||
|
||||
match format {
|
||||
CompressionFormat::Lz4 => {
|
||||
@@ -285,6 +321,185 @@ fn blocking_download_and_extract(url: &str, target_dir: &Path) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Extracts a snapshot from a local file.
|
||||
fn extract_from_file(path: &Path, format: CompressionFormat, target_dir: &Path) -> Result<()> {
|
||||
let file = std::fs::File::open(path)?;
|
||||
let total_size = file.metadata()?.len();
|
||||
extract_archive(file, total_size, format, target_dir)
|
||||
}
|
||||
|
||||
const MAX_DOWNLOAD_RETRIES: u32 = 10;
|
||||
const RETRY_BACKOFF_SECS: u64 = 5;
|
||||
|
||||
/// Wrapper that tracks download progress while writing data.
|
||||
/// Used with [`io::copy`] to display progress during downloads.
|
||||
struct ProgressWriter<W> {
|
||||
inner: W,
|
||||
progress: DownloadProgress,
|
||||
}
|
||||
|
||||
impl<W: Write> Write for ProgressWriter<W> {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
let n = self.inner.write(buf)?;
|
||||
let _ = self.progress.update(n as u64);
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()
|
||||
}
|
||||
}
|
||||
|
||||
/// Downloads a file with resume support using HTTP Range requests.
|
||||
/// Automatically retries on failure, resuming from where it left off.
|
||||
/// Returns the path to the downloaded file and its total size.
|
||||
fn resumable_download(url: &str, target_dir: &Path) -> Result<(PathBuf, u64)> {
|
||||
let file_name = Url::parse(url)
|
||||
.ok()
|
||||
.and_then(|u| u.path_segments()?.next_back().map(|s| s.to_string()))
|
||||
.unwrap_or_else(|| "snapshot.tar".to_string());
|
||||
|
||||
let final_path = target_dir.join(&file_name);
|
||||
let part_path = target_dir.join(format!("{file_name}.part"));
|
||||
|
||||
let client = BlockingClient::builder().timeout(Duration::from_secs(30)).build()?;
|
||||
|
||||
let mut total_size: Option<u64> = None;
|
||||
let mut last_error: Option<eyre::Error> = None;
|
||||
|
||||
for attempt in 1..=MAX_DOWNLOAD_RETRIES {
|
||||
let existing_size = fs::metadata(&part_path).map(|m| m.len()).unwrap_or(0);
|
||||
|
||||
if let Some(total) = total_size &&
|
||||
existing_size >= total
|
||||
{
|
||||
fs::rename(&part_path, &final_path)?;
|
||||
info!(target: "reth::cli", "Download complete: {}", final_path.display());
|
||||
return Ok((final_path, total));
|
||||
}
|
||||
|
||||
if attempt > 1 {
|
||||
info!(target: "reth::cli",
|
||||
"Retry attempt {}/{} - resuming from {} bytes",
|
||||
attempt, MAX_DOWNLOAD_RETRIES, existing_size
|
||||
);
|
||||
}
|
||||
|
||||
let mut request = client.get(url);
|
||||
if existing_size > 0 {
|
||||
request = request.header(RANGE, format!("bytes={existing_size}-"));
|
||||
if attempt == 1 {
|
||||
info!(target: "reth::cli", "Resuming download from {} bytes", existing_size);
|
||||
}
|
||||
}
|
||||
|
||||
let response = match request.send().and_then(|r| r.error_for_status()) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
last_error = Some(e.into());
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
info!(target: "reth::cli",
|
||||
"Download failed, retrying in {} seconds...", RETRY_BACKOFF_SECS
|
||||
);
|
||||
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let is_partial = response.status() == StatusCode::PARTIAL_CONTENT;
|
||||
|
||||
let size = if is_partial {
|
||||
response
|
||||
.headers()
|
||||
.get("Content-Range")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.split('/').next_back())
|
||||
.and_then(|v| v.parse().ok())
|
||||
} else {
|
||||
response.content_length()
|
||||
};
|
||||
|
||||
if total_size.is_none() {
|
||||
total_size = size;
|
||||
}
|
||||
|
||||
let current_total = total_size.ok_or_else(|| {
|
||||
eyre::eyre!("Server did not provide Content-Length or Content-Range header")
|
||||
})?;
|
||||
|
||||
let file = if is_partial && existing_size > 0 {
|
||||
OpenOptions::new()
|
||||
.append(true)
|
||||
.open(&part_path)
|
||||
.map_err(|e| fs::FsPathError::open(e, &part_path))?
|
||||
} else {
|
||||
fs::create_file(&part_path)?
|
||||
};
|
||||
|
||||
let start_offset = if is_partial { existing_size } else { 0 };
|
||||
let mut progress = DownloadProgress::new(current_total);
|
||||
progress.downloaded = start_offset;
|
||||
|
||||
let mut writer = ProgressWriter { inner: BufWriter::new(file), progress };
|
||||
let mut reader = response;
|
||||
|
||||
let copy_result = io::copy(&mut reader, &mut writer);
|
||||
let flush_result = writer.inner.flush();
|
||||
println!();
|
||||
|
||||
if let Err(e) = copy_result.and(flush_result) {
|
||||
last_error = Some(e.into());
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
info!(target: "reth::cli",
|
||||
"Download interrupted, retrying in {} seconds...", RETRY_BACKOFF_SECS
|
||||
);
|
||||
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
fs::rename(&part_path, &final_path)?;
|
||||
info!(target: "reth::cli", "Download complete: {}", final_path.display());
|
||||
return Ok((final_path, current_total));
|
||||
}
|
||||
|
||||
Err(last_error
|
||||
.unwrap_or_else(|| eyre::eyre!("Download failed after {} attempts", MAX_DOWNLOAD_RETRIES)))
|
||||
}
|
||||
|
||||
/// Fetches the snapshot from a remote URL with resume support, then extracts it.
|
||||
fn download_and_extract(url: &str, format: CompressionFormat, target_dir: &Path) -> Result<()> {
|
||||
let (downloaded_path, total_size) = resumable_download(url, target_dir)?;
|
||||
|
||||
info!(target: "reth::cli", "Extracting snapshot...");
|
||||
let file = fs::open(&downloaded_path)?;
|
||||
extract_archive(file, total_size, format, target_dir)?;
|
||||
|
||||
fs::remove_file(&downloaded_path)?;
|
||||
info!(target: "reth::cli", "Removed downloaded archive");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Downloads and extracts a snapshot, blocking until finished.
|
||||
///
|
||||
/// Supports both `file://` URLs for local files and HTTP(S) URLs for remote downloads.
|
||||
fn blocking_download_and_extract(url: &str, target_dir: &Path) -> Result<()> {
|
||||
let format = CompressionFormat::from_url(url)?;
|
||||
|
||||
if let Ok(parsed_url) = Url::parse(url) &&
|
||||
parsed_url.scheme() == "file"
|
||||
{
|
||||
let file_path = parsed_url
|
||||
.to_file_path()
|
||||
.map_err(|_| eyre::eyre!("Invalid file:// URL path: {}", url))?;
|
||||
extract_from_file(&file_path, format, target_dir)
|
||||
} else {
|
||||
download_and_extract(url, format, target_dir)
|
||||
}
|
||||
}
|
||||
|
||||
async fn stream_and_extract(url: &str, target_dir: &Path) -> Result<()> {
|
||||
let target_dir = target_dir.to_path_buf();
|
||||
let url = url.to_string();
|
||||
@@ -343,6 +558,7 @@ mod tests {
|
||||
assert!(help.contains("Available snapshot sources:"));
|
||||
assert!(help.contains("merkle.io"));
|
||||
assert!(help.contains("publicnode.com"));
|
||||
assert!(help.contains("file://"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -367,4 +583,25 @@ mod tests {
|
||||
assert_eq!(defaults.available_snapshots.len(), 4); // 2 defaults + 2 added
|
||||
assert_eq!(defaults.long_help, Some("Custom help for snapshots".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compression_format_detection() {
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("https://example.com/snapshot.tar.lz4"),
|
||||
Ok(CompressionFormat::Lz4)
|
||||
));
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("https://example.com/snapshot.tar.zst"),
|
||||
Ok(CompressionFormat::Zstd)
|
||||
));
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("file:///path/to/snapshot.tar.lz4"),
|
||||
Ok(CompressionFormat::Lz4)
|
||||
));
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("file:///path/to/snapshot.tar.zst"),
|
||||
Ok(CompressionFormat::Zstd)
|
||||
));
|
||||
assert!(CompressionFormat::from_url("https://example.com/snapshot.tar.gz").is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,14 @@ pub struct ImportCommand<C: ChainSpecParser> {
|
||||
#[arg(long, value_name = "CHUNK_LEN", verbatim_doc_comment)]
|
||||
chunk_len: Option<u64>,
|
||||
|
||||
/// Fail immediately when an invalid block is encountered.
|
||||
///
|
||||
/// By default, the import will stop at the last valid block if an invalid block is
|
||||
/// encountered during execution or validation, leaving the database at the last valid
|
||||
/// block state. When this flag is set, the import will instead fail with an error.
|
||||
#[arg(long, verbatim_doc_comment)]
|
||||
fail_on_invalid_block: bool,
|
||||
|
||||
/// The path(s) to block file(s) for import.
|
||||
///
|
||||
/// The online stages (headers and bodies) are replaced by a file import, after which the
|
||||
@@ -52,7 +60,11 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
|
||||
|
||||
info!(target: "reth::cli", "Starting import of {} file(s)", self.paths.len());
|
||||
|
||||
let import_config = ImportConfig { no_state: self.no_state, chunk_len: self.chunk_len };
|
||||
let import_config = ImportConfig {
|
||||
no_state: self.no_state,
|
||||
chunk_len: self.chunk_len,
|
||||
fail_on_invalid_block: self.fail_on_invalid_block,
|
||||
};
|
||||
|
||||
let executor = components.evm_config().clone();
|
||||
let consensus = Arc::new(components.consensus().clone());
|
||||
@@ -81,7 +93,20 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
|
||||
total_decoded_blocks += result.total_decoded_blocks;
|
||||
total_decoded_txns += result.total_decoded_txns;
|
||||
|
||||
if !result.is_complete() {
|
||||
// Check if we stopped due to an invalid block
|
||||
if result.stopped_on_invalid_block {
|
||||
info!(target: "reth::cli",
|
||||
"Stopped at last valid block {} due to invalid block {} in file: {}. Imported {} blocks, {} transactions",
|
||||
result.last_valid_block.unwrap_or(0),
|
||||
result.bad_block.unwrap_or(0),
|
||||
path.display(),
|
||||
result.total_imported_blocks,
|
||||
result.total_imported_txns);
|
||||
// Stop importing further files and exit successfully
|
||||
break;
|
||||
}
|
||||
|
||||
if !result.is_successful() {
|
||||
return Err(eyre::eyre!(
|
||||
"Chain was partially imported from file: {}. Imported {}/{} blocks, {}/{} transactions",
|
||||
path.display(),
|
||||
@@ -98,7 +123,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
|
||||
}
|
||||
|
||||
info!(target: "reth::cli",
|
||||
"All files imported successfully. Total: {}/{} blocks, {}/{} transactions",
|
||||
"Import complete. Total: {}/{} blocks, {}/{} transactions",
|
||||
total_imported_blocks, total_decoded_blocks, total_imported_txns, total_decoded_txns);
|
||||
|
||||
Ok(())
|
||||
@@ -139,4 +164,20 @@ mod tests {
|
||||
assert_eq!(args.paths[1], PathBuf::from("file2.rlp"));
|
||||
assert_eq!(args.paths[2], PathBuf::from("file3.rlp"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_import_command_with_fail_on_invalid_block() {
|
||||
let args: ImportCommand<EthereumChainSpecParser> =
|
||||
ImportCommand::parse_from(["reth", "--fail-on-invalid-block", "chain.rlp"]);
|
||||
assert!(args.fail_on_invalid_block);
|
||||
assert_eq!(args.paths.len(), 1);
|
||||
assert_eq!(args.paths[0], PathBuf::from("chain.rlp"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_import_command_default_stops_on_invalid_block() {
|
||||
let args: ImportCommand<EthereumChainSpecParser> =
|
||||
ImportCommand::parse_from(["reth", "chain.rlp"]);
|
||||
assert!(!args.fail_on_invalid_block);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,11 +22,11 @@ use reth_provider::{
|
||||
StageCheckpointReader,
|
||||
};
|
||||
use reth_prune::PruneModes;
|
||||
use reth_stages::{prelude::*, Pipeline, StageId, StageSet};
|
||||
use reth_stages::{prelude::*, ControlFlow, Pipeline, StageId, StageSet};
|
||||
use reth_static_file::StaticFileProducer;
|
||||
use std::{path::Path, sync::Arc};
|
||||
use tokio::sync::watch;
|
||||
use tracing::{debug, error, info};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
/// Configuration for importing blocks from RLP files.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
@@ -35,6 +35,9 @@ pub struct ImportConfig {
|
||||
pub no_state: bool,
|
||||
/// Chunk byte length to read from file.
|
||||
pub chunk_len: Option<u64>,
|
||||
/// If true, fail immediately when an invalid block is encountered.
|
||||
/// By default (false), the import stops at the last valid block and exits successfully.
|
||||
pub fail_on_invalid_block: bool,
|
||||
}
|
||||
|
||||
/// Result of an import operation.
|
||||
@@ -48,6 +51,12 @@ pub struct ImportResult {
|
||||
pub total_imported_blocks: usize,
|
||||
/// Total number of transactions imported into the database.
|
||||
pub total_imported_txns: usize,
|
||||
/// Whether the import was stopped due to an invalid block.
|
||||
pub stopped_on_invalid_block: bool,
|
||||
/// The block number that was invalid, if any.
|
||||
pub bad_block: Option<u64>,
|
||||
/// The last valid block number when stopped due to invalid block.
|
||||
pub last_valid_block: Option<u64>,
|
||||
}
|
||||
|
||||
impl ImportResult {
|
||||
@@ -56,6 +65,14 @@ impl ImportResult {
|
||||
self.total_decoded_blocks == self.total_imported_blocks &&
|
||||
self.total_decoded_txns == self.total_imported_txns
|
||||
}
|
||||
|
||||
/// Returns true if the import was successful, considering stop-on-invalid-block mode.
|
||||
///
|
||||
/// In stop-on-invalid-block mode, a partial import is considered successful if we
|
||||
/// stopped due to an invalid block (leaving the DB at the last valid block).
|
||||
pub fn is_successful(&self) -> bool {
|
||||
self.is_complete() || self.stopped_on_invalid_block
|
||||
}
|
||||
}
|
||||
|
||||
/// Imports blocks from an RLP-encoded file into the database.
|
||||
@@ -103,6 +120,11 @@ where
|
||||
let static_file_producer =
|
||||
StaticFileProducer::new(provider_factory.clone(), PruneModes::default());
|
||||
|
||||
// Track if we stopped due to an invalid block
|
||||
let mut stopped_on_invalid_block = false;
|
||||
let mut bad_block_number: Option<u64> = None;
|
||||
let mut last_valid_block_number: Option<u64> = None;
|
||||
|
||||
while let Some(file_client) =
|
||||
reader.next_chunk::<BlockTy<N>>(consensus.clone(), Some(sealed_header)).await?
|
||||
{
|
||||
@@ -137,12 +159,51 @@ where
|
||||
|
||||
// Run pipeline
|
||||
info!(target: "reth::import", "Starting sync pipeline");
|
||||
tokio::select! {
|
||||
res = pipeline.run() => res?,
|
||||
_ = tokio::signal::ctrl_c() => {
|
||||
info!(target: "reth::import", "Import interrupted by user");
|
||||
break;
|
||||
},
|
||||
if import_config.fail_on_invalid_block {
|
||||
// Original behavior: fail on unwind
|
||||
tokio::select! {
|
||||
res = pipeline.run() => res?,
|
||||
_ = tokio::signal::ctrl_c() => {
|
||||
info!(target: "reth::import", "Import interrupted by user");
|
||||
break;
|
||||
},
|
||||
}
|
||||
} else {
|
||||
// Default behavior: Use run_loop() to handle unwinds gracefully
|
||||
let result = tokio::select! {
|
||||
res = pipeline.run_loop() => res,
|
||||
_ = tokio::signal::ctrl_c() => {
|
||||
info!(target: "reth::import", "Import interrupted by user");
|
||||
break;
|
||||
},
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(ControlFlow::Unwind { target, bad_block }) => {
|
||||
// An invalid block was encountered; stop at last valid block
|
||||
let bad = bad_block.block.number;
|
||||
warn!(
|
||||
target: "reth::import",
|
||||
bad_block = bad,
|
||||
last_valid_block = target,
|
||||
"Invalid block encountered during import; stopping at last valid block"
|
||||
);
|
||||
stopped_on_invalid_block = true;
|
||||
bad_block_number = Some(bad);
|
||||
last_valid_block_number = Some(target);
|
||||
break;
|
||||
}
|
||||
Ok(ControlFlow::Continue { block_number }) => {
|
||||
debug!(target: "reth::import", block_number, "Pipeline chunk completed");
|
||||
}
|
||||
Ok(ControlFlow::NoProgress { block_number }) => {
|
||||
debug!(target: "reth::import", ?block_number, "Pipeline made no progress");
|
||||
}
|
||||
Err(e) => {
|
||||
// Propagate other pipeline errors
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sealed_header = provider_factory
|
||||
@@ -160,9 +221,20 @@ where
|
||||
total_decoded_txns,
|
||||
total_imported_blocks,
|
||||
total_imported_txns,
|
||||
stopped_on_invalid_block,
|
||||
bad_block: bad_block_number,
|
||||
last_valid_block: last_valid_block_number,
|
||||
};
|
||||
|
||||
if !result.is_complete() {
|
||||
if result.stopped_on_invalid_block {
|
||||
info!(target: "reth::import",
|
||||
total_imported_blocks,
|
||||
total_imported_txns,
|
||||
bad_block = ?result.bad_block,
|
||||
last_valid_block = ?result.last_valid_block,
|
||||
"Import stopped at last valid block due to invalid block"
|
||||
);
|
||||
} else if !result.is_complete() {
|
||||
error!(target: "reth::import",
|
||||
total_decoded_blocks,
|
||||
total_imported_blocks,
|
||||
|
||||
@@ -10,7 +10,8 @@ use reth_node_builder::NodeBuilder;
|
||||
use reth_node_core::{
|
||||
args::{
|
||||
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, EraArgs, MetricArgs,
|
||||
NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
|
||||
NetworkArgs, PayloadBuilderArgs, PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs,
|
||||
TxPoolArgs,
|
||||
},
|
||||
node_config::NodeConfig,
|
||||
version,
|
||||
@@ -102,6 +103,10 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
|
||||
#[command(flatten)]
|
||||
pub pruning: PruningArgs,
|
||||
|
||||
/// All `RocksDB` table routing arguments
|
||||
#[command(flatten)]
|
||||
pub rocksdb: RocksDbArgs,
|
||||
|
||||
/// Engine cli arguments
|
||||
#[command(flatten, next_help_heading = "Engine")]
|
||||
pub engine: EngineArgs,
|
||||
@@ -166,12 +171,16 @@ where
|
||||
db,
|
||||
dev,
|
||||
pruning,
|
||||
rocksdb,
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
ext,
|
||||
} = self;
|
||||
|
||||
// Validate RocksDB arguments
|
||||
rocksdb.validate()?;
|
||||
|
||||
// set up node config
|
||||
let mut node_config = NodeConfig {
|
||||
datadir,
|
||||
@@ -187,6 +196,7 @@ where
|
||||
db,
|
||||
dev,
|
||||
pruning,
|
||||
rocksdb,
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
|
||||
34
crates/cli/commands/src/p2p/enode.rs
Normal file
34
crates/cli/commands/src/p2p/enode.rs
Normal file
@@ -0,0 +1,34 @@
|
||||
//! Enode identifier command
|
||||
|
||||
use clap::Parser;
|
||||
use reth_cli_util::get_secret_key;
|
||||
use reth_network_peers::NodeRecord;
|
||||
use std::{
|
||||
net::{IpAddr, Ipv4Addr, SocketAddr},
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
/// Print the enode identifier for a given secret key.
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct Command {
|
||||
/// Path to the secret key file for discovery.
|
||||
pub discovery_secret: PathBuf,
|
||||
|
||||
/// Optional IP address to include in the enode URL.
|
||||
///
|
||||
/// If not provided, defaults to 0.0.0.0.
|
||||
#[arg(long)]
|
||||
pub ip: Option<IpAddr>,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute the enode command.
|
||||
pub fn execute(self) -> eyre::Result<()> {
|
||||
let sk = get_secret_key(&self.discovery_secret)?;
|
||||
let ip = self.ip.unwrap_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED));
|
||||
let addr = SocketAddr::new(ip, 30303);
|
||||
let enr = NodeRecord::from_secret_key(addr, &sk);
|
||||
println!("{enr}");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,7 @@ use reth_node_core::{
|
||||
};
|
||||
|
||||
pub mod bootnode;
|
||||
pub mod enode;
|
||||
pub mod rlpx;
|
||||
|
||||
/// `reth p2p` command
|
||||
@@ -85,6 +86,9 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
Subcommands::Bootnode(command) => {
|
||||
command.execute().await?;
|
||||
}
|
||||
Subcommands::Enode(command) => {
|
||||
command.execute()?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -99,6 +103,7 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
Subcommands::Body { args, .. } => Some(&args.chain),
|
||||
Subcommands::Rlpx(_) => None,
|
||||
Subcommands::Bootnode(_) => None,
|
||||
Subcommands::Enode(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -126,6 +131,8 @@ pub enum Subcommands<C: ChainSpecParser> {
|
||||
Rlpx(rlpx::Command),
|
||||
/// Bootnode command
|
||||
Bootnode(bootnode::Command),
|
||||
/// Print enode identifier
|
||||
Enode(enode::Command),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Parser)]
|
||||
@@ -225,4 +232,16 @@ mod tests {
|
||||
let _args: Command<EthereumChainSpecParser> =
|
||||
Command::parse_from(["reth", "body", "--chain", "mainnet", "1000"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_enode_cmd() {
|
||||
let _args: Command<EthereumChainSpecParser> =
|
||||
Command::parse_from(["reth", "enode", "/tmp/secret"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_enode_cmd_with_ip() {
|
||||
let _args: Command<EthereumChainSpecParser> =
|
||||
Command::parse_from(["reth", "enode", "/tmp/secret", "--ip", "192.168.1.1"]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,16 @@
|
||||
//! Command that runs pruning without any limits.
|
||||
use crate::common::{AccessRights, CliNodeTypes, EnvironmentArgs};
|
||||
use clap::Parser;
|
||||
use reth_chainspec::{EthChainSpec, EthereumHardforks};
|
||||
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks};
|
||||
use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_node_builder::common::metrics_hooks;
|
||||
use reth_node_core::{args::MetricArgs, version::version_metadata};
|
||||
use reth_node_metrics::{
|
||||
chain::ChainSpecInfo,
|
||||
server::{MetricServer, MetricServerConfig},
|
||||
version::VersionInfo,
|
||||
};
|
||||
use reth_prune::PrunerBuilder;
|
||||
use reth_static_file::StaticFileProducer;
|
||||
use std::sync::Arc;
|
||||
@@ -13,14 +21,42 @@ use tracing::info;
|
||||
pub struct PruneCommand<C: ChainSpecParser> {
|
||||
#[command(flatten)]
|
||||
env: EnvironmentArgs<C>,
|
||||
|
||||
/// Prometheus metrics configuration.
|
||||
#[command(flatten)]
|
||||
metrics: MetricArgs,
|
||||
}
|
||||
|
||||
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneCommand<C> {
|
||||
/// Execute the `prune` command
|
||||
pub async fn execute<N: CliNodeTypes<ChainSpec = C::ChainSpec>>(self) -> eyre::Result<()> {
|
||||
pub async fn execute<N: CliNodeTypes<ChainSpec = C::ChainSpec>>(
|
||||
self,
|
||||
ctx: CliContext,
|
||||
) -> eyre::Result<()> {
|
||||
let env = self.env.init::<N>(AccessRights::RW)?;
|
||||
let provider_factory = env.provider_factory;
|
||||
let config = env.config.prune;
|
||||
let data_dir = env.data_dir;
|
||||
|
||||
if let Some(listen_addr) = self.metrics.prometheus {
|
||||
let config = MetricServerConfig::new(
|
||||
listen_addr,
|
||||
VersionInfo {
|
||||
version: version_metadata().cargo_pkg_version.as_ref(),
|
||||
build_timestamp: version_metadata().vergen_build_timestamp.as_ref(),
|
||||
cargo_features: version_metadata().vergen_cargo_features.as_ref(),
|
||||
git_sha: version_metadata().vergen_git_sha.as_ref(),
|
||||
target_triple: version_metadata().vergen_cargo_target_triple.as_ref(),
|
||||
build_profile: version_metadata().build_profile_name.as_ref(),
|
||||
},
|
||||
ChainSpecInfo { name: provider_factory.chain_spec().chain().to_string() },
|
||||
ctx.task_executor,
|
||||
metrics_hooks(&provider_factory),
|
||||
data_dir.pprof_dumps(),
|
||||
);
|
||||
|
||||
MetricServer::new(config).serve().await?;
|
||||
}
|
||||
|
||||
// Copy data from database to static files
|
||||
info!(target: "reth::cli", "Copying data from database to static files...");
|
||||
|
||||
@@ -42,9 +42,9 @@ pub struct Command<C: ChainSpecParser> {
|
||||
#[arg(long)]
|
||||
to: Option<u64>,
|
||||
|
||||
/// Number of tasks to run in parallel
|
||||
#[arg(long, default_value = "10")]
|
||||
num_tasks: u64,
|
||||
/// Number of tasks to run in parallel. Defaults to the number of available CPUs.
|
||||
#[arg(long)]
|
||||
num_tasks: Option<u64>,
|
||||
|
||||
/// Continues with execution when an invalid block is encountered and collects these blocks.
|
||||
#[arg(long)]
|
||||
@@ -84,12 +84,16 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
}
|
||||
};
|
||||
|
||||
let num_tasks = self.num_tasks.unwrap_or_else(|| {
|
||||
std::thread::available_parallelism().map(|n| n.get() as u64).unwrap_or(10)
|
||||
});
|
||||
|
||||
let total_blocks = max_block - min_block;
|
||||
let total_gas = calculate_gas_used_from_headers(
|
||||
&provider_factory.static_file_provider(),
|
||||
min_block..=max_block,
|
||||
)?;
|
||||
let blocks_per_task = total_blocks / self.num_tasks;
|
||||
let blocks_per_task = total_blocks / num_tasks;
|
||||
|
||||
let db_at = {
|
||||
let provider_factory = provider_factory.clone();
|
||||
@@ -107,10 +111,10 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
let _guard = cancellation.drop_guard();
|
||||
|
||||
let mut tasks = JoinSet::new();
|
||||
for i in 0..self.num_tasks {
|
||||
for i in 0..num_tasks {
|
||||
let start_block = min_block + i * blocks_per_task;
|
||||
let end_block =
|
||||
if i == self.num_tasks - 1 { max_block } else { start_block + blocks_per_task };
|
||||
if i == num_tasks - 1 { max_block } else { start_block + blocks_per_task };
|
||||
|
||||
// Spawn thread executing blocks
|
||||
let provider_factory = provider_factory.clone();
|
||||
@@ -148,7 +152,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
};
|
||||
|
||||
if let Err(err) = consensus
|
||||
.validate_block_post_execution(&block, &result)
|
||||
.validate_block_post_execution(&block, &result, None)
|
||||
.wrap_err_with(|| {
|
||||
format!("Failed to validate block {} {}", block.number(), block.hash())
|
||||
})
|
||||
|
||||
@@ -15,7 +15,8 @@ use reth_db_common::{
|
||||
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
|
||||
use reth_node_core::args::StageEnum;
|
||||
use reth_provider::{
|
||||
DBProvider, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter, TrieWriter,
|
||||
DBProvider, RocksDBProviderFactory, StaticFileProviderFactory, StaticFileWriter,
|
||||
StorageSettingsCache,
|
||||
};
|
||||
use reth_prune::PruneSegment;
|
||||
use reth_stages::StageId;
|
||||
@@ -90,11 +91,14 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
StaticFileSegment::AccountChangeSets => {
|
||||
writer.prune_account_changesets(highest_block)?;
|
||||
}
|
||||
StaticFileSegment::StorageChangeSets => {
|
||||
writer.prune_storage_changesets(highest_block)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let provider_rw = tool.provider_factory.database_provider_rw()?;
|
||||
let provider_rw = tool.provider_factory.unwind_provider_rw()?;
|
||||
let tx = provider_rw.tx_ref();
|
||||
|
||||
match self.stage {
|
||||
@@ -167,13 +171,21 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
None,
|
||||
)?;
|
||||
}
|
||||
StageEnum::MerkleChangeSets => {
|
||||
provider_rw.clear_trie_changesets()?;
|
||||
reset_stage_checkpoint(tx, StageId::MerkleChangeSets)?;
|
||||
}
|
||||
StageEnum::AccountHistory | StageEnum::StorageHistory => {
|
||||
tx.clear::<tables::AccountsHistory>()?;
|
||||
tx.clear::<tables::StoragesHistory>()?;
|
||||
let settings = provider_rw.cached_storage_settings();
|
||||
let rocksdb = tool.provider_factory.rocksdb_provider();
|
||||
|
||||
if settings.account_history_in_rocksdb {
|
||||
rocksdb.clear::<tables::AccountsHistory>()?;
|
||||
} else {
|
||||
tx.clear::<tables::AccountsHistory>()?;
|
||||
}
|
||||
|
||||
if settings.storages_history_in_rocksdb {
|
||||
rocksdb.clear::<tables::StoragesHistory>()?;
|
||||
} else {
|
||||
tx.clear::<tables::StoragesHistory>()?;
|
||||
}
|
||||
|
||||
reset_stage_checkpoint(tx, StageId::IndexAccountHistory)?;
|
||||
reset_stage_checkpoint(tx, StageId::IndexStorageHistory)?;
|
||||
@@ -181,7 +193,14 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
insert_genesis_history(&provider_rw, self.env.chain.genesis().alloc.iter())?;
|
||||
}
|
||||
StageEnum::TxLookup => {
|
||||
tx.clear::<tables::TransactionHashNumbers>()?;
|
||||
if provider_rw.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
|
||||
tool.provider_factory
|
||||
.rocksdb_provider()
|
||||
.clear::<tables::TransactionHashNumbers>()?;
|
||||
} else {
|
||||
tx.clear::<tables::TransactionHashNumbers>()?;
|
||||
}
|
||||
|
||||
reset_prune_checkpoint(tx, PruneSegment::TransactionLookup)?;
|
||||
|
||||
reset_stage_checkpoint(tx, StageId::TransactionLookup)?;
|
||||
|
||||
@@ -11,7 +11,6 @@ use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_cli_util::get_secret_key;
|
||||
use reth_config::config::{HashingConfig, SenderRecoveryConfig, TransactionLookupConfig};
|
||||
use reth_db_api::database_metrics::DatabaseMetrics;
|
||||
use reth_downloaders::{
|
||||
bodies::bodies::BodiesDownloaderBuilder,
|
||||
headers::reverse_headers::ReverseHeadersDownloaderBuilder,
|
||||
@@ -19,19 +18,19 @@ use reth_downloaders::{
|
||||
use reth_exex::ExExManagerHandle;
|
||||
use reth_network::BlockDownloaderProvider;
|
||||
use reth_network_p2p::HeadersClient;
|
||||
use reth_node_builder::common::metrics_hooks;
|
||||
use reth_node_core::{
|
||||
args::{NetworkArgs, StageEnum},
|
||||
version::version_metadata,
|
||||
};
|
||||
use reth_node_metrics::{
|
||||
chain::ChainSpecInfo,
|
||||
hooks::Hooks,
|
||||
server::{MetricServer, MetricServerConfig},
|
||||
version::VersionInfo,
|
||||
};
|
||||
use reth_provider::{
|
||||
ChainSpecProvider, DBProvider, DatabaseProviderFactory, StageCheckpointReader,
|
||||
StageCheckpointWriter, StaticFileProviderFactory,
|
||||
StageCheckpointWriter,
|
||||
};
|
||||
use reth_stages::{
|
||||
stages::{
|
||||
@@ -139,20 +138,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
},
|
||||
ChainSpecInfo { name: provider_factory.chain_spec().chain().to_string() },
|
||||
ctx.task_executor,
|
||||
Hooks::builder()
|
||||
.with_hook({
|
||||
let db = provider_factory.db_ref().clone();
|
||||
move || db.report_metrics()
|
||||
})
|
||||
.with_hook({
|
||||
let sfp = provider_factory.static_file_provider();
|
||||
move || {
|
||||
if let Err(error) = sfp.report_metrics() {
|
||||
error!(%error, "Failed to report metrics from static file provider");
|
||||
}
|
||||
}
|
||||
})
|
||||
.build(),
|
||||
metrics_hooks(&provider_factory),
|
||||
data_dir.pprof_dumps(),
|
||||
);
|
||||
|
||||
|
||||
@@ -26,7 +26,8 @@ rand_08.workspace = true
|
||||
thiserror.workspace = true
|
||||
serde.workspace = true
|
||||
|
||||
tracy-client = { workspace = true, optional = true, features = ["demangle"] }
|
||||
tracy-client = { workspace = true, optional = true }
|
||||
reth-tracing = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
rand.workspace = true
|
||||
@@ -46,7 +47,7 @@ jemalloc-prof = ["jemalloc", "tikv-jemallocator?/profiling"]
|
||||
jemalloc-unprefixed = ["jemalloc", "tikv-jemallocator?/unprefixed_malloc_on_supported_platforms"]
|
||||
|
||||
# Wraps the selected allocator in the tracy profiling allocator
|
||||
tracy-allocator = ["dep:tracy-client"]
|
||||
tracy-allocator = ["dep:tracy-client", "dep:reth-tracing"]
|
||||
|
||||
snmalloc = ["dep:snmalloc-rs"]
|
||||
|
||||
|
||||
@@ -25,7 +25,6 @@ cfg_if::cfg_if! {
|
||||
cfg_if::cfg_if! {
|
||||
if #[cfg(feature = "tracy-allocator")] {
|
||||
type AllocatorWrapper = tracy_client::ProfiledAllocator<AllocatorInner>;
|
||||
tracy_client::register_demangler!();
|
||||
const fn new_allocator_wrapper() -> AllocatorWrapper {
|
||||
AllocatorWrapper::new(AllocatorInner {}, 100)
|
||||
}
|
||||
|
||||
@@ -8,6 +8,9 @@
|
||||
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
|
||||
#![cfg_attr(docsrs, feature(doc_cfg))]
|
||||
|
||||
#[cfg(feature = "tracy-allocator")]
|
||||
use reth_tracing as _;
|
||||
|
||||
pub mod allocator;
|
||||
pub mod cancellation;
|
||||
|
||||
|
||||
@@ -121,7 +121,16 @@ pub fn install() {
|
||||
unsafe {
|
||||
let alt_stack_size: usize = min_sigstack_size() + 64 * 1024;
|
||||
let mut alt_stack: libc::stack_t = mem::zeroed();
|
||||
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, 1).unwrap()).cast();
|
||||
// Both SysV AMD64 ABI and aarch64 ABI require 16 bytes alignment. We are going to be
|
||||
// generous here and just use a size of a page.
|
||||
let raw_page_sz = libc::sysconf(libc::_SC_PAGESIZE);
|
||||
let page_sz = if raw_page_sz == -1 {
|
||||
// Fallback alignment in case sysconf fails.
|
||||
4096_usize
|
||||
} else {
|
||||
raw_page_sz as usize
|
||||
};
|
||||
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, page_sz).unwrap()).cast();
|
||||
alt_stack.ss_size = alt_stack_size;
|
||||
libc::sigaltstack(&raw const alt_stack, ptr::null_mut());
|
||||
|
||||
|
||||
@@ -2,9 +2,8 @@
|
||||
use reth_network_types::{PeersConfig, SessionsConfig};
|
||||
use reth_prune_types::PruneModes;
|
||||
use reth_stages_types::ExecutionStageThresholds;
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_static_file_types::{StaticFileMap, StaticFileSegment};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::{Path, PathBuf},
|
||||
time::Duration,
|
||||
};
|
||||
@@ -439,6 +438,8 @@ pub struct BlocksPerFileConfig {
|
||||
pub transaction_senders: Option<u64>,
|
||||
/// Number of blocks per file for the account changesets segment.
|
||||
pub account_change_sets: Option<u64>,
|
||||
/// Number of blocks per file for the storage changesets segment.
|
||||
pub storage_change_sets: Option<u64>,
|
||||
}
|
||||
|
||||
impl StaticFilesConfig {
|
||||
@@ -452,6 +453,7 @@ impl StaticFilesConfig {
|
||||
receipts,
|
||||
transaction_senders,
|
||||
account_change_sets,
|
||||
storage_change_sets,
|
||||
} = self.blocks_per_file;
|
||||
eyre::ensure!(headers != Some(0), "Headers segment blocks per file must be greater than 0");
|
||||
eyre::ensure!(
|
||||
@@ -470,20 +472,25 @@ impl StaticFilesConfig {
|
||||
account_change_sets != Some(0),
|
||||
"Account changesets segment blocks per file must be greater than 0"
|
||||
);
|
||||
eyre::ensure!(
|
||||
storage_change_sets != Some(0),
|
||||
"Storage changesets segment blocks per file must be greater than 0"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Converts the blocks per file configuration into a [`HashMap`] per segment.
|
||||
pub fn as_blocks_per_file_map(&self) -> HashMap<StaticFileSegment, u64> {
|
||||
/// Converts the blocks per file configuration into a [`StaticFileMap`].
|
||||
pub fn as_blocks_per_file_map(&self) -> StaticFileMap<u64> {
|
||||
let BlocksPerFileConfig {
|
||||
headers,
|
||||
transactions,
|
||||
receipts,
|
||||
transaction_senders,
|
||||
account_change_sets,
|
||||
storage_change_sets,
|
||||
} = self.blocks_per_file;
|
||||
|
||||
let mut map = HashMap::new();
|
||||
let mut map = StaticFileMap::default();
|
||||
// Iterating over all possible segments allows us to do an exhaustive match here,
|
||||
// to not forget to configure new segments in the future.
|
||||
for segment in StaticFileSegment::iter() {
|
||||
@@ -493,6 +500,7 @@ impl StaticFilesConfig {
|
||||
StaticFileSegment::Receipts => receipts,
|
||||
StaticFileSegment::TransactionSenders => transaction_senders,
|
||||
StaticFileSegment::AccountChangeSets => account_change_sets,
|
||||
StaticFileSegment::StorageChangeSets => storage_change_sets,
|
||||
};
|
||||
|
||||
if let Some(blocks_per_file) = blocks_per_file {
|
||||
@@ -551,7 +559,6 @@ impl PruneConfig {
|
||||
/// - `Option<PruneMode>` fields: set from `other` only if `self` is `None`.
|
||||
/// - `block_interval`: set from `other` only if `self.block_interval ==
|
||||
/// DEFAULT_BLOCK_INTERVAL`.
|
||||
/// - `merkle_changesets`: always set from `other`.
|
||||
/// - `receipts_log_filter`: set from `other` only if `self` is empty and `other` is non-empty.
|
||||
pub fn merge(&mut self, other: Self) {
|
||||
let Self {
|
||||
@@ -564,7 +571,6 @@ impl PruneConfig {
|
||||
account_history,
|
||||
storage_history,
|
||||
bodies_history,
|
||||
merkle_changesets,
|
||||
receipts_log_filter,
|
||||
},
|
||||
} = other;
|
||||
@@ -581,8 +587,6 @@ impl PruneConfig {
|
||||
self.segments.account_history = self.segments.account_history.or(account_history);
|
||||
self.segments.storage_history = self.segments.storage_history.or(storage_history);
|
||||
self.segments.bodies_history = self.segments.bodies_history.or(bodies_history);
|
||||
// Merkle changesets is not optional; always take the value from `other`
|
||||
self.segments.merkle_changesets = merkle_changesets;
|
||||
|
||||
if self.segments.receipts_log_filter.0.is_empty() && !receipts_log_filter.0.is_empty() {
|
||||
self.segments.receipts_log_filter = receipts_log_filter;
|
||||
@@ -1079,18 +1083,6 @@ transaction_lookup = 'full'
|
||||
receipts = { distance = 16384 }
|
||||
#";
|
||||
let _conf: Config = toml::from_str(s).unwrap();
|
||||
|
||||
let s = r"#
|
||||
[prune]
|
||||
block_interval = 5
|
||||
|
||||
[prune.segments]
|
||||
sender_recovery = { distance = 16384 }
|
||||
transaction_lookup = 'full'
|
||||
receipts = 'full'
|
||||
#";
|
||||
let err = toml::from_str::<Config>(s).unwrap_err().to_string();
|
||||
assert!(err.contains("invalid value: string \"full\""), "{}", err);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1104,7 +1096,6 @@ receipts = 'full'
|
||||
account_history: None,
|
||||
storage_history: Some(PruneMode::Before(5000)),
|
||||
bodies_history: None,
|
||||
merkle_changesets: PruneMode::Before(0),
|
||||
receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([(
|
||||
Address::random(),
|
||||
PruneMode::Full,
|
||||
@@ -1121,7 +1112,6 @@ receipts = 'full'
|
||||
account_history: Some(PruneMode::Distance(2000)),
|
||||
storage_history: Some(PruneMode::Distance(3000)),
|
||||
bodies_history: None,
|
||||
merkle_changesets: PruneMode::Distance(10000),
|
||||
receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([
|
||||
(Address::random(), PruneMode::Distance(1000)),
|
||||
(Address::random(), PruneMode::Before(2000)),
|
||||
@@ -1140,7 +1130,6 @@ receipts = 'full'
|
||||
assert_eq!(config1.segments.receipts, Some(PruneMode::Distance(1000)));
|
||||
assert_eq!(config1.segments.account_history, Some(PruneMode::Distance(2000)));
|
||||
assert_eq!(config1.segments.storage_history, Some(PruneMode::Before(5000)));
|
||||
assert_eq!(config1.segments.merkle_changesets, PruneMode::Distance(10000));
|
||||
assert_eq!(config1.segments.receipts_log_filter, original_filter);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,14 +1,11 @@
|
||||
//! Collection of methods for block validation.
|
||||
|
||||
use alloy_consensus::{BlockHeader as _, Transaction, EMPTY_OMMER_ROOT_HASH};
|
||||
use alloy_consensus::{BlockHeader as _, EMPTY_OMMER_ROOT_HASH};
|
||||
use alloy_eips::{eip4844::DATA_GAS_PER_BLOB, eip7840::BlobParams};
|
||||
use reth_chainspec::{EthChainSpec, EthereumHardfork, EthereumHardforks};
|
||||
use reth_consensus::{ConsensusError, TxGasLimitTooHighErr};
|
||||
use reth_consensus::ConsensusError;
|
||||
use reth_primitives_traits::{
|
||||
constants::{
|
||||
GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK, MAX_TX_GAS_LIMIT_OSAKA, MINIMUM_GAS_LIMIT,
|
||||
},
|
||||
transaction::TxHashRef,
|
||||
constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK, MINIMUM_GAS_LIMIT},
|
||||
Block, BlockBody, BlockHeader, GotExpected, SealedBlock, SealedHeader,
|
||||
};
|
||||
|
||||
@@ -146,7 +143,7 @@ pub fn validate_block_pre_execution<B, ChainSpec>(
|
||||
) -> Result<(), ConsensusError>
|
||||
where
|
||||
B: Block,
|
||||
ChainSpec: EthereumHardforks,
|
||||
ChainSpec: EthChainSpec + EthereumHardforks,
|
||||
{
|
||||
post_merge_hardfork_fields(block, chain_spec)?;
|
||||
|
||||
@@ -154,19 +151,6 @@ where
|
||||
if let Err(error) = block.ensure_transaction_root_valid() {
|
||||
return Err(ConsensusError::BodyTransactionRootDiff(error.into()))
|
||||
}
|
||||
// EIP-7825 validation
|
||||
if chain_spec.is_osaka_active_at_timestamp(block.timestamp()) {
|
||||
for tx in block.body().transactions() {
|
||||
if tx.gas_limit() > MAX_TX_GAS_LIMIT_OSAKA {
|
||||
return Err(TxGasLimitTooHighErr {
|
||||
tx_hash: *tx.tx_hash(),
|
||||
gas_limit: tx.gas_limit(),
|
||||
max_allowed: MAX_TX_GAS_LIMIT_OSAKA,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -15,6 +15,12 @@ use alloc::{boxed::Box, fmt::Debug, string::String, sync::Arc, vec::Vec};
|
||||
use alloy_consensus::Header;
|
||||
use alloy_primitives::{BlockHash, BlockNumber, Bloom, B256};
|
||||
use core::error::Error;
|
||||
|
||||
/// Pre-computed receipt root and logs bloom.
|
||||
///
|
||||
/// When provided to [`FullConsensus::validate_block_post_execution`], this allows skipping
|
||||
/// the receipt root computation and using the pre-computed values instead.
|
||||
pub type ReceiptRootBloom = (B256, Bloom);
|
||||
use reth_execution_types::BlockExecutionResult;
|
||||
use reth_primitives_traits::{
|
||||
constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK, MINIMUM_GAS_LIMIT},
|
||||
@@ -39,11 +45,15 @@ pub trait FullConsensus<N: NodePrimitives>: Consensus<N::Block> {
|
||||
///
|
||||
/// See the Yellow Paper sections 4.3.2 "Holistic Validity".
|
||||
///
|
||||
/// If `receipt_root_bloom` is provided, the implementation should use the pre-computed
|
||||
/// receipt root and logs bloom instead of computing them from the receipts.
|
||||
///
|
||||
/// Note: validating blocks does not include other validations of the Consensus
|
||||
fn validate_block_post_execution(
|
||||
&self,
|
||||
block: &RecoveredBlock<N::Block>,
|
||||
result: &BlockExecutionResult<N::Receipt>,
|
||||
receipt_root_bloom: Option<ReceiptRootBloom>,
|
||||
) -> Result<(), ConsensusError>;
|
||||
}
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
//!
|
||||
//! **Not for production use** - provides no security guarantees or consensus validation.
|
||||
|
||||
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator};
|
||||
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator, ReceiptRootBloom};
|
||||
use alloc::sync::Arc;
|
||||
use reth_execution_types::BlockExecutionResult;
|
||||
use reth_primitives_traits::{Block, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
|
||||
@@ -76,6 +76,7 @@ impl<N: NodePrimitives> FullConsensus<N> for NoopConsensus {
|
||||
&self,
|
||||
_block: &RecoveredBlock<N::Block>,
|
||||
_result: &BlockExecutionResult<N::Receipt>,
|
||||
_receipt_root_bloom: Option<ReceiptRootBloom>,
|
||||
) -> Result<(), ConsensusError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator};
|
||||
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator, ReceiptRootBloom};
|
||||
use core::sync::atomic::{AtomicBool, Ordering};
|
||||
use reth_execution_types::BlockExecutionResult;
|
||||
use reth_primitives_traits::{Block, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
|
||||
@@ -51,6 +51,7 @@ impl<N: NodePrimitives> FullConsensus<N> for TestConsensus {
|
||||
&self,
|
||||
_block: &RecoveredBlock<N::Block>,
|
||||
_result: &BlockExecutionResult<N::Receipt>,
|
||||
_receipt_root_bloom: Option<ReceiptRootBloom>,
|
||||
) -> Result<(), ConsensusError> {
|
||||
if self.fail_validation() {
|
||||
Err(ConsensusError::BaseFeeMissing)
|
||||
|
||||
@@ -38,12 +38,8 @@ pub trait BlockProvider: Send + Sync + 'static {
|
||||
offset: usize,
|
||||
) -> impl Future<Output = eyre::Result<B256>> + Send {
|
||||
async move {
|
||||
let stored_hash = previous_block_hashes
|
||||
.len()
|
||||
.checked_sub(offset)
|
||||
.and_then(|index| previous_block_hashes.get(index));
|
||||
if let Some(hash) = stored_hash {
|
||||
return Ok(*hash);
|
||||
if let Some(hash) = get_hash_at_offset(previous_block_hashes, offset) {
|
||||
return Ok(hash);
|
||||
}
|
||||
|
||||
// Return zero hash if the chain isn't long enough to have the block at the offset.
|
||||
@@ -83,7 +79,7 @@ where
|
||||
/// Spawn the client to start sending FCUs and new payloads by periodically fetching recent
|
||||
/// blocks.
|
||||
pub async fn run(self) {
|
||||
let mut previous_block_hashes = AllocRingBuffer::new(64);
|
||||
let mut previous_block_hashes = AllocRingBuffer::new(65);
|
||||
let mut block_stream = {
|
||||
let (tx, rx) = mpsc::channel::<P::Block>(64);
|
||||
let block_provider = self.block_provider.clone();
|
||||
@@ -142,3 +138,60 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Looks up a block hash from the ring buffer at the given offset from the most recent entry.
|
||||
///
|
||||
/// Returns `None` if the buffer doesn't have enough entries to satisfy the offset.
|
||||
fn get_hash_at_offset(buffer: &AllocRingBuffer<B256>, offset: usize) -> Option<B256> {
|
||||
buffer.len().checked_sub(offset + 1).and_then(|index| buffer.get(index).copied())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_get_hash_at_offset() {
|
||||
let mut buffer: AllocRingBuffer<B256> = AllocRingBuffer::new(65);
|
||||
|
||||
// Empty buffer returns None for any offset
|
||||
assert_eq!(get_hash_at_offset(&buffer, 0), None);
|
||||
assert_eq!(get_hash_at_offset(&buffer, 1), None);
|
||||
|
||||
// Push hashes 0..65
|
||||
for i in 0..65u8 {
|
||||
buffer.push(B256::with_last_byte(i));
|
||||
}
|
||||
|
||||
// offset=0 should return the most recent (64)
|
||||
assert_eq!(get_hash_at_offset(&buffer, 0), Some(B256::with_last_byte(64)));
|
||||
|
||||
// offset=32 (safe block) should return hash 32
|
||||
assert_eq!(get_hash_at_offset(&buffer, 32), Some(B256::with_last_byte(32)));
|
||||
|
||||
// offset=64 (finalized block) should return hash 0 (the oldest)
|
||||
assert_eq!(get_hash_at_offset(&buffer, 64), Some(B256::with_last_byte(0)));
|
||||
|
||||
// offset=65 exceeds buffer, should return None
|
||||
assert_eq!(get_hash_at_offset(&buffer, 65), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_hash_at_offset_insufficient_entries() {
|
||||
let mut buffer: AllocRingBuffer<B256> = AllocRingBuffer::new(65);
|
||||
|
||||
// With only 1 entry, only offset=0 works
|
||||
buffer.push(B256::with_last_byte(1));
|
||||
assert_eq!(get_hash_at_offset(&buffer, 0), Some(B256::with_last_byte(1)));
|
||||
assert_eq!(get_hash_at_offset(&buffer, 1), None);
|
||||
assert_eq!(get_hash_at_offset(&buffer, 32), None);
|
||||
assert_eq!(get_hash_at_offset(&buffer, 64), None);
|
||||
|
||||
// With 33 entries, offset=32 works but offset=64 doesn't
|
||||
for i in 2..=33u8 {
|
||||
buffer.push(B256::with_last_byte(i));
|
||||
}
|
||||
assert_eq!(get_hash_at_offset(&buffer, 32), Some(B256::with_last_byte(1)));
|
||||
assert_eq!(get_hash_at_offset(&buffer, 64), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,3 +72,11 @@ derive_more.workspace = true
|
||||
[[test]]
|
||||
name = "e2e_testsuite"
|
||||
path = "tests/e2e-testsuite/main.rs"
|
||||
|
||||
[[test]]
|
||||
name = "rocksdb"
|
||||
path = "tests/rocksdb/main.rs"
|
||||
required-features = ["edge"]
|
||||
|
||||
[features]
|
||||
edge = ["reth-node-core/edge", "reth-provider/rocksdb", "reth-cli-commands/edge"]
|
||||
|
||||
@@ -103,7 +103,10 @@ where
|
||||
N: NodeBuilderHelper,
|
||||
{
|
||||
E2ETestSetupBuilder::new(num_nodes, chain_spec, attributes_generator)
|
||||
.with_tree_config_modifier(move |_| tree_config.clone())
|
||||
.with_tree_config_modifier(move |base| {
|
||||
// Apply caller's tree_config but preserve the small cache size from base
|
||||
tree_config.clone().with_cross_block_cache_size(base.cross_block_cache_size())
|
||||
})
|
||||
.with_node_config_modifier(move |config| config.set_dev(is_dev))
|
||||
.with_connect_nodes(connect_nodes)
|
||||
.build()
|
||||
|
||||
@@ -2,11 +2,11 @@ use crate::{network::NetworkTestContext, payload::PayloadTestContext, rpc::RpcTe
|
||||
use alloy_consensus::{transaction::TxHashRef, BlockHeader};
|
||||
use alloy_eips::BlockId;
|
||||
use alloy_primitives::{BlockHash, BlockNumber, Bytes, Sealable, B256};
|
||||
use alloy_rpc_types_engine::ForkchoiceState;
|
||||
use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV5, ForkchoiceState};
|
||||
use alloy_rpc_types_eth::BlockNumberOrTag;
|
||||
use eyre::Ok;
|
||||
use futures_util::Future;
|
||||
use jsonrpsee::http_client::HttpClient;
|
||||
use jsonrpsee::{core::client::ClientT, http_client::HttpClient};
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_network_api::test_utils::PeersHandleProvider;
|
||||
use reth_node_api::{
|
||||
@@ -20,6 +20,7 @@ use reth_provider::{
|
||||
BlockReader, BlockReaderIdExt, CanonStateNotificationStream, CanonStateSubscriptions,
|
||||
HeaderProvider, StageCheckpointReader,
|
||||
};
|
||||
use reth_rpc_api::TestingBuildBlockRequestV1;
|
||||
use reth_rpc_builder::auth::AuthServerHandle;
|
||||
use reth_rpc_eth_api::helpers::{EthApiSpec, EthTransactions, TraceExt};
|
||||
use reth_stages_types::StageId;
|
||||
@@ -319,4 +320,20 @@ where
|
||||
|
||||
Ok(crate::testsuite::NodeClient::new_with_beacon_engine(rpc, auth, url, beacon_handle))
|
||||
}
|
||||
|
||||
/// Calls the `testing_buildBlockV1` RPC on this node.
|
||||
///
|
||||
/// This endpoint builds a block using the provided parent, payload attributes, and
|
||||
/// transactions. Requires the `Testing` RPC module to be enabled.
|
||||
pub async fn testing_build_block_v1(
|
||||
&self,
|
||||
request: TestingBuildBlockRequestV1,
|
||||
) -> eyre::Result<ExecutionPayloadEnvelopeV5> {
|
||||
let client =
|
||||
self.rpc_client().ok_or_else(|| eyre::eyre!("HTTP RPC client not available"))?;
|
||||
|
||||
let res: ExecutionPayloadEnvelopeV5 =
|
||||
client.request("testing_buildBlockV1", [request]).await?;
|
||||
eyre::Ok(res)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,11 +112,13 @@ where
|
||||
..NetworkArgs::default()
|
||||
};
|
||||
|
||||
// Apply tree config modifier if present
|
||||
// Apply tree config modifier if present, with test-appropriate defaults
|
||||
let base_tree_config =
|
||||
reth_node_api::TreeConfig::default().with_cross_block_cache_size(1024 * 1024);
|
||||
let tree_config = if let Some(modifier) = self.tree_config_modifier {
|
||||
modifier(reth_node_api::TreeConfig::default())
|
||||
modifier(base_tree_config)
|
||||
} else {
|
||||
reth_node_api::TreeConfig::default()
|
||||
base_tree_config
|
||||
};
|
||||
|
||||
let mut nodes = (0..self.num_nodes)
|
||||
|
||||
@@ -125,7 +125,10 @@ pub async fn setup_engine_with_chain_import(
|
||||
db.clone(),
|
||||
chain_spec.clone(),
|
||||
reth_provider::providers::StaticFileProvider::read_write(static_files_path.clone())?,
|
||||
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path).build().unwrap(),
|
||||
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
|
||||
.with_default_tables()
|
||||
.build()
|
||||
.unwrap(),
|
||||
)?;
|
||||
|
||||
// Initialize genesis if needed
|
||||
@@ -328,6 +331,7 @@ mod tests {
|
||||
reth_provider::providers::StaticFileProvider::read_write(static_files_path.clone())
|
||||
.unwrap(),
|
||||
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path.clone())
|
||||
.with_default_tables()
|
||||
.build()
|
||||
.unwrap(),
|
||||
)
|
||||
@@ -392,6 +396,7 @@ mod tests {
|
||||
reth_provider::providers::StaticFileProvider::read_only(static_files_path, false)
|
||||
.unwrap(),
|
||||
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
|
||||
.with_default_tables()
|
||||
.build()
|
||||
.unwrap(),
|
||||
)
|
||||
@@ -490,7 +495,10 @@ mod tests {
|
||||
db.clone(),
|
||||
chain_spec.clone(),
|
||||
reth_provider::providers::StaticFileProvider::read_write(static_files_path).unwrap(),
|
||||
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path).build().unwrap(),
|
||||
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
|
||||
.with_default_tables()
|
||||
.build()
|
||||
.unwrap(),
|
||||
)
|
||||
.expect("failed to create provider factory");
|
||||
|
||||
|
||||
@@ -38,6 +38,18 @@ impl TransactionTestContext {
|
||||
signed.encoded_2718().into()
|
||||
}
|
||||
|
||||
/// Creates a transfer with a specific nonce and signs it, returning bytes.
|
||||
/// Uses high `max_fee_per_gas` (1000 gwei) to ensure tx acceptance regardless of basefee.
|
||||
pub async fn transfer_tx_bytes_with_nonce(
|
||||
chain_id: u64,
|
||||
wallet: PrivateKeySigner,
|
||||
nonce: u64,
|
||||
) -> Bytes {
|
||||
let tx = tx(chain_id, 21000, None, None, nonce, Some(1000e9 as u128));
|
||||
let signed = Self::sign_tx(wallet, tx).await;
|
||||
signed.encoded_2718().into()
|
||||
}
|
||||
|
||||
/// Creates a deployment transaction and signs it, returning an envelope.
|
||||
pub async fn deploy_tx(
|
||||
chain_id: u64,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user