Compare commits

..

103 Commits

Author SHA1 Message Date
Matthias Seitz
6cbcfe01a0 Merge branch 'main' into feat/bal-cache 2026-01-23 16:09:45 +01:00
Gigi
830cd5e355 chore: update snmalloc upstream repository link (#21360) 2026-01-23 14:57:46 +00:00
Georgios Konstantopoulos
f77d7d5983 feat(reth-bench): support human-readable gas format in generate-big-block (#21361) 2026-01-23 14:24:34 +00:00
Georgios Konstantopoulos
a2237c534e feat(p2p): add reth p2p enode command (#21357)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-23 13:23:44 +00:00
Arsenii Kulikov
1bd8fab887 feat(txpool): add Block associated type to TransactionValidator trait (#21359) 2026-01-23 13:16:05 +00:00
Matthias Seitz
22a68756c7 fix(tree): evict changeset cache even when finalized block is unset (#21354) 2026-01-23 11:26:57 +00:00
Hwangjae Lee
d99c0ffd62 chore(etc): update ethereum-metrics-exporter GitHub URL (#21348)
Signed-off-by: Hwangjae Lee <meetrick@gmail.com>
2026-01-23 10:59:53 +00:00
Georgios Konstantopoulos
ad476e2b5c chore: add yongkangc as codeowner for crates/storage/provider (#21349) 2026-01-23 07:18:18 +00:00
Matthias Seitz
6df249c1f1 feat(engine): stub Amsterdam engine API endpoints (newPayloadV5, getPayloadV6, BALs) (#21344)
Co-authored-by: Ishika Choudhury <117741714+Rimeeeeee@users.noreply.github.com>
Co-authored-by: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com>
2026-01-22 20:48:11 +00:00
Arsenii Kulikov
5a076df09a feat: allow setting custom debug block provider (#21345)
Co-authored-by: Karl <yh975593284@gmail.com>
2026-01-22 20:40:26 +00:00
YK
f07629eac0 perf: avoid creating RocksDB transactions for legacy MDBX-only nodes (#21325) 2026-01-22 20:30:52 +00:00
Dan Cline
f643e93c35 feat(reth-bench): send-invalid-payload command (#21335) 2026-01-22 19:42:19 +00:00
Matthias Seitz
653362a436 ci: align check-alloy workflow with main clippy job (#21329) 2026-01-22 20:48:53 +01:00
Seola Oh
a02508600c chore(txpool): explicitly deref RwLockReadGuard in PartialEq impl (#21336) 2026-01-22 19:35:00 +00:00
Georgios Konstantopoulos
937a7f226d fix(rpc): use Default for SimulateError to prepare for alloy breaking change (#21319)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-22 19:14:58 +00:00
joshieDo
a0df561117 fix(rocksdb): periodic batch commits in stages to prevent OOM (#21334)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-22 19:04:56 +00:00
Arsenii Kulikov
be5a4ac7a6 feat: bump alloy and alloy-evm (#21337) 2026-01-22 18:43:24 +00:00
Georgios Konstantopoulos
0c854b6f14 fix(net): limit pending pool imports for broadcast transactions (#21254)
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
2026-01-22 18:32:07 +00:00
Georgios Konstantopoulos
28a31cd579 fix: use unwrap_or_else for lazy evaluation of StorageSettings::legacy (#21332) 2026-01-22 17:02:15 +00:00
Brian Picciano
da12451c9c chore(trie): Cleanup unused trie changesets code (#21323) 2026-01-22 16:57:46 +00:00
Georgios Konstantopoulos
247ce3c4e9 feat(storage): warn storage settings diff at startup (#21320)
Co-authored-by: YK <chiayongkang@hotmail.com>
2026-01-22 16:40:10 +00:00
iPLAY888
bf43ebaa29 fix(cli): handle invalid hex in db list --search (#21315) 2026-01-22 16:18:36 +00:00
Matthias Seitz
a01ecce73f test: add e2e tests for invalid payload handling via Engine API (#21288) 2026-01-22 15:55:36 +00:00
Arsenii Kulikov
3e55c6ca6e fix: always check upper subtrie for keys (#21276)
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-01-22 15:47:50 +00:00
Brian Picciano
2ac7d719f3 feat(trie): add V2 account proof computation and refactor proof types (reapply) (#21316)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-22 15:46:01 +00:00
andrewshab
965705ff88 fix: remove collect (#21318)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-22 15:24:51 +00:00
Dan Cline
ebe2ca1366 feat: add StaticFileSegment::StorageChangeSets (#20896) 2026-01-22 15:03:47 +00:00
Matthias Seitz
cc242f83fd feat(rpc): respect history expiry in eth_getLogs per EIP-4444 (#21304) 2026-01-22 14:55:50 +00:00
joshieDo
12cf3d685b fix(provider): add CommitOrder for RocksDB/MDBX unwind atomicity (#21311) 2026-01-22 14:54:47 +00:00
Matthias Seitz
ad5b533ad1 chore: rm patches (#21317) 2026-01-22 15:48:53 +01:00
joshieDo
118f15f345 feat(rocksdb): disable bloom filter for default column family (#21312) 2026-01-22 13:47:34 +00:00
joshieDo
97481f69e5 perf(rocksdb): disable compression and bloom filters for TransactionHashNumbers CF (#21310) 2026-01-22 13:31:16 +00:00
Georgios Konstantopoulos
f692ac7d1e perf(prune): use bulk table clear for PruneMode::Full (#21302) 2026-01-22 13:01:17 +00:00
andrewshab
4b1c341ced fix: remove redundant clone (#21300) 2026-01-22 12:43:19 +00:00
Georgios Konstantopoulos
865f8f8951 perf(prune): sort tx hashes for efficient TransactionLookup pruning (#21297) 2026-01-22 12:10:07 +00:00
joshieDo
492fc20fd1 fix(cli): clear rocksdb tables in drop-stage command (#21299)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-22 12:09:36 +00:00
Sergei Shulepov
ad9886abb8 fix(mdbx): mark reserve as unsafe (#21263) 2026-01-22 12:03:12 +00:00
Matthias Seitz
5c3e45cd6b fix: handle incomplete receipts gracefully in receipt root task (#21285) 2026-01-22 10:52:56 +00:00
Emma Jamieson-Hoare
68fdba32d2 chore(release): prep v1.10.2 release (#21287)
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmai.com>
2026-01-22 10:50:10 +00:00
Matthias Seitz
8f6a0a2992 ci: add on-demand workflow to check alloy breaking changes (#21267) 2026-01-22 10:47:38 +00:00
Matthias Seitz
ec9c7f8d3e perf(db): use ArrayVec for StoredNibbles key encoding (#21279) 2026-01-22 02:05:50 +00:00
Matthias Seitz
dbdaf068f0 fix(engine): clear execution cache when block validation fails (#21282) 2026-01-22 01:01:22 +00:00
Matthias Seitz
055bf63ee9 refactor: use Default::default() for Header in tests (#21277) 2026-01-21 22:50:10 +00:00
Georgios Konstantopoulos
2305c3ebeb feat(rpc): respect history expiry in block() and map to PrunedHistoryUnavailable (#21270) 2026-01-21 22:22:05 +00:00
joshieDo
eb55c3c3da feat(grafana): add RocksDB metrics dashboard (#21243)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-21 22:09:42 +00:00
Alexey Shekhirin
72e1467ba3 fix(prune): avoid panic in tx lookup (#21275) 2026-01-21 21:21:53 +00:00
Alexey Shekhirin
74edce0089 revert: feat(trie): add V2 account proof computation and refactor proof types (#21214) (#21274) 2026-01-21 21:07:13 +00:00
Georgios Konstantopoulos
8c645d5762 feat(reth-bench): accept short notation for --target-gas-limit (#21273) 2026-01-21 21:04:10 +00:00
Georgios Konstantopoulos
b7d2ee2566 feat(engine): add metric for execution cache unavailability due to concurrent use (#21265)
Co-authored-by: Tempo AI <ai@tempo.xyz>
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-01-21 20:17:45 +00:00
Matthias Seitz
7609deddda perf(trie): parallelize merge_ancestors_into_overlay (#21202) 2026-01-21 20:08:03 +00:00
Matthias Seitz
ec50fd40b3 chore(chainspec): use ..Default::default() in create_chain_config (#21266) 2026-01-21 19:19:24 +00:00
YK
624ddc5779 feat(stages): add RocksDB support for IndexStorageHistoryStage (#21175) 2026-01-21 17:05:19 +00:00
Georgios Konstantopoulos
dd72cfe23e refactor: remove static_files.to_settings() and add edge feature to RocksDB flags (#21225) 2026-01-21 16:52:24 +00:00
joshieDo
ff8ac97e33 fix(stages): clear ETL collectors on HeaderStage error paths (#21258) 2026-01-21 16:27:30 +00:00
Alexey Shekhirin
0974485863 feat(reth-bench): add --target-gas-limit option to gas-limit-ramp (#21262) 2026-01-21 16:19:22 +00:00
かりんとう
274394e777 fix: fix payload file filter prefix in replay-payloads (#21255) 2026-01-21 16:11:03 +00:00
Emma Jamieson-Hoare
1954c91a60 chore: update CODEOWNERS (#21223)
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmai.com>
Co-authored-by: YK <chiayongkang@hotmail.com>
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
2026-01-21 14:40:54 +00:00
Sergei Shulepov
9cf82c8403 fix: supply a real ptr to mdbx_dbi_flags_ex (#21230) 2026-01-21 14:23:26 +00:00
Brian Picciano
f85fcba872 feat(trie): add V2 account proof computation and refactor proof types (#21214)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-21 14:18:44 +00:00
joshieDo
ebaa4bda3a feat(rocksdb): add missing observability (#21253) 2026-01-21 14:14:34 +00:00
joshieDo
04d4c9a02f fix(rocksdb): flush all column families on drop and show SST/memtable sizes (#21251)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-21 12:44:08 +00:00
Arsenii Kulikov
3065a328f9 fix: clear overlay_cache in with_extended_hashed_state_overlay (#21233) 2026-01-21 12:08:24 +00:00
Matthias Seitz
a0aac13f75 fix: make len() private to satisfy clippy 2026-01-21 12:36:03 +01:00
Matthias Seitz
9f5cf847cc refactor: simplify BalCache to use HashMap + BTreeMap
Replace LRU-based cache with simpler design:
- Use HashMap<BlockHash, Bytes> for O(1) hash lookups
- Use BTreeMap<BlockNumber, BlockHash> as source of truth for eviction
- Evict oldest (lowest) block numbers when at capacity
- Handle reorgs by removing old hash when block number is replaced

This is simpler, more predictable, and removes schnellru dependency.
2026-01-21 12:24:32 +01:00
Sergei Shulepov
43a84f1231 refactor(engine): move execution logic from metrics to payload_validator (#21226) 2026-01-21 11:17:30 +00:00
Matthias Seitz
df1413167a perf: clone only BAL bytes instead of entire payload
Extract num_hash and BAL before calling new_payload to avoid
cloning the entire ExecutionData payload.
2026-01-21 12:00:38 +01:00
Matthias Seitz
3f50a36191 fix: stop get_by_range at first missing block
Ensures caller knows returned BALs correspond to contiguous blocks
[start, start + len)
2026-01-21 11:49:29 +01:00
Matthias Seitz
d750b4976d fix: address clippy warnings
- Collapse nested if statements using let-chains
- Add backticks around BTreeMap in doc comment
2026-01-21 11:46:37 +01:00
Matthias Seitz
7a65d2595d feat(engine-api): add BAL cache for EIP-7928
Introduces an in-memory LRU cache for Block Access Lists (BALs) in the
Engine API. BALs are cached when payloads are validated as VALID via
newPayload.

- Add BalCache with internal Arc for cheap cloning
- Store BALs keyed by block hash with block number index for range queries
- Implement engine_getBALsByHashV1 and engine_getBALsByRangeV1
- Add metrics for cache inserts/hits/misses

Per EIP-7928, the EL should retain BALs for the weak subjectivity period
(~3533 epochs). This initial implementation uses a configurable LRU cache
(default 1024 entries) as a starting point.
2026-01-21 11:35:11 +01:00
Matthias Seitz
5a5c21cc1b feat(txpool): add IntoIterator for AllPoolTransactions (#21241) 2026-01-21 10:01:32 +00:00
Matthias Seitz
8a8a9126d6 feat(execution-types): add receipts_iter and logs_iter helpers to Chain (#21240) 2026-01-21 09:59:15 +00:00
Emilia Hane
6f73c2447d feat(trie): Add serde-bincode-compat feature to reth-trie (#21235) 2026-01-21 09:42:52 +00:00
Sergei Shulepov
2cae438642 fix: sigsegv handler (#21231)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2026-01-21 09:42:36 +00:00
Georgios Konstantopoulos
37b5db0d47 feat(cli): add RocksDB table stats to reth db stats command (#21221)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-21 08:45:17 +00:00
joshieDo
238433e146 fix(rocksdb): flush memtables before dropping (#21234) 2026-01-21 02:19:36 +00:00
Georgios Konstantopoulos
660964a0f5 feat(node): log storage settings after genesis init (#21229) 2026-01-21 00:58:23 +00:00
Matthias Seitz
22b465dd64 chore(trie): remove unnecessary clone in into_sorted_ref (#21232) 2026-01-20 22:57:08 +00:00
Georgios Konstantopoulos
3ff575b877 feat(engine): add --engine.disable-cache-metrics flag (#21228)
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
2026-01-20 22:03:12 +00:00
かりんとう
d12752dc8a feat(engine): add time_between_forkchoice_updated metric (#21227) 2026-01-20 21:06:11 +00:00
Georgios Konstantopoulos
869b5d0851 feat(edge): enable transaction_hash_numbers_in_rocksdb for edge builds (#21224)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 20:02:02 +00:00
Georgios Konstantopoulos
78de3d8f61 perf(db): use Cow::Borrowed in walk_dup to avoid allocation (#21220) 2026-01-20 19:31:50 +00:00
YK
bc79cc44c9 feat(cli): add --rocksdb.* flags for RocksDB table routing (#21191) 2026-01-20 19:29:05 +00:00
Georgios Konstantopoulos
ff8f434dcd feat(cli): add reth db checksum rocksdb command (#21217)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 19:10:34 +00:00
Arsenii Kulikov
9662dc5271 fix: properly save history indices in pipeline (#21222) 2026-01-20 18:20:28 +00:00
Alexey Shekhirin
3ba37082dc fix(reth-bench): replay-payloads prefix (#21219) 2026-01-20 18:36:35 +01:00
Ahsen Kamal
7934294988 perf(trie): dispatch storage proofs in lexicographical order (#21213)
Signed-off-by: Ahsen Kamal <itsahsenkamal@gmail.com>
2026-01-20 17:09:20 +00:00
Georgios Konstantopoulos
7371bd3f29 chore(db-api): remove sharded_key_encode benchmark (#21215)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 17:01:12 +00:00
Georgios Konstantopoulos
80980b8e4d feat(pruning): add DefaultPruningValues for overridable pruning defaults (#21207)
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-01-20 16:58:29 +00:00
Matthias Seitz
2e2cd67663 perf(chain-state): parallelize into_sorted with rayon (#21193) 2026-01-20 16:42:16 +00:00
Georgios Konstantopoulos
4f009728e2 feat(cli): add reth db checksum mdbx/static-file command (#21211)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 16:11:51 +00:00
Georgios Konstantopoulos
39d5ae73e8 feat(storage): add read-only mode for RocksDB provider (#21210)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 16:09:51 +00:00
Georgios Konstantopoulos
5ef200eaad perf(db): stack-allocate ShardedKey and StorageShardedKey encoding (#21200)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 15:58:43 +00:00
ethfanWilliam
d002dacc13 chore: remove deprecated and unused ExecuteOutput struct (#20887)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-20 15:06:26 +00:00
Alexey Shekhirin
bb39cba504 ci: partition bench codspeed job (#20332)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-20 14:29:48 +00:00
YK
bd144a4c42 feat(stages): add RocksDB support for IndexAccountHistoryStage (#21165)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 14:23:29 +00:00
tonis
a0845bab18 feat: Check CL/Reth capability compatibility (#20348)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 14:19:31 +00:00
Brian Picciano
346cc0da71 feat(trie): add AsyncAccountValueEncoder for V2 proof computation (#21197)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 13:50:29 +00:00
Matthias Seitz
ea3d4663ae perf(trie): use HashMap reserve heuristic in MultiProof::extend (#21199) 2026-01-20 13:34:41 +00:00
Hwangjae Lee
3667d3b5aa perf(trie): defer child RLP conversion in proof_v2 for async encoder support (#20873)
Signed-off-by: Hwangjae Lee <meetrick@gmail.com>
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-01-20 13:33:08 +00:00
Brian Picciano
7cfb19c98e feat(trie): Add V2 reveal method and target types (#21196)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 13:25:54 +00:00
joshieDo
5a38871489 fix: set StaticFileArgs defaults for edge (#21208) 2026-01-20 12:39:36 +00:00
Brian Picciano
c825c8c187 chore(trie): Move hybrid check for trie input merges into common code (#21198) 2026-01-20 12:38:46 +00:00
Matthias Seitz
8f37cd08fc feat(engine-api): add EIP-7928 BAL stub methods (#21204) 2026-01-20 11:33:27 +00:00
240 changed files with 14083 additions and 4090 deletions

43
.github/CODEOWNERS vendored
View File

@@ -1,45 +1,52 @@
* @gakonst
crates/blockchain-tree-api/ @rakita @mattsse @Rjected
crates/blockchain-tree/ @rakita @mattsse @Rjected
crates/chain-state/ @fgimenez @mattsse
crates/chainspec/ @Rjected @joshieDo @mattsse
crates/cli/ @mattsse
crates/config/ @shekhirin @mattsse @Rjected
crates/consensus/ @mattsse @Rjected
crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez
crates/engine/ @mattsse @Rjected @fgimenez @mediocregopher @yongkangc
crates/era/ @mattsse @RomanHodulak
crates/engine/ @mattsse @Rjected @mediocregopher @yongkangc
crates/era/ @mattsse
crates/era-downloader/ @mattsse
crates/era-utils/ @mattsse
crates/errors/ @mattsse
crates/ethereum-forks/ @mattsse @Rjected
crates/ethereum/ @mattsse @Rjected
crates/etl/ @joshieDo @shekhirin
crates/evm/ @rakita @mattsse @Rjected
crates/evm/ @mattsse @Rjected @klkvr
crates/exex/ @shekhirin
crates/fs-util/ @mattsse
crates/metrics/ @mattsse @Rjected
crates/net/ @mattsse @Rjected
crates/net/downloaders/ @Rjected
crates/node/ @mattsse @Rjected @klkvr
crates/optimism/ @mattsse @Rjected @fgimenez
crates/optimism/ @mattsse @Rjected
crates/payload/ @mattsse @Rjected
crates/primitives-traits/ @Rjected @RomanHodulak @mattsse @klkvr
crates/primitives-traits/ @Rjected @mattsse @klkvr
crates/primitives/ @Rjected @mattsse @klkvr
crates/prune/ @shekhirin @joshieDo
crates/ress @shekhirin @Rjected
crates/revm/ @mattsse @rakita
crates/rpc/ @mattsse @Rjected @RomanHodulak
crates/ress/ @shekhirin @Rjected
crates/revm/ @mattsse
crates/rpc/ @mattsse @Rjected
crates/stages/ @shekhirin @mediocregopher
crates/static-file/ @joshieDo @shekhirin
crates/stateless/ @mattsse
crates/storage/codecs/ @joshieDo
crates/storage/db-api/ @joshieDo @rakita
crates/storage/db-api/ @joshieDo
crates/storage/db-common/ @Rjected
crates/storage/db/ @joshieDo @rakita
crates/storage/errors/ @rakita
crates/storage/libmdbx-rs/ @rakita @shekhirin
crates/storage/db/ @joshieDo
crates/storage/errors/ @joshieDo
crates/storage/libmdbx-rs/ @shekhirin
crates/storage/nippy-jar/ @joshieDo @shekhirin
crates/storage/provider/ @rakita @joshieDo @shekhirin
crates/storage/provider/ @joshieDo @shekhirin @yongkangc
crates/storage/storage-api/ @joshieDo
crates/tasks/ @mattsse
crates/tokio-util/ @fgimenez
crates/tokio-util/ @mattsse
crates/tracing/ @mattsse @shekhirin
crates/tracing-otlp/ @mattsse @Rjected
crates/transaction-pool/ @mattsse @yongkangc
crates/trie/ @Rjected @shekhirin @mediocregopher
crates/trie/ @Rjected @shekhirin @mediocregopher @yongkangc
bin/reth/ @mattsse @shekhirin @Rjected
bin/reth-bench/ @mattsse @Rjected @shekhirin @yongkangc
bin/reth-bench-compare/ @mediocregopher @shekhirin @yongkangc
etc/ @Rjected @shekhirin
.github/ @gakonst @DaniPopes

View File

@@ -1,14 +0,0 @@
#!/usr/bin/env bash
set -eo pipefail
# TODO: Benchmarks run WAY too slow due to excessive amount of iterations.
cmd=(cargo codspeed build --profile profiling)
crates=(
-p reth-primitives
-p reth-trie
-p reth-trie-common
-p reth-trie-sparse
)
"${cmd[@]}" --features test-utils "${crates[@]}"

View File

@@ -17,6 +17,16 @@ name: bench
jobs:
codspeed:
runs-on: depot-ubuntu-latest
strategy:
matrix:
partition: [1, 2]
total_partitions: [2]
include:
- partition: 1
crates: "-p reth-primitives -p reth-trie-common -p reth-trie-sparse"
- partition: 2
crates: "-p reth-trie"
name: codspeed (${{ matrix.partition }}/${{ matrix.total_partitions }})
steps:
- uses: actions/checkout@v6
with:
@@ -32,10 +42,10 @@ jobs:
with:
tool: cargo-codspeed
- name: Build the benchmark target(s)
run: ./.github/scripts/codspeed-build.sh
run: cargo codspeed build --profile profiling --features test-utils ${{ matrix.crates }}
- name: Run the benchmarks
uses: CodSpeedHQ/action@v4
with:
run: cargo codspeed run --workspace
run: cargo codspeed run ${{ matrix.crates }}
mode: instrumentation
token: ${{ secrets.CODSPEED_TOKEN }}

65
.github/workflows/check-alloy.yml vendored Normal file
View File

@@ -0,0 +1,65 @@
# Checks reth compilation against alloy branches to detect breaking changes.
# Run on-demand via workflow_dispatch.
name: Check Alloy Breaking Changes
on:
workflow_dispatch:
inputs:
alloy_branch:
description: 'Branch/rev for alloy-rs/alloy (leave empty to skip)'
required: false
type: string
alloy_evm_branch:
description: 'Branch/rev for alloy-rs/evm (alloy-evm, alloy-op-evm) (leave empty to skip)'
required: false
type: string
op_alloy_branch:
description: 'Branch/rev for alloy-rs/op-alloy (leave empty to skip)'
required: false
type: string
env:
CARGO_TERM_COLOR: always
jobs:
check:
name: Check compilation with patched alloy
runs-on: depot-ubuntu-latest-16
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Apply alloy patches
run: |
ARGS=""
if [ -n "${{ inputs.alloy_branch }}" ]; then
ARGS="$ARGS --alloy ${{ inputs.alloy_branch }}"
fi
if [ -n "${{ inputs.alloy_evm_branch }}" ]; then
ARGS="$ARGS --evm ${{ inputs.alloy_evm_branch }}"
fi
if [ -n "${{ inputs.op_alloy_branch }}" ]; then
ARGS="$ARGS --op ${{ inputs.op_alloy_branch }}"
fi
if [ -z "$ARGS" ]; then
echo "No branches specified, nothing to patch"
exit 1
fi
./scripts/patch-alloy.sh $ARGS
echo "=== Final patch section ==="
tail -50 Cargo.toml
- name: Check workspace
run: cargo clippy --workspace --lib --examples --tests --benches --all-features --locked
env:
RUSTFLAGS: -D warnings

143
Cargo.lock generated
View File

@@ -106,9 +106,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
[[package]]
name = "alloy-chains"
version = "0.2.27"
version = "0.2.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25db5bcdd086f0b1b9610140a12c59b757397be90bd130d8d836fc8da0815a34"
checksum = "3842d8c52fcd3378039f4703dba392dca8b546b1c8ed6183048f8dab95b2be78"
dependencies = [
"alloy-primitives",
"alloy-rlp",
@@ -121,9 +121,9 @@ dependencies = [
[[package]]
name = "alloy-consensus"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c3a590d13de3944675987394715f37537b50b856e3b23a0e66e97d963edbf38"
checksum = "ed1958f0294ecc05ebe7b3c9a8662a3e221c2523b7f2bcd94c7a651efbd510bf"
dependencies = [
"alloy-eips",
"alloy-primitives",
@@ -149,9 +149,9 @@ dependencies = [
[[package]]
name = "alloy-consensus-any"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f28f769d5ea999f0d8a105e434f483456a15b4e1fcb08edbbbe1650a497ff6d"
checksum = "f752e99497ddc39e22d547d7dfe516af10c979405a034ed90e69b914b7dddeae"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -164,9 +164,9 @@ dependencies = [
[[package]]
name = "alloy-contract"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "990fa65cd132a99d3c3795a82b9f93ec82b81c7de3bab0bf26ca5c73286f7186"
checksum = "f2140796bc79150b1b7375daeab99750f0ff5e27b1f8b0aa81ccde229c7f02a2"
dependencies = [
"alloy-consensus",
"alloy-dyn-abi",
@@ -255,19 +255,21 @@ checksum = "6adac476434bf024279164dcdca299309f0c7d1e3557024eb7a83f8d9d01c6b5"
dependencies = [
"alloy-primitives",
"alloy-rlp",
"arbitrary",
"borsh",
"serde",
]
[[package]]
name = "alloy-eips"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09535cbc646b0e0c6fcc12b7597eaed12cf86dff4c4fba9507a61e71b94f30eb"
checksum = "813a67f87e56b38554d18b182616ee5006e8e2bf9df96a0df8bf29dff1d52e3f"
dependencies = [
"alloy-eip2124",
"alloy-eip2930",
"alloy-eip7702",
"alloy-eip7928",
"alloy-primitives",
"alloy-rlp",
"alloy-serde",
@@ -287,9 +289,9 @@ dependencies = [
[[package]]
name = "alloy-evm"
version = "0.26.3"
version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a96827207397445a919a8adc49289b53cc74e48e460411740bba31cec2fc307d"
checksum = "1582933a9fc27c0953220eb4f18f6492ff577822e9a8d848890ff59f6b4f5beb"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -309,9 +311,9 @@ dependencies = [
[[package]]
name = "alloy-genesis"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1005520ccf89fa3d755e46c1d992a9e795466c2e7921be2145ef1f749c5727de"
checksum = "05864eef929c4d28895ae4b4d8ac9c6753c4df66e873b9c8fafc8089b59c1502"
dependencies = [
"alloy-eips",
"alloy-primitives",
@@ -350,9 +352,9 @@ dependencies = [
[[package]]
name = "alloy-json-rpc"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b626409c98ba43aaaa558361bca21440c88fd30df7542c7484b9c7a1489cdb"
checksum = "d2dd146b3de349a6ffaa4e4e319ab3a90371fb159fb0bddeb1c7bbe8b1792eff"
dependencies = [
"alloy-primitives",
"alloy-sol-types",
@@ -365,9 +367,9 @@ dependencies = [
[[package]]
name = "alloy-network"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89924fdcfeee0e0fa42b1f10af42f92802b5d16be614a70897382565663bf7cf"
checksum = "8c12278ffbb8872dfba3b2f17d8ea5e8503c2df5155d9bc5ee342794bde505c3"
dependencies = [
"alloy-consensus",
"alloy-consensus-any",
@@ -391,9 +393,9 @@ dependencies = [
[[package]]
name = "alloy-network-primitives"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f0dbe56ff50065713ff8635d8712a0895db3ad7f209db9793ad8fcb6b1734aa"
checksum = "833037c04917bc2031541a60e8249e4ab5500e24c637c1c62e95e963a655d66f"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -404,9 +406,9 @@ dependencies = [
[[package]]
name = "alloy-op-evm"
version = "0.26.3"
version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54dc5c46a92fc7267055a174d30efb34e2599a0047102a4d38a025ae521435ba"
checksum = "6f19214adae08ea95600c3ede76bcbf0c40b36a263534a8f441a4c732f60e868"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -467,9 +469,9 @@ dependencies = [
[[package]]
name = "alloy-provider"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b56f7a77513308a21a2ba0e9d57785a9d9d2d609e77f4e71a78a1192b83ff2d"
checksum = "eafa840b0afe01c889a3012bb2fde770a544f74eab2e2870303eb0a5fb869c48"
dependencies = [
"alloy-chains",
"alloy-consensus",
@@ -512,9 +514,9 @@ dependencies = [
[[package]]
name = "alloy-pubsub"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94813abbd7baa30c700ea02e7f92319dbcb03bff77aeea92a3a9af7ba19c5c70"
checksum = "57b3a3b3e4efc9f4d30e3326b6bd6811231d16ef94837e18a802b44ca55119e6"
dependencies = [
"alloy-json-rpc",
"alloy-primitives",
@@ -556,9 +558,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-client"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff01723afc25ec4c5b04de399155bef7b6a96dfde2475492b1b7b4e7a4f46445"
checksum = "12768ae6303ec764905a8a7cd472aea9072f9f9c980d18151e26913da8ae0123"
dependencies = [
"alloy-json-rpc",
"alloy-primitives",
@@ -582,9 +584,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f91bf006bb06b7d812591b6ac33395cb92f46c6a65cda11ee30b348338214f0f"
checksum = "0622d8bcac2f16727590aa33f4c3f05ea98130e7e4b4924bce8be85da5ad0dae"
dependencies = [
"alloy-primitives",
"alloy-rpc-types-engine",
@@ -595,9 +597,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-admin"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b934c3bcdc6617563b45deb36a40881c8230b94d0546ea739dff7edb3aa2f6fd"
checksum = "c38c5ac70457ecc74e87fe1a5a19f936419224ded0eb0636241452412ca92733"
dependencies = [
"alloy-genesis",
"alloy-primitives",
@@ -607,9 +609,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-anvil"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e82145856df8abb1fefabef58cdec0f7d9abf337d4abd50c1ed7e581634acdd"
checksum = "ae8eb0e5d6c48941b61ab76fabab4af66f7d88309a98aa14ad3dec7911c1eba3"
dependencies = [
"alloy-primitives",
"alloy-rpc-types-eth",
@@ -619,9 +621,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-any"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "212ca1c1dab27f531d3858f8b1a2d6bfb2da664be0c1083971078eb7b71abe4b"
checksum = "a1cf5a093e437dfd62df48e480f24e1a3807632358aad6816d7a52875f1c04aa"
dependencies = [
"alloy-consensus-any",
"alloy-rpc-types-eth",
@@ -630,9 +632,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-beacon"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d92a9b4b268fac505ef7fb1dac9bb129d4fd7de7753f22a5b6e9f666f7f7de6"
checksum = "e07949e912479ef3b848e1cf8db54b534bdd7bc58e6c23f28ea9488960990c8c"
dependencies = [
"alloy-eips",
"alloy-primitives",
@@ -650,9 +652,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-debug"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bab1ebed118b701c497e6541d2d11dfa6f3c6ae31a3c52999daa802fcdcc16b7"
checksum = "925ff0f48c2169c050f0ae7a82769bdf3f45723d6742ebb6a5efb4ed2f491b26"
dependencies = [
"alloy-primitives",
"derive_more",
@@ -662,9 +664,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-engine"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "232f00fcbcd3ee3b9399b96223a8fc884d17742a70a44f9d7cef275f93e6e872"
checksum = "336ef381c7409f23c69f6e79bddc1917b6e832cff23e7a5cf84b9381d53582e6"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -683,9 +685,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-eth"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5715d0bf7efbd360873518bd9f6595762136b5327a9b759a8c42ccd9b5e44945"
checksum = "28e97603095020543a019ab133e0e3dc38cd0819f19f19bdd70c642404a54751"
dependencies = [
"alloy-consensus",
"alloy-consensus-any",
@@ -705,9 +707,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-mev"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7b61941d2add2ee64646612d3eda92cbbde8e6c933489760b6222c8898c79be"
checksum = "2805153975e25d38e37ee100880e642d5b24e421ed3014a7d2dae1d9be77562e"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -720,9 +722,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-trace"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9763cc931a28682bd4b9a68af90057b0fbe80e2538a82251afd69d7ae00bbebf"
checksum = "f1aec4e1c66505d067933ea1a949a4fb60a19c4cfc2f109aa65873ea99e62ea8"
dependencies = [
"alloy-primitives",
"alloy-rpc-types-eth",
@@ -734,9 +736,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-txpool"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "359a8caaa98cb49eed62d03f5bc511dd6dd5dee292238e8627a6e5690156df0f"
checksum = "25b73c1d6e4f1737a20d246dad5a0abd6c1b76ec4c3d153684ef8c6f1b6bb4f4"
dependencies = [
"alloy-primitives",
"alloy-rpc-types-eth",
@@ -746,9 +748,9 @@ dependencies = [
[[package]]
name = "alloy-serde"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ed8531cae8d21ee1c6571d0995f8c9f0652a6ef6452fde369283edea6ab7138"
checksum = "946a0d413dbb5cd9adba0de5f8a1a34d5b77deda9b69c1d7feed8fc875a1aa26"
dependencies = [
"alloy-primitives",
"arbitrary",
@@ -758,9 +760,9 @@ dependencies = [
[[package]]
name = "alloy-signer"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb10ccd49d0248df51063fce6b716f68a315dd912d55b32178c883fd48b4021d"
checksum = "2f7481dc8316768f042495eaf305d450c32defbc9bce09d8bf28afcd956895bb"
dependencies = [
"alloy-primitives",
"async-trait",
@@ -773,9 +775,9 @@ dependencies = [
[[package]]
name = "alloy-signer-local"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4d992d44e6c414ece580294abbadb50e74cfd4eaa69787350a4dfd4b20eaa1b"
checksum = "1259dac1f534a4c66c1d65237c89915d0010a2a91d6c3b0bada24dc5ee0fb917"
dependencies = [
"alloy-consensus",
"alloy-network",
@@ -862,9 +864,9 @@ dependencies = [
[[package]]
name = "alloy-transport"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f50a9516736d22dd834cc2240e5bf264f338667cc1d9e514b55ec5a78b987ca"
checksum = "78f169b85eb9334871db986e7eaf59c58a03d86a30cc68b846573d47ed0656bb"
dependencies = [
"alloy-json-rpc",
"auto_impl",
@@ -885,9 +887,9 @@ dependencies = [
[[package]]
name = "alloy-transport-http"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a18b541a6197cf9a084481498a766fdf32fefda0c35ea6096df7d511025e9f1"
checksum = "019821102e70603e2c141954418255bec539ef64ac4117f8e84fb493769acf73"
dependencies = [
"alloy-json-rpc",
"alloy-transport",
@@ -900,9 +902,9 @@ dependencies = [
[[package]]
name = "alloy-transport-ipc"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8075911680ebc537578cacf9453464fd394822a0f68614884a9c63f9fbaf5e89"
checksum = "e574ca2f490fb5961d2cdd78188897392c46615cd88b35c202d34bbc31571a81"
dependencies = [
"alloy-json-rpc",
"alloy-pubsub",
@@ -920,9 +922,9 @@ dependencies = [
[[package]]
name = "alloy-transport-ws"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "921d37a57e2975e5215f7dd0f28873ed5407c7af630d4831a4b5c737de4b0b8b"
checksum = "b92dea6996269769f74ae56475570e3586910661e037b7b52d50c9641f76c68f"
dependencies = [
"alloy-pubsub",
"alloy-transport",
@@ -957,9 +959,9 @@ dependencies = [
[[package]]
name = "alloy-tx-macros"
version = "1.4.3"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2289a842d02fe63f8c466db964168bb2c7a9fdfb7b24816dbb17d45520575fb"
checksum = "45ceac797eb8a56bdf5ab1fab353072c17d472eab87645ca847afe720db3246d"
dependencies = [
"darling 0.21.3",
"proc-macro2",
@@ -7788,6 +7790,7 @@ dependencies = [
"parking_lot",
"pin-project",
"rand 0.9.2",
"rayon",
"reth-chainspec",
"reth-errors",
"reth-ethereum-primitives",
@@ -7906,6 +7909,7 @@ dependencies = [
"reth-stages-types",
"reth-static-file",
"reth-static-file-types",
"reth-storage-api",
"reth-tasks",
"reth-trie",
"reth-trie-common",
@@ -8098,6 +8102,7 @@ dependencies = [
"alloy-genesis",
"alloy-primitives",
"arbitrary",
"arrayvec",
"bytes",
"derive_more",
"metrics",
@@ -9441,7 +9446,6 @@ dependencies = [
"reth-network-p2p",
"reth-network-peers",
"reth-primitives-traits",
"reth-provider",
"reth-prune-types",
"reth-rpc-convert",
"reth-rpc-eth-types",
@@ -10624,6 +10628,7 @@ dependencies = [
"jsonrpsee-core",
"jsonrpsee-types",
"metrics",
"parking_lot",
"reth-chainspec",
"reth-engine-primitives",
"reth-ethereum-engine-primitives",
@@ -14605,9 +14610,9 @@ dependencies = [
[[package]]
name = "zmij"
version = "1.0.15"
version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94f63c051f4fe3c1509da62131a678643c5b6fbdc9273b2b79d4378ebda003d2"
checksum = "dfcd145825aace48cff44a8844de64bf75feec3080e0aa5cdbde72961ae51a65"
[[package]]
name = "zstd"

View File

@@ -485,10 +485,10 @@ revm-inspectors = "0.34.0"
# eth
alloy-chains = { version = "0.2.5", default-features = false }
alloy-dyn-abi = "1.4.3"
alloy-dyn-abi = "1.5.2"
alloy-eip2124 = { version = "0.2.0", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-evm = { version = "0.26.3", default-features = false }
alloy-evm = { version = "0.27.0", default-features = false }
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
alloy-sol-macro = "1.5.0"
@@ -497,36 +497,36 @@ alloy-trie = { version = "0.9.1", default-features = false }
alloy-hardforks = "0.4.5"
alloy-consensus = { version = "1.4.3", default-features = false }
alloy-contract = { version = "1.4.3", default-features = false }
alloy-eips = { version = "1.4.3", default-features = false }
alloy-genesis = { version = "1.4.3", default-features = false }
alloy-json-rpc = { version = "1.4.3", default-features = false }
alloy-network = { version = "1.4.3", default-features = false }
alloy-network-primitives = { version = "1.4.3", default-features = false }
alloy-provider = { version = "1.4.3", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.4.3", default-features = false }
alloy-rpc-client = { version = "1.4.3", default-features = false }
alloy-rpc-types = { version = "1.4.3", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.4.3", default-features = false }
alloy-rpc-types-anvil = { version = "1.4.3", default-features = false }
alloy-rpc-types-beacon = { version = "1.4.3", default-features = false }
alloy-rpc-types-debug = { version = "1.4.3", default-features = false }
alloy-rpc-types-engine = { version = "1.4.3", default-features = false }
alloy-rpc-types-eth = { version = "1.4.3", default-features = false }
alloy-rpc-types-mev = { version = "1.4.3", default-features = false }
alloy-rpc-types-trace = { version = "1.4.3", default-features = false }
alloy-rpc-types-txpool = { version = "1.4.3", default-features = false }
alloy-serde = { version = "1.4.3", default-features = false }
alloy-signer = { version = "1.4.3", default-features = false }
alloy-signer-local = { version = "1.4.3", default-features = false }
alloy-transport = { version = "1.4.3" }
alloy-transport-http = { version = "1.4.3", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.4.3", default-features = false }
alloy-transport-ws = { version = "1.4.3", default-features = false }
alloy-consensus = { version = "1.5.2", default-features = false }
alloy-contract = { version = "1.5.2", default-features = false }
alloy-eips = { version = "1.5.2", default-features = false }
alloy-genesis = { version = "1.5.2", default-features = false }
alloy-json-rpc = { version = "1.5.2", default-features = false }
alloy-network = { version = "1.5.2", default-features = false }
alloy-network-primitives = { version = "1.5.2", default-features = false }
alloy-provider = { version = "1.5.2", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.5.2", default-features = false }
alloy-rpc-client = { version = "1.5.2", default-features = false }
alloy-rpc-types = { version = "1.5.2", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.5.2", default-features = false }
alloy-rpc-types-anvil = { version = "1.5.2", default-features = false }
alloy-rpc-types-beacon = { version = "1.5.2", default-features = false }
alloy-rpc-types-debug = { version = "1.5.2", default-features = false }
alloy-rpc-types-engine = { version = "1.5.2", default-features = false }
alloy-rpc-types-eth = { version = "1.5.2", default-features = false }
alloy-rpc-types-mev = { version = "1.5.2", default-features = false }
alloy-rpc-types-trace = { version = "1.5.2", default-features = false }
alloy-rpc-types-txpool = { version = "1.5.2", default-features = false }
alloy-serde = { version = "1.5.2", default-features = false }
alloy-signer = { version = "1.5.2", default-features = false }
alloy-signer-local = { version = "1.5.2", default-features = false }
alloy-transport = { version = "1.5.2" }
alloy-transport-http = { version = "1.5.2", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.5.2", default-features = false }
alloy-transport-ws = { version = "1.5.2", default-features = false }
# op
alloy-op-evm = { version = "0.26.3", default-features = false }
alloy-op-evm = { version = "0.27.0", default-features = false }
alloy-op-hardforks = "0.4.4"
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
@@ -790,8 +790,8 @@ ipnet = "2.11"
# jsonrpsee-http-client = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# jsonrpsee-types = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "3020ea8" }

View File

@@ -32,7 +32,7 @@ alloy-eips.workspace = true
alloy-json-rpc.workspace = true
alloy-consensus.workspace = true
alloy-network.workspace = true
alloy-primitives.workspace = true
alloy-primitives = { workspace = true, features = ["rand"] }
alloy-provider = { workspace = true, features = ["engine-api", "pubsub", "reqwest-rustls-tls"], default-features = false }
alloy-pubsub.workspace = true
alloy-rpc-client = { workspace = true, features = ["pubsub"] }

View File

@@ -3,7 +3,7 @@
use crate::{
authenticated_transport::AuthenticatedTransportConnect,
bench::{
helpers::{build_payload, prepare_payload_request, rpc_block_to_header},
helpers::{build_payload, parse_gas_limit, prepare_payload_request, rpc_block_to_header},
output::GasRampPayloadFile,
},
valid_payload::{call_forkchoice_updated, call_new_payload, payload_to_new_payload},
@@ -25,9 +25,16 @@ use tracing::info;
/// `reth benchmark gas-limit-ramp` command.
#[derive(Debug, Parser)]
pub struct Command {
/// Number of blocks to generate.
#[arg(long, value_name = "BLOCKS")]
blocks: u64,
/// Number of blocks to generate. Mutually exclusive with --target-gas-limit.
#[arg(long, value_name = "BLOCKS", conflicts_with = "target_gas_limit")]
blocks: Option<u64>,
/// Target gas limit to ramp up to. The benchmark will generate blocks until the gas limit
/// reaches or exceeds this value. Mutually exclusive with --blocks.
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 2G = 2
/// billion).
#[arg(long, value_name = "TARGET_GAS_LIMIT", conflicts_with = "blocks", value_parser = parse_gas_limit)]
target_gas_limit: Option<u64>,
/// The Engine API RPC URL.
#[arg(long = "engine-rpc-url", value_name = "ENGINE_RPC_URL")]
@@ -42,12 +49,37 @@ pub struct Command {
output: PathBuf,
}
/// Mode for determining when to stop ramping.
#[derive(Debug, Clone, Copy)]
enum RampMode {
/// Ramp for a fixed number of blocks.
Blocks(u64),
/// Ramp until reaching or exceeding target gas limit.
TargetGasLimit(u64),
}
impl Command {
/// Execute `benchmark gas-limit-ramp` command.
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
if self.blocks == 0 {
return Err(eyre::eyre!("--blocks must be greater than 0"));
}
let mode = match (self.blocks, self.target_gas_limit) {
(Some(blocks), None) => {
if blocks == 0 {
return Err(eyre::eyre!("--blocks must be greater than 0"));
}
RampMode::Blocks(blocks)
}
(None, Some(target)) => {
if target == 0 {
return Err(eyre::eyre!("--target-gas-limit must be greater than 0"));
}
RampMode::TargetGasLimit(target)
}
_ => {
return Err(eyre::eyre!(
"Exactly one of --blocks or --target-gas-limit must be specified"
));
}
};
// Ensure output directory exists
if self.output.is_file() {
@@ -84,14 +116,31 @@ impl Command {
let canonical_parent = parent_header.number;
let start_block = canonical_parent + 1;
let end_block = start_block + self.blocks - 1;
info!(canonical_parent, start_block, end_block, "Starting gas limit ramp benchmark");
match mode {
RampMode::Blocks(blocks) => {
info!(
canonical_parent,
start_block,
end_block = start_block + blocks - 1,
"Starting gas limit ramp benchmark (block count mode)"
);
}
RampMode::TargetGasLimit(target) => {
info!(
canonical_parent,
start_block,
current_gas_limit = parent_header.gas_limit,
target_gas_limit = target,
"Starting gas limit ramp benchmark (target gas limit mode)"
);
}
}
let mut next_block_number = start_block;
let mut blocks_processed = 0u64;
let total_benchmark_duration = Instant::now();
while next_block_number <= end_block {
while !should_stop(mode, blocks_processed, parent_header.gas_limit) {
let timestamp = parent_header.timestamp.saturating_add(1);
let request = prepare_payload_request(&chain_spec, timestamp, parent_hash);
@@ -140,13 +189,13 @@ impl Command {
parent_header = block.header;
parent_hash = block_hash;
next_block_number += 1;
blocks_processed += 1;
}
let final_gas_limit = parent_header.gas_limit;
info!(
total_duration=?total_benchmark_duration.elapsed(),
blocks_processed = self.blocks,
blocks_processed,
final_gas_limit,
"Benchmark complete"
);
@@ -158,3 +207,10 @@ impl Command {
const fn max_gas_limit_increase(parent_gas_limit: u64) -> u64 {
(parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR).saturating_sub(1)
}
const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u64) -> bool {
match mode {
RampMode::Blocks(target_blocks) => blocks_processed >= target_blocks,
RampMode::TargetGasLimit(target) => current_gas_limit >= target,
}
}

View File

@@ -3,7 +3,9 @@
//! This command fetches transactions from existing blocks and packs them into a single
//! large block using the `testing_buildBlockV1` RPC endpoint.
use crate::authenticated_transport::AuthenticatedTransportConnect;
use crate::{
authenticated_transport::AuthenticatedTransportConnect, bench::helpers::parse_gas_limit,
};
use alloy_eips::{BlockNumberOrTag, Typed2718};
use alloy_primitives::{Bytes, B256};
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
@@ -202,7 +204,9 @@ pub struct Command {
jwt_secret: std::path::PathBuf,
/// Target gas to pack into the block.
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000")]
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 1G = 1
/// billion).
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000", value_parser = parse_gas_limit)]
target_gas: u64,
/// Starting block number to fetch transactions from.

View File

@@ -1,6 +1,29 @@
//! Common helpers for reth-bench commands.
use crate::valid_payload::call_forkchoice_updated;
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
///
/// Examples: "30000000", "30M", "1G", "2G"
pub(crate) fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
let s = s.trim();
if s.is_empty() {
return Err(eyre::eyre!("empty value"));
}
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
(prefix, 1_000_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
(prefix, 1_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
(prefix, 1_000u64)
} else {
(s, 1u64)
};
let base: u64 = num_str.trim().parse()?;
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
}
use alloy_consensus::Header;
use alloy_eips::eip4844::kzg_to_versioned_hash;
use alloy_primitives::{Address, B256};
@@ -194,3 +217,50 @@ pub(crate) async fn get_payload_with_sidecar(
_ => panic!("This tool does not support getPayload versions past v5"),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_gas_limit_plain_number() {
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("1").unwrap(), 1);
assert_eq!(parse_gas_limit("0").unwrap(), 0);
}
#[test]
fn test_parse_gas_limit_k_suffix() {
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
}
#[test]
fn test_parse_gas_limit_m_suffix() {
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
}
#[test]
fn test_parse_gas_limit_g_suffix() {
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
}
#[test]
fn test_parse_gas_limit_with_whitespace() {
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
}
#[test]
fn test_parse_gas_limit_errors() {
assert!(parse_gas_limit("").is_err());
assert!(parse_gas_limit("abc").is_err());
assert!(parse_gas_limit("G").is_err());
assert!(parse_gas_limit("-1G").is_err());
}
}

View File

@@ -16,6 +16,7 @@ mod new_payload_fcu;
mod new_payload_only;
mod output;
mod replay_payloads;
mod send_invalid_payload;
mod send_payload;
/// `reth bench` command
@@ -74,6 +75,18 @@ pub enum Subcommands {
/// `reth-bench replay-payloads --payload-dir ./payloads --engine-rpc-url
/// http://localhost:8551 --jwt-secret ~/.local/share/reth/mainnet/jwt.hex`
ReplayPayloads(replay_payloads::Command),
/// Generate and send an invalid `engine_newPayload` request for testing.
///
/// Takes a valid block and modifies fields to make it invalid, allowing you to test
/// Engine API rejection behavior. Block hash is recalculated after modifications
/// unless `--invalid-block-hash` or `--skip-hash-recalc` is used.
///
/// Example:
///
/// `cast block latest --full --json | reth-bench send-invalid-payload --rpc-url localhost:5000
/// --jwt-secret $(cat ~/.local/share/reth/mainnet/jwt.hex) --invalid-state-root`
SendInvalidPayload(Box<send_invalid_payload::Command>),
}
impl BenchmarkCommand {
@@ -89,6 +102,7 @@ impl BenchmarkCommand {
Subcommands::SendPayload(command) => command.execute(ctx).await,
Subcommands::GenerateBigBlock(command) => command.execute(ctx).await,
Subcommands::ReplayPayloads(command) => command.execute(ctx).await,
Subcommands::SendInvalidPayload(command) => (*command).execute(ctx).await,
}
}

View File

@@ -180,7 +180,7 @@ impl Command {
.filter_map(|e| e.ok())
.filter(|e| {
e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
e.file_name().to_string_lossy().starts_with("payload_")
e.file_name().to_string_lossy().starts_with("payload_block_")
})
.collect();
@@ -191,7 +191,7 @@ impl Command {
let name = e.file_name();
let name_str = name.to_string_lossy();
// Extract index from "payload_NNN.json"
let index_str = name_str.strip_prefix("payload_")?.strip_suffix(".json")?;
let index_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
let index: u64 = index_str.parse().ok()?;
Some((index, e.path()))
})

View File

@@ -0,0 +1,219 @@
use alloy_eips::eip4895::Withdrawal;
use alloy_primitives::{Address, Bloom, Bytes, B256, U256};
use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3};
/// Configuration for invalidating payload fields
#[derive(Debug, Default)]
pub(super) struct InvalidationConfig {
// Explicit value overrides (Option<T>)
pub(super) parent_hash: Option<B256>,
pub(super) fee_recipient: Option<Address>,
pub(super) state_root: Option<B256>,
pub(super) receipts_root: Option<B256>,
pub(super) logs_bloom: Option<Bloom>,
pub(super) prev_randao: Option<B256>,
pub(super) block_number: Option<u64>,
pub(super) gas_limit: Option<u64>,
pub(super) gas_used: Option<u64>,
pub(super) timestamp: Option<u64>,
pub(super) extra_data: Option<Bytes>,
pub(super) base_fee_per_gas: Option<u64>,
pub(super) block_hash: Option<B256>,
pub(super) blob_gas_used: Option<u64>,
pub(super) excess_blob_gas: Option<u64>,
// Auto-invalidation flags
pub(super) invalidate_parent_hash: bool,
pub(super) invalidate_state_root: bool,
pub(super) invalidate_receipts_root: bool,
pub(super) invalidate_gas_used: bool,
pub(super) invalidate_block_number: bool,
pub(super) invalidate_timestamp: bool,
pub(super) invalidate_base_fee: bool,
pub(super) invalidate_transactions: bool,
pub(super) invalidate_block_hash: bool,
pub(super) invalidate_withdrawals: bool,
pub(super) invalidate_blob_gas_used: bool,
pub(super) invalidate_excess_blob_gas: bool,
}
impl InvalidationConfig {
/// Returns true if `block_hash` is being explicitly set or auto-invalidated.
/// When true, the caller should skip recalculating the block hash since it will be overwritten.
pub(super) const fn should_skip_hash_recalc(&self) -> bool {
self.block_hash.is_some() || self.invalidate_block_hash
}
/// Applies invalidations to a V1 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v1(&self, payload: &mut ExecutionPayloadV1) -> Vec<String> {
let mut changes = Vec::new();
// Explicit value overrides
if let Some(parent_hash) = self.parent_hash {
payload.parent_hash = parent_hash;
changes.push(format!("parent_hash = {parent_hash}"));
}
if let Some(fee_recipient) = self.fee_recipient {
payload.fee_recipient = fee_recipient;
changes.push(format!("fee_recipient = {fee_recipient}"));
}
if let Some(state_root) = self.state_root {
payload.state_root = state_root;
changes.push(format!("state_root = {state_root}"));
}
if let Some(receipts_root) = self.receipts_root {
payload.receipts_root = receipts_root;
changes.push(format!("receipts_root = {receipts_root}"));
}
if let Some(logs_bloom) = self.logs_bloom {
payload.logs_bloom = logs_bloom;
changes.push("logs_bloom = <custom>".to_string());
}
if let Some(prev_randao) = self.prev_randao {
payload.prev_randao = prev_randao;
changes.push(format!("prev_randao = {prev_randao}"));
}
if let Some(block_number) = self.block_number {
payload.block_number = block_number;
changes.push(format!("block_number = {block_number}"));
}
if let Some(gas_limit) = self.gas_limit {
payload.gas_limit = gas_limit;
changes.push(format!("gas_limit = {gas_limit}"));
}
if let Some(gas_used) = self.gas_used {
payload.gas_used = gas_used;
changes.push(format!("gas_used = {gas_used}"));
}
if let Some(timestamp) = self.timestamp {
payload.timestamp = timestamp;
changes.push(format!("timestamp = {timestamp}"));
}
if let Some(ref extra_data) = self.extra_data {
payload.extra_data = extra_data.clone();
changes.push(format!("extra_data = {} bytes", extra_data.len()));
}
if let Some(base_fee_per_gas) = self.base_fee_per_gas {
payload.base_fee_per_gas = U256::from_limbs([base_fee_per_gas, 0, 0, 0]);
changes.push(format!("base_fee_per_gas = {base_fee_per_gas}"));
}
if let Some(block_hash) = self.block_hash {
payload.block_hash = block_hash;
changes.push(format!("block_hash = {block_hash}"));
}
// Auto-invalidation flags
if self.invalidate_parent_hash {
let random_hash = B256::random();
payload.parent_hash = random_hash;
changes.push(format!("parent_hash = {random_hash} (auto-invalidated: random)"));
}
if self.invalidate_state_root {
payload.state_root = B256::ZERO;
changes.push("state_root = ZERO (auto-invalidated: empty trie root)".to_string());
}
if self.invalidate_receipts_root {
payload.receipts_root = B256::ZERO;
changes.push("receipts_root = ZERO (auto-invalidated)".to_string());
}
if self.invalidate_gas_used {
let invalid_gas = payload.gas_limit + 1;
payload.gas_used = invalid_gas;
changes.push(format!("gas_used = {invalid_gas} (auto-invalidated: exceeds gas_limit)"));
}
if self.invalidate_block_number {
let invalid_number = payload.block_number + 999;
payload.block_number = invalid_number;
changes.push(format!("block_number = {invalid_number} (auto-invalidated: huge gap)"));
}
if self.invalidate_timestamp {
payload.timestamp = 0;
changes.push("timestamp = 0 (auto-invalidated: impossibly old)".to_string());
}
if self.invalidate_base_fee {
payload.base_fee_per_gas = U256::ZERO;
changes
.push("base_fee_per_gas = 0 (auto-invalidated: invalid post-London)".to_string());
}
if self.invalidate_transactions {
let invalid_tx = Bytes::from_static(&[0xff, 0xff, 0xff]);
payload.transactions.insert(0, invalid_tx);
changes.push("transactions = prepended invalid RLP (auto-invalidated)".to_string());
}
if self.invalidate_block_hash {
let random_hash = B256::random();
payload.block_hash = random_hash;
changes.push(format!("block_hash = {random_hash} (auto-invalidated: random)"));
}
changes
}
/// Applies invalidations to a V2 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v2(&self, payload: &mut ExecutionPayloadV2) -> Vec<String> {
let mut changes = self.apply_to_payload_v1(&mut payload.payload_inner);
// Handle withdrawals invalidation (V2+)
if self.invalidate_withdrawals {
let fake_withdrawal = Withdrawal {
index: u64::MAX,
validator_index: u64::MAX,
address: Address::ZERO,
amount: u64::MAX,
};
payload.withdrawals.push(fake_withdrawal);
changes.push("withdrawals = added fake withdrawal (auto-invalidated)".to_string());
}
changes
}
/// Applies invalidations to a V3 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v3(&self, payload: &mut ExecutionPayloadV3) -> Vec<String> {
let mut changes = self.apply_to_payload_v2(&mut payload.payload_inner);
// Explicit overrides for V3 fields
if let Some(blob_gas_used) = self.blob_gas_used {
payload.blob_gas_used = blob_gas_used;
changes.push(format!("blob_gas_used = {blob_gas_used}"));
}
if let Some(excess_blob_gas) = self.excess_blob_gas {
payload.excess_blob_gas = excess_blob_gas;
changes.push(format!("excess_blob_gas = {excess_blob_gas}"));
}
// Auto-invalidation for V3 fields
if self.invalidate_blob_gas_used {
payload.blob_gas_used = u64::MAX;
changes.push("blob_gas_used = MAX (auto-invalidated)".to_string());
}
if self.invalidate_excess_blob_gas {
payload.excess_blob_gas = u64::MAX;
changes.push("excess_blob_gas = MAX (auto-invalidated)".to_string());
}
changes
}
}

View File

@@ -0,0 +1,367 @@
//! Command for sending invalid payloads to test Engine API rejection.
mod invalidation;
use invalidation::InvalidationConfig;
use alloy_primitives::{Address, B256};
use alloy_provider::network::AnyRpcBlock;
use alloy_rpc_types_engine::ExecutionPayload;
use clap::Parser;
use eyre::{OptionExt, Result};
use op_alloy_consensus::OpTxEnvelope;
use reth_cli_runner::CliContext;
use std::io::{BufReader, Read, Write};
/// Command for generating and sending an invalid `engine_newPayload` request.
///
/// Takes a valid block and modifies fields to make it invalid for testing
/// Engine API rejection behavior. Block hash is recalculated after modifications
/// unless `--invalidate-block-hash` or `--skip-hash-recalc` is used.
#[derive(Debug, Parser)]
pub struct Command {
// ==================== Input Options ====================
/// Path to the JSON file containing the block. If not specified, stdin will be used.
#[arg(short, long, help_heading = "Input Options")]
path: Option<String>,
/// The engine RPC URL to use.
#[arg(
short,
long,
help_heading = "Input Options",
required_if_eq_any([("mode", "execute"), ("mode", "cast")]),
required_unless_present("mode")
)]
rpc_url: Option<String>,
/// The JWT secret to use. Can be either a path to a file containing the secret or the secret
/// itself.
#[arg(short, long, help_heading = "Input Options")]
jwt_secret: Option<String>,
/// The newPayload version to use (3 or 4).
#[arg(long, default_value_t = 3, help_heading = "Input Options")]
new_payload_version: u8,
/// The output mode to use.
#[arg(long, value_enum, default_value = "execute", help_heading = "Input Options")]
mode: Mode,
// ==================== Explicit Value Overrides ====================
/// Override the parent hash with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
parent_hash: Option<B256>,
/// Override the fee recipient (coinbase) with a specific address.
#[arg(long, value_name = "ADDR", help_heading = "Explicit Value Overrides")]
fee_recipient: Option<Address>,
/// Override the state root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
state_root: Option<B256>,
/// Override the receipts root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
receipts_root: Option<B256>,
/// Override the block number with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
block_number: Option<u64>,
/// Override the gas limit with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
gas_limit: Option<u64>,
/// Override the gas used with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
gas_used: Option<u64>,
/// Override the timestamp with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
timestamp: Option<u64>,
/// Override the base fee per gas with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
base_fee_per_gas: Option<u64>,
/// Override the block hash with a specific value (skips hash recalculation).
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
block_hash: Option<B256>,
/// Override the blob gas used with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
blob_gas_used: Option<u64>,
/// Override the excess blob gas with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
excess_blob_gas: Option<u64>,
/// Override the parent beacon block root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
parent_beacon_block_root: Option<B256>,
/// Override the requests hash with a specific value (EIP-7685).
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
requests_hash: Option<B256>,
// ==================== Auto-Invalidation Flags ====================
/// Invalidate the parent hash by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_parent_hash: bool,
/// Invalidate the state root by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_state_root: bool,
/// Invalidate the receipts root by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_receipts_root: bool,
/// Invalidate the gas used by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_gas_used: bool,
/// Invalidate the block number by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_block_number: bool,
/// Invalidate the timestamp by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_timestamp: bool,
/// Invalidate the base fee by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_base_fee: bool,
/// Invalidate the transactions by modifying them.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_transactions: bool,
/// Invalidate the block hash by not recalculating it after modifications.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_block_hash: bool,
/// Invalidate the withdrawals by modifying them.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_withdrawals: bool,
/// Invalidate the blob gas used by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_blob_gas_used: bool,
/// Invalidate the excess blob gas by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_excess_blob_gas: bool,
/// Invalidate the requests hash by setting it to a random value (EIP-7685).
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_requests_hash: bool,
// ==================== Meta Flags ====================
/// Skip block hash recalculation after modifications.
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
skip_hash_recalc: bool,
/// Print what would be done without actually sending the payload.
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
dry_run: bool,
}
#[derive(Debug, Clone, clap::ValueEnum)]
enum Mode {
/// Execute the `cast` command. This works with blocks of any size, because it pipes the
/// payload into the `cast` command.
Execute,
/// Print the `cast` command. Caution: this may not work with large blocks because of the
/// command length limit.
Cast,
/// Print the JSON payload. Can be piped into `cast` command if the block is small enough.
Json,
}
impl Command {
/// Read input from either a file or stdin
fn read_input(&self) -> Result<String> {
Ok(match &self.path {
Some(path) => reth_fs_util::read_to_string(path)?,
None => String::from_utf8(
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
)?,
})
}
/// Load JWT secret from either a file or use the provided string directly
fn load_jwt_secret(&self) -> Result<Option<String>> {
match &self.jwt_secret {
Some(secret) => match std::fs::read_to_string(secret) {
Ok(contents) => Ok(Some(contents.trim().to_string())),
Err(_) => Ok(Some(secret.clone())),
},
None => Ok(None),
}
}
/// Build `InvalidationConfig` from command flags
const fn build_invalidation_config(&self) -> InvalidationConfig {
InvalidationConfig {
parent_hash: self.parent_hash,
fee_recipient: self.fee_recipient,
state_root: self.state_root,
receipts_root: self.receipts_root,
logs_bloom: None,
prev_randao: None,
block_number: self.block_number,
gas_limit: self.gas_limit,
gas_used: self.gas_used,
timestamp: self.timestamp,
extra_data: None,
base_fee_per_gas: self.base_fee_per_gas,
block_hash: self.block_hash,
blob_gas_used: self.blob_gas_used,
excess_blob_gas: self.excess_blob_gas,
invalidate_parent_hash: self.invalidate_parent_hash,
invalidate_state_root: self.invalidate_state_root,
invalidate_receipts_root: self.invalidate_receipts_root,
invalidate_gas_used: self.invalidate_gas_used,
invalidate_block_number: self.invalidate_block_number,
invalidate_timestamp: self.invalidate_timestamp,
invalidate_base_fee: self.invalidate_base_fee,
invalidate_transactions: self.invalidate_transactions,
invalidate_block_hash: self.invalidate_block_hash,
invalidate_withdrawals: self.invalidate_withdrawals,
invalidate_blob_gas_used: self.invalidate_blob_gas_used,
invalidate_excess_blob_gas: self.invalidate_excess_blob_gas,
}
}
/// Execute the command
pub async fn execute(self, _ctx: CliContext) -> Result<()> {
let block_json = self.read_input()?;
let jwt_secret = self.load_jwt_secret()?;
let block = serde_json::from_str::<AnyRpcBlock>(&block_json)?
.into_inner()
.map_header(|header| header.map(|h| h.into_header_with_defaults()))
.try_map_transactions(|tx| tx.try_into_either::<OpTxEnvelope>())?
.into_consensus();
let config = self.build_invalidation_config();
let parent_beacon_block_root =
self.parent_beacon_block_root.or(block.header.parent_beacon_block_root);
let blob_versioned_hashes =
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
let use_v4 = block.header.requests_hash.is_some();
let requests_hash = self.requests_hash.or(block.header.requests_hash);
let mut execution_payload = ExecutionPayload::from_block_slow(&block).0;
let changes = match &mut execution_payload {
ExecutionPayload::V1(p) => config.apply_to_payload_v1(p),
ExecutionPayload::V2(p) => config.apply_to_payload_v2(p),
ExecutionPayload::V3(p) => config.apply_to_payload_v3(p),
};
let skip_recalc = self.skip_hash_recalc || config.should_skip_hash_recalc();
if !skip_recalc {
let new_hash = match execution_payload.clone().into_block_raw() {
Ok(block) => block.header.hash_slow(),
Err(e) => {
eprintln!(
"Warning: Could not recalculate block hash: {e}. Using original hash."
);
match &execution_payload {
ExecutionPayload::V1(p) => p.block_hash,
ExecutionPayload::V2(p) => p.payload_inner.block_hash,
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash,
}
}
};
match &mut execution_payload {
ExecutionPayload::V1(p) => p.block_hash = new_hash,
ExecutionPayload::V2(p) => p.payload_inner.block_hash = new_hash,
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash = new_hash,
}
}
if self.dry_run {
println!("=== Dry Run ===");
println!("Changes that would be applied:");
for change in &changes {
println!(" - {}", change);
}
if changes.is_empty() {
println!(" (no changes)");
}
if skip_recalc {
println!(" - Block hash recalculation: SKIPPED");
} else {
println!(" - Block hash recalculation: PERFORMED");
}
println!("\nResulting payload JSON:");
let json = serde_json::to_string_pretty(&execution_payload)?;
println!("{}", json);
return Ok(());
}
let json_request = if use_v4 {
serde_json::to_string(&(
execution_payload,
blob_versioned_hashes,
parent_beacon_block_root,
requests_hash.unwrap_or_default(),
))?
} else {
serde_json::to_string(&(
execution_payload,
blob_versioned_hashes,
parent_beacon_block_root,
))?
};
match self.mode {
Mode::Execute => {
let mut command = std::process::Command::new("cast");
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
command.arg("rpc").arg(method).arg("--raw");
if let Some(rpc_url) = self.rpc_url {
command.arg("--rpc-url").arg(rpc_url);
}
if let Some(secret) = &jwt_secret {
command.arg("--jwt-secret").arg(secret);
}
let mut process = command.stdin(std::process::Stdio::piped()).spawn()?;
process
.stdin
.take()
.ok_or_eyre("stdin not available")?
.write_all(json_request.as_bytes())?;
process.wait()?;
}
Mode::Cast => {
let mut cmd = format!(
"cast rpc engine_newPayloadV{} --raw '{}'",
self.new_payload_version, json_request
);
if let Some(rpc_url) = self.rpc_url {
cmd += &format!(" --rpc-url {rpc_url}");
}
if let Some(secret) = &jwt_secret {
cmd += &format!(" --jwt-secret {secret}");
}
println!("{cmd}");
}
Mode::Json => {
println!("{json_request}");
}
}
Ok(())
}
}

View File

@@ -25,8 +25,8 @@
//! - `jemalloc-unprefixed`: Uses unprefixed jemalloc symbols.
//! - `tracy-allocator`: Enables [Tracy](https://github.com/wolfpld/tracy) profiler allocator
//! integration for memory profiling.
//! - `snmalloc`: Uses [snmalloc](https://github.com/snmalloc/snmalloc) as the global allocator. Use
//! `--no-default-features` when enabling this, as jemalloc takes precedence.
//! - `snmalloc`: Uses [snmalloc](https://github.com/microsoft/snmalloc) as the global allocator.
//! Use `--no-default-features` when enabling this, as jemalloc takes precedence.
//! - `snmalloc-native`: Uses snmalloc with native CPU optimizations. Use `--no-default-features`
//! when enabling this.
//!

View File

@@ -41,6 +41,7 @@ derive_more.workspace = true
metrics.workspace = true
parking_lot.workspace = true
pin-project.workspace = true
rayon = { workspace = true, optional = true }
serde = { workspace = true, optional = true }
# optional deps for test-utils
@@ -84,6 +85,7 @@ test-utils = [
"reth-trie/test-utils",
"reth-ethereum-primitives/test-utils",
]
rayon = ["dep:rayon"]
[[bench]]
name = "canonical_hashes_range"

View File

@@ -163,14 +163,29 @@ impl DeferredTrieData {
anchor_hash: B256,
ancestors: &[Self],
) -> ComputedTrieData {
let sorted_hashed_state = match Arc::try_unwrap(hashed_state) {
Ok(state) => state.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
};
let sorted_trie_updates = match Arc::try_unwrap(trie_updates) {
Ok(updates) => updates.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
};
#[cfg(feature = "rayon")]
let (sorted_hashed_state, sorted_trie_updates) = rayon::join(
|| match Arc::try_unwrap(hashed_state) {
Ok(state) => state.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
|| match Arc::try_unwrap(trie_updates) {
Ok(updates) => updates.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
);
#[cfg(not(feature = "rayon"))]
let (sorted_hashed_state, sorted_trie_updates) = (
match Arc::try_unwrap(hashed_state) {
Ok(state) => state.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
match Arc::try_unwrap(trie_updates) {
Ok(updates) => updates.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
);
// Reuse parent's overlay if available and anchors match.
// We can only reuse the parent's overlay if it was built on top of the same
@@ -228,8 +243,53 @@ impl DeferredTrieData {
/// In normal operation, the parent always has a cached overlay and this
/// function is never called.
///
/// Iterates ancestors oldest -> newest, then extends with current block's data,
/// so later state takes precedence.
/// When the `rayon` feature is enabled, uses parallel collection and merge:
/// 1. Collects ancestor data in parallel (each `wait_cloned()` may compute)
/// 2. Merges hashed state and trie updates in parallel with each other
/// 3. Uses tree reduction within each merge for O(log n) depth
#[cfg(feature = "rayon")]
fn merge_ancestors_into_overlay(
ancestors: &[Self],
sorted_hashed_state: &HashedPostStateSorted,
sorted_trie_updates: &TrieUpdatesSorted,
) -> TrieInputSorted {
// Early exit: no ancestors means just wrap current block's data
if ancestors.is_empty() {
return TrieInputSorted::new(
Arc::new(sorted_trie_updates.clone()),
Arc::new(sorted_hashed_state.clone()),
Default::default(),
);
}
// Collect ancestor data, unzipping states and updates into Arc slices
let (states, updates): (Vec<_>, Vec<_>) = ancestors
.iter()
.map(|a| {
let data = a.wait_cloned();
(data.hashed_state, data.trie_updates)
})
.unzip();
// Merge state and nodes in parallel with each other using tree reduction
let (state, nodes) = rayon::join(
|| {
let mut merged = HashedPostStateSorted::merge_parallel(&states);
merged.extend_ref_and_sort(sorted_hashed_state);
merged
},
|| {
let mut merged = TrieUpdatesSorted::merge_parallel(&updates);
merged.extend_ref_and_sort(sorted_trie_updates);
merged
},
);
TrieInputSorted::new(Arc::new(nodes), Arc::new(state), Default::default())
}
/// Merge all ancestors and current block's data into a single overlay (sequential fallback).
#[cfg(not(feature = "rayon"))]
fn merge_ancestors_into_overlay(
ancestors: &[Self],
sorted_hashed_state: &HashedPostStateSorted,

View File

@@ -123,60 +123,18 @@ impl LazyOverlay {
/// Merge all blocks' trie data into a single [`TrieInputSorted`].
///
/// Blocks are ordered newest to oldest. Uses hybrid merge algorithm that
/// switches between `extend_ref` (small batches) and k-way merge (large batches).
/// Blocks are ordered newest to oldest.
fn merge_blocks(blocks: &[DeferredTrieData]) -> TrieInputSorted {
const MERGE_BATCH_THRESHOLD: usize = 64;
if blocks.is_empty() {
return TrieInputSorted::default();
}
// Single block: use its data directly (no allocation)
if blocks.len() == 1 {
let data = blocks[0].wait_cloned();
return TrieInputSorted {
state: data.hashed_state,
nodes: data.trie_updates,
prefix_sets: Default::default(),
};
}
let state =
HashedPostStateSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().hashed_state));
let nodes =
TrieUpdatesSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().trie_updates));
if blocks.len() < MERGE_BATCH_THRESHOLD {
// Small k: extend_ref loop with Arc::make_mut is faster.
// Uses copy-on-write - only clones inner data if Arc has multiple refs.
// Iterate oldest->newest so newer values override older ones.
let mut blocks_iter = blocks.iter().rev();
let first = blocks_iter.next().expect("blocks is non-empty");
let data = first.wait_cloned();
let mut state = data.hashed_state;
let mut nodes = data.trie_updates;
for block in blocks_iter {
let block_data = block.wait_cloned();
Arc::make_mut(&mut state).extend_ref_and_sort(block_data.hashed_state.as_ref());
Arc::make_mut(&mut nodes).extend_ref_and_sort(block_data.trie_updates.as_ref());
}
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
} else {
// Large k: k-way merge is faster (O(n log k)).
// Collect is unavoidable here - we need all data materialized for k-way merge.
let trie_data: Vec<_> = blocks.iter().map(|b| b.wait_cloned()).collect();
let merged_state = HashedPostStateSorted::merge_batch(
trie_data.iter().map(|d| d.hashed_state.as_ref()),
);
let merged_nodes =
TrieUpdatesSorted::merge_batch(trie_data.iter().map(|d| d.trie_updates.as_ref()));
TrieInputSorted {
state: Arc::new(merged_state),
nodes: Arc::new(merged_nodes),
prefix_sets: Default::default(),
}
}
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
}
}

View File

@@ -278,6 +278,7 @@ pub fn create_chain_config(
// Check if DAO fork is supported (it has an activation block)
let dao_fork_support = hardforks.fork(EthereumHardfork::Dao) != ForkCondition::Never;
#[allow(clippy::needless_update)]
ChainConfig {
chain_id: chain.map(|c| c.id()).unwrap_or(0),
homestead_block: block_num(EthereumHardfork::Homestead),
@@ -313,6 +314,7 @@ pub fn create_chain_config(
extra_fields: Default::default(),
deposit_contract_address,
blob_schedule,
..Default::default()
}
}

View File

@@ -50,6 +50,7 @@ reth-stages-types = { workspace = true, optional = true }
reth-static-file-types = { workspace = true, features = ["clap"] }
reth-static-file.workspace = true
reth-tasks.workspace = true
reth-storage-api.workspace = true
reth-trie = { workspace = true, features = ["metrics"] }
reth-trie-db = { workspace = true, features = ["metrics"] }
reth-trie-common.workspace = true
@@ -131,4 +132,4 @@ arbitrary = [
"reth-ethereum-primitives/arbitrary",
]
edge = ["reth-db-common/edge", "reth-stages/rocksdb"]
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb"]

View File

@@ -19,7 +19,7 @@ use reth_node_builder::{
Node, NodeComponents, NodeComponentsBuilder, NodeTypes, NodeTypesWithDBAdapter,
};
use reth_node_core::{
args::{DatabaseArgs, DatadirArgs, StaticFilesArgs},
args::{DatabaseArgs, DatadirArgs, RocksDbArgs, StaticFilesArgs},
dirs::{ChainPath, DataDirPath},
};
use reth_provider::{
@@ -27,7 +27,7 @@ use reth_provider::{
BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider,
StaticFileProviderBuilder,
},
ProviderFactory, StaticFileProviderFactory,
ProviderFactory, StaticFileProviderFactory, StorageSettings,
};
use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget};
use reth_static_file::StaticFileProducer;
@@ -66,9 +66,24 @@ pub struct EnvironmentArgs<C: ChainSpecParser> {
/// All static files related arguments
#[command(flatten)]
pub static_files: StaticFilesArgs,
/// All `RocksDB` related arguments
#[command(flatten)]
pub rocksdb: RocksDbArgs,
}
impl<C: ChainSpecParser> EnvironmentArgs<C> {
/// Returns the effective storage settings derived from static-file and `RocksDB` CLI args.
pub fn storage_settings(&self) -> StorageSettings {
StorageSettings::base()
.with_receipts_in_static_files(self.static_files.receipts)
.with_transaction_senders_in_static_files(self.static_files.transaction_senders)
.with_account_changesets_in_static_files(self.static_files.account_changesets)
.with_transaction_hash_numbers_in_rocksdb(self.rocksdb.all || self.rocksdb.tx_hash)
.with_storages_history_in_rocksdb(self.rocksdb.all || self.rocksdb.storages_history)
.with_account_history_in_rocksdb(self.rocksdb.all || self.rocksdb.account_history)
}
/// Initializes environment according to [`AccessRights`] and returns an instance of
/// [`Environment`].
pub fn init<N: CliNodeTypes>(&self, access: AccessRights) -> eyre::Result<Environment<N>>
@@ -121,17 +136,17 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
})
}
};
// TransactionDB only support read-write mode
let rocksdb_provider = RocksDBProvider::builder(data_dir.rocksdb())
.with_default_tables()
.with_database_log_level(self.db.log_level)
.with_read_only(!access.is_read_write())
.build()?;
let provider_factory =
self.create_provider_factory(&config, db, sfp, rocksdb_provider, access)?;
if access.is_read_write() {
debug!(target: "reth::cli", chain=%self.chain.chain(), genesis=?self.chain.genesis_hash(), "Initializing genesis");
init_genesis_with_settings(&provider_factory, self.static_files.to_settings())?;
init_genesis_with_settings(&provider_factory, self.storage_settings())?;
}
Ok(Environment { config, provider_factory, data_dir })

View File

@@ -1,145 +0,0 @@
use crate::{
common::CliNodeTypes,
db::get::{maybe_json_value_parser, table_key},
};
use alloy_primitives::map::foldhash::fast::FixedState;
use clap::Parser;
use reth_chainspec::EthereumHardforks;
use reth_db::DatabaseEnv;
use reth_db_api::{
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
Tables,
};
use reth_db_common::DbTool;
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_provider::{providers::ProviderNodeTypes, DBProvider};
use std::{
hash::{BuildHasher, Hasher},
sync::Arc,
time::{Duration, Instant},
};
use tracing::{info, warn};
#[derive(Parser, Debug)]
/// The arguments for the `reth db checksum` command
pub struct Command {
/// The table name
table: Tables,
/// The start of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
start_key: Option<String>,
/// The end of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
end_key: Option<String>,
/// The maximum number of records that are queried and used to compute the
/// checksum.
#[arg(long)]
limit: Option<usize>,
}
impl Command {
/// Execute `db checksum` command
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
) -> eyre::Result<()> {
warn!("This command should be run without the node running!");
self.table.view(&ChecksumViewer {
tool,
start_key: self.start_key,
end_key: self.end_key,
limit: self.limit,
})?;
Ok(())
}
}
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
tool: &'a DbTool<N>,
start_key: Option<String>,
end_key: Option<String>,
limit: Option<usize>,
}
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
}
}
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
type Error = eyre::Report;
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
let provider =
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
let tx = provider.tx_ref();
info!(
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
self.start_key, self.end_key, self.limit
);
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
(Some(start), Some(end)) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(start_key..=end_key)?
}
(None, Some(end)) => {
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(..=end_key)?
}
(Some(start), None) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
cursor.walk_range(start_key..)?
}
(None, None) => cursor.walk_range(..)?,
};
let start_time = Instant::now();
let mut hasher = FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher();
let mut total = 0;
let limit = self.limit.unwrap_or(usize::MAX);
let mut enumerate_start_key = None;
let mut enumerate_end_key = None;
for (index, entry) in walker.enumerate() {
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
if index.is_multiple_of(100_000) {
info!("Hashed {index} entries.");
}
hasher.write(k.raw_key());
hasher.write(v.raw_value());
if enumerate_start_key.is_none() {
enumerate_start_key = Some(k.clone());
}
enumerate_end_key = Some(k);
total = index + 1;
if total >= limit {
break
}
}
info!("Hashed {total} entries.");
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
Ok((checksum, elapsed))
}
}

View File

@@ -0,0 +1,288 @@
use crate::{
common::CliNodeTypes,
db::get::{maybe_json_value_parser, table_key},
};
use alloy_primitives::map::foldhash::fast::FixedState;
use clap::Parser;
use itertools::Itertools;
use reth_chainspec::EthereumHardforks;
use reth_db::{static_file::iter_static_files, DatabaseEnv};
use reth_db_api::{
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
Tables,
};
use reth_db_common::DbTool;
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProviderFactory};
use reth_static_file_types::StaticFileSegment;
use std::{
hash::{BuildHasher, Hasher},
sync::Arc,
time::{Duration, Instant},
};
use tracing::{info, warn};
#[cfg(all(unix, feature = "edge"))]
mod rocksdb;
/// Interval for logging progress during checksum computation.
const PROGRESS_LOG_INTERVAL: usize = 100_000;
#[derive(Parser, Debug)]
/// The arguments for the `reth db checksum` command
pub struct Command {
#[command(subcommand)]
subcommand: Subcommand,
}
#[derive(clap::Subcommand, Debug)]
enum Subcommand {
/// Calculates the checksum of a database table
Mdbx {
/// The table name
table: Tables,
/// The start of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
start_key: Option<String>,
/// The end of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
end_key: Option<String>,
/// The maximum number of records that are queried and used to compute the
/// checksum.
#[arg(long)]
limit: Option<usize>,
},
/// Calculates the checksum of a static file segment
StaticFile {
/// The static file segment
#[arg(value_enum)]
segment: StaticFileSegment,
/// The block number to start from (inclusive).
#[arg(long)]
start_block: Option<u64>,
/// The block number to end at (inclusive).
#[arg(long)]
end_block: Option<u64>,
/// The maximum number of rows to checksum.
#[arg(long)]
limit: Option<usize>,
},
/// Calculates the checksum of a RocksDB table
#[cfg(all(unix, feature = "edge"))]
Rocksdb {
/// The RocksDB table
#[arg(value_enum)]
table: rocksdb::RocksDbTable,
/// The maximum number of records to checksum.
#[arg(long)]
limit: Option<usize>,
},
}
impl Command {
/// Execute `db checksum` command
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
) -> eyre::Result<()> {
warn!("This command should be run without the node running!");
match self.subcommand {
Subcommand::Mdbx { table, start_key, end_key, limit } => {
table.view(&ChecksumViewer { tool, start_key, end_key, limit })?;
}
Subcommand::StaticFile { segment, start_block, end_block, limit } => {
checksum_static_file(tool, segment, start_block, end_block, limit)?;
}
#[cfg(all(unix, feature = "edge"))]
Subcommand::Rocksdb { table, limit } => {
rocksdb::checksum_rocksdb(tool, table, limit)?;
}
}
Ok(())
}
}
/// Creates a new hasher with the standard seed used for checksum computation.
fn checksum_hasher() -> impl Hasher {
FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher()
}
fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
segment: StaticFileSegment,
start_block: Option<u64>,
end_block: Option<u64>,
limit: Option<usize>,
) -> eyre::Result<()> {
let static_file_provider = tool.provider_factory.static_file_provider();
if let Err(err) = static_file_provider.check_consistency(&tool.provider_factory.provider()?) {
warn!("Error checking consistency of static files: {err}");
}
let static_files = iter_static_files(static_file_provider.directory())?;
let ranges = static_files
.get(segment)
.ok_or_else(|| eyre::eyre!("No static files found for segment: {}", segment))?;
let start_time = Instant::now();
let mut hasher = checksum_hasher();
let mut total = 0usize;
let limit = limit.unwrap_or(usize::MAX);
let start_block = start_block.unwrap_or(0);
let end_block = end_block.unwrap_or(u64::MAX);
info!(
"Computing checksum for {} static files, start_block={}, end_block={}, limit={:?}",
segment,
start_block,
end_block,
if limit == usize::MAX { None } else { Some(limit) }
);
'outer: for (block_range, _header) in ranges.iter().sorted_by_key(|(range, _)| range.start()) {
if block_range.end() < start_block || block_range.start() > end_block {
continue;
}
let fixed_block_range = static_file_provider.find_fixed_range(segment, block_range.start());
let jar_provider = static_file_provider
.get_segment_provider_for_range(segment, || Some(fixed_block_range), None)?
.ok_or_else(|| {
eyre::eyre!(
"Failed to get segment provider for segment {} at range {}",
segment,
block_range
)
})?;
let mut cursor = jar_provider.cursor()?;
while let Ok(Some(row)) = cursor.next_row() {
for col_data in row.iter() {
hasher.write(col_data);
}
total += 1;
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {total} entries.");
}
if total >= limit {
break 'outer;
}
}
// Explicitly drop provider before removing from cache to avoid deadlock
drop(jar_provider);
static_file_provider.remove_cached_provider(segment, fixed_block_range.end());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!(
"Checksum for static file segment `{}`: {:#x} ({} entries, elapsed: {:?})",
segment, checksum, total, elapsed
);
Ok(())
}
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
tool: &'a DbTool<N>,
start_key: Option<String>,
end_key: Option<String>,
limit: Option<usize>,
}
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
}
}
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
type Error = eyre::Report;
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
let provider =
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
let tx = provider.tx_ref();
info!(
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
self.start_key, self.end_key, self.limit
);
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
(Some(start), Some(end)) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(start_key..=end_key)?
}
(None, Some(end)) => {
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(..=end_key)?
}
(Some(start), None) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
cursor.walk_range(start_key..)?
}
(None, None) => cursor.walk_range(..)?,
};
let start_time = Instant::now();
let mut hasher = checksum_hasher();
let mut total = 0;
let limit = self.limit.unwrap_or(usize::MAX);
let mut enumerate_start_key = None;
let mut enumerate_end_key = None;
for (index, entry) in walker.enumerate() {
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
if index.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {index} entries.");
}
hasher.write(k.raw_key());
hasher.write(v.raw_value());
if enumerate_start_key.is_none() {
enumerate_start_key = Some(k.clone());
}
enumerate_end_key = Some(k);
total = index + 1;
if total >= limit {
break
}
}
info!("Hashed {total} entries.");
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
Ok((checksum, elapsed))
}
}

View File

@@ -0,0 +1,106 @@
//! RocksDB checksum implementation.
use super::{checksum_hasher, PROGRESS_LOG_INTERVAL};
use crate::common::CliNodeTypes;
use clap::ValueEnum;
use reth_chainspec::EthereumHardforks;
use reth_db::{tables, DatabaseEnv};
use reth_db_api::table::Table;
use reth_db_common::DbTool;
use reth_node_builder::NodeTypesWithDBAdapter;
use reth_provider::RocksDBProviderFactory;
use std::{hash::Hasher, sync::Arc, time::Instant};
use tracing::info;
/// RocksDB tables that can be checksummed.
#[derive(Debug, Clone, Copy, ValueEnum)]
pub enum RocksDbTable {
/// Transaction hash to transaction number mapping
TransactionHashNumbers,
/// Account history indices
AccountsHistory,
/// Storage history indices
StoragesHistory,
}
impl RocksDbTable {
/// Returns the table name as a string
const fn name(&self) -> &'static str {
match self {
Self::TransactionHashNumbers => tables::TransactionHashNumbers::NAME,
Self::AccountsHistory => tables::AccountsHistory::NAME,
Self::StoragesHistory => tables::StoragesHistory::NAME,
}
}
}
/// Computes a checksum for a RocksDB table.
pub fn checksum_rocksdb<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
table: RocksDbTable,
limit: Option<usize>,
) -> eyre::Result<()> {
let rocksdb = tool.provider_factory.rocksdb_provider();
let start_time = Instant::now();
let limit = limit.unwrap_or(usize::MAX);
info!(
"Computing checksum for RocksDB table `{}`, limit={:?}",
table.name(),
if limit == usize::MAX { None } else { Some(limit) }
);
let (checksum, total) = match table {
RocksDbTable::TransactionHashNumbers => {
checksum_rocksdb_table::<tables::TransactionHashNumbers>(&rocksdb, limit)?
}
RocksDbTable::AccountsHistory => {
checksum_rocksdb_table::<tables::AccountsHistory>(&rocksdb, limit)?
}
RocksDbTable::StoragesHistory => {
checksum_rocksdb_table::<tables::StoragesHistory>(&rocksdb, limit)?
}
};
let elapsed = start_time.elapsed();
info!(
"Checksum for RocksDB table `{}`: {:#x} ({} entries, elapsed: {:?})",
table.name(),
checksum,
total,
elapsed
);
Ok(())
}
/// Computes checksum for a specific RocksDB table by iterating over rows.
fn checksum_rocksdb_table<T: Table>(
rocksdb: &reth_provider::providers::RocksDBProvider,
limit: usize,
) -> eyre::Result<(u64, usize)> {
let iter = rocksdb.raw_iter::<T>()?;
let mut hasher = checksum_hasher();
let mut total = 0usize;
for entry in iter {
let (key_bytes, value_bytes) = entry?;
hasher.write(&key_bytes);
hasher.write(&value_bytes);
total += 1;
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {total} entries.");
}
if total >= limit {
break;
}
}
Ok((hasher.finish(), total))
}

View File

@@ -21,6 +21,7 @@ use reth_node_builder::NodeTypesWithDB;
use reth_primitives_traits::ValueWithSubKey;
use reth_provider::{providers::ProviderNodeTypes, ChangeSetReader, StaticFileProviderFactory};
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::StorageChangeSetReader;
use tracing::error;
/// The arguments for the `reth db get` command
@@ -82,6 +83,33 @@ impl Command {
table.view(&GetValueViewer { tool, key, subkey, end_key, end_subkey, raw })?
}
Subcommand::StaticFile { segment, key, subkey, raw } => {
if let StaticFileSegment::StorageChangeSets = segment {
let storage_key =
table_subkey::<tables::StorageChangeSets>(subkey.as_deref()).ok();
let key = table_key::<tables::StorageChangeSets>(&key)?;
let provider = tool.provider_factory.static_file_provider();
if let Some(storage_key) = storage_key {
let entry = provider.get_storage_before_block(
key.block_number(),
key.address(),
storage_key,
)?;
if let Some(entry) = entry {
println!("{}", serde_json::to_string_pretty(&entry)?);
} else {
error!(target: "reth::cli", "No content for the given table key.");
}
return Ok(());
}
let changesets = provider.storage_changeset(key.block_number())?;
println!("{}", serde_json::to_string_pretty(&changesets)?);
return Ok(());
}
let (key, subkey, mask): (u64, _, _) = match segment {
StaticFileSegment::Headers => (
table_key::<tables::Headers>(&key)?,
@@ -112,6 +140,9 @@ impl Command {
AccountChangesetMask::MASK,
)
}
StaticFileSegment::StorageChangeSets => {
unreachable!("storage changesets handled above");
}
};
// handle account changesets differently if a subkey is provided.
@@ -190,6 +221,9 @@ impl Command {
StaticFileSegment::AccountChangeSets => {
unreachable!("account changeset static files are special cased before this match")
}
StaticFileSegment::StorageChangeSets => {
unreachable!("storage changeset static files are special cased before this match")
}
}
}
}

View File

@@ -61,19 +61,21 @@ impl Command {
}
/// Generate [`ListFilter`] from command.
pub fn list_filter(&self) -> ListFilter {
let search = self
.search
.as_ref()
.map(|search| {
pub fn list_filter(&self) -> eyre::Result<ListFilter> {
let search = match self.search.as_deref() {
Some(search) => {
if let Some(search) = search.strip_prefix("0x") {
return hex::decode(search).unwrap()
hex::decode(search).wrap_err(
"Invalid hex content after 0x prefix in --search (expected valid hex like 0xdeadbeef).",
)?
} else {
search.as_bytes().to_vec()
}
search.as_bytes().to_vec()
})
.unwrap_or_default();
}
None => Vec::new(),
};
ListFilter {
Ok(ListFilter {
skip: self.skip,
len: self.len,
search,
@@ -82,7 +84,7 @@ impl Command {
min_value_size: self.min_value_size,
reverse: self.reverse,
only_count: self.count,
}
})
}
}
@@ -115,7 +117,7 @@ impl<N: NodeTypes> TableViewer<()> for ListTableViewer<'_, N> {
}
let list_filter = self.args.list_filter();
let list_filter = self.args.list_filter()?;
if self.args.json || self.args.count {
let (list, count) = self.tool.list::<T>(&list_filter)?;

View File

@@ -39,7 +39,7 @@ pub enum Subcommands {
Stats(stats::Command),
/// Lists the contents of a table
List(list::Command),
/// Calculates the content checksum of a table
/// Calculates the content checksum of a table or static file segment
Checksum(checksum::Command),
/// Create a diff between two database tables or two entire databases.
Diff(diff::Command),
@@ -162,7 +162,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
let access_rights =
if command.dry_run { AccessRights::RO } else { AccessRights::RW };
db_exec!(self.env, tool, N, access_rights, {
command.execute(&tool, ctx.task_executor.clone(), &data_dir)?;
command.execute(&tool, ctx.task_executor, &data_dir)?;
});
}
Subcommands::StaticFileHeader(command) => {

View File

@@ -69,6 +69,11 @@ pub enum SetCommand {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store storage changesets in static files instead of the database
StorageChangesets {
#[clap(action(ArgAction::Set))]
value: bool,
},
}
impl Command {
@@ -115,6 +120,7 @@ impl Command {
transaction_hash_numbers_in_rocksdb: _,
account_history_in_rocksdb: _,
account_changesets_in_static_files: _,
storage_changesets_in_static_files: _,
} = settings.unwrap_or_else(StorageSettings::legacy);
// Update the setting based on the key
@@ -167,6 +173,14 @@ impl Command {
settings.account_history_in_rocksdb = value;
println!("Set account_history_in_rocksdb = {}", value);
}
SetCommand::StorageChangesets { value } => {
if settings.storage_changesets_in_static_files == value {
println!("storage_changesets_in_static_files is already set to {}", value);
return Ok(());
}
settings.storage_changesets_in_static_files = value;
println!("Set storage_changesets_in_static_files = {}", value);
}
}
// Write updated settings

View File

@@ -11,7 +11,10 @@ use reth_db_common::DbTool;
use reth_fs_util as fs;
use reth_node_builder::{NodePrimitives, NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_provider::providers::{ProviderNodeTypes, StaticFileProvider};
use reth_provider::{
providers::{ProviderNodeTypes, StaticFileProvider},
RocksDBProviderFactory,
};
use reth_static_file_types::SegmentRangeInclusive;
use std::{sync::Arc, time::Duration};
@@ -61,6 +64,11 @@ impl Command {
let db_stats_table = self.db_stats_table(tool)?;
println!("{db_stats_table}");
println!("\n");
let rocksdb_stats_table = self.rocksdb_stats_table(tool);
println!("{rocksdb_stats_table}");
Ok(())
}
@@ -148,6 +156,60 @@ impl Command {
Ok(table)
}
fn rocksdb_stats_table<N: NodeTypesWithDB>(&self, tool: &DbTool<N>) -> ComfyTable {
let mut table = ComfyTable::new();
table.load_preset(comfy_table::presets::ASCII_MARKDOWN);
table.set_header([
"RocksDB Table Name",
"# Entries",
"SST Size",
"Memtable Size",
"Total Size",
"Pending Compaction",
]);
let stats = tool.provider_factory.rocksdb_provider().table_stats();
let mut total_sst: u64 = 0;
let mut total_memtable: u64 = 0;
let mut total_size: u64 = 0;
let mut total_pending: u64 = 0;
for stat in &stats {
total_sst += stat.sst_size_bytes;
total_memtable += stat.memtable_size_bytes;
total_size += stat.estimated_size_bytes;
total_pending += stat.pending_compaction_bytes;
let mut row = Row::new();
row.add_cell(Cell::new(&stat.name))
.add_cell(Cell::new(stat.estimated_num_keys))
.add_cell(Cell::new(human_bytes(stat.sst_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.memtable_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.estimated_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.pending_compaction_bytes as f64)));
table.add_row(row);
}
if !stats.is_empty() {
let max_widths = table.column_max_content_widths();
let mut separator = Row::new();
for width in max_widths {
separator.add_cell(Cell::new("-".repeat(width as usize)));
}
table.add_row(separator);
let mut row = Row::new();
row.add_cell(Cell::new("RocksDB Total"))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(total_sst as f64)))
.add_cell(Cell::new(human_bytes(total_memtable as f64)))
.add_cell(Cell::new(human_bytes(total_size as f64)))
.add_cell(Cell::new(human_bytes(total_pending as f64)));
table.add_row(row);
}
table
}
fn static_files_stats_table<N: NodePrimitives>(
&self,
data_dir: ChainPath<DataDirPath>,

View File

@@ -10,7 +10,8 @@ use reth_node_builder::NodeBuilder;
use reth_node_core::{
args::{
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, EraArgs, MetricArgs,
NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
NetworkArgs, PayloadBuilderArgs, PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs,
TxPoolArgs,
},
node_config::NodeConfig,
version,
@@ -102,6 +103,10 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
#[command(flatten)]
pub pruning: PruningArgs,
/// All `RocksDB` table routing arguments
#[command(flatten)]
pub rocksdb: RocksDbArgs,
/// Engine cli arguments
#[command(flatten, next_help_heading = "Engine")]
pub engine: EngineArgs,
@@ -166,12 +171,16 @@ where
db,
dev,
pruning,
rocksdb,
engine,
era,
static_files,
ext,
} = self;
// Validate RocksDB arguments
rocksdb.validate()?;
// set up node config
let mut node_config = NodeConfig {
datadir,
@@ -187,6 +196,7 @@ where
db,
dev,
pruning,
rocksdb,
engine,
era,
static_files,

View File

@@ -0,0 +1,34 @@
//! Enode identifier command
use clap::Parser;
use reth_cli_util::get_secret_key;
use reth_network_peers::NodeRecord;
use std::{
net::{IpAddr, Ipv4Addr, SocketAddr},
path::PathBuf,
};
/// Print the enode identifier for a given secret key.
#[derive(Parser, Debug)]
pub struct Command {
/// Path to the secret key file for discovery.
pub discovery_secret: PathBuf,
/// Optional IP address to include in the enode URL.
///
/// If not provided, defaults to 0.0.0.0.
#[arg(long)]
pub ip: Option<IpAddr>,
}
impl Command {
/// Execute the enode command.
pub fn execute(self) -> eyre::Result<()> {
let sk = get_secret_key(&self.discovery_secret)?;
let ip = self.ip.unwrap_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED));
let addr = SocketAddr::new(ip, 30303);
let enr = NodeRecord::from_secret_key(addr, &sk);
println!("{enr}");
Ok(())
}
}

View File

@@ -18,6 +18,7 @@ use reth_node_core::{
};
pub mod bootnode;
pub mod enode;
pub mod rlpx;
/// `reth p2p` command
@@ -85,6 +86,9 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
Subcommands::Bootnode(command) => {
command.execute().await?;
}
Subcommands::Enode(command) => {
command.execute()?;
}
}
Ok(())
@@ -99,6 +103,7 @@ impl<C: ChainSpecParser> Command<C> {
Subcommands::Body { args, .. } => Some(&args.chain),
Subcommands::Rlpx(_) => None,
Subcommands::Bootnode(_) => None,
Subcommands::Enode(_) => None,
}
}
}
@@ -126,6 +131,8 @@ pub enum Subcommands<C: ChainSpecParser> {
Rlpx(rlpx::Command),
/// Bootnode command
Bootnode(bootnode::Command),
/// Print enode identifier
Enode(enode::Command),
}
#[derive(Debug, Clone, Parser)]
@@ -225,4 +232,16 @@ mod tests {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "body", "--chain", "mainnet", "1000"]);
}
#[test]
fn parse_enode_cmd() {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "enode", "/tmp/secret"]);
}
#[test]
fn parse_enode_cmd_with_ip() {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "enode", "/tmp/secret", "--ip", "192.168.1.1"]);
}
}

View File

@@ -15,7 +15,8 @@ use reth_db_common::{
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
use reth_node_core::args::StageEnum;
use reth_provider::{
DBProvider, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter,
DBProvider, RocksDBProviderFactory, StaticFileProviderFactory, StaticFileWriter,
StorageSettingsCache,
};
use reth_prune::PruneSegment;
use reth_stages::StageId;
@@ -90,11 +91,14 @@ impl<C: ChainSpecParser> Command<C> {
StaticFileSegment::AccountChangeSets => {
writer.prune_account_changesets(highest_block)?;
}
StaticFileSegment::StorageChangeSets => {
writer.prune_storage_changesets(highest_block)?;
}
}
}
}
let provider_rw = tool.provider_factory.database_provider_rw()?;
let provider_rw = tool.provider_factory.unwind_provider_rw()?;
let tx = provider_rw.tx_ref();
match self.stage {
@@ -168,8 +172,20 @@ impl<C: ChainSpecParser> Command<C> {
)?;
}
StageEnum::AccountHistory | StageEnum::StorageHistory => {
tx.clear::<tables::AccountsHistory>()?;
tx.clear::<tables::StoragesHistory>()?;
let settings = provider_rw.cached_storage_settings();
let rocksdb = tool.provider_factory.rocksdb_provider();
if settings.account_history_in_rocksdb {
rocksdb.clear::<tables::AccountsHistory>()?;
} else {
tx.clear::<tables::AccountsHistory>()?;
}
if settings.storages_history_in_rocksdb {
rocksdb.clear::<tables::StoragesHistory>()?;
} else {
tx.clear::<tables::StoragesHistory>()?;
}
reset_stage_checkpoint(tx, StageId::IndexAccountHistory)?;
reset_stage_checkpoint(tx, StageId::IndexStorageHistory)?;
@@ -177,7 +193,14 @@ impl<C: ChainSpecParser> Command<C> {
insert_genesis_history(&provider_rw, self.env.chain.genesis().alloc.iter())?;
}
StageEnum::TxLookup => {
tx.clear::<tables::TransactionHashNumbers>()?;
if provider_rw.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
tool.provider_factory
.rocksdb_provider()
.clear::<tables::TransactionHashNumbers>()?;
} else {
tx.clear::<tables::TransactionHashNumbers>()?;
}
reset_prune_checkpoint(tx, PruneSegment::TransactionLookup)?;
reset_stage_checkpoint(tx, StageId::TransactionLookup)?;

View File

@@ -121,7 +121,16 @@ pub fn install() {
unsafe {
let alt_stack_size: usize = min_sigstack_size() + 64 * 1024;
let mut alt_stack: libc::stack_t = mem::zeroed();
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, 1).unwrap()).cast();
// Both SysV AMD64 ABI and aarch64 ABI require 16 bytes alignment. We are going to be
// generous here and just use a size of a page.
let raw_page_sz = libc::sysconf(libc::_SC_PAGESIZE);
let page_sz = if raw_page_sz == -1 {
// Fallback alignment in case sysconf fails.
4096_usize
} else {
raw_page_sz as usize
};
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, page_sz).unwrap()).cast();
alt_stack.ss_size = alt_stack_size;
libc::sigaltstack(&raw const alt_stack, ptr::null_mut());

View File

@@ -438,6 +438,8 @@ pub struct BlocksPerFileConfig {
pub transaction_senders: Option<u64>,
/// Number of blocks per file for the account changesets segment.
pub account_change_sets: Option<u64>,
/// Number of blocks per file for the storage changesets segment.
pub storage_change_sets: Option<u64>,
}
impl StaticFilesConfig {
@@ -451,6 +453,7 @@ impl StaticFilesConfig {
receipts,
transaction_senders,
account_change_sets,
storage_change_sets,
} = self.blocks_per_file;
eyre::ensure!(headers != Some(0), "Headers segment blocks per file must be greater than 0");
eyre::ensure!(
@@ -469,6 +472,10 @@ impl StaticFilesConfig {
account_change_sets != Some(0),
"Account changesets segment blocks per file must be greater than 0"
);
eyre::ensure!(
storage_change_sets != Some(0),
"Storage changesets segment blocks per file must be greater than 0"
);
Ok(())
}
@@ -480,6 +487,7 @@ impl StaticFilesConfig {
receipts,
transaction_senders,
account_change_sets,
storage_change_sets,
} = self.blocks_per_file;
let mut map = StaticFileMap::default();
@@ -492,6 +500,7 @@ impl StaticFilesConfig {
StaticFileSegment::Receipts => receipts,
StaticFileSegment::TransactionSenders => transaction_senders,
StaticFileSegment::AccountChangeSets => account_change_sets,
StaticFileSegment::StorageChangeSets => storage_change_sets,
};
if let Some(blocks_per_file) = blocks_per_file {

View File

@@ -34,6 +34,11 @@ fn default_account_worker_count() -> usize {
/// The size of proof targets chunk to spawn in one multiproof calculation.
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE: usize = 60;
/// The size of proof targets chunk to spawn in one multiproof calculation when V2 proofs are
/// enabled. This is 4x the default chunk size to take advantage of more efficient V2 proof
/// computation.
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2: usize = DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE * 4;
/// Default number of reserved CPU cores for non-reth processes.
///
/// This will be deducted from the thread count of main reth global threadpool.
@@ -137,6 +142,8 @@ pub struct TreeConfig {
account_worker_count: usize,
/// Whether to enable V2 storage proofs.
enable_proof_v2: bool,
/// Whether to disable cache metrics recording (can be expensive with large cached state).
disable_cache_metrics: bool,
}
impl Default for TreeConfig {
@@ -166,6 +173,7 @@ impl Default for TreeConfig {
storage_worker_count: default_storage_worker_count(),
account_worker_count: default_account_worker_count(),
enable_proof_v2: false,
disable_cache_metrics: false,
}
}
}
@@ -198,6 +206,7 @@ impl TreeConfig {
storage_worker_count: usize,
account_worker_count: usize,
enable_proof_v2: bool,
disable_cache_metrics: bool,
) -> Self {
Self {
persistence_threshold,
@@ -224,6 +233,7 @@ impl TreeConfig {
storage_worker_count,
account_worker_count,
enable_proof_v2,
disable_cache_metrics,
}
}
@@ -262,6 +272,17 @@ impl TreeConfig {
self.multiproof_chunk_size
}
/// Return the multiproof task chunk size, using the V2 default if V2 proofs are enabled
/// and the chunk size is at the default value.
pub const fn effective_multiproof_chunk_size(&self) -> usize {
if self.enable_proof_v2 && self.multiproof_chunk_size == DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE
{
DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2
} else {
self.multiproof_chunk_size
}
}
/// Return the number of reserved CPU cores for non-reth processes
pub const fn reserved_cpu_cores(&self) -> usize {
self.reserved_cpu_cores
@@ -516,4 +537,15 @@ impl TreeConfig {
self.enable_proof_v2 = enable_proof_v2;
self
}
/// Returns whether cache metrics recording is disabled.
pub const fn disable_cache_metrics(&self) -> bool {
self.disable_cache_metrics
}
/// Setter for whether to disable cache metrics recording.
pub const fn without_cache_metrics(mut self, disable_cache_metrics: bool) -> Self {
self.disable_cache_metrics = disable_cache_metrics;
self
}
}

View File

@@ -62,7 +62,8 @@ pub trait EngineTypes:
+ TryInto<Self::ExecutionPayloadEnvelopeV2>
+ TryInto<Self::ExecutionPayloadEnvelopeV3>
+ TryInto<Self::ExecutionPayloadEnvelopeV4>
+ TryInto<Self::ExecutionPayloadEnvelopeV5>,
+ TryInto<Self::ExecutionPayloadEnvelopeV5>
+ TryInto<Self::ExecutionPayloadEnvelopeV6>,
> + DeserializeOwned
+ Serialize
{
@@ -106,6 +107,14 @@ pub trait EngineTypes:
+ Send
+ Sync
+ 'static;
/// Execution Payload V6 envelope type.
type ExecutionPayloadEnvelopeV6: DeserializeOwned
+ Serialize
+ Clone
+ Unpin
+ Send
+ Sync
+ 'static;
}
/// Type that validates the payloads processed by the engine API.

View File

@@ -12,7 +12,7 @@ workspace = true
[dependencies]
# reth
reth-chain-state.workspace = true
reth-chain-state = { workspace = true, features = ["rayon"] }
reth-chainspec = { workspace = true, optional = true }
reth-consensus.workspace = true
reth-db.workspace = true

View File

@@ -606,12 +606,21 @@ pub(crate) struct SavedCache {
/// A guard to track in-flight usage of this cache.
/// The cache is considered available if the strong count is 1.
usage_guard: Arc<()>,
/// Whether to skip cache metrics recording (can be expensive with large cached state).
disable_cache_metrics: bool,
}
impl SavedCache {
/// Creates a new instance with the internals
pub(super) fn new(hash: B256, caches: ExecutionCache, metrics: CachedStateMetrics) -> Self {
Self { hash, caches, metrics, usage_guard: Arc::new(()) }
Self { hash, caches, metrics, usage_guard: Arc::new(()), disable_cache_metrics: false }
}
/// Sets whether to disable cache metrics recording.
pub(super) const fn with_disable_cache_metrics(mut self, disable: bool) -> Self {
self.disable_cache_metrics = disable;
self
}
/// Returns the hash for this cache
@@ -619,9 +628,9 @@ impl SavedCache {
self.hash
}
/// Splits the cache into its caches and metrics, consuming it.
pub(crate) fn split(self) -> (ExecutionCache, CachedStateMetrics) {
(self.caches, self.metrics)
/// Splits the cache into its caches, metrics, and `disable_cache_metrics` flag, consuming it.
pub(crate) fn split(self) -> (ExecutionCache, CachedStateMetrics, bool) {
(self.caches, self.metrics, self.disable_cache_metrics)
}
/// Returns true if the cache is available for use (no other tasks are currently using it).
@@ -645,7 +654,13 @@ impl SavedCache {
}
/// Updates the metrics for the [`ExecutionCache`].
///
/// Note: This can be expensive with large cached state as it iterates over
/// all storage entries. Use `with_disable_cache_metrics(true)` to skip.
pub(crate) fn update_metrics(&self) {
if self.disable_cache_metrics {
return;
}
self.metrics.storage_cache_size.set(self.caches.total_storage_slots() as f64);
self.metrics.account_cache_size.set(self.caches.account_cache.entry_count() as f64);
self.metrics.code_cache_size.set(self.caches.code_cache.entry_count() as f64);

View File

@@ -1,25 +1,15 @@
use crate::tree::{error::InsertBlockFatalError, MeteredStateHook, TreeOutcome};
use alloy_consensus::transaction::TxHashRef;
use alloy_evm::{
block::{BlockExecutor, ExecutableTx},
Evm,
};
use crate::tree::{error::InsertBlockFatalError, TreeOutcome};
use alloy_rpc_types_engine::{PayloadStatus, PayloadStatusEnum};
use core::borrow::BorrowMut;
use reth_engine_primitives::{ForkchoiceStatus, OnForkChoiceUpdated};
use reth_errors::{BlockExecutionError, ProviderError};
use reth_evm::{metrics::ExecutorMetrics, OnStateHook};
use reth_errors::ProviderError;
use reth_evm::metrics::ExecutorMetrics;
use reth_execution_types::BlockExecutionOutput;
use reth_metrics::{
metrics::{Counter, Gauge, Histogram},
Metrics,
};
use reth_primitives_traits::SignedTransaction;
use reth_trie::updates::TrieUpdates;
use revm::database::{states::bundle_state::BundleRetention, State};
use revm_primitives::Address;
use std::time::Instant;
use tracing::{debug_span, trace};
use std::time::{Duration, Instant};
/// Metrics for the `EngineApi`.
#[derive(Debug, Default)]
@@ -35,114 +25,24 @@ pub(crate) struct EngineApiMetrics {
}
impl EngineApiMetrics {
/// Helper function for metered execution
fn metered<F, R>(&self, f: F) -> R
where
F: FnOnce() -> (u64, R),
{
// Execute the block and record the elapsed time.
let execute_start = Instant::now();
let (gas_used, output) = f();
let execution_duration = execute_start.elapsed().as_secs_f64();
// Update gas metrics.
self.executor.gas_processed_total.increment(gas_used);
self.executor.gas_per_second.set(gas_used as f64 / execution_duration);
self.executor.gas_used_histogram.record(gas_used as f64);
self.executor.execution_histogram.record(execution_duration);
self.executor.execution_duration.set(execution_duration);
output
}
/// Execute the given block using the provided [`BlockExecutor`] and update metrics for the
/// execution.
/// Records metrics for block execution.
///
/// This method updates metrics for execution time, gas usage, and the number
/// of accounts, storage slots and bytecodes loaded and updated.
///
/// The optional `on_receipt` callback is invoked after each transaction with the receipt
/// index and a reference to all receipts collected so far. This allows callers to stream
/// receipts to a background task for incremental receipt root computation.
pub(crate) fn execute_metered<E, DB, F>(
/// of accounts, storage slots and bytecodes updated.
pub(crate) fn record_block_execution<R>(
&self,
executor: E,
mut transactions: impl Iterator<Item = Result<impl ExecutableTx<E>, BlockExecutionError>>,
transaction_count: usize,
state_hook: Box<dyn OnStateHook>,
mut on_receipt: F,
) -> Result<(BlockExecutionOutput<E::Receipt>, Vec<Address>), BlockExecutionError>
where
DB: alloy_evm::Database,
E: BlockExecutor<Evm: Evm<DB: BorrowMut<State<DB>>>, Transaction: SignedTransaction>,
F: FnMut(&[E::Receipt]),
{
// clone here is cheap, all the metrics are Option<Arc<_>>. additionally
// they are globally registered so that the data recorded in the hook will
// be accessible.
let wrapper = MeteredStateHook { metrics: self.executor.clone(), inner_hook: state_hook };
output: &BlockExecutionOutput<R>,
execution_duration: Duration,
) {
let execution_secs = execution_duration.as_secs_f64();
let gas_used = output.result.gas_used;
let mut senders = Vec::with_capacity(transaction_count);
let mut executor = executor.with_state_hook(Some(Box::new(wrapper)));
let f = || {
let start = Instant::now();
debug_span!(target: "engine::tree", "pre execution")
.entered()
.in_scope(|| executor.apply_pre_execution_changes())?;
self.executor.pre_execution_histogram.record(start.elapsed());
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
loop {
let start = Instant::now();
let Some(tx) = transactions.next() else { break };
self.executor.transaction_wait_histogram.record(start.elapsed());
let tx = tx?;
senders.push(*tx.signer());
let span = debug_span!(
target: "engine::tree",
"execute tx",
tx_hash = ?tx.tx().tx_hash(),
gas_used = tracing::field::Empty,
);
let enter = span.entered();
trace!(target: "engine::tree", "Executing transaction");
let start = Instant::now();
let gas_used = executor.execute_transaction(tx)?;
self.executor.transaction_execution_histogram.record(start.elapsed());
// Invoke callback with the latest receipt
on_receipt(executor.receipts());
// record the tx gas used
enter.record("gas_used", gas_used);
}
drop(exec_span);
let start = Instant::now();
let result = debug_span!(target: "engine::tree", "finish")
.entered()
.in_scope(|| executor.finish())
.map(|(evm, result)| (evm.into_db(), result));
self.executor.post_execution_histogram.record(start.elapsed());
result
};
// Use metered to execute and track timing/gas metrics
let (mut db, result) = self.metered(|| {
let res = f();
let gas_used = res.as_ref().map(|r| r.1.gas_used).unwrap_or(0);
(gas_used, res)
})?;
// merge transitions into bundle state
debug_span!(target: "engine::tree", "merge transitions")
.entered()
.in_scope(|| db.borrow_mut().merge_transitions(BundleRetention::Reverts));
let output = BlockExecutionOutput { result, state: db.borrow_mut().take_bundle() };
// Update gas metrics
self.executor.gas_processed_total.increment(gas_used);
self.executor.gas_per_second.set(gas_used as f64 / execution_secs);
self.executor.gas_used_histogram.record(gas_used as f64);
self.executor.execution_histogram.record(execution_secs);
self.executor.execution_duration.set(execution_secs);
// Update the metrics for the number of accounts, storage slots and bytecodes updated
let accounts = output.state.state.len();
@@ -153,8 +53,31 @@ impl EngineApiMetrics {
self.executor.accounts_updated_histogram.record(accounts as f64);
self.executor.storage_slots_updated_histogram.record(storage_slots as f64);
self.executor.bytecodes_updated_histogram.record(bytecodes as f64);
}
Ok((output, senders))
/// Returns a reference to the executor metrics for use in state hooks.
pub(crate) const fn executor_metrics(&self) -> &ExecutorMetrics {
&self.executor
}
/// Records the duration of block pre-execution changes (e.g., beacon root update).
pub(crate) fn record_pre_execution(&self, elapsed: Duration) {
self.executor.pre_execution_histogram.record(elapsed);
}
/// Records the duration of block post-execution changes (e.g., finalization).
pub(crate) fn record_post_execution(&self, elapsed: Duration) {
self.executor.post_execution_histogram.record(elapsed);
}
/// Records the time spent waiting for the next transaction from the iterator.
pub(crate) fn record_transaction_wait(&self, elapsed: Duration) {
self.executor.transaction_wait_histogram.record(elapsed);
}
/// Records the duration of a single transaction execution.
pub(crate) fn record_transaction_execution(&self, elapsed: Duration) {
self.executor.transaction_execution_histogram.record(elapsed);
}
}
@@ -210,6 +133,12 @@ pub(crate) struct EngineMetrics {
#[derive(Metrics)]
#[metrics(scope = "consensus.engine.beacon")]
pub(crate) struct ForkchoiceUpdatedMetrics {
/// Finish time of the latest forkchoice updated call.
#[metric(skip)]
pub(crate) latest_finish_at: Option<Instant>,
/// Start time of the latest forkchoice updated call.
#[metric(skip)]
pub(crate) latest_start_at: Option<Instant>,
/// The total count of forkchoice updated messages received.
pub(crate) forkchoice_updated_messages: Counter,
/// The total count of forkchoice updated messages with payload received.
@@ -232,18 +161,35 @@ pub(crate) struct ForkchoiceUpdatedMetrics {
pub(crate) forkchoice_updated_last: Gauge,
/// Time diff between new payload call response and the next forkchoice updated call request.
pub(crate) new_payload_forkchoice_updated_time_diff: Histogram,
/// Time from previous forkchoice updated finish to current forkchoice updated start (idle
/// time).
pub(crate) time_between_forkchoice_updated: Histogram,
/// Time from previous forkchoice updated start to current forkchoice updated start (total
/// interval).
pub(crate) forkchoice_updated_interval: Histogram,
}
impl ForkchoiceUpdatedMetrics {
/// Increment the forkchoiceUpdated counter based on the given result
pub(crate) fn update_response_metrics(
&self,
&mut self,
start: Instant,
latest_new_payload_at: &mut Option<Instant>,
has_attrs: bool,
result: &Result<TreeOutcome<OnForkChoiceUpdated>, ProviderError>,
) {
let elapsed = start.elapsed();
let finish = Instant::now();
let elapsed = finish - start;
if let Some(prev_finish) = self.latest_finish_at {
self.time_between_forkchoice_updated.record(start - prev_finish);
}
if let Some(prev_start) = self.latest_start_at {
self.forkchoice_updated_interval.record(start - prev_start);
}
self.latest_finish_at = Some(finish);
self.latest_start_at = Some(start);
match result {
Ok(outcome) => match outcome.outcome.forkchoice_status() {
ForkchoiceStatus::Valid => self.forkchoice_updated_valid.increment(1),
@@ -410,138 +356,10 @@ pub(crate) struct BlockBufferMetrics {
mod tests {
use super::*;
use alloy_eips::eip7685::Requests;
use alloy_evm::block::StateChangeSource;
use alloy_primitives::{B256, U256};
use metrics_util::debugging::{DebuggingRecorder, Snapshotter};
use reth_ethereum_primitives::{Receipt, TransactionSigned};
use reth_evm_ethereum::EthEvm;
use reth_ethereum_primitives::Receipt;
use reth_execution_types::BlockExecutionResult;
use reth_primitives_traits::RecoveredBlock;
use revm::{
context::result::{ExecutionResult, Output, ResultAndState, SuccessReason},
database::State,
database_interface::EmptyDB,
inspector::NoOpInspector,
state::{Account, AccountInfo, AccountStatus, EvmState, EvmStorage, EvmStorageSlot},
Context, MainBuilder, MainContext,
};
use revm_primitives::Bytes;
use std::sync::mpsc;
/// A simple mock executor for testing that doesn't require complex EVM setup
struct MockExecutor {
state: EvmState,
receipts: Vec<Receipt>,
hook: Option<Box<dyn OnStateHook>>,
}
impl MockExecutor {
fn new(state: EvmState) -> Self {
Self { state, receipts: vec![], hook: None }
}
}
// Mock Evm type for testing
type MockEvm = EthEvm<State<EmptyDB>, NoOpInspector>;
impl BlockExecutor for MockExecutor {
type Transaction = TransactionSigned;
type Receipt = Receipt;
type Evm = MockEvm;
fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> {
Ok(())
}
fn execute_transaction_without_commit(
&mut self,
_tx: impl ExecutableTx<Self>,
) -> Result<ResultAndState<<Self::Evm as Evm>::HaltReason>, BlockExecutionError> {
// Call hook with our mock state for each transaction
if let Some(hook) = self.hook.as_mut() {
hook.on_state(StateChangeSource::Transaction(0), &self.state);
}
Ok(ResultAndState::new(
ExecutionResult::Success {
reason: SuccessReason::Return,
gas_used: 1000, // Mock gas used
gas_refunded: 0,
logs: vec![],
output: Output::Call(Bytes::from(vec![])),
},
Default::default(),
))
}
fn commit_transaction(
&mut self,
_output: ResultAndState<<Self::Evm as Evm>::HaltReason>,
_tx: impl ExecutableTx<Self>,
) -> Result<u64, BlockExecutionError> {
Ok(1000)
}
fn finish(
self,
) -> Result<(Self::Evm, BlockExecutionResult<Self::Receipt>), BlockExecutionError> {
let Self { hook, state, .. } = self;
// Call hook with our mock state
if let Some(mut hook) = hook {
hook.on_state(StateChangeSource::Transaction(0), &state);
}
// Create a mock EVM
let db = State::builder()
.with_database(EmptyDB::default())
.with_bundle_update()
.without_state_clear()
.build();
let evm = EthEvm::new(
Context::mainnet().with_db(db).build_mainnet_with_inspector(NoOpInspector {}),
false,
);
// Return successful result like the original tests
Ok((
evm,
BlockExecutionResult {
receipts: vec![],
requests: Requests::default(),
gas_used: 1000,
blob_gas_used: 0,
},
))
}
fn set_state_hook(&mut self, hook: Option<Box<dyn OnStateHook>>) {
self.hook = hook;
}
fn evm_mut(&mut self) -> &mut Self::Evm {
panic!("Mock executor evm_mut() not implemented")
}
fn evm(&self) -> &Self::Evm {
panic!("Mock executor evm() not implemented")
}
fn receipts(&self) -> &[Self::Receipt] {
&self.receipts
}
}
struct ChannelStateHook {
output: i32,
sender: mpsc::Sender<i32>,
}
impl OnStateHook for ChannelStateHook {
fn on_state(&mut self, _source: StateChangeSource, _state: &EvmState) {
let _ = self.sender.send(self.output);
}
}
use reth_revm::db::BundleState;
fn setup_test_recorder() -> Snapshotter {
let recorder = DebuggingRecorder::new();
@@ -551,38 +369,7 @@ mod tests {
}
#[test]
fn test_executor_metrics_hook_called() {
let metrics = EngineApiMetrics::default();
let input = RecoveredBlock::<reth_ethereum_primitives::Block>::default();
let (tx, rx) = mpsc::channel();
let expected_output = 42;
let state_hook = Box::new(ChannelStateHook { sender: tx, output: expected_output });
let state = EvmState::default();
let executor = MockExecutor::new(state);
// This will fail to create the EVM but should still call the hook
let _result = metrics.execute_metered::<_, EmptyDB, _>(
executor,
input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>),
input.transaction_count(),
state_hook,
|_| {},
);
// Check if hook was called (it might not be if finish() fails early)
match rx.try_recv() {
Ok(actual_output) => assert_eq!(actual_output, expected_output),
Err(_) => {
// Hook wasn't called, which is expected if the mock fails early
// The test still validates that the code compiles and runs
}
}
}
#[test]
fn test_executor_metrics_hook_metrics_recorded() {
fn test_record_block_execution_metrics() {
let snapshotter = setup_test_recorder();
let metrics = EngineApiMetrics::default();
@@ -591,45 +378,17 @@ mod tests {
metrics.executor.gas_per_second.set(0.0);
metrics.executor.gas_used_histogram.record(0.0);
let input = RecoveredBlock::<reth_ethereum_primitives::Block>::default();
let (tx, _rx) = mpsc::channel();
let state_hook = Box::new(ChannelStateHook { sender: tx, output: 42 });
// Create a state with some data
let state = {
let mut state = EvmState::default();
let storage =
EvmStorage::from_iter([(U256::from(1), EvmStorageSlot::new(U256::from(2), 0))]);
state.insert(
Default::default(),
Account {
info: AccountInfo {
balance: U256::from(100),
nonce: 10,
code_hash: B256::random(),
code: Default::default(),
account_id: None,
},
original_info: Box::new(AccountInfo::default()),
storage,
status: AccountStatus::default(),
transaction_id: 0,
},
);
state
let output = BlockExecutionOutput::<Receipt> {
state: BundleState::default(),
result: BlockExecutionResult {
receipts: vec![],
requests: Requests::default(),
gas_used: 21000,
blob_gas_used: 0,
},
};
let executor = MockExecutor::new(state);
// Execute (will fail but should still update some metrics)
let _result = metrics.execute_metered::<_, EmptyDB, _>(
executor,
input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>),
input.transaction_count(),
state_hook,
|_| {},
);
metrics.record_block_execution(&output, Duration::from_millis(100));
let snapshot = snapshotter.snapshot().into_vec();

View File

@@ -32,7 +32,8 @@ use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedBlock, Sealed
use reth_provider::{
BlockExecutionOutput, BlockExecutionResult, BlockNumReader, BlockReader, ChangeSetReader,
DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader,
StateProviderBox, StateProviderFactory, StateReader, TransactionVariant,
StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader,
TransactionVariant,
};
use reth_revm::database::StateProviderDatabase;
use reth_stages_api::ControlFlow;
@@ -84,6 +85,12 @@ pub mod state;
/// backfill this gap.
pub(crate) const MIN_BLOCKS_FOR_PIPELINE_RUN: u64 = EPOCH_SLOTS;
/// The minimum number of blocks to retain in the changeset cache after eviction.
///
/// This ensures that recent trie changesets are kept in memory for potential reorgs,
/// even when the finalized block is not set (e.g., on L2s like Optimism).
const CHANGESET_CACHE_RETENTION_BLOCKS: u64 = 64;
/// A builder for creating state providers that can be used across threads.
#[derive(Clone, Debug)]
pub struct StateProviderBuilder<N: NodePrimitives, P> {
@@ -317,6 +324,7 @@ where
<P as DatabaseProviderFactory>::Provider: BlockReader<Block = N::Block, Header = N::BlockHeader>
+ StageCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader,
C: ConfigureEvm<Primitives = N> + 'static,
T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>,
@@ -1376,19 +1384,27 @@ where
debug!(target: "engine::tree", ?last_persisted_block_hash, ?last_persisted_block_number, elapsed=?start_time.elapsed(), "Finished persisting, calling finish");
self.persistence_state.finish(last_persisted_block_hash, last_persisted_block_number);
// Evict trie changesets for blocks below the finalized block, but keep at least 64 blocks
if let Some(finalized) = self.canonical_in_memory_state.get_finalized_num_hash() {
let min_threshold = last_persisted_block_number.saturating_sub(64);
let eviction_threshold = finalized.number.min(min_threshold);
debug!(
target: "engine::tree",
last_persisted = last_persisted_block_number,
finalized_number = finalized.number,
eviction_threshold,
"Evicting changesets below threshold"
);
self.changeset_cache.evict(eviction_threshold);
}
// Evict trie changesets for blocks below the eviction threshold.
// Keep at least CHANGESET_CACHE_RETENTION_BLOCKS from the persisted tip, and also respect
// the finalized block if set.
let min_threshold =
last_persisted_block_number.saturating_sub(CHANGESET_CACHE_RETENTION_BLOCKS);
let eviction_threshold =
if let Some(finalized) = self.canonical_in_memory_state.get_finalized_num_hash() {
// Use the minimum of finalized block and retention threshold to be conservative
finalized.number.min(min_threshold)
} else {
// When finalized is not set (e.g., on L2s), use the retention threshold
min_threshold
};
debug!(
target: "engine::tree",
last_persisted = last_persisted_block_number,
finalized_number = ?self.canonical_in_memory_state.get_finalized_num_hash().map(|f| f.number),
eviction_threshold,
"Evicting changesets below threshold"
);
self.changeset_cache.evict(eviction_threshold);
self.on_new_persisted_block()?;
Ok(())

View File

@@ -15,19 +15,22 @@ use crate::tree::{
};
use alloy_eip7928::BlockAccessList;
use alloy_eips::eip1898::BlockWithParent;
use alloy_evm::{block::StateChangeSource, ToTxEnv};
use alloy_evm::block::StateChangeSource;
use alloy_primitives::B256;
use crossbeam_channel::Sender as CrossbeamSender;
use executor::WorkloadExecutor;
use metrics::Counter;
use multiproof::{SparseTrieUpdate, *};
use parking_lot::RwLock;
use prewarm::PrewarmMetrics;
use rayon::prelude::*;
use reth_evm::{
block::ExecutableTxParts,
execute::{ExecutableTxFor, WithTxEnv},
ConfigureEvm, EvmEnvFor, ExecutableTxIterator, ExecutableTxTuple, OnStateHook, SpecFor,
TxEnvFor,
};
use reth_metrics::Metrics;
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
BlockExecutionOutput, BlockReader, DatabaseProviderROFactory, StateProvider,
@@ -99,7 +102,7 @@ pub const SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY: usize = 1_000_000;
/// Type alias for [`PayloadHandle`] returned by payload processor spawn methods.
type IteratorPayloadHandle<Evm, I, N> = PayloadHandle<
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxTuple>::Tx>,
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxIterator<Evm>>::Recovered>,
<I as ExecutableTxTuple>::Error,
<N as NodePrimitives>::Receipt,
>;
@@ -139,6 +142,8 @@ where
disable_parallel_sparse_trie: bool,
/// Maximum concurrency for prewarm task.
prewarm_max_concurrency: usize,
/// Whether to disable cache metrics recording.
disable_cache_metrics: bool,
}
impl<N, Evm> PayloadProcessor<Evm>
@@ -171,6 +176,7 @@ where
sparse_state_trie: Arc::default(),
disable_parallel_sparse_trie: config.disable_parallel_sparse_trie(),
prewarm_max_concurrency: config.prewarm_max_concurrency(),
disable_cache_metrics: config.disable_cache_metrics(),
}
}
}
@@ -242,6 +248,9 @@ where
let (to_sparse_trie, sparse_trie_rx) = channel();
let (to_multi_proof, from_multi_proof) = crossbeam_channel::unbounded();
// Extract V2 proofs flag early so we can pass it to prewarm
let v2_proofs_enabled = config.enable_proof_v2();
// Handle BAL-based optimization if available
let prewarm_handle = if let Some(bal) = bal {
// When BAL is present, use BAL prewarming and send BAL to multiproof
@@ -258,6 +267,7 @@ where
provider_builder.clone(),
None, // Don't send proof targets when BAL is present
Some(bal),
v2_proofs_enabled,
)
} else {
// Normal path: spawn with transaction prewarming
@@ -268,6 +278,7 @@ where
provider_builder.clone(),
Some(to_multi_proof.clone()),
None,
v2_proofs_enabled,
)
};
@@ -275,7 +286,6 @@ where
let task_ctx = ProofTaskCtx::new(multiproof_provider_factory);
let storage_worker_count = config.storage_worker_count();
let account_worker_count = config.account_worker_count();
let v2_proofs_enabled = config.enable_proof_v2();
let proof_handle = ProofWorkerHandle::new(
self.executor.handle().clone(),
task_ctx,
@@ -287,10 +297,13 @@ where
let multi_proof_task = MultiProofTask::new(
proof_handle.clone(),
to_sparse_trie,
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size()),
config
.multiproof_chunking_enabled()
.then_some(config.effective_multiproof_chunk_size()),
to_multi_proof.clone(),
from_multi_proof,
);
)
.with_v2_proofs_enabled(v2_proofs_enabled);
// spawn multi-proof task
let parent_span = span.clone();
@@ -300,7 +313,7 @@ where
// Build a state provider for the multiproof task
let provider = provider_builder.build().expect("failed to build provider");
let provider = if let Some(saved_cache) = saved_cache {
let (cache, metrics) = saved_cache.split();
let (cache, metrics, _) = saved_cache.split();
Box::new(CachedStateProvider::new(provider, cache, metrics))
as Box<dyn StateProvider>
} else {
@@ -339,8 +352,9 @@ where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
{
let (prewarm_rx, execution_rx, size_hint) = self.spawn_tx_iterator(transactions);
// This path doesn't use multiproof, so V2 proofs flag doesn't matter
let prewarm_handle =
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal);
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal, false);
PayloadHandle {
to_multi_proof: None,
prewarm_handle,
@@ -356,8 +370,8 @@ where
&self,
transactions: I,
) -> (
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Tx>>,
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Tx>, I::Error>>,
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Recovered>>,
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Recovered>, I::Error>>,
usize,
) {
let (transactions, convert) = transactions.into();
@@ -372,7 +386,10 @@ where
self.executor.spawn_blocking(move || {
transactions.enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
let tx = convert(tx);
let tx = tx.map(|tx| WithTxEnv { tx_env: tx.to_tx_env(), tx: Arc::new(tx) });
let tx = tx.map(|tx| {
let (tx_env, tx) = tx.into_parts();
WithTxEnv { tx_env, tx: Arc::new(tx) }
});
// Only send Ok(_) variants to prewarming task.
if let Ok(tx) = &tx {
let _ = prewarm_tx.send(tx.clone());
@@ -407,6 +424,7 @@ where
}
/// Spawn prewarming optionally wired to the multiproof task for target updates.
#[expect(clippy::too_many_arguments)]
fn spawn_caching_with<P>(
&self,
env: ExecutionEnv<Evm>,
@@ -415,6 +433,7 @@ where
provider_builder: StateProviderBuilder<N, P>,
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
bal: Option<Arc<BlockAccessList>>,
v2_proofs_enabled: bool,
) -> CacheTaskHandle<N::Receipt>
where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
@@ -437,6 +456,7 @@ where
terminate_execution: Arc::new(AtomicBool::new(false)),
precompile_cache_disabled: self.precompile_cache_disabled,
precompile_cache_map: self.precompile_cache_map.clone(),
v2_proofs_enabled,
};
let (prewarm_task, to_prewarm_task) = PrewarmCacheTask::new(
@@ -477,6 +497,7 @@ where
debug!("creating new execution cache on cache miss");
let cache = ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size);
SavedCache::new(parent_hash, cache, CachedStateMetrics::zeroed())
.with_disable_cache_metrics(self.disable_cache_metrics)
}
}
@@ -558,6 +579,7 @@ where
block_with_parent: BlockWithParent,
bundle_state: &BundleState,
) {
let disable_cache_metrics = self.disable_cache_metrics;
self.execution_cache.update_with_guard(|cached| {
if cached.as_ref().is_some_and(|c| c.executed_block_hash() != block_with_parent.parent) {
debug!(
@@ -571,7 +593,8 @@ where
// Take existing cache (if any) or create fresh caches
let (caches, cache_metrics) = match cached.take() {
Some(existing) => {
existing.split()
let (c, m, _) = existing.split();
(c, m)
}
None => (
ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size),
@@ -580,7 +603,8 @@ where
};
// Insert the block's bundle state into cache
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics);
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics)
.with_disable_cache_metrics(disable_cache_metrics);
if new_cache.cache().insert_state(bundle_state).is_err() {
*cached = None;
debug!(target: "engine::caching", "cleared execution cache on update error");
@@ -770,6 +794,8 @@ impl<R> Drop for CacheTaskHandle<R> {
struct ExecutionCache {
/// Guarded cloneable cache identified by a block hash.
inner: Arc<RwLock<Option<SavedCache>>>,
/// Metrics for cache operations.
metrics: ExecutionCacheMetrics,
}
impl ExecutionCache {
@@ -811,6 +837,10 @@ impl ExecutionCache {
if hash_matches && available {
return Some(c.clone());
}
if hash_matches && !available {
self.metrics.execution_cache_in_use.increment(1);
}
} else {
debug!(target: "engine::caching", %parent_hash, "No cache found");
}
@@ -846,6 +876,15 @@ impl ExecutionCache {
}
}
/// Metrics for execution cache operations.
#[derive(Metrics, Clone)]
#[metrics(scope = "consensus.engine.beacon")]
pub(crate) struct ExecutionCacheMetrics {
/// Counter for when the execution cache was unavailable because other threads
/// (e.g., prewarming) are still using it.
pub(crate) execution_cache_in_use: Counter,
}
/// EVM context required to execute a block.
#[derive(Debug, Clone)]
pub struct ExecutionEnv<Evm: ConfigureEvm> {

View File

@@ -11,14 +11,18 @@ use reth_metrics::Metrics;
use reth_provider::AccountReader;
use reth_revm::state::EvmState;
use reth_trie::{
added_removed_keys::MultiAddedRemovedKeys, DecodedMultiProof, HashedPostState, HashedStorage,
added_removed_keys::MultiAddedRemovedKeys, proof_v2, HashedPostState, HashedStorage,
MultiProofTargets,
};
#[cfg(test)]
use reth_trie_parallel::stats::ParallelTrieTracker;
use reth_trie_parallel::{
proof::ParallelProof,
proof_task::{
AccountMultiproofInput, ProofResultContext, ProofResultMessage, ProofWorkerHandle,
AccountMultiproofInput, ProofResult, ProofResultContext, ProofResultMessage,
ProofWorkerHandle,
},
targets_v2::{ChunkedMultiProofTargetsV2, MultiProofTargetsV2},
};
use revm_primitives::map::{hash_map, B256Map};
use std::{collections::BTreeMap, sync::Arc, time::Instant};
@@ -63,12 +67,12 @@ const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300;
/// A trie update that can be applied to sparse trie alongside the proofs for touched parts of the
/// state.
#[derive(Default, Debug)]
#[derive(Debug)]
pub struct SparseTrieUpdate {
/// The state update that was used to calculate the proof
pub(crate) state: HashedPostState,
/// The calculated multiproof
pub(crate) multiproof: DecodedMultiProof,
pub(crate) multiproof: ProofResult,
}
impl SparseTrieUpdate {
@@ -80,7 +84,11 @@ impl SparseTrieUpdate {
/// Construct update from multiproof.
#[cfg(test)]
pub(super) fn from_multiproof(multiproof: reth_trie::MultiProof) -> alloy_rlp::Result<Self> {
Ok(Self { multiproof: multiproof.try_into()?, ..Default::default() })
let stats = ParallelTrieTracker::default().finish();
Ok(Self {
state: HashedPostState::default(),
multiproof: ProofResult::Legacy(multiproof.try_into()?, stats),
})
}
/// Extend update with contents of the other.
@@ -94,7 +102,7 @@ impl SparseTrieUpdate {
#[derive(Debug)]
pub(super) enum MultiProofMessage {
/// Prefetch proof targets
PrefetchProofs(MultiProofTargets),
PrefetchProofs(VersionedMultiProofTargets),
/// New state update from transaction execution with its source
StateUpdate(Source, EvmState),
/// State update that can be applied to the sparse trie without any new proofs.
@@ -223,12 +231,155 @@ pub(crate) fn evm_state_to_hashed_post_state(update: EvmState) -> HashedPostStat
hashed_state
}
/// Extends a `MultiProofTargets` with the contents of a `VersionedMultiProofTargets`,
/// regardless of which variant the latter is.
fn extend_multiproof_targets(dest: &mut MultiProofTargets, src: &VersionedMultiProofTargets) {
match src {
VersionedMultiProofTargets::Legacy(targets) => {
dest.extend_ref(targets);
}
VersionedMultiProofTargets::V2(targets) => {
// Add all account targets
for target in &targets.account_targets {
dest.entry(target.key()).or_default();
}
// Add all storage targets
for (hashed_address, slots) in &targets.storage_targets {
let slot_set = dest.entry(*hashed_address).or_default();
for slot in slots {
slot_set.insert(slot.key());
}
}
}
}
}
/// A set of multiproof targets which can be either in the legacy or V2 representations.
#[derive(Debug)]
pub(super) enum VersionedMultiProofTargets {
/// Legacy targets
Legacy(MultiProofTargets),
/// V2 targets
V2(MultiProofTargetsV2),
}
impl VersionedMultiProofTargets {
/// Returns true if there are no account or storage targets.
fn is_empty(&self) -> bool {
match self {
Self::Legacy(targets) => targets.is_empty(),
Self::V2(targets) => targets.is_empty(),
}
}
/// Returns the number of account targets in the multiproof target
fn account_targets_len(&self) -> usize {
match self {
Self::Legacy(targets) => targets.len(),
Self::V2(targets) => targets.account_targets.len(),
}
}
/// Returns the number of storage targets in the multiproof target
fn storage_targets_len(&self) -> usize {
match self {
Self::Legacy(targets) => targets.values().map(|slots| slots.len()).sum::<usize>(),
Self::V2(targets) => {
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
}
}
}
/// Returns the number of accounts in the multiproof targets.
fn len(&self) -> usize {
match self {
Self::Legacy(targets) => targets.len(),
Self::V2(targets) => targets.account_targets.len(),
}
}
/// Returns the total storage slot count across all accounts.
fn storage_count(&self) -> usize {
match self {
Self::Legacy(targets) => targets.values().map(|slots| slots.len()).sum(),
Self::V2(targets) => targets.storage_targets.values().map(|slots| slots.len()).sum(),
}
}
/// Returns the number of items that will be considered during chunking.
fn chunking_length(&self) -> usize {
match self {
Self::Legacy(targets) => targets.chunking_length(),
Self::V2(targets) => {
// For V2, count accounts + storage slots
targets.account_targets.len() +
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
}
}
}
/// Retains the targets representing the difference with another `MultiProofTargets`.
/// Removes all targets that are already present in `other`.
fn retain_difference(&mut self, other: &MultiProofTargets) {
match self {
Self::Legacy(targets) => {
targets.retain_difference(other);
}
Self::V2(targets) => {
// Remove account targets that exist in other
targets.account_targets.retain(|target| !other.contains_key(&target.key()));
// For each account in storage_targets, remove slots that exist in other
targets.storage_targets.retain(|hashed_address, slots| {
if let Some(other_slots) = other.get(hashed_address) {
slots.retain(|slot| !other_slots.contains(&slot.key()));
!slots.is_empty()
} else {
true
}
});
}
}
}
/// Extends this `VersionedMultiProofTargets` with the contents of another.
///
/// Panics if the variants do not match.
fn extend(&mut self, other: Self) {
match (self, other) {
(Self::Legacy(dest), Self::Legacy(src)) => {
dest.extend(src);
}
(Self::V2(dest), Self::V2(src)) => {
dest.account_targets.extend(src.account_targets);
for (addr, slots) in src.storage_targets {
dest.storage_targets.entry(addr).or_default().extend(slots);
}
}
_ => panic!("Cannot extend VersionedMultiProofTargets with mismatched variants"),
}
}
/// Chunks this `VersionedMultiProofTargets` into smaller chunks of the given size.
fn chunks(self, chunk_size: usize) -> Box<dyn Iterator<Item = Self>> {
match self {
Self::Legacy(targets) => {
Box::new(MultiProofTargets::chunks(targets, chunk_size).map(Self::Legacy))
}
Self::V2(targets) => {
Box::new(ChunkedMultiProofTargetsV2::new(targets, chunk_size).map(Self::V2))
}
}
}
}
/// Input parameters for dispatching a multiproof calculation.
#[derive(Debug)]
struct MultiproofInput {
source: Option<Source>,
hashed_state_update: HashedPostState,
proof_targets: MultiProofTargets,
proof_targets: VersionedMultiProofTargets,
proof_sequence_number: u64,
state_root_message_sender: CrossbeamSender<MultiProofMessage>,
multi_added_removed_keys: Option<Arc<MultiAddedRemovedKeys>>,
@@ -263,8 +414,6 @@ pub struct MultiproofManager {
proof_result_tx: CrossbeamSender<ProofResultMessage>,
/// Metrics
metrics: MultiProofTaskMetrics,
/// Whether to use V2 storage proofs
v2_proofs_enabled: bool,
}
impl MultiproofManager {
@@ -278,9 +427,7 @@ impl MultiproofManager {
metrics.max_storage_workers.set(proof_worker_handle.total_storage_workers() as f64);
metrics.max_account_workers.set(proof_worker_handle.total_account_workers() as f64);
let v2_proofs_enabled = proof_worker_handle.v2_proofs_enabled();
Self { metrics, proof_worker_handle, proof_result_tx, v2_proofs_enabled }
Self { metrics, proof_worker_handle, proof_result_tx }
}
/// Dispatches a new multiproof calculation to worker pools.
@@ -325,41 +472,48 @@ impl MultiproofManager {
multi_added_removed_keys,
} = multiproof_input;
let account_targets = proof_targets.len();
let storage_targets = proof_targets.values().map(|slots| slots.len()).sum::<usize>();
trace!(
target: "engine::tree::payload_processor::multiproof",
proof_sequence_number,
?proof_targets,
account_targets,
storage_targets,
account_targets = proof_targets.account_targets_len(),
storage_targets = proof_targets.storage_targets_len(),
?source,
"Dispatching multiproof to workers"
);
let start = Instant::now();
// Extend prefix sets with targets
let frozen_prefix_sets =
ParallelProof::extend_prefix_sets_with_targets(&Default::default(), &proof_targets);
// Workers will send ProofResultMessage directly to proof_result_rx
let proof_result_sender = ProofResultContext::new(
self.proof_result_tx.clone(),
proof_sequence_number,
hashed_state_update,
start,
);
// Dispatch account multiproof to worker pool with result sender
let input = AccountMultiproofInput {
targets: proof_targets,
prefix_sets: frozen_prefix_sets,
collect_branch_node_masks: true,
multi_added_removed_keys,
// Workers will send ProofResultMessage directly to proof_result_rx
proof_result_sender: ProofResultContext::new(
self.proof_result_tx.clone(),
proof_sequence_number,
hashed_state_update,
start,
),
v2_proofs_enabled: self.v2_proofs_enabled,
let input = match proof_targets {
VersionedMultiProofTargets::Legacy(proof_targets) => {
// Extend prefix sets with targets
let frozen_prefix_sets = ParallelProof::extend_prefix_sets_with_targets(
&Default::default(),
&proof_targets,
);
AccountMultiproofInput::Legacy {
targets: proof_targets,
prefix_sets: frozen_prefix_sets,
collect_branch_node_masks: true,
multi_added_removed_keys,
proof_result_sender,
}
}
VersionedMultiProofTargets::V2(proof_targets) => {
AccountMultiproofInput::V2 { targets: proof_targets, proof_result_sender }
}
};
// Dispatch account multiproof to worker pool with result sender
if let Err(e) = self.proof_worker_handle.dispatch_account_multiproof(input) {
error!(target: "engine::tree::payload_processor::multiproof", ?e, "Failed to dispatch account multiproof");
return;
@@ -561,6 +715,9 @@ pub(super) struct MultiProofTask {
/// there are any active workers and force chunking across workers. This is to prevent tasks
/// which are very long from hitting a single worker.
max_targets_for_chunking: usize,
/// Whether or not V2 proof calculation is enabled. If enabled then [`MultiProofTargetsV2`]
/// will be produced by state updates.
v2_proofs_enabled: bool,
}
impl MultiProofTask {
@@ -592,9 +749,16 @@ impl MultiProofTask {
),
metrics,
max_targets_for_chunking: DEFAULT_MAX_TARGETS_FOR_CHUNKING,
v2_proofs_enabled: false,
}
}
/// Enables V2 proof target generation on state updates.
pub(super) const fn with_v2_proofs_enabled(mut self, v2_proofs_enabled: bool) -> Self {
self.v2_proofs_enabled = v2_proofs_enabled;
self
}
/// Handles request for proof prefetch.
///
/// Returns how many multiproof tasks were dispatched for the prefetch request.
@@ -602,37 +766,29 @@ impl MultiProofTask {
level = "debug",
target = "engine::tree::payload_processor::multiproof",
skip_all,
fields(accounts = targets.len(), chunks = 0)
fields(accounts = targets.account_targets_len(), chunks = 0)
)]
fn on_prefetch_proof(&mut self, mut targets: MultiProofTargets) -> u64 {
fn on_prefetch_proof(&mut self, mut targets: VersionedMultiProofTargets) -> u64 {
// Remove already fetched proof targets to avoid redundant work.
targets.retain_difference(&self.fetched_proof_targets);
self.fetched_proof_targets.extend_ref(&targets);
extend_multiproof_targets(&mut self.fetched_proof_targets, &targets);
// Make sure all target accounts have an `AddedRemovedKeySet` in the
// For Legacy multiproofs, make sure all target accounts have an `AddedRemovedKeySet` in the
// [`MultiAddedRemovedKeys`]. Even if there are not any known removed keys for the account,
// we still want to optimistically fetch extension children for the leaf addition case.
self.multi_added_removed_keys.touch_accounts(targets.keys().copied());
// Clone+Arc MultiAddedRemovedKeys for sharing with the dispatched multiproof tasks
let multi_added_removed_keys = Arc::new(MultiAddedRemovedKeys {
account: self.multi_added_removed_keys.account.clone(),
storages: targets
.keys()
.filter_map(|account| {
self.multi_added_removed_keys
.storages
.get(account)
.cloned()
.map(|keys| (*account, keys))
})
.collect(),
});
// V2 multiproofs don't need this.
let multi_added_removed_keys =
if let VersionedMultiProofTargets::Legacy(legacy_targets) = &targets {
self.multi_added_removed_keys.touch_accounts(legacy_targets.keys().copied());
Some(Arc::new(self.multi_added_removed_keys.clone()))
} else {
None
};
self.metrics.prefetch_proof_targets_accounts_histogram.record(targets.len() as f64);
self.metrics
.prefetch_proof_targets_storages_histogram
.record(targets.values().map(|slots| slots.len()).sum::<usize>() as f64);
.record(targets.storage_count() as f64);
let chunking_len = targets.chunking_length();
let available_account_workers =
@@ -646,7 +802,7 @@ impl MultiProofTask {
self.max_targets_for_chunking,
available_account_workers,
available_storage_workers,
MultiProofTargets::chunks,
VersionedMultiProofTargets::chunks,
|proof_targets| {
self.multiproof_manager.dispatch(MultiproofInput {
source: None,
@@ -654,7 +810,7 @@ impl MultiProofTask {
proof_targets,
proof_sequence_number: self.proof_sequencer.next_sequence(),
state_root_message_sender: self.tx.clone(),
multi_added_removed_keys: Some(multi_added_removed_keys.clone()),
multi_added_removed_keys: multi_added_removed_keys.clone(),
});
},
);
@@ -757,6 +913,7 @@ impl MultiProofTask {
self.multiproof_manager.proof_worker_handle.available_account_workers();
let available_storage_workers =
self.multiproof_manager.proof_worker_handle.available_storage_workers();
let num_chunks = dispatch_with_chunking(
not_fetched_state_update,
chunking_len,
@@ -770,8 +927,9 @@ impl MultiProofTask {
&hashed_state_update,
&self.fetched_proof_targets,
&multi_added_removed_keys,
self.v2_proofs_enabled,
);
spawned_proof_targets.extend_ref(&proof_targets);
extend_multiproof_targets(&mut spawned_proof_targets, &proof_targets);
self.multiproof_manager.dispatch(MultiproofInput {
source: Some(source),
@@ -871,7 +1029,10 @@ impl MultiProofTask {
batch_metrics.proofs_processed += 1;
if let Some(combined_update) = self.on_proof(
sequence_number,
SparseTrieUpdate { state, multiproof: Default::default() },
SparseTrieUpdate {
state,
multiproof: ProofResult::empty(self.v2_proofs_enabled),
},
) {
let _ = self.to_sparse_trie.send(combined_update);
}
@@ -898,8 +1059,7 @@ impl MultiProofTask {
}
let account_targets = merged_targets.len();
let storage_targets =
merged_targets.values().map(|slots| slots.len()).sum::<usize>();
let storage_targets = merged_targets.storage_count();
batch_metrics.prefetch_proofs_requested += self.on_prefetch_proof(merged_targets);
trace!(
target: "engine::tree::payload_processor::multiproof",
@@ -1003,7 +1163,10 @@ impl MultiProofTask {
if let Some(combined_update) = self.on_proof(
sequence_number,
SparseTrieUpdate { state, multiproof: Default::default() },
SparseTrieUpdate {
state,
multiproof: ProofResult::empty(self.v2_proofs_enabled),
},
) {
let _ = self.to_sparse_trie.send(combined_update);
}
@@ -1106,7 +1269,7 @@ impl MultiProofTask {
let update = SparseTrieUpdate {
state: proof_result.state,
multiproof: proof_result_data.proof,
multiproof: proof_result_data,
};
if let Some(combined_update) =
@@ -1196,7 +1359,7 @@ struct MultiproofBatchCtx {
/// received.
updates_finished_time: Option<Instant>,
/// Reusable buffer for accumulating prefetch targets during batching.
accumulated_prefetch_targets: Vec<MultiProofTargets>,
accumulated_prefetch_targets: Vec<VersionedMultiProofTargets>,
}
impl MultiproofBatchCtx {
@@ -1242,40 +1405,77 @@ fn get_proof_targets(
state_update: &HashedPostState,
fetched_proof_targets: &MultiProofTargets,
multi_added_removed_keys: &MultiAddedRemovedKeys,
) -> MultiProofTargets {
let mut targets = MultiProofTargets::default();
v2_enabled: bool,
) -> VersionedMultiProofTargets {
if v2_enabled {
let mut targets = MultiProofTargetsV2::default();
// first collect all new accounts (not previously fetched)
for hashed_address in state_update.accounts.keys() {
if !fetched_proof_targets.contains_key(hashed_address) {
targets.insert(*hashed_address, HashSet::default());
// first collect all new accounts (not previously fetched)
for &hashed_address in state_update.accounts.keys() {
if !fetched_proof_targets.contains_key(&hashed_address) {
targets.account_targets.push(hashed_address.into());
}
}
// then process storage slots for all accounts in the state update
for (hashed_address, storage) in &state_update.storages {
let fetched = fetched_proof_targets.get(hashed_address);
// If the storage is wiped, we still need to fetch the account proof.
if storage.wiped && fetched.is_none() {
targets.account_targets.push(Into::<proof_v2::Target>::into(*hashed_address));
continue
}
let changed_slots = storage
.storage
.keys()
.filter(|slot| !fetched.is_some_and(|f| f.contains(*slot)))
.map(|slot| Into::<proof_v2::Target>::into(*slot))
.collect::<Vec<_>>();
if !changed_slots.is_empty() {
targets.account_targets.push((*hashed_address).into());
targets.storage_targets.insert(*hashed_address, changed_slots);
}
}
VersionedMultiProofTargets::V2(targets)
} else {
let mut targets = MultiProofTargets::default();
// first collect all new accounts (not previously fetched)
for hashed_address in state_update.accounts.keys() {
if !fetched_proof_targets.contains_key(hashed_address) {
targets.insert(*hashed_address, HashSet::default());
}
}
// then process storage slots for all accounts in the state update
for (hashed_address, storage) in &state_update.storages {
let fetched = fetched_proof_targets.get(hashed_address);
let storage_added_removed_keys = multi_added_removed_keys.get_storage(hashed_address);
let mut changed_slots = storage
.storage
.keys()
.filter(|slot| {
!fetched.is_some_and(|f| f.contains(*slot)) ||
storage_added_removed_keys.is_some_and(|k| k.is_removed(slot))
})
.peekable();
// If the storage is wiped, we still need to fetch the account proof.
if storage.wiped && fetched.is_none() {
targets.entry(*hashed_address).or_default();
}
if changed_slots.peek().is_some() {
targets.entry(*hashed_address).or_default().extend(changed_slots);
}
}
VersionedMultiProofTargets::Legacy(targets)
}
// then process storage slots for all accounts in the state update
for (hashed_address, storage) in &state_update.storages {
let fetched = fetched_proof_targets.get(hashed_address);
let storage_added_removed_keys = multi_added_removed_keys.get_storage(hashed_address);
let mut changed_slots = storage
.storage
.keys()
.filter(|slot| {
!fetched.is_some_and(|f| f.contains(*slot)) ||
storage_added_removed_keys.is_some_and(|k| k.is_removed(slot))
})
.peekable();
// If the storage is wiped, we still need to fetch the account proof.
if storage.wiped && fetched.is_none() {
targets.entry(*hashed_address).or_default();
}
if changed_slots.peek().is_some() {
targets.entry(*hashed_address).or_default().extend(changed_slots);
}
}
targets
}
/// Dispatches work items as a single unit or in chunks based on target size and worker
@@ -1323,7 +1523,7 @@ mod tests {
use reth_provider::{
providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory,
BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, LatestStateProvider,
PruneCheckpointReader, StageCheckpointReader, StateProviderBox,
PruneCheckpointReader, StageCheckpointReader, StateProviderBox, StorageChangeSetReader,
};
use reth_trie::MultiProof;
use reth_trie_db::ChangesetCache;
@@ -1350,6 +1550,7 @@ mod tests {
+ StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader,
> + Clone
+ Send
@@ -1481,12 +1682,24 @@ mod tests {
state
}
fn unwrap_legacy_targets(targets: VersionedMultiProofTargets) -> MultiProofTargets {
match targets {
VersionedMultiProofTargets::Legacy(targets) => targets,
VersionedMultiProofTargets::V2(_) => panic!("Expected Legacy targets"),
}
}
#[test]
fn test_get_proof_targets_new_account_targets() {
let state = create_get_proof_targets_state();
let fetched = MultiProofTargets::default();
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
// should return all accounts as targets since nothing was fetched before
assert_eq!(targets.len(), state.accounts.len());
@@ -1500,7 +1713,12 @@ mod tests {
let state = create_get_proof_targets_state();
let fetched = MultiProofTargets::default();
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
// verify storage slots are included for accounts with storage
for (addr, storage) in &state.storages {
@@ -1528,7 +1746,12 @@ mod tests {
// mark the account as already fetched
fetched.insert(*fetched_addr, HashSet::default());
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
// should not include the already fetched account since it has no storage updates
assert!(!targets.contains_key(fetched_addr));
@@ -1548,7 +1771,12 @@ mod tests {
fetched_slots.insert(fetched_slot);
fetched.insert(*addr, fetched_slots);
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
// should not include the already fetched storage slot
let target_slots = &targets[addr];
@@ -1561,7 +1789,12 @@ mod tests {
let state = HashedPostState::default();
let fetched = MultiProofTargets::default();
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
assert!(targets.is_empty());
}
@@ -1588,7 +1821,12 @@ mod tests {
fetched_slots.insert(slot1);
fetched.insert(addr1, fetched_slots);
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
assert!(targets.contains_key(&addr2));
assert!(!targets[&addr1].contains(&slot1));
@@ -1614,7 +1852,12 @@ mod tests {
assert!(!state.accounts.contains_key(&addr));
assert!(!fetched.contains_key(&addr));
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
// verify that we still get the storage slots for the unmodified account
assert!(targets.contains_key(&addr));
@@ -1656,7 +1899,12 @@ mod tests {
removed_state.storages.insert(addr, removed_storage);
multi_added_removed_keys.update_with_state(&removed_state);
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&multi_added_removed_keys,
false,
));
// slot1 should be included despite being fetched, because it's marked as removed
assert!(targets.contains_key(&addr));
@@ -1683,7 +1931,12 @@ mod tests {
storage.storage.insert(slot1, U256::from(100));
state.storages.insert(addr, storage);
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&multi_added_removed_keys,
false,
));
// account should be included because storage is wiped and account wasn't fetched
assert!(targets.contains_key(&addr));
@@ -1726,7 +1979,12 @@ mod tests {
removed_state.storages.insert(addr, removed_storage);
multi_added_removed_keys.update_with_state(&removed_state);
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&multi_added_removed_keys,
false,
));
// only slots in the state update can be included, so slot3 should not appear
assert!(!targets.contains_key(&addr));
@@ -1753,9 +2011,12 @@ mod tests {
targets3.insert(addr3, HashSet::default());
let tx = task.tx.clone();
tx.send(MultiProofMessage::PrefetchProofs(targets1)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets2)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets3)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets1)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets2)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets3)))
.unwrap();
let proofs_requested =
if let Ok(MultiProofMessage::PrefetchProofs(targets)) = task.rx.recv() {
@@ -1769,11 +2030,12 @@ mod tests {
assert_eq!(num_batched, 3);
assert_eq!(merged_targets.len(), 3);
assert!(merged_targets.contains_key(&addr1));
assert!(merged_targets.contains_key(&addr2));
assert!(merged_targets.contains_key(&addr3));
let legacy_targets = unwrap_legacy_targets(merged_targets);
assert!(legacy_targets.contains_key(&addr1));
assert!(legacy_targets.contains_key(&addr2));
assert!(legacy_targets.contains_key(&addr3));
task.on_prefetch_proof(merged_targets)
task.on_prefetch_proof(VersionedMultiProofTargets::Legacy(legacy_targets))
} else {
panic!("Expected PrefetchProofs message");
};
@@ -1848,11 +2110,16 @@ mod tests {
// Queue: [PrefetchProofs1, PrefetchProofs2, StateUpdate1, StateUpdate2, PrefetchProofs3]
let tx = task.tx.clone();
tx.send(MultiProofMessage::PrefetchProofs(targets1)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets2)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets1)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets2)))
.unwrap();
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update1)).unwrap();
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update2)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets3.clone())).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(
targets3.clone(),
)))
.unwrap();
// Step 1: Receive and batch PrefetchProofs (should get targets1 + targets2)
let mut pending_msg: Option<MultiProofMessage> = None;
@@ -1878,9 +2145,10 @@ mod tests {
// Should have batched exactly 2 PrefetchProofs (not 3!)
assert_eq!(num_batched, 2, "Should batch only until different message type");
assert_eq!(merged_targets.len(), 2);
assert!(merged_targets.contains_key(&addr1));
assert!(merged_targets.contains_key(&addr2));
assert!(!merged_targets.contains_key(&addr3), "addr3 should NOT be in first batch");
let legacy_targets = unwrap_legacy_targets(merged_targets);
assert!(legacy_targets.contains_key(&addr1));
assert!(legacy_targets.contains_key(&addr2));
assert!(!legacy_targets.contains_key(&addr3), "addr3 should NOT be in first batch");
} else {
panic!("Expected PrefetchProofs message");
}
@@ -1905,7 +2173,8 @@ mod tests {
match task.rx.try_recv() {
Ok(MultiProofMessage::PrefetchProofs(targets)) => {
assert_eq!(targets.len(), 1);
assert!(targets.contains_key(&addr3));
let legacy_targets = unwrap_legacy_targets(targets);
assert!(legacy_targets.contains_key(&addr3));
}
_ => panic!("PrefetchProofs3 was lost!"),
}
@@ -1951,9 +2220,13 @@ mod tests {
let source = StateChangeSource::Transaction(99);
let tx = task.tx.clone();
tx.send(MultiProofMessage::PrefetchProofs(prefetch1)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(prefetch1)))
.unwrap();
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(prefetch2.clone())).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(
prefetch2.clone(),
)))
.unwrap();
let mut ctx = MultiproofBatchCtx::new(Instant::now());
let mut batch_metrics = MultiproofBatchMetrics::default();
@@ -1986,7 +2259,8 @@ mod tests {
match task.rx.try_recv() {
Ok(MultiProofMessage::PrefetchProofs(targets)) => {
assert_eq!(targets.len(), 1);
assert!(targets.contains_key(&prefetch_addr2));
let legacy_targets = unwrap_legacy_targets(targets);
assert!(legacy_targets.contains_key(&prefetch_addr2));
}
other => panic!("Expected PrefetchProofs2 in channel, got {:?}", other),
}

View File

@@ -16,7 +16,7 @@ use crate::tree::{
payload_processor::{
bal::{total_slots, BALSlotIter},
executor::WorkloadExecutor,
multiproof::MultiProofMessage,
multiproof::{MultiProofMessage, VersionedMultiProofTargets},
ExecutionCache as PayloadExecutionCache,
},
precompile_cache::{CachedPrecompile, PrecompileCacheMap},
@@ -29,7 +29,7 @@ use alloy_evm::Database;
use alloy_primitives::{keccak256, map::B256Set, B256};
use crossbeam_channel::Sender as CrossbeamSender;
use metrics::{Counter, Gauge, Histogram};
use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, SpecFor};
use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, RecoveredTx, SpecFor};
use reth_metrics::Metrics;
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
@@ -237,7 +237,7 @@ where
}
/// If configured and the tx returned proof targets, emit the targets the transaction produced
fn send_multi_proof_targets(&self, targets: Option<MultiProofTargets>) {
fn send_multi_proof_targets(&self, targets: Option<VersionedMultiProofTargets>) {
if self.is_execution_terminated() {
// if execution is already terminated then we dont need to send more proof fetch
// messages
@@ -278,8 +278,9 @@ where
execution_cache.update_with_guard(|cached| {
// consumes the `SavedCache` held by the prewarming task, which releases its usage
// guard
let (caches, cache_metrics) = saved_cache.split();
let new_cache = SavedCache::new(hash, caches, cache_metrics);
let (caches, cache_metrics, disable_cache_metrics) = saved_cache.split();
let new_cache = SavedCache::new(hash, caches, cache_metrics)
.with_disable_cache_metrics(disable_cache_metrics);
// Insert state into cache while holding the lock
// Access the BundleState through the shared ExecutionOutcome
@@ -483,6 +484,8 @@ where
pub(super) terminate_execution: Arc<AtomicBool>,
pub(super) precompile_cache_disabled: bool,
pub(super) precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
/// Whether V2 proof calculation is enabled.
pub(super) v2_proofs_enabled: bool,
}
impl<N, P, Evm> PrewarmContext<N, P, Evm>
@@ -491,10 +494,12 @@ where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
Evm: ConfigureEvm<Primitives = N> + 'static,
{
/// Splits this context into an evm, an evm config, metrics, and the atomic bool for terminating
/// execution.
/// Splits this context into an evm, an evm config, metrics, the atomic bool for terminating
/// execution, and whether V2 proofs are enabled.
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
fn evm_for_ctx(self) -> Option<(EvmFor<Evm, impl Database>, PrewarmMetrics, Arc<AtomicBool>)> {
fn evm_for_ctx(
self,
) -> Option<(EvmFor<Evm, impl Database>, PrewarmMetrics, Arc<AtomicBool>, bool)> {
let Self {
env,
evm_config,
@@ -504,6 +509,7 @@ where
terminate_execution,
precompile_cache_disabled,
precompile_cache_map,
v2_proofs_enabled,
} = self;
let mut state_provider = match provider.build() {
@@ -553,7 +559,7 @@ where
});
}
Some((evm, metrics, terminate_execution))
Some((evm, metrics, terminate_execution, v2_proofs_enabled))
}
/// Accepts an [`mpsc::Receiver`] of transactions and a handle to prewarm task. Executes
@@ -574,7 +580,10 @@ where
) where
Tx: ExecutableTxFor<Evm>,
{
let Some((mut evm, metrics, terminate_execution)) = self.evm_for_ctx() else { return };
let Some((mut evm, metrics, terminate_execution, v2_proofs_enabled)) = self.evm_for_ctx()
else {
return
};
while let Ok(IndexedTransaction { index, tx }) = {
let _enter = debug_span!(target: "engine::tree::payload_processor::prewarm", "recv tx")
@@ -600,7 +609,8 @@ where
break
}
let res = match evm.transact(&tx) {
let (tx_env, tx) = tx.into_parts();
let res = match evm.transact(tx_env) {
Ok(res) => res,
Err(err) => {
trace!(
@@ -637,7 +647,8 @@ where
let _enter =
debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm outcome", index, tx_hash=%tx.tx().tx_hash())
.entered();
let (targets, storage_targets) = multiproof_targets_from_state(res.state);
let (targets, storage_targets) =
multiproof_targets_from_state(res.state, v2_proofs_enabled);
metrics.prefetch_storage_targets.record(storage_targets as f64);
let _ = sender.send(PrewarmTaskEvent::Outcome { proof_targets: Some(targets) });
drop(_enter);
@@ -782,9 +793,22 @@ where
}
}
/// Returns a set of [`MultiProofTargets`] and the total amount of storage targets, based on the
/// Returns a set of [`VersionedMultiProofTargets`] and the total amount of storage targets, based
/// on the given state.
fn multiproof_targets_from_state(
state: EvmState,
v2_enabled: bool,
) -> (VersionedMultiProofTargets, usize) {
if v2_enabled {
multiproof_targets_v2_from_state(state)
} else {
multiproof_targets_legacy_from_state(state)
}
}
/// Returns legacy [`MultiProofTargets`] and the total amount of storage targets, based on the
/// given state.
fn multiproof_targets_from_state(state: EvmState) -> (MultiProofTargets, usize) {
fn multiproof_targets_legacy_from_state(state: EvmState) -> (VersionedMultiProofTargets, usize) {
let mut targets = MultiProofTargets::with_capacity(state.len());
let mut storage_targets = 0;
for (addr, account) in state {
@@ -814,7 +838,50 @@ fn multiproof_targets_from_state(state: EvmState) -> (MultiProofTargets, usize)
targets.insert(keccak256(addr), storage_set);
}
(targets, storage_targets)
(VersionedMultiProofTargets::Legacy(targets), storage_targets)
}
/// Returns V2 [`reth_trie_parallel::targets_v2::MultiProofTargetsV2`] and the total amount of
/// storage targets, based on the given state.
fn multiproof_targets_v2_from_state(state: EvmState) -> (VersionedMultiProofTargets, usize) {
use reth_trie::proof_v2;
use reth_trie_parallel::targets_v2::MultiProofTargetsV2;
let mut targets = MultiProofTargetsV2::default();
let mut storage_target_count = 0;
for (addr, account) in state {
// if the account was not touched, or if the account was selfdestructed, do not
// fetch proofs for it
//
// Since selfdestruct can only happen in the same transaction, we can skip
// prefetching proofs for selfdestructed accounts
//
// See: https://eips.ethereum.org/EIPS/eip-6780
if !account.is_touched() || account.is_selfdestructed() {
continue
}
let hashed_address = keccak256(addr);
targets.account_targets.push(hashed_address.into());
let mut storage_slots = Vec::with_capacity(account.storage.len());
for (key, slot) in account.storage {
// do nothing if unchanged
if !slot.is_changed() {
continue
}
let hashed_slot = keccak256(B256::new(key.to_be_bytes()));
storage_slots.push(proof_v2::Target::from(hashed_slot));
}
storage_target_count += storage_slots.len();
if !storage_slots.is_empty() {
targets.storage_targets.insert(hashed_address, storage_slots);
}
}
(VersionedMultiProofTargets::V2(targets), storage_target_count)
}
/// The events the pre-warm task can handle.
@@ -839,7 +906,7 @@ pub(super) enum PrewarmTaskEvent<R> {
/// The outcome of a pre-warm task
Outcome {
/// The prepared proof targets based on the evm state outcome
proof_targets: Option<MultiProofTargets>,
proof_targets: Option<VersionedMultiProofTargets>,
},
/// Finished executing all transactions
FinishedTxExecution {

View File

@@ -4,7 +4,7 @@ use crate::tree::payload_processor::multiproof::{MultiProofTaskMetrics, SparseTr
use alloy_primitives::B256;
use rayon::iter::{ParallelBridge, ParallelIterator};
use reth_trie::{updates::TrieUpdates, Nibbles};
use reth_trie_parallel::root::ParallelStateRootError;
use reth_trie_parallel::{proof_task::ProofResult, root::ParallelStateRootError};
use reth_trie_sparse::{
errors::{SparseStateTrieResult, SparseTrieErrorKind},
provider::{TrieNodeProvider, TrieNodeProviderFactory},
@@ -97,8 +97,8 @@ where
debug!(
target: "engine::root",
num_updates,
account_proofs = update.multiproof.account_subtree.len(),
storage_proofs = update.multiproof.storages.len(),
account_proofs = update.multiproof.account_proofs_len(),
storage_proofs = update.multiproof.storage_proofs_len(),
"Updating sparse trie"
);
@@ -157,7 +157,14 @@ where
let started_at = Instant::now();
// Reveal new accounts and storage slots.
trie.reveal_decoded_multiproof(multiproof)?;
match multiproof {
ProofResult::Legacy(decoded, _) => {
trie.reveal_decoded_multiproof(decoded)?;
}
ProofResult::V2(decoded_v2) => {
trie.reveal_decoded_multiproof_v2(decoded_v2)?;
}
}
let reveal_multiproof_elapsed = started_at.elapsed();
trace!(
target: "engine::root::sparse",

View File

@@ -7,10 +7,10 @@ use crate::tree::{
payload_processor::{executor::WorkloadExecutor, PayloadProcessor},
precompile_cache::{CachedPrecompile, CachedPrecompileMetrics, PrecompileCacheMap},
sparse_trie::StateRootComputeOutcome,
EngineApiMetrics, EngineApiTreeState, ExecutionEnv, PayloadHandle, StateProviderBuilder,
StateProviderDatabase, TreeConfig,
EngineApiMetrics, EngineApiTreeState, ExecutionEnv, MeteredStateHook, PayloadHandle,
StateProviderBuilder, StateProviderDatabase, TreeConfig,
};
use alloy_consensus::transaction::Either;
use alloy_consensus::transaction::{Either, TxHashRef};
use alloy_eip7928::BlockAccessList;
use alloy_eips::{eip1898::BlockWithParent, NumHash};
use alloy_evm::Evm;
@@ -39,9 +39,9 @@ use reth_provider::{
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader,
ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider,
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
StateProviderFactory, StateReader,
StateProviderFactory, StateReader, StorageChangeSetReader,
};
use reth_revm::db::State;
use reth_revm::db::{states::bundle_state::BundleRetention, State};
use reth_trie::{updates::TrieUpdates, HashedPostState, StateRoot};
use reth_trie_db::ChangesetCache;
use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError};
@@ -144,6 +144,7 @@ where
+ StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader,
> + BlockReader<Header = N::BlockHeader>
+ ChangeSetReader
@@ -642,7 +643,13 @@ where
Ok(())
}
/// Executes a block with the given state provider
/// Executes a block with the given state provider.
///
/// This method orchestrates block execution:
/// 1. Sets up the EVM with state database and precompile caching
/// 2. Spawns a background task for incremental receipt root computation
/// 3. Executes transactions with metrics collection via state hooks
/// 4. Merges state transitions and records execution metrics
#[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)]
#[expect(clippy::type_complexity)]
fn execute_block<S, Err, T>(
@@ -705,31 +712,117 @@ where
let task_handle = ReceiptRootTaskHandle::new(receipt_rx, result_tx);
self.payload_processor.executor().spawn_blocking(move || task_handle.run(receipts_len));
// Wrap the state hook with metrics collection
let inner_hook = Box::new(handle.state_hook());
let state_hook =
MeteredStateHook { metrics: self.metrics.executor_metrics().clone(), inner_hook };
let transaction_count = input.transaction_count();
let executor = executor.with_state_hook(Some(Box::new(state_hook)));
let execution_start = Instant::now();
let state_hook = Box::new(handle.state_hook());
let (output, senders) = self.metrics.execute_metered(
// Execute all transactions and finalize
let (executor, senders) = self.execute_transactions(
executor,
handle.iter_transactions().map(|res| res.map_err(BlockExecutionError::other)),
input.transaction_count(),
state_hook,
|receipts| {
// Send the latest receipt to the background task for incremental root computation.
// The receipt is cloned here; encoding happens in the background thread.
if let Some(receipt) = receipts.last() {
// Infer tx_index from the number of receipts collected so far
let tx_index = receipts.len() - 1;
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
}
},
transaction_count,
handle.iter_transactions(),
&receipt_tx,
)?;
drop(receipt_tx);
let execution_finish = Instant::now();
let execution_time = execution_finish.duration_since(execution_start);
debug!(target: "engine::tree::payload_validator", elapsed = ?execution_time, "Executed block");
// Finish execution and get the result
let post_exec_start = Instant::now();
let (_evm, result) = debug_span!(target: "engine::tree", "finish")
.in_scope(|| executor.finish())
.map(|(evm, result)| (evm.into_db(), result))?;
self.metrics.record_post_execution(post_exec_start.elapsed());
// Merge transitions into bundle state
debug_span!(target: "engine::tree", "merge transitions")
.in_scope(|| db.merge_transitions(BundleRetention::Reverts));
let output = BlockExecutionOutput { result, state: db.take_bundle() };
let execution_duration = execution_start.elapsed();
self.metrics.record_block_execution(&output, execution_duration);
debug!(target: "engine::tree::payload_validator", elapsed = ?execution_duration, "Executed block");
Ok((output, senders, result_rx))
}
/// Executes transactions and collects senders, streaming receipts to a background task.
///
/// This method handles:
/// - Applying pre-execution changes (e.g., beacon root updates)
/// - Executing each transaction with timing metrics
/// - Streaming receipts to the receipt root computation task
/// - Collecting transaction senders for later use
///
/// Returns the executor (for finalization) and the collected senders.
fn execute_transactions<E, Tx, InnerTx, Err>(
&self,
mut executor: E,
transaction_count: usize,
transactions: impl Iterator<Item = Result<Tx, Err>>,
receipt_tx: &crossbeam_channel::Sender<IndexedReceipt<N::Receipt>>,
) -> Result<(E, Vec<Address>), BlockExecutionError>
where
E: BlockExecutor<Receipt = N::Receipt>,
Tx: alloy_evm::block::ExecutableTx<E> + alloy_evm::RecoveredTx<InnerTx>,
InnerTx: TxHashRef,
Err: core::error::Error + Send + Sync + 'static,
{
let mut senders = Vec::with_capacity(transaction_count);
// Apply pre-execution changes (e.g., beacon root update)
let pre_exec_start = Instant::now();
debug_span!(target: "engine::tree", "pre execution")
.in_scope(|| executor.apply_pre_execution_changes())?;
self.metrics.record_pre_execution(pre_exec_start.elapsed());
// Execute transactions
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
let mut transactions = transactions.into_iter();
loop {
// Measure time spent waiting for next transaction from iterator
// (e.g., parallel signature recovery)
let wait_start = Instant::now();
let Some(tx_result) = transactions.next() else { break };
self.metrics.record_transaction_wait(wait_start.elapsed());
let tx = tx_result.map_err(BlockExecutionError::other)?;
let tx_signer = *<Tx as alloy_evm::RecoveredTx<InnerTx>>::signer(&tx);
let tx_hash = <Tx as alloy_evm::RecoveredTx<InnerTx>>::tx(&tx).tx_hash();
senders.push(tx_signer);
let span = debug_span!(
target: "engine::tree",
"execute tx",
?tx_hash,
gas_used = tracing::field::Empty,
);
let enter = span.entered();
trace!(target: "engine::tree", "Executing transaction");
let tx_start = Instant::now();
let gas_used = executor.execute_transaction(tx)?;
self.metrics.record_transaction_execution(tx_start.elapsed());
// Send the latest receipt to the background task for incremental root computation
if let Some(receipt) = executor.receipts().last() {
let tx_index = executor.receipts().len() - 1;
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
}
enter.record("gas_used", gas_used);
}
drop(exec_span);
Ok((executor, senders))
}
/// Compute state root for the given hashed post state in parallel.
///
/// Uses an overlay factory which provides the state of the parent block, along with the
@@ -1244,6 +1337,7 @@ where
+ StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader,
> + BlockReader<Header = N::BlockHeader>
+ StateProviderFactory

View File

@@ -17,10 +17,11 @@ pub use payload::{payload_id, BlobSidecars, EthBuiltPayload, EthPayloadBuilderAt
mod error;
pub use error::*;
use alloy_rpc_types_engine::{ExecutionData, ExecutionPayload, ExecutionPayloadEnvelopeV5};
use alloy_rpc_types_engine::{ExecutionData, ExecutionPayload};
pub use alloy_rpc_types_engine::{
ExecutionPayloadEnvelopeV2, ExecutionPayloadEnvelopeV3, ExecutionPayloadEnvelopeV4,
ExecutionPayloadV1, PayloadAttributes as EthPayloadAttributes,
ExecutionPayloadEnvelopeV5, ExecutionPayloadEnvelopeV6, ExecutionPayloadV1,
PayloadAttributes as EthPayloadAttributes,
};
use reth_engine_primitives::EngineTypes;
use reth_payload_primitives::{BuiltPayload, PayloadTypes};
@@ -66,13 +67,15 @@ where
+ TryInto<ExecutionPayloadEnvelopeV2>
+ TryInto<ExecutionPayloadEnvelopeV3>
+ TryInto<ExecutionPayloadEnvelopeV4>
+ TryInto<ExecutionPayloadEnvelopeV5>,
+ TryInto<ExecutionPayloadEnvelopeV5>
+ TryInto<ExecutionPayloadEnvelopeV6>,
{
type ExecutionPayloadEnvelopeV1 = ExecutionPayloadV1;
type ExecutionPayloadEnvelopeV2 = ExecutionPayloadEnvelopeV2;
type ExecutionPayloadEnvelopeV3 = ExecutionPayloadEnvelopeV3;
type ExecutionPayloadEnvelopeV4 = ExecutionPayloadEnvelopeV4;
type ExecutionPayloadEnvelopeV5 = ExecutionPayloadEnvelopeV5;
type ExecutionPayloadEnvelopeV6 = ExecutionPayloadEnvelopeV6;
}
/// A default payload type for [`EthEngineTypes`]

View File

@@ -11,8 +11,8 @@ use alloy_primitives::{Address, B256, U256};
use alloy_rlp::Encodable;
use alloy_rpc_types_engine::{
BlobsBundleV1, BlobsBundleV2, ExecutionPayloadEnvelopeV2, ExecutionPayloadEnvelopeV3,
ExecutionPayloadEnvelopeV4, ExecutionPayloadEnvelopeV5, ExecutionPayloadFieldV2,
ExecutionPayloadV1, ExecutionPayloadV3, PayloadAttributes, PayloadId,
ExecutionPayloadEnvelopeV4, ExecutionPayloadEnvelopeV5, ExecutionPayloadEnvelopeV6,
ExecutionPayloadFieldV2, ExecutionPayloadV1, ExecutionPayloadV3, PayloadAttributes, PayloadId,
};
use core::convert::Infallible;
use reth_ethereum_primitives::EthPrimitives;
@@ -160,6 +160,13 @@ impl EthBuiltPayload {
execution_requests: requests.unwrap_or_default(),
})
}
/// Try converting built payload into [`ExecutionPayloadEnvelopeV6`].
///
/// Note: Amsterdam fork is not yet implemented, so this conversion is not yet supported.
pub fn try_into_v6(self) -> Result<ExecutionPayloadEnvelopeV6, BuiltPayloadConversionError> {
unimplemented!("ExecutionPayloadEnvelopeV6 not yet supported")
}
}
impl<N: NodePrimitives> BuiltPayload for EthBuiltPayload<N> {
@@ -227,6 +234,14 @@ impl TryFrom<EthBuiltPayload> for ExecutionPayloadEnvelopeV5 {
}
}
impl TryFrom<EthBuiltPayload> for ExecutionPayloadEnvelopeV6 {
type Error = BuiltPayloadConversionError;
fn try_from(value: EthBuiltPayload) -> Result<Self, Self::Error> {
value.try_into_v6()
}
}
/// An enum representing blob transaction sidecars belonging to [`EthBuiltPayload`].
#[derive(Clone, Default, Debug)]
pub enum BlobSidecars {

View File

@@ -1,3 +1,4 @@
use alloy_consensus::TxType;
use alloy_evm::eth::receipt_builder::{ReceiptBuilder, ReceiptBuilderCtx};
use reth_ethereum_primitives::{Receipt, TransactionSigned};
use reth_evm::Evm;
@@ -12,13 +13,10 @@ impl ReceiptBuilder for RethReceiptBuilder {
type Transaction = TransactionSigned;
type Receipt = Receipt;
fn build_receipt<E: Evm>(
&self,
ctx: ReceiptBuilderCtx<'_, Self::Transaction, E>,
) -> Self::Receipt {
let ReceiptBuilderCtx { tx, result, cumulative_gas_used, .. } = ctx;
fn build_receipt<E: Evm>(&self, ctx: ReceiptBuilderCtx<'_, TxType, E>) -> Self::Receipt {
let ReceiptBuilderCtx { tx_type, result, cumulative_gas_used, .. } = ctx;
Receipt {
tx_type: tx.tx_type(),
tx_type,
// Success flag was added in `EIP-658: Embedding transaction status code in
// receipts`.
success: result.is_success(),

View File

@@ -1,6 +1,6 @@
use crate::EthEvmConfig;
use alloc::{boxed::Box, sync::Arc, vec, vec::Vec};
use alloy_consensus::Header;
use alloy_consensus::{Header, TxType};
use alloy_eips::eip7685::Requests;
use alloy_evm::precompiles::PrecompilesMap;
use alloy_primitives::Bytes;
@@ -11,14 +11,14 @@ use reth_evm::{
block::{
BlockExecutionError, BlockExecutor, BlockExecutorFactory, BlockExecutorFor, ExecutableTx,
},
eth::{EthBlockExecutionCtx, EthEvmContext},
eth::{EthBlockExecutionCtx, EthEvmContext, EthTxResult},
ConfigureEngineEvm, ConfigureEvm, Database, EthEvm, EthEvmFactory, Evm, EvmEnvFor, EvmFactory,
ExecutableTxIterator, ExecutionCtxFor,
ExecutableTxIterator, ExecutionCtxFor, RecoveredTx,
};
use reth_execution_types::{BlockExecutionResult, ExecutionOutcome};
use reth_primitives_traits::{BlockTy, SealedBlock, SealedHeader};
use revm::{
context::result::{ExecutionResult, Output, ResultAndState, SuccessReason},
context::result::{ExecutionResult, HaltReason, Output, ResultAndState, SuccessReason},
database::State,
Inspector,
};
@@ -90,6 +90,7 @@ impl<'a, DB: Database, I: Inspector<EthEvmContext<&'a mut State<DB>>>> BlockExec
type Evm = EthEvm<&'a mut State<DB>, I, PrecompilesMap>;
type Transaction = TransactionSigned;
type Receipt = Receipt;
type Result = EthTxResult<HaltReason, TxType>;
fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> {
Ok(())
@@ -101,25 +102,25 @@ impl<'a, DB: Database, I: Inspector<EthEvmContext<&'a mut State<DB>>>> BlockExec
fn execute_transaction_without_commit(
&mut self,
_tx: impl ExecutableTx<Self>,
) -> Result<ResultAndState<<Self::Evm as Evm>::HaltReason>, BlockExecutionError> {
Ok(ResultAndState::new(
ExecutionResult::Success {
reason: SuccessReason::Return,
gas_used: 0,
gas_refunded: 0,
logs: vec![],
output: Output::Call(Bytes::from(vec![])),
},
Default::default(),
))
tx: impl ExecutableTx<Self>,
) -> Result<Self::Result, BlockExecutionError> {
Ok(EthTxResult {
result: ResultAndState::new(
ExecutionResult::Success {
reason: SuccessReason::Return,
gas_used: 0,
gas_refunded: 0,
logs: vec![],
output: Output::Call(Bytes::from(vec![])),
},
Default::default(),
),
tx_type: tx.into_parts().1.tx().tx_type(),
blob_gas_used: 0,
})
}
fn commit_transaction(
&mut self,
_output: ResultAndState<<Self::Evm as Evm>::HaltReason>,
_tx: impl ExecutableTx<Self>,
) -> Result<u64, BlockExecutionError> {
fn commit_transaction(&mut self, _output: Self::Result) -> Result<u64, BlockExecutionError> {
Ok(0)
}

View File

@@ -18,7 +18,7 @@ use reth_evm::{
};
use reth_network::{primitives::BasicNetworkPrimitives, NetworkHandle, PeersInfo};
use reth_node_api::{
AddOnsContext, FullNodeComponents, HeaderTy, NodeAddOns, NodePrimitives,
AddOnsContext, BlockTy, FullNodeComponents, HeaderTy, NodeAddOns, NodePrimitives,
PayloadAttributesBuilder, PrimitivesTy, TxTy,
};
use reth_node_builder::{
@@ -53,8 +53,8 @@ use reth_rpc_eth_types::{error::FromEvmError, EthApiError};
use reth_rpc_server_types::RethRpcModule;
use reth_tracing::tracing::{debug, info};
use reth_transaction_pool::{
blobstore::DiskFileBlobStore, EthTransactionPool, PoolPooledTx, PoolTransaction,
TransactionPool, TransactionValidationTaskExecutor,
blobstore::DiskFileBlobStore, EthPooledTransaction, EthTransactionPool, PoolPooledTx,
PoolTransaction, TransactionPool, TransactionValidationTaskExecutor,
};
use revm::context::TxEnv;
use std::{marker::PhantomData, sync::Arc, time::SystemTime};
@@ -464,7 +464,8 @@ where
>,
Node: FullNodeTypes<Types = Types>,
{
type Pool = EthTransactionPool<Node::Provider, DiskFileBlobStore>;
type Pool =
EthTransactionPool<Node::Provider, DiskFileBlobStore, EthPooledTransaction, BlockTy<Types>>;
async fn build_pool(self, ctx: &BuilderContext<Node>) -> eyre::Result<Self::Pool> {
let pool_config = ctx.pool_config();

View File

@@ -0,0 +1,357 @@
//! Tests for handling invalid payloads via Engine API.
//!
//! This module tests the scenario where a node receives invalid payloads (e.g., with modified
//! state roots) before receiving valid ones, ensuring the node can recover and continue.
use crate::utils::eth_payload_attributes;
use alloy_primitives::B256;
use alloy_rpc_types_engine::{ExecutionPayloadV3, PayloadStatusEnum};
use rand::{rngs::StdRng, Rng, SeedableRng};
use reth_chainspec::{ChainSpecBuilder, MAINNET};
use reth_e2e_test_utils::{setup_engine, transaction::TransactionTestContext};
use reth_node_ethereum::EthereumNode;
use reth_rpc_api::EngineApiClient;
use std::sync::Arc;
/// Tests that a node can handle receiving an invalid payload (with wrong state root)
/// followed by the correct payload, and continue operating normally.
///
/// Setup:
/// - Node 1: Produces valid payloads and advances the chain
/// - Node 2: Receives payloads from node 1, but we also inject modified payloads with invalid state
/// roots in between to verify error handling
#[tokio::test]
async fn can_handle_invalid_payload_then_valid() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
let seed: [u8; 32] = rand::rng().random();
let mut rng = StdRng::from_seed(seed);
println!("Seed: {seed:?}");
let chain_spec = Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
.cancun_activated()
.build(),
);
let (mut nodes, _tasks, wallet) = setup_engine::<EthereumNode>(
2,
chain_spec.clone(),
false,
Default::default(),
eth_payload_attributes,
)
.await?;
let mut producer = nodes.pop().unwrap();
let receiver = nodes.pop().unwrap();
// Get engine API client for the receiver node
let receiver_engine = receiver.auth_server_handle().http_client();
// Inject a transaction to allow block building (advance_block waits for transactions)
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
producer.rpc.inject_tx(raw_tx).await?;
// Build a valid payload on the producer
let payload = producer.advance_block().await?;
let valid_block = payload.block().clone();
// Create valid payload first, then corrupt the state root
let mut invalid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
let original_state_root = invalid_payload.payload_inner.payload_inner.state_root;
invalid_payload.payload_inner.payload_inner.state_root = B256::random_with(&mut rng);
// Send the invalid payload to the receiver - should be rejected
let invalid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
invalid_payload.clone(),
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!(
"Invalid payload response: {:?} (state_root changed from {original_state_root} to {})",
invalid_result.status, invalid_payload.payload_inner.payload_inner.state_root
);
// The invalid payload should be rejected
assert!(
matches!(
invalid_result.status,
PayloadStatusEnum::Invalid { .. } | PayloadStatusEnum::Syncing
),
"Expected INVALID or SYNCING status for invalid payload, got {:?}",
invalid_result.status
);
// Now send the valid payload - should be accepted
let valid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
let valid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
valid_payload,
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!("Valid payload response: {:?}", valid_result.status);
// The valid payload should be accepted
assert!(
matches!(
valid_result.status,
PayloadStatusEnum::Valid | PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted
),
"Expected VALID/SYNCING/ACCEPTED status for valid payload, got {:?}",
valid_result.status
);
// Update forkchoice on receiver to the valid block
receiver.update_forkchoice(valid_block.hash(), valid_block.hash()).await?;
// Verify the receiver node is at the expected block
let receiver_head = receiver.block_hash(1);
let producer_head = producer.block_hash(1);
assert_eq!(
receiver_head, producer_head,
"Receiver should have synced to the same chain as producer"
);
println!(
"Test passed: Receiver successfully handled invalid payloads and synced to valid chain"
);
Ok(())
}
/// Tests that a node can handle multiple consecutive invalid payloads
/// before receiving a valid one.
#[tokio::test]
async fn can_handle_multiple_invalid_payloads() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
let seed: [u8; 32] = rand::rng().random();
let mut rng = StdRng::from_seed(seed);
println!("Seed: {seed:?}");
let chain_spec = Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
.cancun_activated()
.build(),
);
let (mut nodes, _tasks, wallet) = setup_engine::<EthereumNode>(
2,
chain_spec.clone(),
false,
Default::default(),
eth_payload_attributes,
)
.await?;
let mut producer = nodes.pop().unwrap();
let receiver = nodes.pop().unwrap();
let receiver_engine = receiver.auth_server_handle().http_client();
// Inject a transaction to allow block building
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
producer.rpc.inject_tx(raw_tx).await?;
// Produce a valid block
let payload = producer.advance_block().await?;
let valid_block = payload.block().clone();
// Send multiple invalid payloads with different corruptions
for i in 0..3 {
// Create valid payload first, then corrupt the state root
let mut invalid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
invalid_payload.payload_inner.payload_inner.state_root = B256::random_with(&mut rng);
let result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
invalid_payload,
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!("Invalid payload {i}: status = {:?}", result.status);
assert!(
matches!(result.status, PayloadStatusEnum::Invalid { .. } | PayloadStatusEnum::Syncing),
"Expected INVALID or SYNCING for invalid payload {i}, got {:?}",
result.status
);
}
// Now send the valid payload
let valid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
let valid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
valid_payload,
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!("Valid payload: status = {:?}", valid_result.status);
assert!(
matches!(
valid_result.status,
PayloadStatusEnum::Valid | PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted
),
"Expected valid status for correct payload, got {:?}",
valid_result.status
);
// Finalize the valid block
receiver.update_forkchoice(valid_block.hash(), valid_block.hash()).await?;
println!("Test passed: Receiver handled multiple invalid payloads and accepted valid one");
Ok(())
}
/// Tests invalid payload handling with blocks that contain transactions.
///
/// This test sends real transactions to node 1, produces blocks with those transactions,
/// then sends invalid (corrupted state root) and valid payloads to node 2.
#[tokio::test]
async fn can_handle_invalid_payload_with_transactions() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
let seed: [u8; 32] = rand::rng().random();
let mut rng = StdRng::from_seed(seed);
println!("Seed: {seed:?}");
let chain_spec = Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
.cancun_activated()
.build(),
);
let (mut nodes, _tasks, wallet) = setup_engine::<EthereumNode>(
2,
chain_spec.clone(),
false,
Default::default(),
eth_payload_attributes,
)
.await?;
let mut producer = nodes.pop().unwrap();
let receiver = nodes.pop().unwrap();
let receiver_engine = receiver.auth_server_handle().http_client();
// Create and send a transaction to the producer node
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
let tx_hash = producer.rpc.inject_tx(raw_tx).await?;
println!("Injected transaction {tx_hash}");
// Build a block containing the transaction
let payload = producer.advance_block().await?;
let valid_block = payload.block().clone();
// Verify the block contains a transaction
let tx_count = valid_block.body().transactions().count();
println!("Block contains {tx_count} transaction(s)");
assert!(tx_count > 0, "Block should contain at least one transaction");
// Create invalid payload by corrupting the state root
let mut invalid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
let original_state_root = invalid_payload.payload_inner.payload_inner.state_root;
invalid_payload.payload_inner.payload_inner.state_root = B256::random_with(&mut rng);
// Send invalid payload - should be rejected
let invalid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
invalid_payload.clone(),
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!(
"Invalid payload (with tx) response: {:?} (state_root changed from {original_state_root} to {})",
invalid_result.status,
invalid_payload.payload_inner.payload_inner.state_root
);
assert!(
matches!(
invalid_result.status,
PayloadStatusEnum::Invalid { .. } | PayloadStatusEnum::Syncing
),
"Expected INVALID or SYNCING for invalid payload with transactions, got {:?}",
invalid_result.status
);
// Send valid payload - should be accepted
let valid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
let valid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
valid_payload,
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!("Valid payload (with tx) response: {:?}", valid_result.status);
assert!(
matches!(
valid_result.status,
PayloadStatusEnum::Valid | PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted
),
"Expected valid status for correct payload with transactions, got {:?}",
valid_result.status
);
// Update forkchoice
receiver.update_forkchoice(valid_block.hash(), valid_block.hash()).await?;
// Verify both nodes are at the same head
let receiver_head = receiver.block_hash(1);
let producer_head = producer.block_hash(1);
assert_eq!(
receiver_head, producer_head,
"Receiver should have synced to the same chain as producer"
);
println!("Test passed: Receiver handled invalid payloads with transactions correctly");
Ok(())
}

View File

@@ -4,6 +4,7 @@ mod blobs;
mod custom_genesis;
mod dev;
mod eth;
mod invalid_payload;
mod p2p;
mod pool;
mod prestate;

View File

@@ -1,5 +1,7 @@
use crate::{execute::ExecutableTxFor, ConfigureEvm, EvmEnvFor, ExecutionCtxFor};
use crate::{execute::ExecutableTxFor, ConfigureEvm, EvmEnvFor, ExecutionCtxFor, TxEnvFor};
use alloy_evm::{block::ExecutableTxParts, RecoveredTx};
use rayon::prelude::*;
use reth_primitives_traits::TxTy;
/// [`ConfigureEvm`] extension providing methods for executing payloads.
pub trait ConfigureEngineEvm<ExecutionData>: ConfigureEvm {
@@ -61,11 +63,16 @@ where
/// Iterator over executable transactions.
pub trait ExecutableTxIterator<Evm: ConfigureEvm>:
ExecutableTxTuple<Tx: ExecutableTxFor<Evm>>
ExecutableTxTuple<Tx: ExecutableTxFor<Evm, Recovered = Self::Recovered>>
{
/// HACK: for some reason, this duplicated AT is the only way to enforce the inner Recovered:
/// Send + Sync bound. Effectively alias for `Self::Tx::Recovered`.
type Recovered: RecoveredTx<TxTy<Evm::Primitives>> + Send + Sync;
}
impl<T, Evm: ConfigureEvm> ExecutableTxIterator<Evm> for T where
T: ExecutableTxTuple<Tx: ExecutableTxFor<Evm>>
impl<T, Evm: ConfigureEvm> ExecutableTxIterator<Evm> for T
where
T: ExecutableTxTuple<Tx: ExecutableTxFor<Evm, Recovered: Send + Sync>>,
{
type Recovered = <T::Tx as ExecutableTxParts<TxEnvFor<Evm>, TxTy<Evm::Primitives>>>::Recovered;
}

View File

@@ -6,7 +6,7 @@ use alloy_consensus::{BlockHeader, Header};
use alloy_eips::eip2718::WithEncoded;
pub use alloy_evm::block::{BlockExecutor, BlockExecutorFactory};
use alloy_evm::{
block::{CommitChanges, ExecutableTx},
block::{CommitChanges, ExecutableTxParts},
Evm, EvmEnv, EvmFactory, RecoveredTx, ToTxEnv,
};
use alloy_primitives::{Address, B256};
@@ -148,20 +148,6 @@ pub trait Executor<DB: Database>: Sized {
fn size_hint(&self) -> usize;
}
/// Helper type for the output of executing a block.
///
/// Deprecated: this type is unused within reth and will be removed in the next
/// major release. Use `reth_execution_types::BlockExecutionResult` or
/// `reth_execution_types::BlockExecutionOutput`.
#[deprecated(note = "Use reth_execution_types::BlockExecutionResult or BlockExecutionOutput")]
#[derive(Debug, Clone)]
pub struct ExecuteOutput<R> {
/// Receipts obtained after executing a block.
pub receipts: Vec<R>,
/// Cumulative gas used in the block execution.
pub gas_used: u64,
}
/// Input for block building. Consumed by [`BlockAssembler`].
///
/// This struct contains all the data needed by the [`BlockAssembler`] to create
@@ -415,49 +401,31 @@ where
/// Conversions for executable transactions.
pub trait ExecutorTx<Executor: BlockExecutor> {
/// Converts the transaction into [`ExecutableTx`].
fn as_executable(&self) -> impl ExecutableTx<Executor>;
/// Converts the transaction into [`Recovered`].
fn into_recovered(self) -> Recovered<Executor::Transaction>;
/// Converts the transaction into a tuple of [`TxEnvFor`] and [`Recovered`].
fn into_parts(self) -> (<Executor::Evm as Evm>::Tx, Recovered<Executor::Transaction>);
}
impl<Executor: BlockExecutor> ExecutorTx<Executor>
for WithEncoded<Recovered<Executor::Transaction>>
{
fn as_executable(&self) -> impl ExecutableTx<Executor> {
self
}
fn into_recovered(self) -> Recovered<Executor::Transaction> {
self.1
fn into_parts(self) -> (<Executor::Evm as Evm>::Tx, Recovered<Executor::Transaction>) {
(self.to_tx_env(), self.1)
}
}
impl<Executor: BlockExecutor> ExecutorTx<Executor> for Recovered<Executor::Transaction> {
fn as_executable(&self) -> impl ExecutableTx<Executor> {
self
}
fn into_recovered(self) -> Self {
self
fn into_parts(self) -> (<Executor::Evm as Evm>::Tx, Self) {
(self.to_tx_env(), self)
}
}
impl<T, Executor> ExecutorTx<Executor>
for WithTxEnv<<<Executor as BlockExecutor>::Evm as Evm>::Tx, T>
impl<Executor> ExecutorTx<Executor>
for WithTxEnv<<Executor::Evm as Evm>::Tx, Recovered<Executor::Transaction>>
where
T: ExecutorTx<Executor> + Clone,
Executor: BlockExecutor,
<<Executor as BlockExecutor>::Evm as Evm>::Tx: Clone,
Self: RecoveredTx<Executor::Transaction>,
Executor: BlockExecutor<Transaction: Clone>,
{
fn as_executable(&self) -> impl ExecutableTx<Executor> {
self
}
fn into_recovered(self) -> Recovered<Executor::Transaction> {
Arc::unwrap_or_clone(self.tx).into_recovered()
fn into_parts(self) -> (<Executor::Evm as Evm>::Tx, Recovered<Executor::Transaction>) {
(self.tx_env, Arc::unwrap_or_clone(self.tx))
}
}
@@ -493,10 +461,11 @@ where
&ExecutionResult<<<Self::Executor as BlockExecutor>::Evm as Evm>::HaltReason>,
) -> CommitChanges,
) -> Result<Option<u64>, BlockExecutionError> {
let (tx_env, tx) = tx.into_parts();
if let Some(gas_used) =
self.executor.execute_transaction_with_commit_condition(tx.as_executable(), f)?
self.executor.execute_transaction_with_commit_condition((tx_env, &tx), f)?
{
self.transactions.push(tx.into_recovered());
self.transactions.push(tx);
Ok(Some(gas_used))
} else {
Ok(None)
@@ -623,20 +592,20 @@ where
}
}
/// A helper trait marking a 'static type that can be converted into an [`ExecutableTx`] for block
/// executor.
/// A helper trait marking a 'static type that can be converted into an [`ExecutableTxParts`] for
/// block executor.
pub trait ExecutableTxFor<Evm: ConfigureEvm>:
ToTxEnv<TxEnvFor<Evm>> + RecoveredTx<TxTy<Evm::Primitives>>
ExecutableTxParts<TxEnvFor<Evm>, TxTy<Evm::Primitives>> + RecoveredTx<TxTy<Evm::Primitives>>
{
}
impl<T, Evm: ConfigureEvm> ExecutableTxFor<Evm> for T where
T: ToTxEnv<TxEnvFor<Evm>> + RecoveredTx<TxTy<Evm::Primitives>>
T: ExecutableTxParts<TxEnvFor<Evm>, TxTy<Evm::Primitives>> + RecoveredTx<TxTy<Evm::Primitives>>
{
}
/// A container for a transaction and a transaction environment.
#[derive(Debug, Clone)]
#[derive(Debug)]
pub struct WithTxEnv<TxEnv, T> {
/// The transaction environment for EVM.
pub tx_env: TxEnv,
@@ -644,6 +613,12 @@ pub struct WithTxEnv<TxEnv, T> {
pub tx: Arc<T>,
}
impl<TxEnv: Clone, T> Clone for WithTxEnv<TxEnv, T> {
fn clone(&self) -> Self {
Self { tx_env: self.tx_env.clone(), tx: self.tx.clone() }
}
}
impl<TxEnv, Tx, T: RecoveredTx<Tx>> RecoveredTx<Tx> for WithTxEnv<TxEnv, T> {
fn tx(&self) -> &Tx {
self.tx.tx()
@@ -654,9 +629,11 @@ impl<TxEnv, Tx, T: RecoveredTx<Tx>> RecoveredTx<Tx> for WithTxEnv<TxEnv, T> {
}
}
impl<TxEnv: Clone, T> ToTxEnv<TxEnv> for WithTxEnv<TxEnv, T> {
fn to_tx_env(&self) -> TxEnv {
self.tx_env.clone()
impl<TxEnv, T: RecoveredTx<Tx>, Tx> ExecutableTxParts<TxEnv, Tx> for WithTxEnv<TxEnv, T> {
type Recovered = Arc<T>;
fn into_parts(self) -> (TxEnv, Self::Recovered) {
(self.tx_env, self.tx)
}
}

View File

@@ -2,9 +2,9 @@
use crate::ExecutionOutcome;
use alloc::{borrow::Cow, collections::BTreeMap, vec::Vec};
use alloy_consensus::{transaction::Recovered, BlockHeader};
use alloy_consensus::{transaction::Recovered, BlockHeader, TxReceipt};
use alloy_eips::{eip1898::ForkBlock, eip2718::Encodable2718, BlockNumHash};
use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash};
use alloy_primitives::{Address, BlockHash, BlockNumber, Log, TxHash};
use core::{fmt, ops::RangeInclusive};
use reth_primitives_traits::{
transaction::signed::SignedTransaction, Block, BlockBody, IndexedTx, NodePrimitives,
@@ -184,6 +184,19 @@ impl<N: NodePrimitives> Chain<N> {
self.execution_outcome.receipts().iter()
}
/// Returns an iterator over all receipts in the chain.
pub fn receipts_iter(&self) -> impl Iterator<Item = &N::Receipt> + '_ {
self.block_receipts_iter().flatten()
}
/// Returns an iterator over all logs in the chain.
pub fn logs_iter(&self) -> impl Iterator<Item = &Log> + '_
where
N::Receipt: TxReceipt<Log = Log>,
{
self.receipts_iter().flat_map(|receipt| receipt.logs())
}
/// Returns an iterator over all blocks in the chain with increasing block number.
pub fn blocks_iter(&self) -> impl Iterator<Item = &RecoveredBlock<N::Block>> + '_ {
self.blocks().iter().map(|block| block.1)

View File

@@ -131,6 +131,8 @@ pub struct TransactionsManagerMetrics {
/// capacity. Note, this is not a limit to the number of inflight requests, but a health
/// measure.
pub(crate) capacity_pending_pool_imports: Counter,
/// Total number of transactions ignored because pending pool imports are at capacity.
pub(crate) skipped_transactions_pending_pool_imports_at_capacity: Counter,
/// The time it took to prepare transactions for import. This is mostly sender recovery.
pub(crate) pool_import_prepare_duration: Histogram,

View File

@@ -429,11 +429,22 @@ impl<Pool: TransactionPool, N: NetworkPrimitives> TransactionsManager<Pool, N> {
/// Returns `true` if [`TransactionsManager`] has capacity to request pending hashes. Returns
/// `false` if [`TransactionsManager`] is operating close to full capacity.
fn has_capacity_for_fetching_pending_hashes(&self) -> bool {
self.pending_pool_imports_info
.has_capacity(self.pending_pool_imports_info.max_pending_pool_imports) &&
self.has_capacity_for_pending_pool_imports() &&
self.transaction_fetcher.has_capacity_for_fetching_pending_hashes()
}
/// Returns `true` if [`TransactionsManager`] has capacity for more pending pool imports.
fn has_capacity_for_pending_pool_imports(&self) -> bool {
self.remaining_pool_import_capacity() > 0
}
/// Returns the remaining capacity for pending pool imports.
fn remaining_pool_import_capacity(&self) -> usize {
self.pending_pool_imports_info.max_pending_pool_imports.saturating_sub(
self.pending_pool_imports_info.pending_pool_imports.load(Ordering::Relaxed),
)
}
fn report_peer_bad_transactions(&self, peer_id: PeerId) {
self.report_peer(peer_id, ReputationChangeKind::BadTransactions);
self.metrics.reported_bad_transactions.increment(1);
@@ -1285,6 +1296,7 @@ where
trace!(target: "net::tx", peer_id=format!("{peer_id:#}"), policy=?self.config.ingress_policy, "Ignoring full transactions from peer blocked by ingress policy");
return;
}
// ensure we didn't receive any blob transactions as these are disallowed to be
// broadcasted in full
@@ -1335,7 +1347,13 @@ where
return
}
// Early return if we don't have capacity for any imports
if !self.has_capacity_for_pending_pool_imports() {
return
}
let Some(peer) = self.peers.get_mut(&peer_id) else { return };
let client_version = peer.client_version.clone();
let mut transactions = transactions.0;
let start = Instant::now();
@@ -1378,7 +1396,7 @@ where
trace!(target: "net::tx",
peer_id=format!("{peer_id:#}"),
hash=%tx.tx_hash(),
client_version=%peer.client_version,
%client_version,
"received a known bad transaction from peer"
);
has_bad_transactions = true;
@@ -1387,6 +1405,18 @@ where
true
});
// Truncate to remaining capacity before recovery to avoid wasting CPU on transactions
// that won't be imported anyway.
let capacity = self.remaining_pool_import_capacity();
if transactions.len() > capacity {
let skipped = transactions.len() - capacity;
transactions.truncate(capacity);
self.metrics
.skipped_transactions_pending_pool_imports_at_capacity
.increment(skipped as u64);
trace!(target: "net::tx", skipped, capacity, "Truncated transactions batch to capacity");
}
let txs_len = transactions.len();
let new_txs = transactions
@@ -1397,7 +1427,7 @@ where
trace!(target: "net::tx",
peer_id=format!("{peer_id:#}"),
hash=%badtx.tx_hash(),
client_version=%peer.client_version,
client_version=%client_version,
"failed ecrecovery for transaction"
);
None
@@ -1448,7 +1478,7 @@ where
self.metrics
.occurrences_of_transaction_already_seen_by_peer
.increment(num_already_seen_by_peer);
trace!(target: "net::tx", num_txs=%num_already_seen_by_peer, ?peer_id, client=?peer.client_version, "Peer sent already seen transactions");
trace!(target: "net::tx", num_txs=%num_already_seen_by_peer, ?peer_id, client=%client_version, "Peer sent already seen transactions");
}
if has_bad_transactions {

View File

@@ -4,7 +4,7 @@ use crate::{BuilderContext, FullNodeTypes};
use alloy_primitives::Address;
use reth_chain_state::CanonStateSubscriptions;
use reth_chainspec::EthereumHardforks;
use reth_node_api::{NodeTypes, TxTy};
use reth_node_api::{BlockTy, NodeTypes, TxTy};
use reth_transaction_pool::{
blobstore::DiskFileBlobStore, BlobStore, CoinbaseTipOrdering, PoolConfig, PoolTransaction,
SubPoolLimit, TransactionPool, TransactionValidationTaskExecutor, TransactionValidator,
@@ -129,7 +129,7 @@ impl<'a, Node: FullNodeTypes, V> TxPoolBuilder<'a, Node, V> {
impl<'a, Node, V> TxPoolBuilder<'a, Node, TransactionValidationTaskExecutor<V>>
where
Node: FullNodeTypes<Types: NodeTypes<ChainSpec: EthereumHardforks>>,
V: TransactionValidator + 'static,
V: TransactionValidator<Block = BlockTy<Node::Types>> + 'static,
V::Transaction:
PoolTransaction<Consensus = TxTy<Node::Types>> + reth_transaction_pool::EthPoolTransaction,
{
@@ -248,7 +248,7 @@ fn spawn_pool_maintenance_task<Node, Pool>(
) -> eyre::Result<()>
where
Node: FullNodeTypes<Types: NodeTypes<ChainSpec: EthereumHardforks>>,
Pool: reth_transaction_pool::TransactionPoolExt + Clone + 'static,
Pool: reth_transaction_pool::TransactionPoolExt<Block = BlockTy<Node::Types>> + Clone + 'static,
Pool::Transaction: PoolTransaction<Consensus = TxTy<Node::Types>>,
{
let chain_events = ctx.provider().canonical_state_stream();
@@ -280,7 +280,7 @@ pub fn spawn_maintenance_tasks<Node, Pool>(
) -> eyre::Result<()>
where
Node: FullNodeTypes<Types: NodeTypes<ChainSpec: EthereumHardforks>>,
Pool: reth_transaction_pool::TransactionPoolExt + Clone + 'static,
Pool: reth_transaction_pool::TransactionPoolExt<Block = BlockTy<Node::Types>> + Clone + 'static,
Pool::Transaction: PoolTransaction<Consensus = TxTy<Node::Types>>,
{
spawn_local_backup_task(ctx, pool.clone())?;

View File

@@ -676,19 +676,13 @@ where
/// Convenience function to [`Self::init_genesis`]
pub fn with_genesis(self) -> Result<Self, InitStorageError> {
init_genesis_with_settings(
self.provider_factory(),
self.node_config().static_files.to_settings(),
)?;
init_genesis_with_settings(self.provider_factory(), self.node_config().storage_settings())?;
Ok(self)
}
/// Write the genesis block and state if it has not already been written
pub fn init_genesis(&self) -> Result<B256, InitStorageError> {
init_genesis_with_settings(
self.provider_factory(),
self.node_config().static_files.to_settings(),
)
init_genesis_with_settings(self.provider_factory(), self.node_config().storage_settings())
}
/// Creates a new `WithMeteredProvider` container and attaches it to the
@@ -1283,6 +1277,10 @@ pub fn metrics_hooks<N: NodeTypesWithDB>(provider_factory: &ProviderFactory<N>)
})
}
})
.with_hook({
let rocksdb = provider_factory.rocksdb_provider();
move || throttle!(Duration::from_secs(5 * 60), || rocksdb.report_metrics())
})
.build()
}

View File

@@ -4,10 +4,13 @@ use alloy_consensus::transaction::Either;
use alloy_provider::network::AnyNetwork;
use jsonrpsee::core::{DeserializeOwned, Serialize};
use reth_chainspec::EthChainSpec;
use reth_consensus_debug_client::{DebugConsensusClient, EtherscanBlockProvider, RpcBlockProvider};
use reth_consensus_debug_client::{
BlockProvider, DebugConsensusClient, EtherscanBlockProvider, RpcBlockProvider,
};
use reth_engine_local::LocalMiner;
use reth_node_api::{
BlockTy, FullNodeComponents, HeaderTy, PayloadAttrTy, PayloadAttributesBuilder, PayloadTypes,
BlockTy, FullNodeComponents, FullNodeTypes, HeaderTy, PayloadAttrTy, PayloadAttributesBuilder,
PayloadTypes,
};
use std::{
future::{Future, IntoFuture},
@@ -109,9 +112,16 @@ impl<L> DebugNodeLauncher<L> {
}
}
/// Type alias for the default debug block provider. We use etherscan provider to satisfy the
/// bounds.
pub type DefaultDebugBlockProvider<N> = EtherscanBlockProvider<
<<N as FullNodeTypes>::Types as DebugNode<N>>::RpcBlock,
BlockTy<<N as FullNodeTypes>::Types>,
>;
/// Future for the [`DebugNodeLauncher`].
#[expect(missing_debug_implementations, clippy::type_complexity)]
pub struct DebugNodeLauncherFuture<L, Target, N>
pub struct DebugNodeLauncherFuture<L, Target, N, B = DefaultDebugBlockProvider<N>>
where
N: FullNodeComponents<Types: DebugNode<N>>,
{
@@ -121,14 +131,17 @@ where
Option<Box<dyn PayloadAttributesBuilder<PayloadAttrTy<N::Types>, HeaderTy<N::Types>>>>,
map_attributes:
Option<Box<dyn Fn(PayloadAttrTy<N::Types>) -> PayloadAttrTy<N::Types> + Send + Sync>>,
debug_block_provider: Option<B>,
}
impl<L, Target, N, AddOns> DebugNodeLauncherFuture<L, Target, N>
impl<L, Target, N, AddOns, B> DebugNodeLauncherFuture<L, Target, N, B>
where
N: FullNodeComponents<Types: DebugNode<N>>,
AddOns: RethRpcAddOns<N>,
L: LaunchNode<Target, Node = NodeHandle<N, AddOns>>,
B: BlockProvider<Block = BlockTy<N::Types>> + Clone,
{
/// Sets a custom payload attributes builder for local mining in dev mode.
pub fn with_payload_attributes_builder(
self,
builder: impl PayloadAttributesBuilder<PayloadAttrTy<N::Types>, HeaderTy<N::Types>>,
@@ -138,9 +151,11 @@ where
target: self.target,
local_payload_attributes_builder: Some(Box::new(builder)),
map_attributes: None,
debug_block_provider: self.debug_block_provider,
}
}
/// Sets a function to map payload attributes before building.
pub fn map_debug_payload_attributes(
self,
f: impl Fn(PayloadAttrTy<N::Types>) -> PayloadAttrTy<N::Types> + Send + Sync + 'static,
@@ -150,16 +165,58 @@ where
target: self.target,
local_payload_attributes_builder: None,
map_attributes: Some(Box::new(f)),
debug_block_provider: self.debug_block_provider,
}
}
/// Sets a custom block provider for the debug consensus client.
///
/// When set, this provider will be used instead of creating an `EtherscanBlockProvider`
/// or `RpcBlockProvider` from CLI arguments.
pub fn with_debug_block_provider<B2>(
self,
provider: B2,
) -> DebugNodeLauncherFuture<L, Target, N, B2>
where
B2: BlockProvider<Block = BlockTy<N::Types>> + Clone,
{
DebugNodeLauncherFuture {
inner: self.inner,
target: self.target,
local_payload_attributes_builder: self.local_payload_attributes_builder,
map_attributes: self.map_attributes,
debug_block_provider: Some(provider),
}
}
async fn launch_node(self) -> eyre::Result<NodeHandle<N, AddOns>> {
let Self { inner, target, local_payload_attributes_builder, map_attributes } = self;
let Self {
inner,
target,
local_payload_attributes_builder,
map_attributes,
debug_block_provider,
} = self;
let handle = inner.launch_node(target).await?;
let config = &handle.node.config;
if let Some(url) = config.debug.rpc_consensus_url.clone() {
if let Some(provider) = debug_block_provider {
info!(target: "reth::cli", "Using custom debug block provider");
let rpc_consensus_client = DebugConsensusClient::new(
handle.node.add_ons_handle.beacon_engine_handle.clone(),
Arc::new(provider),
);
handle
.node
.task_executor
.spawn_critical("custom debug block provider consensus client", async move {
rpc_consensus_client.run().await
});
} else if let Some(url) = config.debug.rpc_consensus_url.clone() {
info!(target: "reth::cli", "Using RPC consensus client: {}", url);
let block_provider =
@@ -180,14 +237,11 @@ where
handle.node.task_executor.spawn_critical("rpc-ws consensus client", async move {
rpc_consensus_client.run().await
});
}
if let Some(maybe_custom_etherscan_url) = config.debug.etherscan.clone() {
} else if let Some(maybe_custom_etherscan_url) = config.debug.etherscan.clone() {
info!(target: "reth::cli", "Using etherscan as consensus client");
let chain = config.chain.chain();
let etherscan_url = maybe_custom_etherscan_url.map(Ok).unwrap_or_else(|| {
// If URL isn't provided, use default Etherscan URL for the chain if it is known
chain
.etherscan_urls()
.map(|urls| urls.0.to_string())
@@ -252,12 +306,13 @@ where
}
}
impl<L, Target, N, AddOns> IntoFuture for DebugNodeLauncherFuture<L, Target, N>
impl<L, Target, N, AddOns, B> IntoFuture for DebugNodeLauncherFuture<L, Target, N, B>
where
Target: Send + 'static,
N: FullNodeComponents<Types: DebugNode<N>>,
AddOns: RethRpcAddOns<N> + 'static,
L: LaunchNode<Target, Node = NodeHandle<N, AddOns>> + 'static,
B: BlockProvider<Block = BlockTy<N::Types>> + Clone + 'static,
{
type Output = eyre::Result<NodeHandle<N, AddOns>>;
type IntoFuture = Pin<Box<dyn Future<Output = eyre::Result<NodeHandle<N, AddOns>>> + Send>>;
@@ -273,6 +328,7 @@ where
N: FullNodeComponents<Types: DebugNode<N>>,
AddOns: RethRpcAddOns<N> + 'static,
L: LaunchNode<Target, Node = NodeHandle<N, AddOns>> + 'static,
DefaultDebugBlockProvider<N>: BlockProvider<Block = BlockTy<N::Types>> + Clone,
{
type Node = NodeHandle<N, AddOns>;
type Future = DebugNodeLauncherFuture<L, Target, N>;
@@ -283,6 +339,7 @@ where
target,
local_payload_attributes_builder: None,
map_attributes: None,
debug_block_provider: None,
}
}
}

View File

@@ -32,7 +32,7 @@ use reth_node_core::{
use reth_node_events::node;
use reth_provider::{
providers::{BlockchainProvider, NodeTypesForProvider},
BlockNumReader, MetadataProvider,
BlockNumReader, StorageSettingsCache,
};
use reth_tasks::TaskExecutor;
use reth_tokio_util::EventSender;
@@ -41,7 +41,6 @@ use reth_trie_db::ChangesetCache;
use std::{future::Future, pin::Pin, sync::Arc};
use tokio::sync::{mpsc::unbounded_channel, oneshot};
use tokio_stream::wrappers::UnboundedReceiverStream;
use tracing::warn;
/// The engine node launcher.
#[derive(Debug)]
@@ -104,24 +103,8 @@ impl EngineNodeLauncher {
.with_adjusted_configs()
// Create the provider factory with changeset cache
.with_provider_factory::<_, <CB::Components as NodeComponents<T>>::Evm>(changeset_cache.clone()).await?
.inspect(|ctx| {
.inspect(|_| {
info!(target: "reth::cli", "Database opened");
match ctx.provider_factory().storage_settings() {
Ok(settings) => {
info!(
target: "reth::cli",
?settings,
"Storage settings"
);
},
Err(err) => {
warn!(
target: "reth::cli",
?err,
"Failed to get storage settings"
);
},
}
})
.with_prometheus_server().await?
.inspect(|this| {
@@ -130,6 +113,8 @@ impl EngineNodeLauncher {
.with_genesis()?
.inspect(|this: &LaunchContextWith<Attached<WithConfigs<<T::Types as NodeTypes>::ChainSpec>, _>>| {
info!(target: "reth::cli", "\n{}", this.chain_spec().display_hardforks());
let settings = this.provider_factory().cached_storage_settings();
info!(target: "reth::cli", ?settings, "Loaded storage settings");
})
.with_metrics_task()
// passing FullNodeTypes as type parameter here so that we can build

View File

@@ -31,7 +31,7 @@ pub use builder::{add_ons::AddOns, *};
mod launch;
pub use launch::{
debug::{DebugNode, DebugNodeLauncher},
debug::{DebugNode, DebugNodeLauncher, DebugNodeLauncherFuture, DefaultDebugBlockProvider},
engine::EngineNodeLauncher,
*,
};

View File

@@ -19,7 +19,6 @@ reth-cli-util.workspace = true
reth-db = { workspace = true, features = ["mdbx"] }
reth-storage-errors.workspace = true
reth-storage-api = { workspace = true, features = ["std", "db-api"] }
reth-provider.workspace = true
reth-network = { workspace = true, features = ["serde"] }
reth-network-p2p.workspace = true
reth-rpc-eth-types.workspace = true
@@ -92,7 +91,7 @@ min-debug-logs = ["tracing/release_max_level_debug"]
min-trace-logs = ["tracing/release_max_level_trace"]
# Marker feature for edge/unstable builds - captured by vergen in build.rs
edge = []
edge = ["reth-storage-api/edge"]
[build-dependencies]
vergen = { workspace = true, features = ["build", "cargo", "emit_and_set"] }

View File

@@ -37,6 +37,7 @@ pub struct DefaultEngineValues {
storage_worker_count: Option<usize>,
account_worker_count: Option<usize>,
enable_proof_v2: bool,
cache_metrics_disabled: bool,
}
impl DefaultEngineValues {
@@ -172,6 +173,12 @@ impl DefaultEngineValues {
self.enable_proof_v2 = v;
self
}
/// Set whether to disable cache metrics by default
pub const fn with_cache_metrics_disabled(mut self, v: bool) -> Self {
self.cache_metrics_disabled = v;
self
}
}
impl Default for DefaultEngineValues {
@@ -197,6 +204,7 @@ impl Default for DefaultEngineValues {
storage_worker_count: None,
account_worker_count: None,
enable_proof_v2: false,
cache_metrics_disabled: false,
}
}
}
@@ -320,6 +328,10 @@ pub struct EngineArgs {
/// Enable V2 storage proofs for state root calculations
#[arg(long = "engine.enable-proof-v2", default_value_t = DefaultEngineValues::get_global().enable_proof_v2)]
pub enable_proof_v2: bool,
/// Disable cache metrics recording, which can take up to 50ms with large cached state.
#[arg(long = "engine.disable-cache-metrics", default_value_t = DefaultEngineValues::get_global().cache_metrics_disabled)]
pub cache_metrics_disabled: bool,
}
#[allow(deprecated)]
@@ -346,6 +358,7 @@ impl Default for EngineArgs {
storage_worker_count,
account_worker_count,
enable_proof_v2,
cache_metrics_disabled,
} = DefaultEngineValues::get_global().clone();
Self {
persistence_threshold,
@@ -371,6 +384,7 @@ impl Default for EngineArgs {
storage_worker_count,
account_worker_count,
enable_proof_v2,
cache_metrics_disabled,
}
}
}
@@ -407,6 +421,7 @@ impl EngineArgs {
}
config = config.with_enable_proof_v2(self.enable_proof_v2);
config = config.without_cache_metrics(self.cache_metrics_disabled);
config
}
@@ -458,6 +473,7 @@ mod tests {
storage_worker_count: Some(16),
account_worker_count: Some(8),
enable_proof_v2: false,
cache_metrics_disabled: true,
};
let parsed_args = CommandParser::<EngineArgs>::parse_from([
@@ -488,6 +504,7 @@ mod tests {
"16",
"--engine.account-worker-count",
"8",
"--engine.disable-cache-metrics",
])
.args;

View File

@@ -54,7 +54,7 @@ pub use dev::DevArgs;
/// PruneArgs for configuring the pruning and full node
mod pruning;
pub use pruning::PruningArgs;
pub use pruning::{DefaultPruningValues, PruningArgs};
/// DatadirArgs for configuring data storage paths
mod datadir_args;
@@ -80,5 +80,9 @@ pub use era::{DefaultEraHost, EraArgs, EraSourceArgs};
mod static_files;
pub use static_files::{StaticFilesArgs, MINIMAL_BLOCKS_PER_FILE};
/// `RocksDbArgs` for configuring RocksDB table routing.
mod rocksdb;
pub use rocksdb::{RocksDbArgs, RocksDbArgsError};
mod error;
pub mod types;

View File

@@ -6,7 +6,88 @@ use clap::{builder::RangedU64ValueParser, Args};
use reth_chainspec::EthereumHardforks;
use reth_config::config::PruneConfig;
use reth_prune_types::{PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_PRUNING_DISTANCE};
use std::{collections::BTreeMap, ops::Not};
use std::{collections::BTreeMap, ops::Not, sync::OnceLock};
/// Global static pruning defaults
static PRUNING_DEFAULTS: OnceLock<DefaultPruningValues> = OnceLock::new();
/// Default values for `--full` and `--minimal` pruning modes that can be customized.
///
/// Global defaults can be set via [`DefaultPruningValues::try_init`].
#[derive(Debug, Clone)]
pub struct DefaultPruningValues {
/// Prune modes for `--full` flag.
///
/// Note: `bodies_history` is ignored when `full_bodies_history_use_pre_merge` is `true`.
pub full_prune_modes: PruneModes,
/// If `true`, `--full` will set `bodies_history` to prune everything before the merge block
/// (Paris hardfork). If `false`, uses `full_prune_modes.bodies_history` directly.
pub full_bodies_history_use_pre_merge: bool,
/// Prune modes for `--minimal` flag.
pub minimal_prune_modes: PruneModes,
}
impl DefaultPruningValues {
/// Initialize the global pruning defaults with this configuration.
///
/// Returns `Err(self)` if already initialized.
pub fn try_init(self) -> Result<(), Self> {
PRUNING_DEFAULTS.set(self)
}
/// Get a reference to the global pruning defaults.
pub fn get_global() -> &'static Self {
PRUNING_DEFAULTS.get_or_init(Self::default)
}
/// Set the prune modes for `--full` flag.
pub fn with_full_prune_modes(mut self, modes: PruneModes) -> Self {
self.full_prune_modes = modes;
self
}
/// Set whether `--full` should use pre-merge pruning for bodies history.
///
/// When `true` (default), bodies are pruned before the Paris hardfork block.
/// When `false`, uses `full_prune_modes.bodies_history` directly.
pub const fn with_full_bodies_history_use_pre_merge(mut self, use_pre_merge: bool) -> Self {
self.full_bodies_history_use_pre_merge = use_pre_merge;
self
}
/// Set the prune modes for `--minimal` flag.
pub fn with_minimal_prune_modes(mut self, modes: PruneModes) -> Self {
self.minimal_prune_modes = modes;
self
}
}
impl Default for DefaultPruningValues {
fn default() -> Self {
Self {
full_prune_modes: PruneModes {
sender_recovery: Some(PruneMode::Full),
transaction_lookup: None,
receipts: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
account_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
storage_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
// This field is ignored when full_bodies_history_use_pre_merge is true
bodies_history: None,
receipts_log_filter: Default::default(),
},
full_bodies_history_use_pre_merge: true,
minimal_prune_modes: PruneModes {
sender_recovery: Some(PruneMode::Full),
transaction_lookup: Some(PruneMode::Full),
receipts: Some(PruneMode::Full),
account_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
storage_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
bodies_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
receipts_log_filter: Default::default(),
},
}
}
}
/// Parameters for pruning and full node
#[derive(Debug, Clone, Args, PartialEq, Eq, Default)]
@@ -128,36 +209,22 @@ impl PruningArgs {
// If --full is set, use full node defaults.
if self.full {
config = PruneConfig {
block_interval: config.block_interval,
segments: PruneModes {
sender_recovery: Some(PruneMode::Full),
transaction_lookup: None,
receipts: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
account_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
storage_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)),
bodies_history: chain_spec
.ethereum_fork_activation(EthereumHardfork::Paris)
.block_number()
.map(PruneMode::Before),
receipts_log_filter: Default::default(),
},
let defaults = DefaultPruningValues::get_global();
let mut segments = defaults.full_prune_modes.clone();
if defaults.full_bodies_history_use_pre_merge {
segments.bodies_history = chain_spec
.ethereum_fork_activation(EthereumHardfork::Paris)
.block_number()
.map(PruneMode::Before);
}
config = PruneConfig { block_interval: config.block_interval, segments }
}
// If --minimal is set, use minimal storage mode with aggressive pruning.
if self.minimal {
config = PruneConfig {
block_interval: config.block_interval,
segments: PruneModes {
sender_recovery: Some(PruneMode::Full),
transaction_lookup: Some(PruneMode::Full),
receipts: Some(PruneMode::Full),
account_history: Some(PruneMode::Distance(10064)),
storage_history: Some(PruneMode::Distance(10064)),
bodies_history: Some(PruneMode::Distance(10064)),
receipts_log_filter: Default::default(),
},
segments: DefaultPruningValues::get_global().minimal_prune_modes.clone(),
}
}

View File

@@ -0,0 +1,160 @@
//! clap [Args](clap::Args) for `RocksDB` table routing configuration
use clap::{ArgAction, Args};
/// Default value for `RocksDB` routing flags.
///
/// When the `edge` feature is enabled, defaults to `true` to enable edge storage features.
/// Otherwise defaults to `false` for legacy behavior.
const fn default_rocksdb_flag() -> bool {
cfg!(feature = "edge")
}
/// Parameters for `RocksDB` table routing configuration.
///
/// These flags control which database tables are stored in `RocksDB` instead of MDBX.
/// All flags are genesis-initialization-only: changing them after genesis requires a re-sync.
#[derive(Debug, Args, PartialEq, Eq, Clone, Copy)]
#[command(next_help_heading = "RocksDB")]
pub struct RocksDbArgs {
/// Route all supported tables to `RocksDB` instead of MDBX.
///
/// This enables `RocksDB` for `tx-hash`, `storages-history`, and `account-history` tables.
/// Cannot be combined with individual flags set to false.
#[arg(long = "rocksdb.all", action = ArgAction::SetTrue)]
pub all: bool,
/// Route tx hash -> number table to `RocksDB` instead of MDBX.
///
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
/// Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
#[arg(long = "rocksdb.tx-hash", default_value_t = default_rocksdb_flag(), action = ArgAction::Set)]
pub tx_hash: bool,
/// Route storages history tables to `RocksDB` instead of MDBX.
///
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
/// Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
#[arg(long = "rocksdb.storages-history", default_value_t = default_rocksdb_flag(), action = ArgAction::Set)]
pub storages_history: bool,
/// Route account history tables to `RocksDB` instead of MDBX.
///
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
/// Defaults to `true` when the `edge` feature is enabled, `false` otherwise.
#[arg(long = "rocksdb.account-history", default_value_t = default_rocksdb_flag(), action = ArgAction::Set)]
pub account_history: bool,
}
impl Default for RocksDbArgs {
fn default() -> Self {
Self {
all: false,
tx_hash: default_rocksdb_flag(),
storages_history: default_rocksdb_flag(),
account_history: default_rocksdb_flag(),
}
}
}
impl RocksDbArgs {
/// Validates the `RocksDB` arguments.
///
/// Returns an error if `--rocksdb.all` is used with any individual flag set to `false`.
pub const fn validate(&self) -> Result<(), RocksDbArgsError> {
if self.all {
if !self.tx_hash {
return Err(RocksDbArgsError::ConflictingFlags("tx-hash"));
}
if !self.storages_history {
return Err(RocksDbArgsError::ConflictingFlags("storages-history"));
}
if !self.account_history {
return Err(RocksDbArgsError::ConflictingFlags("account-history"));
}
}
Ok(())
}
}
/// Error type for `RocksDB` argument validation.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum RocksDbArgsError {
/// `--rocksdb.all` cannot be combined with an individual flag set to false.
#[error("--rocksdb.all cannot be combined with --rocksdb.{0}=false")]
ConflictingFlags(&'static str),
}
#[cfg(test)]
mod tests {
use super::*;
use clap::Parser;
#[derive(Parser)]
struct CommandParser<T: Args> {
#[command(flatten)]
args: T,
}
#[test]
fn test_default_rocksdb_args() {
let args = CommandParser::<RocksDbArgs>::parse_from(["reth"]).args;
assert_eq!(args, RocksDbArgs::default());
}
#[test]
fn test_parse_all_flag() {
let args = CommandParser::<RocksDbArgs>::parse_from(["reth", "--rocksdb.all"]).args;
assert!(args.all);
assert_eq!(args.tx_hash, default_rocksdb_flag());
}
#[test]
fn test_parse_individual_flags() {
let args = CommandParser::<RocksDbArgs>::parse_from([
"reth",
"--rocksdb.tx-hash=true",
"--rocksdb.storages-history=false",
"--rocksdb.account-history=true",
])
.args;
assert!(!args.all);
assert!(args.tx_hash);
assert!(!args.storages_history);
assert!(args.account_history);
}
#[test]
fn test_validate_all_with_true_ok() {
let args =
RocksDbArgs { all: true, tx_hash: true, storages_history: true, account_history: true };
assert!(args.validate().is_ok());
}
#[test]
fn test_validate_all_with_false_errors() {
let args = RocksDbArgs {
all: true,
tx_hash: false,
storages_history: true,
account_history: true,
};
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("tx-hash")));
let args = RocksDbArgs {
all: true,
tx_hash: true,
storages_history: false,
account_history: true,
};
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("storages-history")));
let args = RocksDbArgs {
all: true,
tx_hash: true,
storages_history: true,
account_history: false,
};
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("account-history")));
}
}

View File

@@ -2,15 +2,23 @@
use clap::Args;
use reth_config::config::{BlocksPerFileConfig, StaticFilesConfig};
use reth_provider::StorageSettings;
use reth_storage_api::StorageSettings;
/// Blocks per static file when running in `--minimal` node.
///
/// 10000 blocks per static file allows us to prune all history every 10k blocks.
pub const MINIMAL_BLOCKS_PER_FILE: u64 = 10000;
/// Default value for static file storage flags.
///
/// When the `edge` feature is enabled, defaults to `true` to enable edge storage features.
/// Otherwise defaults to `false` for legacy behavior.
const fn default_static_file_flag() -> bool {
cfg!(feature = "edge")
}
/// Parameters for static files configuration
#[derive(Debug, Args, PartialEq, Eq, Default, Clone, Copy)]
#[derive(Debug, Args, PartialEq, Eq, Clone, Copy)]
#[command(next_help_heading = "Static Files")]
pub struct StaticFilesArgs {
/// Number of blocks per file for the headers segment.
@@ -33,13 +41,17 @@ pub struct StaticFilesArgs {
#[arg(long = "static-files.blocks-per-file.account-change-sets")]
pub blocks_per_file_account_change_sets: Option<u64>,
/// Number of blocks per file for the storage changesets segment.
#[arg(long = "static-files.blocks-per-file.storage-change-sets")]
pub blocks_per_file_storage_change_sets: Option<u64>,
/// Store receipts in static files instead of the database.
///
/// When enabled, receipts will be written to static files on disk instead of the database.
///
/// Note: This setting can only be configured at genesis initialization. Once
/// the node has been initialized, changing this flag requires re-syncing from scratch.
#[arg(long = "static-files.receipts")]
#[arg(long = "static-files.receipts", default_value_t = default_static_file_flag(), action = clap::ArgAction::Set)]
pub receipts: bool,
/// Store transaction senders in static files instead of the database.
@@ -49,7 +61,7 @@ pub struct StaticFilesArgs {
///
/// Note: This setting can only be configured at genesis initialization. Once
/// the node has been initialized, changing this flag requires re-syncing from scratch.
#[arg(long = "static-files.transaction-senders")]
#[arg(long = "static-files.transaction-senders", default_value_t = default_static_file_flag(), action = clap::ArgAction::Set)]
pub transaction_senders: bool,
/// Store account changesets in static files.
@@ -59,8 +71,18 @@ pub struct StaticFilesArgs {
///
/// Note: This setting can only be configured at genesis initialization. Once
/// the node has been initialized, changing this flag requires re-syncing from scratch.
#[arg(long = "static-files.account-change-sets")]
#[arg(long = "static-files.account-change-sets", default_value_t = default_static_file_flag(), action = clap::ArgAction::Set)]
pub account_changesets: bool,
/// Store storage changesets in static files.
///
/// When enabled, storage changesets will be written to static files on disk instead of the
/// database.
///
/// Note: This setting can only be configured at genesis initialization. Once
/// the node has been initialized, changing this flag requires re-syncing from scratch.
#[arg(long = "static-files.storage-change-sets", default_value_t = default_static_file_flag(), action = clap::ArgAction::Set)]
pub storage_changesets: bool,
}
impl StaticFilesArgs {
@@ -91,15 +113,40 @@ impl StaticFilesArgs {
account_change_sets: self
.blocks_per_file_account_change_sets
.or(config.blocks_per_file.account_change_sets),
storage_change_sets: self
.blocks_per_file_storage_change_sets
.or(config.blocks_per_file.storage_change_sets),
},
}
}
/// Converts the static files arguments into [`StorageSettings`].
pub const fn to_settings(&self) -> StorageSettings {
StorageSettings::legacy()
.with_receipts_in_static_files(self.receipts)
#[cfg(feature = "edge")]
let base = StorageSettings::edge();
#[cfg(not(feature = "edge"))]
let base = StorageSettings::legacy();
base.with_receipts_in_static_files(self.receipts)
.with_transaction_senders_in_static_files(self.transaction_senders)
.with_account_changesets_in_static_files(self.account_changesets)
.with_storage_changesets_in_static_files(self.storage_changesets)
}
}
impl Default for StaticFilesArgs {
fn default() -> Self {
Self {
blocks_per_file_headers: None,
blocks_per_file_transactions: None,
blocks_per_file_receipts: None,
blocks_per_file_transaction_senders: None,
blocks_per_file_account_change_sets: None,
blocks_per_file_storage_change_sets: None,
receipts: default_static_file_flag(),
transaction_senders: default_static_file_flag(),
account_changesets: default_static_file_flag(),
storage_changesets: default_static_file_flag(),
}
}
}

View File

@@ -3,7 +3,7 @@
use crate::{
args::{
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, NetworkArgs, PayloadBuilderArgs,
PruningArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
},
dirs::{ChainPath, DataDirPath},
utils::get_single_header,
@@ -21,6 +21,7 @@ use reth_primitives_traits::SealedHeader;
use reth_stages_types::StageId;
use reth_storage_api::{
BlockHashReader, DatabaseProviderFactory, HeaderProvider, StageCheckpointReader,
StorageSettings,
};
use reth_storage_errors::provider::ProviderResult;
use reth_transaction_pool::TransactionPool;
@@ -150,6 +151,9 @@ pub struct NodeConfig<ChainSpec> {
/// All static files related arguments
pub static_files: StaticFilesArgs,
/// All `RocksDB` table routing arguments
pub rocksdb: RocksDbArgs,
}
impl NodeConfig<ChainSpec> {
@@ -181,6 +185,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
engine: EngineArgs::default(),
era: EraArgs::default(),
static_files: StaticFilesArgs::default(),
rocksdb: RocksDbArgs::default(),
}
}
@@ -255,6 +260,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
engine,
era,
static_files,
rocksdb,
..
} = self;
NodeConfig {
@@ -274,6 +280,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
engine,
era,
static_files,
rocksdb,
}
}
@@ -350,6 +357,18 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
self.pruning.prune_config(&self.chain)
}
/// Returns the effective storage settings derived from static-file and `RocksDB` CLI args.
pub const fn storage_settings(&self) -> StorageSettings {
StorageSettings::base()
.with_receipts_in_static_files(self.static_files.receipts)
.with_transaction_senders_in_static_files(self.static_files.transaction_senders)
.with_account_changesets_in_static_files(self.static_files.account_changesets)
.with_storage_changesets_in_static_files(self.static_files.storage_changesets)
.with_transaction_hash_numbers_in_rocksdb(self.rocksdb.all || self.rocksdb.tx_hash)
.with_storages_history_in_rocksdb(self.rocksdb.all || self.rocksdb.storages_history)
.with_account_history_in_rocksdb(self.rocksdb.all || self.rocksdb.account_history)
}
/// Returns the max block that the node should run to, looking it up from the network if
/// necessary
pub async fn max_block<Provider, Client>(
@@ -544,6 +563,7 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
engine: self.engine,
era: self.era,
static_files: self.static_files,
rocksdb: self.rocksdb,
}
}
@@ -585,6 +605,7 @@ impl<ChainSpec> Clone for NodeConfig<ChainSpec> {
engine: self.engine.clone(),
era: self.era.clone(),
static_files: self.static_files,
rocksdb: self.rocksdb,
}
}
}

View File

@@ -106,6 +106,7 @@ impl MetricServer {
// Describe metrics after recorder installation
describe_db_metrics();
describe_static_file_metrics();
describe_rocksdb_metrics();
Collector::default().describe();
describe_memory_stats();
describe_io_stats();
@@ -238,6 +239,26 @@ fn describe_static_file_metrics() {
);
}
fn describe_rocksdb_metrics() {
describe_gauge!(
"rocksdb.table_size",
Unit::Bytes,
"The estimated size of a RocksDB table (SST + memtable)"
);
describe_gauge!("rocksdb.table_entries", "The estimated number of keys in a RocksDB table");
describe_gauge!(
"rocksdb.pending_compaction_bytes",
Unit::Bytes,
"Bytes pending compaction for a RocksDB table"
);
describe_gauge!("rocksdb.sst_size", Unit::Bytes, "The size of SST files for a RocksDB table");
describe_gauge!(
"rocksdb.memtable_size",
Unit::Bytes,
"The size of memtables for a RocksDB table"
);
}
#[cfg(all(feature = "jemalloc", unix))]
fn describe_memory_stats() {
describe_gauge!(

View File

@@ -17,9 +17,9 @@ impl OpReceiptBuilder for OpRethReceiptBuilder {
fn build_receipt<'a, E: Evm>(
&self,
ctx: ReceiptBuilderCtx<'a, OpTransactionSigned, E>,
) -> Result<Self::Receipt, ReceiptBuilderCtx<'a, OpTransactionSigned, E>> {
match ctx.tx.tx_type() {
ctx: ReceiptBuilderCtx<'a, OpTxType, E>,
) -> Result<Self::Receipt, ReceiptBuilderCtx<'a, OpTxType, E>> {
match ctx.tx_type {
OpTxType::Deposit => Err(ctx),
ty => {
let receipt = Receipt {

View File

@@ -62,6 +62,7 @@ where
type ExecutionPayloadEnvelopeV3 = OpExecutionPayloadEnvelopeV3;
type ExecutionPayloadEnvelopeV4 = OpExecutionPayloadEnvelopeV4;
type ExecutionPayloadEnvelopeV5 = OpExecutionPayloadEnvelopeV4;
type ExecutionPayloadEnvelopeV6 = OpExecutionPayloadEnvelopeV4;
}
/// Validator for Optimism engine API.

View File

@@ -16,7 +16,7 @@ use reth_network::{
PeersInfo,
};
use reth_node_api::{
AddOnsContext, BuildNextEnv, EngineTypes, FullNodeComponents, HeaderTy, NodeAddOns,
AddOnsContext, BlockTy, BuildNextEnv, EngineTypes, FullNodeComponents, HeaderTy, NodeAddOns,
NodePrimitives, PayloadAttributesBuilder, PayloadTypes, PrimitivesTy, TxTy,
};
use reth_node_builder::{
@@ -962,7 +962,7 @@ where
Node: FullNodeTypes<Types: NodeTypes<ChainSpec: OpHardforks>>,
T: EthPoolTransaction<Consensus = TxTy<Node::Types>> + OpPooledTx,
{
type Pool = OpTransactionPool<Node::Provider, DiskFileBlobStore, T>;
type Pool = OpTransactionPool<Node::Provider, DiskFileBlobStore, T, BlockTy<Node::Types>>;
async fn build_pool(self, ctx: &BuilderContext<Node>) -> eyre::Result<Self::Pool> {
let Self { pool_config_overrides, .. } = self;

View File

@@ -9,6 +9,7 @@
#![cfg_attr(docsrs, feature(doc_cfg))]
mod validator;
use op_alloy_consensus::OpBlock;
pub use validator::{OpL1BlockInfo, OpTransactionValidator};
pub mod conditional;
@@ -24,8 +25,8 @@ pub mod estimated_da_size;
use reth_transaction_pool::{CoinbaseTipOrdering, Pool, TransactionValidationTaskExecutor};
/// Type alias for default optimism transaction pool
pub type OpTransactionPool<Client, S, T = OpPooledTransaction> = Pool<
TransactionValidationTaskExecutor<OpTransactionValidator<Client, T>>,
pub type OpTransactionPool<Client, S, T = OpPooledTransaction, B = OpBlock> = Pool<
TransactionValidationTaskExecutor<OpTransactionValidator<Client, T, B>>,
CoinbaseTipOrdering<T>,
S,
>;

View File

@@ -325,10 +325,11 @@ mod tests {
#[tokio::test]
async fn validate_optimism_transaction() {
let client = MockEthProvider::default().with_chain_spec(OP_MAINNET.clone());
let validator = EthTransactionValidatorBuilder::new(client)
.no_shanghai()
.no_cancun()
.build(InMemoryBlobStore::default());
let validator =
EthTransactionValidatorBuilder::new(client)
.no_shanghai()
.no_cancun()
.build::<_, _, reth_optimism_primitives::OpBlock>(InMemoryBlobStore::default());
let validator = OpTransactionValidator::new(validator);
let origin = TransactionOrigin::External;

View File

@@ -1,5 +1,6 @@
use crate::{supervisor::SupervisorClient, InvalidCrossTx, OpPooledTx};
use alloy_consensus::{BlockHeader, Transaction};
use op_alloy_consensus::OpBlock;
use op_revm::L1BlockInfo;
use parking_lot::RwLock;
use reth_chainspec::ChainSpecProvider;
@@ -39,9 +40,9 @@ impl OpL1BlockInfo {
/// Validator for Optimism transactions.
#[derive(Debug, Clone)]
pub struct OpTransactionValidator<Client, Tx> {
pub struct OpTransactionValidator<Client, Tx, B = OpBlock> {
/// The type that performs the actual validation.
inner: Arc<EthTransactionValidator<Client, Tx>>,
inner: Arc<EthTransactionValidator<Client, Tx, B>>,
/// Additional block info required for validation.
block_info: Arc<OpL1BlockInfo>,
/// If true, ensure that the transaction's sender has enough balance to cover the L1 gas fee
@@ -54,7 +55,7 @@ pub struct OpTransactionValidator<Client, Tx> {
fork_tracker: Arc<OpForkTracker>,
}
impl<Client, Tx> OpTransactionValidator<Client, Tx> {
impl<Client, Tx, B: Block> OpTransactionValidator<Client, Tx, B> {
/// Returns the configured chain spec
pub fn chain_spec(&self) -> Arc<Client::ChainSpec>
where
@@ -86,14 +87,15 @@ impl<Client, Tx> OpTransactionValidator<Client, Tx> {
}
}
impl<Client, Tx> OpTransactionValidator<Client, Tx>
impl<Client, Tx, B> OpTransactionValidator<Client, Tx, B>
where
Client:
ChainSpecProvider<ChainSpec: OpHardforks> + StateProviderFactory + BlockReaderIdExt + Sync,
Tx: EthPoolTransaction + OpPooledTx,
B: Block,
{
/// Create a new [`OpTransactionValidator`].
pub fn new(inner: EthTransactionValidator<Client, Tx>) -> Self {
pub fn new(inner: EthTransactionValidator<Client, Tx, B>) -> Self {
let this = Self::with_block_info(inner, OpL1BlockInfo::default());
if let Ok(Some(block)) =
this.inner.client().block_by_number_or_tag(alloy_eips::BlockNumberOrTag::Latest)
@@ -112,7 +114,7 @@ where
/// Create a new [`OpTransactionValidator`] with the given [`OpL1BlockInfo`].
pub fn with_block_info(
inner: EthTransactionValidator<Client, Tx>,
inner: EthTransactionValidator<Client, Tx, B>,
block_info: OpL1BlockInfo,
) -> Self {
Self {
@@ -288,13 +290,15 @@ where
}
}
impl<Client, Tx> TransactionValidator for OpTransactionValidator<Client, Tx>
impl<Client, Tx, B> TransactionValidator for OpTransactionValidator<Client, Tx, B>
where
Client:
ChainSpecProvider<ChainSpec: OpHardforks> + StateProviderFactory + BlockReaderIdExt + Sync,
Tx: EthPoolTransaction + OpPooledTx,
B: Block,
{
type Transaction = Tx;
type Block = B;
async fn validate_transaction(
&self,
@@ -325,10 +329,7 @@ where
.await
}
fn on_new_head_block<B>(&self, new_tip_block: &SealedBlock<B>)
where
B: Block,
{
fn on_new_head_block(&self, new_tip_block: &SealedBlock<Self::Block>) {
self.inner.on_new_head_block(new_tip_block);
self.update_l1_block_info(
new_tip_block.header(),

View File

@@ -510,27 +510,12 @@ mod tests {
fn test_sealed_block_rlp_roundtrip() {
// Create a sample block using alloy_consensus::Block
let header = alloy_consensus::Header {
parent_hash: B256::ZERO,
ommers_hash: B256::ZERO,
beneficiary: Address::ZERO,
state_root: B256::ZERO,
transactions_root: B256::ZERO,
receipts_root: B256::ZERO,
logs_bloom: Default::default(),
difficulty: Default::default(),
number: 42,
gas_limit: 30_000_000,
gas_used: 21_000,
timestamp: 1_000_000,
extra_data: Default::default(),
mix_hash: B256::ZERO,
nonce: Default::default(),
base_fee_per_gas: Some(1_000_000_000),
withdrawals_root: None,
blob_gas_used: None,
excess_blob_gas: None,
parent_beacon_block_root: None,
requests_hash: None,
..Default::default()
};
// Create a simple transaction
@@ -585,27 +570,12 @@ mod tests {
fn test_decode_sealed_produces_correct_hash() {
// Create a sample block using alloy_consensus::Block
let header = alloy_consensus::Header {
parent_hash: B256::ZERO,
ommers_hash: B256::ZERO,
beneficiary: Address::ZERO,
state_root: B256::ZERO,
transactions_root: B256::ZERO,
receipts_root: B256::ZERO,
logs_bloom: Default::default(),
difficulty: Default::default(),
number: 42,
gas_limit: 30_000_000,
gas_used: 21_000,
timestamp: 1_000_000,
extra_data: Default::default(),
mix_hash: B256::ZERO,
nonce: Default::default(),
base_fee_per_gas: Some(1_000_000_000),
withdrawals_root: None,
blob_gas_used: None,
excess_blob_gas: None,
parent_beacon_block_root: None,
requests_hash: None,
..Default::default()
};
// Create a simple transaction

View File

@@ -9,6 +9,16 @@ use std::{fmt::Debug, ops::RangeBounds};
use tracing::debug;
pub(crate) trait DbTxPruneExt: DbTxMut + DbTx {
/// Clear the entire table in a single operation.
///
/// This is much faster than iterating entry-by-entry for `PruneMode::Full`.
/// Returns the number of entries that were in the table.
fn clear_table<T: Table>(&self) -> Result<usize, DatabaseError> {
let count = self.entries::<T>()?;
<Self as DbTxMut>::clear::<T>(self)?;
Ok(count)
}
/// Prune the table for the specified pre-sorted key iterator.
///
/// Returns number of rows pruned.

View File

@@ -1,119 +0,0 @@
use crate::{
db_ext::DbTxPruneExt,
segments::{PruneInput, Segment},
PrunerError,
};
use alloy_primitives::B256;
use reth_db_api::{models::BlockNumberHashedAddress, table::Value, tables, transaction::DbTxMut};
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
errors::provider::ProviderResult, BlockReader, ChainStateBlockReader, DBProvider,
NodePrimitivesProvider, PruneCheckpointWriter, TransactionsProvider,
};
use reth_prune_types::{
PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint,
};
use reth_stages_types::StageId;
use tracing::{instrument, trace};
#[derive(Debug)]
pub struct MerkleChangeSets {
mode: PruneMode,
}
impl MerkleChangeSets {
pub const fn new(mode: PruneMode) -> Self {
Self { mode }
}
}
impl<Provider> Segment<Provider> for MerkleChangeSets
where
Provider: DBProvider<Tx: DbTxMut>
+ PruneCheckpointWriter
+ TransactionsProvider
+ BlockReader
+ ChainStateBlockReader
+ NodePrimitivesProvider<Primitives: NodePrimitives<Receipt: Value>>,
{
fn segment(&self) -> PruneSegment {
PruneSegment::MerkleChangeSets
}
fn mode(&self) -> Option<PruneMode> {
Some(self.mode)
}
fn purpose(&self) -> PrunePurpose {
PrunePurpose::User
}
fn required_stage(&self) -> Option<StageId> {
Some(StageId::MerkleChangeSets)
}
#[instrument(level = "trace", target = "pruner", skip(self, provider), ret)]
fn prune(&self, provider: &Provider, input: PruneInput) -> Result<SegmentOutput, PrunerError> {
let Some(block_range) = input.get_next_block_range() else {
trace!(target: "pruner", "No change sets to prune");
return Ok(SegmentOutput::done())
};
let block_range_end = *block_range.end();
let mut limiter = input.limiter;
// Create range for StoragesTrieChangeSets which uses BlockNumberHashedAddress as key
let storage_range_start: BlockNumberHashedAddress =
(*block_range.start(), B256::ZERO).into();
let storage_range_end: BlockNumberHashedAddress =
(*block_range.end() + 1, B256::ZERO).into();
let storage_range = storage_range_start..storage_range_end;
let mut last_storages_pruned_block = None;
let (storages_pruned, done) =
provider.tx_ref().prune_dupsort_table_with_range::<tables::StoragesTrieChangeSets>(
storage_range,
&mut limiter,
|(BlockNumberHashedAddress((block_number, _)), _)| {
last_storages_pruned_block = Some(block_number);
},
)?;
trace!(target: "pruner", %storages_pruned, %done, "Pruned storages change sets");
let mut last_accounts_pruned_block = block_range_end;
let last_storages_pruned_block = last_storages_pruned_block
// If there's more storage changesets to prune, set the checkpoint block number to
// previous, so we could finish pruning its storage changesets on the next run.
.map(|block_number| if done { block_number } else { block_number.saturating_sub(1) })
.unwrap_or(block_range_end);
let (accounts_pruned, done) =
provider.tx_ref().prune_dupsort_table_with_range::<tables::AccountsTrieChangeSets>(
block_range,
&mut limiter,
|row| last_accounts_pruned_block = row.0,
)?;
trace!(target: "pruner", %accounts_pruned, %done, "Pruned accounts change sets");
let progress = limiter.progress(done);
Ok(SegmentOutput {
progress,
pruned: accounts_pruned + storages_pruned,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: Some(last_storages_pruned_block.min(last_accounts_pruned_block)),
tx_number: None,
}),
})
}
fn save_checkpoint(
&self,
provider: &Provider,
checkpoint: PruneCheckpoint,
) -> ProviderResult<()> {
provider.save_prune_checkpoint(PruneSegment::MerkleChangeSets, checkpoint)
}
}

View File

@@ -6,7 +6,7 @@ use crate::{
use reth_db_api::{tables, transaction::DbTxMut};
use reth_provider::{BlockReader, DBProvider, TransactionsProvider};
use reth_prune_types::{
PruneMode, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint,
PruneMode, PruneProgress, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint,
};
use tracing::{instrument, trace};
@@ -48,6 +48,25 @@ where
};
let tx_range_end = *tx_range.end();
// For PruneMode::Full, clear the entire table in one operation
if self.mode.is_full() {
let pruned = provider.tx_ref().clear_table::<tables::TransactionSenders>()?;
trace!(target: "pruner", %pruned, "Cleared transaction senders table");
let last_pruned_block = provider
.block_by_transaction_id(tx_range_end)?
.ok_or(PrunerError::InconsistentData("Block for transaction is not found"))?;
return Ok(SegmentOutput {
progress: PruneProgress::Finished,
pruned,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: Some(last_pruned_block),
tx_number: Some(tx_range_end),
}),
});
}
let mut limiter = input.limiter;
let mut last_pruned_transaction = tx_range_end;

View File

@@ -8,7 +8,7 @@ use rayon::prelude::*;
use reth_db_api::{tables, transaction::DbTxMut};
use reth_provider::{BlockReader, DBProvider, PruneCheckpointReader, StaticFileProviderFactory};
use reth_prune_types::{
PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment, SegmentOutputCheckpoint,
PruneCheckpoint, PruneMode, PruneProgress, PrunePurpose, PruneSegment, SegmentOutputCheckpoint,
};
use reth_static_file_types::StaticFileSegment;
use tracing::{debug, instrument, trace};
@@ -82,19 +82,51 @@ where
}
}
.into_inner();
// For PruneMode::Full, clear the entire table in one operation
if self.mode.is_full() {
let pruned = provider.tx_ref().clear_table::<tables::TransactionHashNumbers>()?;
trace!(target: "pruner", %pruned, "Cleared transaction lookup table");
let last_pruned_block = provider
.block_by_transaction_id(end)?
.ok_or(PrunerError::InconsistentData("Block for transaction is not found"))?;
return Ok(SegmentOutput {
progress: PruneProgress::Finished,
pruned,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: Some(last_pruned_block),
tx_number: Some(end),
}),
});
}
let tx_range = start..=
Some(end)
.min(input.limiter.deleted_entries_limit_left().map(|left| start + left as u64 - 1))
.min(
input
.limiter
.deleted_entries_limit_left()
// Use saturating addition here to avoid panicking on
// `deleted_entries_limit == usize::MAX`
.map(|left| start.saturating_add(left as u64) - 1),
)
.unwrap();
let tx_range_end = *tx_range.end();
// Retrieve transactions in the range and calculate their hashes in parallel
let hashes = provider
let mut hashes = provider
.transactions_by_tx_range(tx_range.clone())?
.into_par_iter()
.map(|transaction| transaction.trie_hash())
.collect::<Vec<_>>();
// Sort hashes to enable efficient cursor traversal through the TransactionHashNumbers
// table, which is keyed by hash. Without sorting, each seek would be O(log n) random
// access; with sorting, the cursor advances sequentially through the B+tree.
hashes.sort_unstable();
// Number of transactions retrieved from the database should match the tx range count
let tx_count = tx_range.count();
if hashes.len() != tx_count {

View File

@@ -12,7 +12,8 @@ use alloy_json_rpc::RpcObject;
use alloy_primitives::{Address, BlockHash, Bytes, B256, U256, U64};
use alloy_rpc_types_engine::{
ClientVersionV1, ExecutionPayloadBodiesV1, ExecutionPayloadInputV2, ExecutionPayloadV1,
ExecutionPayloadV3, ForkchoiceState, ForkchoiceUpdated, PayloadId, PayloadStatus,
ExecutionPayloadV3, ExecutionPayloadV4, ForkchoiceState, ForkchoiceUpdated, PayloadId,
PayloadStatus,
};
use alloy_rpc_types_eth::{
state::StateOverride, BlockOverrides, EIP1186AccountProofResponse, Filter, Log, SyncStatus,
@@ -73,6 +74,18 @@ pub trait EngineApi<Engine: EngineTypes> {
execution_requests: RequestsOrHash,
) -> RpcResult<PayloadStatus>;
/// Post Amsterdam payload handler
///
/// See also <https://github.com/ethereum/execution-apis/blob/main/src/engine/amsterdam.md#engine_newpayloadv5>
#[method(name = "newPayloadV5")]
async fn new_payload_v5(
&self,
payload: ExecutionPayloadV4,
versioned_hashes: Vec<B256>,
parent_beacon_block_root: B256,
execution_requests: RequestsOrHash,
) -> RpcResult<PayloadStatus>;
/// See also <https://github.com/ethereum/execution-apis/blob/6709c2a795b707202e93c4f2867fa0bf2640a84f/src/engine/paris.md#engine_forkchoiceupdatedv1>
///
/// Caution: This should not accept the `withdrawals` field in the payload attributes.
@@ -178,6 +191,19 @@ pub trait EngineApi<Engine: EngineTypes> {
payload_id: PayloadId,
) -> RpcResult<Engine::ExecutionPayloadEnvelopeV5>;
/// Post Amsterdam payload handler.
///
/// See also <https://github.com/ethereum/execution-apis/blob/main/src/engine/amsterdam.md#engine_getpayloadv6>
///
/// Returns the most recent version of the payload that is available in the corresponding
/// payload build process at the time of receiving this call. Note:
/// > Provider software MAY stop the corresponding build process after serving this call.
#[method(name = "getPayloadV6")]
async fn get_payload_v6(
&self,
payload_id: PayloadId,
) -> RpcResult<Engine::ExecutionPayloadEnvelopeV6>;
/// See also <https://github.com/ethereum/execution-apis/blob/6452a6b194d7db269bf1dbd087a267251d3cc7f8/src/engine/shanghai.md#engine_getpayloadbodiesbyhashv1>
#[method(name = "getPayloadBodiesByHashV1")]
async fn get_payload_bodies_by_hash_v1(
@@ -252,6 +278,18 @@ pub trait EngineApi<Engine: EngineTypes> {
&self,
versioned_hashes: Vec<B256>,
) -> RpcResult<Option<Vec<Option<BlobAndProofV2>>>>;
/// Returns the Block Access Lists for the given block hashes.
///
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
#[method(name = "getBALsByHashV1")]
async fn get_bals_by_hash_v1(&self, block_hashes: Vec<BlockHash>) -> RpcResult<Vec<Bytes>>;
/// Returns the Block Access Lists for the given block range.
///
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
#[method(name = "getBALsByRangeV1")]
async fn get_bals_by_range_v1(&self, start: U64, count: U64) -> RpcResult<Vec<Bytes>>;
}
/// A subset of the ETH rpc interface: <https://ethereum.github.io/execution-apis/api-documentation>

View File

@@ -41,6 +41,7 @@ metrics.workspace = true
async-trait.workspace = true
jsonrpsee-core.workspace = true
jsonrpsee-types.workspace = true
parking_lot.workspace = true
serde.workspace = true
thiserror.workspace = true
tracing.workspace = true

View File

@@ -0,0 +1,274 @@
//! Block Access List (BAL) cache for EIP-7928.
//!
//! This module provides an in-memory cache for storing Block Access Lists received via
//! the Engine API. BALs are stored for valid payloads and can be retrieved via
//! `engine_getBALsByHashV1` and `engine_getBALsByRangeV1`.
//!
//! According to EIP-7928, the EL MUST retain BALs for at least the duration of the
//! weak subjectivity period (~3533 epochs) to support synchronization with re-execution.
//! This initial implementation uses a simple in-memory cache with configurable capacity.
use alloy_primitives::{BlockHash, BlockNumber, Bytes};
use parking_lot::RwLock;
use reth_metrics::{
metrics::{Counter, Gauge},
Metrics,
};
use std::{
collections::{BTreeMap, HashMap},
sync::Arc,
};
/// Default capacity for the BAL cache.
///
/// This is a conservative default - production deployments should configure based on
/// weak subjectivity period requirements (~3533 epochs ≈ 113,000 blocks).
const DEFAULT_BAL_CACHE_CAPACITY: u32 = 1024;
/// In-memory cache for Block Access Lists (BALs).
///
/// Provides O(1) lookups by block hash and O(log n) range queries by block number.
/// Evicts the oldest (lowest) block numbers when capacity is exceeded.
///
/// This type is cheaply cloneable as it wraps an `Arc` internally.
#[derive(Debug, Clone)]
pub struct BalCache {
inner: Arc<BalCacheInner>,
}
#[derive(Debug)]
struct BalCacheInner {
/// Maximum number of entries to store.
capacity: u32,
/// Mapping from block hash to BAL bytes.
entries: RwLock<HashMap<BlockHash, Bytes>>,
/// Index mapping block number to block hash for range queries.
/// Uses `BTreeMap` for efficient range iteration and eviction of oldest blocks.
block_index: RwLock<BTreeMap<BlockNumber, BlockHash>>,
/// Cache metrics.
metrics: BalCacheMetrics,
}
impl BalCache {
/// Creates a new BAL cache with the default capacity.
pub fn new() -> Self {
Self::with_capacity(DEFAULT_BAL_CACHE_CAPACITY)
}
/// Creates a new BAL cache with the specified capacity.
pub fn with_capacity(capacity: u32) -> Self {
Self {
inner: Arc::new(BalCacheInner {
capacity,
entries: RwLock::new(HashMap::new()),
block_index: RwLock::new(BTreeMap::new()),
metrics: BalCacheMetrics::default(),
}),
}
}
/// Inserts a BAL into the cache.
///
/// If a different hash already exists for this block number (reorg), the old entry
/// is removed first. If the cache is at capacity, the oldest block number is evicted.
pub fn insert(&self, block_hash: BlockHash, block_number: BlockNumber, bal: Bytes) {
let mut entries = self.inner.entries.write();
let mut block_index = self.inner.block_index.write();
// If this block number already has a different hash, remove the old entry
if let Some(old_hash) = block_index.get(&block_number) &&
*old_hash != block_hash
{
entries.remove(old_hash);
}
// Evict oldest block if at capacity and this is a new entry
if !entries.contains_key(&block_hash) &&
entries.len() as u32 >= self.inner.capacity &&
let Some((&oldest_num, &oldest_hash)) = block_index.first_key_value()
{
entries.remove(&oldest_hash);
block_index.remove(&oldest_num);
}
entries.insert(block_hash, bal);
block_index.insert(block_number, block_hash);
self.inner.metrics.inserts.increment(1);
self.inner.metrics.count.set(entries.len() as f64);
}
/// Retrieves BALs for the given block hashes.
///
/// Returns a vector with the same length as `block_hashes`, where each element
/// is `Some(bal)` if found or `None` if not in cache.
pub fn get_by_hashes(&self, block_hashes: &[BlockHash]) -> Vec<Option<Bytes>> {
let entries = self.inner.entries.read();
block_hashes
.iter()
.map(|hash| {
let result = entries.get(hash).cloned();
if result.is_some() {
self.inner.metrics.hits.increment(1);
} else {
self.inner.metrics.misses.increment(1);
}
result
})
.collect()
}
/// Retrieves BALs for a range of blocks starting at `start` for `count` blocks.
///
/// Returns a vector of contiguous BALs in block number order, stopping at the first
/// missing block. This ensures the caller knows the returned BALs correspond to
/// blocks `[start, start + len)`.
pub fn get_by_range(&self, start: BlockNumber, count: u64) -> Vec<Bytes> {
let entries = self.inner.entries.read();
let block_index = self.inner.block_index.read();
let mut result = Vec::new();
for block_num in start..start.saturating_add(count) {
let Some(hash) = block_index.get(&block_num) else {
break;
};
let Some(bal) = entries.get(hash) else {
break;
};
result.push(bal.clone());
}
result
}
/// Returns the number of entries in the cache.
#[cfg(test)]
fn len(&self) -> usize {
self.inner.entries.read().len()
}
}
impl Default for BalCache {
fn default() -> Self {
Self::new()
}
}
/// Metrics for the BAL cache.
#[derive(Metrics)]
#[metrics(scope = "engine.bal_cache")]
struct BalCacheMetrics {
/// The total number of BALs in the cache.
count: Gauge,
/// The number of cache inserts.
inserts: Counter,
/// The number of cache hits.
hits: Counter,
/// The number of cache misses.
misses: Counter,
}
#[cfg(test)]
mod tests {
use super::*;
use alloy_primitives::B256;
#[test]
fn test_insert_and_get_by_hash() {
let cache = BalCache::with_capacity(10);
let hash1 = B256::random();
let hash2 = B256::random();
let bal1 = Bytes::from_static(b"bal1");
let bal2 = Bytes::from_static(b"bal2");
cache.insert(hash1, 1, bal1.clone());
cache.insert(hash2, 2, bal2.clone());
let results = cache.get_by_hashes(&[hash1, hash2, B256::random()]);
assert_eq!(results.len(), 3);
assert_eq!(results[0], Some(bal1));
assert_eq!(results[1], Some(bal2));
assert_eq!(results[2], None);
}
#[test]
fn test_get_by_range() {
let cache = BalCache::with_capacity(10);
for i in 1..=5 {
let hash = B256::random();
let bal = Bytes::from(format!("bal{i}").into_bytes());
cache.insert(hash, i, bal);
}
let results = cache.get_by_range(2, 3);
assert_eq!(results.len(), 3);
}
#[test]
fn test_get_by_range_stops_at_gap() {
let cache = BalCache::with_capacity(10);
// Insert blocks 1, 2, 4, 5 (missing block 3)
for i in [1, 2, 4, 5] {
let hash = B256::random();
let bal = Bytes::from(format!("bal{i}").into_bytes());
cache.insert(hash, i, bal);
}
// Requesting range starting at 1 should stop at the gap (block 3)
let results = cache.get_by_range(1, 5);
assert_eq!(results.len(), 2); // Only blocks 1 and 2
// Requesting range starting at 4 should return 4 and 5
let results = cache.get_by_range(4, 3);
assert_eq!(results.len(), 2);
}
#[test]
fn test_eviction_oldest_first() {
let cache = BalCache::with_capacity(3);
// Insert blocks 10, 20, 30
for i in [10, 20, 30] {
let hash = B256::random();
cache.insert(hash, i, Bytes::from_static(b"bal"));
}
assert_eq!(cache.len(), 3);
// Insert block 40, should evict block 10 (oldest/lowest)
let hash40 = B256::random();
cache.insert(hash40, 40, Bytes::from_static(b"bal40"));
assert_eq!(cache.len(), 3);
// Block 10 should be gone, block 20 should still be there
let results = cache.get_by_range(10, 1);
assert_eq!(results.len(), 0);
let results = cache.get_by_range(20, 1);
assert_eq!(results.len(), 1);
}
#[test]
fn test_reorg_replaces_hash() {
let cache = BalCache::with_capacity(10);
let hash1 = B256::random();
let hash2 = B256::random();
let bal1 = Bytes::from_static(b"bal1");
let bal2 = Bytes::from_static(b"bal2");
// Insert block 100 with hash1
cache.insert(hash1, 100, bal1.clone());
assert_eq!(cache.get_by_hashes(&[hash1])[0], Some(bal1));
// Reorg: insert block 100 with hash2
cache.insert(hash2, 100, bal2.clone());
// hash1 should be gone, hash2 should be there
assert_eq!(cache.get_by_hashes(&[hash1])[0], None);
assert_eq!(cache.get_by_hashes(&[hash2])[0], Some(bal2));
assert_eq!(cache.len(), 1);
}
}

View File

@@ -1,6 +1,11 @@
use std::collections::HashSet;
//! Engine API capabilities.
/// The list of all supported Engine capabilities available over the engine endpoint.
use std::collections::HashSet;
use tracing::warn;
/// All Engine API capabilities supported by Reth (Ethereum mainnet).
///
/// See <https://github.com/ethereum/execution-apis/tree/main/src/engine> for updates.
pub const CAPABILITIES: &[&str] = &[
"engine_forkchoiceUpdatedV1",
"engine_forkchoiceUpdatedV2",
@@ -22,43 +27,150 @@ pub const CAPABILITIES: &[&str] = &[
"engine_getBlobsV3",
];
// The list of all supported Engine capabilities available over the engine endpoint.
///
/// Latest spec: Prague
/// Engine API capabilities set.
#[derive(Debug, Clone)]
pub struct EngineCapabilities {
inner: HashSet<String>,
}
impl EngineCapabilities {
/// Creates a new `EngineCapabilities` instance with the given capabilities.
pub fn new(capabilities: impl IntoIterator<Item: Into<String>>) -> Self {
/// Creates from an iterator of capability strings.
pub fn new(capabilities: impl IntoIterator<Item = impl Into<String>>) -> Self {
Self { inner: capabilities.into_iter().map(Into::into).collect() }
}
/// Returns the list of all supported Engine capabilities for Prague spec.
fn prague() -> Self {
Self { inner: CAPABILITIES.iter().copied().map(str::to_owned).collect() }
}
/// Returns the list of all supported Engine capabilities.
/// Returns the capabilities as a list of strings.
pub fn list(&self) -> Vec<String> {
self.inner.iter().cloned().collect()
}
/// Inserts a new capability.
pub fn add_capability(&mut self, capability: impl Into<String>) {
self.inner.insert(capability.into());
/// Returns a reference to the inner set.
pub const fn as_set(&self) -> &HashSet<String> {
&self.inner
}
/// Removes a capability.
pub fn remove_capability(&mut self, capability: &str) -> Option<String> {
self.inner.take(capability)
/// Compares CL capabilities with this EL's capabilities and returns any mismatches.
///
/// Called during `engine_exchangeCapabilities` to detect version mismatches
/// between the consensus layer and execution layer.
pub fn get_capability_mismatches(&self, cl_capabilities: &[String]) -> CapabilityMismatches {
let cl_set: HashSet<&str> = cl_capabilities.iter().map(String::as_str).collect();
// CL has methods EL doesn't support
let mut missing_in_el: Vec<_> = cl_capabilities
.iter()
.filter(|cap| !self.inner.contains(cap.as_str()))
.cloned()
.collect();
missing_in_el.sort();
// EL has methods CL doesn't support
let mut missing_in_cl: Vec<_> =
self.inner.iter().filter(|cap| !cl_set.contains(cap.as_str())).cloned().collect();
missing_in_cl.sort();
CapabilityMismatches { missing_in_el, missing_in_cl }
}
/// Logs warnings if CL and EL capabilities don't match.
///
/// Called during `engine_exchangeCapabilities` to warn operators about
/// version mismatches between the consensus layer and execution layer.
pub fn log_capability_mismatches(&self, cl_capabilities: &[String]) {
let mismatches = self.get_capability_mismatches(cl_capabilities);
if !mismatches.missing_in_el.is_empty() {
warn!(
target: "rpc::engine",
missing = ?mismatches.missing_in_el,
"CL supports Engine API methods that Reth doesn't. Consider upgrading Reth."
);
}
if !mismatches.missing_in_cl.is_empty() {
warn!(
target: "rpc::engine",
missing = ?mismatches.missing_in_cl,
"Reth supports Engine API methods that CL doesn't. Consider upgrading your consensus client."
);
}
}
}
impl Default for EngineCapabilities {
fn default() -> Self {
Self::prague()
Self::new(CAPABILITIES.iter().copied())
}
}
/// Result of comparing CL and EL capabilities.
#[derive(Debug, Default, PartialEq, Eq)]
pub struct CapabilityMismatches {
/// Methods supported by CL but not by EL (Reth).
/// Operators should consider upgrading Reth.
pub missing_in_el: Vec<String>,
/// Methods supported by EL (Reth) but not by CL.
/// Operators should consider upgrading their consensus client.
pub missing_in_cl: Vec<String>,
}
impl CapabilityMismatches {
/// Returns `true` if there are no mismatches.
pub const fn is_empty(&self) -> bool {
self.missing_in_el.is_empty() && self.missing_in_cl.is_empty()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_no_mismatches() {
let el = EngineCapabilities::new(["method_a", "method_b"]);
let cl = vec!["method_a".to_string(), "method_b".to_string()];
let result = el.get_capability_mismatches(&cl);
assert!(result.is_empty());
}
#[test]
fn test_cl_has_extra_methods() {
let el = EngineCapabilities::new(["method_a"]);
let cl = vec!["method_a".to_string(), "method_b".to_string()];
let result = el.get_capability_mismatches(&cl);
assert_eq!(result.missing_in_el, vec!["method_b"]);
assert!(result.missing_in_cl.is_empty());
}
#[test]
fn test_el_has_extra_methods() {
let el = EngineCapabilities::new(["method_a", "method_b"]);
let cl = vec!["method_a".to_string()];
let result = el.get_capability_mismatches(&cl);
assert!(result.missing_in_el.is_empty());
assert_eq!(result.missing_in_cl, vec!["method_b"]);
}
#[test]
fn test_both_have_extra_methods() {
let el = EngineCapabilities::new(["method_a", "method_c"]);
let cl = vec!["method_a".to_string(), "method_b".to_string()];
let result = el.get_capability_mismatches(&cl);
assert_eq!(result.missing_in_el, vec!["method_b"]);
assert_eq!(result.missing_in_cl, vec!["method_c"]);
}
#[test]
fn test_results_are_sorted() {
let el = EngineCapabilities::new(["z_method", "a_method"]);
let cl = vec!["z_other".to_string(), "a_other".to_string()];
let result = el.get_capability_mismatches(&cl);
assert_eq!(result.missing_in_el, vec!["a_other", "z_other"]);
assert_eq!(result.missing_in_cl, vec!["a_method", "z_method"]);
}
}

View File

@@ -1,18 +1,20 @@
use crate::{
capabilities::EngineCapabilities, metrics::EngineApiMetrics, EngineApiError, EngineApiResult,
bal_cache::BalCache, capabilities::EngineCapabilities, metrics::EngineApiMetrics,
EngineApiError, EngineApiResult,
};
use alloy_eips::{
eip1898::BlockHashOrNumber,
eip4844::{BlobAndProofV1, BlobAndProofV2},
eip4895::Withdrawals,
eip7685::RequestsOrHash,
BlockNumHash,
};
use alloy_primitives::{BlockHash, BlockNumber, B256, U64};
use alloy_primitives::{BlockHash, BlockNumber, Bytes, B256, U64};
use alloy_rpc_types_engine::{
CancunPayloadFields, ClientVersionV1, ExecutionData, ExecutionPayloadBodiesV1,
ExecutionPayloadBodyV1, ExecutionPayloadInputV2, ExecutionPayloadSidecar, ExecutionPayloadV1,
ExecutionPayloadV3, ForkchoiceState, ForkchoiceUpdated, PayloadId, PayloadStatus,
PraguePayloadFields,
ExecutionPayloadV3, ExecutionPayloadV4, ForkchoiceState, ForkchoiceUpdated, PayloadId,
PayloadStatus, PraguePayloadFields,
};
use async_trait::async_trait;
use jsonrpsee_core::{server::RpcModule, RpcResult};
@@ -21,7 +23,7 @@ use reth_engine_primitives::{ConsensusEngineHandle, EngineApiValidator, EngineTy
use reth_network_api::NetworkInfo;
use reth_payload_builder::PayloadStore;
use reth_payload_primitives::{
validate_payload_timestamp, EngineApiMessageVersion, MessageValidationKind,
validate_payload_timestamp, EngineApiMessageVersion, ExecutionPayload, MessageValidationKind,
PayloadOrAttributes, PayloadTypes,
};
use reth_primitives_traits::{Block, BlockBody};
@@ -96,6 +98,38 @@ where
validator: Validator,
accept_execution_requests_hash: bool,
network: impl NetworkInfo + 'static,
) -> Self {
Self::with_bal_cache(
provider,
chain_spec,
beacon_consensus,
payload_store,
tx_pool,
task_spawner,
client,
capabilities,
validator,
accept_execution_requests_hash,
network,
BalCache::new(),
)
}
/// Create new instance of [`EngineApi`] with a custom BAL cache.
#[expect(clippy::too_many_arguments)]
pub fn with_bal_cache(
provider: Provider,
chain_spec: Arc<ChainSpec>,
beacon_consensus: ConsensusEngineHandle<PayloadT>,
payload_store: PayloadStore<PayloadT>,
tx_pool: Pool,
task_spawner: Box<dyn TaskSpawner>,
client: ClientVersionV1,
capabilities: EngineCapabilities,
validator: Validator,
accept_execution_requests_hash: bool,
network: impl NetworkInfo + 'static,
bal_cache: BalCache,
) -> Self {
let is_syncing = Arc::new(move || network.is_syncing());
let inner = Arc::new(EngineApiInner {
@@ -111,10 +145,25 @@ where
validator,
accept_execution_requests_hash,
is_syncing,
bal_cache,
});
Self { inner }
}
/// Returns a reference to the BAL cache.
pub fn bal_cache(&self) -> &BalCache {
&self.inner.bal_cache
}
/// Caches the BAL if the status is valid.
fn maybe_cache_bal(&self, num_hash: BlockNumHash, bal: Option<Bytes>, status: &PayloadStatus) {
if status.is_valid() &&
let Some(bal) = bal
{
self.inner.bal_cache.insert(num_hash.hash, num_hash.number, bal);
}
}
/// Fetches the client version.
pub fn get_client_version_v1(
&self,
@@ -149,7 +198,11 @@ where
.validator
.validate_version_specific_fields(EngineApiMessageVersion::V1, payload_or_attrs)?;
Ok(self.inner.beacon_consensus.new_payload(payload).await?)
let num_hash = payload.num_hash();
let bal = payload.block_access_list().cloned();
let status = self.inner.beacon_consensus.new_payload(payload).await?;
self.maybe_cache_bal(num_hash, bal, &status);
Ok(status)
}
/// Metered version of `new_payload_v1`.
@@ -177,7 +230,12 @@ where
self.inner
.validator
.validate_version_specific_fields(EngineApiMessageVersion::V2, payload_or_attrs)?;
Ok(self.inner.beacon_consensus.new_payload(payload).await?)
let num_hash = payload.num_hash();
let bal = payload.block_access_list().cloned();
let status = self.inner.beacon_consensus.new_payload(payload).await?;
self.maybe_cache_bal(num_hash, bal, &status);
Ok(status)
}
/// Metered version of `new_payload_v2`.
@@ -206,7 +264,11 @@ where
.validator
.validate_version_specific_fields(EngineApiMessageVersion::V3, payload_or_attrs)?;
Ok(self.inner.beacon_consensus.new_payload(payload).await?)
let num_hash = payload.num_hash();
let bal = payload.block_access_list().cloned();
let status = self.inner.beacon_consensus.new_payload(payload).await?;
self.maybe_cache_bal(num_hash, bal, &status);
Ok(status)
}
/// Metrics version of `new_payload_v3`
@@ -236,7 +298,11 @@ where
.validator
.validate_version_specific_fields(EngineApiMessageVersion::V4, payload_or_attrs)?;
Ok(self.inner.beacon_consensus.new_payload(payload).await?)
let num_hash = payload.num_hash();
let bal = payload.block_access_list().cloned();
let status = self.inner.beacon_consensus.new_payload(payload).await?;
self.maybe_cache_bal(num_hash, bal, &status);
Ok(status)
}
/// Metrics version of `new_payload_v4`
@@ -881,6 +947,22 @@ where
res
}
/// Retrieves BALs for the given block hashes from the cache.
///
/// Returns the RLP-encoded BALs for blocks found in the cache.
/// Missing blocks are returned as empty bytes.
pub fn get_bals_by_hash(&self, block_hashes: Vec<BlockHash>) -> Vec<alloy_primitives::Bytes> {
let results = self.inner.bal_cache.get_by_hashes(&block_hashes);
results.into_iter().map(|opt| opt.unwrap_or_default()).collect()
}
/// Retrieves BALs for a range of blocks from the cache.
///
/// Returns the RLP-encoded BALs for blocks in the range `[start, start + count)`.
pub fn get_bals_by_range(&self, start: u64, count: u64) -> Vec<alloy_primitives::Bytes> {
self.inner.bal_cache.get_by_range(start, count)
}
}
// This is the concrete ethereum engine API implementation.
@@ -963,6 +1045,24 @@ where
Ok(self.new_payload_v4_metered(payload).await?)
}
/// Handler for `engine_newPayloadV5`
///
/// Post Amsterdam payload handler. Currently returns unsupported fork error.
///
/// See also <https://github.com/ethereum/execution-apis/blob/main/src/engine/amsterdam.md#engine_newpayloadv5>
async fn new_payload_v5(
&self,
_payload: ExecutionPayloadV4,
_versioned_hashes: Vec<B256>,
_parent_beacon_block_root: B256,
_execution_requests: RequestsOrHash,
) -> RpcResult<PayloadStatus> {
trace!(target: "rpc::engine", "Serving engine_newPayloadV5");
Err(EngineApiError::EngineObjectValidationError(
reth_payload_primitives::EngineObjectValidationError::UnsupportedFork,
))?
}
/// Handler for `engine_forkchoiceUpdatedV1`
/// See also <https://github.com/ethereum/execution-apis/blob/3d627c95a4d3510a8187dd02e0250ecb4331d27e/src/engine/paris.md#engine_forkchoiceupdatedv1>
///
@@ -1086,6 +1186,21 @@ where
Ok(self.get_payload_v5_metered(payload_id).await?)
}
/// Handler for `engine_getPayloadV6`
///
/// Post Amsterdam payload handler. Currently returns unsupported fork error.
///
/// See also <https://github.com/ethereum/execution-apis/blob/main/src/engine/amsterdam.md#engine_getpayloadv6>
async fn get_payload_v6(
&self,
_payload_id: PayloadId,
) -> RpcResult<EngineT::ExecutionPayloadEnvelopeV6> {
trace!(target: "rpc::engine", "Serving engine_getPayloadV6");
Err(EngineApiError::EngineObjectValidationError(
reth_payload_primitives::EngineObjectValidationError::UnsupportedFork,
))?
}
/// Handler for `engine_getPayloadBodiesByHashV1`
/// See also <https://github.com/ethereum/execution-apis/blob/6452a6b194d7db269bf1dbd087a267251d3cc7f8/src/engine/shanghai.md#engine_getpayloadbodiesbyhashv1>
async fn get_payload_bodies_by_hash_v1(
@@ -1134,8 +1249,13 @@ where
/// Handler for `engine_exchangeCapabilitiesV1`
/// See also <https://github.com/ethereum/execution-apis/blob/6452a6b194d7db269bf1dbd087a267251d3cc7f8/src/engine/common.md#capabilities>
async fn exchange_capabilities(&self, _capabilities: Vec<String>) -> RpcResult<Vec<String>> {
Ok(self.capabilities().list())
async fn exchange_capabilities(&self, capabilities: Vec<String>) -> RpcResult<Vec<String>> {
trace!(target: "rpc::engine", "Serving engine_exchangeCapabilities");
let el_caps = self.capabilities();
el_caps.log_capability_mismatches(&capabilities);
Ok(el_caps.list())
}
async fn get_blobs_v1(
@@ -1161,6 +1281,29 @@ where
trace!(target: "rpc::engine", "Serving engine_getBlobsV3");
Ok(self.get_blobs_v3_metered(versioned_hashes)?)
}
/// Handler for `engine_getBALsByHashV1`
///
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
async fn get_bals_by_hash_v1(
&self,
block_hashes: Vec<BlockHash>,
) -> RpcResult<Vec<alloy_primitives::Bytes>> {
trace!(target: "rpc::engine", "Serving engine_getBALsByHashV1");
Ok(self.get_bals_by_hash(block_hashes))
}
/// Handler for `engine_getBALsByRangeV1`
///
/// See also <https://eips.ethereum.org/EIPS/eip-7928>
async fn get_bals_by_range_v1(
&self,
start: U64,
count: U64,
) -> RpcResult<Vec<alloy_primitives::Bytes>> {
trace!(target: "rpc::engine", "Serving engine_getBALsByRangeV1");
Ok(self.get_bals_by_range(start.to(), count.to()))
}
}
impl<Provider, EngineT, Pool, Validator, ChainSpec> IntoEngineApiRpcModule
@@ -1219,6 +1362,8 @@ struct EngineApiInner<Provider, PayloadT: PayloadTypes, Pool, Validator, ChainSp
accept_execution_requests_hash: bool,
/// Returns `true` if the node is currently syncing.
is_syncing: Arc<dyn Fn() -> bool + Send + Sync>,
/// Cache for Block Access Lists (BALs) per EIP-7928.
bal_cache: BalCache,
}
#[cfg(test)]

View File

@@ -12,6 +12,10 @@
/// The Engine API implementation.
mod engine_api;
/// Block Access List (BAL) cache for EIP-7928.
mod bal_cache;
pub use bal_cache::BalCache;
/// Engine API capabilities.
pub mod capabilities;
pub use capabilities::EngineCapabilities;

View File

@@ -507,6 +507,7 @@ impl From<reth_errors::ProviderError> for EthApiError {
ProviderError::BlockNumberForTransactionIndexNotFound => Self::UnknownBlockOrTxIndex,
ProviderError::FinalizedBlockNotFound => Self::HeaderNotFound(BlockId::finalized()),
ProviderError::SafeBlockNotFound => Self::HeaderNotFound(BlockId::safe()),
ProviderError::BlockExpired { .. } => Self::PrunedHistoryUnavailable,
err => Self::Internal(err.into()),
}
}

View File

@@ -258,11 +258,13 @@ where
let call = match result {
ExecutionResult::Halt { reason, gas_used } => {
let error = Err::from_evm_halt(reason, tx.gas_limit());
#[allow(clippy::needless_update)]
SimCallResult {
return_data: Bytes::new(),
error: Some(SimulateError {
message: error.to_string(),
code: error.into().code(),
..SimulateError::invalid_params()
}),
gas_used,
logs: Vec::new(),
@@ -271,11 +273,13 @@ where
}
ExecutionResult::Revert { output, gas_used } => {
let error = Err::from_revert(output.clone());
#[allow(clippy::needless_update)]
SimCallResult {
return_data: output,
error: Some(SimulateError {
message: error.to_string(),
code: error.into().code(),
..SimulateError::invalid_params()
}),
gas_used,
status: false,

View File

@@ -483,6 +483,12 @@ where
.ok_or_else(|| ProviderError::HeaderNotFound(block_hash.into()))?
};
// Check if the block has been pruned (EIP-4444)
let earliest_block = self.provider().earliest_block_number()?;
if header.number() < earliest_block {
return Err(EthApiError::PrunedHistoryUnavailable.into());
}
let block_num_hash = BlockNumHash::new(header.number(), block_hash);
let mut all_logs = Vec::new();
@@ -565,6 +571,12 @@ where
let (from_block_number, to_block_number) =
logs_utils::get_filter_block_range(from, to, start_block, info)?;
// Check if the requested range overlaps with pruned history (EIP-4444)
let earliest_block = self.provider().earliest_block_number()?;
if from_block_number < earliest_block {
return Err(EthApiError::PrunedHistoryUnavailable.into());
}
self.get_logs_in_block_range(filter, from_block_number, to_block_number, limits)
.await
}

View File

@@ -316,7 +316,7 @@ impl<N: ProviderNodeTypes> Pipeline<N> {
let _locked_sf_producer = self.static_file_producer.lock();
let mut provider_rw =
self.provider_factory.database_provider_rw()?.disable_long_read_transaction_safety();
self.provider_factory.unwind_provider_rw()?.disable_long_read_transaction_safety();
for stage in unwind_pipeline {
let stage_id = stage.id();
@@ -396,7 +396,7 @@ impl<N: ProviderNodeTypes> Pipeline<N> {
stage.post_unwind_commit()?;
provider_rw = self.provider_factory.database_provider_rw()?;
provider_rw = self.provider_factory.unwind_provider_rw()?;
}
Err(err) => {
self.event_sender.notify(PipelineEvent::Error { stage_id });

View File

@@ -86,6 +86,14 @@ where
}
}
/// Clear all ETL state. Called on error paths to prevent buffer pollution on retry.
fn clear_etl_state(&mut self) {
self.sync_gap = None;
self.hash_collector.clear();
self.header_collector.clear();
self.is_etl_ready = false;
}
/// Write downloaded headers to storage from ETL.
///
/// Writes to static files ( `Header | HeaderTD | HeaderHash` ) and [`tables::HeaderNumbers`]
@@ -258,7 +266,7 @@ where
}
Some(Err(HeadersDownloaderError::DetachedHead { local_head, header, error })) => {
error!(target: "sync::stages::headers", %error, "Cannot attach header to head");
self.sync_gap = None;
self.clear_etl_state();
return Poll::Ready(Err(StageError::DetachedHead {
local_head: Box::new(local_head.block_with_parent()),
header: Box::new(header.block_with_parent()),
@@ -266,7 +274,7 @@ where
}))
}
None => {
self.sync_gap = None;
self.clear_etl_state();
return Poll::Ready(Err(StageError::ChannelClosed))
}
}
@@ -324,7 +332,7 @@ where
provider: &Provider,
input: UnwindInput,
) -> Result<UnwindOutput, StageError> {
self.sync_gap.take();
self.clear_etl_state();
// First unwind the db tables, until the unwind_to block number. use the walker to unwind
// HeaderNumbers based on the index in CanonicalHeaders

View File

@@ -1,11 +1,10 @@
use crate::stages::utils::collect_history_indices;
use super::{collect_account_history_indices, load_history_indices};
use alloy_primitives::Address;
use super::collect_account_history_indices;
use crate::stages::utils::{collect_history_indices, load_account_history};
use reth_config::config::{EtlConfig, IndexHistoryConfig};
use reth_db_api::{models::ShardedKey, table::Decode, tables, transaction::DbTxMut};
use reth_db_api::{models::ShardedKey, tables, transaction::DbTxMut};
use reth_provider::{
DBProvider, HistoryWriter, PruneCheckpointReader, PruneCheckpointWriter, StorageSettingsCache,
DBProvider, EitherWriter, HistoryWriter, PruneCheckpointReader, PruneCheckpointWriter,
RocksDBProviderFactory, StorageSettingsCache,
};
use reth_prune_types::{PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment};
use reth_stages_api::{
@@ -53,7 +52,8 @@ where
+ PruneCheckpointWriter
+ reth_storage_api::ChangeSetReader
+ reth_provider::StaticFileProviderFactory
+ StorageSettingsCache,
+ StorageSettingsCache
+ RocksDBProviderFactory,
{
/// Return the id of the stage
fn id(&self) -> StageId {
@@ -101,15 +101,25 @@ where
let mut range = input.next_block_range();
let first_sync = input.checkpoint().block_number == 0;
let use_rocksdb = provider.cached_storage_settings().account_history_in_rocksdb;
// On first sync we might have history coming from genesis. We clear the table since it's
// faster to rebuild from scratch.
if first_sync {
provider.tx_ref().clear::<tables::AccountsHistory>()?;
if use_rocksdb {
// Note: RocksDB clear() executes immediately (not deferred to commit like MDBX),
// but this is safe for first_sync because if we crash before commit, the
// checkpoint stays at 0 and we'll just clear and rebuild again on restart. The
// source data (changesets) is intact.
#[cfg(all(unix, feature = "rocksdb"))]
provider.rocksdb_provider().clear::<tables::AccountsHistory>()?;
} else {
provider.tx_ref().clear::<tables::AccountsHistory>()?;
}
range = 0..=*input.next_block_range().end();
}
info!(target: "sync::stages::index_account_history::exec", ?first_sync, "Collecting indices");
info!(target: "sync::stages::index_account_history::exec", ?first_sync, ?use_rocksdb, "Collecting indices");
let collector = if provider.cached_storage_settings().account_changesets_in_static_files {
// Use the provider-based collection that can read from static files.
@@ -125,14 +135,13 @@ where
};
info!(target: "sync::stages::index_account_history::exec", "Loading indices into database");
load_history_indices::<_, tables::AccountsHistory, _>(
provider,
collector,
first_sync,
ShardedKey::new,
ShardedKey::<Address>::decode_owned,
|key| key.key,
)?;
provider.with_rocksdb_batch_auto_commit(|rocksdb_batch| {
let mut writer = EitherWriter::new_accounts_history(provider, rocksdb_batch)?;
load_account_history(collector, first_sync, &mut writer)
.map_err(|e| reth_provider::ProviderError::other(Box::new(e)))?;
Ok(((), writer.into_raw_rocksdb_batch()))
})?;
Ok(ExecOutput { checkpoint: StageCheckpoint::new(*range.end()), done: true })
}
@@ -160,7 +169,7 @@ mod tests {
stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, TestRunnerError,
TestStageDB, UnwindStageTestRunner,
};
use alloy_primitives::{address, BlockNumber, B256};
use alloy_primitives::{address, Address, BlockNumber, B256};
use itertools::Itertools;
use reth_db_api::{
cursor::DbCursorRO,
@@ -646,4 +655,169 @@ mod tests {
Ok(())
}
}
#[cfg(all(unix, feature = "rocksdb"))]
mod rocksdb_tests {
use super::*;
use reth_provider::RocksDBProviderFactory;
use reth_storage_api::StorageSettings;
/// Test that when `account_history_in_rocksdb` is enabled, the stage
/// writes account history indices to `RocksDB` instead of MDBX.
#[tokio::test]
async fn execute_writes_to_rocksdb_when_enabled() {
// init
let db = TestStageDB::default();
// Enable RocksDB for account history
db.factory.set_storage_settings_cache(
StorageSettings::legacy().with_account_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=10 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::AccountChangeSets>(block, acc())?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(10), ..Default::default() };
let mut stage = IndexAccountHistoryStage::default();
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
provider.commit().unwrap();
// Verify MDBX table is empty (data should be in RocksDB)
let mdbx_table = db.table::<tables::AccountsHistory>().unwrap();
assert!(
mdbx_table.is_empty(),
"MDBX AccountsHistory should be empty when RocksDB is enabled"
);
// Verify RocksDB has the data
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should contain account history");
let block_list = result.unwrap();
let blocks: Vec<u64> = block_list.iter().collect();
assert_eq!(blocks, (0..=10).collect::<Vec<_>>());
}
/// Test that unwind works correctly when `account_history_in_rocksdb` is enabled.
#[tokio::test]
async fn unwind_works_when_rocksdb_enabled() {
let db = TestStageDB::default();
db.factory.set_storage_settings_cache(
StorageSettings::legacy().with_account_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=10 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::AccountChangeSets>(block, acc())?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(10), ..Default::default() };
let mut stage = IndexAccountHistoryStage::default();
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
provider.commit().unwrap();
// Verify RocksDB has blocks 0-10 before unwind
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should have data before unwind");
let blocks_before: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks_before, (0..=10).collect::<Vec<_>>());
// Unwind to block 5 (remove blocks 6-10)
let unwind_input =
UnwindInput { checkpoint: StageCheckpoint::new(10), unwind_to: 5, bad_block: None };
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.unwind(&provider, unwind_input).unwrap();
assert_eq!(out, UnwindOutput { checkpoint: StageCheckpoint::new(5) });
provider.commit().unwrap();
// Verify RocksDB now only has blocks 0-5 (blocks 6-10 removed)
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should still have data after unwind");
let blocks_after: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks_after, (0..=5).collect::<Vec<_>>(), "Should only have blocks 0-5");
}
/// Test incremental sync merges new data with existing shards.
#[tokio::test]
async fn execute_incremental_sync() {
let db = TestStageDB::default();
db.factory.set_storage_settings_cache(
StorageSettings::legacy().with_account_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=5 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::AccountChangeSets>(block, acc())?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(5), ..Default::default() };
let mut stage = IndexAccountHistoryStage::default();
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(5), done: true });
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some());
let blocks: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks, (0..=5).collect::<Vec<_>>());
db.commit(|tx| {
for block in 6..=10 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::AccountChangeSets>(block, acc())?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(10), checkpoint: Some(StageCheckpoint::new(5)) };
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::AccountsHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should have merged data");
let blocks: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks, (0..=10).collect::<Vec<_>>());
}
}
}

View File

@@ -1,19 +1,22 @@
use super::{collect_history_indices, load_history_indices};
use crate::{StageCheckpoint, StageId};
use super::{collect_history_indices, collect_storage_history_indices};
use crate::{stages::utils::load_storage_history, StageCheckpoint, StageId};
use reth_config::config::{EtlConfig, IndexHistoryConfig};
use reth_db_api::{
models::{storage_sharded_key::StorageShardedKey, AddressStorageKey, BlockNumberAddress},
table::Decode,
tables,
transaction::DbTxMut,
};
use reth_provider::{DBProvider, HistoryWriter, PruneCheckpointReader, PruneCheckpointWriter};
use reth_provider::{
DBProvider, EitherWriter, HistoryWriter, PruneCheckpointReader, PruneCheckpointWriter,
RocksDBProviderFactory, StaticFileProviderFactory, StorageChangeSetReader,
StorageSettingsCache,
};
use reth_prune_types::{PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment};
use reth_stages_api::{ExecInput, ExecOutput, Stage, StageError, UnwindInput, UnwindOutput};
use std::fmt::Debug;
use tracing::info;
/// Stage is indexing history the account changesets generated in
/// Stage is indexing history the storage changesets generated in
/// [`ExecutionStage`][crate::stages::ExecutionStage]. For more information
/// on index sharding take a look at [`tables::StoragesHistory`].
#[derive(Debug)]
@@ -34,7 +37,7 @@ impl IndexStorageHistoryStage {
etl_config: EtlConfig,
prune_mode: Option<PruneMode>,
) -> Self {
Self { commit_threshold: config.commit_threshold, prune_mode, etl_config }
Self { commit_threshold: config.commit_threshold, etl_config, prune_mode }
}
}
@@ -46,8 +49,15 @@ impl Default for IndexStorageHistoryStage {
impl<Provider> Stage<Provider> for IndexStorageHistoryStage
where
Provider:
DBProvider<Tx: DbTxMut> + PruneCheckpointWriter + HistoryWriter + PruneCheckpointReader,
Provider: DBProvider<Tx: DbTxMut>
+ HistoryWriter
+ PruneCheckpointReader
+ PruneCheckpointWriter
+ StorageSettingsCache
+ RocksDBProviderFactory
+ StorageChangeSetReader
+ StaticFileProviderFactory
+ reth_provider::NodePrimitivesProvider,
{
/// Return the id of the stage
fn id(&self) -> StageId {
@@ -95,16 +105,28 @@ where
let mut range = input.next_block_range();
let first_sync = input.checkpoint().block_number == 0;
let use_rocksdb = provider.cached_storage_settings().storages_history_in_rocksdb;
// On first sync we might have history coming from genesis. We clear the table since it's
// faster to rebuild from scratch.
if first_sync {
provider.tx_ref().clear::<tables::StoragesHistory>()?;
if use_rocksdb {
// Note: RocksDB clear() executes immediately (not deferred to commit like MDBX),
// but this is safe for first_sync because if we crash before commit, the
// checkpoint stays at 0 and we'll just clear and rebuild again on restart. The
// source data (changesets) is intact.
#[cfg(all(unix, feature = "rocksdb"))]
provider.rocksdb_provider().clear::<tables::StoragesHistory>()?;
} else {
provider.tx_ref().clear::<tables::StoragesHistory>()?;
}
range = 0..=*input.next_block_range().end();
}
info!(target: "sync::stages::index_storage_history::exec", ?first_sync, "Collecting indices");
let collector =
info!(target: "sync::stages::index_storage_history::exec", ?first_sync, ?use_rocksdb, "Collecting indices");
let collector = if provider.cached_storage_settings().storage_changesets_in_static_files {
collect_storage_history_indices(provider, range.clone(), &self.etl_config)?
} else {
collect_history_indices::<_, tables::StorageChangeSets, tables::StoragesHistory, _>(
provider,
BlockNumberAddress::range(range.clone()),
@@ -113,19 +135,17 @@ where
},
|(key, value)| (key.block_number(), AddressStorageKey((key.address(), value.key))),
&self.etl_config,
)?;
)?
};
info!(target: "sync::stages::index_storage_history::exec", "Loading indices into database");
load_history_indices::<_, tables::StoragesHistory, _>(
provider,
collector,
first_sync,
|AddressStorageKey((address, storage_key)), highest_block_number| {
StorageShardedKey::new(address, storage_key, highest_block_number)
},
StorageShardedKey::decode_owned,
|key| AddressStorageKey((key.address, key.sharded_key.key)),
)?;
provider.with_rocksdb_batch_auto_commit(|rocksdb_batch| {
let mut writer = EitherWriter::new_storages_history(provider, rocksdb_batch)?;
load_storage_history(collector, first_sync, &mut writer)
.map_err(|e| reth_provider::ProviderError::other(Box::new(e)))?;
Ok(((), writer.into_raw_rocksdb_batch()))
})?;
Ok(ExecOutput { checkpoint: StageCheckpoint::new(*range.end()), done: true })
}
@@ -382,12 +402,12 @@ mod tests {
async fn insert_index_second_half_shard() {
// init
let db = TestStageDB::default();
let mut close_full_list = (1..=LAST_BLOCK_IN_FULL_SHARD - 1).collect::<Vec<_>>();
let mut almost_full_list = (1..=LAST_BLOCK_IN_FULL_SHARD - 1).collect::<Vec<_>>();
// setup
partial_setup(&db);
db.commit(|tx| {
tx.put::<tables::StoragesHistory>(shard(u64::MAX), list(&close_full_list)).unwrap();
tx.put::<tables::StoragesHistory>(shard(u64::MAX), list(&almost_full_list)).unwrap();
Ok(())
})
.unwrap();
@@ -396,12 +416,12 @@ mod tests {
run(&db, LAST_BLOCK_IN_FULL_SHARD + 1, Some(LAST_BLOCK_IN_FULL_SHARD - 1));
// verify
close_full_list.push(LAST_BLOCK_IN_FULL_SHARD);
almost_full_list.push(LAST_BLOCK_IN_FULL_SHARD);
let table = cast(db.table::<tables::StoragesHistory>().unwrap());
assert_eq!(
table,
BTreeMap::from([
(shard(LAST_BLOCK_IN_FULL_SHARD), close_full_list.clone()),
(shard(LAST_BLOCK_IN_FULL_SHARD), almost_full_list.clone()),
(shard(u64::MAX), vec![LAST_BLOCK_IN_FULL_SHARD + 1])
])
);
@@ -410,9 +430,9 @@ mod tests {
unwind(&db, LAST_BLOCK_IN_FULL_SHARD, LAST_BLOCK_IN_FULL_SHARD - 1);
// verify initial state
close_full_list.pop();
almost_full_list.pop();
let table = cast(db.table::<tables::StoragesHistory>().unwrap());
assert_eq!(table, BTreeMap::from([(shard(u64::MAX), close_full_list)]));
assert_eq!(table, BTreeMap::from([(shard(u64::MAX), almost_full_list)]));
}
#[tokio::test]
@@ -663,4 +683,294 @@ mod tests {
Ok(())
}
}
#[cfg(all(unix, feature = "rocksdb"))]
mod rocksdb_tests {
use super::*;
use reth_provider::RocksDBProviderFactory;
use reth_storage_api::StorageSettings;
/// Test that when `storages_history_in_rocksdb` is enabled, the stage
/// writes storage history indices to `RocksDB` instead of MDBX.
#[tokio::test]
async fn execute_writes_to_rocksdb_when_enabled() {
let db = TestStageDB::default();
db.factory.set_storage_settings_cache(
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=10 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(10), ..Default::default() };
let mut stage = IndexStorageHistoryStage::default();
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
provider.commit().unwrap();
let mdbx_table = db.table::<tables::StoragesHistory>().unwrap();
assert!(
mdbx_table.is_empty(),
"MDBX StoragesHistory should be empty when RocksDB is enabled"
);
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should contain storage history");
let block_list = result.unwrap();
let blocks: Vec<u64> = block_list.iter().collect();
assert_eq!(blocks, (0..=10).collect::<Vec<_>>());
}
/// Test that unwind works correctly when `storages_history_in_rocksdb` is enabled.
#[tokio::test]
async fn unwind_works_when_rocksdb_enabled() {
let db = TestStageDB::default();
db.factory.set_storage_settings_cache(
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=10 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(10), ..Default::default() };
let mut stage = IndexStorageHistoryStage::default();
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should have data before unwind");
let blocks_before: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks_before, (0..=10).collect::<Vec<_>>());
let unwind_input =
UnwindInput { checkpoint: StageCheckpoint::new(10), unwind_to: 5, bad_block: None };
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.unwind(&provider, unwind_input).unwrap();
assert_eq!(out, UnwindOutput { checkpoint: StageCheckpoint::new(5) });
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should still have data after partial unwind");
let blocks_after: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(
blocks_after,
(0..=5).collect::<Vec<_>>(),
"Should only have blocks 0-5 after unwind to block 5"
);
}
/// Test that unwind to block 0 keeps only block 0's history.
#[tokio::test]
async fn unwind_to_zero_keeps_block_zero() {
let db = TestStageDB::default();
db.factory.set_storage_settings_cache(
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=5 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(5), ..Default::default() };
let mut stage = IndexStorageHistoryStage::default();
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(5), done: true });
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should have data before unwind");
let unwind_input =
UnwindInput { checkpoint: StageCheckpoint::new(5), unwind_to: 0, bad_block: None };
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.unwind(&provider, unwind_input).unwrap();
assert_eq!(out, UnwindOutput { checkpoint: StageCheckpoint::new(0) });
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should still have block 0 history");
let blocks_after: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks_after, vec![0], "Should only have block 0 after unwinding to 0");
}
/// Test incremental sync merges new data with existing shards.
#[tokio::test]
async fn execute_incremental_sync() {
let db = TestStageDB::default();
db.factory.set_storage_settings_cache(
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
);
db.commit(|tx| {
for block in 0..=5 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(5), ..Default::default() };
let mut stage = IndexStorageHistoryStage::default();
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(5), done: true });
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some());
let blocks: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks, (0..=5).collect::<Vec<_>>());
db.commit(|tx| {
for block in 6..=10 {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(10), checkpoint: Some(StageCheckpoint::new(5)) };
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
assert_eq!(out, ExecOutput { checkpoint: StageCheckpoint::new(10), done: true });
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let result = rocksdb.get::<tables::StoragesHistory>(shard(u64::MAX)).unwrap();
assert!(result.is_some(), "RocksDB should have merged data");
let blocks: Vec<u64> = result.unwrap().iter().collect();
assert_eq!(blocks, (0..=10).collect::<Vec<_>>());
}
/// Test multi-shard unwind correctly handles shards that span across unwind boundary.
#[tokio::test]
async fn unwind_multi_shard() {
use reth_db_api::models::sharded_key::NUM_OF_INDICES_IN_SHARD;
let db = TestStageDB::default();
db.factory.set_storage_settings_cache(
StorageSettings::legacy().with_storages_history_in_rocksdb(true),
);
let num_blocks = (NUM_OF_INDICES_IN_SHARD * 2 + 100) as u64;
db.commit(|tx| {
for block in 0..num_blocks {
tx.put::<tables::BlockBodyIndices>(
block,
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
)?;
tx.put::<tables::StorageChangeSets>(
block_number_address(block),
storage(STORAGE_KEY),
)?;
}
Ok(())
})
.unwrap();
let input = ExecInput { target: Some(num_blocks - 1), ..Default::default() };
let mut stage = IndexStorageHistoryStage::default();
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.execute(&provider, input).unwrap();
assert_eq!(
out,
ExecOutput { checkpoint: StageCheckpoint::new(num_blocks - 1), done: true }
);
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let shards = rocksdb.storage_history_shards(ADDRESS, STORAGE_KEY).unwrap();
assert!(shards.len() >= 2, "Should have at least 2 shards for {} blocks", num_blocks);
let unwind_to = NUM_OF_INDICES_IN_SHARD as u64 + 50;
let unwind_input = UnwindInput {
checkpoint: StageCheckpoint::new(num_blocks - 1),
unwind_to,
bad_block: None,
};
let provider = db.factory.database_provider_rw().unwrap();
let out = stage.unwind(&provider, unwind_input).unwrap();
assert_eq!(out, UnwindOutput { checkpoint: StageCheckpoint::new(unwind_to) });
provider.commit().unwrap();
let rocksdb = db.factory.rocksdb_provider();
let shards_after = rocksdb.storage_history_shards(ADDRESS, STORAGE_KEY).unwrap();
assert!(!shards_after.is_empty(), "Should still have shards after unwind");
let all_blocks: Vec<u64> =
shards_after.iter().flat_map(|(_, list)| list.iter()).collect();
assert_eq!(
all_blocks,
(0..=unwind_to).collect::<Vec<_>>(),
"Should only have blocks 0 to {} after unwind",
unwind_to
);
}
}
}

View File

@@ -9,7 +9,7 @@ use reth_db_api::{
use reth_primitives_traits::{GotExpected, SealedHeader};
use reth_provider::{
ChangeSetReader, DBProvider, HeaderProvider, ProviderError, StageCheckpointReader,
StageCheckpointWriter, StatsReader, TrieWriter,
StageCheckpointWriter, StatsReader, StorageChangeSetReader, TrieWriter,
};
use reth_stages_api::{
BlockErrorKind, EntitiesCheckpoint, ExecInput, ExecOutput, MerkleCheckpoint, Stage,
@@ -159,6 +159,7 @@ where
+ StatsReader
+ HeaderProvider
+ ChangeSetReader
+ StorageChangeSetReader
+ StageCheckpointReader
+ StageCheckpointWriter,
{

View File

@@ -1,447 +0,0 @@
use crate::stages::merkle::INVALID_STATE_ROOT_ERROR_MESSAGE;
use alloy_consensus::BlockHeader;
use alloy_primitives::BlockNumber;
use reth_consensus::ConsensusError;
use reth_primitives_traits::{GotExpected, SealedHeader};
use reth_provider::{
BlockNumReader, ChainStateBlockReader, ChangeSetReader, DBProvider, HeaderProvider,
ProviderError, PruneCheckpointReader, PruneCheckpointWriter, StageCheckpointReader,
StageCheckpointWriter, TrieWriter,
};
use reth_prune_types::{
PruneCheckpoint, PruneMode, PruneSegment, MERKLE_CHANGESETS_RETENTION_BLOCKS,
};
use reth_stages_api::{
BlockErrorKind, ExecInput, ExecOutput, Stage, StageCheckpoint, StageError, StageId,
UnwindInput, UnwindOutput,
};
use reth_trie::{
updates::TrieUpdates, HashedPostStateSorted, KeccakKeyHasher, StateRoot, TrieInputSorted,
};
use reth_trie_db::{DatabaseHashedPostState, DatabaseStateRoot};
use std::{ops::Range, sync::Arc};
use tracing::{debug, error};
/// The `MerkleChangeSets` stage.
///
/// This stage processes and maintains trie changesets from the finalized block to the latest block.
#[derive(Debug, Clone)]
pub struct MerkleChangeSets {
/// The number of blocks to retain changesets for, used as a fallback when the finalized block
/// is not found. Defaults to [`MERKLE_CHANGESETS_RETENTION_BLOCKS`] (2 epochs in beacon
/// chain).
retention_blocks: u64,
}
impl MerkleChangeSets {
/// Creates a new `MerkleChangeSets` stage with the default retention blocks.
pub const fn new() -> Self {
Self { retention_blocks: MERKLE_CHANGESETS_RETENTION_BLOCKS }
}
/// Creates a new `MerkleChangeSets` stage with a custom finalized block height.
pub const fn with_retention_blocks(retention_blocks: u64) -> Self {
Self { retention_blocks }
}
/// Returns the range of blocks which are already computed. Will return an empty range if none
/// have been computed.
fn computed_range<Provider>(
provider: &Provider,
checkpoint: Option<StageCheckpoint>,
) -> Result<Range<BlockNumber>, StageError>
where
Provider: PruneCheckpointReader,
{
let to = checkpoint.map(|chk| chk.block_number).unwrap_or_default();
// Get the prune checkpoint for MerkleChangeSets to use as the lower bound. If there's no
// prune checkpoint or if the pruned block number is None, return empty range
let Some(from) = provider
.get_prune_checkpoint(PruneSegment::MerkleChangeSets)?
.and_then(|chk| chk.block_number)
// prune checkpoint indicates the last block pruned, so the block after is the start of
// the computed data
.map(|block_number| block_number + 1)
else {
return Ok(0..0)
};
Ok(from..to + 1)
}
/// Determines the target range for changeset computation based on the checkpoint and provider
/// state.
///
/// Returns the target range (exclusive end) to compute changesets for.
fn determine_target_range<Provider>(
&self,
provider: &Provider,
) -> Result<Range<BlockNumber>, StageError>
where
Provider: StageCheckpointReader + ChainStateBlockReader,
{
// Get merkle checkpoint which represents our target end block
let merkle_checkpoint = provider
.get_stage_checkpoint(StageId::MerkleExecute)?
.map(|checkpoint| checkpoint.block_number)
.unwrap_or(0);
let target_end = merkle_checkpoint + 1; // exclusive
// Calculate the target range based on the finalized block and the target block.
// We maintain changesets from the finalized block to the latest block.
let finalized_block = provider.last_finalized_block_number()?;
// Calculate the fallback start position based on retention blocks
let retention_based_start = merkle_checkpoint.saturating_sub(self.retention_blocks);
// If the finalized block was way in the past then we don't want to generate changesets for
// all of those past blocks; we only care about the recent history.
//
// Use maximum of finalized_block and retention_based_start if finalized_block exists,
// otherwise just use retention_based_start.
let mut target_start = finalized_block
.map(|finalized| finalized.saturating_add(1).max(retention_based_start))
.unwrap_or(retention_based_start);
// We cannot revert the genesis block; target_start must be >0
target_start = target_start.max(1);
Ok(target_start..target_end)
}
/// Calculates the trie updates given a [`TrieInputSorted`], asserting that the resulting state
/// root matches the expected one for the block.
fn calculate_block_trie_updates<Provider: DBProvider + HeaderProvider>(
provider: &Provider,
block_number: BlockNumber,
input: TrieInputSorted,
) -> Result<TrieUpdates, StageError> {
let (root, trie_updates) =
StateRoot::overlay_root_from_nodes_with_updates(provider.tx_ref(), input).map_err(
|e| {
error!(
target: "sync::stages::merkle_changesets",
%e,
?block_number,
"Incremental state root failed! {INVALID_STATE_ROOT_ERROR_MESSAGE}");
StageError::Fatal(Box::new(e))
},
)?;
let block = provider
.header_by_number(block_number)?
.ok_or_else(|| ProviderError::HeaderNotFound(block_number.into()))?;
let (got, expected) = (root, block.state_root());
if got != expected {
// Only seal the header when we need it for the error
let header = SealedHeader::seal_slow(block);
error!(
target: "sync::stages::merkle_changesets",
?block_number,
?got,
?expected,
"Failed to verify block state root! {INVALID_STATE_ROOT_ERROR_MESSAGE}",
);
return Err(StageError::Block {
error: BlockErrorKind::Validation(ConsensusError::BodyStateRootDiff(
GotExpected { got, expected }.into(),
)),
block: Box::new(header.block_with_parent()),
})
}
Ok(trie_updates)
}
fn populate_range<Provider>(
provider: &Provider,
target_range: Range<BlockNumber>,
) -> Result<(), StageError>
where
Provider: StageCheckpointReader
+ TrieWriter
+ DBProvider
+ HeaderProvider
+ ChainStateBlockReader
+ BlockNumReader
+ ChangeSetReader,
{
let target_start = target_range.start;
let target_end = target_range.end;
debug!(
target: "sync::stages::merkle_changesets",
?target_range,
"Starting trie changeset computation",
);
// We need to distinguish a cumulative revert and a per-block revert. A cumulative revert
// reverts changes starting at db tip all the way to a block. A per-block revert only
// reverts a block's changes.
//
// We need to calculate the cumulative HashedPostState reverts for every block in the
// target range. The cumulative HashedPostState revert for block N can be calculated as:
//
//
// ```
// // where `extend` overwrites any shared keys
// cumulative_state_revert(N) = cumulative_state_revert(N + 1).extend(get_block_state_revert(N))
// ```
//
// We need per-block reverts to calculate the prefix set for each individual block. By
// using the per-block reverts to calculate cumulative reverts on-the-fly we can save a
// bunch of memory.
debug!(
target: "sync::stages::merkle_changesets",
?target_range,
"Computing per-block state reverts",
);
let range_len = target_end - target_start;
let mut per_block_state_reverts = Vec::with_capacity(range_len as usize);
for block_number in target_range.clone() {
per_block_state_reverts.push(HashedPostStateSorted::from_reverts::<KeccakKeyHasher>(
provider,
block_number..=block_number,
)?);
}
// Helper to retrieve state revert data for a specific block from the pre-computed array
let get_block_state_revert = |block_number: BlockNumber| -> &HashedPostStateSorted {
let index = (block_number - target_start) as usize;
&per_block_state_reverts[index]
};
// Helper to accumulate state reverts from a given block to the target end
let compute_cumulative_state_revert = |block_number: BlockNumber| -> HashedPostStateSorted {
let mut cumulative_revert = HashedPostStateSorted::default();
for n in (block_number..target_end).rev() {
cumulative_revert.extend_ref_and_sort(get_block_state_revert(n))
}
cumulative_revert
};
// To calculate the changeset for a block, we first need the TrieUpdates which are
// generated as a result of processing the block. To get these we need:
// 1) The TrieUpdates which revert the db's trie to _prior_ to the block
// 2) The HashedPostStateSorted to revert the db's state to _after_ the block
//
// To get (1) for `target_start` we need to do a big state root calculation which takes
// into account all changes between that block and db tip. For each block after the
// `target_start` we can update (1) using the TrieUpdates which were output by the previous
// block, only targeting the state changes of that block.
debug!(
target: "sync::stages::merkle_changesets",
?target_start,
"Computing trie state at starting block",
);
let initial_state = compute_cumulative_state_revert(target_start);
let initial_prefix_sets = initial_state.construct_prefix_sets();
let initial_input =
TrieInputSorted::new(Arc::default(), Arc::new(initial_state), initial_prefix_sets);
// target_start will be >= 1, see `determine_target_range`.
let mut nodes = Arc::new(
Self::calculate_block_trie_updates(provider, target_start - 1, initial_input)?
.into_sorted(),
);
for block_number in target_range {
debug!(
target: "sync::stages::merkle_changesets",
?block_number,
"Computing trie updates for block",
);
// Revert the state so that this block has been just processed, meaning we take the
// cumulative revert of the subsequent block.
let state = Arc::new(compute_cumulative_state_revert(block_number + 1));
// Construct prefix sets from only this block's `HashedPostStateSorted`, because we only
// care about trie updates which occurred as a result of this block being processed.
let prefix_sets = get_block_state_revert(block_number).construct_prefix_sets();
let input = TrieInputSorted::new(Arc::clone(&nodes), state, prefix_sets);
// Calculate the trie updates for this block, then apply those updates to the reverts.
// We calculate the overlay which will be passed into the next step using the trie
// reverts prior to them being updated.
let this_trie_updates =
Self::calculate_block_trie_updates(provider, block_number, input)?.into_sorted();
let trie_overlay = Arc::clone(&nodes);
let mut nodes_mut = Arc::unwrap_or_clone(nodes);
nodes_mut.extend_ref_and_sort(&this_trie_updates);
nodes = Arc::new(nodes_mut);
// Write the changesets to the DB using the trie updates produced by the block, and the
// trie reverts as the overlay.
debug!(
target: "sync::stages::merkle_changesets",
?block_number,
"Writing trie changesets for block",
);
provider.write_trie_changesets(
block_number,
&this_trie_updates,
Some(&trie_overlay),
)?;
}
Ok(())
}
}
impl Default for MerkleChangeSets {
fn default() -> Self {
Self::new()
}
}
impl<Provider> Stage<Provider> for MerkleChangeSets
where
Provider: StageCheckpointReader
+ TrieWriter
+ DBProvider
+ HeaderProvider
+ ChainStateBlockReader
+ StageCheckpointWriter
+ PruneCheckpointReader
+ PruneCheckpointWriter
+ ChangeSetReader
+ BlockNumReader,
{
fn id(&self) -> StageId {
StageId::MerkleChangeSets
}
fn execute(&mut self, provider: &Provider, input: ExecInput) -> Result<ExecOutput, StageError> {
// Get merkle checkpoint and assert that the target is the same.
let merkle_checkpoint = provider
.get_stage_checkpoint(StageId::MerkleExecute)?
.map(|checkpoint| checkpoint.block_number)
.unwrap_or(0);
if input.target.is_none_or(|target| merkle_checkpoint != target) {
return Err(StageError::Fatal(eyre::eyre!("Cannot sync stage to block {:?} when MerkleExecute is at block {merkle_checkpoint:?}", input.target).into()))
}
let mut target_range = self.determine_target_range(provider)?;
// Get the previously computed range. This will be updated to reflect the populating of the
// target range.
let mut computed_range = Self::computed_range(provider, input.checkpoint)?;
debug!(
target: "sync::stages::merkle_changesets",
?computed_range,
?target_range,
"Got computed and target ranges",
);
// We want the target range to not include any data already computed previously, if
// possible, so we start the target range from the end of the computed range if that is
// greater.
//
// ------------------------------> Block #
// |------computed-----|
// |-----target-----|
// |--actual--|
//
// However, if the target start is less than the previously computed start, we don't want to
// do this, as it would leave a gap of data at `target_range.start..=computed_range.start`.
//
// ------------------------------> Block #
// |---computed---|
// |-------target-------|
// |-------actual-------|
//
if target_range.start >= computed_range.start {
target_range.start = target_range.start.max(computed_range.end);
}
// If target range is empty (target_start >= target_end), stage is already successfully
// executed.
if target_range.start >= target_range.end {
return Ok(ExecOutput::done(StageCheckpoint::new(target_range.end.saturating_sub(1))));
}
// If our target range is a continuation of the already computed range then we can keep the
// already computed data.
if target_range.start == computed_range.end {
// Clear from target_start onwards to ensure no stale data exists
provider.clear_trie_changesets_from(target_range.start)?;
computed_range.end = target_range.end;
} else {
// If our target range is not a continuation of the already computed range then we
// simply clear the computed data, to make sure there's no gaps or conflicts.
provider.clear_trie_changesets()?;
computed_range = target_range.clone();
}
// Populate the target range with changesets
Self::populate_range(provider, target_range)?;
// Update the prune checkpoint to reflect that all data before `computed_range.start`
// is not available.
provider.save_prune_checkpoint(
PruneSegment::MerkleChangeSets,
PruneCheckpoint {
block_number: Some(computed_range.start.saturating_sub(1)),
tx_number: None,
prune_mode: PruneMode::Before(computed_range.start),
},
)?;
// `computed_range.end` is exclusive.
let checkpoint = StageCheckpoint::new(computed_range.end.saturating_sub(1));
Ok(ExecOutput::done(checkpoint))
}
fn unwind(
&mut self,
provider: &Provider,
input: UnwindInput,
) -> Result<UnwindOutput, StageError> {
// Unwinding is trivial; just clear everything after the target block.
provider.clear_trie_changesets_from(input.unwind_to + 1)?;
let mut computed_range = Self::computed_range(provider, Some(input.checkpoint))?;
computed_range.end = input.unwind_to + 1;
if computed_range.start > computed_range.end {
computed_range.start = computed_range.end;
}
// If we've unwound so far that there are no longer enough trie changesets available then
// simply clear them and the checkpoints, so that on next pipeline startup they will be
// regenerated.
//
// We don't do this check if the target block is not greater than the retention threshold
// (which happens near genesis), as in that case would could still have all possible
// changesets even if the total count doesn't meet the threshold.
debug!(
target: "sync::stages::merkle_changesets",
?computed_range,
retention_blocks=?self.retention_blocks,
"Checking if computed range is over retention threshold",
);
if input.unwind_to > self.retention_blocks &&
computed_range.end - computed_range.start < self.retention_blocks
{
debug!(
target: "sync::stages::merkle_changesets",
?computed_range,
retention_blocks=?self.retention_blocks,
"Clearing checkpoints completely",
);
provider.clear_trie_changesets()?;
provider
.save_stage_checkpoint(StageId::MerkleChangeSets, StageCheckpoint::default())?;
return Ok(UnwindOutput { checkpoint: StageCheckpoint::default() })
}
// `computed_range.end` is exclusive
let checkpoint = StageCheckpoint::new(computed_range.end.saturating_sub(1));
Ok(UnwindOutput { checkpoint })
}
}

View File

@@ -1,5 +1,6 @@
/// The bodies stage.
mod bodies;
mod era;
/// The execution stage that generates state diff.
mod execution;
/// The finish stage
@@ -36,9 +37,7 @@ pub use prune::*;
pub use sender_recovery::*;
pub use tx_lookup::*;
mod era;
mod utils;
use utils::*;
#[cfg(test)]

View File

@@ -158,15 +158,13 @@ where
let append_only =
provider.count_entries::<tables::TransactionHashNumbers>()?.is_zero();
// Create RocksDB batch if feature is enabled
// Auto-commits on threshold; consistency check heals any crash.
#[cfg(all(unix, feature = "rocksdb"))]
let rocksdb = provider.rocksdb_provider();
#[cfg(all(unix, feature = "rocksdb"))]
let rocksdb_batch = rocksdb.batch();
let rocksdb_batch = rocksdb.batch_with_auto_commit();
#[cfg(not(all(unix, feature = "rocksdb")))]
let rocksdb_batch = ();
// Create writer that routes to either MDBX or RocksDB based on settings
let mut writer =
EitherWriter::new_transaction_hash_numbers(provider, rocksdb_batch)?;
@@ -217,15 +215,12 @@ where
) -> Result<UnwindOutput, StageError> {
let (range, unwind_to, _) = input.unwind_block_range_with_threshold(self.chunk_size);
// Create RocksDB batch if feature is enabled
#[cfg(all(unix, feature = "rocksdb"))]
let rocksdb = provider.rocksdb_provider();
#[cfg(all(unix, feature = "rocksdb"))]
let rocksdb_batch = rocksdb.batch();
#[cfg(not(all(unix, feature = "rocksdb")))]
let rocksdb_batch = ();
// Create writer that routes to either MDBX or RocksDB based on settings
let mut writer = EitherWriter::new_transaction_hash_numbers(provider, rocksdb_batch)?;
let static_file_provider = provider.static_file_provider();

View File

@@ -1,21 +1,25 @@
//! Utils for `stages`.
use alloy_primitives::{Address, BlockNumber, TxNumber};
use alloy_primitives::{Address, BlockNumber, TxNumber, B256};
use reth_config::config::EtlConfig;
use reth_db_api::{
cursor::{DbCursorRO, DbCursorRW},
models::{sharded_key::NUM_OF_INDICES_IN_SHARD, AccountBeforeTx, ShardedKey},
table::{Decompress, Table},
transaction::{DbTx, DbTxMut},
BlockNumberList, DatabaseError,
models::{
sharded_key::NUM_OF_INDICES_IN_SHARD, storage_sharded_key::StorageShardedKey,
AccountBeforeTx, AddressStorageKey, BlockNumberAddress, ShardedKey,
},
table::{Decode, Decompress, Table},
transaction::DbTx,
BlockNumberList,
};
use reth_etl::Collector;
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
providers::StaticFileProvider, to_range, BlockReader, DBProvider, ProviderError,
providers::StaticFileProvider, to_range, BlockReader, DBProvider, EitherWriter, ProviderError,
StaticFileProviderFactory,
};
use reth_stages_api::StageError;
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::ChangeSetReader;
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader};
use std::{collections::HashMap, hash::Hash, ops::RangeBounds};
use tracing::info;
@@ -98,17 +102,17 @@ where
}
/// Allows collecting indices from a cache with a custom insert fn
fn collect_indices<F>(
cache: impl Iterator<Item = (Address, Vec<u64>)>,
fn collect_indices<K, F>(
cache: impl Iterator<Item = (K, Vec<u64>)>,
mut insert_fn: F,
) -> Result<(), StageError>
where
F: FnMut(Address, Vec<u64>) -> Result<(), StageError>,
F: FnMut(K, Vec<u64>) -> Result<(), StageError>,
{
for (address, indices) in cache {
insert_fn(address, indices)?
for (key, indices) in cache {
insert_fn(key, indices)?
}
Ok::<(), StageError>(())
Ok(())
}
/// Collects account history indices using a provider that implements `ChangeSetReader`.
@@ -124,12 +128,12 @@ where
let mut cache: HashMap<Address, Vec<u64>> = HashMap::default();
let mut insert_fn = |address: Address, indices: Vec<u64>| {
let last = indices.last().expect("qed");
let last = indices.last().expect("indices is non-empty");
collector.insert(
ShardedKey::new(address, *last),
BlockNumberList::new_pre_sorted(indices.into_iter()),
)?;
Ok::<(), StageError>(())
Ok(())
};
// Convert range bounds to concrete range
@@ -170,154 +174,232 @@ where
Ok(collector)
}
/// Given a [`Collector`] created by [`collect_history_indices`] it iterates all entries, loading
/// the indices into the database in shards.
///
/// ## Process
/// Iterates over elements, grouping indices by their partial keys (e.g., `Address` or
/// `Address.StorageKey`). It flushes indices to disk when reaching a shard's max length
/// (`NUM_OF_INDICES_IN_SHARD`) or when the partial key changes, ensuring the last previous partial
/// key shard is stored.
pub(crate) fn load_history_indices<Provider, H, P>(
/// Collects storage history indices using a provider that implements `StorageChangeSetReader`.
pub(crate) fn collect_storage_history_indices<Provider>(
provider: &Provider,
mut collector: Collector<H::Key, H::Value>,
range: impl RangeBounds<BlockNumber>,
etl_config: &EtlConfig,
) -> Result<Collector<StorageShardedKey, BlockNumberList>, StageError>
where
Provider: DBProvider + StorageChangeSetReader + StaticFileProviderFactory,
{
let mut collector = Collector::new(etl_config.file_size, etl_config.dir.clone());
let mut cache: HashMap<AddressStorageKey, Vec<u64>> = HashMap::default();
let mut insert_fn = |key: AddressStorageKey, indices: Vec<u64>| {
let last = indices.last().expect("qed");
collector.insert(
StorageShardedKey::new(key.0 .0, key.0 .1, *last),
BlockNumberList::new_pre_sorted(indices.into_iter()),
)?;
Ok::<(), StageError>(())
};
let range = to_range(range);
let static_file_provider = provider.static_file_provider();
let total_changesets = static_file_provider.storage_changeset_count()?;
let interval = (total_changesets / 1000).max(1);
let walker = static_file_provider.walk_storage_changeset_range(range);
let mut flush_counter = 0;
let mut current_block_number = u64::MAX;
for (idx, changeset_result) in walker.enumerate() {
let (BlockNumberAddress((block_number, address)), storage) = changeset_result?;
cache.entry(AddressStorageKey((address, storage.key))).or_default().push(block_number);
if idx > 0 && idx % interval == 0 && total_changesets > 1000 {
info!(target: "sync::stages::index_history", progress = %format!("{:.4}%", (idx as f64 / total_changesets as f64) * 100.0), "Collecting indices");
}
if block_number != current_block_number {
current_block_number = block_number;
flush_counter += 1;
}
if flush_counter > DEFAULT_CACHE_THRESHOLD {
collect_indices(cache.drain(), &mut insert_fn)?;
flush_counter = 0;
}
}
collect_indices(cache.into_iter(), insert_fn)?;
Ok(collector)
}
/// Loads account history indices into the database via `EitherWriter`.
///
/// Works with [`EitherWriter`] to support both MDBX and `RocksDB` backends.
///
/// ## Process
/// Iterates over elements, grouping indices by their address. It flushes indices to disk
/// when reaching a shard's max length (`NUM_OF_INDICES_IN_SHARD`) or when the address changes,
/// ensuring the last previous address shard is stored.
///
/// Uses `Option<Address>` instead of `Address::default()` as the sentinel to avoid
/// incorrectly treating `Address::ZERO` as "no previous address".
pub(crate) fn load_account_history<N, CURSOR>(
mut collector: Collector<ShardedKey<Address>, BlockNumberList>,
append_only: bool,
sharded_key_factory: impl Clone + Fn(P, u64) -> <H as Table>::Key,
decode_key: impl Fn(Vec<u8>) -> Result<<H as Table>::Key, DatabaseError>,
get_partial: impl Fn(<H as Table>::Key) -> P,
writer: &mut EitherWriter<'_, CURSOR, N>,
) -> Result<(), StageError>
where
Provider: DBProvider<Tx: DbTxMut>,
H: Table<Value = BlockNumberList>,
P: Copy + Default + Eq,
N: NodePrimitives,
CURSOR: DbCursorRW<reth_db_api::tables::AccountsHistory>
+ DbCursorRO<reth_db_api::tables::AccountsHistory>,
{
let mut write_cursor = provider.tx_ref().cursor_write::<H>()?;
let mut current_partial = P::default();
let mut current_address: Option<Address> = None;
// Accumulator for block numbers where the current address changed.
let mut current_list = Vec::<u64>::new();
// observability
let total_entries = collector.len();
let interval = (total_entries / 10).max(1);
for (index, element) in collector.iter()?.enumerate() {
let (k, v) = element?;
let sharded_key = decode_key(k)?;
let sharded_key = ShardedKey::<Address>::decode_owned(k)?;
let new_list = BlockNumberList::decompress_owned(v)?;
if index > 0 && index.is_multiple_of(interval) && total_entries > 10 {
info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices");
}
// AccountsHistory: `Address`.
// StorageHistory: `Address.StorageKey`.
let partial_key = get_partial(sharded_key);
let address = sharded_key.key;
if current_partial != partial_key {
// We have reached the end of this subset of keys so
// we need to flush its last indice shard.
load_indices(
&mut write_cursor,
current_partial,
&mut current_list,
&sharded_key_factory,
append_only,
LoadMode::Flush,
)?;
// When address changes, flush the previous address's shards and start fresh.
if current_address != Some(address) {
// Flush all remaining shards for the previous address (uses u64::MAX for last shard).
if let Some(prev_addr) = current_address {
flush_account_history_shards(prev_addr, &mut current_list, append_only, writer)?;
}
current_partial = partial_key;
current_address = Some(address);
current_list.clear();
// If it's not the first sync, there might an existing shard already, so we need to
// merge it with the one coming from the collector
// On incremental sync, merge with the existing last shard from the database.
// The last shard is stored with key (address, u64::MAX) so we can find it.
if !append_only &&
let Some((_, last_database_shard)) =
write_cursor.seek_exact(sharded_key_factory(current_partial, u64::MAX))?
let Some(last_shard) = writer.get_last_account_history_shard(address)?
{
current_list.extend(last_database_shard.iter());
current_list.extend(last_shard.iter());
}
}
// Append new block numbers to the accumulator.
current_list.extend(new_list.iter());
load_indices(
&mut write_cursor,
current_partial,
&mut current_list,
&sharded_key_factory,
append_only,
LoadMode::KeepLast,
)?;
// Flush complete shards, keeping the last (partial) shard buffered.
flush_account_history_shards_partial(address, &mut current_list, append_only, writer)?;
}
// There will be one remaining shard that needs to be flushed to DB.
load_indices(
&mut write_cursor,
current_partial,
&mut current_list,
&sharded_key_factory,
append_only,
LoadMode::Flush,
)?;
// Flush the final address's remaining shard.
if let Some(addr) = current_address {
flush_account_history_shards(addr, &mut current_list, append_only, writer)?;
}
Ok(())
}
/// Shard and insert the indices list according to [`LoadMode`] and its length.
pub(crate) fn load_indices<H, C, P>(
cursor: &mut C,
partial_key: P,
list: &mut Vec<BlockNumber>,
sharded_key_factory: &impl Fn(P, BlockNumber) -> <H as Table>::Key,
/// Flushes complete shards for account history, keeping the trailing partial shard buffered.
///
/// Only flushes when we have more than one shard's worth of data, keeping the last
/// (possibly partial) shard for continued accumulation. This avoids writing a shard
/// that may need to be updated when more indices arrive.
fn flush_account_history_shards_partial<N, CURSOR>(
address: Address,
list: &mut Vec<u64>,
append_only: bool,
mode: LoadMode,
writer: &mut EitherWriter<'_, CURSOR, N>,
) -> Result<(), StageError>
where
C: DbCursorRO<H> + DbCursorRW<H>,
H: Table<Value = BlockNumberList>,
P: Copy,
N: NodePrimitives,
CURSOR: DbCursorRW<reth_db_api::tables::AccountsHistory>
+ DbCursorRO<reth_db_api::tables::AccountsHistory>,
{
if list.len() > NUM_OF_INDICES_IN_SHARD || mode.is_flush() {
let chunks = list
.chunks(NUM_OF_INDICES_IN_SHARD)
.map(|chunks| chunks.to_vec())
.collect::<Vec<Vec<u64>>>();
// Nothing to flush if we haven't filled a complete shard yet.
if list.len() <= NUM_OF_INDICES_IN_SHARD {
return Ok(());
}
let mut iter = chunks.into_iter().peekable();
while let Some(chunk) = iter.next() {
let mut highest = *chunk.last().expect("at least one index");
let num_full_shards = list.len() / NUM_OF_INDICES_IN_SHARD;
if !mode.is_flush() && iter.peek().is_none() {
*list = chunk;
} else {
if iter.peek().is_none() {
highest = u64::MAX;
}
let key = sharded_key_factory(partial_key, highest);
let value = BlockNumberList::new_pre_sorted(chunk);
// Always keep at least one shard buffered for continued accumulation.
// If len is exact multiple of shard size, keep the last full shard.
let shards_to_flush = if list.len().is_multiple_of(NUM_OF_INDICES_IN_SHARD) {
num_full_shards - 1
} else {
num_full_shards
};
if append_only {
cursor.append(key, &value)?;
} else {
cursor.upsert(key, &value)?;
}
}
if shards_to_flush == 0 {
return Ok(());
}
// Split: flush the first N shards, keep the remainder buffered.
let flush_len = shards_to_flush * NUM_OF_INDICES_IN_SHARD;
let remainder = list.split_off(flush_len);
// Write each complete shard with its highest block number as the key.
for chunk in list.chunks(NUM_OF_INDICES_IN_SHARD) {
let highest = *chunk.last().expect("chunk is non-empty");
let key = ShardedKey::new(address, highest);
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
if append_only {
writer.append_account_history(key, &value)?;
} else {
writer.upsert_account_history(key, &value)?;
}
}
// Keep the remaining indices for the next iteration.
*list = remainder;
Ok(())
}
/// Mode on how to load index shards into the database.
pub(crate) enum LoadMode {
/// Keep the last shard in memory and don't flush it to the database.
KeepLast,
/// Flush all shards into the database.
Flush,
}
impl LoadMode {
const fn is_flush(&self) -> bool {
matches!(self, Self::Flush)
/// Flushes all remaining shards for account history, using `u64::MAX` for the last shard.
///
/// The `u64::MAX` key for the final shard is an invariant that allows `seek_exact(address,
/// u64::MAX)` to find the last shard during incremental sync for merging with new indices.
fn flush_account_history_shards<N, CURSOR>(
address: Address,
list: &mut Vec<u64>,
append_only: bool,
writer: &mut EitherWriter<'_, CURSOR, N>,
) -> Result<(), StageError>
where
N: NodePrimitives,
CURSOR: DbCursorRW<reth_db_api::tables::AccountsHistory>
+ DbCursorRO<reth_db_api::tables::AccountsHistory>,
{
if list.is_empty() {
return Ok(());
}
let num_chunks = list.len().div_ceil(NUM_OF_INDICES_IN_SHARD);
for (i, chunk) in list.chunks(NUM_OF_INDICES_IN_SHARD).enumerate() {
let is_last = i == num_chunks - 1;
// Use u64::MAX for the final shard's key. This invariant allows incremental sync
// to find the last shard via seek_exact(address, u64::MAX) for merging.
let highest = if is_last { u64::MAX } else { *chunk.last().expect("chunk is non-empty") };
let key = ShardedKey::new(address, highest);
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
if append_only {
writer.append_account_history(key, &value)?;
} else {
writer.upsert_account_history(key, &value)?;
}
}
list.clear();
Ok(())
}
/// Called when database is ahead of static files. Attempts to find the first block we are missing
@@ -355,3 +437,191 @@ where
segment,
})
}
/// Loads storage history indices into the database via `EitherWriter`.
///
/// Works with [`EitherWriter`] to support both MDBX and `RocksDB` backends.
///
/// ## Process
/// Iterates over elements, grouping indices by their (address, `storage_key`) pairs. It flushes
/// indices to disk when reaching a shard's max length (`NUM_OF_INDICES_IN_SHARD`) or when the
/// (address, `storage_key`) pair changes, ensuring the last previous shard is stored.
///
/// Uses `Option<(Address, B256)>` instead of default values as the sentinel to avoid
/// incorrectly treating `(Address::ZERO, B256::ZERO)` as "no previous key".
pub(crate) fn load_storage_history<N, CURSOR>(
mut collector: Collector<StorageShardedKey, BlockNumberList>,
append_only: bool,
writer: &mut EitherWriter<'_, CURSOR, N>,
) -> Result<(), StageError>
where
N: NodePrimitives,
CURSOR: DbCursorRW<reth_db_api::tables::StoragesHistory>
+ DbCursorRO<reth_db_api::tables::StoragesHistory>,
{
let mut current_key: Option<(Address, B256)> = None;
// Accumulator for block numbers where the current (address, storage_key) changed.
let mut current_list = Vec::<u64>::new();
let total_entries = collector.len();
let interval = (total_entries / 10).max(1);
for (index, element) in collector.iter()?.enumerate() {
let (k, v) = element?;
let sharded_key = StorageShardedKey::decode_owned(k)?;
let new_list = BlockNumberList::decompress_owned(v)?;
if index > 0 && index.is_multiple_of(interval) && total_entries > 10 {
info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices");
}
let partial_key = (sharded_key.address, sharded_key.sharded_key.key);
// When (address, storage_key) changes, flush the previous key's shards and start fresh.
if current_key != Some(partial_key) {
// Flush all remaining shards for the previous key (uses u64::MAX for last shard).
if let Some((prev_addr, prev_storage_key)) = current_key {
flush_storage_history_shards(
prev_addr,
prev_storage_key,
&mut current_list,
append_only,
writer,
)?;
}
current_key = Some(partial_key);
current_list.clear();
// On incremental sync, merge with the existing last shard from the database.
// The last shard is stored with key (address, storage_key, u64::MAX) so we can find it.
if !append_only &&
let Some(last_shard) =
writer.get_last_storage_history_shard(partial_key.0, partial_key.1)?
{
current_list.extend(last_shard.iter());
}
}
// Append new block numbers to the accumulator.
current_list.extend(new_list.iter());
// Flush complete shards, keeping the last (partial) shard buffered.
flush_storage_history_shards_partial(
partial_key.0,
partial_key.1,
&mut current_list,
append_only,
writer,
)?;
}
// Flush the final key's remaining shard.
if let Some((addr, storage_key)) = current_key {
flush_storage_history_shards(addr, storage_key, &mut current_list, append_only, writer)?;
}
Ok(())
}
/// Flushes complete shards for storage history, keeping the trailing partial shard buffered.
///
/// Only flushes when we have more than one shard's worth of data, keeping the last
/// (possibly partial) shard for continued accumulation. This avoids writing a shard
/// that may need to be updated when more indices arrive.
fn flush_storage_history_shards_partial<N, CURSOR>(
address: Address,
storage_key: B256,
list: &mut Vec<u64>,
append_only: bool,
writer: &mut EitherWriter<'_, CURSOR, N>,
) -> Result<(), StageError>
where
N: NodePrimitives,
CURSOR: DbCursorRW<reth_db_api::tables::StoragesHistory>
+ DbCursorRO<reth_db_api::tables::StoragesHistory>,
{
// Nothing to flush if we haven't filled a complete shard yet.
if list.len() <= NUM_OF_INDICES_IN_SHARD {
return Ok(());
}
let num_full_shards = list.len() / NUM_OF_INDICES_IN_SHARD;
// Always keep at least one shard buffered for continued accumulation.
// If len is exact multiple of shard size, keep the last full shard.
let shards_to_flush = if list.len().is_multiple_of(NUM_OF_INDICES_IN_SHARD) {
num_full_shards - 1
} else {
num_full_shards
};
if shards_to_flush == 0 {
return Ok(());
}
// Split: flush the first N shards, keep the remainder buffered.
let flush_len = shards_to_flush * NUM_OF_INDICES_IN_SHARD;
let remainder = list.split_off(flush_len);
// Write each complete shard with its highest block number as the key.
for chunk in list.chunks(NUM_OF_INDICES_IN_SHARD) {
let highest = *chunk.last().expect("chunk is non-empty");
let key = StorageShardedKey::new(address, storage_key, highest);
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
if append_only {
writer.append_storage_history(key, &value)?;
} else {
writer.upsert_storage_history(key, &value)?;
}
}
// Keep the remaining indices for the next iteration.
*list = remainder;
Ok(())
}
/// Flushes all remaining shards for storage history, using `u64::MAX` for the last shard.
///
/// The `u64::MAX` key for the final shard is an invariant that allows
/// `seek_exact(address, storage_key, u64::MAX)` to find the last shard during incremental
/// sync for merging with new indices.
fn flush_storage_history_shards<N, CURSOR>(
address: Address,
storage_key: B256,
list: &mut Vec<u64>,
append_only: bool,
writer: &mut EitherWriter<'_, CURSOR, N>,
) -> Result<(), StageError>
where
N: NodePrimitives,
CURSOR: DbCursorRW<reth_db_api::tables::StoragesHistory>
+ DbCursorRO<reth_db_api::tables::StoragesHistory>,
{
if list.is_empty() {
return Ok(());
}
let num_chunks = list.len().div_ceil(NUM_OF_INDICES_IN_SHARD);
for (i, chunk) in list.chunks(NUM_OF_INDICES_IN_SHARD).enumerate() {
let is_last = i == num_chunks - 1;
// Use u64::MAX for the final shard's key. This invariant allows incremental sync
// to find the last shard via seek_exact(address, storage_key, u64::MAX) for merging.
let highest = if is_last { u64::MAX } else { *chunk.last().expect("chunk is non-empty") };
let key = StorageShardedKey::new(address, storage_key, highest);
let value = BlockNumberList::new_pre_sorted(chunk.iter().copied());
if append_only {
writer.append_storage_history(key, &value)?;
} else {
writer.upsert_storage_history(key, &value)?;
}
}
list.clear();
Ok(())
}

Some files were not shown because too many files have changed in this diff Show More